diff --git a/src/core/resolver/hb_opts.erl b/src/core/resolver/hb_opts.erl index b1acd6be8..569ea86d6 100644 --- a/src/core/resolver/hb_opts.erl +++ b/src/core/resolver/hb_opts.erl @@ -484,6 +484,8 @@ raw_default_message() -> % default_index => #{ <<"device">> => <<"hyperbuddy@1.0">> }, % Should we use the latest cached state of a process when computing? <<"process-now-from-cache">> => false, + % Maximum age, in seconds, for `/now' to serve from the process cache. + <<"process-now-max-age">> => infinity, % Should we trust the GraphQL API when converting to ANS-104? Some GQL % services do not provide the `anchor' or `last_tx' fields, so their % responses are not verifiable. diff --git a/src/preloaded/arweave/dev_arweave.erl b/src/preloaded/arweave/dev_arweave.erl index 580b069cf..7dcec5ece 100644 --- a/src/preloaded/arweave/dev_arweave.erl +++ b/src/preloaded/arweave/dev_arweave.erl @@ -885,31 +885,7 @@ request(Method, Path, Extra, LogExtra, Opts) -> <<"cache-control">> => [<<"no-cache">>, <<"no-store">>] } ), - to_message(Path, Method, best_response(Res), LogExtra, Opts). - -%% @doc Select the best response from a list of responses by sorting them -%% ascending by HTTP status code. Returns the first (best) response tuple. -best_response({error, {no_viable_responses, Responses}}) -> - best_response(Responses); -best_response([]) -> - {error, no_viable_responses}; -best_response(Responses) when is_list(Responses) -> - Sorted = lists:sort( - fun({_, ResponseA}, {_, ResponseB}) -> - StatusA = response_status(ResponseA), - StatusB = response_status(ResponseB), - StatusA =< StatusB - end, - Responses - ), - hd(Sorted); -best_response(Response) -> - Response. - -response_status(Response) when is_map(Response) -> - maps:get(<<"status">>, Response, 999); -response_status(_Response) -> - 999. + to_message(Path, Method, lib_arweave_common:best_response(Res), LogExtra, Opts). %% @doc Transform a response from the Arweave node into an AO-Core message. to_message(Path, Method, {error, #{ <<"status">> := 404 }}, LogExtra, _Opts) -> @@ -1206,7 +1182,7 @@ best_response_handles_failed_connect_entries_test_parallel() -> ], ?assertEqual( {ok, #{ <<"status">> => 200, <<"body">> => <<"OK-2">> }}, - best_response(Responses) + lib_arweave_common:best_response(Responses) ). best_response_non_map_error_round_trips_test_parallel() -> diff --git a/src/preloaded/arweave/dev_manifest.erl b/src/preloaded/arweave/dev_manifest.erl index 8e339a604..3a22246fe 100644 --- a/src/preloaded/arweave/dev_manifest.erl +++ b/src/preloaded/arweave/dev_manifest.erl @@ -102,10 +102,7 @@ request(Base, Req, Opts) -> % message replaced with the casted manifest. case {Rest, maybe_cast_manifest(Loaded, Opts)} of {_, ignored} -> - ?event( - debug_manifest, - {non_manifest_returning_loaded, {loaded, Loaded}, {rest, Rest}}), - {ok, Req#{ <<"body">> => [Loaded|Rest] }}; + maybe_cast_resolved_path(Loaded, Rest, Req, Opts); {[], {ok, Casted}} -> ?event(debug_manifest, {manifest_returning_index, {req, Req}}), {ok, Req#{ <<"body">> => [Casted, #{<<"path">> => <<"index">>}] }}; @@ -129,6 +126,36 @@ request(Base, Req, Opts) -> {ok, Req} end. +maybe_cast_resolved_path(Loaded, [Next | Rest], Req, Opts) -> + case catch hb_ao:resolve(Loaded, Next, Opts) of + {ok, Resolved} -> + case catch load(Resolved, Opts) of + {ok, NextLoaded} -> + case maybe_cast_manifest(NextLoaded, Opts) of + {ok, Casted} -> manifest_path(Casted, Rest, Req); + ignored -> non_manifest_path(Loaded, [Next | Rest], Req) + end; + _ -> non_manifest_path(Loaded, [Next | Rest], Req) + end; + _ -> non_manifest_path(Loaded, [Next | Rest], Req) + end; +maybe_cast_resolved_path(Loaded, [], Req, _Opts) -> + non_manifest_path(Loaded, [], Req). + +manifest_path(Casted, [], Req) -> + ?event(debug_manifest, {manifest_returning_index, {req, Req}}), + {ok, Req#{ <<"body">> => [Casted, #{<<"path">> => <<"index">>}] }}; +manifest_path(Casted, Rest, Req) -> + ?event(debug_manifest, {manifest_returning_subpath, {req, Req}}), + {ok, Req#{ <<"body">> => [Casted | Rest] }}. + +non_manifest_path(Loaded, Rest, Req) -> + ?event( + debug_manifest, + {non_manifest_returning_loaded, {loaded, Loaded}, {rest, Rest}} + ), + {ok, Req#{ <<"body">> => [Loaded | Rest] }}. + %% @doc Cast a message to `manifest@1.0` if it has the correct content-type but %% no other device is specified. load(Msg, _Opts) when is_map(Msg) -> {ok, Msg}; diff --git a/src/preloaded/codec/lib_arweave_common.erl b/src/preloaded/codec/lib_arweave_common.erl index 755b9492c..6430ad2de 100644 --- a/src/preloaded/codec/lib_arweave_common.erl +++ b/src/preloaded/codec/lib_arweave_common.erl @@ -6,6 +6,7 @@ -export([bundle_hint/4, data/3, tags/5, excluded_tags/3]). -export([to/3, to/6, siginfo/4, fields_to_tx/4]). -export([bundle_header/2, bundle_header/3]). +-export([best_response/1]). -include("include/hb.hrl"). -define(ANS104_BASE_FIELDS, [<<"anchor">>, <<"target">>]). @@ -770,3 +771,29 @@ read_bundle_header(BundleStartOffset, HeaderSize, FirstChunk, Opts) -> Error -> Error end. + +%% @doc Select the best response from a (potentially multi-node) request result +%% by sorting the responses ascending by HTTP status code and returning the +%% first (best) one. Shared by the Arweave devices, which broadcast requests to +%% several nodes and take the most successful reply. +best_response({error, {no_viable_responses, Responses}}) -> + best_response(Responses); +best_response([]) -> + {error, no_viable_responses}; +best_response(Responses) when is_list(Responses) -> + Sorted = lists:sort( + fun({_, ResponseA}, {_, ResponseB}) -> + StatusA = response_status(ResponseA), + StatusB = response_status(ResponseB), + StatusA =< StatusB + end, + Responses + ), + hd(Sorted); +best_response(Response) -> + Response. + +response_status(Response) when is_map(Response) -> + maps:get(<<"status">>, Response, 999); +response_status(_Response) -> + 999. diff --git a/src/preloaded/name/dev_name.erl b/src/preloaded/name/dev_name.erl index 04aa4689c..63ae81a12 100644 --- a/src/preloaded/name/dev_name.erl +++ b/src/preloaded/name/dev_name.erl @@ -83,21 +83,24 @@ request(HookMsg, HookReq, Opts) -> maybe {ok, Req} ?= hb_maps:find(<<"request">>, HookReq, Opts), {ok, Host} ?= hb_maps:find(<<"host">>, Req, Opts), - {ok, Name} ?= + {ok, RawName} ?= name_from_host( Host, hb_opts:get(node_host, hb_opts:get(host, no_host, Opts), Opts) ), + {Name, NamePath} = name_path(RawName), {ok, ResolvedMsg} ?= resolve(Name, HookMsg, HookReq, Opts), ModReq = - maybe_append_named_message( + append_name_path( ResolvedMsg, + NamePath, hb_util:ok(hb_maps:find(<<"body">>, HookReq, Opts)), Opts ), ?event( {request_with_prepended_path, {name, Name}, + {name_path, NamePath}, {full_host, Host}, {resolved_msg, ResolvedMsg}, {to_execute, ModReq} @@ -121,6 +124,19 @@ request(HookMsg, HookReq, Opts) -> end end. +%% @doc Split an underscore-encoded subdomain into root name and path parts. +%% `x_y_z' resolves `z', then `y', then `x'. +name_path(Name) -> + case lists:reverse(binary:split(Name, <<"_">>, [global, trim_all])) of + [Root | Path] -> {Root, Path}; + [] -> {Name, []} + end. + +%% @doc Append the resolved name as the base, then any path encoded in the host. +append_name_path(ResolvedMsg, NamePath, OldReq, Opts) -> + [Base | Rest] = maybe_append_named_message(ResolvedMsg, OldReq, Opts), + [Base | ([#{ <<"path">> => Part } || Part <- NamePath] ++ Rest)]. + %% @doc After finding a hit for a named message, we should ensure that it is the %% base message for the evaluation. If it is already present in the request, %% however, we should not add it twice. Instead, we must add the version that @@ -209,6 +225,19 @@ device_resolver(Msg) -> } }. +host_test_opts() -> + #{ + <<"port">> => 0, + <<"name-resolvers">> => + [ + device_resolver( + #{ <<"permabytes">> => #{ <<"content-type">> => <<"text/html">> } } + ) + ], + <<"on">> => + #{ <<"request">> => #{ <<"device">> => <<"name@1.0">> } } + }. + single_resolver_test_parallel() -> ?assertEqual( {ok, <<"world">>}, @@ -303,8 +332,8 @@ arns_json_snapshot_test_parallel() -> ) ). -arns_host_resolution_test_parallel() -> - Opts = hb_name_test_utils:arns_opts(), +host_resolution_test_parallel() -> + Opts = host_test_opts(), Node = hb_http_server:start_node(Opts), ?assertMatch( {ok, <<"text/html">>}, @@ -312,17 +341,14 @@ arns_host_resolution_test_parallel() -> Node, #{ <<"path">> => <<"content-type">>, - <<"host">> => <<"001_permabytes.localhost">> + <<"host">> => <<"permabytes.localhost">> }, Opts ) ). -arns_host_resolution_with_node_host_test_parallel() -> - Opts = (hb_name_test_utils:arns_opts())#{ - <<"node-host">> => <<"http://localhost">>, - <<"port">> => 0 - }, +host_resolution_with_node_host_test_parallel() -> + Opts = (host_test_opts())#{ <<"node-host">> => <<"http://localhost">> }, Node = hb_http_server:start_node(Opts), ?assertMatch( {ok, <<"text/html">>}, @@ -330,12 +356,58 @@ arns_host_resolution_with_node_host_test_parallel() -> Node, #{ <<"path">> => <<"content-type">>, - <<"host">> => <<"001_permabytes.localhost">> + <<"host">> => <<"permabytes.localhost">> }, Opts ) ). +underscore_host_parts_resolve_manifest_test_parallel() -> + Opts = hb_name_test_utils:manifest_opts(), + ManifestID = <<"42jky7O3rzKkMOfHBXgK-304YjulzEYqHc9qyjT3efA">>, + {ok, IndexPageID} = + hb_cache:write( + #{ + <<"content-type">> => <<"text/html">>, + <<"body">> => <<"Index page.">> + } + ), + Subrealm = + #{ + <<"sub2">> => ManifestID + <<"index">> => IndexPageID + } + Opts = + (hb_name_test_utils:manifest_opts())#{ + <<"port">> => 0, + <<"http-client-hackney-recv-timeout">> => 30_000, + <<"name-resolvers">> => + [ + device_resolver( + #{ + <<"sub2_sub1">> => <<"not-the-manifest">>, + <<"sub1">> => IndexPageID + } + ) + ], + <<"on">> => + #{ + <<"request">> => + [ + #{<<"device">> => <<"name@1.0">>}, + #{<<"device">> => <<"manifest@1.0">>} + ] + } + }, + Node = hb_http_server:start_node(Opts), + hb_test_utils:assert_manifest_response( + Node, + #{ <<"path">> => <<"/">>, <<"host">> => <<"sub2_sub1.localhost">> }, + <<"text/html">>, + <<"Portal">>, + Opts + ). + root_request_skips_name_resolution_test_parallel() -> BaseOpts = #{ @@ -371,4 +443,9 @@ name_from_host_test_parallel() -> ?assertEqual( {ok, <<"sub3.sub2">>}, name_from_host(<<"sub3.sub2.sub1.abc.xyz">>, <<"sub1.abc.xyz">>) + ), + ?assertEqual({<<"sub1">>, [<<"sub2">>]}, name_path(<<"sub2_sub1">>)), + ?assertEqual( + {<<"sub1">>, [<<"sub2">>, <<"sub3">>]}, + name_path(<<"sub3_sub2_sub1">>) ). diff --git a/src/preloaded/process/dev_arweave_scheduler.erl b/src/preloaded/process/dev_arweave_scheduler.erl new file mode 100644 index 000000000..23a4b81e5 --- /dev/null +++ b/src/preloaded/process/dev_arweave_scheduler.erl @@ -0,0 +1,1335 @@ +%%% @doc A scheduler for AO processes that uses the Arweave network itself +%%% as the sequencing layer. The device implements the same interface as +%%% `~scheduler@1.0' from the perspective of consumers (`~process@1.0' et +%%% al), but assignments are not minted by a scheduling authority: they are +%%% implicit in the order that Arweave blocks include L1 transactions that +%%% target the process. +%%% +%%% The model is as follows: +%%% +%%% +%%% A process message may widen what it is sequenced by, with a +%%% `scheduler-mode' key: +%%% +%%% The mode is read from the process message itself -- an L1 transaction -- +%%% so it is chain data like the rest of the schedule, and is fixed for the +%%% life of the process. +%%% +%%% Assignment bodies are transaction headers only: each is the +%%% data-free header that `~copycat@1.0/arweave' cached while indexing the +%%% range (read from the node's stores), so the schedule never carries (nor +%%% depends on the availability of) a transaction's data. A header still +%%% carries the tx@1.0 signature, so its committed ID is the transaction ID +%%% and it verifies independently. Assignments are +%%% deterministic derivations of chain data and are left uncommitted: every +%%% node that reads the same blocks converges on an identical schedule +%%% without trusting a scheduler wallet. Only `tx@1.0' commitments are +%%% accepted for dispatch -- ANS-104 data items and HTTPSig messages cannot +%%% be sequenced by the base layer. +%%% +%%% `POST /schedule' dispatches a user's presigned transaction to the +%%% network on their behalf. Only its header is relayed: because the schedule +%%% is built from headers, a message carrying data is rejected rather than +%%% dispatched, so its slot can never depend on chunk availability. No slot is +%%% returned at dispatch time: the transaction receives its slot once it is +%%% included in a block. Note +%%% that Arweave refuses to create accounts with a zero balance +%%% (`validate_overspend'), and a process ID is a fresh account: the first +%%% message addressed to a process must therefore carry a `quantity' of at +%%% least 1 winston (and, until the account exists, the sender's `reward' +%%% must cover the network's new-account fee). +%%% +%%% Each process's synchronization tracks the contiguous block range +%%% `[spawn-height, synced-to]' it has indexed -- the range the node can +%%% attest to. A sync only indexes the blocks above `synced-to' (up to the +%%% confirmed tip), so a repeated read with no new blocks does no network +%%% work, and `/status' reports each tracked process's range. The device +%%% honors the following node options: +%%% +-module(dev_arweave_scheduler). +-implements(<<"arweave-scheduler@1.0">>). +-device_libraries([lib_process, lib_scheduler, lib_arweave_common]). +%%% AO-Core API functions: +-export([info/0]). +%%% Scheduling functions: +-export([schedule/3, router/4]). +%%% CU-flow functions: +-export([slot/3, status/3, next/3]). +-export([checkpoint/1]). +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +%%% The default number of blocks beneath the network tip at which we +%%% consider a block final. +-define(DEFAULT_CONFIRMATION_DEPTH, 10). +%%% The default number of blocks indexed per synchronization pass. A sync +%%% persists its progress after each chunk, so this bounds the work redone if +%%% a pass is interrupted and keeps the first sync of a long-lived process +%%% resumable. Overridable with the `arweave_scheduler_sync_chunk' option. +-define(DEFAULT_SYNC_CHUNK_BLOCKS, 1000). +%%% The number of transactions requested per query page. +-define(QUERY_PAGE_SIZE, 100). +%%% The Arweave device that we resolve chain data through. +-define(ARWEAVE_DEVICE, <<"~arweave@2.9">>). +%%% The sequencing mode of a process whose message does not name one: only the +%%% transactions addressed to it are its messages. +-define(DEFAULT_MODE, <<"target">>). + +%% @doc This device uses a default handler to route requests to the correct +%% function. +info() -> + #{ + exports => + [ + <<"status">>, + <<"next">>, + <<"schedule">>, + <<"slot">>, + <<"init">>, + <<"checkpoint">> + ], + excludes => [set, keys], + default => fun router/4 + }. + +%% @doc The default handler for the device: route all unrecognized requests +%% to `schedule'. +router(_, Base, Req, Opts) -> + ?event({arweave_scheduler_router_called, {req, Req}}), + schedule(Base, Req, Opts). + +%% @doc Return the next assignment for a process. Assumes that `Base' is a +%% `dev_process' or similar message, having an `at-slot' key. If the next +%% slot is not present in the local cache, the schedule is synchronized +%% from Arweave before the read is retried. +next(Base, Req, Opts) -> + ProcID = lib_process:process_id(Base, Req, Opts), + LastProcessed = lib_scheduler:at_slot(Base, Opts), + ?event(next, {arweave_next, {proc_id, ProcID}, {last, LastProcessed}}), + maybe + {ok, Assignment} ?= find_assignment(ProcID, LastProcessed + 1, Opts), + {ok, #{ <<"body">> => Assignment, <<"state">> => Base }} + end. + +%% @doc Read an assignment from the local cache, synchronizing the schedule +%% from Arweave if it is not yet present. +find_assignment(ProcID, Slot, Opts) -> + case dev_arweave_scheduler_cache:read(ProcID, Slot, Opts) of + {ok, Assignment} -> {ok, Assignment}; + not_found -> + maybe + {ok, _} ?= sync(ProcID, Opts), + case dev_arweave_scheduler_cache:read(ProcID, Slot, Opts) of + {ok, Assignment} -> {ok, Assignment}; + not_found -> lib_scheduler:slot_unavailable() + end + end + end. + +%% @doc A router for choosing between getting the existing schedule, or +%% dispatching a new message to Arweave. +schedule(Base, Req, Opts) -> + ?event({resolving_arweave_schedule_request, {req, Req}}), + case hb_util:key_to_atom(hb_ao:get(<<"method">>, Req, <<"GET">>, Opts)) of + post -> post_schedule(Base, Req, Opts); + get -> get_schedule(Base, Req, Opts) + end. + +%% @doc Generate and return the schedule for a process, optionally between +%% two slots -- labelled as `from' and `to'. +get_schedule(Base, Req, Opts) -> + ProcID = hb_util:human_id(lib_scheduler:find_target_id(Base, Req, Opts)), + {From, To} = lib_scheduler:parse_slot_range(Req, Opts), + maybe + {ok, #{ <<"next-slot">> := NextSlot }} ?= sync(ProcID, Opts), + Latest = NextSlot - 1, + RequestedTo = + case To of + undefined -> Latest; + _ -> min(To, Latest) + end, + {Assignments, More} = + lib_scheduler:read_assignment_range( + dev_arweave_scheduler_cache, ProcID, From, RequestedTo, Opts), + dev_arweave_scheduler_cache:assignments_to_bundle( + ProcID, + Assignments, + More, + Opts + ) + end. + +%% @doc Dispatch a new message to Arweave. The message must carry a signed +%% `tx@1.0' commitment: the caller signs the L1 transaction themselves +%% (including its `anchor' and `reward'), and this device relays it to the +%% network. The message is also written to the local cache, so that it is +%% servable by ID before the network has propagated and indexed it. +post_schedule(Base, Req, Opts) -> + maybe + {ok, ToSched} ?= lib_scheduler:load_message_to_schedule(Base, Req, Opts), + ProcID = lib_scheduler:find_target_id(Base, Req, ToSched, Opts), + ?event({arweave_post_schedule, {proc_id, ProcID}}), + {ok, OnlyCommitted} ?= lib_scheduler:only_committed(ToSched, Opts), + ok ?= ensure_tx_committed(OnlyCommitted, Opts), + dispatch(ProcID, OnlyCommitted, Opts) + end. + +%% @doc Ensure that the given message carries a signed, valid `tx@1.0' +%% commitment. The Arweave scheduler accepts only L1 transactions: ANS-104 +%% and HTTPSig commitments cannot be sequenced by the base layer. +ensure_tx_committed(Msg, Opts) -> + Devices = hb_message:commitment_devices(Msg, Opts), + Signers = hb_message:signers(Msg, Opts), + case {lists:member(<<"tx@1.0">>, Devices), Signers} of + {false, _} -> + {error, + #{ + <<"status">> => 422, + <<"body">> => + <<"The Arweave scheduler only accepts messages ", + "committed with tx@1.0.">> + } + }; + {true, []} -> + {error, + #{ + <<"status">> => 422, + <<"body">> => <<"Message must be signed.">> + } + }; + {true, _} -> + case hb_message:verify(Msg, signers, Opts) of + true -> ok; + false -> + {error, + #{ + <<"status">> => 400, + <<"body">> => <<"Message is not valid.">> + } + } + end + end. + +%% @doc Relay a committed `tx@1.0' message to the Arweave network. Only the +%% transaction header is dispatched: this scheduler sequences headers, never +%% data. A message carrying data is therefore rejected rather than uploaded -- +%% its chunks might be unavailable when the schedule is later read. The same +%% header-only rule governs synchronization, so a data-carrying transaction +%% posted directly to Arweave is still sequenced safely, by its header alone. +dispatch(ProcID, Msg, Opts) -> + TX = hb_message:convert(Msg, <<"tx@1.0">>, Opts), + case TX#tx.data of + <<>> -> dispatch_header(ProcID, TX, Msg, Opts); + _ -> + {error, + #{ + <<"status">> => 422, + <<"body">> => + <<"The Arweave scheduler sequences transaction ", + "headers only; a message carrying data cannot ", + "be scheduled.">> + } + } + end. + +dispatch_header(ProcID, TX, Msg, Opts) -> + TXID = hb_util:human_id(TX#tx.id), + ?event({dispatching_tx, {id, {explicit, TXID}}, {proc_id, ProcID}}), + maybe + {ok, _} ?= post_tx_header(TX, Opts), + {ok, _} = hb_cache:write(Msg, Opts), + % `tx-id' rather than `id': the reserved `id' key would be shadowed + % by the receipt message's own ID when read by consumers. + {ok, + #{ + <<"status">> => 202, + <<"tx-id">> => TXID, + <<"process">> => hb_util:human_id(ProcID), + <<"body">> => + <<"Transaction dispatched to Arweave. It will receive ", + "a slot in the schedule once mined.">> + } + } + end. + +%% @doc Post a transaction header to the network, via the node's `/arweave' +%% route. The route may broadcast to multiple nodes: acceptance by any one +%% of them places the transaction in the network's mempool, so the best +%% response wins, mirroring `~arweave@2.9's handling. +post_tx_header(TX, Opts) -> + Res = + hb_http:request( + #{ + <<"method">> => <<"POST">>, + <<"path">> => <<"/arweave/tx">>, + <<"body">> => hb_json:encode(ar_tx:tx_to_json_struct(TX)) + }, + no_result_cache(Opts) + ), + lib_arweave_common:best_response(Res). + +%% @doc Returns information about the current slot for a process. +slot(Base, Req, Opts) -> + ProcID = hb_util:human_id(lib_scheduler:find_target_id(Base, Req, Opts)), + ?event({getting_current_slot, {proc_id, ProcID}}), + maybe + {ok, #{ <<"next-slot">> := NextSlot }} ?= sync(ProcID, Opts), + {ok, + #{ + <<"process">> => ProcID, + <<"current">> => NextSlot - 1, + <<"cache-control">> => <<"no-store">> + } + } + end. + +%% @doc Returns information about the scheduler, including -- for each process +%% it tracks -- the contiguous block range it has indexed and can attest to. +status(_Base, _Req, Opts) -> + Wallet = hb_opts:get(priv_wallet, hb:wallet(), Opts), + ProcIDs = dev_arweave_scheduler_cache:list_processes(Opts), + {ok, + #{ + <<"address">> => hb_util:human_id(ar_wallet:to_address(Wallet)), + <<"processes">> => + hb_maps:from_list([ {P, attestable_range(P, Opts)} || P <- ProcIDs ]), + <<"cache-control">> => <<"no-store">> + } + }. + +%% @doc The block range a process has been indexed over, as it currently stands +%% in the cache (no synchronization is triggered). `from' is the spawn block and +%% `to' is the confirmed height up to which the schedule is complete; `current' +%% is the latest materialized slot. +attestable_range(ProcID, Opts) -> + case dev_arweave_scheduler_cache:read_state(ProcID, Opts) of + {ok, #{ + <<"spawn-height">> := Spawn, + <<"synced-to">> := SyncedTo, + <<"next-slot">> := NextSlot + }} -> + #{ + <<"from">> => Spawn, + <<"to">> => SyncedTo, + <<"current">> => NextSlot - 1 + }; + _ -> #{} + end. + +%% @doc Returns the current state of the scheduler. +checkpoint(State) -> {ok, State}. + +%%% Synchronization + +%% @doc Synchronize the schedule of a process from Arweave, extending the +%% contiguously-indexed block range `[spawn-height, synced-to]' up to the +%% confirmed network tip. Only the blocks above `synced-to' are indexed; a +%% process already synced to the tip does no network work at all. +sync(ProcID, Opts) -> + maybe + {ok, State} ?= ensure_initialized(ProcID, Opts), + {ok, Upper} ?= confirmed_tip(Opts), + do_sync(ProcID, State, Upper, Opts) + end. + +%% @doc Extend the synced range up to `Upper', one bounded chunk at a time. The +%% range grows contiguously from `synced-to + 1', and `{next-slot, synced-to}' +%% are persisted together after each chunk, so an interrupted sync resumes at +%% the start of the unfinished chunk (re-materializing is idempotent) rather +%% than re-deriving the whole history. +do_sync(_ProcID, State = #{ <<"synced-to">> := SyncedTo }, Upper, _Opts) + when SyncedTo >= Upper -> + {ok, State}; +do_sync(ProcID, State = #{ <<"synced-to">> := SyncedTo, <<"mode">> := Mode }, + Upper, Opts) -> + From = SyncedTo + 1, + To = min(SyncedTo + sync_chunk_blocks(Opts), Upper), + ?event( + {arweave_scheduler_sync, + {proc_id, ProcID}, + {mode, Mode}, + {from, From}, + {to, To}, + {target, Upper} + } + ), + maybe + {ok, Ordered} ?= discover(ProcID, Mode, From, To, Opts), + {ok, NewState} ?= materialize(ProcID, State, Ordered, To, Opts), + do_sync(ProcID, NewState, Upper, Opts) + end. + +%% @doc The number of blocks to index per synchronization pass. +sync_chunk_blocks(Opts) -> + hb_util:int( + hb_opts:get(arweave_scheduler_sync_chunk, ?DEFAULT_SYNC_CHUNK_BLOCKS, Opts) + ). + +%% @doc Side-effecting and mutable internal resolutions must never be served +%% from (or written to) the resolution cache: nodes default to caching every +%% HTTP resolution, which would otherwise freeze the indexing runs, queries +%% and tip lookups at their first result. +no_result_cache(Opts) -> + Opts#{ + <<"hashpath">> => ignore, + <<"cache-control">> => [<<"no-cache">>, <<"no-store">>] + }. + +%% @doc Discover the base-layer transactions that a process is sequenced by +%% within a block range, in canonical weave order, entirely from the node's own +%% index. `target' mode records each transaction's weave offset and caches its +%% header. `all' mode caches only data-free headers. +%% +%% In `target' mode the recipient match is served from the node's own +%% `~query@1.0' GraphQL endpoint, and each match is annotated with its offset -- +%% the sort key and the `offset' recorded on the assignment. No gateway is +%% queried by default. In `all' mode there is nothing to match, so no query is +%% run: the range's block headers are walked directly (see +%% `enumerate_blocks/4'). +%% +%% Returns `{Extra, TXID}' pairs in the order they are to be assigned, where +%% `Extra' is the sequencing detail recorded on each assignment. Bundled data +%% items are excluded, as this scheduler sequences the base layer only. +discover(ProcID, <<"all">>, From, To, Opts) -> + maybe + ok ?= ensure_headers(From, To, Opts), + {ok, Located} ?= enumerate_blocks(ProcID, From, To, Opts), + base_layer_blocks(Located, Opts) + end; +discover(ProcID, _Mode, From, To, Opts) -> + maybe + ok ?= ensure_offsets(From, To, Opts), + {ok, IDs} ?= query_recipients(ProcID, From, To, Opts), + {ok, + [ + {#{ <<"offset">> => Offset }, TXID} + || + {Offset, TXID} <- base_layer_offsets(IDs, Opts) + ] + } + end. + +%% @doc Enumerate every transaction in a block range from the block headers +%% themselves, in canonical chain order: blocks ascending by height, then each +%% block's transactions in the order that block lists them. This is the same +%% source `~copycat@1.0/arweave' walks while indexing, and `ensure_offsets' has +%% already cached the blocks locally, so the enumeration is normally a local +%% read. The process's own transaction is skipped: it is already slot 0. +%% +%% Returns `{Height, TXID}' pairs. Unlike a query result these are ordered by +%% construction, and that order is total -- weave offset is not, because every +%% transaction that carries no data shares the offset of the one before it. +enumerate_blocks(_ProcID, From, To, _Opts) when From > To -> {ok, []}; +enumerate_blocks(ProcID, From, To, Opts) -> + maybe + {ok, Block} ?= + hb_ao:resolve( + <>, + no_result_cache(Opts) + ), + {ok, Rest} ?= enumerate_blocks(ProcID, From + 1, To, Opts), + {ok, + [ + {From, TXID} + || + TXID <- hb_maps:get(<<"txs">>, Block, [], Opts), + TXID =/= hb_util:human_id(ProcID) + ] ++ Rest + } + end. + +%% @doc Ensure every data-free transaction header in a block range is cached. +ensure_headers(From, To, _Opts) when From > To -> ok; +ensure_headers(From, To, Opts) -> + maybe + ok ?= index_range(<<"headers">>, From, To, Opts), + ok ?= headers_indexed(From, To, Opts) + end. + +headers_indexed(From, To, Opts) -> + #{ <<"index-store">> := Store } = hb_store_arweave:store_from_opts(Opts), + Missing = + lists:dropwhile( + fun(Height) -> + case hb_store:read( + Store, + <<"block/", (hb_util:bin(Height))/binary, "/mode">>, + Opts + ) of + {ok, Mode} -> + lists:member( + Mode, + [<<"headers">>, <<"shallow">>, <<"deep">>, <<"full">>] + ); + _ -> + false + end + end, + lists:seq(From, To) + ), + case Missing of + [] -> ok; + [Height | _] -> + {error, + #{ + <<"status">> => 503, + <<"reason">> => + <<"Block range is not fully indexed locally.">>, + <<"block-height">> => Height + } + } + end. + +%% @doc Ensure the node's local Arweave index covers the block range, so that +%% every transaction in it has both a weave offset and a locally-cached header. +%% `~copycat@1.0/arweave' in `shallow' mode records an ID->offset entry for each +%% transaction and caches its (data-free) header -- populating the `field-target' +%% match index that `query_recipients' reads. `reindex=false' skips blocks the +%% node has already indexed (for any process), so the run is idempotent, +%% incremental, and shared: overlapping ranges across processes are not +%% re-fetched. +ensure_offsets(From, To, Opts) -> index_range(<<"shallow">>, From, To, Opts). + +index_range(_Mode, From, To, _Opts) when From > To -> ok; +index_range(Mode, From, To, Opts) -> + maybe + {ok, _} ?= + hb_ao:resolve( + << + "~copycat@1.0/arweave&mode=", Mode/binary, "&reindex=false", + "&from=", (hb_util:bin(To))/binary, + "&to=", (hb_util:bin(From))/binary + >>, + no_result_cache(Opts) + ), + ok + end. + +%% @doc Query for the transactions addressed to a process within a block +%% range, ordered by weave position, returning their IDs. The query is served +%% by the node's own `~query@1.0' index, whose `field-target' +%% matches are populated by `~copycat@1.0/arweave' as it indexes the range (see +%% `ensure_offsets'). +query_recipients(ProcID, From, To, Opts) -> + Query = + << + "query($after: String) { transactions(", + "recipients: [\"", (hb_util:human_id(ProcID))/binary, "\"], ", + "block: { min: ", (hb_util:bin(From))/binary, + ", max: ", (hb_util:bin(To))/binary, " }, ", + "sort: HEIGHT_ASC, ", + "first: ", (hb_util:bin(?QUERY_PAGE_SIZE))/binary, + ", after: $after", + ") { pageInfo { hasNextPage } edges { cursor node { id } } } }" + >>, + query_pages(Query, undefined, [], Opts). + +query_pages(Query, After, Acc, Opts) -> + Variables = + case After of + undefined -> #{}; + _ -> #{ <<"after">> => After } + end, + maybe + {ok, Transactions} ?= run_query(Query, Variables, Opts), + Edges = hb_maps:get(<<"edges">>, Transactions, [], Opts), + IDs = Acc ++ [ edge_id(E, Opts) || E <- Edges ], + HasNext = + hb_util:atom( + hb_ao:get( + <<"pageInfo/hasNextPage">>, + Transactions, + false, + Opts#{ <<"hashpath">> => ignore } + ) + ), + case {HasNext, Edges} of + {true, [_ | _]} -> + Cursor = + hb_maps:get(<<"cursor">>, lists:last(Edges), undefined, Opts), + query_pages(Query, Cursor, IDs, Opts); + _ -> + {ok, [ ID || ID <- IDs, is_binary(ID) ]} + end + end. + +edge_id(Edge, Opts) -> + hb_maps:get(<<"id">>, hb_maps:get(<<"node">>, Edge, #{}, Opts), undefined, Opts). + +%% @doc Run a GraphQL `transactions' query against the node's local index. +run_query(Query, Variables, Opts) -> + to_transactions( + hb_ao:resolve( + #{ <<"device">> => <<"query@1.0">> }, + #{ + <<"path">> => <<"graphql">>, + <<"method">> => <<"POST">>, + <<"query">> => Query, + <<"variables">> => Variables + }, + no_result_cache(Opts) + ), + Opts + ). + +to_transactions({ok, Response}, Opts) -> + Decoded = + hb_json:decode(hb_ao:get(<<"body">>, Response, <<"{}">>, Opts)), + {ok, + hb_ao:get( + <<"data/transactions">>, + Decoded, + #{}, + Opts#{ <<"hashpath">> => ignore } + ) + }; +to_transactions({error, Reason}, _Opts) -> + {error, + #{ + <<"status">> => 502, + <<"reason">> => <<"Arweave query failed.">>, + <<"detail">> => Reason + } + }. + +%% @doc Annotate each matched transaction with its weave offset from the +%% local index, keeping only the base-layer (`tx@1.0') transactions and +%% dropping bundled data items (indexed under the `ans104@1.0' codec) and any +%% that are not yet locally indexed. The result is sorted by offset, which is +%% the canonical weave order. +base_layer_offsets(IDs, Opts) -> + Store = hb_store_arweave:store_from_opts(Opts), + lists:keysort( + 1, + lists:filtermap( + fun(ID) -> base_layer_offset(Store, ID, Opts) end, + IDs + ) + ). + +base_layer_offset(Store, ID, Opts) -> + case offset_entry(Store, ID, Opts) of + {ok, <<"tx@1.0">>, Offset} -> {true, {Offset, ID}}; + _ -> false + end. + +%% @doc Keep the data-free headers cached by the headers-mode pass. +base_layer_blocks(Located, Opts) -> + {ok, + [ + {#{ <<"block-height">> => Height }, ID} + || + {Height, ID} <- Located, + data_free(ID, Opts) + ] + }. + +data_free(ID, Opts) -> + LocalOpts = hb_store:scope(Opts, local), + try + case hb_cache:read(ID, LocalOpts) of + {ok, Header} -> + (hb_message:convert( + Header, <<"tx@1.0">>, LocalOpts))#tx.data_size =:= 0; + _ -> + false + end + catch + _:_ -> false + end. + +%% @doc Read a transaction's local index entry, returning its codec device and +%% weave offset. +offset_entry(Store, ID, Opts) -> + case hb_store_arweave:read_offset(Store, ID, Opts) of + {ok, + #{ + <<"codec-device">> := Codec, + <<"start-offset">> := Offset + }} when is_integer(Offset) -> + {ok, Codec, Offset}; + _ -> not_found + end. + +%% @doc Materialize a chunk's assignments and record that the range is synced +%% to `To'. The discovered list holds only the messages in blocks above the +%% already-synced range, so they are exactly the next slots -- appended from +%% `next-slot' with no de-duplication. `next-slot' and `synced-to' are then +%% persisted together, so the two never diverge. +materialize(ProcID, State = #{ <<"next-slot">> := NextSlot }, Ordered, To, Opts) -> + assign(ProcID, State, NextSlot, Ordered, To, Opts). + +assign(ProcID, State, Slot, [], To, Opts) -> + NewState = State#{ <<"next-slot">> => Slot, <<"synced-to">> => To }, + ok = dev_arweave_scheduler_cache:write_state(ProcID, NewState, Opts), + {ok, NewState}; +assign(ProcID, State, Slot, [{Extra, TXID} | Rest], To, Opts) -> + case read_tx_header(TXID, Opts) of + {ok, Msg} -> + ok = write_assignment(ProcID, Slot, Extra, Msg, Opts), + assign(ProcID, State, Slot + 1, Rest, To, Opts); + {error, Err} -> + ?event( + error, + {arweave_scheduler_tx_unavailable, + {proc_id, ProcID}, + {tx, {explicit, TXID}} + } + ), + {error, Err} + end. + +%% @doc Generate and store the synthetic assignment for a message. Mirrors the +%% assignments minted by `~scheduler@1.0', but records the chain position that +%% sequences its mode: weave `offset' for `target', `block-height' for `all'. +%% Every field derives from chain data, so the assignment is deterministic +%% across nodes and is left uncommitted. +write_assignment(ProcID, Slot, Extra, Msg, Opts) -> + BaseAssignment = + lib_scheduler:base_assignment( + hb_util:human_id(ProcID), + Slot, + Msg, + Opts + ), + Assignment = hb_maps:merge(BaseAssignment, Extra, Opts), + ?event( + {minting_assignment, + {proc_id, ProcID}, + {slot, Slot}, + {extra, Extra} + } + ), + dev_arweave_scheduler_cache:write(Assignment, Opts). + +%% @doc Find or create the persisted synchronization state for a process. +ensure_initialized(ProcID, Opts) -> + case dev_arweave_scheduler_cache:read_state(ProcID, Opts) of + {ok, State} -> {ok, State}; + _ -> initialize(ProcID, Opts) + end. + +%% @doc First contact with a process: locate its spawn block, read the process +%% header, and mint the slot 0 assignment from the process message itself. The +%% canonical process header is also written to the cache so that +%% `~process@1.0' resolves to the verifying tx@1.0 form ahead of any +%% lossier gateway-derived copy. If the spawn is not yet confirmed, +%% initialization fails and is retried on the next synchronization. +initialize(ProcID, Opts) -> + ?event({initializing_arweave_schedule, {proc_id, ProcID}}), + maybe + {ok, SpawnHeight} ?= spawn_height(ProcID, Opts), + {ok, Process} ?= read_tx_header(ProcID, Opts), + {ok, _} = hb_cache:write(Process, Opts), + Mode = mode(Process, Opts), + {ok, Zero} ?= slot_zero(Mode, ProcID, SpawnHeight, Opts), + ok = write_assignment(ProcID, 0, Zero, Process, Opts), + % `synced-to' starts one below the spawn block: slot 0 is the process + % itself, and no message-bearing block has been indexed yet. The first + % sync begins its range at the spawn block, catching any messages mined + % alongside the process. + State = + #{ + <<"next-slot">> => 1, + <<"spawn-height">> => SpawnHeight, + <<"synced-to">> => SpawnHeight - 1, + <<"mode">> => Mode + }, + ok = dev_arweave_scheduler_cache:write_state(ProcID, State, Opts), + {ok, State} + end. + +%% @doc The sequencing mode a process message asks for. A mode the device does +%% not implement falls back to `target' rather than erroring: the process +%% message is immutable, so a typo in a spawn tag would otherwise wedge the +%% process permanently. +mode(Process, Opts) -> + case hb_ao:get(<<"scheduler-mode">>, Process, ?DEFAULT_MODE, Opts) of + <<"all">> -> <<"all">>; + _ -> ?DEFAULT_MODE + end. + +%% @doc The sequencing detail recorded on slot 0. The process message is its +%% own first message, so in `all' mode it carries the height of its spawn block +%% -- the process's clock starts at the block it was created in -- and in the +%% modes that sort by weave position, its offset. Only those modes index the +%% spawn block to read that offset. +slot_zero(<<"all">>, _ProcID, SpawnHeight, _Opts) -> + {ok, #{ <<"block-height">> => SpawnHeight }}; +slot_zero(_Mode, ProcID, SpawnHeight, Opts) -> + maybe + ok ?= ensure_offsets(SpawnHeight, SpawnHeight, Opts), + {ok, Offset} ?= tx_offset(ProcID, Opts), + {ok, #{ <<"offset">> => Offset }} + end. + +%% @doc Read an L1 transaction as a header-only message from the node's stores. +%% `~copycat@1.0/arweave' caches the (data-free) header locally while indexing, +%% so the read is normally served straight from the local store; a miss falls +%% through the rest of the store chain (which may reach the header faster than a +%% specific Arweave host). The header carries the transaction's tags and its +%% tx@1.0 signature -- so its committed ID is the transaction ID and it verifies +%% -- and no data is attached at this layer, so the schedule never depends on +%% the availability of any transaction's data. +%% The header is taken as it is read, links and all. Forcing it to load in full +%% would demand a transaction's data -- the very thing this schedule promises +%% never to depend on -- and any transaction on the network can be a message +%% here, so most of them carry some. Loading only what happens to be available +%% locally would be worse than either: two nodes would mint different +%% assignments for the same transaction. +%% Only the node's own stores are consulted, because only they are known to +%% hold what this schedule wants: the indexing pass caches a data-free header +%% for every plain transaction it walks. Letting the read fall through to a +%% gateway would answer with the whole transaction instead -- data, or a bundle +%% decoded into items whose contents are not on this node -- and the schedule +%% would then depend on the availability of data it promises never to touch. +%% Anything not cached locally is fetched as a header in its own right. +read_tx_header(TXID, Opts) -> + case hb_cache:read(TXID, hb_store:scope(Opts, local)) of + {ok, Msg} -> {ok, Msg}; + _ -> fetch_tx_header(TXID, Opts) + end. + +%% @doc Fetch a transaction's data-free header from Arweave. The indexing pass +%% caches the headers of plain L1 transactions as it goes, but deliberately not +%% those of bundles -- and an `all'-mode schedule sequences every transaction in +%% a block, bundles included. Their headers are fetched on demand rather than +%% failing the whole range; `exclude-data' keeps the schedule header-only, so it +%% still never depends on the availability of a transaction's data. +fetch_tx_header(TXID, Opts) -> + Res = + hb_ao:resolve( + << + ?ARWEAVE_DEVICE/binary, "/tx", + "&tx=", TXID/binary, + "&exclude-data=true" + >>, + no_result_cache(Opts) + ), + case Res of + {ok, Msg} -> {ok, Msg}; + _ -> + {error, + #{ + <<"status">> => 503, + <<"reason">> => + <<"Transaction header is not yet retrievable ", + "from Arweave.">>, + <<"tx">> => TXID + } + } + end. + +%% @doc Read a transaction's weave offset from the local index. A transaction +%% that has not yet been indexed has no offset. +tx_offset(TXID, Opts) -> + Store = hb_store_arweave:store_from_opts(Opts), + case offset_entry(Store, TXID, Opts) of + {ok, _Codec, Offset} -> {ok, Offset}; + not_found -> + {error, + #{ + <<"status">> => 503, + <<"reason">> => + <<"Transaction is not yet indexed locally.">>, + <<"tx">> => TXID + } + } + end. + +%% @doc Locate the block height that includes a transaction, via the Arweave +%% `tx status' API. An unconfirmed transaction has no block height: a process +%% cannot be scheduled against until its spawn is confirmed. +spawn_height(TXID, Opts) -> + Res = + hb_http:request( + #{ + <<"method">> => <<"GET">>, + <<"path">> => <<"/arweave/tx/", TXID/binary, "/status">> + }, + no_result_cache(Opts) + ), + case lib_arweave_common:best_response(Res) of + {ok, Response} -> + Status = + hb_json:decode(hb_ao:get(<<"body">>, Response, <<"{}">>, Opts)), + case hb_maps:get(<<"block_height">>, Status, undefined, Opts) of + undefined -> {error, unconfirmed_process(TXID)}; + Height -> {ok, hb_util:int(Height)} + end; + _ -> {error, unconfirmed_process(TXID)} + end. + +unconfirmed_process(TXID) -> + #{ + <<"status">> => 404, + <<"reason">> => <<"Process is not yet confirmed on Arweave.">>, + <<"process">> => TXID + }. + +%% @doc The height up to which the schedule may safely be extended: the +%% network tip less the configured confirmation depth, optionally capped by +%% the `arweave_scheduler_max_height' node option. +confirmed_tip(Opts) -> + maybe + {ok, Height} ?= + hb_ao:resolve( + <>, + no_result_cache(Opts) + ), + Depth = + hb_util:int( + hb_opts:get( + arweave_scheduler_confirmation_depth, + ?DEFAULT_CONFIRMATION_DEPTH, + Opts + ) + ), + Confirmed = hb_util:int(Height) - Depth, + case hb_opts:get(arweave_scheduler_max_height, undefined, Opts) of + undefined -> {ok, Confirmed}; + Max -> {ok, min(Confirmed, hb_util:int(Max))} + end + end. + +%%% Tests + +%%% The permanent test fixture: a `lua@5.3a' process scheduled by this +%%% device, seeded onto Arweave with three state-transformation messages +%%% (`test/arweave-scheduler-test.lua'; all transactions carry +%%% `test-suite: arweave-scheduler' tags). `?FIXTURE_MSG2' was posted +%%% directly to arweave.net, never passing through a HyperBEAM node: its +%%% presence in the schedule proves foreign-message indexing. Arweave is +%%% permanent, so these tests are repeatable against the live network. +%%% Spawn block 1958986; messages at 1958993, 1958994 and 1958995. +-define(FIXTURE_PROCESS, <<"q3SycbYpO1lz-S6V2kd7FG3DIZ3AU0NrKKxWw4C3yos">>). +-define(FIXTURE_MSG1, <<"Y8GnKytC57VUk7W7FlGnD1oH4OAwFHyP-pJPGCJBa3Y">>). +-define(FIXTURE_MSG2, <<"luf1fFmhi0RMIZNnFmv1QgK2SJU5plAFQ3ZxndUsLpk">>). +-define(FIXTURE_MSG3, <<"UThcMtT6zy0pJ7iXUsMTFUMu3Xhv51EAjn5BzqHMhR4">>). +-define(FIXTURE_MAX_HEIGHT, 1958995). + +test_opts() -> + TestStore = hb_test_utils:test_store(), + IndexStore = hb_test_utils:test_store(), + % The full default config is merged in so the node has the Arweave routes + % that `~copycat@1.0' and `~arweave@2.9' resolve through. + (hb_opts:default_message())#{ + <<"store">> => [ + TestStore, + % The local Arweave offset index that discovery orders by. + #{ + <<"store-module">> => hb_store_arweave, + <<"name">> => <<"cache-arweave">>, + <<"index-store">> => [IndexStore] + }, + % Serves the Lua module transaction's data (its source) and the + % transaction headers `~copycat@1.0/arweave' fetches while indexing. + #{ + <<"store-module">> => hb_store_gateway, + <<"local-store">> => [TestStore] + } + ], + <<"arweave-index-store">> => #{ <<"index-store">> => [IndexStore] }, + <<"arweave-index-workers">> => 8, + <<"arweave-scheduler-confirmation-depth">> => 1, + <<"arweave-scheduler-max-height">> => ?FIXTURE_MAX_HEIGHT, + <<"priv-wallet">> => ar_wallet:new() + }. + +%% @doc Scheduling a message that does not carry a `tx@1.0' commitment must +%% be rejected. +reject_non_tx_message_test() -> + Opts = #{ <<"priv-wallet">> => ar_wallet:new() }, + Msg = + hb_message:commit( + #{ + <<"target">> => hb_util:human_id(crypto:strong_rand_bytes(32)), + <<"test-key">> => <<"test-value">> + }, + Opts + ), + ?assertMatch( + {error, #{ <<"status">> := 422 }}, + hb_ao:resolve( + #{ <<"device">> => <<"arweave-scheduler@1.0">> }, + #{ + <<"path">> => <<"schedule">>, + <<"method">> => <<"POST">>, + <<"body">> => Msg + }, + Opts + ) + ). + +%% @doc A validly `tx@1.0'-committed message that carries data must also be +%% rejected: this scheduler sequences transaction headers only, so it will not +%% dispatch (nor later depend on the chunk availability of) a message's data. +reject_data_message_test() -> + Opts = #{ <<"priv-wallet">> => ar_wallet:new() }, + Msg = + hb_message:commit( + #{ + <<"target">> => hb_util:human_id(crypto:strong_rand_bytes(32)), + <<"data">> => <<"unschedulable-data">> + }, + Opts, + #{ <<"commitment-device">> => <<"tx@1.0">> } + ), + ?assertMatch( + {error, #{ <<"status">> := 422 }}, + hb_ao:resolve( + #{ <<"device">> => <<"arweave-scheduler@1.0">> }, + #{ + <<"path">> => <<"schedule">>, + <<"method">> => <<"POST">>, + <<"body">> => Msg + }, + Opts + ) + ). + +%% @doc Base-layer annotation keeps only tx@1.0 offsets and sorts by offset. +base_layer_offsets_test() -> + Store = hb_test_utils:test_store(), + hb_store:start(Store), + ArwStore = #{ <<"index-store">> => [Store] }, + Opts = #{ <<"arweave-index-store">> => ArwStore }, + Late = hb_util:human_id(crypto:strong_rand_bytes(32)), + Early = hb_util:human_id(crypto:strong_rand_bytes(32)), + Bundled = hb_util:human_id(crypto:strong_rand_bytes(32)), + Unindexed = hb_util:human_id(crypto:strong_rand_bytes(32)), + ok = hb_store_arweave:write_offset(ArwStore, Late, <<"tx@1.0">>, 200, 1), + ok = hb_store_arweave:write_offset(ArwStore, Early, <<"tx@1.0">>, 100, 1), + ok = hb_store_arweave:write_offset(ArwStore, Bundled, <<"ans104@1.0">>, 150, 1), + ?assertEqual( + [{100, Early}, {200, Late}], + base_layer_offsets([Late, Bundled, Early, Unindexed], Opts) + ). + +%% @doc The sequencing mode is read from the process message, and anything the +%% device does not implement leaves the process sequenced as normal rather than +%% wedging it: the process message cannot be corrected once it is on-chain. +mode_test() -> + ?assertEqual(<<"all">>, mode(#{ <<"scheduler-mode">> => <<"all">> }, #{})), + ?assertEqual(?DEFAULT_MODE, mode(#{}, #{})), + ?assertEqual( + ?DEFAULT_MODE, + mode(#{ <<"scheduler-mode">> => <<"sideways">> }, #{}) + ). + +%% @doc A process sequenced by the whole chain has a clock from slot 0. +slot_zero_test() -> + ?assertEqual( + {ok, #{ <<"block-height">> => 3 }}, + slot_zero(<<"all">>, ignored, 3, #{}) + ). + +%% @doc Write a block into the node's block cache at the height pseudo-path +%% `~copycat@1.0/arweave' caches it under, so the enumerator reads it locally +%% rather than from the network. +write_test_block(Height, TXs, Opts) -> + {ok, MsgID} = + hb_cache:write(#{ <<"height">> => Height, <<"txs">> => TXs }, Opts), + hb_cache:link( + MsgID, + hb_path:to_binary( + [?ARWEAVE_DEVICE, <<"block">>, <<"height">>, hb_util:bin(Height)] + ), + Opts + ), + ok. + +%% @doc In `all' mode the schedule is enumerated from the block headers, in +%% chain order, and the process's own transaction is not re-assigned: it is +%% already slot 0. +enumerate_blocks_test() -> + Store = hb_test_utils:test_store(), + hb_store:start(Store), + Opts = #{ <<"store">> => [Store] }, + ProcID = hb_util:human_id(crypto:strong_rand_bytes(32)), + First = hb_util:human_id(crypto:strong_rand_bytes(32)), + Second = hb_util:human_id(crypto:strong_rand_bytes(32)), + Third = hb_util:human_id(crypto:strong_rand_bytes(32)), + ok = write_test_block(10, [First, ProcID], Opts), + ok = write_test_block(11, [Second, Third], Opts), + ?assertEqual( + {ok, [{10, First}, {11, Second}, {11, Third}]}, + enumerate_blocks(ProcID, 10, 11, Opts) + ). + +%% @doc Block-enumerated transactions keep chain order and only locally cached, +%% data-free transaction headers. +base_layer_blocks_test() -> + Store = hb_test_utils:test_store(), + hb_store:start(Store), + Opts = #{ <<"store">> => [Store], <<"priv-wallet">> => ar_wallet:new() }, + DataFree = + hb_message:commit( + #{ <<"target">> => hb_util:human_id(crypto:strong_rand_bytes(32)) }, + Opts, + #{ <<"commitment-device">> => <<"tx@1.0">> } + ), + DataBearing = + hb_message:commit( + #{ <<"data">> => <<"not-a-message">> }, + Opts, + #{ <<"commitment-device">> => <<"tx@1.0">> } + ), + DataFreeID = hb_message:id(DataFree, signed, Opts), + DataBearingID = hb_message:id(DataBearing, signed, Opts), + {ok, _} = hb_cache:write(DataFree, Opts), + {ok, _} = hb_cache:write(DataBearing, Opts), + ?assertEqual( + {ok, [{#{ <<"block-height">> => 10 }, DataFreeID}]}, + base_layer_blocks( + [{10, DataFreeID}, {10, DataBearingID}, {10, <<"missing">>}], + Opts + ) + ). + +%% @doc An `all'-mode assignment records the height that sequenced it, and it +%% survives the cache round trip that a process reads its schedule back through. +%% This is the contract `~arweave-swap@1.0' reads its clock from. +all_mode_assignment_test() -> + Store = hb_test_utils:test_store(), + hb_store:start(Store), + Opts = #{ <<"store">> => [Store], <<"priv-wallet">> => ar_wallet:new() }, + ProcID = hb_util:human_id(crypto:strong_rand_bytes(32)), + Msg = + hb_message:commit( + #{ <<"target">> => ProcID }, + Opts, + #{ <<"commitment-device">> => <<"tx@1.0">> } + ), + ok = + write_assignment( + ProcID, + 1, + #{ <<"block-height">> => 1958986 }, + Msg, + Opts + ), + {ok, Assignment} = dev_arweave_scheduler_cache:read(ProcID, 1, Opts), + ?assertEqual(1, hb_util:int(hb_ao:get(<<"slot">>, Assignment, Opts))), + ?assertEqual( + 1958986, + hb_util:int(hb_ao:get(<<"block-height">>, Assignment, Opts)) + ). + +%% @doc The mode is pinned in the persisted state, so a schedule can never be +%% half-derived in each mode. +state_mode_round_trip_test() -> + Store = hb_test_utils:test_store(), + hb_store:start(Store), + Opts = #{ <<"store">> => [Store] }, + ProcID = hb_util:human_id(crypto:strong_rand_bytes(32)), + State = + #{ + <<"next-slot">> => 1, + <<"spawn-height">> => 10, + <<"synced-to">> => 9, + <<"mode">> => <<"all">> + }, + ok = dev_arweave_scheduler_cache:write_state(ProcID, State, Opts), + ?assertEqual({ok, State}, dev_arweave_scheduler_cache:read_state(ProcID, Opts)). + +%% @doc `/status' reports each tracked process's contiguously-indexed block +%% range straight from the cache, without triggering a synchronization. +status_attestable_range_test() -> + Store = hb_test_utils:test_store(), + hb_store:start(Store), + Opts = #{ <<"store">> => [Store], <<"priv-wallet">> => ar_wallet:new() }, + ProcID = hb_util:human_id(crypto:strong_rand_bytes(32)), + ok = + dev_arweave_scheduler_cache:write_state( + ProcID, + #{ + <<"next-slot">> => 4, + <<"spawn-height">> => 100, + <<"synced-to">> => 150, + <<"mode">> => ?DEFAULT_MODE + }, + Opts + ), + {ok, #{ <<"processes">> := Processes }} = status(#{}, #{}, Opts), + ?assertEqual( + #{ <<"from">> => 100, <<"to">> => 150, <<"current">> => 3 }, + hb_maps:get(ProcID, Processes, not_found, Opts) + ). + +%% @doc Synchronize the fixture schedule from the live network, retrying on +%% transient indexing failures (for example, gateway rate limits while +%% fetching block transaction headers). +fixture_sync(_Opts, 0) -> {error, fixture_sync_failed}; +fixture_sync(Opts, Attempts) -> + Res = + hb_ao:resolve( + #{ <<"device">> => <<"arweave-scheduler@1.0">> }, + #{ + <<"path">> => <<"schedule">>, + <<"method">> => <<"GET">>, + <<"target">> => ?FIXTURE_PROCESS + }, + Opts + ), + case Res of + {ok, Schedule} -> {ok, Schedule}; + {error, _} -> + timer:sleep(5000), + fixture_sync(Opts, Attempts - 1) + end. + +%% @doc Read the fixture schedule from the live network and check that the +%% assignments arrive in seeded order, including the message that never +%% passed through a HyperBEAM node. +fixture_schedule_test_() -> + {timeout, 1200, fun fixture_schedule/0}. +fixture_schedule() -> + Opts = test_opts(), + {ok, Schedule} = fixture_sync(Opts, 5), + Assignments = + hb_ao:normalize_keys( + hb_ao:get(<<"assignments">>, Schedule, Opts), + Opts + ), + SlotIDs = + lists:map( + fun(Slot) -> + Assignment = + hb_maps:get(hb_util:bin(Slot), Assignments, not_found, Opts), + ?assertEqual( + Slot, + hb_util:int(hb_ao:get(<<"slot">>, Assignment, Opts)) + ), + Body = hb_ao:get(<<"body">>, Assignment, Opts), + % The body is a header: it verifies and its committed ID is + % the transaction ID, but it carries no data payload -- the + % schedule never depends on data availability. + ?assert(hb_message:verify(Body, all, Opts)), + BodyKeys = hb_maps:keys(hb_message:uncommitted(Body, Opts), Opts), + ?assertNot(lists:member(<<"data">>, BodyKeys)), + ?assertNot(lists:member(<<"body">>, BodyKeys)), + hb_message:id(Body, signed, Opts) + end, + [0, 1, 2, 3] + ), + ?assertEqual( + [?FIXTURE_PROCESS, ?FIXTURE_MSG1, ?FIXTURE_MSG2, ?FIXTURE_MSG3], + SlotIDs + ). + +%% @doc Compute the fixture process from its Arweave schedule and check the +%% state transformations applied in order: `setstate 1000', `addstate 337', +%% then `querystate' reporting the result. +fixture_lua_e2e_test_() -> + {timeout, 1200, fun fixture_lua_e2e/0}. +fixture_lua_e2e() -> + Opts = test_opts(), + % Prime the schedule first: synchronization indexes the spawn block, so + % the process message is read back as its canonical tx@1.0 decoding + % (rather than the gateway store's lossy representation). + {ok, _} = fixture_sync(Opts, 5), + {ok, RawProcess} = hb_cache:read(?FIXTURE_PROCESS, Opts), + Process = hb_cache:ensure_all_loaded(RawProcess, Opts), + % Luerl represents Lua numbers as floats. + ?assertEqual( + {ok, 1000.0}, + hb_ao:resolve( + Process, + #{ <<"path">> => <<"compute/state">>, <<"slot">> => 1 }, + Opts + ) + ), + ?assertEqual( + {ok, 1337.0}, + hb_ao:resolve( + Process, + #{ <<"path">> => <<"compute/state">>, <<"slot">> => 2 }, + Opts + ) + ), + ?assertEqual( + {ok, <<"state=1337.0">>}, + hb_ao:resolve( + Process, + #{ + <<"path">> => <<"compute/results/output/body">>, + <<"slot">> => 3 + }, + Opts + ) + ), + ?assertEqual({ok, 1337.0}, hb_ao:resolve(Process, <<"now/state">>, Opts)). + +%% @doc A first sync indexes the range in bounded chunks, persisting its +%% progress after each. With a chunk size smaller than the fixture's span -- +%% so the messages fall in different chunks -- the schedule is still assembled +%% in order, and `/status' reports the range synced all the way to the tip. +chunked_sync_test_() -> + {timeout, 1200, fun chunked_sync/0}. +chunked_sync() -> + Opts = (test_opts())#{ <<"arweave-scheduler-sync-chunk">> => 4 }, + {ok, Schedule} = fixture_sync(Opts, 5), + Assignments = + hb_ao:normalize_keys( + hb_ao:get(<<"assignments">>, Schedule, Opts), + Opts + ), + SlotIDs = + lists:map( + fun(Slot) -> + Assignment = + hb_maps:get(hb_util:bin(Slot), Assignments, not_found, Opts), + hb_message:id(hb_ao:get(<<"body">>, Assignment, Opts), signed, Opts) + end, + [0, 1, 2, 3] + ), + ?assertEqual( + [?FIXTURE_PROCESS, ?FIXTURE_MSG1, ?FIXTURE_MSG2, ?FIXTURE_MSG3], + SlotIDs + ), + % A completed sync makes the whole fixture range attestable: the spawn + % block through the tip. + {ok, #{ <<"processes">> := Processes }} = status(#{}, #{}, Opts), + Range = hb_maps:get(?FIXTURE_PROCESS, Processes, not_found, Opts), + ?assertEqual(?FIXTURE_MAX_HEIGHT, hb_util:int(hb_ao:get(<<"to">>, Range, Opts))), + ?assertEqual(3, hb_util:int(hb_ao:get(<<"current">>, Range, Opts))). diff --git a/src/preloaded/process/dev_arweave_scheduler_cache.erl b/src/preloaded/process/dev_arweave_scheduler_cache.erl new file mode 100644 index 000000000..14beb5e37 --- /dev/null +++ b/src/preloaded/process/dev_arweave_scheduler_cache.erl @@ -0,0 +1,118 @@ +%%% @doc A cache for the `~arweave-scheduler@1.0' device: stores the +%%% synthetic assignments derived from Arweave L1 transactions, as well as +%%% the per-process synchronization state of the indexer. +-module(dev_arweave_scheduler_cache). +-export([write/2, read/3, list_processes/1]). +-export([read_state/2, write_state/3]). +-export([assignments_to_bundle/4]). +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +%%% The pseudo-path prefix which the arweave-scheduler cache should use. +-define(CACHE_PREFIX, <<"~arweave-scheduler@1.0">>). + +%% @doc Merge the scheduler store with the main store. Used before writing +%% to the cache. +opts(Opts) -> lib_scheduler:cache_opts(Opts). + +%% @doc Write an assignment message into the cache. Assignments are +%% deterministic derivations of chain data, so writes are idempotent: +%% concurrent synchronizations of the same process converge on identical +%% messages at identical paths. +write(Assignment, Opts) -> + lib_scheduler:write_assignment(?CACHE_PREFIX, Assignment, Opts). + +%% @doc Get an assignment message from the cache. +read(ProcID, Slot, Opts) -> + lib_scheduler:read_assignment(?CACHE_PREFIX, ProcID, Slot, Opts). + +%% @doc List the processes that the device holds synchronization state for. +list_processes(RawOpts) -> + Opts = opts(RawOpts), + Store = hb_opts:get(store, no_viable_store, Opts), + case hb_store:list(Store, <>, Opts) of + {ok, Names} -> Names; + _ -> [] + end. + +%% @doc Read the persisted synchronization state for a process. Returns +%% `{ok, State}' with the `next-slot' to be assigned, the process's +%% `spawn-height', `synced-to' (the highest block whose messages have been +%% contiguously indexed and materialized), and the `mode' the process is +%% sequenced in, or propagates the store's `not_found'. The mode is pinned here +%% at first contact: it is read from the process message, which cannot change, +%% and a schedule half-derived in each mode would be undetectable. +read_state(ProcID, RawOpts) -> + Opts = opts(RawOpts), + Store = hb_opts:get(store, no_viable_store, Opts), + maybe + {ok, NextSlot} ?= + hb_store:read(Store, state_path(ProcID, <<"next-slot">>), Opts), + {ok, SpawnHeight} ?= + hb_store:read(Store, state_path(ProcID, <<"spawn-height">>), Opts), + {ok, SyncedTo} ?= + hb_store:read(Store, state_path(ProcID, <<"synced-to">>), Opts), + {ok, Mode} ?= + hb_store:read(Store, state_path(ProcID, <<"mode">>), Opts), + {ok, + #{ + <<"next-slot">> => hb_util:int(NextSlot), + <<"spawn-height">> => hb_util:int(SpawnHeight), + <<"synced-to">> => hb_util:int(SyncedTo), + <<"mode">> => Mode + } + } + end. + +%% @doc Persist the synchronization state for a process. +write_state(ProcID, State, RawOpts) -> + Opts = opts(RawOpts), + Store = hb_opts:get(store, no_viable_store, Opts), + #{ + <<"next-slot">> := NextSlot, + <<"spawn-height">> := SpawnHeight, + <<"synced-to">> := SyncedTo, + <<"mode">> := Mode + } = State, + hb_store:write( + Store, + #{ + state_path(ProcID, <<"next-slot">>) => hb_util:bin(NextSlot), + state_path(ProcID, <<"spawn-height">>) => hb_util:bin(SpawnHeight), + state_path(ProcID, <<"synced-to">>) => hb_util:bin(SyncedTo), + state_path(ProcID, <<"mode">>) => Mode + }, + Opts + ). + +state_path(ProcID, Key) -> + hb_path:to_binary( + [ + ?CACHE_PREFIX, + <<"state">>, + hb_util:human_id(ProcID), + Key + ] + ). + +%% @doc Generate a `GET /schedule' response for a process. Mirrors +%% `dev_scheduler_formats:assignments_to_bundle/4' (which cannot be called +%% across device namespaces), less the bundle-level `timestamp', +%% `block-height' and `block-hash'. Those describe the current weave tip +%% rather than the blocks that sequenced these assignments, so they would be +%% misleading on a schedule that is a deterministic read of historical chain +%% data. Each assignment carries the position that sequences its mode. +assignments_to_bundle(ProcID, Assignments, More, RawOpts) -> + Opts = lib_scheduler:format_opts(RawOpts), + {ok, #{ + <<"type">> => <<"schedule">>, + <<"process">> => hb_util:human_id(ProcID), + <<"continues">> => hb_util:atom(More), + <<"assignments">> => + hb_message:normalize_commitments( + hb_maps:from_list( + [ {hb_ao:get(<<"slot">>, A, Opts), A} || A <- Assignments ] + ), + Opts + ) + }}. diff --git a/src/preloaded/process/dev_arweave_swap.erl b/src/preloaded/process/dev_arweave_swap.erl new file mode 100644 index 000000000..7c59c7aa6 --- /dev/null +++ b/src/preloaded/process/dev_arweave_swap.erl @@ -0,0 +1,1448 @@ +%%% @doc A device for selling a process's tokens for native Arweave value. +%%% +%%% The two halves of the trade live in different places, and that asymmetry +%%% is the whole design. The token half lives here: this device debits and +%%% credits the `balances' submessage that a `~token@1.0' implementation keeps +%%% in the same process's base, so it can hold tokens in escrow and pay them +%%% out with certainty. The Arweave half does not live here at all -- AR moves +%%% directly from buyer to seller as an ordinary layer-1 transfer, which no +%%% process can hold, redirect or refund. +%%% +%%% A process using this device is therefore sequenced by +%%% `~arweave-scheduler@1.0' in its `all' mode: every data-free base-layer +%%% transaction becomes a slot, so a payment between two addresses that the +%%% process is not a party to is nonetheless something the process sees, and +%%% can settle against. Its process message reads: +%%% 
+%%%     scheduler-device: arweave-scheduler@1.0
+%%%     scheduler-mode:   all
+%%%     execution-device: arweave-swap@1.0
+%%%
+%%% +%%% The protocol is four messages: +%%% +%%% +%%% An offer does not expire: it stands until its seller withdraws it or a +%%% buyer completes it. +%%% +%%% The `deposit' is the buyer's collateral against taking exclusivity +%%% and then abandoning it -- pledged when they register, returned when they +%%% pay, forfeit to the seller when their reservation lapses unpaid. It is +%%% denominated in the process's own token because that is the only thing this +%%% device can decline to honour: a registration naming an offer that is +%%% already gone takes nothing. AR could not serve, because AR settles on the +%%% weave whatever any process decides. +%%% +%%% Paying an open offer without registering first is permitted and unwise: +%%% the seller may withdraw, or another buyer complete it, while the payment is +%%% in flight. +%%% +%%% The book holds live offers only, so opening an offer and withdrawing it +%%% leaves the process exactly as it was. Escrowed goods, pledged collateral +%%% and balances always sum to the supply. +%%% +%%% Reservations are measured in Arweave block heights, read from the +%%% `block-height' that `all'-mode assignments carry. That is the only clock +%%% the device has, and deliberately so: reading the chain tip during a compute +%%% would be non-deterministic, and `~process@1.0' caches every slot result +%%% forever. +-module(dev_arweave_swap). +-implements(<<"arweave-swap@1.0">>). +%%% AO-Core API functions: +-export([info/0, compute/3, set/3]). +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +%%% The balances submessage this device settles in. Owned by the process's +%%% token implementation; the swap only moves value inside it. +-define(BALANCES, <<"balances">>). +%%% How long a reservation lasts when an offer does not say, counted in blocks +%%% from the buyer's own `register-interest'. +-define(DEFAULT_DEADLINE, 20). + +%% @doc Every state transition in this device is driven by the schedule, never +%% by a direct request, so every key routes to `compute'. +%% +%% The key a slot resolves is chosen by the scheduled transaction's own `path' +%% tag, and this process is sequenced by every data-free transaction on Arweave: +%% whatever a stranger writes there must be applied like any other message. So +%% there is no `exports' list -- a key outside it would fall through to `~message@1.0', +%% answer `not_found', fail its slot and wedge the process permanently -- and no +%% `excludes' list either, since an excluded key is handed to `~message@1.0' +%% instead, whose `set' would let a passer-by write the process's own balances +%% and whose `keys' would replace the state with a list of key names. `set' is +%% therefore implemented here. +%% +%% `info' is deliberately arity 0. A device's `info' is always exported, and had +%% it taken the base as an argument it would also answer the `info' key, so a +%% transaction tagged `path: info' would replace the process state with this +%% map. +info() -> + #{ default => fun router/4 }. + +%% @doc Apply any scheduled message, whatever it asked to be routed to. +router(_Key, Base, Assignment, Opts) -> + compute(Base, Assignment, Opts). + +%% @doc Setting the device is honoured, and nothing else is. `lib_process' puts +%% the process's own device back with a `set' after every slot, and reading the +%% state as a message is itself a device set, so refusing those would leave the +%% device unable to read itself. A scheduled message that asks to be routed to +%% `set' is applied like any other message instead of being allowed to write the +%% state directly. +%% +%% The device is written here rather than delegated to `~message@1.0', because +%% delegating means viewing this state as a message, which is another device +%% set: it would arrive back here forever. +set(Base, Req, Opts) -> + case hb_maps:keys(Req, Opts) -- [<<"path">>, <<"set-mode">>] of + [<<"device">>] -> + {ok, + Base#{ + <<"device">> => + hb_maps:get(<<"device">>, Req, undefined, Opts) + } + }; + _ -> compute(Base, Req, Opts) + end. + +%% @doc Apply one assignment to the swap's state. +%% +%% In `all' mode the overwhelming majority of slots are unrelated Arweave +%% traffic, so the classification below is ordered by cost: advance the clock, +%% then compare a single field -- the transaction's `target' -- against the +%% process, and only then look at tags. A transaction that is neither addressed +%% to the process nor a payment against a live order leaves the state exactly as +%% it was. +%% The process is identified from the assignment rather than from the process +%% message, which `lib_process:process_id/3' would re-verify the signature of on +%% every data-free transaction. +compute(Base, Assignment, Opts) -> + Height = hb_util:int(field(<<"block-height">>, Assignment, 0, Opts)), + ProcID = field(<<"process">>, Assignment, <<>>, Opts), + Body = field(<<"body">>, Assignment, #{}, Opts), + Advanced = advance(Base, Height, Opts), + case tx_field(Body, <<"target">>, <<>>, Opts) of + ProcID -> {ok, control(Advanced, Body, Height, Opts)}; + Target -> {ok, payment(Advanced, Body, Target, Height, Opts)} + end. + +%%% Order lifecycle + +%% @doc Route a transaction addressed to the process by its `action'. An +%% unknown action is not an error: in this mode anyone may send the process +%% anything, and a process that could be wedged by a stranger's transaction +%% would not survive its own schedule. +control(Base, Body, Height, Opts) -> + case field(<<"action">>, Body, <<>>, Opts) of + <<"make-offer">> -> make_offer(Base, Body, Height, Opts); + <<"cancel-order">> -> cancel_order(Base, Body, Opts); + <<"register-interest">> -> register_interest(Base, Body, Height, Opts); + _ -> Base + end. + +%% @doc Open an offer, moving the goods into escrow. +%% Nothing is written unless the whole offer is admissible, so a rejected offer +%% is indistinguishable from a transaction that was never sent. +make_offer(Base, Body, Height, Opts) -> + maybe + {ok, Seller} ?= signer(Body, Opts), + {ok, Quantity} ?= amount(<<"offer-quantity">>, Body, Opts), + {ok, Asking} ?= amount(<<"asking">>, Body, Opts), + {ok, Deposit} ?= amount(<<"deposit">>, Body, Opts), + {ok, Fee} ?= amount(<<"minimum-fee">>, Body, Opts), + {ok, Deadline} ?= amount(<<"deadline">>, Body, ?DEFAULT_DEADLINE, Opts), + Recipient = field(<<"recipient">>, Body, Seller, Opts), + true ?= Quantity >= 1, + true ?= Asking >= 1, + true ?= Deposit >= 0, + true ?= Fee >= 0, + true ?= Deadline >= 1, + % The seller escrows the goods and nothing else: the deposit is asked + % of the buyer, not staked by the seller. A seller whose whole holding + % is the thing being sold -- the single unit of a name -- can therefore + % still make an offer that asks collateral. + true ?= balance(Base, Seller, Opts) >= Quantity, + OrderID = hb_util:human_id(hb_message:id(Body, signed, Opts)), + not_found ?= hb_maps:get(OrderID, order_book(Base, Opts), not_found, Opts), + Order = + #{ + <<"order-id">> => OrderID, + <<"creator">> => Seller, + <<"recipient">> => Recipient, + <<"quantity">> => Quantity, + <<"asking">> => Asking, + <<"deposit">> => Deposit, + <<"minimum-fee">> => Fee, + <<"deadline">> => Deadline, + <<"created-at">> => Height, + <<"status">> => <<"open">> + }, + ?event( + {swap_order_opened, + {order, OrderID}, + {seller, Seller}, + {quantity, Quantity}, + {asking, Asking} + } + ), + put_order(debit(Base, Seller, Quantity, Opts), Order, Opts) + else + _ -> Base + end. + +%% @doc Withdraw an offer nobody has reserved, returning the escrowed goods and +%% removing it from the book. A reservation blocks it: that is what the buyer's +%% fee and collateral bought. +cancel_order(Base, Body, Opts) -> + maybe + {ok, Signer} ?= signer(Body, Opts), + {ok, Order} ?= find_order(Base, Body, Opts), + #{ + <<"order-id">> := OrderID, + <<"status">> := <<"open">>, + <<"creator">> := Signer, + <<"quantity">> := Quantity + } ?= Order, + ?event({swap_order_cancelled, {order, OrderID}}), + drop_order(credit(Base, Signer, Quantity, Opts), Order, Opts) + else + _ -> Base + end. + +%% @doc Buy the exclusive right to complete an order. +%% +%% For `deadline' blocks the order is this buyer's alone: the seller cannot +%% withdraw it and nobody else can complete it. That is what makes it safe to +%% send AR, which no process can hold, redirect or refund. +%% +%% Two things are asked, either of which an offer may set to zero. The +%% `minimum-fee' is paid as this transaction's own `reward' -- burned to miners +%% and the endowment rather than collected, so it costs a buyer only what they +%% were spending anyway and makes idle registrations expensive. The `deposit' +%% is collateral, pledged from the buyer's own balance and forfeit to the +%% seller if the reservation lapses unpaid. It is denominated in the token +%% because a pledge is only honoured while there is an offer to honour it +%% against: a registration naming an order that has gone takes nothing at all. +register_interest(Base, Body, Height, Opts) -> + maybe + {ok, Buyer} ?= signer(Body, Opts), + {ok, Order} ?= find_order(Base, Body, Opts), + #{ + <<"order-id">> := OrderID, + <<"status">> := <<"open">>, + <<"minimum-fee">> := Fee, + <<"deposit">> := Deposit, + <<"deadline">> := Deadline + } ?= Order, + {ok, Paid} ?= amount(<<"reward">>, Body, Opts), + true ?= Paid >= Fee, + true ?= balance(Base, Buyer, Opts) >= Deposit, + Until = Height + Deadline, + ?event( + {swap_interest_registered, + {order, OrderID}, + {buyer, Buyer}, + {deposit, Deposit}, + {until, Until} + } + ), + deadlines( + put_order( + debit(Base, Buyer, Deposit, Opts), + Order#{ + <<"status">> => <<"reserved">>, + <<"buyer">> => Buyer, + <<"reserved-until">> => Until + }, + Opts + ), + Opts + ) + else + _ -> Base + end. + +%% @doc Settle against a layer-1 payment. The transaction is not addressed to +%% the process at all: it is a transfer between two user addresses that names an +%% `order-id', and the process sees it only because it is sequenced by every +%% data-free transaction on the network. It counts as payment for an order when +%% it is addressed to that order's `recipient' and carries at least the asking +%% winston. +%% +%% Underpayment is ignored rather than partially filled -- the value never +%% passed through the process, so there is nothing to refund the difference +%% from. +payment(Base, Body, Target, Height, Opts) -> + maybe + {ok, Buyer} ?= signer(Body, Opts), + {ok, Order} ?= find_order(Base, Body, Opts), + #{ + <<"creator">> := Creator, + <<"recipient">> := Recipient, + <<"asking">> := Asking + } = Order, + true ?= Target =:= Recipient, + % A seller completing their own order from the address that made it + % would take the goods straight back. These two comparisons close that; + % they cannot close a second wallet, and nothing here can. + false ?= Buyer =:= Creator, + false ?= Buyer =:= Recipient, + {ok, Paid} ?= hb_util:safe_int(tx_field(Body, <<"quantity">>, 0, Opts)), + true ?= Paid >= Asking, + % An open order is first-come; a reserved one is its buyer's alone + % until the reservation lapses. + true ?= claimable(Order, Buyer, Height), + settle(Base, Order, Buyer, Opts) + else + _ -> Base + end. + +%% @doc Complete an order: the goods pass to the buyer, and the collateral they +%% pledged to reserve it comes back to them. The AR was always the seller's +%% directly -- no process ever held it, and none could return it. +%% +%% The order then leaves the book, so a later payment naming it finds nothing +%% and does nothing. That is the only honest answer available: the goods are no +%% longer there to give, and the AR is not there to refund. +settle(Base, Order, Buyer, Opts) -> + #{ + <<"order-id">> := OrderID, + <<"status">> := Status, + <<"quantity">> := Quantity, + <<"deposit">> := Deposit + } = Order, + % A buyer who reserved the offer pledged its deposit; one who paid an open + % offer outright pledged nothing, and has nothing to get back. + Pledged = + case Status of + <<"reserved">> -> Deposit; + _ -> 0 + end, + ?event( + {swap_order_settled, + {order, OrderID}, + {buyer, Buyer}, + {quantity, Quantity} + } + ), + deadlines( + drop_order( + credit(credit(Base, Buyer, Quantity, Opts), Buyer, Pledged, Opts), + Order, + Opts + ), + Opts + ). + +%% @doc Whether an order's goods are the payer's to take: an order nobody has +%% reserved is first-come, and a reserved one is its buyer's until the +%% reservation lapses. +claimable(#{ <<"status">> := <<"open">> }, _Buyer, _Height) -> true; +claimable( + #{ + <<"status">> := <<"reserved">>, + <<"buyer">> := Buyer, + <<"reserved-until">> := Until + }, + Buyer, + Height) -> + Height =< Until; +claimable(_Order, _Buyer, _Height) -> false. + +%%% The clock + +%% @doc Advance the process's notion of the chain to the height of the +%% assignment being applied, retiring whatever fell due in between. +%% +%% Every transaction on the network is a slot here, so this runs on the order of +%% a thousand times per block: the common path is one integer comparison against +%% the next height at which anything at all happens, and only crossing it walks +%% the orders. +advance(Base, Height, Opts) -> + Dated = Base#{ <<"swap-height">> => Height }, + case state(<<"next-deadline">>, Base, 0, Opts) of + Next when Next > 0, Height >= Next -> + deadlines( + lists:foldl( + fun(Order, Acc) -> expire(Acc, Order, Height, Opts) end, + Dated, + orders(Base, Opts) + ), + Opts + ); + _ -> Dated + end. + +%% @doc Lapse a reservation the height has outlived: the buyer's collateral goes +%% to the seller, who has been held all this time, and the offer opens again. An +%% offer never expires, so this is the only thing the clock does here. + +expire( + Base, + Order = + #{ + <<"order-id">> := OrderID, + <<"status">> := <<"reserved">>, + <<"creator">> := Seller, + <<"deposit">> := Deposit, + <<"reserved-until">> := Until + }, + Height, + Opts) when Height > Until -> + ?event({swap_reservation_lapsed, {order, OrderID}, {forfeit, Deposit}}), + put_order( + credit(Base, Seller, Deposit, Opts), + hb_maps:without( + [<<"buyer">>, <<"reserved-until">>], + Order#{ <<"status">> => <<"open">> }, + Opts + ), + Opts + ); +expire(Base, _Order, _Height, _Opts) -> Base. + +%% @doc Record the next height at which a reservation lapses, so that the slots +%% in between cost a single comparison. Zero means nothing is pending. +deadlines(Base, Opts) -> + Lapses = + [ + Until + 1 + || + #{ <<"status">> := <<"reserved">>, <<"reserved-until">> := Until } + <- orders(Base, Opts) + ], + Base#{ + <<"next-deadline">> => + case Lapses of + [] -> 0; + _ -> lists:min(Lapses) + end + }. + +%%% State helpers + +%% @doc Read a key of the process's own state. +%% +%% While a slot is being computed the state carries this device, so a plain read +%% of one of its keys would resolve that key *through this device* and land back +%% in `compute'. Every read of the state is therefore taken as a message. Writes +%% have the same hazard one level down -- setting a nested path resolves the +%% keys above it on the way -- so the device only ever writes whole top-level +%% keys. +state(Key, Base, Default, Opts) -> + hb_ao:get(Key, {as, <<"message@1.0">>, Base}, Default, Opts). + +%% @doc Read a field from an untrusted scheduled message as plain data. +field(Key, Msg, Default, Opts) -> + hb_maps:get(Key, Msg, Default, Opts). + +%% @doc Read the orders currently held, as plain maps. The state may have been +%% written to the process cache and read back since it was last touched, so it +%% is loaded through the link layer, and anything that is not an order is +%% ignored rather than assumed away. +orders(Base, Opts) -> + [ + Order + || + Order = #{ <<"order-id">> := _ } <- + hb_maps:values( + hb_cache:ensure_all_loaded(order_book(Base, Opts), Opts), + Opts + ) + ]. + +order_book(Base, Opts) -> state(<<"orders">>, Base, #{}, Opts). + +%% @doc Read the order a message names, if the process holds it. +%% +%% The name comes from a stranger's transaction, so it is looked up as a key of +%% the order book rather than resolved as a path: a path would let +%% `order-id: /creator' reach an address, and a reserved name like `keys' +%% reach a list, either of which would then be read as an order and fail the +%% slot. Whatever comes back must look like an order before it is treated as +%% one. +find_order(Base, Body, Opts) -> + case + hb_cache:ensure_all_loaded( + hb_maps:get( + field(<<"order-id">>, Body, <<>>, Opts), + order_book(Base, Opts), + not_found, + Opts + ), + Opts + ) + of + Order = #{ <<"order-id">> := _ } -> {ok, Order}; + _ -> not_found + end. + +%% @doc Write an order back, replacing the one held rather than merging over +%% it, so that a lapsed reservation leaves no buyer behind. +put_order(Base, Order = #{ <<"order-id">> := OrderID }, Opts) -> + Base#{ + <<"orders">> => + hb_maps:put( + OrderID, + Order, + order_book(Base, Opts), + Opts + ) + }. + +%% @doc Forget an order. What the book holds is the offers still open to be +%% taken, so a completed or withdrawn one leaves no trace: opening an offer and +%% withdrawing it again returns the process to exactly where it was. +drop_order(Base, #{ <<"order-id">> := OrderID }, Opts) -> + Base#{ + <<"orders">> => + hb_maps:without([OrderID], order_book(Base, Opts), Opts) + }. + +%% @doc Read an address's token balance from the ledger this process shares. +%% Only the one entry is read: the ledger may be large, and the rest of it is +%% none of this device's business. +balance(Base, Address, Opts) -> + hb_util:int(state([?BALANCES, Address], Base, 0, Opts)). + +credit(Base, _Address, 0, _Opts) -> Base; +credit(Base, Address, Amount, Opts) -> + Base#{ + ?BALANCES => + hb_maps:put( + Address, + balance(Base, Address, Opts) + Amount, + state(?BALANCES, Base, #{}, Opts), + Opts + ) + }. + +debit(Base, Address, Amount, Opts) -> credit(Base, Address, -Amount, Opts). + +%% @doc Read a number a stranger wrote. Every figure in the protocol arrives as +%% a tag on somebody else's transaction, and this process is sequenced by all of +%% them: coercing `deadline: tomorrow' with `hb_util:int/1' would raise out of +%% the enclosing `maybe' -- which catches mismatches, not exceptions -- and fail +%% that slot on every node, for good. A value that is not a number is simply not +%% an admissible message. +amount(Key, Body, Opts) -> amount(Key, Body, 0, Opts). + +amount(Key, Body, Default, Opts) -> + hb_util:safe_int(field(Key, Body, Default, Opts)). + +%% @doc Read a value from the real L1 transaction fields recorded in the +%% `tx@1.0' commitment. Top-level keys may come from tags with the same names, so +%% payment routing and amount checks must not use them. +tx_field(Body, Field, Default, Opts) -> + case hb_message:commitment(#{ <<"commitment-device">> => <<"tx@1.0">> }, Body, Opts) of + {ok, _ID, Commitment} -> + hb_maps:get(<<"field-", Field/binary>>, Commitment, Default, Opts); + _ -> + Default + end. + +%% @doc The single signer of a message. A message with any other number of +%% signers is not attributable to one party, so it cannot open, cancel, reserve +%% or pay for anything. +signer(Body, Opts) -> + case hb_message:signers(Body, Opts) of + [Signer] -> {ok, hb_util:human_id(Signer)}; + _ -> not_found + end. + +%%% Tests + +%%% The tests drive `compute/3' directly with synthetic assignments, exactly as +%%% `~process@1.0' would: the device reads chain data but performs no I/O, so a +%%% whole trade can be played out without a weave. + +-define(PROCESS, <<"pRoCeSs000000000000000000000000000000000000">>). + +test_opts() -> #{ <<"priv-wallet">> => ar_wallet:new() }. + +%% @doc A party to a trade: a wallet and the address it signs as. +party() -> + Wallet = ar_wallet:new(), + {Wallet, hb_util:human_id(ar_wallet:to_address(Wallet))}. + +%% @doc A process base holding the given balances and nothing else. +base(Balances) -> + #{ ?BALANCES => Balances }. + +%% @doc An L1 transaction, committed as the base layer commits them. +tx(Wallet, Fields) -> + hb_message:commit( + Fields, + #{ <<"priv-wallet">> => Wallet }, + #{ <<"commitment-device">> => <<"tx@1.0">> } + ). + +%% @doc Do an order's numbers survive the process cache as numbers? +%% +%% `order/1' re-coerces every numeric key on the way out, on the stated grounds +%% that a value written between slots may come back as a binary. If that is not +%% so, the coercion is doing nothing at all. +order_numbers_survive_the_cache_test() -> + Opts = test_opts(), + Written = #{ <<"orders">> => #{ <<"o">> => #{ + <<"order-id">> => <<"o">>, + <<"quantity">> => 10, + <<"deadline">> => 20, + <<"reserved-until">> => 1966680 + } } }, + {ok, ID} = hb_cache:write(Written, Opts), + {ok, Read} = hb_cache:read(ID, Opts), + Loaded = hb_cache:ensure_all_loaded(Read, Opts), + Order = hb_maps:get(<<"o">>, hb_maps:get(<<"orders">>, Loaded, #{}, Opts), #{}, Opts), + ?event({round_tripped, {order, Order}}), + ?assertEqual(10, maps:get(<<"quantity">>, Order)), + ?assertEqual(1966680, maps:get(<<"reserved-until">>, Order)). + +%% @doc A message this device reads is a stranger's, and a key it does not carry +%% must be answered by this device's default, not by code the stranger picked. +%% +%% `all' mode hands every data-free transaction on Arweave to this process, so +%% the `device' key of each is the author's to choose. Reading plainly +%% dispatches on it, and a device answers only for absent keys -- so the attack +%% is to omit one. Here +%% `recipient' is omitted: the seller of an order is meant to be paid by the +%% default, and a chosen device answering that key would redirect the goods. +strangers_device_cannot_answer_for_an_absent_key_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + {_, ThiefAddr} = party(), + Hostile = + tx( + Seller, + #{ + <<"target">> => ?PROCESS, + <<"action">> => <<"make-offer">>, + <<"offer-quantity">> => <<"1">>, + <<"asking">> => <<"10">>, + <<"deposit">> => <<"0">>, + <<"deadline">> => <<"100">>, + %% No `recipient'. The script answers for it. + <<"device">> => <<"lua@5.3a">>, + <<"module">> => + #{ + <<"content-type">> => <<"text/x-lua">>, + <<"body">> => + <<"function recipient(base, req)\n" + " return \"ok\", \"", ThiefAddr/binary, "\"\n" + "end\n">> + } + } + ), + Opened = apply_tx(base(#{ SellerAddr => 1 }), Hostile, 10, Opts), + [Order] = orders(Opened, Opts), + %% The goods come back to the seller, because that is what this device + %% decided in the absence of an instruction. + ?assertEqual(SellerAddr, maps:get(<<"recipient">>, Order, undefined)), + ?assertNotEqual(ThiefAddr, maps:get(<<"recipient">>, Order, undefined)). + +%% @doc A transaction that carries trade keys as tags only. +tag_only_tx(Wallet, Tags) -> + Signed = ar_tx:sign(#tx{ format = 2, reward = 1, tags = Tags }, Wallet), + hb_message:convert(Signed, <<"structured@1.0">>, <<"tx@1.0">>, #{}). + +%% @doc Sequence a transaction into the process at a block height, as +%% `~arweave-scheduler@1.0' in `all' mode does. +apply_tx(Base, Body, Height, Opts) -> + {ok, New} = + compute( + Base, + #{ + <<"process">> => ?PROCESS, + <<"slot">> => 1, + <<"block-height">> => Height, + <<"body">> => Body + }, + Opts + ), + New. + +%% @doc Advance the process to a height without anything happening, which is +%% what the network's unrelated traffic does. +tick(Base, Height, Opts) -> + apply_tx(Base, #{ <<"target">> => <<"someone-else">> }, Height, Opts). + +offer(Wallet, Quantity, Asking, Deposit, Deadline) -> + tx( + Wallet, + #{ + <<"target">> => ?PROCESS, + <<"action">> => <<"make-offer">>, + <<"offer-quantity">> => hb_util:bin(Quantity), + <<"asking">> => hb_util:bin(Asking), + <<"deposit">> => hb_util:bin(Deposit), + <<"deadline">> => hb_util:bin(Deadline) + } + ). + +order_action(Wallet, Action, OrderID) -> + tx( + Wallet, + #{ + <<"target">> => ?PROCESS, + <<"action">> => Action, + <<"order-id">> => OrderID + } + ). + +%% @doc The payment leg: an ordinary transfer to the seller that the process is +%% not addressed by, naming the order it settles. +pay(Wallet, To, Winston, OrderID) -> + tx( + Wallet, + #{ + <<"target">> => To, + <<"quantity">> => hb_util:bin(Winston), + <<"order-id">> => OrderID + } + ). + +tag_only_transfer(Wallet, To, Winston, OrderID) -> + tag_only_tx( + Wallet, + [ + {<<"target">>, To}, + {<<"quantity">>, hb_util:bin(Winston)}, + {<<"order-id">>, OrderID} + ] + ). + +only_order(Base, Opts) -> + [Order] = orders(Base, Opts), + Order. + +%% @doc A foreign transaction's device cannot interpret absent control fields. +foreign_device_is_data_test() -> + Opts = test_opts(), + {Sender, SenderAddr} = party(), + {_, Recipient} = party(), + Base = base(#{ SenderAddr => 1 }), + Foreign = + tx( + Sender, + #{ + <<"target">> => Recipient, + <<"device">> => <<"reference@1.0">> + } + ), + Untouched = apply_tx(Base, Foreign, 100, Opts), + ?assertEqual(1, balance(Untouched, SenderAddr, Opts)), + ?assertEqual([], orders(Untouched, Opts)). + +%% @doc Opening an offer moves the goods into escrow, and leaves +%% the order open. +make_offer_escrows_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + Base = base(#{ SellerAddr => 100 }), + Opened = apply_tx(Base, offer(Seller, 10, 500, 5, 20), 100, Opts), + Order = only_order(Opened, Opts), + ?assertEqual(90, balance(Opened, SellerAddr, Opts)), + ?assertEqual(<<"open">>, maps:get(<<"status">>, Order)), + ?assertEqual(10, maps:get(<<"quantity">>, Order)), + ?assertEqual(5, maps:get(<<"deposit">>, Order)), + ?assertEqual(SellerAddr, maps:get(<<"recipient">>, Order)). + +%% @doc An offer for more than the seller holds changes nothing at all. +make_offer_insufficient_balance_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + Base = base(#{ SellerAddr => 4 }), + Result = apply_tx(Base, offer(Seller, 10, 500, 5, 20), 100, Opts), + ?assertEqual([], orders(Result, Opts)), + ?assertEqual(4, balance(Result, SellerAddr, Opts)). + +%% @doc Tag-only trade keys are metadata and do not route control slots. +tag_only_target_is_metadata_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + Tagged = + tag_only_tx( + Seller, + [ + {<<"target">>, ?PROCESS}, + {<<"action">>, <<"make-offer">>}, + {<<"offer-quantity">>, <<"10">>}, + {<<"asking">>, <<"500">>}, + {<<"deposit">>, <<"5">>}, + {<<"deadline">>, <<"200">>} + ] + ), + ?assertEqual(?PROCESS, hb_ao:get(<<"target">>, Tagged, not_found, Opts)), + ?assertEqual(<<>>, tx_field(Tagged, <<"target">>, <<>>, Opts)), + Result = apply_tx(base(#{ SellerAddr => 100 }), Tagged, 100, Opts), + ?assertEqual([], orders(Result, Opts)), + ?assertEqual(100, balance(Result, SellerAddr, Opts)). + +%% @doc The whole trade: the buyer pays the seller directly on layer one, and +%% the process -- which is not a party to that payment -- settles it. +settlement_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + {Buyer, BuyerAddr} = party(), + Base = base(#{ SellerAddr => 100 }), + Opened = apply_tx(Base, offer(Seller, 10, 500, 5, 20), 100, Opts), + OrderID = maps:get(<<"order-id">>, only_order(Opened, Opts)), + Settled = + apply_tx( + Opened, + pay(Buyer, SellerAddr, 500, OrderID), + 120, + Opts + ), + % The offer is complete, so it is no longer an offer. + ?assertEqual([], orders(Settled, Opts)), + ?assertEqual(10, balance(Settled, BuyerAddr, Opts)), + ?assertEqual(90, balance(Settled, SellerAddr, Opts)). + +%% @doc Paying less than the asking price settles nothing: the process never +%% held the value, so it cannot refund a partial fill. +underpayment_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + {Buyer, BuyerAddr} = party(), + Opened = + apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 20), 100, Opts), + OrderID = maps:get(<<"order-id">>, only_order(Opened, Opts)), + Result = apply_tx(Opened, pay(Buyer, SellerAddr, 499, OrderID), 120, Opts), + ?assertEqual(<<"open">>, maps:get(<<"status">>, only_order(Result, Opts))), + ?assertEqual(0, balance(Result, BuyerAddr, Opts)). + +%% @doc A payment that names the order but is addressed to somebody else is not +%% a payment for it. +payment_to_wrong_address_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + {Buyer, BuyerAddr} = party(), + {_, Stranger} = party(), + Opened = + apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 20), 100, Opts), + OrderID = maps:get(<<"order-id">>, only_order(Opened, Opts)), + Result = apply_tx(Opened, pay(Buyer, Stranger, 500, OrderID), 120, Opts), + ?assertEqual(<<"open">>, maps:get(<<"status">>, only_order(Result, Opts))), + ?assertEqual(0, balance(Result, BuyerAddr, Opts)). + +%% @doc Tag-only trade keys are metadata and do not count as the payment leg. +tag_only_transfer_is_metadata_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + {Buyer, BuyerAddr} = party(), + Opened = + apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 20), 100, Opts), + OrderID = maps:get(<<"order-id">>, only_order(Opened, Opts)), + Tagged = tag_only_transfer(Buyer, SellerAddr, 500, OrderID), + ?assertEqual(SellerAddr, hb_ao:get(<<"target">>, Tagged, not_found, Opts)), + ?assertEqual(<<"500">>, hb_ao:get(<<"quantity">>, Tagged, not_found, Opts)), + ?assertEqual(<<>>, tx_field(Tagged, <<"target">>, <<>>, Opts)), + ?assertEqual({ok, 0}, hb_util:safe_int(tx_field(Tagged, <<"quantity">>, 0, Opts))), + Result = apply_tx(Opened, Tagged, 120, Opts), + ?assertEqual(<<"open">>, maps:get(<<"status">>, only_order(Result, Opts))), + ?assertEqual(0, balance(Result, BuyerAddr, Opts)), + ?assertEqual(90, balance(Result, SellerAddr, Opts)). + +%% @doc Only the seller may cancel their order. +cancel_by_stranger_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + {Stranger, _} = party(), + Opened = + apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 20), 100, Opts), + OrderID = maps:get(<<"order-id">>, only_order(Opened, Opts)), + Result = + apply_tx(Opened, order_action(Stranger, <<"cancel-order">>, OrderID), 110, Opts), + ?assertEqual(<<"open">>, maps:get(<<"status">>, only_order(Result, Opts))). + +%% @doc A reserved order cannot be pulled out from under the buyer who reserved +%% it. This is the guarantee that makes paying safe. +reservation_prevents_cancellation_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + {Buyer, BuyerAddr} = party(), + Opened = + apply_tx( + base(#{ SellerAddr => 100, BuyerAddr => 5 }), + offer(Seller, 10, 500, 5, 20), + 100, + Opts + ), + OrderID = maps:get(<<"order-id">>, only_order(Opened, Opts)), + Reserved = + apply_tx( + Opened, + order_action(Buyer, <<"register-interest">>, OrderID), + 110, + Opts + ), + ?assertEqual(<<"reserved">>, maps:get(<<"status">>, only_order(Reserved, Opts))), + Attempted = + apply_tx( + Reserved, + order_action(Seller, <<"cancel-order">>, OrderID), + 111, + Opts + ), + ?assertEqual(<<"reserved">>, maps:get(<<"status">>, only_order(Attempted, Opts))). + +%% @doc A reservation is exclusive while it lasts: somebody else's payment does +%% not take the goods. +reservation_is_exclusive_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + {Buyer, BuyerAddr} = party(), + {Interloper, InterloperAddr} = party(), + Opened = + apply_tx( + base(#{ SellerAddr => 100, BuyerAddr => 5 }), + offer(Seller, 10, 500, 5, 20), + 100, + Opts + ), + OrderID = maps:get(<<"order-id">>, only_order(Opened, Opts)), + Reserved = + apply_tx(Opened, order_action(Buyer, <<"register-interest">>, OrderID), 110, Opts), + % The buyer's collateral is pledged out of their balance. + ?assertEqual(0, balance(Reserved, BuyerAddr, Opts)), + Result = apply_tx(Reserved, pay(Interloper, SellerAddr, 500, OrderID), 111, Opts), + Order = only_order(Result, Opts), + % The stranger's AR bought nothing: the order is not theirs to complete, and + % there is nothing here that could give it back. + ?assertEqual(<<"reserved">>, maps:get(<<"status">>, Order)), + ?assertEqual(BuyerAddr, maps:get(<<"buyer">>, Order)), + ?assertEqual(10, maps:get(<<"quantity">>, Order)), + ?assertEqual(0, balance(Result, InterloperAddr, Opts)). + +%% @doc A reservation lapses on its own, without anybody sending anything: the +%% network's own traffic carries the clock forward. +reservation_lapses_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + {Buyer, BuyerAddr} = party(), + Opened = + apply_tx( + base(#{ SellerAddr => 100, BuyerAddr => 5 }), + offer(Seller, 10, 500, 5, 20), + 100, + Opts + ), + OrderID = maps:get(<<"order-id">>, only_order(Opened, Opts)), + Reserved = + apply_tx(Opened, order_action(Buyer, <<"register-interest">>, OrderID), 110, Opts), + ?assertEqual( + 110 + 20, + maps:get(<<"reserved-until">>, only_order(Reserved, Opts)) + ), + Lapsed = tick(Reserved, 110 + 20 + 1, Opts), + ?assertEqual(<<"open">>, maps:get(<<"status">>, only_order(Lapsed, Opts))). + +%% @doc Settling twice is not possible: the goods left with the first payment. +double_settlement_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + {Buyer, BuyerAddr} = party(), + {Late, LateAddr} = party(), + Opened = + apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 20), 100, Opts), + OrderID = maps:get(<<"order-id">>, only_order(Opened, Opts)), + Settled = apply_tx(Opened, pay(Buyer, SellerAddr, 500, OrderID), 120, Opts), + Again = apply_tx(Settled, pay(Late, SellerAddr, 500, OrderID), 121, Opts), + ?assertEqual(10, balance(Again, BuyerAddr, Opts)), + ?assertEqual(0, balance(Again, LateAddr, Opts)), + ?assertEqual(90, balance(Again, SellerAddr, Opts)). + +%% @doc A payment naming an order the process has never heard of is ordinary +%% Arweave traffic, and is ignored. +unknown_order_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + {Buyer, BuyerAddr} = party(), + Opened = + apply_tx( + base(#{ SellerAddr => 100, BuyerAddr => 5 }), + offer(Seller, 10, 500, 5, 20), + 100, + Opts + ), + Result = + apply_tx( + Opened, + pay(Buyer, SellerAddr, 500, <<"nOtAnOrDeR0000000000000000000000000000000000">>), + 120, + Opts + ), + ?assertEqual(<<"open">>, maps:get(<<"status">>, only_order(Result, Opts))). + +%% @doc Unrelated traffic -- the overwhelming majority of what this process is +%% sequenced by -- moves the clock and nothing else. +unrelated_traffic_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + Opened = + apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 0, 20), 100, Opts), + Ticked = tick(Opened, 150, Opts), + ?assertEqual(150, hb_ao:get(<<"swap-height">>, Ticked, 0, Opts)), + ?assertEqual(<<"open">>, maps:get(<<"status">>, only_order(Ticked, Opts))), + ?assertEqual(90, balance(Ticked, SellerAddr, Opts)). + +%% @doc A stranger's transaction may ask to be routed anywhere -- the key a slot +%% resolves comes from the sender's own `path' tag, and this process is +%% sequenced by every data-free transaction on Arweave. It must be applied like +%% any other, not fail its slot. +stray_path_is_applied_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + {Stranger, _} = party(), + Opened = + apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 20), 100, Opts), + Strayed = + hb_ao:resolve( + Opened#{ <<"device">> => <<"arweave-swap@1.0">> }, + #{ + <<"path">> => <<"withdraw-everything">>, + <<"process">> => ?PROCESS, + <<"slot">> => 2, + <<"block-height">> => 150, + <<"body">> => + tx(Stranger, #{ <<"target">> => <<"somebody-else">> }) + }, + Opts + ), + ?assertMatch({ok, _}, Strayed), + {ok, State} = Strayed, + % The state still carries this device here, because the test resolves it + % directly rather than through `lib_process:run_as', which puts the + % process's own device back afterwards. Read it as a message. + ?assertEqual(150, hb_util:int(state(<<"swap-height">>, State, 0, Opts))), + ?assertEqual(90, balance(State, SellerAddr, Opts)). + +%% @doc Collateral returns to the buyer when they complete the sale. +settlement_returns_the_collateral_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + {Buyer, BuyerAddr} = party(), + Opened = + apply_tx( + base(#{ SellerAddr => 100, BuyerAddr => 5 }), + offer(Seller, 10, 500, 5, 20), + 100, + Opts + ), + OrderID = maps:get(<<"order-id">>, only_order(Opened, Opts)), + Reserved = + apply_tx(Opened, order_action(Buyer, <<"register-interest">>, OrderID), 110, Opts), + ?assertEqual(0, balance(Reserved, BuyerAddr, Opts)), + Settled = apply_tx(Reserved, pay(Buyer, SellerAddr, 500, OrderID), 115, Opts), + ?assertEqual([], orders(Settled, Opts)), + % The goods, and the collateral back. + ?assertEqual(15, balance(Settled, BuyerAddr, Opts)), + ?assertEqual(90, balance(Settled, SellerAddr, Opts)). + +%% @doc A buyer who cannot cover the collateral cannot reserve. +collateral_must_be_held_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + {Buyer, BuyerAddr} = party(), + Opened = + apply_tx( + base(#{ SellerAddr => 100, BuyerAddr => 4 }), + offer(Seller, 10, 500, 5, 20), + 100, + Opts + ), + OrderID = maps:get(<<"order-id">>, only_order(Opened, Opts)), + Tried = + apply_tx(Opened, order_action(Buyer, <<"register-interest">>, OrderID), 110, Opts), + ?assertEqual(<<"open">>, maps:get(<<"status">>, only_order(Tried, Opts))), + ?assertEqual(4, balance(Tried, BuyerAddr, Opts)). + +%% @doc A registration naming an offer that has gone takes nothing. +%% +%% This is why collateral is denominated in the token: the process can decline +%% to honour a pledge when there is nothing to honour it against. AR could not +%% serve, because AR settles on the weave whatever this device decides. +collateral_is_not_taken_for_a_vanished_offer_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + {Buyer, BuyerAddr} = party(), + Before = base(#{ SellerAddr => 100, BuyerAddr => 5 }), + Opened = apply_tx(Before, offer(Seller, 10, 500, 5, 20), 100, Opts), + OrderID = maps:get(<<"order-id">>, only_order(Opened, Opts)), + Withdrawn = + apply_tx(Opened, order_action(Seller, <<"cancel-order">>, OrderID), 105, Opts), + Late = + apply_tx( + Withdrawn, + order_action(Buyer, <<"register-interest">>, OrderID), + 110, + Opts + ), + ?assertEqual(5, balance(Late, BuyerAddr, Opts)), + ?assertEqual([], orders(Late, Opts)). + +%% @doc Opening an offer and withdrawing it leaves the process as it was. +offer_and_withdrawal_are_a_round_trip_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + Before = base(#{ SellerAddr => 100 }), + Opened = apply_tx(Before, offer(Seller, 10, 500, 5, 20), 100, Opts), + OrderID = maps:get(<<"order-id">>, only_order(Opened, Opts)), + After = + apply_tx(Opened, order_action(Seller, <<"cancel-order">>, OrderID), 105, Opts), + ?assertEqual([], orders(After, Opts)), + ?assertEqual(100, balance(After, SellerAddr, Opts)), + ?assertEqual(0, hb_util:int(state(<<"next-deadline">>, After, 0, Opts))). + +%% @doc Units are conserved across a whole trade: what is escrowed, what is +%% pledged and what is held always sum to the supply. +supply_is_conserved_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + {Buyer, BuyerAddr} = party(), + Held = + fun(State) -> + balance(State, SellerAddr, Opts) + balance(State, BuyerAddr, Opts) + + lists:sum( + [ + % Goods are escrowed by every offer; collateral only by + % one somebody has reserved. + case maps:get(<<"status">>, Order) of + <<"reserved">> -> maps:get(<<"quantity">>, Order) + maps:get(<<"deposit">>, Order); + _ -> maps:get(<<"quantity">>, Order) + end + || + Order <- orders(State, Opts) + ] + ) + end, + Before = base(#{ SellerAddr => 100, BuyerAddr => 5 }), + ?assertEqual(105, Held(Before)), + Opened = apply_tx(Before, offer(Seller, 10, 500, 5, 20), 100, Opts), + ?assertEqual(105, Held(Opened)), + OrderID = maps:get(<<"order-id">>, only_order(Opened, Opts)), + Reserved = + apply_tx(Opened, order_action(Buyer, <<"register-interest">>, OrderID), 110, Opts), + ?assertEqual(105, Held(Reserved)), + Settled = apply_tx(Reserved, pay(Buyer, SellerAddr, 500, OrderID), 115, Opts), + ?assertEqual(105, Held(Settled)), + Lapsed = tick(apply_tx(Before, offer(Seller, 10, 500, 5, 20), 100, Opts), 300, Opts), + ?assertEqual(105, Held(Lapsed)). + +%% @doc A lapsed reservation pays the seller and reopens the offer. +%% +%% The collateral is the buyer's answer for having frozen the seller out. When +%% the window passes unpaid it is the seller's, and the offer -- which never +%% expires -- is open to anyone again. +lapse_forfeits_collateral_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + {Buyer, BuyerAddr} = party(), + Opened = + apply_tx( + base(#{ SellerAddr => 100, BuyerAddr => 5 }), + offer(Seller, 10, 500, 5, 20), + 100, + Opts + ), + OrderID = maps:get(<<"order-id">>, only_order(Opened, Opts)), + Reserved = + apply_tx(Opened, order_action(Buyer, <<"register-interest">>, OrderID), 190, Opts), + ?assertEqual(0, balance(Reserved, BuyerAddr, Opts)), + Lapsed = tick(Reserved, 190 + 20 + 1, Opts), + Order = only_order(Lapsed, Opts), + ?assertEqual(<<"open">>, maps:get(<<"status">>, Order)), + ?assertEqual(false, maps:is_key(<<"buyer">>, Order)), + % The seller keeps 90 escrowed-out plus the 5 forfeited to them. + ?assertEqual(95, balance(Lapsed, SellerAddr, Opts)), + ?assertEqual(0, balance(Lapsed, BuyerAddr, Opts)). + +%% @doc The offered amount is not carried as `quantity'. That key is the +%% transaction's own value field, so the codec would send it as winston of AR to +%% the process -- an address with no key. An offer that uses it opens nothing. +offer_quantity_is_not_winston_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + Wrong = + tx( + Seller, + #{ + <<"target">> => ?PROCESS, + <<"action">> => <<"make-offer">>, + <<"quantity">> => <<"10">>, + <<"asking">> => <<"500">>, + <<"deposit">> => <<"5">>, + <<"deadline">> => <<"200">> + } + ), + Result = apply_tx(base(#{ SellerAddr => 100 }), Wrong, 100, Opts), + ?assertEqual([], orders(Result, Opts)), + ?assertEqual(100, balance(Result, SellerAddr, Opts)). + +%% @doc A figure that is not a number is an inadmissible message, not a failed +%% slot: anyone may send this process anything, and a slot that raises can never +%% be recomputed. +non_numeric_tag_is_ignored_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + Nonsense = + tx( + Seller, + #{ + <<"target">> => ?PROCESS, + <<"action">> => <<"make-offer">>, + <<"offer-quantity">> => <<"10">>, + <<"asking">> => <<"500">>, + <<"deposit">> => <<"5">>, + <<"deadline">> => <<"tomorrow">> + } + ), + Result = apply_tx(base(#{ SellerAddr => 100 }), Nonsense, 100, Opts), + ?assertEqual([], orders(Result, Opts)), + ?assertEqual(100, balance(Result, SellerAddr, Opts)). + +%% @doc An `order-id' is caller-supplied text. Path-like values and reserved +%% keys must not reach anything that is then read as an order. +reserved_order_id_names_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + {Buyer, BuyerAddr} = party(), + Opened = + apply_tx( + base(#{ SellerAddr => 100, BuyerAddr => 5 }), + offer(Seller, 10, 500, 5, 20), + 100, + Opts + ), + OrderID = maps:get(<<"order-id">>, only_order(Opened, Opts)), + Names = + [ + <>, + <<"keys">>, + <<"id">>, + <<"commitments">> + ], + lists:foreach( + fun(Name) -> + Result = apply_tx(Opened, pay(Buyer, SellerAddr, 500, Name), 120, Opts), + ?assertEqual( + <<"open">>, + maps:get(<<"status">>, only_order(Result, Opts)) + ) + end, + Names + ). + +%% @doc A seller cannot buy their own order. Paying oneself costs only a network +%% fee, so it would otherwise take back the goods, leaving a real +%% buyer to pay for an order that is already spent. +seller_cannot_settle_own_order_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + Opened = + apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 20), 100, Opts), + OrderID = maps:get(<<"order-id">>, only_order(Opened, Opts)), + Result = apply_tx(Opened, pay(Seller, SellerAddr, 500, OrderID), 120, Opts), + ?assertEqual(<<"open">>, maps:get(<<"status">>, only_order(Result, Opts))), + ?assertEqual(90, balance(Result, SellerAddr, Opts)). + +%% @doc A scheduled message routed to `set' or `keys' is applied like any other. +%% Handing either to `~message@1.0' would let a passer-by write the process's own +%% balances, or replace its state with a list of key names. +reserved_paths_are_applied_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + {Stranger, _} = party(), + Opened = + apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 20), 100, Opts), + lists:foreach( + fun(Path) -> + {ok, State} = + hb_ao:resolve( + Opened#{ <<"device">> => <<"arweave-swap@1.0">> }, + #{ + <<"path">> => Path, + <<"process">> => ?PROCESS, + <<"slot">> => 2, + <<"block-height">> => 150, + <<"balances">> => #{ SellerAddr => 10000000 }, + <<"body">> => + tx(Stranger, #{ <<"target">> => <<"somebody-else">> }) + }, + Opts + ), + ?assertEqual(90, balance(State, SellerAddr, Opts)), + ?assertEqual( + <<"open">>, + maps:get(<<"status">>, only_order(State, Opts)) + ) + end, + [<<"set">>, <<"keys">>, <<"info">>] + ). + +%% @doc An offer that charges to register turns away a registration that does +%% not pay the fee, and lets one that does through. The fee is the +%% registration's own reward, so it is paid to the network rather than to an +%% address nobody holds. +minimum_fee_gates_registration_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + {Buyer, _} = party(), + Charged = + tx( + Seller, + #{ + <<"target">> => ?PROCESS, + <<"action">> => <<"make-offer">>, + <<"offer-quantity">> => <<"10">>, + <<"asking">> => <<"500">>, + <<"deposit">> => <<"0">>, + <<"minimum-fee">> => <<"1000">>, + <<"deadline">> => <<"200">> + } + ), + Opened = apply_tx(base(#{ SellerAddr => 100 }), Charged, 100, Opts), + OrderID = maps:get(<<"order-id">>, only_order(Opened, Opts)), + Free = + apply_tx( + Opened, + order_action(Buyer, <<"register-interest">>, OrderID), + 110, + Opts + ), + ?assertEqual(<<"open">>, maps:get(<<"status">>, only_order(Free, Opts))), + % Value sent to the process is not the fee: it would be stranded there, and + % it is the reward that the order asks for. + Stranded = + apply_tx( + Opened, + tx( + Buyer, + #{ + <<"target">> => ?PROCESS, + <<"action">> => <<"register-interest">>, + <<"order-id">> => OrderID, + <<"quantity">> => <<"100000">> + } + ), + 110, + Opts + ), + ?assertEqual(<<"open">>, maps:get(<<"status">>, only_order(Stranded, Opts))), + Underpaid = + apply_tx( + Opened, + registration(Buyer, OrderID, 999), + 110, + Opts + ), + ?assertEqual(<<"open">>, maps:get(<<"status">>, only_order(Underpaid, Opts))), + Reserved = apply_tx(Opened, registration(Buyer, OrderID, 1000), 110, Opts), + ?assertEqual( + <<"reserved">>, + maps:get(<<"status">>, only_order(Reserved, Opts)) + ). + +%% @doc An order that charges nothing is still free to register on. +no_minimum_fee_registers_free_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + {Buyer, BuyerAddr} = party(), + Opened = + apply_tx( + base(#{ SellerAddr => 100, BuyerAddr => 5 }), + offer(Seller, 10, 500, 5, 20), + 100, + Opts + ), + OrderID = maps:get(<<"order-id">>, only_order(Opened, Opts)), + Reserved = + apply_tx( + Opened, + order_action(Buyer, <<"register-interest">>, OrderID), + 110, + Opts + ), + ?assertEqual( + <<"reserved">>, + maps:get(<<"status">>, only_order(Reserved, Opts)) + ). + +%% @doc A seller whose whole holding is the single unit they are selling can +%% still offer it: an offer escrows the goods and asks nothing else of them. +single_unit_offer_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + Opened = + apply_tx(base(#{ SellerAddr => 1 }), offer(Seller, 1, 500, 0, 20), 100, Opts), + Order = only_order(Opened, Opts), + ?assertEqual(1, maps:get(<<"quantity">>, Order)), + ?assertEqual(0, maps:get(<<"deposit">>, Order)), + ?assertEqual(0, balance(Opened, SellerAddr, Opts)). + +%% @doc A registration paying a fee, as the reward on its own transaction. +registration(Wallet, OrderID, Winston) -> + tx( + Wallet, + #{ + <<"target">> => ?PROCESS, + <<"action">> => <<"register-interest">>, + <<"order-id">> => OrderID, + <<"reward">> => hb_util:bin(Winston) + } + ). + +%% @doc The next height at which anything happens is recorded, so that the +%% slots in between cost a single comparison rather than a walk of the orders. +next_deadline_tracked_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + {Buyer, BuyerAddr} = party(), + Opened = + apply_tx( + base(#{ SellerAddr => 100, BuyerAddr => 5 }), + offer(Seller, 10, 500, 5, 20), + 100, + Opts + ), + % An offer alone has nothing pending: it never expires. + ?assertEqual(0, hb_ao:get(<<"next-deadline">>, Opened, 0, Opts)), + OrderID = maps:get(<<"order-id">>, only_order(Opened, Opts)), + Reserved = + apply_tx(Opened, order_action(Buyer, <<"register-interest">>, OrderID), 110, Opts), + % A reservation is the only thing a clock waits for. + ?assertEqual( + 110 + 20 + 1, + hb_ao:get(<<"next-deadline">>, Reserved, 0, Opts) + ). diff --git a/src/preloaded/process/dev_name_token.erl b/src/preloaded/process/dev_name_token.erl new file mode 100644 index 000000000..34c963208 --- /dev/null +++ b/src/preloaded/process/dev_name_token.erl @@ -0,0 +1,1225 @@ +%%% @doc A name: a token with a single unit, whose holder decides what the +%%% name resolves to. +%%% +%%% The device is `token-1.0''s shape carrying `~reference@1.0''s payload. From +%%% the token it takes the state layout -- `name', `ticker', `denomination', +%%% `total-supply' and `balances' on a flat `~process@1.0' message -- the +%%% `transfer' action and its notices, and the rule that decides who may `set': +%%% the holder of the whole supply. From the reference it takes what `set' +%%% actually writes -- a linked message whose keys the name then resolves +%%% through to. +%%% +%%% Because the supply is one indivisible unit, those two things compose into a +%%% name: exactly one address holds it, that address alone may say what the name +%%% points at, and transferring the unit hands over that right with it. Selling +%%% the unit -- with `~arweave-swap@1.0', which settles against the same +%%% `balances' -- sells the name. +%%% +%%% The protocol is two messages: +%%%
    +%%%
  • `transfer', carrying `recipient' and `quantity'. The sender is the +%%% message's signer. A `Debit-Notice' and a `Credit-Notice' are emitted +%%% with the same keys `token-1.0' uses.
    • +%%%
  • `set', carrying either a `reference-value' -- the message the name is +%%% to resolve to -- or nothing, in which case the name inherits the keys +%%% of the `set' message itself, as `~reference@1.0' defines it. Only the +%%% holder of the whole supply may send it, judged against the balances as +%%% they stand at that slot.
    • +%%%
+%%% +%%% There is no mint path, so the supply is fixed at spawn: a name cannot be +%%% diluted into existing twice. +%%% +%%% Ordering is the schedule's. `~reference@1.0' has to reconstruct which of +%%% several competing `set's won -- by a signer-declared timestamp, tie-broken +%%% by weave position -- because it is assembled from loose layer-1 messages. A +%%% process is handed that order already: the latest `set' in slot order wins, +%%% and there is nothing to declare and nothing to tie-break. +%%% +%%% Reads are paths into the state -- `/now/balances/
', +%%% `/now/total-supply', `/now/value/' -- and not device keys, which is how +%%% `token-1.0' reads too. The device deliberately exports no key list: a name +%%% is meant to be sold by `~arweave-swap@1.0', which requires its process to be +%%% sequenced by every data-free transaction on Arweave, so the key a slot +%%% resolves is chosen by a stranger's `path' tag. An exported `balance' key +%%% would let a passer-by's transaction hand back a balance as the new process +%%% state. +-module(dev_name_token). +-implements(<<"name-token@1.0">>). +%%% AO-Core API functions: +-export([info/0, compute/3, set/3]). +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +%%% The balances submessage, shared with any device that settles in this token. +-define(BALANCES, <<"balances">>). +%%% The linked message whose keys the name resolves through to. +-define(VALUE, <<"value">>). +%%% The share of the supply a signer must hold to set the name, in basis +%%% points. The whole supply, unless the token says otherwise. +-define(DEFAULT_THRESHOLD_BPS, 10000). + +%% @doc Every key routes to `compute': the schedule drives this device, and +%% under `~arweave-scheduler@1.0''s `all' mode the key a slot resolves comes +%% from a stranger's `path' tag. See `dev_arweave_swap:info/0', which carries +%% the same reasoning at length. +info() -> + #{ default => fun router/4 }. + +%% @doc Apply any scheduled message, whatever it asked to be routed to. +router(_Key, Base, Assignment, Opts) -> + compute(Base, Assignment, Opts). + +%% @doc Apply one assignment to the name's state. +%% +%% A name that is for sale is settled in the same balances it keeps, so every +%% message is offered to the selling device first -- including the ones this +%% device would otherwise ignore, since the payment that buys a name is a +%% transfer between two other addresses. Which device that is, is a scalar key +%% on the process; see `swap/3' for why it is not a `~stack@1.0'. +compute(Base, Assignment, Opts) -> + Sold = swap(seed(Base, Opts), Assignment, Opts), + Body = hb_maps:get(<<"body">>, Assignment, #{}, Opts), + ProcID = hb_maps:get(<<"process">>, Assignment, <<>>, Opts), + case tx_field(Body, <<"target">>, <<>>, Opts) of + <<>> -> {ok, Sold}; + ProcID -> {ok, action(Sold, Body, Opts)}; + _ -> + % Not addressed to this name. Under `all' mode that is almost every + % data-free transaction on the network. + {ok, Sold} + end. + +%% @doc Hand the message to the device that sells this name, if it has one, and +%% take back the state it produces. +%% +%% This is a `~stack@1.0' written out by hand, and deliberately so. A process +%% spawned as an Arweave transaction can only carry flat, scalar tags: the +%% codec turns a submessage or a list into a `+link' to content that is not on +%% the weave, so nothing else can read it back. `device-stack' is a list, so a +%% stack cannot survive the spawn -- but `swap-device' is one word. +swap(Base, Assignment, Opts) -> + case state(<<"swap-device">>, Base, not_found, Opts) of + not_found -> Base; + Device -> + try hb_ao:resolve(Base#{ <<"device">> => Device }, Assignment, Opts) of + {ok, Settled} -> Settled#{ <<"device">> => <<"name-token@1.0">> }; + _ -> Base + catch + _:_ -> Base + end + end. + +%% @doc Give the name its single unit, and whatever it was minted pointing at, +%% the first time it computes. +%% +%% Neither can be written into the spawn directly: `balances' and the linked +%% message are submessages, and a submessage does not cross the chain. What does +%% cross is a scalar -- `initial-holder', an address, and `initial-value', the id +%% of whatever the name should resolve to. Values keep their case, where keys are +%% lowercased and a lowercased address is a different address. +%% +%% Seeding a value at spawn is what makes a name cheap to mint. Without it, a +%% freshly spawned name says nothing until somebody sends it a `set', and the +%% first message addressed to a process pays Arweave's new-account fee -- so +%% every name would cost that before it could resolve at all. +seed(Base, Opts) -> + seed_value(seed_holding(Base, Opts), Opts). + +seed_holding(Base, Opts) -> + case state(<<"initial-holder">>, Base, not_found, Opts) of + not_found -> Base; + Holder -> + case state(?BALANCES, Base, not_found, Opts) of + not_found -> + Supply = supply(Base, Opts), + ?event({name_token_seeded, {holder, Holder}, {supply, Supply}}), + Base#{ ?BALANCES => #{ Holder => Supply } }; + _ -> Base + end + end. + +%% @doc A name minted pointing somewhere resolves there from its first slot. The +%% value is a message of its own -- `{ target: }' -- so that a `set' can +%% later replace it with anything at all without the shape changing underneath +%% whatever is reading it. +seed_value(Base, Opts) -> + case state(<<"initial-value">>, Base, not_found, Opts) of + not_found -> Base; + Target -> + case state(?VALUE, Base, not_found, Opts) of + not_found -> + ?event({name_token_seeded_value, {target, Target}}), + Base#{ ?VALUE => #{ <<"target">> => Target } }; + _ -> Base + end + end. + +%% @doc Route a message addressed to the name by its `action'. Matching is +%% case-insensitive, as `token-1.0' matches. An unknown action leaves the state +%% untouched rather than failing the slot, which would stop the process on every +%% node for good. +action(Base, Body, Opts) -> + case hb_util:to_lower(field(<<"action">>, Body, <<>>, Opts)) of + <<"transfer">> -> transfer(Base, Body, Opts); + <<"set">> -> set_value(Base, Body, Opts); + _ -> Base + end. + +%%% The token + +%% @doc Move units between balances. Nothing is written unless the whole +%% transfer is admissible, so a rejected one is indistinguishable from a +%% message that was never sent. +transfer(Base, Body, Opts) -> + maybe + {ok, Sender} ?= signer(Body, Opts), + Recipient = field(<<"recipient">>, Body, not_found, Opts), + true ?= is_binary(Recipient), + {ok, Quantity} ?= hb_util:safe_int(field(<<"quantity">>, Body, 0, Opts)), + true ?= Quantity >= 1, + true ?= balance(Base, Sender, Opts) >= Quantity, + ?event( + {name_token_transfer, + {from, Sender}, + {to, Recipient}, + {quantity, Quantity} + } + ), + notices( + credit( + debit(Base, Sender, Quantity, Opts), + Recipient, + Quantity, + Opts + ), + Sender, + Recipient, + Quantity + ) + else + _ -> Base + end. + +%% @doc Emit the pair of notices that `token-1.0' emits for a transfer, with +%% the same keys. They are the slot's results; whether anything delivers them +%% is the process's business, not this device's. +notices(Base, Sender, Recipient, Quantity) -> + Base#{ + <<"results">> => + #{ + <<"outbox">> => + #{ + <<"1">> => + #{ + <<"target">> => Sender, + <<"action">> => <<"Debit-Notice">>, + <<"recipient">> => Recipient, + <<"quantity">> => Quantity + }, + <<"2">> => + #{ + <<"target">> => Recipient, + <<"action">> => <<"Credit-Notice">>, + <<"sender">> => Sender, + <<"quantity">> => Quantity + } + } + } + }. + +%%% The name + +%% @doc Write the message the name resolves to. The value is the message's +%% `reference-value' if it carries one, and otherwise the message itself, less +%% the keys that carried it here -- exactly the choice `~reference@1.0' offers. +set_value(Base, Body, Opts) -> + maybe + {ok, Signer} ?= signer(Body, Opts), + true ?= owns_supply(Base, Signer, Opts), + Value = value_of(Body, Opts), + ?event({name_token_set, {by, Signer}}), + Base#{ ?VALUE => Value } + else + _ -> Base + end. + +%% @doc The message a `set' is asking the name to resolve to. +value_of(Body, Opts) -> + case field(<<"reference-value">>, Body, not_found, Opts) of + not_found -> + % The set message itself is the value. The keys that addressed it to + % this name are not part of what the name says. + hb_maps:without( + [ + <<"action">>, + <<"target">>, + <<"quantity">>, + <<"anchor">>, + <<"reward">>, + <<"last_tx">>, + <<"owner">>, + <<"signature">>, + <<"commitments">>, + <<"priv">> + ], + hb_cache:ensure_all_loaded(Body, Opts), + Opts + ); + Value -> hb_cache:ensure_all_loaded(Value, Opts) + end. + +%% @doc Whether an address may speak for the name: it must hold the share of the +%% supply the token requires, which is all of it unless the token says +%% otherwise. This is `token-1.0''s `supply-threshold-owner' rule, and it is +%% evaluated against the balances as they stand -- so the authority moves with +%% the unit, with no separate owner field to keep in step. +owns_supply(Base, Address, Opts) -> + Supply = supply(Base, Opts), + Threshold = + hb_util:int( + state( + <<"set-authority-threshold-bps">>, + Base, + ?DEFAULT_THRESHOLD_BPS, + Opts + ) + ), + balance(Base, Address, Opts) * 10000 >= Supply * Threshold. + +%% @doc Read the name's supply, falling back to its single unit if malformed. +supply(Base, Opts) -> + hb_util:ok_or( + hb_util:safe_int(state(<<"total-supply">>, Base, 1, Opts)), + 1 + ). + +%%% State + +%% @doc Read a key of the process's own state. While a slot is being computed +%% the state carries this device, so a plain read would resolve the key back +%% through `compute'. See `dev_arweave_swap:state/4'. +state(Key, Base, Default, Opts) -> + hb_ao:get(Key, {as, <<"message@1.0">>, Base}, Default, Opts). + +%% @doc Read a field from an untrusted scheduled message as plain data. +field(Key, Msg, Default, Opts) -> + hb_maps:get(Key, Msg, Default, Opts). + +%% @doc Read a value from the real layer-1 transaction fields. +tx_field(Body, Field, Default, Opts) -> + case + hb_message:commitment( + #{ <<"commitment-device">> => <<"tx@1.0">> }, + Body, + Opts + ) + of + {ok, _ID, Commitment} -> + hb_maps:get(<<"field-", Field/binary>>, Commitment, Default, Opts); + _ -> Default + end. + +balance(Base, Address, Opts) -> + hb_util:int(state([?BALANCES, Address], Base, 0, Opts)). + +%% @doc Credit one balance by replacing the whole balances submessage. +credit(Base, Address, Amount, Opts) -> + Base#{ + ?BALANCES => + hb_maps:put( + Address, + balance(Base, Address, Opts) + Amount, + state(?BALANCES, Base, #{}, Opts), + Opts + ) + }. + +debit(Base, Address, Amount, Opts) -> credit(Base, Address, -Amount, Opts). + +%% @doc Setting the device is honoured, and nothing else is: `lib_process' puts +%% the process's own device back after every slot, and reading this state as a +%% message is itself a device set. Anything else is a scheduled message and is +%% applied as one. See `dev_arweave_swap:set/3'. +set(Base, Req, Opts) -> + case hb_maps:keys(Req, Opts) -- [<<"path">>, <<"set-mode">>] of + [<<"device">>] -> + {ok, + Base#{ + <<"device">> => + hb_maps:get(<<"device">>, Req, undefined, Opts) + } + }; + _ -> compute(Base, Req, Opts) + end. + +%% @doc The single signer of a message. A message with any other number of +%% signers is not attributable to one party, so it can neither move the unit nor +%% speak for the name. +signer(Body, Opts) -> + case hb_message:signers(Body, Opts) of + [Signer] -> {ok, hb_util:human_id(Signer)}; + _ -> not_found + end. + +%%% Tests + +%%% The tests drive `compute/3' directly with synthetic assignments, exactly as +%%% `~process@1.0' would. The process id must be a real 43-character address so +%%% the transaction codec can carry it as the layer-1 target. +-define(PROCESS, <<"nAmEtOkEn0000000000000000000000000000000000">>). + +test_opts() -> #{ <<"priv-wallet">> => ar_wallet:new() }. + +party() -> + Wallet = ar_wallet:new(), + {Wallet, hb_util:human_id(ar_wallet:to_address(Wallet))}. + +%% @doc A name held by one address, as it stands at spawn. +name_held_by(Address) -> + #{ + <<"name">> => <<"test-name">>, + <<"ticker">> => <<"NAME">>, + <<"denomination">> => 0, + <<"total-supply">> => 1, + ?BALANCES => #{ Address => 1 } + }. + +tx(Wallet, Fields) -> + hb_message:commit( + Fields, + #{ <<"priv-wallet">> => Wallet }, + #{ <<"commitment-device">> => <<"tx@1.0">> } + ). + +apply_tx(Base, Body, Opts) -> + {ok, New} = + compute( + Base, + #{ + <<"process">> => ?PROCESS, + <<"slot">> => 1, + <<"body">> => Body + }, + Opts + ), + New. + +%% @doc A `target' tag is not an address. A transaction addressed to nobody +%% carries the process's own id at the key a plain read would find, so the gate +%% reads the field the base layer moved value to instead. +tags_are_not_transaction_fields_test() -> + Opts = test_opts(), + {_, OwnerAddr} = party(), + {Stranger, StrangerAddr} = party(), + Held = name_held_by(OwnerAddr), + Steal = + tag_only_tx( + Stranger, + [ + {<<"target">>, ?PROCESS}, + {<<"action">>, <<"transfer">>}, + {<<"recipient">>, StrangerAddr}, + {<<"quantity">>, <<"1">>} + ] + ), + ?assertEqual(?PROCESS, hb_ao:get(<<"target">>, Steal, not_found, Opts)), + ?assertEqual(<<>>, tx_field(Steal, <<"target">>, <<>>, Opts)), + Tried = apply_tx(Held, Steal, Opts), + ?assertEqual(1, held_by(Tried, OwnerAddr, Opts)), + ?assertEqual(0, held_by(Tried, StrangerAddr, Opts)). + +%% @doc A transaction that carries its keys as tags only, addressed to nobody. +tag_only_tx(Wallet, Tags) -> + Signed = ar_tx:sign(#tx{ format = 2, reward = 1, tags = Tags }, Wallet), + hb_message:convert(Signed, <<"structured@1.0">>, <<"tx@1.0">>, #{}). + +transfer_tx(Wallet, Recipient, Quantity) -> + tx( + Wallet, + #{ + <<"target">> => ?PROCESS, + <<"action">> => <<"transfer">>, + <<"recipient">> => Recipient, + <<"quantity">> => hb_util:bin(Quantity) + } + ). + +set_tx(Wallet, Fields) -> + tx( + Wallet, + Fields#{ <<"target">> => ?PROCESS, <<"action">> => <<"set">> } + ). + +held_by(Base, Address, Opts) -> balance(Base, Address, Opts). + +value(Base, Opts) -> + hb_cache:ensure_all_loaded(state(?VALUE, Base, #{}, Opts), Opts). + +%% @doc A foreign transaction's device cannot interpret absent envelope fields. +foreign_device_is_data_test() -> + Opts = test_opts(), + {Owner, OwnerAddr} = party(), + Foreign = + tx( + Owner, + #{ + <<"device">> => <<"reference@1.0">> + } + ), + Untouched = apply_tx(name_held_by(OwnerAddr), Foreign, Opts), + ?assertEqual(1, held_by(Untouched, OwnerAddr, Opts)). + +%% @doc The unit moves, and the pair of notices `token-1.0' emits go with it. +transfer_moves_the_unit_test() -> + Opts = test_opts(), + {Owner, OwnerAddr} = party(), + {_, BuyerAddr} = party(), + Moved = + apply_tx(name_held_by(OwnerAddr), transfer_tx(Owner, BuyerAddr, 1), Opts), + ?assertEqual(0, held_by(Moved, OwnerAddr, Opts)), + ?assertEqual(1, held_by(Moved, BuyerAddr, Opts)), + Outbox = hb_ao:get(<<"results/outbox">>, {as, <<"message@1.0">>, Moved}, #{}, Opts), + ?assertEqual( + <<"Debit-Notice">>, + hb_ao:get(<<"1/action">>, Outbox, not_found, Opts) + ), + ?assertEqual( + <<"Credit-Notice">>, + hb_ao:get(<<"2/action">>, Outbox, not_found, Opts) + ), + ?assertEqual( + BuyerAddr, + hb_ao:get(<<"1/recipient">>, Outbox, not_found, Opts) + ), + ?assertEqual( + OwnerAddr, + hb_ao:get(<<"2/sender">>, Outbox, not_found, Opts) + ). + +%% @doc Nobody can send what they do not hold. +transfer_beyond_balance_test() -> + Opts = test_opts(), + {Owner, OwnerAddr} = party(), + {Stranger, _} = party(), + {_, ElsewhereAddr} = party(), + Base = name_held_by(OwnerAddr), + ?assertEqual( + 1, + held_by(apply_tx(Base, transfer_tx(Owner, ElsewhereAddr, 2), Opts), OwnerAddr, Opts) + ), + ?assertEqual( + 0, + held_by( + apply_tx(Base, transfer_tx(Stranger, ElsewhereAddr, 1), Opts), + ElsewhereAddr, + Opts + ) + ). + +%% @doc The holder says what the name resolves to, by handing over a message. +set_writes_the_linked_message_test() -> + Opts = test_opts(), + {Owner, OwnerAddr} = party(), + Set = + apply_tx( + name_held_by(OwnerAddr), + set_tx( + Owner, + #{ + <<"reference-value">> => + #{ + <<"content-type">> => <<"text/html">>, + <<"body">> => <<"

hello

">> + } + } + ), + Opts + ), + ?assertEqual( + <<"text/html">>, + hb_ao:get(<<"content-type">>, value(Set, Opts), not_found, Opts) + ), + ?assertEqual( + <<"

hello

">>, + hb_ao:get(<<"body">>, value(Set, Opts), not_found, Opts) + ). + +%% @doc With no `reference-value', the name inherits the keys of the `set' +%% itself -- less the keys that carried it here. +set_without_value_inherits_own_keys_test() -> + Opts = test_opts(), + {Owner, OwnerAddr} = party(), + Set = + apply_tx( + name_held_by(OwnerAddr), + set_tx(Owner, #{ <<"greeting">> => <<"ahoy">> }), + Opts + ), + Value = value(Set, Opts), + ?assertEqual(<<"ahoy">>, hb_ao:get(<<"greeting">>, Value, not_found, Opts)), + ?assertEqual(not_found, hb_ao:get(<<"action">>, Value, not_found, Opts)), + ?assertEqual(not_found, hb_ao:get(<<"target">>, Value, not_found, Opts)). + +%% @doc Anybody who does not hold the name cannot speak for it. +set_by_stranger_test() -> + Opts = test_opts(), + {_, OwnerAddr} = party(), + {Stranger, _} = party(), + Result = + apply_tx( + name_held_by(OwnerAddr), + set_tx(Stranger, #{ <<"greeting">> => <<"mine now">> }), + Opts + ), + ?assertEqual(#{}, value(Result, Opts)). + +%% @doc The authority is the holding, not a recorded owner: it goes with the +%% unit and needs nothing kept in step. This is what makes a name sellable. +authority_follows_the_unit_test() -> + Opts = test_opts(), + {Owner, OwnerAddr} = party(), + {Buyer, BuyerAddr} = party(), + Base = name_held_by(OwnerAddr), + Before = apply_tx(Base, set_tx(Owner, #{ <<"points-at">> => <<"seller">> }), Opts), + ?assertEqual( + <<"seller">>, + hb_ao:get(<<"points-at">>, value(Before, Opts), not_found, Opts) + ), + Sold = apply_tx(Before, transfer_tx(Owner, BuyerAddr, 1), Opts), + % The seller has handed over the unit, and with it the right to speak. + Stale = apply_tx(Sold, set_tx(Owner, #{ <<"points-at">> => <<"seller again">> }), Opts), + ?assertEqual( + <<"seller">>, + hb_ao:get(<<"points-at">>, value(Stale, Opts), not_found, Opts) + ), + Fresh = apply_tx(Sold, set_tx(Buyer, #{ <<"points-at">> => <<"buyer">> }), Opts), + ?assertEqual( + <<"buyer">>, + hb_ao:get(<<"points-at">>, value(Fresh, Opts), not_found, Opts) + ), + ?assertEqual(1, held_by(Fresh, BuyerAddr, Opts)), + ?assertEqual(0, held_by(Fresh, OwnerAddr, Opts)). + +%% @doc A partial holding is not the whole supply, so it does not carry the +%% authority. (A name is indivisible, but the rule is the supply threshold.) +partial_holding_cannot_set_test() -> + Opts = test_opts(), + {Owner, OwnerAddr} = party(), + {_, OtherAddr} = party(), + Split = + (name_held_by(OwnerAddr))#{ + <<"total-supply">> => 2, + ?BALANCES => #{ OwnerAddr => 1, OtherAddr => 1 } + }, + Result = apply_tx(Split, set_tx(Owner, #{ <<"greeting">> => <<"half mine">> }), Opts), + ?assertEqual(#{}, value(Result, Opts)). + +%% @doc There is no mint path, so the supply cannot grow -- an unknown action +%% is ignored rather than failing the slot. +supply_is_fixed_test() -> + Opts = test_opts(), + {Owner, OwnerAddr} = party(), + Result = + apply_tx( + name_held_by(OwnerAddr), + tx( + Owner, + #{ + <<"target">> => ?PROCESS, + <<"action">> => <<"mint">>, + <<"quantity">> => <<"1000">> + } + ), + Opts + ), + ?assertEqual(1, held_by(Result, OwnerAddr, Opts)), + ?assertEqual(1, hb_util:int(state(<<"total-supply">>, Result, 0, Opts))). + +%% @doc Actions are matched however they are cased, as `token-1.0' matches. +action_case_is_ignored_test() -> + Opts = test_opts(), + {Owner, OwnerAddr} = party(), + {_, BuyerAddr} = party(), + Moved = + apply_tx( + name_held_by(OwnerAddr), + tx( + Owner, + #{ + <<"target">> => ?PROCESS, + <<"action">> => <<"Transfer">>, + <<"recipient">> => BuyerAddr, + <<"quantity">> => <<"1">> + } + ), + Opts + ), + ?assertEqual(1, held_by(Moved, BuyerAddr, Opts)). + +%% @doc A transaction that is not addressed to this name is left alone. Under +%% `all' mode that is almost every data-free transaction on the network. +unrelated_traffic_test() -> + Opts = test_opts(), + {Stranger, _} = party(), + {_, OwnerAddr} = party(), + Base = name_held_by(OwnerAddr), + Result = + apply_tx( + Base, + tx(Stranger, #{ <<"target">> => <<"somebody-else">>, <<"quantity">> => <<"1">> }), + Opts + ), + ?assertEqual(1, held_by(Result, OwnerAddr, Opts)). + +%% @doc A name spawned as an Arweave transaction carries only scalars, so it +%% names its first holder rather than holding a balances submessage. The unit +%% appears the first time it computes. +seeded_from_initial_holder_test() -> + Opts = test_opts(), + {Owner, OwnerAddr} = party(), + Spawned = + #{ + <<"name">> => <<"test-name">>, + <<"total-supply">> => 1, + <<"initial-holder">> => OwnerAddr + }, + % Any message at all brings the name to life, including one that is not + % addressed to it. + Alive = + apply_tx(Spawned, tx(Owner, #{ <<"target">> => <<"elsewhere">> }), Opts), + ?assertEqual(1, held_by(Alive, OwnerAddr, Opts)), + % And the holder can immediately speak for it. + Set = apply_tx(Alive, set_tx(Owner, #{ <<"greeting">> => <<"mine">> }), Opts), + ?assertEqual( + <<"mine">>, + hb_ao:get(<<"greeting">>, value(Set, Opts), not_found, Opts) + ). + +%% @doc A name minted pointing somewhere resolves there immediately, without +%% anybody having to send it a message -- which matters because the first message +%% addressed to a process pays to create its account. +seeded_value_test() -> + Opts = test_opts(), + {Owner, OwnerAddr} = party(), + Spawned = + #{ + <<"name">> => <<"pn-test-1">>, + <<"total-supply">> => 1, + <<"initial-holder">> => OwnerAddr, + <<"initial-value">> => <<"aBcDeFgHiJkLmNoPqRsTuVwXyZ0123456789_-aBcDe">> + }, + Alive = apply_tx(Spawned, tx(Owner, #{ <<"target">> => <<"elsewhere">> }), Opts), + ?assertEqual( + <<"aBcDeFgHiJkLmNoPqRsTuVwXyZ0123456789_-aBcDe">>, + hb_ao:get(<<"target">>, value(Alive, Opts), not_found, Opts) + ), + % And the holder can still point it somewhere else afterwards. + Reset = apply_tx(Alive, set_tx(Owner, #{ <<"greeting">> => <<"moved">> }), Opts), + ?assertEqual( + <<"moved">>, + hb_ao:get(<<"greeting">>, value(Reset, Opts), not_found, Opts) + ), + ?assertEqual(not_found, hb_ao:get(<<"target">>, value(Reset, Opts), not_found, Opts)). + +%% @doc An unreadable supply falls back to the single unit a name represents. +unreadable_supply_is_one_unit_test() -> + Opts = test_opts(), + {Owner, OwnerAddr} = party(), + Alive = + apply_tx( + #{ + <<"name">> => <<"test-name">>, + <<"total-supply">> => <<"one">>, + <<"initial-holder">> => OwnerAddr + }, + tx(Owner, #{ <<"target">> => <<"elsewhere">> }), + Opts + ), + ?assertEqual(1, held_by(Alive, OwnerAddr, Opts)), + Set = apply_tx(Alive, set_tx(Owner, #{ <<"greeting">> => <<"mine">> }), Opts), + ?assertEqual( + <<"mine">>, + hb_ao:get(<<"greeting">>, value(Set, Opts), not_found, Opts) + ). + +%% @doc Seeding happens once. A name whose unit has moved on is not handed a +%% fresh one by the next message that arrives. +seeding_happens_once_test() -> + Opts = test_opts(), + {Owner, OwnerAddr} = party(), + {Stranger, StrangerAddr} = party(), + Spawned = + #{ + <<"name">> => <<"test-name">>, + <<"total-supply">> => 1, + <<"initial-holder">> => OwnerAddr + }, + Alive = apply_tx(Spawned, tx(Owner, #{ <<"target">> => <<"elsewhere">> }), Opts), + Moved = apply_tx(Alive, transfer_tx(Owner, StrangerAddr, 1), Opts), + Later = + apply_tx(Moved, tx(Stranger, #{ <<"target">> => <<"elsewhere">> }), Opts), + ?assertEqual(0, held_by(Later, OwnerAddr, Opts)), + ?assertEqual(1, held_by(Later, StrangerAddr, Opts)). + +%% @doc A missing or invalid swap device cannot wedge the name. +without_a_working_swap_device_test() -> + Opts = test_opts(), + {Owner, OwnerAddr} = party(), + lists:foreach( + fun(Base) -> + Set = + apply_tx( + Base, + set_tx(Owner, #{ <<"greeting">> => <<"no swap here">> }), + Opts + ), + ?assertEqual( + <<"no swap here">>, + hb_ao:get(<<"greeting">>, value(Set, Opts), not_found, Opts) + ), + ?assertEqual(not_found, state(<<"orders">>, Set, not_found, Opts)) + end, + [ + name_held_by(OwnerAddr), + (name_held_by(OwnerAddr))#{ <<"swap-device">> => <<"nOt-A-dEvIcE@1.0">> } + ] + ). + +%% @doc A stranger's transaction may ask to be routed anywhere, including to +%% keys `~message@1.0' would otherwise answer. Each is applied like any other +%% message rather than handing back the state, a list of key names, or an info +%% map as the new state. +reserved_paths_are_applied_test() -> + Opts = test_opts(), + {_, OwnerAddr} = party(), + {Stranger, _} = party(), + Base = name_held_by(OwnerAddr), + lists:foreach( + fun(Path) -> + {ok, State} = + hb_ao:resolve( + Base#{ <<"device">> => <<"name-token@1.0">> }, + #{ + <<"path">> => Path, + <<"process">> => ?PROCESS, + <<"slot">> => 2, + ?BALANCES => #{ OwnerAddr => 1000000 }, + <<"body">> => + tx(Stranger, #{ <<"target">> => <<"somebody-else">> }) + }, + Opts + ), + ?assertEqual(1, held_by(State, OwnerAddr, Opts)) + end, + [<<"set">>, <<"keys">>, <<"info">>, <<"balances">>, <<"anything-else">>] + ). + +%%% The permanent fixtures +%%% +%%% The driver that posted these stories spends real AR and does not belong in +%%% the test battery. It remains recoverable from the commit that created them: +%%% +%%% git show 824816e7c:src/preloaded/process/dev_name_token.erl +%%% +%%% Story one, replayed: a name that had to be paid for +%%% +%%% Every transaction below is on mainnet. The reads walk the process forward +%%% one slot at a time, so what the schedule did to the state is visible in the +%%% order it happened, rather than only at the end. +%%% +%%% make-offer the seller's unit goes into escrow, the order opens +%%% register:underpaid a reward of 33,039,920 does not clear a 100,000,000 +%%% fee -- the order stays open +%%% register:stranded 150,000,000 sent *to the process* is not the fee +%%% either: it would only be stranded there +%%% register:paid a reward of 150,000,000 does clear it -- reserved +%%% payment:interloper the underpayer pays anyway, but the order is not +%%% theirs and there is no bond to compensate them with +%%% payment:buyer the registrant pays -- settled, and the name moves +%%% set:former-owner the seller no longer holds it, so is no longer heard +%%% set:owner the buyer says what the name means +%%% stray:path-info a stranger routes a slot at `info'; nothing breaks +%%% make-offer:nonsense a deadline of `tomorrow' opens no order +-define(SALE_PROCESS, <<"an95oAK9MlahZI_tKKeG4ykzNN01qfMi2WfJO58o_UU">>). +-define(SALE_SELLER, <<"ggltHF0Cnv9ylH3vM1p7amR2vXLMoPLQIUQmAEwLP-k">>). +-define(SALE_UNDERPAYER, <<"2yvAwMDrF62hpH_kKTfguatzB9mKVzcM2edAn8KauTQ">>). +-define(SALE_BUYER, <<"LW0myHWuv7XcLec19OCDzFJW0P6jXPG_Ao49kfy9Slc">>). +-define(SALE_OFFER, <<"r6lleOybw5_Pz-3EDHjLwAYv8XNGTlLMjT_8pA9e6o0">>). +-define(SALE_UNDERPAID, <<"060IcKkUJ4ggdojenpcvSpTQjbuGDsoDz_jEhs-C4E8">>). +-define(SALE_STRANDED, <<"f31_VyEl5NumgmKUPeokDgPAxbOZj32bFBf3D5wwf_A">>). +-define(SALE_PAID, <<"RJXzg_GbIo7mUs3oNXfG4DI2-1EK7rWipN5YlvFxZFI">>). +-define(SALE_INTERLOPER, <<"L1mrhwV8JY0-n7B2XZu2R9Ox5MjCsdnsSohpNl_UHAg">>). +-define(SALE_PAYMENT, <<"B4F3TSTcLBuivjQ9Rzf-2IyY2svHUIMnfQjHGbegn6c">>). +-define(SALE_STALE_SET, <<"ndgQ_1zZCm3cMlmC42jhLFZHFwo60wVs5znQ14Mip4Q">>). +-define(SALE_OWNER_SET, <<"vc3eQypdoGc4--NMQkah5tYYnG62KkYtpme1JVTpeAU">>). +-define(SALE_STRAY, <<"bwEotEyzbH4AYP_-5WZCZB7TgTRI4bFjVdJl6fjmuhE">>). +-define(SALE_NONSENSE, <<"aYIqfkDstZTyewfRdqNcKHYdfCeC2jqyICD0juRCoC0">>). +-define(SALE_MAX_HEIGHT, 1966084). + +%% @doc Node options pinned to a story's last block, so its answer is +%% immutable: no block after it can reach the process. +story_opts(MaxHeight, Process) -> + TestStore = hb_test_utils:test_store(), + IndexStore = hb_test_utils:test_store(), + (hb_opts:default_message())#{ + <<"store">> => [ + TestStore, + #{ + <<"store-module">> => hb_store_arweave, + <<"name">> => <<"cache-arweave">>, + <<"index-store">> => [IndexStore] + }, + #{ + <<"store-module">> => hb_store_gateway, + <<"local-store">> => [TestStore] + } + ], + <<"arweave-index-store">> => #{ <<"index-store">> => [IndexStore] }, + <<"arweave-index-workers">> => 8, + <<"arweave-scheduler-confirmation-depth">> => 1, + <<"arweave-scheduler-max-height">> => MaxHeight, + <<"name-resolvers">> => [#{ <<"test-name">> => Process }], + <<"node-host">> => <<"host">>, + <<"priv-wallet">> => ar_wallet:new() + }. + +%% @doc Synchronize a story's schedule, retrying while the gateway rate-limits +%% us -- the same allowance the scheduler's own fixture tests make. +story_sync(_Process, _Opts, 0) -> {error, sync_failed}; +story_sync(Process, Opts, Attempts) -> + case + hb_ao:resolve( + #{ <<"device">> => <<"arweave-scheduler@1.0">> }, + #{ + <<"path">> => <<"schedule">>, + <<"method">> => <<"GET">>, + <<"target">> => Process + }, + Opts + ) + of + {ok, Schedule} -> {ok, Schedule}; + _ -> + timer:sleep(5000), + story_sync(Process, Opts, Attempts - 1) + end. + +%% @doc The slot a transaction was given. The stories pin transaction ids +%% rather than slot numbers, because a slot number is a fact about the weave -- +%% every data-free transaction on the network takes one -- while the id is the +%% message itself. +slot_of(Schedule, TXID, Opts) -> + Assignments = + hb_ao:normalize_keys(hb_ao:get(<<"assignments">>, Schedule, Opts), Opts), + Found = + [ + Slot + || + {Key, Assignment} <- hb_maps:to_list(Assignments, Opts), + {ok, Slot} <- [hb_util:safe_int(Key)], + is_map(Assignment), + hb_util:human_id( + hb_message:id(hb_ao:get(<<"body">>, Assignment, Opts), signed, Opts) + ) =:= TXID + ], + case Found of + [Slot | _] -> Slot; + [] -> error({not_scheduled, TXID}) + end. + +%% @doc Read a process's state as it stood at the end of a given slot, in the +%% form a reader can follow: `~process@1.0/compute&slot=/'. +at(Process, Slot, Path, Opts) -> + hb_ao:resolve( + << + Process/binary, + "~process@1.0/compute&slot=", + (hb_util:bin(Slot))/binary, + "/", + Path/binary + >>, + Opts#{ <<"hashpath">> => ignore } + ). + +sale_story_test_() -> {timeout, 3600, fun sale_story/0}. +sale_story() -> + Process = ?SALE_PROCESS, + Opts = story_opts(?SALE_MAX_HEIGHT, Process), + {ok, Schedule} = story_sync(Process, Opts, 5), + Slot = fun(TXID) -> slot_of(Schedule, TXID, Opts) end, + Seller = ?SALE_SELLER, + Buyer = ?SALE_BUYER, + Poor = ?SALE_UNDERPAYER, + Order = ?SALE_OFFER, + % The offer escrows the seller's only unit, and opens the order. + Offered = Slot(?SALE_OFFER), + ?assertEqual({ok, 0}, at(Process, Offered, <<"balances/", Seller/binary>>, Opts)), + ?assertEqual( + {ok, <<"open">>}, + at(Process, Offered, <<"orders/", Order/binary, "/status">>, Opts) + ), + ?assertEqual( + {ok, 100000000}, + at(Process, Offered, <<"orders/", Order/binary, "/minimum-fee">>, Opts) + ), + % A registration paying only what the network charges does not clear a fee + % set above it, and neither does value sent to the process. + Underpaid = Slot(?SALE_UNDERPAID), + ?assertEqual( + {ok, <<"open">>}, + at(Process, Underpaid, <<"orders/", Order/binary, "/status">>, Opts) + ), + Stranded = Slot(?SALE_STRANDED), + ?assertEqual( + {ok, <<"open">>}, + at(Process, Stranded, <<"orders/", Order/binary, "/status">>, Opts) + ), + % Overpaying the reward does. + Paid = Slot(?SALE_PAID), + ?assertEqual( + {ok, <<"reserved">>}, + at(Process, Paid, <<"orders/", Order/binary, "/status">>, Opts) + ), + ?assertEqual( + {ok, Buyer}, + at(Process, Paid, <<"orders/", Order/binary, "/buyer">>, Opts) + ), + % Somebody else's payment buys nothing while it is reserved, and with no + % bond posted there is nothing to compensate them with either. + Interloped = Slot(?SALE_INTERLOPER), + ?assertEqual( + {ok, <<"reserved">>}, + at(Process, Interloped, <<"orders/", Order/binary, "/status">>, Opts) + ), + ?assertMatch( + {error, not_found}, + at(Process, Interloped, <<"balances/", Poor/binary>>, Opts) + ), + % The registrant's payment settles it, and the name moves. + Settled = Slot(?SALE_PAYMENT), + ?assertMatch( + {error, _}, + at(Process, Settled, <<"orders/", Order/binary, "/status">>, Opts) + ), + ?assertEqual({ok, 1}, at(Process, Settled, <<"balances/", Buyer/binary>>, Opts)), + ?assertEqual({ok, 0}, at(Process, Settled, <<"balances/", Seller/binary>>, Opts)), + % The seller no longer holds the name, so is no longer heard. + Stale = Slot(?SALE_STALE_SET), + ?assertMatch({error, not_found}, at(Process, Stale, <<"value/greeting">>, Opts)), + % The buyer is. + Spoken = Slot(?SALE_OWNER_SET), + ?assertEqual( + {ok, <<"hello from the new owner">>}, + at(Process, Spoken, <<"value/greeting">>, Opts) + ), + % A stranger routing a slot at `info' changes nothing and breaks nothing: + % the process still computes, and still holds what it held. + Stray = Slot(?SALE_STRAY), + ?assertEqual({ok, 1}, at(Process, Stray, <<"balances/", Buyer/binary>>, Opts)), + ?assertEqual( + {ok, <<"hello from the new owner">>}, + at(Process, Stray, <<"value/greeting">>, Opts) + ), + % And an offer whose deadline is `tomorrow' opens nothing. + Nonsense = Slot(?SALE_NONSENSE), + ?assertMatch( + {error, not_found}, + at(Process, Nonsense, <<"orders/", (?SALE_NONSENSE)/binary, "/status">>, Opts) + ), + ?assertEqual({ok, 1}, at(Process, Nonsense, <<"balances/", Buyer/binary>>, Opts)). + +%%% Story two, replayed: an offer withdrawn +%%% +%%% A seller may take back an order nobody has reserved. Once they have, the +%%% order is spent: registering against it is refused however much is paid, and +%%% a payment against it buys nothing and -- there being no bond -- compensates +%%% nobody. A stranger may not withdraw somebody else's order. +-define(WITHDRAWN_PROCESS, <<"petNFJyilEh0YvFb39FqoL7CeBUvmnuQ73eHSlGLYxI">>). +-define(WITHDRAWN_OFFER, <<"H1aRe1UoXqSW1IA1H4fZf8oUkHRFzgfzv2cw6hsRYGE">>). +-define(WITHDRAWN_CANCEL, <<"zr8xLGjdXJWvWa3VYMjigEk4c47mteGRfUE0cZzX_hc">>). +-define(WITHDRAWN_LATE_REGISTER, <<"m0KwPLpX1zPiR-k9H7Hx16xwGQHRM82O_WgS8E3H1X8">>). +-define(WITHDRAWN_LATE_PAYMENT, <<"nUalA_4J6M-1fV9mx0dDaT4V1GQHGWmcW09G7ZuEOjA">>). +-define(WITHDRAWN_SECOND, <<"0--Z1ngGMHPCrT5Wj9wxFXGapoXoDCq97u2jCuIlZkQ">>). +-define(WITHDRAWN_STRANGER, <<"o6N4lmA5tV318cOEoXTnN5pmBRYGXtIB8K3VaSPCONw">>). +-define(WITHDRAWN_MAX_HEIGHT, 1966093). + +withdrawn_story_test_() -> {timeout, 3600, fun withdrawn_story/0}. +withdrawn_story() -> + Process = ?WITHDRAWN_PROCESS, + Opts = story_opts(?WITHDRAWN_MAX_HEIGHT, Process), + {ok, Schedule} = story_sync(Process, Opts, 5), + Slot = fun(TXID) -> slot_of(Schedule, TXID, Opts) end, + Seller = ?SALE_SELLER, + Buyer = ?SALE_BUYER, + First = ?WITHDRAWN_OFFER, + Second = ?WITHDRAWN_SECOND, + % The offer escrows the unit. + Offered = Slot(?WITHDRAWN_OFFER), + ?assertEqual({ok, 0}, at(Process, Offered, <<"balances/", Seller/binary>>, Opts)), + ?assertEqual( + {ok, <<"open">>}, + at(Process, Offered, <<"orders/", First/binary, "/status">>, Opts) + ), + % Withdrawing it gives the unit back. + Cancelled = Slot(?WITHDRAWN_CANCEL), + ?assertMatch( + {error, _}, + at(Process, Cancelled, <<"orders/", First/binary, "/status">>, Opts) + ), + ?assertEqual({ok, 1}, at(Process, Cancelled, <<"balances/", Seller/binary>>, Opts)), + % A registration that pays the fee in full is still refused: the order is + % no longer open. + LateRegister = Slot(?WITHDRAWN_LATE_REGISTER), + ?assertMatch( + {error, _}, + at(Process, LateRegister, <<"orders/", First/binary, "/status">>, Opts) + ), + ?assertMatch( + {error, not_found}, + at(Process, LateRegister, <<"orders/", First/binary, "/buyer">>, Opts) + ), + % And a payment against it moves nothing, in either direction: the goods + % are back with the seller and there was no bond to compensate anyone from. + LatePayment = Slot(?WITHDRAWN_LATE_PAYMENT), + ?assertMatch( + {error, _}, + at(Process, LatePayment, <<"orders/", First/binary, "/status">>, Opts) + ), + ?assertEqual( + {ok, 1}, + at(Process, LatePayment, <<"balances/", Seller/binary>>, Opts) + ), + ?assertMatch( + {error, not_found}, + at(Process, LatePayment, <<"balances/", Buyer/binary>>, Opts) + ), + % The seller offers it again, and a stranger tries to withdraw it. + Reoffered = Slot(?WITHDRAWN_SECOND), + ?assertEqual( + {ok, <<"open">>}, + at(Process, Reoffered, <<"orders/", Second/binary, "/status">>, Opts) + ), + Meddled = Slot(?WITHDRAWN_STRANGER), + ?assertEqual( + {ok, <<"open">>}, + at(Process, Meddled, <<"orders/", Second/binary, "/status">>, Opts) + ), + ?assertEqual({ok, 0}, at(Process, Meddled, <<"balances/", Seller/binary>>, Opts)). + +%%% Story three, replayed: a name handed over +%%% +%%% No sale at all -- the token half on its own. The unit moves by `transfer', +%%% and the authority moves with it: the former holder's word stops counting +%%% the moment it does, and the new holder's starts. +-define(HANDOVER_PROCESS, <<"D4uhF_nO_vyPoIhPDZ0kFMyfOnk1ZCFJFkmnxVw7vSs">>). +-define(HANDOVER_FIRST_SET, <<"dx_Dmvnp2FDZdb3wlDpfVvc4k3UCbQRWLXIEwwgGxIA">>). +-define(HANDOVER_TRANSFER, <<"WgX3Ih8Ef7aDzQ_8Ziio_qYSA5z_-OUcCOVjmdC8WV0">>). +-define(HANDOVER_STALE_SET, <<"-epGyJadxPyQK_bibsic_btn133f_J8UCX1AndWIWRs">>). +-define(HANDOVER_NEW_SET, <<"mqpK8TsoWFUJW345gcYLrFmw4nxqyGqpavt7xUaniC0">>). +-define(HANDOVER_MAX_HEIGHT, 1966100). + +handover_story_test_() -> {timeout, 3600, fun handover_story/0}. +handover_story() -> + Process = ?HANDOVER_PROCESS, + Opts = story_opts(?HANDOVER_MAX_HEIGHT, Process), + {ok, Schedule} = story_sync(Process, Opts, 5), + Slot = fun(TXID) -> slot_of(Schedule, TXID, Opts) end, + Seller = ?SALE_SELLER, + Buyer = ?SALE_BUYER, + % While the seller holds it, the seller speaks for it. + Spoke = Slot(?HANDOVER_FIRST_SET), + ?assertEqual({ok, 1}, at(Process, Spoke, <<"balances/", Seller/binary>>, Opts)), + ?assertEqual({ok, <<"seller">>}, at(Process, Spoke, <<"value/points-at">>, Opts)), + % The unit moves. + Moved = Slot(?HANDOVER_TRANSFER), + ?assertEqual({ok, 0}, at(Process, Moved, <<"balances/", Seller/binary>>, Opts)), + ?assertEqual({ok, 1}, at(Process, Moved, <<"balances/", Buyer/binary>>, Opts)), + % The notices `token-1.0' emits for a transfer are the slot's results. + ?assertEqual( + {ok, <<"Debit-Notice">>}, + at(Process, Moved, <<"results/outbox/1/action">>, Opts) + ), + ?assertEqual( + {ok, <<"Credit-Notice">>}, + at(Process, Moved, <<"results/outbox/2/action">>, Opts) + ), + % The former holder is no longer heard: the name still says what it said. + Stale = Slot(?HANDOVER_STALE_SET), + ?assertEqual({ok, <<"seller">>}, at(Process, Stale, <<"value/points-at">>, Opts)), + % The new holder is. + Spoken = Slot(?HANDOVER_NEW_SET), + ?assertEqual({ok, <<"buyer">>}, at(Process, Spoken, <<"value/points-at">>, Opts)), + ?assertEqual({ok, 1}, at(Process, Spoken, <<"balances/", Buyer/binary>>, Opts)). + +%%% Story four: a name, read as a name. +%%% +%%% The three stories above are read slot by slot, through +%%% `~process@1.0/compute&slot='. This one is read the way a person reads a +%%% name: `~name@1.0' resolves the label to the process, and `/now' computes the +%%% whole schedule at once rather than one slot at a time. The name really was +%%% sold -- the buyer holds the unit and has said what it means -- and the +%%% payment that did it was an ordinary transfer between two addresses that the +%%% process was never a party to. +-define(NAMED_PROCESS, <<"yWRe7v4SZ4_NKV6LkYyNPrFdzEaGh0ckblu-CaGXqG4">>). +-define(NAMED_ORDER, <<"3BApJHeatc9pVuLgjZ_P-HT5hZgRE1Q3I1bdTESgRDM">>). +-define(NAMED_MAX_HEIGHT, 1966044). + +name_resolution_test_() -> {timeout, 1800, fun name_resolution/0}. +name_resolution() -> + Opts = story_opts(?NAMED_MAX_HEIGHT, ?NAMED_PROCESS), + % A node that serves a name holds it: priming the schedule puts the process + % in the node's own cache, which is what the resolver then loads. + {ok, _} = story_sync(?NAMED_PROCESS, Opts, 5), + ?assertEqual( + {ok, ?NAMED_PROCESS}, + hb_ao:resolve_many( + [ + #{ <<"device">> => <<"name@1.0">> }, + #{ <<"path">> => <<"test-name">>, <<"load">> => false } + ], + Opts + ) + ), + % `test-name.host' is the same lookup: the node's own host is stripped from + % the request's host, leaving the label to resolve. + {ok, Resolved} = + hb_ao:resolve( + #{ <<"device">> => <<"name@1.0">> }, + #{ + <<"path">> => <<"request">>, + <<"request">> => #{ <<"host">> => <<"test-name.host">> }, + <<"body">> => [#{ <<"path">> => <<"now">> }] + }, + Opts + ), + [Named | _] = hb_ao:get(<<"body">>, Resolved, [], Opts), + Loaded = hb_cache:ensure_all_loaded(Named, Opts), + % The message the host resolved to is this name: it carries the name's own + % spawn keys, and nothing else on the weave does. + ?assertEqual(<<"test-name">>, hb_ao:get(<<"name">>, Loaded, not_found, Opts)), + ?assertEqual( + ?SALE_SELLER, + hb_ao:get(<<"initial-holder">>, Loaded, not_found, Opts) + ), + ?assertEqual( + <<"name-token@1.0">>, + hb_ao:get(<<"execution-device">>, Loaded, not_found, Opts) + ), + % And the whole schedule, computed at once: the name is the buyer's, the + % order it was sold by has left the book, and the new owner has spoken. The + % process is read from the node's cache, so that it is its canonical `tx@1.0' + % decoding rather than a gateway store's lossier one. + {ok, Raw} = hb_cache:read(?NAMED_PROCESS, Opts), + {ok, State} = + hb_ao:resolve(hb_cache:ensure_all_loaded(Raw, Opts), <<"now">>, Opts), + Read = + fun(Path) -> + hb_ao:get(Path, {as, <<"message@1.0">>, State}, not_found, Opts) + end, + ?assertEqual(1, hb_util:int(Read([?BALANCES, ?SALE_BUYER]))), + ?assertEqual(0, hb_util:int(Read([?BALANCES, ?SALE_SELLER]))), + ?assertEqual(1, hb_util:int(Read(<<"total-supply">>))), + ?assertEqual(not_found, Read([<<"orders">>, ?NAMED_ORDER, <<"creator">>])), + ?assertEqual(<<"hello from the new owner">>, Read([?VALUE, <<"greeting">>])), + ?assertEqual(<<"text/plain">>, Read([?VALUE, <<"content-type">>])). diff --git a/src/preloaded/process/dev_process.erl b/src/preloaded/process/dev_process.erl index 55eea8feb..85b19caf3 100644 --- a/src/preloaded/process/dev_process.erl +++ b/src/preloaded/process/dev_process.erl @@ -387,11 +387,11 @@ compute_slot(ProcID, State, RawInputMsg, InitReq, TargetSlot, Opts) -> {store_ms, StoreTimeMicroSecs div 1000}, {computed_slot_size, erlang:external_size(NewProcStateMsgWithSlot)}, {action, - hb_ao:get( - <<"body/action">>, - Req, + hb_maps:get( + <<"action">>, + hb_maps:get(<<"body">>, Req, #{}, Opts), no_action_set, - Opts#{ <<"hashpath">> => ignore } + Opts ) } } @@ -646,35 +646,56 @@ now(RawBase, Req, Opts) -> LatestKnown = dev_process_cache:latest(ProcessID, [], Opts), case LatestKnown of {ok, LatestSlot, RawLatestMsg} -> - LatestMsg = without_snapshot(RawLatestMsg, Opts), - ?event(compute_cache, - {serving_latest_cached_state, - {proc_id, ProcessID}, - {slot, LatestSlot} - }, - Opts - ), - dev_process_worker:notify_compute( - ProcessID, - LatestSlot, - {ok, LatestMsg}, - Opts - ), - {ok, LatestMsg}; + case dev_process_cache:fresh(ProcessID, Req, Opts) of + true -> + LatestMsg = without_snapshot(RawLatestMsg, Opts), + ?event(compute_cache, + {serving_latest_cached_state, + {proc_id, ProcessID}, + {slot, LatestSlot} + }, + Opts + ), + dev_process_worker:notify_compute( + ProcessID, + LatestSlot, + {ok, LatestMsg}, + Opts + ), + {ok, LatestMsg}; + false -> + ?event(compute_cache, + {latest_cached_state_stale, + {proc_id, ProcessID}, + {slot, LatestSlot} + }, + Opts + ), + uncached_now( + CacheParam, + Base, + Req, + Opts, + <<"No fresh cached state available.">> + ) + end; _ -> - if CacheParam =/= always -> - % The node is configured to use the cache if possible, - % but forcing computation is also admissible. Subsequently, - % as no other option is available, we compute the state. - now(Base, Req, Opts#{ <<"process-now-from-cache">> => false }); - true -> - % The node is configured to only serve the latest known - % state from the cache, so we return the latest slot. - {failure, <<"No cached state available.">>} - end + uncached_now(CacheParam, Base, Req, Opts) end end. +uncached_now(CacheParam, Base, Req, Opts) -> + uncached_now(CacheParam, Base, Req, Opts, <<"No cached state available.">>). + +uncached_now(always, _Base, _Req, _Opts, Failure) -> + {failure, Failure}; +uncached_now(<<"always">>, _Base, _Req, _Opts, Failure) -> + {failure, Failure}; +uncached_now(_CacheParam, Base, Req, Opts, _Failure) -> + % The node is configured to use the cache if possible, but forcing + % computation is also admissible. + now(Base, Req, Opts#{ <<"process-now-from-cache">> => false }). + %% @doc Recursively push messages to the scheduler until we find a message %% that does not lead to any further messages being scheduled. push(Base, Req, Opts) -> diff --git a/src/preloaded/process/dev_process_cache.erl b/src/preloaded/process/dev_process_cache.erl index b2617ee6e..d969e7ea2 100644 --- a/src/preloaded/process/dev_process_cache.erl +++ b/src/preloaded/process/dev_process_cache.erl @@ -3,20 +3,22 @@ %%% convenient interface for reading the result of a process at a given slot or %%% message ID. -module(dev_process_cache). --export([latest/2, latest/3, latest/4, read/2, read/3, write/4]). +-export([fresh/3, latest/2, latest/3, latest/4, read/2, read/3, write/4]). -include_lib("eunit/include/eunit.hrl"). -include("include/hb.hrl"). %% @doc Read the result of a process at a given slot. read(ProcID, Opts) -> hb_util:ok(latest(ProcID, Opts)). -read(ProcID, SlotRef, Opts) -> +read(ProcID, SlotRef, RawOpts) -> + Opts = lib_process:cache_opts(RawOpts), ?event({reading_computed_result, ProcID, SlotRef}), Path = path(ProcID, SlotRef, Opts), hb_cache:read(Path, Opts). %% @doc Write a process computation result to the cache. -write(ProcID, Slot, Msg, Opts) -> +write(ProcID, Slot, Msg, RawOpts) -> + Opts = lib_process:cache_opts(RawOpts), % Write the item to the cache in the root of the store. {ok, Root} = hb_cache:write(hb_private:reset(Msg), Opts), % Link the item to the path in the store by slot number. @@ -38,9 +40,77 @@ write(ProcID, Slot, Msg, Opts) -> } ), hb_cache:link(Root, MsgIDPath, Opts), + write_refreshed_at(ProcID, Opts), % Return the slot number path. {ok, SlotNumPath}. +%% @doc Mark the process as refreshed at the current clock time. +write_refreshed_at(ProcID, Opts) -> + Store = hb_opts:get(store, no_viable_store, Opts), + ok = hb_store:write( + Store, + #{ refreshed_path(ProcID) => hb_util:bin(clock(Opts)) }, + Opts + ). + +refreshed_path(ProcID) -> + path(ProcID, <<"refreshed-at">>, #{}). + +%% @doc Return whether the latest cached process output is fresh enough for +%% `/now' to serve from cache under the effective `max-age'. +fresh(ProcID, Req, RawOpts) -> + Opts = lib_process:scoped_opts(RawOpts), + case effective_max_age(Req, Opts) of + infinity -> + true; + MaxAge -> + case read_refreshed_at(ProcID, Opts) of + undefined -> false; + RefreshedAt -> clock(Opts) =< RefreshedAt + MaxAge + end + end. + +%% @doc Read the timestamp of the last refresh of a process. +read_refreshed_at(ProcID, Opts) -> + Store = hb_opts:get(store, no_viable_store, Opts), + case hb_store:read(Store, refreshed_path(ProcID), Opts) of + {ok, Timestamp} -> hb_util:int(Timestamp); + _ -> undefined + end. + +%% @doc Calculate the effective maximum age of a process cache entry. +effective_max_age(Req, Opts) -> + case lib_process:only_if_cached(Req, Opts) of + true -> + infinity; + false -> + case max_age_from_request(Req, Opts) of + {ok, MaxAge} -> + normalize_max_age(MaxAge); + error -> + normalize_max_age( + hb_opts:get(process_now_max_age, infinity, Opts) + ) + end + end. + +max_age_from_request(Req, Opts) when is_map(Req) -> + hb_maps:find(<<"max-age">>, Req, Opts); +max_age_from_request(_Req, _Opts) -> + error. + +normalize_max_age(infinity) -> infinity; +normalize_max_age(<<"infinity">>) -> infinity; +normalize_max_age(RawMaxAge) -> hb_util:int(RawMaxAge). + +%% @doc Return the current clock time. Allows the option to override the clock +%% time with a custom value for test use. +clock(Opts) -> + case hb_opts:get(process_clock, undefined, Opts) of + undefined -> erlang:system_time(second); + Time -> hb_util:int(Time) + end. + %% @doc Calculate the path of a result, given a process ID and a slot. path(ProcID, Ref, Opts) -> path(ProcID, Ref, [], Opts). @@ -65,16 +135,7 @@ latest(ProcID, Opts) -> latest(ProcID, [], Opts). latest(ProcID, RequiredPath, Opts) -> latest(ProcID, RequiredPath, undefined, Opts). latest(ProcID, RawRequiredPath, Limit, RawOpts) -> - Scope = hb_opts:get(process_cache_scope, local, RawOpts), - % Normalize the store descriptor to a list of stores. - UnscopedStore = - case hb_opts:get(store, no_viable_store, RawOpts) of - StoreMsg when is_map(StoreMsg) -> [StoreMsg]; - Other -> Other - end, - % Apply the scope to the store and update the options message. - ScopedStore = hb_store:scope(UnscopedStore, Scope), - Opts = RawOpts#{ <<"store">> => ScopedStore }, + Opts = lib_process:scoped_opts(RawOpts), % Convert the required path to a list of _binary_ keys. RequiredPath = case RawRequiredPath of @@ -126,43 +187,60 @@ latest(ProcID, RawRequiredPath, Limit, RawOpts) -> end. %% @doc Find the latest assignment with the requested path suffix. -first_with_path(ProcID, RequiredPath, Slots, Opts) -> - first_with_path( - ProcID, - RequiredPath, - Slots, - Opts, - hb_opts:get(store, no_viable_store, Opts) - ). -first_with_path(_ProcID, _Required, [], _Opts, _Store) -> +first_with_path(_ProcID, _Required, [], _Opts) -> not_found; -first_with_path(ProcID, RequiredPath, [Slot | Rest], Opts, Store) -> - RawPath = path(ProcID, Slot, RequiredPath, Opts), +first_with_path(ProcID, RequiredPath, [Slot | Rest], Opts) -> + RawPath = path(ProcID, Slot, Opts), ?event({trying_slot, {slot, Slot}, {path, RawPath}}), - case hb_store:read(Store, RawPath, Opts) of + case hb_cache:read(RawPath, Opts) of {error, not_found} -> - first_with_path(ProcID, RequiredPath, Rest, Opts, Store); + first_with_path(ProcID, RequiredPath, Rest, Opts); {failure, _} = Failure -> Failure; {error, _} = Error -> Error; - _ -> - Slot + {ok, Msg} -> + case path_exists(RequiredPath, hb_cache:ensure_all_loaded(Msg, Opts)) of + true -> Slot; + false -> first_with_path(ProcID, RequiredPath, Rest, Opts) + end end. +path_exists([], _Msg) -> + true; +path_exists([Key | Rest], Msg) when is_map(Msg) -> + case maps:find(Key, Msg) of + {ok, Next} -> path_exists(Rest, Next); + error -> false + end; +path_exists(_Path, _Msg) -> + false. + %%% Tests process_cache_suite_test_() -> hb_store:generate_test_suite( [ - {"write and read process outputs", fun test_write_and_read_output/1}, - {"find latest output (with path)", fun find_latest_outputs/1} + { + "write and read process outputs", + fun(Store) -> + test_write_and_read_output(#{ <<"store">> => [Store] }) + end + }, + { + "find latest output (with path)", + fun(Store) -> + find_latest_outputs(#{ <<"store">> => [Store] }) + end + }, + { + "honor max-age when checking process cache freshness", + fun(Store) -> + freshness_max_age(#{ <<"store">> => [Store] }) + end + } ], - [ - {Name, Opts} - || - {Name, Opts} <- hb_store:test_stores() - ] + hb_store:test_stores() ). %% @doc Test for writing multiple computed outputs, then getting them by @@ -196,20 +274,20 @@ find_latest_outputs(Opts) -> Store = hb_opts:get(store, no_viable_store, Opts), ResetRes = hb_store:reset(Store), ?event({reset_store, {result, ResetRes}, {store, Store}}), - Proc1 = hb_process_test_vectors:aos_process(), - ProcID = hb_util:human_id(hb_ao:get(id, Proc1, Opts)), + ProcID = hb_util:human_id(crypto:strong_rand_bytes(32)), + ProcessRef = <<"test-process-ref">>, % Create messages for the slots, with only the middle slot having a % `/Process' field, while the top slot has a `/Deep/Process' field. Msg0 = #{ <<"Results">> => #{ <<"Result-Number">> => 0 } }, Base = #{ <<"Results">> => #{ <<"Result-Number">> => 1 }, - <<"Process">> => Proc1 + <<"Process">> => ProcessRef }, Req = #{ <<"Results">> => #{ <<"Result-Number">> => 2 }, - <<"Deep">> => #{ <<"Process">> => Proc1 } + <<"Deep">> => #{ <<"Process">> => ProcessRef } }, % Write the messages to the cache. {ok, _} = write(ProcID, 0, Msg0, Opts), @@ -217,16 +295,141 @@ find_latest_outputs(Opts) -> {ok, _} = write(ProcID, 2, Req, Opts), ?event(wrote_items), % Read the messages with various qualifiers. - {ok, 2, ReadReq} = latest(ProcID, Opts), + {ok, 2, RawReadReq} = latest(ProcID, Opts), + ReadReq = hb_cache:ensure_all_loaded(RawReadReq, Opts), ?event({read_latest, ReadReq}), - ?assert(hb_message:match(Req, ReadReq)), + ?assertEqual(2, maps:get(<<"Result-Number">>, maps:get(<<"Results">>, ReadReq))), + ?assertEqual(ProcessRef, maps:get(<<"Process">>, maps:get(<<"Deep">>, ReadReq))), ?event(read_latest_slot_without_qualifiers), - {ok, 1, ReadBaseRequired} = latest(ProcID, <<"Process">>, Opts), + {ok, 1, RawReadBaseRequired} = latest(ProcID, <<"Process">>, Opts), + ReadBaseRequired = hb_cache:ensure_all_loaded(RawReadBaseRequired, Opts), ?event({read_latest_with_process, ReadBaseRequired}), - ?assert(hb_message:match(Base, ReadBaseRequired)), + ?assertEqual( + 1, + maps:get(<<"Result-Number">>, maps:get(<<"Results">>, ReadBaseRequired)) + ), + ?assertEqual(ProcessRef, maps:get(<<"Process">>, ReadBaseRequired)), ?event(read_latest_slot_with_shallow_key), - {ok, 2, ReadReqRequired} = latest(ProcID, <<"Deep/Process">>, Opts), - ?assert(hb_message:match(Req, ReadReqRequired)), + {ok, 2, RawReadReqRequired} = latest(ProcID, <<"Deep/Process">>, Opts), + ReadReqRequired = hb_cache:ensure_all_loaded(RawReadReqRequired, Opts), + ?assertEqual( + 2, + maps:get(<<"Result-Number">>, maps:get(<<"Results">>, ReadReqRequired)) + ), + ?assertEqual(ProcessRef, maps:get(<<"Process">>, maps:get(<<"Deep">>, ReadReqRequired))), ?event(read_latest_slot_with_deep_key), - {ok, 1, ReadBase} = latest(ProcID, [], 1, Opts), - ?assert(hb_message:match(Base, ReadBase)). + {ok, 1, RawReadBase} = latest(ProcID, [], 1, Opts), + ReadBase = hb_cache:ensure_all_loaded(RawReadBase, Opts), + ?assertEqual(1, maps:get(<<"Result-Number">>, maps:get(<<"Results">>, ReadBase))), + ?assertEqual(ProcessRef, maps:get(<<"Process">>, ReadBase)). + +%% @doc Test for serving `/now' from cache only while the cache is fresh enough. +freshness_max_age(Opts) -> + ProcID = hb_util:human_id(crypto:strong_rand_bytes(32)), + Slot = 1, + SlotResult = #{ + <<"device">> => <<"process@1.0">>, + <<"at-slot">> => Slot, + <<"results">> => #{ <<"number">> => 1 } + }, + % Assert that the process is not fresh by default. + ?assertEqual( + false, + fresh( + ProcID, + #{ <<"max-age">> => 60 }, + Opts#{ <<"process-clock">> => 100 } + ) + ), + % Write the slot result to the cache at clock time 100. + {ok, _} = write(ProcID, Slot, SlotResult, Opts#{ <<"process-clock">> => 100 }), + {ok, 1, RawReadSlotResult} = latest(ProcID, Opts), + ReadSlotResult = hb_cache:ensure_all_loaded(RawReadSlotResult, Opts), + ?assertEqual(<<"process@1.0">>, maps:get(<<"device">>, ReadSlotResult)), + ?assertEqual(1, maps:get(<<"number">>, maps:get(<<"results">>, ReadSlotResult))), + % Assert that the process is fresh exactly at the max-age. + ?assertEqual( + true, + fresh( + ProcID, + #{ <<"max-age">> => 60 }, + Opts#{ <<"process-clock">> => 160 } + ) + ), + % Assert that the process is not fresh after the max-age. + ?assertEqual( + false, + fresh( + ProcID, + #{ <<"max-age">> => 60 }, + Opts#{ <<"process-clock">> => 161 } + ) + ), + % Assert that the process is fresh if the max-age is infinity. + ?assertEqual( + true, + fresh( + ProcID, + #{ <<"max-age">> => <<"infinity">> }, + Opts#{ <<"process-clock">> => 1000 } + ) + ), + % Assert that the process is fresh if the only-if-cached flag is set. + ?assertEqual( + true, + fresh( + ProcID, + #{ + <<"cache-control">> => [<<"only-if-cached">>], + <<"max-age">> => 0 + }, + Opts#{ <<"process-clock">> => 1000 } + ) + ), + % Assert that the max age is read as a fallback from the node opts + ?assertEqual( + true, + fresh( + ProcID, + #{}, + Opts#{ + <<"process-clock">> => 160, + <<"process-now-max-age">> => 60 + } + ) + ), + ?assertEqual( + false, + fresh( + ProcID, + #{}, + Opts#{ + <<"process-clock">> => 161, + <<"process-now-max-age">> => 60 + } + ) + ). + +%% @doc Process cache writes go only to `process-store' when configured. +isolated_process_store_test() -> + MainStore = hb_test_utils:test_store(hb_store_fs, <<"process-main">>), + ProcessStore = hb_test_utils:test_store(hb_store_fs, <<"process-isolated">>), + Opts = #{ + <<"store">> => [MainStore], + <<"process-store">> => [ProcessStore] + }, + hb_store:start(MainStore), + hb_store:start(ProcessStore), + ProcID = hb_util:human_id(crypto:strong_rand_bytes(32)), + Msg = #{ <<"results">> => #{ <<"ok">> => <<"stored">> } }, + {ok, Path} = write(ProcID, 1, Msg, Opts), + MainOpts = #{ <<"store">> => [MainStore] }, + ?assertMatch({error, not_found}, hb_cache:read(Path, MainOpts)), + ?assertMatch({ok, _}, read(ProcID, 1, Opts)), + ?assertMatch({ok, 1, _}, latest(ProcID, Opts)), + hb_store:reset(MainStore), + ?assertMatch({ok, _}, read(ProcID, 1, Opts)), + ?assertMatch({ok, 1, _}, latest(ProcID, Opts)), + hb_store:reset(ProcessStore), + ?assertMatch({error, not_found}, read(ProcID, 1, Opts)), + ?assertMatch({error, not_found}, latest(ProcID, Opts)). diff --git a/src/preloaded/process/dev_scheduler.erl b/src/preloaded/process/dev_scheduler.erl index fe8dff675..0333080b9 100644 --- a/src/preloaded/process/dev_scheduler.erl +++ b/src/preloaded/process/dev_scheduler.erl @@ -15,7 +15,7 @@ %%% -module(dev_scheduler). --device_libraries([lib_process]). +-device_libraries([lib_process, lib_scheduler]). %%% AO-Core API functions: -export([info/0]). %%% Local scheduling functions: @@ -30,8 +30,6 @@ -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). -%%% The maximum number of assignments that we will query/return at a time. --define(MAX_ASSIGNMENT_QUERY_LEN, 1000). %%% The timeout for a lookahead worker. -define(LOOKAHEAD_TIMEOUT, 1500). @@ -84,14 +82,7 @@ next(Base, Req, Opts) -> ?event(next, started_next), ?event(next_profiling, started_next), Schedule = message_cached_assignments(Base, Opts), - LastProcessed = - hb_util:int( - hb_ao:get( - <<"at-slot">>, - Base, - Opts#{ <<"hashpath">> => ignore } - ) - ), + LastProcessed = lib_scheduler:at_slot(Base, Opts), ?event(next_profiling, got_last_processed), ?event(debug_next, {in_message_cache, {schedule, Schedule}}), ?event(next, {last_processed, LastProcessed, {message_cache, length(Schedule)}}), @@ -111,12 +102,7 @@ next(Base, Req, Opts) -> ?event(next_profiling, got_no_assignments), {error, Reason}; {ok, [], _} -> - {error, #{ - <<"status">> => 404, - <<"reason">> => - <<"Requested slot not yet available in schedule.">> - } - }; + lib_scheduler:slot_unavailable(); {ok, Assignments, Lookahead} -> ?event(next_profiling, got_assignments), validate_next_slot(Base, Assignments, Lookahead, LastProcessed, Opts) @@ -375,21 +361,9 @@ schedule(Base, Req, Opts) -> %% for this scheduler. If so, it schedules the message and returns the assignment. post_schedule(Base, Req, Opts) -> ?event(scheduling_message), - % Find the target message to schedule: - RawToSched = find_message_to_schedule(Base, Req, Opts), - % If the message can not be properly loaded, this will throw an error - % before scheduling the message. - try hb_cache:ensure_all_loaded(RawToSched, Opts) of - ToSched -> - do_post_schedule(Base, Req, ToSched, Opts) - catch - error:{necessary_message_not_found, _, _} -> - {error, - #{ - <<"status">> => 404, - <<"body">> => <<"Cannot fully load message to schedule.">> - } - } + maybe + {ok, ToSched} ?= lib_scheduler:load_message_to_schedule(Base, Req, Opts), + do_post_schedule(Base, Req, ToSched, Opts) end. do_post_schedule(Base, Req, ToSched, Opts) -> @@ -397,53 +371,44 @@ do_post_schedule(Base, Req, ToSched, Opts) -> % Find the ProcessID of the target message: % - If it is a Process, use the ID of the message. % - If not, use the target as the ProcessID. - ProcID = find_target_id(Base, Req, ToSched, Opts), + ProcID = lib_scheduler:find_target_id(Base, Req, ToSched, Opts), ?event({proc_id, ProcID}), % Filter all unsigned keys from the source message. - case hb_message:with_only_committed(ToSched, Opts) of - {ok, OnlyCommitted} -> - ?event( - {post_schedule, - {schedule_id, ProcID}, - {message, ToSched} - } - ), - % Find the relevant scheduler server for the given process and - % message, start a new one if necessary, or return a redirect to the - % correct remote scheduler. - case find_server(ProcID, Base, ToSched, Opts) of - {local, PID} -> - ?event({scheduling_locally, {proc_id, ProcID}, {pid, PID}}), - post_local_schedule(ProcID, PID, OnlyCommitted, Opts); - {redirect, Redirect} -> - ?event({process_is_remote, {redirect, Redirect}}), - case hb_opts:get(scheduler_follow_redirects, true, Opts) of - true -> - ?event({proxying_to_remote_scheduler, - {redirect, Redirect}, - {msg, OnlyCommitted} - }), - post_remote_schedule( - ProcID, - Redirect, - OnlyCommitted, - Opts - ); - false -> {ok, Redirect} - end; - {error, Error} -> - ?event({error_finding_scheduler, {error, Error}}), - {error, Error} - end; - {error, Err} -> - {error, - #{ - <<"status">> => 400, - <<"body">> => <<"Message invalid: ", - "Committed components cannot be validated.">>, - <<"reason">> => Err - } + maybe + {ok, OnlyCommitted} ?= lib_scheduler:only_committed(ToSched, Opts), + ?event( + {post_schedule, + {schedule_id, ProcID}, + {message, ToSched} } + ), + % Find the relevant scheduler server for the given process and + % message, start a new one if necessary, or return a redirect to the + % correct remote scheduler. + case find_server(ProcID, Base, ToSched, Opts) of + {local, PID} -> + ?event({scheduling_locally, {proc_id, ProcID}, {pid, PID}}), + post_local_schedule(ProcID, PID, OnlyCommitted, Opts); + {redirect, Redirect} -> + ?event({process_is_remote, {redirect, Redirect}}), + case hb_opts:get(scheduler_follow_redirects, true, Opts) of + true -> + ?event({proxying_to_remote_scheduler, + {redirect, Redirect}, + {msg, OnlyCommitted} + }), + post_remote_schedule( + ProcID, + Redirect, + OnlyCommitted, + Opts + ); + false -> {ok, Redirect} + end; + {error, Error} -> + ?event({error_finding_scheduler, {error, Error}}), + {error, Error} + end end. %% @doc Post schedule the message. `Req' by this point has been refined to only @@ -717,7 +682,7 @@ find_remote_scheduler(ProcID, Scheduler, Opts) -> %% @doc Returns information about the current slot for a process. slot(M1, M2, Opts) -> ?event({getting_current_slot, {msg, M1}}), - ProcID = find_target_id(M1, M2, Opts), + ProcID = lib_scheduler:find_target_id(M1, M2, Opts), case find_server(ProcID, M1, Opts) of {local, PID} -> ?event({getting_current_slot, {proc_id, ProcID}}), @@ -813,18 +778,8 @@ remote_slot(<<"ao.TN.1">>, ProcID, Node, Opts) -> %% two slots -- labelled as `from' and `to'. If the schedule is not local, %% we redirect to the remote scheduler or proxy based on the node opts. get_schedule(Base, Req, Opts) -> - ProcID = hb_util:human_id(find_target_id(Base, Req, Opts)), - From = - case hb_ao:get(<<"from">>, Req, not_found, Opts) of - not_found -> 0; - X when X < 0 -> 0; - FromRes -> hb_util:int(FromRes) - end, - To = - case hb_ao:get(<<"to">>, Req, not_found, Opts) of - not_found -> undefined; - ToRes -> hb_util:int(ToRes) - end, + ProcID = hb_util:human_id(lib_scheduler:find_target_id(Base, Req, Opts)), + {From, To} = lib_scheduler:parse_slot_range(Req, Opts), Format = hb_ao:get(<<"accept">>, Req, <<"application/http">>, Opts), ?event( {parsed_get_schedule, @@ -963,7 +918,7 @@ do_get_remote_schedule(ProcID, LocalAssignments, From, To, Redirect, Opts) -> << ProcID/binary, "?process-id=", ProcID/binary, FromBin/binary, ToParam/binary, - "&limit=", (hb_util:bin(?MAX_ASSIGNMENT_QUERY_LEN))/binary + "&limit=", (hb_util:bin(lib_scheduler:max_assignment_query_len()))/binary >> end, ?event({getting_remote_schedule, {node, {string, Node}}, {path, {string, Path}}}), @@ -1295,80 +1250,6 @@ post_legacy_schedule(ProcID, OnlyCommitted, Node, Opts) -> %%% Private methods -%% @doc Find the schedule ID from a given request. The precidence order for -%% search is as follows: -%% 1. `ToSched/id' when `ToSched' has `type: Process' -%% 2. `ToSched/target' when `ToSched' has a `target' key -%% 2. `Req/target' -%% 3. `Req/id' when `Req' has `type: Process' -%% 4. `Base/process/id' -%% 5. `Base/id' when `Base' has `type: Process' -%% 6. `Req/id' -find_target_id(Base, Req, ToSched, Opts) -> - case hb_ao:get(<<"type">>, ToSched, not_found, Opts) of - <<"Process">> -> - lib_process:process_id(ToSched, #{}, Opts); - _ -> - case hb_ao:get(<<"target">>, ToSched, not_found, Opts) of - not_found -> find_target_id(Base, Req, Opts); - Target -> hb_util:human_id(Target) - end - end. -find_target_id(Base, Req, Opts) -> - TempOpts = Opts#{ <<"hashpath">> => ignore }, - Res = case hb_ao:resolve(Req, <<"target">>, TempOpts) of - {ok, Target} -> - % ID found at Req/target - Target; - _ -> - case hb_ao:resolve(Req, <<"type">>, TempOpts) of - {ok, <<"Process">>} -> - % Req is a Process, so the ID is at Req/id - lib_process:process_id(Req, #{}, Opts); - _ -> - case hb_ao:resolve(Base, <<"process">>, TempOpts) of - {ok, _Process} -> - lib_process:process_id(Base, #{}, Opts); - _ -> - % Does the message have a type of process? - case hb_ao:get(<<"type">>, Base, TempOpts) of - <<"Process">> -> - % Yes: Base is the process. - lib_process:process_id(Base, #{}, Opts); - _ -> - % No: Req is the target process. - lib_process:process_id(Req, #{}, Opts) - end - end - end - end, - ?event({found_id, {id, Res}, {base, Base}, {req, Req}}), - Res. - -%% @doc Search the given base and request message pair to find the message to -%% schedule. The precidence order for search is as follows: -%% 1. A key in `Req' with the value `self', indicating that the entire message -%% is the subject. -%% 2. A key in `Req' with another value, present in that message. -%% 3. The body of the message. -%% 4. The message itself. -find_message_to_schedule(Base, Req, Opts) -> - Subject = - hb_ao:get( - <<"subject">>, - Req, - not_found, - Opts#{ <<"hashpath">> => ignore } - ), - case Subject of - <<"base">> -> Base; - <<"self">> -> Req; - not_found -> - hb_ao:get(<<"body">>, Req, Req, Opts#{ <<"hashpath">> => ignore }); - Subject -> - hb_ao:get(Subject, Req, Opts#{ <<"hashpath">> => ignore }) - end. - %% @doc Generate a `GET /schedule' response for a process. generate_local_schedule(Format, ProcID, From, To, Opts) -> ?event( @@ -1405,35 +1286,8 @@ get_local_assignments(ProcID, From, undefined, Opts) -> end; get_local_assignments(ProcID, From, RequestedTo, Opts) -> ?event({handling_req_to_get_assignments, ProcID, {from, From}, {to, RequestedTo}}), - ComputedTo = - case (RequestedTo - From) > ?MAX_ASSIGNMENT_QUERY_LEN of - true -> From + ?MAX_ASSIGNMENT_QUERY_LEN; - false -> RequestedTo - end, - { - read_local_assignments(ProcID, From, ComputedTo, Opts), - ComputedTo < RequestedTo - }. - -%% @doc Get the assignments for a process. -read_local_assignments(_ProcID, From, To, _Opts) when From > To -> - []; -read_local_assignments(ProcID, CurrentSlot, To, Opts) -> - case dev_scheduler_cache:read(ProcID, CurrentSlot, Opts) of - not_found -> - % No assignment found in cache. - []; - {ok, Assignment} -> - [ - Assignment - | read_local_assignments( - ProcID, - CurrentSlot + 1, - To, - Opts - ) - ] - end. + lib_scheduler:read_assignment_range( + dev_scheduler_cache, ProcID, From, RequestedTo, Opts). %% @doc Returns the current state of the scheduler. checkpoint(State) -> {ok, State}. diff --git a/src/preloaded/process/dev_scheduler_cache.erl b/src/preloaded/process/dev_scheduler_cache.erl index 4ec202c77..43481f4b6 100644 --- a/src/preloaded/process/dev_scheduler_cache.erl +++ b/src/preloaded/process/dev_scheduler_cache.erl @@ -10,52 +10,11 @@ %% @doc Merge the scheduler store with the main store. Used before writing %% to the cache. -opts(Opts) -> - Opts#{ - <<"store">> => - hb_opts:get( - scheduler_store, - hb_opts:get(store, no_viable_store, Opts), - Opts - ) - }. +opts(Opts) -> lib_scheduler:cache_opts(Opts). %% @doc Write an assignment message into the cache. -write(RawAssignment, RawOpts) -> - Assignment = hb_cache:ensure_all_loaded(RawAssignment, RawOpts), - Opts = opts(RawOpts), - Store = hb_opts:get(store, no_viable_store, Opts), - % Write the message into the main cache - ProcID = hb_ao:get(<<"process">>, Assignment, Opts), - Slot = hb_ao:get(<<"slot">>, Assignment, Opts), - ?event( - {writing_assignment, - {proc_id, ProcID}, - {slot, Slot}, - {assignment, Assignment} - } - ), - case hb_cache:write(Assignment, Opts) of - {ok, _UnsignedID} -> - % Create symlinks from the message on the process and the - % slot on the process to the underlying data. - ok = hb_store:link( - Store, - #{ - hb_path:to_binary([ - ?SCHEDULER_CACHE_PREFIX, - <<"assignments">>, - hb_util:human_id(ProcID), - hb_ao:normalize_key(Slot) - ]) => hb_message:id(Assignment, signed, Opts) - }, - Opts - ), - ok; - {error, Reason} -> - ?event(error, {failed_to_write_assignment, {reason, Reason}}), - {error, Reason} - end. +write(Assignment, Opts) -> + lib_scheduler:write_assignment(?SCHEDULER_CACHE_PREFIX, Assignment, Opts). %% @doc Write the initial assignment message to the cache. write_spawn(RawInitMessage, Opts) -> @@ -63,53 +22,8 @@ write_spawn(RawInitMessage, Opts) -> hb_cache:write(InitMessage, opts(Opts)). %% @doc Get an assignment message from the cache. -read(ProcID, Slot, Opts) when is_integer(Slot) -> - read(ProcID, hb_util:bin(Slot), Opts); -read(ProcID, Slot, RawOpts) -> - Opts = opts(RawOpts), - Store = hb_opts:get(store, no_viable_store, Opts), - P1 = hb_path:to_binary([ - ?SCHEDULER_CACHE_PREFIX, - <<"assignments">>, - hb_util:human_id(ProcID), - Slot - ]), - ?event( - {read_assignment, - {proc_id, ProcID}, - {slot, Slot}, - {store, Store} - } - ), - case hb_store:resolve(Store, P1, Opts) of - {ok, ResolvedPath} -> - ?event({resolved_path, {p1, P1}, {p2, ResolvedPath}, {resolved, ResolvedPath}}), - case hb_cache:read(ResolvedPath, Opts) of - {ok, RawAssignment} -> - % `hb_cache:read' no longer normalizes commitments; the - % scheduler relies on each assignment carrying its unsigned - % commitment ID, so we restore it here. - Assignment = - hb_message:normalize_commitments(RawAssignment, Opts), - % If the slot key is not present, the format of the assignment is - % AOS2, so we need to convert it to the canonical format. - case hb_ao:get(<<"variant">>, Assignment, Opts) of - <<"ao.TN.1">> -> - Loaded = hb_cache:ensure_all_loaded(Assignment, Opts), - Norm = dev_scheduler_formats:aos2_to_assignment(Loaded, Opts), - ?event({normalized_aos2_assignment, Norm}), - {ok, Norm}; - <<"ao.N.1">> -> - {ok, hb_cache:ensure_all_loaded(Assignment, Opts)} - end; - {error, not_found} -> - ?event(debug_sched, {read_assignment, {res, not_found}}), - not_found - end; - {error, not_found} -> - ?event(debug_sched, {read_assignment, {res, not_found}}), - not_found - end. +read(ProcID, Slot, Opts) -> + lib_scheduler:read_assignment(?SCHEDULER_CACHE_PREFIX, ProcID, Slot, Opts). %% @doc Get the assignments for a process. list(ProcID, RawOpts) -> diff --git a/src/preloaded/process/dev_scheduler_formats.erl b/src/preloaded/process/dev_scheduler_formats.erl index 915114f87..b12633075 100644 --- a/src/preloaded/process/dev_scheduler_formats.erl +++ b/src/preloaded/process/dev_scheduler_formats.erl @@ -238,9 +238,4 @@ aos2_normalize_types(Msg) -> %% @doc For all scheduler format operations, we do not calculate hashpaths, %% perform cache lookups, or await inprogress results. -format_opts(Opts) -> - Opts#{ - <<"hashpath">> => ignore, - <<"cache-control">> => [<<"no-cache">>, <<"no-store">>], - <<"await-inprogress">> => false - }. +format_opts(Opts) -> lib_scheduler:format_opts(Opts). diff --git a/src/preloaded/process/dev_scheduler_server.erl b/src/preloaded/process/dev_scheduler_server.erl index 3a4f4ea95..45b94438b 100644 --- a/src/preloaded/process/dev_scheduler_server.erl +++ b/src/preloaded/process/dev_scheduler_server.erl @@ -212,27 +212,23 @@ do_assign(State, Message, ReplyPID) -> BaseStateHashpath = base_state(State), NextSlot = maps:get(current, State) + 1, {Timestamp, Height, Hash} = ar_timestamp:get(), + BaseAssignment = + lib_scheduler:base_assignment( + hb_util:id(maps:get(id, State)), + NextSlot, + Message, + Opts + ), Assignment = commit_assignment( - #{ - <<"path">> => - case hb_path:from_message(request, Message, Opts) of - undefined -> <<"compute">>; - Path -> hb_path:to_binary(Path) - end, - <<"data-protocol">> => <<"ao">>, - <<"variant">> => <<"ao.N.1">>, - <<"process">> => hb_util:id(maps:get(id, State)), - <<"epoch">> => <<"0">>, - <<"slot">> => NextSlot, + BaseAssignment#{ <<"block-height">> => Height, <<"block-hash">> => hb_util:human_id(Hash), <<"block-timestamp">> => Timestamp, % Note: Local time on the SU, not Arweave <<"timestamp">> => scheduler_time(), <<"base-hashpath">> => BaseStateHashpath, - <<"body">> => OnlyAttested, - <<"type">> => <<"Assignment">> + <<"body">> => OnlyAttested }, State ), diff --git a/src/preloaded/process/lib_process.erl b/src/preloaded/process/lib_process.erl index c86c1f6f8..270a7a218 100644 --- a/src/preloaded/process/lib_process.erl +++ b/src/preloaded/process/lib_process.erl @@ -6,6 +6,11 @@ as_process/2, run_as/4, process_id/3, + cache_opts/1, + cache_control/2, + normalize_cache_control/1, + only_if_cached/2, + scoped_opts/1, set_results/3, ensure_process_key/2, default_device/3 @@ -34,6 +39,56 @@ process_id(Base, Req, Opts) -> end end. +%% @doc Merge the process store with the main store. Used before reading +%% from or writing to a process cache. +cache_opts(Opts) -> + Opts#{ + <<"store">> => + hb_opts:get( + process_store, + hb_opts:get(store, no_viable_store, Opts), + Opts + ) + }. + +%% @doc Apply the process cache store and configured store scope to opts. +scoped_opts(RawOpts) -> + CacheOpts = cache_opts(RawOpts), + Scope = hb_opts:get(process_cache_scope, local, CacheOpts), + % Normalize the selected store descriptor to a list of stores. + UnscopedStore = + case hb_opts:get(store, no_viable_store, CacheOpts) of + StoreMsg when is_map(StoreMsg) -> [StoreMsg]; + Other -> Other + end, + % Apply the scope to the process store and update the options message. + CacheOpts#{ <<"store">> => hb_store:scope(UnscopedStore, Scope) }. + +%% @doc Return the effective cache-control directives from a request, falling +%% back to the node options when the request does not specify them. +cache_control(Req, Opts) when is_map(Req) -> + normalize_cache_control( + hb_maps:get( + <<"cache-control">>, + Req, + hb_opts:get(cache_control, [], Opts), + Opts + ) + ); +cache_control(_Req, Opts) -> + normalize_cache_control(hb_opts:get(cache_control, [], Opts)). + +%% @doc Normalize cache-control values to the binary directive form used by +%% process devices. +normalize_cache_control(CC) when is_list(CC) -> + lists:map(fun hb_ao:normalize_key/1, CC); +normalize_cache_control(CC) -> + normalize_cache_control([CC]). + +%% @doc Return whether the request or node opts require cached results only. +only_if_cached(Req, Opts) -> + lists:member(<<"only-if-cached">>, cache_control(Req, Opts)). + %% @doc Run a message against Base, with the device being swapped out for %% the device found at `Key'. After execution, the device is swapped back %% to the original device if the device is the same as we left it. diff --git a/src/preloaded/process/lib_scheduler.erl b/src/preloaded/process/lib_scheduler.erl new file mode 100644 index 000000000..ad0fa4b83 --- /dev/null +++ b/src/preloaded/process/lib_scheduler.erl @@ -0,0 +1,322 @@ +%%% @doc A library of functions shared by the scheduler devices +%%% (`~scheduler@1.0' and `~arweave-scheduler@1.0'). Both present the same +%%% consumer interface, so the logic for interpreting a scheduling request -- +%%% locating the target process and the message to schedule -- is identical +%%% regardless of how the schedule is sourced. Reading a contiguous range of +%%% assignments from a scheduler cache is likewise shared, parameterized by +%%% the cache module. +-module(lib_scheduler). +-include("include/hb.hrl"). +-export([find_target_id/4, find_target_id/3, at_slot/2]). +-export([find_message_to_schedule/3, load_message_to_schedule/3]). +-export([only_committed/2, base_assignment/4, slot_unavailable/0]). +-export([parse_slot_range/2, read_assignment_range/5, read_local_assignments/5]). +-export([cache_opts/1, write_assignment/3, read_assignment/4, format_opts/1]). +-export([max_assignment_query_len/0]). + +%%% The maximum number of assignments that a schedule request returns at a +%%% time. +-define(MAX_ASSIGNMENT_QUERY_LEN, 1000). + +%% @doc The maximum number of assignments that a schedule request returns at a +%% time, for callers that need the limit itself (for example, to bound a +%% remote schedule request). +max_assignment_query_len() -> ?MAX_ASSIGNMENT_QUERY_LEN. + +%% @doc Find the schedule ID from a given request. The precedence order for +%% search is as follows: +%% 1. `ToSched/id' when `ToSched' has `type: Process' +%% 2. `ToSched/target' when `ToSched' has a `target' key +%% 3. `Req/target' +%% 4. `Req/id' when `Req' has `type: Process' +%% 5. `Base/process/id' +%% 6. `Base/id' when `Base' has `type: Process' +%% 7. `Req/id' +find_target_id(Base, Req, ToSched, Opts) -> + case hb_ao:get(<<"type">>, ToSched, not_found, Opts) of + <<"Process">> -> + lib_process:process_id(ToSched, #{}, Opts); + _ -> + case hb_ao:get(<<"target">>, ToSched, not_found, Opts) of + not_found -> find_target_id(Base, Req, Opts); + Target -> hb_util:human_id(Target) + end + end. +find_target_id(Base, Req, Opts) -> + TempOpts = Opts#{ <<"hashpath">> => ignore }, + Res = case hb_ao:resolve(Req, <<"target">>, TempOpts) of + {ok, Target} -> + % ID found at Req/target + Target; + _ -> + case hb_ao:resolve(Req, <<"type">>, TempOpts) of + {ok, <<"Process">>} -> + % Req is a Process, so the ID is at Req/id + lib_process:process_id(Req, #{}, Opts); + _ -> + case hb_ao:resolve(Base, <<"process">>, TempOpts) of + {ok, _Process} -> + lib_process:process_id(Base, #{}, Opts); + _ -> + % Does the message have a type of process? + case hb_ao:get(<<"type">>, Base, TempOpts) of + <<"Process">> -> + % Yes: Base is the process. + lib_process:process_id(Base, #{}, Opts); + _ -> + % No: Req is the target process. + lib_process:process_id(Req, #{}, Opts) + end + end + end + end, + ?event({found_id, {id, Res}, {base, Base}, {req, Req}}), + Res. + +%% @doc Search the given base and request message pair to find the message to +%% schedule. The precedence order for search is as follows: +%% 1. A key in `Req' with the value `self', indicating that the entire message +%% is the subject. +%% 2. A key in `Req' with another value, present in that message. +%% 3. The body of the message. +%% 4. The message itself. +find_message_to_schedule(Base, Req, Opts) -> + Subject = + hb_ao:get( + <<"subject">>, + Req, + not_found, + Opts#{ <<"hashpath">> => ignore } + ), + case Subject of + <<"base">> -> Base; + <<"self">> -> Req; + not_found -> + hb_ao:get(<<"body">>, Req, Req, Opts#{ <<"hashpath">> => ignore }); + Subject -> + hb_ao:get(Subject, Req, Opts#{ <<"hashpath">> => ignore }) + end. + +%% @doc The slot that the given (process-shaped) base message has been +%% computed to. +at_slot(Base, Opts) -> + hb_util:int( + hb_ao:get( + <<"at-slot">>, + Base, + Opts#{ <<"hashpath">> => ignore } + ) + ). + +%% @doc Reduce a message to its committed components, returning a 400 error +%% message if they cannot be validated. +only_committed(Msg, Opts) -> + case hb_message:with_only_committed(Msg, Opts) of + {ok, OnlyCommitted} -> {ok, OnlyCommitted}; + {error, Err} -> + {error, + #{ + <<"status">> => 400, + <<"body">> => <<"Message invalid: ", + "Committed components cannot be validated.">>, + <<"reason">> => Err + } + } + end. + +%% @doc The keys of an assignment that are shared by every scheduler device: +%% everything except the fields that describe the assignment's position in the +%% scheduler's specific sequencing space (block info, weave offset, et al), +%% which the caller merges in. The `path' is taken from `PathMsg' -- normally +%% the assigned message itself. +base_assignment(ProcID, Slot, PathMsg, Opts) -> + #{ + <<"path">> => + case hb_path:from_message(request, PathMsg, Opts) of + undefined -> <<"compute">>; + Path -> hb_path:to_binary(Path) + end, + <<"data-protocol">> => <<"ao">>, + <<"variant">> => <<"ao.N.1">>, + <<"process">> => ProcID, + <<"epoch">> => <<"0">>, + <<"slot">> => Slot, + <<"body">> => PathMsg, + <<"type">> => <<"Assignment">> + }. + +%% @doc The error returned when a requested slot is not yet present in the +%% schedule. +slot_unavailable() -> + {error, + #{ + <<"status">> => 404, + <<"reason">> => + <<"Requested slot not yet available in schedule.">> + } + }. + +%% @doc Find and fully load the message to schedule from the given base and +%% request pair. If the message cannot be completely loaded, a 404 error +%% message is returned instead. +load_message_to_schedule(Base, Req, Opts) -> + RawToSched = find_message_to_schedule(Base, Req, Opts), + try {ok, hb_cache:ensure_all_loaded(RawToSched, Opts)} + catch + error:{necessary_message_not_found, _, _} -> + {error, + #{ + <<"status">> => 404, + <<"body">> => <<"Cannot fully load message to schedule.">> + } + } + end. + +%% @doc Parse the requested slot range -- `from' and `to' -- from a schedule +%% request. `from' defaults to 0 and is clamped to be non-negative; `to' +%% defaults to `undefined' (the latest slot known to the caller). +parse_slot_range(Req, Opts) -> + From = + case hb_ao:get(<<"from">>, Req, not_found, Opts) of + not_found -> 0; + FromRes -> max(0, hb_util:int(FromRes)) + end, + To = + case hb_ao:get(<<"to">>, Req, not_found, Opts) of + not_found -> undefined; + ToRes -> hb_util:int(ToRes) + end, + {From, To}. + +%% @doc Read the assignments for slots `From'..`RequestedTo' from a scheduler +%% cache, truncating the range to at most `?MAX_ASSIGNMENT_QUERY_LEN' slots. +%% Returns the assignments and whether the request was truncated. +read_assignment_range(CacheMod, ProcID, From, RequestedTo, Opts) -> + ComputedTo = min(RequestedTo, From + ?MAX_ASSIGNMENT_QUERY_LEN), + { + read_local_assignments(CacheMod, ProcID, From, ComputedTo, Opts), + ComputedTo < RequestedTo + }. + +%% @doc Read a contiguous range of assignments (slots `From'..`To', inclusive) +%% from a scheduler cache, stopping at the first slot that is not present. The +%% cache module is passed in so both scheduler devices can share the traversal: +%% `CacheMod:read(ProcID, Slot, Opts)' must return `{ok, Assignment}' or +%% `not_found'. +read_local_assignments(_CacheMod, _ProcID, From, To, _Opts) when From > To -> + []; +read_local_assignments(CacheMod, ProcID, From, To, Opts) -> + case CacheMod:read(ProcID, From, Opts) of + not_found -> []; + {ok, Assignment} -> + [ + Assignment + | read_local_assignments(CacheMod, ProcID, From + 1, To, Opts) + ] + end. + +%% @doc Options for generating (and reading messages for) schedule responses: +%% formatting is deterministic, so its resolutions are neither cached nor +%% awaited. +format_opts(Opts) -> + Opts#{ + <<"hashpath">> => ignore, + <<"cache-control">> => [<<"no-cache">>, <<"no-store">>], + <<"await-inprogress">> => false + }. + +%% @doc Merge the scheduler store with the main store. Used before reading +%% from or writing to a scheduler cache. +cache_opts(Opts) -> + Opts#{ + <<"store">> => + hb_opts:get( + scheduler_store, + hb_opts:get(store, no_viable_store, Opts), + Opts + ) + }. + +%% @doc Write an assignment message into a scheduler cache: the message goes +%% into the main cache, and `/assignments//' is linked to +%% its signed ID. The pseudo-path prefix distinguishes the caches of the +%% scheduler devices that share this helper. +write_assignment(Prefix, RawAssignment, RawOpts) -> + Assignment = hb_cache:ensure_all_loaded(RawAssignment, RawOpts), + Opts = cache_opts(RawOpts), + Store = hb_opts:get(store, no_viable_store, Opts), + ProcID = hb_ao:get(<<"process">>, Assignment, Opts), + Slot = hb_ao:get(<<"slot">>, Assignment, Opts), + ?event( + {writing_assignment, + {prefix, Prefix}, + {proc_id, ProcID}, + {slot, Slot} + } + ), + case hb_cache:write(Assignment, Opts) of + {ok, _UnsignedID} -> + ok = hb_store:link( + Store, + #{ + assignment_path(Prefix, ProcID, Slot) => + hb_message:id(Assignment, signed, Opts) + }, + Opts + ), + ok; + {error, Reason} -> + ?event(error, {failed_to_write_assignment, {reason, Reason}}), + {error, Reason} + end. + +%% @doc Get an assignment message from a scheduler cache. Restores the +%% assignment's unsigned commitment ID (`hb_cache:read' does not normalize +%% commitments) and converts legacy `ao.TN.1' (AOS2) assignments to the +%% canonical format via the scheduler package's formats module. +read_assignment(Prefix, ProcID, Slot, Opts) when is_integer(Slot) -> + read_assignment(Prefix, ProcID, hb_util:bin(Slot), Opts); +read_assignment(Prefix, ProcID, Slot, RawOpts) -> + Opts = cache_opts(RawOpts), + Store = hb_opts:get(store, no_viable_store, Opts), + Path = assignment_path(Prefix, ProcID, Slot), + ?event( + {read_assignment, + {prefix, Prefix}, + {proc_id, ProcID}, + {slot, Slot} + } + ), + case hb_store:resolve(Store, Path, Opts) of + {ok, ResolvedPath} -> + case hb_cache:read(ResolvedPath, Opts) of + {ok, RawAssignment} -> + Assignment = + hb_message:normalize_commitments(RawAssignment, Opts), + case hb_ao:get(<<"variant">>, Assignment, Opts) of + <<"ao.TN.1">> -> + Loaded = + hb_cache:ensure_all_loaded(Assignment, Opts), + {ok, + dev_scheduler_formats:aos2_to_assignment( + Loaded, + Opts + ) + }; + <<"ao.N.1">> -> + {ok, hb_cache:ensure_all_loaded(Assignment, Opts)} + end; + {error, not_found} -> not_found + end; + {error, not_found} -> not_found + end. + +assignment_path(Prefix, ProcID, Slot) -> + hb_path:to_binary( + [ + Prefix, + <<"assignments">>, + hb_util:human_id(ProcID), + hb_ao:normalize_key(Slot) + ] + ). diff --git a/src/preloaded/query/dev_copycat_arweave.erl b/src/preloaded/query/dev_copycat_arweave.erl index 0b06b0666..66532b5df 100644 --- a/src/preloaded/query/dev_copycat_arweave.erl +++ b/src/preloaded/query/dev_copycat_arweave.erl @@ -35,11 +35,12 @@ arweave(_Base, Request, Opts) -> end; {error, Mode} -> {error, <<"Unsupported mode `", (hb_util:bin(Mode))/binary, - "`. Supported modes are: shallow, deep, full, list">>} + "`. Supported modes are: headers, shallow, deep, full, list">>} end. request_mode(Request, Opts) -> case hb_maps:get(<<"mode">>, Request, <<"shallow">>, Opts) of + <<"headers">> -> {ok, headers}; <<"shallow">> -> {ok, shallow}; <<"deep">> -> {ok, deep}; <<"full">> -> {ok, full}; @@ -340,9 +341,20 @@ block_indexed_path(Height) -> write_block_index(Height, IndexMode, Opts) -> #{ <<"index-store">> := Store } = hb_store_arweave:store_from_opts(Opts), + Path = block_indexed_path(Height), + Mode = + case hb_store:read(Store, Path, Opts) of + {ok, Existing} -> + case mode_rank(Existing) >= mode_rank(IndexMode) of + true -> Existing; + false -> mode_name(IndexMode) + end; + _ -> + mode_name(IndexMode) + end, hb_store:write( Store, - #{ block_indexed_path(Height) => mode_name(IndexMode) }, + #{ Path => Mode }, Opts ). @@ -359,18 +371,44 @@ is_block_indexed(Height, IndexMode, Opts) -> end end. +mode_name(headers) -> <<"headers">>; mode_name(shallow) -> <<"shallow">>; mode_name(deep) -> <<"deep">>; mode_name(full) -> <<"full">>. +mode_rank(headers) -> 0; mode_rank(shallow) -> 1; mode_rank(deep) -> 2; mode_rank(full) -> 3; +mode_rank(<<"headers">>) -> mode_rank(headers); mode_rank(<<"shallow">>) -> mode_rank(shallow); mode_rank(<<"deep">>) -> mode_rank(deep); mode_rank(<<"full">>) -> mode_rank(full); -mode_rank(_Other) -> 0. +mode_rank(_Other) -> -1. +%% @doc Cache only the data-free L1 transaction headers from the block. +maybe_index_ids(Block, headers, Opts) -> + TXIDs = hb_maps:get(<<"txs">>, Block, [], Opts), + case fetch_block_tx_layout(Block, Opts) of + error -> + {block_skipped, #{ + skipped_count => length(TXIDs), + total_txs => length(TXIDs) + }}; + {ok, Layout} when length(Layout) =:= length(TXIDs) -> + Results = parallel_map( + Layout, + fun(Entry) -> cache_data_free_header(Entry, Opts) end, + Opts + ), + {block_cached, + (sum_counters(Results))#{ total_txs => length(TXIDs) }}; + {ok, _} -> + {block_skipped, #{ + skipped_count => length(TXIDs), + total_txs => length(TXIDs) + }} + end; %% @doc Index the IDs of all transactions in the block if configured to do so. maybe_index_ids(Block, IndexMode, Opts) -> TXIDs = hb_maps:get(<<"txs">>, Block, [], Opts), @@ -590,6 +628,25 @@ skip_bundle(EncodedTXID, Reason) -> ), counters(0, 1, 1). +cache_data_free_header({_TXID, DataSize}, _Opts) + when is_integer(DataSize), DataSize > 0 -> + counters(0, 0, 0); +cache_data_free_header({TXID, DataSize}, Opts) -> + EncodedTXID = hb_util:encode(TXID), + case resolve_tx_header(EncodedTXID, Opts) of + {ok, #tx{ data_size = 0 } = TX} -> + try + {ok, _} = cache_item(TX, <<"tx@1.0">>, Opts), + counters(1, 0, 0) + catch + _:_ -> counters(0, 0, 1) + end; + {ok, #tx{}} when DataSize =:= unknown -> + counters(0, 0, 0); + _ -> + counters(0, 0, 1) + end. + index_full_bundle_bytes(BundleData, BundleStartOffset, IndexMode, Store, Opts) -> case ar_bundles:decode_bundle_header(BundleData) of invalid_bundle_header -> @@ -621,6 +678,8 @@ index_pending(IndexMode, Opts) -> Error end. +process_pending_tx(_TXID, headers, _Opts) -> + counters(0, 0, 0); process_pending_tx(TXID, IndexMode, Opts) -> case resolve_pending_tx_header(TXID, Opts) of {ok, TX} -> @@ -736,10 +795,7 @@ index_full_bundle_items( ok = case {IndexMode, ParseResult} of {full, {ok, _, Parsed}} -> - LocalOpts = hb_store:scope(Opts, local), - Msg = hb_message:convert( - Parsed, <<"structured@1.0">>, <<"ans104@1.0">>, LocalOpts), - {ok, _Path} = hb_cache:write(Msg, LocalOpts), + {ok, _Path} = cache_item(Parsed, <<"ans104@1.0">>, Opts), ok; _ -> ok end, @@ -800,12 +856,142 @@ download_bundle_header(EndOffset, Size, Opts) -> lib_arweave_common:bundle_header(EndOffset - Size, Size, Opts) end). +%% @doc Read transaction IDs and data sizes from one `/block2' response. +fetch_block_tx_layout(Block, Opts) -> + Height = hb_maps:get(<<"height">>, Block, 0, Opts), + BlockID = hb_maps:get(<<"indep_hash">>, Block, <<>>, Opts), + Res = observe_event(<<"block_headers">>, fun() -> + hb_http:request( + #{ + <<"path">> => + <<"/arweave/block2/hash/", BlockID/binary>>, + <<"method">> => <<"GET">>, + % `/block2' accepts one selection bit for each possible TX. + <<"body">> => + binary:copy( + <<255>>, + (length(hb_maps:get(<<"txs">>, Block, [], Opts)) + 7) + div 8 + ), + <<"route-by">> => Height + }, + Opts#{ + <<"cache-control">> => [<<"no-cache">>, <<"no-store">>], + <<"http-client">> => hackney + } + ) + end), + case lib_arweave_common:best_response(Res) of + {ok, #{ <<"body">> := Body }} -> + parse_block2_transactions(Body); + _ -> + error + end. + +%% @doc Decode the transaction section of an Arweave `/block2' response. +parse_block2_transactions( + << + _:48/binary, + PrevHashSize:8, _:PrevHashSize/binary, + TimestampSize:8, _:(TimestampSize * 8), + NonceSize:16, _:NonceSize/binary, + HeightSize:8, _:(HeightSize * 8), + DiffSize:16, _:(DiffSize * 8), + CumulativeDiffSize:16, _:(CumulativeDiffSize * 8), + LastRetargetSize:8, _:(LastRetargetSize * 8), + HashSize:8, _:HashSize/binary, + BlockSizeSize:16, _:(BlockSizeSize * 8), + WeaveSizeSize:16, _:(WeaveSizeSize * 8), + RewardAddrSize:8, _:RewardAddrSize/binary, + TXRootSize:8, _:TXRootSize/binary, + WalletListSize:8, _:WalletListSize/binary, + HashListMerkleSize:8, _:HashListMerkleSize/binary, + RewardPoolSize:8, _:(RewardPoolSize * 8), + PackingThresholdSize:8, _:(PackingThresholdSize * 8), + StrictChunkThresholdSize:8, _:(StrictChunkThresholdSize * 8), + RateDividendSize:8, _:(RateDividendSize * 8), + RateDivisorSize:8, _:(RateDivisorSize * 8), + ScheduledRateDividendSize:8, _:(ScheduledRateDividendSize * 8), + ScheduledRateDivisorSize:8, _:(ScheduledRateDivisorSize * 8), + PoAOptionSize:8, _:(PoAOptionSize * 8), + ChunkSize:24, _:ChunkSize/binary, + TXPathSize:24, _:TXPathSize/binary, + DataPathSize:24, _:DataPathSize/binary, + Rest/binary + >> +) -> + parse_block2_tags(Rest); +parse_block2_transactions(_Bin) -> + error. + +parse_block2_tags(<< Count:16, Rest/binary >>) when Count =< 2048 -> + parse_block2_tags(Count, Rest); +parse_block2_tags(_Bin) -> + error. + +parse_block2_tags(0, Rest) -> + parse_block2_txs(Rest); +parse_block2_tags(Count, << Size:16, _:Size/binary, Rest/binary >>) -> + parse_block2_tags(Count - 1, Rest); +parse_block2_tags(_Count, _Bin) -> + error. + +parse_block2_txs(<< Count:16, Rest/binary >>) when Count =< 1000 -> + parse_block2_txs(Count, Rest, []); +parse_block2_txs(_Bin) -> + error. + +parse_block2_txs(0, _Rest, TXs) -> + {ok, TXs}; +parse_block2_txs( + Count, << Size:24, TXBin:Size/binary, Rest/binary >>, + TXs) when Count > 0 -> + case parse_block2_tx(TXBin) of + {ok, TX} -> + parse_block2_txs(Count - 1, Rest, [TX | TXs]); + error -> + error + end; +parse_block2_txs(_Count, _Bin, _TXs) -> + error. + +parse_block2_tx(<< TXID:32/binary >>) -> + {ok, {TXID, unknown}}; +parse_block2_tx( + << + Format:8, TXID:32/binary, + AnchorSize:8, _:AnchorSize/binary, + OwnerSize:16, _:OwnerSize/binary, + TargetSize:8, _:TargetSize/binary, + QuantitySize:8, _:(QuantitySize * 8), + DataSizeSize:16, DataSize:(DataSizeSize * 8), + DataRootSize:8, _:DataRootSize/binary, + SignatureSize:16, _:SignatureSize/binary, + RewardSize:8, _:(RewardSize * 8), + DataEncodingSize:24, _:DataEncodingSize/binary, + _/binary + >> +) when Format =:= 1; Format =:= 2 -> + {ok, + { + TXID, + case Format of + 1 -> DataEncodingSize; + 2 -> DataSize + end + }}; +parse_block2_tx(_Bin) -> + error. + resolve_tx_headers(TXIDs, Opts) -> Results = parallel_map( TXIDs, fun(TXID) -> resolve_tx_header(TXID, Opts) end, Opts ), + collect_tx_headers(Results). + +collect_tx_headers(Results) -> case lists:any(fun(Res) -> Res =:= error end, Results) of true -> error; false -> @@ -876,12 +1062,8 @@ cache_tx_header(TX, Opts) -> end. write_tx_header(TX, Opts) -> - LocalOpts = hb_store:scope(Opts, local), try - Msg = - hb_message:convert( - TX, <<"structured@1.0">>, <<"tx@1.0">>, LocalOpts), - {ok, _} = hb_cache:write(Msg, LocalOpts), + {ok, _} = cache_item(TX, <<"tx@1.0">>, Opts), ok catch Class:Reason -> @@ -896,6 +1078,14 @@ write_tx_header(TX, Opts) -> ok end. +%% @doc Cache an item decoded during the block scan in the local store as a +%% structured message, keyed by the codec it is committed with. Its fields +%% (notably `target') then become locally matchable. +cache_item(Item, Codec, Opts) -> + LocalOpts = hb_store:scope(Opts, local), + Msg = hb_message:convert(Item, <<"structured@1.0">>, Codec, LocalOpts), + hb_cache:write(Msg, LocalOpts). + %% @doc Record event metrics (count and duration) using hb_event:record. record_event_metrics(MetricName, Count, Duration) -> hb_event:record(<<"arweave_block_count">>, MetricName, #{}, Count), @@ -910,6 +1100,51 @@ observe_event(MetricName, Fun) -> %%% Tests +headers_mode_test() -> + Block = 1967269, + BlockBin = hb_util:bin(Block), + ZeroTXID = <<"enIzMI_6vVcY80ZqfiCdWq56chbft3HSZnXMj7X7BdE">>, + DataTXID = <<"NEsEjfjjawZ_fqMEnIS9aURTIvgrZ5TqFwC9hcsE2nM">>, + TestStore = hb_test_utils:test_store(), + StoreOpts = #{ <<"index-store">> => [TestStore] }, + Opts = #{ + <<"store">> => [TestStore], + <<"arweave-index-store">> => StoreOpts, + <<"arweave-index-workers">> => 2 + }, + LocalOpts = hb_store:scope(Opts, local), + ?assertEqual({error, not_found}, hb_cache:read(ZeroTXID, LocalOpts)), + {ok, Block} = + hb_ao:resolve( + << + "~copycat@1.0/arweave&" + "from=", BlockBin/binary, "&" + "to=", BlockBin/binary, "&" + "mode=headers" + >>, + Opts + ), + ?assertMatch( + {ok, _}, + hb_cache:read( + <<"~arweave@2.9/block/height/", BlockBin/binary>>, + LocalOpts + ) + ), + ?assertMatch({ok, _}, hb_cache:read(ZeroTXID, LocalOpts)), + ?assertEqual({error, not_found}, hb_cache:read(DataTXID, LocalOpts)), + lists:foreach( + fun(TXID) -> + ?assertEqual( + not_found, + hb_store_arweave:read_offset(StoreOpts, TXID, Opts) + ) + end, + [ZeroTXID, DataTXID] + ), + ?assert(is_block_indexed(Block, headers, Opts)), + ?assertNot(is_block_indexed(Block, shallow, Opts)). + index_ids_test_parallel() -> %% Test block: https://viewblock.io/arweave/block/1827942 %% Note: this block includes a data item with an Ethereum signature. This diff --git a/src/preloaded/query/dev_query_arweave.erl b/src/preloaded/query/dev_query_arweave.erl index 5ffb05f3c..6b91e847d 100644 --- a/src/preloaded/query/dev_query_arweave.erl +++ b/src/preloaded/query/dev_query_arweave.erl @@ -87,6 +87,13 @@ query(Obj, <<"transactions">>, Args, Opts) -> ), {ok, connection([], Args, Opts)} end; +query(#{ <<"offset">> := Offset }, <<"block">>, _Args, Opts) + when is_integer(Offset) -> + % Resolve the `block' field of a transaction node: a cached transaction + % holds no block metadata of its own, so the block is found by the weave + % offset carried on the node (see `node_with_offset/3'), which also + % distinguishes this from the top-level `block(id/height)' query. + block_at_offset(Offset, Opts); query(Obj, <<"block">>, Args, Opts) -> case query(Obj, <<"blocks">>, Args, Opts) of {ok, []} -> {ok, null}; @@ -264,11 +271,21 @@ read_ids(_, 0, _Opts) -> []; read_ids([AnnotatedID = #{ <<"id">> := ID } | Rest], Count, Opts) -> case hb_cache:read(ID, Opts) of {ok, Msg} -> - [AnnotatedID#{ <<"node">> => Msg } | read_ids(Rest, Count - 1, Opts)]; + [ + AnnotatedID#{ <<"node">> => node_with_offset(Msg, AnnotatedID, Opts) } + | read_ids(Rest, Count - 1, Opts) + ]; _ -> read_ids(Rest, Count, Opts) end. +%% @doc Carry the transaction's weave offset onto its node, so that the `block' +%% field resolver can locate the block that includes it (see `query/4'). +node_with_offset(Msg, #{ <<"offset">> := Offset }, _Opts) + when is_map(Msg), is_integer(Offset) -> + Msg#{ <<"offset">> => Offset }; +node_with_offset(Msg, _Annotated, _Opts) -> Msg. + %% @doc Drop to the cursor position, returning the list of items after the cursor. drop_to_cursor(Args, Ordered, Opts) -> drop_to_cursor( @@ -380,14 +397,8 @@ block_range_to_offset_range(Heights, Opts) -> RawMin -> case read_block(hb_util:int(RawMin), Opts) of {ok, MinBlock} -> - % The `weave_size` is the size at the _end_ of the block, - % so we must subtract the start from it to find the - % starting byte of the block. - WeaveSize = hb_util:int( - hb_maps:get(<<"weave_size">>, MinBlock, 0, Opts)), - BlockSize = hb_util:int( - hb_maps:get(<<"block_size">>, MinBlock, 0, Opts)), - WeaveSize - BlockSize; + {BlockStart, _} = block_bounds(MinBlock, Opts), + BlockStart; {error, not_found} -> 0 end end, @@ -456,18 +467,47 @@ read_cached_block(Height, Opts) -> %% @doc Return the latest block height indexed in the Arweave pseudo-path cache. latest_cached_block(Opts) -> - Blocks = - hb_cache:list_numbered( - hb_path:to_binary([ - <<"~arweave@2.9">>, - <<"block">>, - <<"height">> - ]), - Opts - ), - case Blocks of + case cached_heights(Opts) of [] -> not_found; - _ -> {ok, lists:max(Blocks)} + Blocks -> {ok, lists:max(Blocks)} + end. + +%% @doc List the block heights present in the Arweave pseudo-path cache. +cached_heights(Opts) -> + hb_cache:list_numbered( + hb_path:to_binary([ + <<"~arweave@2.9">>, + <<"block">>, + <<"height">> + ]), + Opts + ). + +%% @doc The weave byte range `{StartOffset, EndOffset}' that a block covers. +%% The block's `weave_size' is the size of the weave at the block's _end_, so +%% its starting byte is that size less the block's own size. +block_bounds(Block, Opts) -> + WeaveSize = hb_util:int(hb_maps:get(<<"weave_size">>, Block, 0, Opts)), + BlockSize = hb_util:int(hb_maps:get(<<"block_size">>, Block, 0, Opts)), + {WeaveSize - BlockSize, WeaveSize}. + +%% @doc Find the cached block that includes the given weave offset -- the block +%% whose byte range `[weave_size - block_size, weave_size)' contains it. Scans +%% the locally cached blocks, so it is only reached when the `block' field is +%% explicitly selected on a transaction. Returns `{ok, null}' when no cached +%% block covers the offset. +block_at_offset(Offset, Opts) -> + block_covering(Offset, lists:sort(cached_heights(Opts)), Opts). + +block_covering(_Offset, [], _Opts) -> {ok, null}; +block_covering(Offset, [Height | Rest], Opts) -> + case read_cached_block(Height, Opts) of + {ok, Block} -> + case block_bounds(Block, Opts) of + {Start, End} when Start =< Offset, Offset < End -> {ok, Block}; + _ -> block_covering(Offset, Rest, Opts) + end; + _ -> block_covering(Offset, Rest, Opts) end. %%% Match argument processing diff --git a/test/arweave-scheduler-test.lua b/test/arweave-scheduler-test.lua new file mode 100644 index 000000000..ec816ccb4 --- /dev/null +++ b/test/arweave-scheduler-test.lua @@ -0,0 +1,34 @@ +--- @module arweave-scheduler-test +--- State-transformation test module for the `~arweave-scheduler@1.0` device. +--- Messages scheduled as Arweave L1 transactions invoke these functions via +--- their `path` tag. The function names deliberately avoid the keys that +--- `~lua@5.3a` excludes from its default handler (`set`, `path`, ...) and +--- the keys present on the process message itself. + +--- Executed for assignments without a `path` tag -- most notably slot 0, +--- the process message itself. +function compute(base, req, opts) + return base +end + +--- Overwrite the process state with the `value` tag of the message. +function setstate(base, req, opts) + base.state = tonumber(req.body.value) + return base +end + +--- Add the `value` tag of the message to the process state. +function addstate(base, req, opts) + base.state = (base.state or 0) + tonumber(req.body.value) + return base +end + +--- Report the process state in the results of the slot. +function querystate(base, req, opts) + base.results = { + output = { + body = "state=" .. tostring(base.state) + } + } + return base +end