From 654af74456d1c4305964d2fa5ac202237f4d3d36 Mon Sep 17 00:00:00 2001 From: Rani Elhusseini Date: Mon, 27 Jul 2026 20:17:42 +0200 Subject: [PATCH 01/27] feat: support underscore name subdomains --- src/preloaded/arweave/dev_manifest.erl | 35 +++++++++-- src/preloaded/name/dev_name.erl | 86 ++++++++++++++++++++++---- 2 files changed, 106 insertions(+), 15 deletions(-) diff --git a/src/preloaded/arweave/dev_manifest.erl b/src/preloaded/arweave/dev_manifest.erl index 8e339a6041..3a22246fee 100644 --- a/src/preloaded/arweave/dev_manifest.erl +++ b/src/preloaded/arweave/dev_manifest.erl @@ -102,10 +102,7 @@ request(Base, Req, Opts) -> % message replaced with the casted manifest. case {Rest, maybe_cast_manifest(Loaded, Opts)} of {_, ignored} -> - ?event( - debug_manifest, - {non_manifest_returning_loaded, {loaded, Loaded}, {rest, Rest}}), - {ok, Req#{ <<"body">> => [Loaded|Rest] }}; + maybe_cast_resolved_path(Loaded, Rest, Req, Opts); {[], {ok, Casted}} -> ?event(debug_manifest, {manifest_returning_index, {req, Req}}), {ok, Req#{ <<"body">> => [Casted, #{<<"path">> => <<"index">>}] }}; @@ -129,6 +126,36 @@ request(Base, Req, Opts) -> {ok, Req} end. +maybe_cast_resolved_path(Loaded, [Next | Rest], Req, Opts) -> + case catch hb_ao:resolve(Loaded, Next, Opts) of + {ok, Resolved} -> + case catch load(Resolved, Opts) of + {ok, NextLoaded} -> + case maybe_cast_manifest(NextLoaded, Opts) of + {ok, Casted} -> manifest_path(Casted, Rest, Req); + ignored -> non_manifest_path(Loaded, [Next | Rest], Req) + end; + _ -> non_manifest_path(Loaded, [Next | Rest], Req) + end; + _ -> non_manifest_path(Loaded, [Next | Rest], Req) + end; +maybe_cast_resolved_path(Loaded, [], Req, _Opts) -> + non_manifest_path(Loaded, [], Req). + +manifest_path(Casted, [], Req) -> + ?event(debug_manifest, {manifest_returning_index, {req, Req}}), + {ok, Req#{ <<"body">> => [Casted, #{<<"path">> => <<"index">>}] }}; +manifest_path(Casted, Rest, Req) -> + ?event(debug_manifest, {manifest_returning_subpath, {req, Req}}), + {ok, Req#{ <<"body">> => [Casted | Rest] }}. + +non_manifest_path(Loaded, Rest, Req) -> + ?event( + debug_manifest, + {non_manifest_returning_loaded, {loaded, Loaded}, {rest, Rest}} + ), + {ok, Req#{ <<"body">> => [Loaded | Rest] }}. + %% @doc Cast a message to `manifest@1.0` if it has the correct content-type but %% no other device is specified. load(Msg, _Opts) when is_map(Msg) -> {ok, Msg}; diff --git a/src/preloaded/name/dev_name.erl b/src/preloaded/name/dev_name.erl index 04aa4689cc..4b6a5e2bf4 100644 --- a/src/preloaded/name/dev_name.erl +++ b/src/preloaded/name/dev_name.erl @@ -83,21 +83,24 @@ request(HookMsg, HookReq, Opts) -> maybe {ok, Req} ?= hb_maps:find(<<"request">>, HookReq, Opts), {ok, Host} ?= hb_maps:find(<<"host">>, Req, Opts), - {ok, Name} ?= + {ok, RawName} ?= name_from_host( Host, hb_opts:get(node_host, hb_opts:get(host, no_host, Opts), Opts) ), + {Name, NamePath} = name_path(RawName), {ok, ResolvedMsg} ?= resolve(Name, HookMsg, HookReq, Opts), ModReq = - maybe_append_named_message( + append_name_path( ResolvedMsg, + NamePath, hb_util:ok(hb_maps:find(<<"body">>, HookReq, Opts)), Opts ), ?event( {request_with_prepended_path, {name, Name}, + {name_path, NamePath}, {full_host, Host}, {resolved_msg, ResolvedMsg}, {to_execute, ModReq} @@ -121,6 +124,19 @@ request(HookMsg, HookReq, Opts) -> end end. +%% @doc Split an underscore-encoded subdomain into root name and path parts. +%% `x_y_z' resolves `z', then `y', then `x'. +name_path(Name) -> + case lists:reverse(binary:split(Name, <<"_">>, [global, trim_all])) of + [Root | Path] -> {Root, Path}; + [] -> {Name, []} + end. + +%% @doc Append the resolved name as the base, then any path encoded in the host. +append_name_path(ResolvedMsg, NamePath, OldReq, Opts) -> + [Base | Rest] = maybe_append_named_message(ResolvedMsg, OldReq, Opts), + [Base | ([#{ <<"path">> => Part } || Part <- NamePath] ++ Rest)]. + %% @doc After finding a hit for a named message, we should ensure that it is the %% base message for the evaluation. If it is already present in the request, %% however, we should not add it twice. Instead, we must add the version that @@ -209,6 +225,19 @@ device_resolver(Msg) -> } }. +host_test_opts() -> + #{ + <<"port">> => 0, + <<"name-resolvers">> => + [ + device_resolver( + #{ <<"permabytes">> => #{ <<"content-type">> => <<"text/html">> } } + ) + ], + <<"on">> => + #{ <<"request">> => #{ <<"device">> => <<"name@1.0">> } } + }. + single_resolver_test_parallel() -> ?assertEqual( {ok, <<"world">>}, @@ -303,8 +332,8 @@ arns_json_snapshot_test_parallel() -> ) ). -arns_host_resolution_test_parallel() -> - Opts = hb_name_test_utils:arns_opts(), +host_resolution_test_parallel() -> + Opts = host_test_opts(), Node = hb_http_server:start_node(Opts), ?assertMatch( {ok, <<"text/html">>}, @@ -312,17 +341,14 @@ arns_host_resolution_test_parallel() -> Node, #{ <<"path">> => <<"content-type">>, - <<"host">> => <<"001_permabytes.localhost">> + <<"host">> => <<"permabytes.localhost">> }, Opts ) ). -arns_host_resolution_with_node_host_test_parallel() -> - Opts = (hb_name_test_utils:arns_opts())#{ - <<"node-host">> => <<"http://localhost">>, - <<"port">> => 0 - }, +host_resolution_with_node_host_test_parallel() -> + Opts = (host_test_opts())#{ <<"node-host">> => <<"http://localhost">> }, Node = hb_http_server:start_node(Opts), ?assertMatch( {ok, <<"text/html">>}, @@ -330,12 +356,45 @@ arns_host_resolution_with_node_host_test_parallel() -> Node, #{ <<"path">> => <<"content-type">>, - <<"host">> => <<"001_permabytes.localhost">> + <<"host">> => <<"permabytes.localhost">> }, Opts ) ). +underscore_host_parts_resolve_manifest_test_parallel() -> + ManifestID = <<"42jky7O3rzKkMOfHBXgK-304YjulzEYqHc9qyjT3efA">>, + Opts = + (hb_name_test_utils:manifest_opts())#{ + <<"port">> => 0, + <<"http-client-hackney-recv-timeout">> => 30_000, + <<"name-resolvers">> => + [ + device_resolver( + #{ + <<"sub2_sub1">> => <<"not-the-manifest">>, + <<"sub1">> => #{ <<"sub2">> => ManifestID } + } + ) + ], + <<"on">> => + #{ + <<"request">> => + [ + #{<<"device">> => <<"name@1.0">>}, + #{<<"device">> => <<"manifest@1.0">>} + ] + } + }, + Node = hb_http_server:start_node(Opts), + hb_test_utils:assert_manifest_response( + Node, + #{ <<"path">> => <<"/">>, <<"host">> => <<"sub2_sub1.localhost">> }, + <<"text/html">>, + <<"Portal">>, + Opts + ). + root_request_skips_name_resolution_test_parallel() -> BaseOpts = #{ @@ -371,4 +430,9 @@ name_from_host_test_parallel() -> ?assertEqual( {ok, <<"sub3.sub2">>}, name_from_host(<<"sub3.sub2.sub1.abc.xyz">>, <<"sub1.abc.xyz">>) + ), + ?assertEqual({<<"sub1">>, [<<"sub2">>]}, name_path(<<"sub2_sub1">>)), + ?assertEqual( + {<<"sub1">>, [<<"sub2">>, <<"sub3">>]}, + name_path(<<"sub3_sub2_sub1">>) ). From a093cc4ee2b0e36105e57444b8d556fd54f80c05 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Mon, 27 Jul 2026 17:46:58 -0400 Subject: [PATCH 02/27] wip: `index` route validation --- src/preloaded/name/dev_name.erl | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/preloaded/name/dev_name.erl b/src/preloaded/name/dev_name.erl index 4b6a5e2bf4..63ae81a129 100644 --- a/src/preloaded/name/dev_name.erl +++ b/src/preloaded/name/dev_name.erl @@ -363,7 +363,20 @@ host_resolution_with_node_host_test_parallel() -> ). underscore_host_parts_resolve_manifest_test_parallel() -> + Opts = hb_name_test_utils:manifest_opts(), ManifestID = <<"42jky7O3rzKkMOfHBXgK-304YjulzEYqHc9qyjT3efA">>, + {ok, IndexPageID} = + hb_cache:write( + #{ + <<"content-type">> => <<"text/html">>, + <<"body">> => <<"Index page.">> + } + ), + Subrealm = + #{ + <<"sub2">> => ManifestID + <<"index">> => IndexPageID + } Opts = (hb_name_test_utils:manifest_opts())#{ <<"port">> => 0, @@ -373,7 +386,7 @@ underscore_host_parts_resolve_manifest_test_parallel() -> device_resolver( #{ <<"sub2_sub1">> => <<"not-the-manifest">>, - <<"sub1">> => #{ <<"sub2">> => ManifestID } + <<"sub1">> => IndexPageID } ) ], From de4dfc3f984308df873a6f9f239e1864044b3a81 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Thu, 16 Jul 2026 14:14:00 -0400 Subject: [PATCH 03/27] refactor(copycat): share the local item-caching helper Caching an L1 transaction header (every mode) and caching a fully-parsed bundle item (full mode) used the same scope-convert-write sequence with different codecs. Share it as `cache_item/3'; behaviour is unchanged (the header path keeps its log-and-skip guard, the full-mode path still asserts the write). --- src/preloaded/query/dev_copycat_arweave.erl | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/preloaded/query/dev_copycat_arweave.erl b/src/preloaded/query/dev_copycat_arweave.erl index 0b06b06660..5b5b6675d8 100644 --- a/src/preloaded/query/dev_copycat_arweave.erl +++ b/src/preloaded/query/dev_copycat_arweave.erl @@ -736,10 +736,7 @@ index_full_bundle_items( ok = case {IndexMode, ParseResult} of {full, {ok, _, Parsed}} -> - LocalOpts = hb_store:scope(Opts, local), - Msg = hb_message:convert( - Parsed, <<"structured@1.0">>, <<"ans104@1.0">>, LocalOpts), - {ok, _Path} = hb_cache:write(Msg, LocalOpts), + {ok, _Path} = cache_item(Parsed, <<"ans104@1.0">>, Opts), ok; _ -> ok end, @@ -876,12 +873,8 @@ cache_tx_header(TX, Opts) -> end. write_tx_header(TX, Opts) -> - LocalOpts = hb_store:scope(Opts, local), try - Msg = - hb_message:convert( - TX, <<"structured@1.0">>, <<"tx@1.0">>, LocalOpts), - {ok, _} = hb_cache:write(Msg, LocalOpts), + {ok, _} = cache_item(TX, <<"tx@1.0">>, Opts), ok catch Class:Reason -> @@ -896,6 +889,14 @@ write_tx_header(TX, Opts) -> ok end. +%% @doc Cache an item decoded during the block scan in the local store as a +%% structured message, keyed by the codec it is committed with. Its fields +%% (notably `target') then become locally matchable. +cache_item(Item, Codec, Opts) -> + LocalOpts = hb_store:scope(Opts, local), + Msg = hb_message:convert(Item, <<"structured@1.0">>, Codec, LocalOpts), + hb_cache:write(Msg, LocalOpts). + %% @doc Record event metrics (count and duration) using hb_event:record. record_event_metrics(MetricName, Count, Duration) -> hb_event:record(<<"arweave_block_count">>, MetricName, #{}, Count), From b2c741f4858379d9b2f17c0692b0172bd39c7f13 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Thu, 16 Jul 2026 14:11:27 -0400 Subject: [PATCH 04/27] refactor(scheduler): extract the logic shared by scheduler devices into lib_scheduler `~scheduler@1.0' and the `~arweave-scheduler@1.0' device added later in this branch present the same consumer interface, so everything about interpreting a scheduling request -- and storing and serving its results -- is identical regardless of how the schedule is sourced. Move that shared logic into a new `lib_scheduler' device library: - `find_target_id/4,3' and `find_message_to_schedule/3' (request parsing), `load_message_to_schedule/3' (find + fully load, 404 on failure), and `only_committed/2' (reduce to committed components, 400 on failure). - `parse_slot_range/2' (the `from'/`to' slot parse; `from' is now clamped non-negative after coercion, which also catches a negative binary input) and `read_assignment_range/5' (bounded range read + truncation flag), with the single `?MAX_ASSIGNMENT_QUERY_LEN' definition and `max_assignment_query_len/0' for callers that bound remote requests by it. - The assignment cache: `cache_opts/1', `write_assignment/3' and `read_assignment/4', parameterized by each device's pseudo-path prefix. The read restores unsigned commitment IDs and converts legacy `ao.TN.1' assignments, exactly as `dev_scheduler_cache:read/3' did. - `base_assignment/4', the assignment keys every scheduler shares -- `dev_scheduler_server:do_assign' merges its block fields and commits; other schedulers merge their own position fields. - `at_slot/2', `slot_unavailable/0' and `format_opts/1' (the no-cache/no-await formatting options, previously inlined in several places). `dev_scheduler', `dev_scheduler_cache', `dev_scheduler_formats' and `dev_scheduler_server' delegate to the library; no behavioural change. --- src/preloaded/process/dev_scheduler.erl | 240 +++---------- src/preloaded/process/dev_scheduler_cache.erl | 96 +----- .../process/dev_scheduler_formats.erl | 7 +- .../process/dev_scheduler_server.erl | 22 +- src/preloaded/process/lib_scheduler.erl | 322 ++++++++++++++++++ 5 files changed, 384 insertions(+), 303 deletions(-) create mode 100644 src/preloaded/process/lib_scheduler.erl diff --git a/src/preloaded/process/dev_scheduler.erl b/src/preloaded/process/dev_scheduler.erl index fe8dff675b..0333080b9f 100644 --- a/src/preloaded/process/dev_scheduler.erl +++ b/src/preloaded/process/dev_scheduler.erl @@ -15,7 +15,7 @@ %%% -module(dev_scheduler). --device_libraries([lib_process]). +-device_libraries([lib_process, lib_scheduler]). %%% AO-Core API functions: -export([info/0]). %%% Local scheduling functions: @@ -30,8 +30,6 @@ -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). -%%% The maximum number of assignments that we will query/return at a time. --define(MAX_ASSIGNMENT_QUERY_LEN, 1000). %%% The timeout for a lookahead worker. -define(LOOKAHEAD_TIMEOUT, 1500). @@ -84,14 +82,7 @@ next(Base, Req, Opts) -> ?event(next, started_next), ?event(next_profiling, started_next), Schedule = message_cached_assignments(Base, Opts), - LastProcessed = - hb_util:int( - hb_ao:get( - <<"at-slot">>, - Base, - Opts#{ <<"hashpath">> => ignore } - ) - ), + LastProcessed = lib_scheduler:at_slot(Base, Opts), ?event(next_profiling, got_last_processed), ?event(debug_next, {in_message_cache, {schedule, Schedule}}), ?event(next, {last_processed, LastProcessed, {message_cache, length(Schedule)}}), @@ -111,12 +102,7 @@ next(Base, Req, Opts) -> ?event(next_profiling, got_no_assignments), {error, Reason}; {ok, [], _} -> - {error, #{ - <<"status">> => 404, - <<"reason">> => - <<"Requested slot not yet available in schedule.">> - } - }; + lib_scheduler:slot_unavailable(); {ok, Assignments, Lookahead} -> ?event(next_profiling, got_assignments), validate_next_slot(Base, Assignments, Lookahead, LastProcessed, Opts) @@ -375,21 +361,9 @@ schedule(Base, Req, Opts) -> %% for this scheduler. If so, it schedules the message and returns the assignment. post_schedule(Base, Req, Opts) -> ?event(scheduling_message), - % Find the target message to schedule: - RawToSched = find_message_to_schedule(Base, Req, Opts), - % If the message can not be properly loaded, this will throw an error - % before scheduling the message. - try hb_cache:ensure_all_loaded(RawToSched, Opts) of - ToSched -> - do_post_schedule(Base, Req, ToSched, Opts) - catch - error:{necessary_message_not_found, _, _} -> - {error, - #{ - <<"status">> => 404, - <<"body">> => <<"Cannot fully load message to schedule.">> - } - } + maybe + {ok, ToSched} ?= lib_scheduler:load_message_to_schedule(Base, Req, Opts), + do_post_schedule(Base, Req, ToSched, Opts) end. do_post_schedule(Base, Req, ToSched, Opts) -> @@ -397,53 +371,44 @@ do_post_schedule(Base, Req, ToSched, Opts) -> % Find the ProcessID of the target message: % - If it is a Process, use the ID of the message. % - If not, use the target as the ProcessID. - ProcID = find_target_id(Base, Req, ToSched, Opts), + ProcID = lib_scheduler:find_target_id(Base, Req, ToSched, Opts), ?event({proc_id, ProcID}), % Filter all unsigned keys from the source message. - case hb_message:with_only_committed(ToSched, Opts) of - {ok, OnlyCommitted} -> - ?event( - {post_schedule, - {schedule_id, ProcID}, - {message, ToSched} - } - ), - % Find the relevant scheduler server for the given process and - % message, start a new one if necessary, or return a redirect to the - % correct remote scheduler. - case find_server(ProcID, Base, ToSched, Opts) of - {local, PID} -> - ?event({scheduling_locally, {proc_id, ProcID}, {pid, PID}}), - post_local_schedule(ProcID, PID, OnlyCommitted, Opts); - {redirect, Redirect} -> - ?event({process_is_remote, {redirect, Redirect}}), - case hb_opts:get(scheduler_follow_redirects, true, Opts) of - true -> - ?event({proxying_to_remote_scheduler, - {redirect, Redirect}, - {msg, OnlyCommitted} - }), - post_remote_schedule( - ProcID, - Redirect, - OnlyCommitted, - Opts - ); - false -> {ok, Redirect} - end; - {error, Error} -> - ?event({error_finding_scheduler, {error, Error}}), - {error, Error} - end; - {error, Err} -> - {error, - #{ - <<"status">> => 400, - <<"body">> => <<"Message invalid: ", - "Committed components cannot be validated.">>, - <<"reason">> => Err - } + maybe + {ok, OnlyCommitted} ?= lib_scheduler:only_committed(ToSched, Opts), + ?event( + {post_schedule, + {schedule_id, ProcID}, + {message, ToSched} } + ), + % Find the relevant scheduler server for the given process and + % message, start a new one if necessary, or return a redirect to the + % correct remote scheduler. + case find_server(ProcID, Base, ToSched, Opts) of + {local, PID} -> + ?event({scheduling_locally, {proc_id, ProcID}, {pid, PID}}), + post_local_schedule(ProcID, PID, OnlyCommitted, Opts); + {redirect, Redirect} -> + ?event({process_is_remote, {redirect, Redirect}}), + case hb_opts:get(scheduler_follow_redirects, true, Opts) of + true -> + ?event({proxying_to_remote_scheduler, + {redirect, Redirect}, + {msg, OnlyCommitted} + }), + post_remote_schedule( + ProcID, + Redirect, + OnlyCommitted, + Opts + ); + false -> {ok, Redirect} + end; + {error, Error} -> + ?event({error_finding_scheduler, {error, Error}}), + {error, Error} + end end. %% @doc Post schedule the message. `Req' by this point has been refined to only @@ -717,7 +682,7 @@ find_remote_scheduler(ProcID, Scheduler, Opts) -> %% @doc Returns information about the current slot for a process. slot(M1, M2, Opts) -> ?event({getting_current_slot, {msg, M1}}), - ProcID = find_target_id(M1, M2, Opts), + ProcID = lib_scheduler:find_target_id(M1, M2, Opts), case find_server(ProcID, M1, Opts) of {local, PID} -> ?event({getting_current_slot, {proc_id, ProcID}}), @@ -813,18 +778,8 @@ remote_slot(<<"ao.TN.1">>, ProcID, Node, Opts) -> %% two slots -- labelled as `from' and `to'. If the schedule is not local, %% we redirect to the remote scheduler or proxy based on the node opts. get_schedule(Base, Req, Opts) -> - ProcID = hb_util:human_id(find_target_id(Base, Req, Opts)), - From = - case hb_ao:get(<<"from">>, Req, not_found, Opts) of - not_found -> 0; - X when X < 0 -> 0; - FromRes -> hb_util:int(FromRes) - end, - To = - case hb_ao:get(<<"to">>, Req, not_found, Opts) of - not_found -> undefined; - ToRes -> hb_util:int(ToRes) - end, + ProcID = hb_util:human_id(lib_scheduler:find_target_id(Base, Req, Opts)), + {From, To} = lib_scheduler:parse_slot_range(Req, Opts), Format = hb_ao:get(<<"accept">>, Req, <<"application/http">>, Opts), ?event( {parsed_get_schedule, @@ -963,7 +918,7 @@ do_get_remote_schedule(ProcID, LocalAssignments, From, To, Redirect, Opts) -> << ProcID/binary, "?process-id=", ProcID/binary, FromBin/binary, ToParam/binary, - "&limit=", (hb_util:bin(?MAX_ASSIGNMENT_QUERY_LEN))/binary + "&limit=", (hb_util:bin(lib_scheduler:max_assignment_query_len()))/binary >> end, ?event({getting_remote_schedule, {node, {string, Node}}, {path, {string, Path}}}), @@ -1295,80 +1250,6 @@ post_legacy_schedule(ProcID, OnlyCommitted, Node, Opts) -> %%% Private methods -%% @doc Find the schedule ID from a given request. The precidence order for -%% search is as follows: -%% 1. `ToSched/id' when `ToSched' has `type: Process' -%% 2. `ToSched/target' when `ToSched' has a `target' key -%% 2. `Req/target' -%% 3. `Req/id' when `Req' has `type: Process' -%% 4. `Base/process/id' -%% 5. `Base/id' when `Base' has `type: Process' -%% 6. `Req/id' -find_target_id(Base, Req, ToSched, Opts) -> - case hb_ao:get(<<"type">>, ToSched, not_found, Opts) of - <<"Process">> -> - lib_process:process_id(ToSched, #{}, Opts); - _ -> - case hb_ao:get(<<"target">>, ToSched, not_found, Opts) of - not_found -> find_target_id(Base, Req, Opts); - Target -> hb_util:human_id(Target) - end - end. -find_target_id(Base, Req, Opts) -> - TempOpts = Opts#{ <<"hashpath">> => ignore }, - Res = case hb_ao:resolve(Req, <<"target">>, TempOpts) of - {ok, Target} -> - % ID found at Req/target - Target; - _ -> - case hb_ao:resolve(Req, <<"type">>, TempOpts) of - {ok, <<"Process">>} -> - % Req is a Process, so the ID is at Req/id - lib_process:process_id(Req, #{}, Opts); - _ -> - case hb_ao:resolve(Base, <<"process">>, TempOpts) of - {ok, _Process} -> - lib_process:process_id(Base, #{}, Opts); - _ -> - % Does the message have a type of process? - case hb_ao:get(<<"type">>, Base, TempOpts) of - <<"Process">> -> - % Yes: Base is the process. - lib_process:process_id(Base, #{}, Opts); - _ -> - % No: Req is the target process. - lib_process:process_id(Req, #{}, Opts) - end - end - end - end, - ?event({found_id, {id, Res}, {base, Base}, {req, Req}}), - Res. - -%% @doc Search the given base and request message pair to find the message to -%% schedule. The precidence order for search is as follows: -%% 1. A key in `Req' with the value `self', indicating that the entire message -%% is the subject. -%% 2. A key in `Req' with another value, present in that message. -%% 3. The body of the message. -%% 4. The message itself. -find_message_to_schedule(Base, Req, Opts) -> - Subject = - hb_ao:get( - <<"subject">>, - Req, - not_found, - Opts#{ <<"hashpath">> => ignore } - ), - case Subject of - <<"base">> -> Base; - <<"self">> -> Req; - not_found -> - hb_ao:get(<<"body">>, Req, Req, Opts#{ <<"hashpath">> => ignore }); - Subject -> - hb_ao:get(Subject, Req, Opts#{ <<"hashpath">> => ignore }) - end. - %% @doc Generate a `GET /schedule' response for a process. generate_local_schedule(Format, ProcID, From, To, Opts) -> ?event( @@ -1405,35 +1286,8 @@ get_local_assignments(ProcID, From, undefined, Opts) -> end; get_local_assignments(ProcID, From, RequestedTo, Opts) -> ?event({handling_req_to_get_assignments, ProcID, {from, From}, {to, RequestedTo}}), - ComputedTo = - case (RequestedTo - From) > ?MAX_ASSIGNMENT_QUERY_LEN of - true -> From + ?MAX_ASSIGNMENT_QUERY_LEN; - false -> RequestedTo - end, - { - read_local_assignments(ProcID, From, ComputedTo, Opts), - ComputedTo < RequestedTo - }. - -%% @doc Get the assignments for a process. -read_local_assignments(_ProcID, From, To, _Opts) when From > To -> - []; -read_local_assignments(ProcID, CurrentSlot, To, Opts) -> - case dev_scheduler_cache:read(ProcID, CurrentSlot, Opts) of - not_found -> - % No assignment found in cache. - []; - {ok, Assignment} -> - [ - Assignment - | read_local_assignments( - ProcID, - CurrentSlot + 1, - To, - Opts - ) - ] - end. + lib_scheduler:read_assignment_range( + dev_scheduler_cache, ProcID, From, RequestedTo, Opts). %% @doc Returns the current state of the scheduler. checkpoint(State) -> {ok, State}. diff --git a/src/preloaded/process/dev_scheduler_cache.erl b/src/preloaded/process/dev_scheduler_cache.erl index 4ec202c771..43481f4b62 100644 --- a/src/preloaded/process/dev_scheduler_cache.erl +++ b/src/preloaded/process/dev_scheduler_cache.erl @@ -10,52 +10,11 @@ %% @doc Merge the scheduler store with the main store. Used before writing %% to the cache. -opts(Opts) -> - Opts#{ - <<"store">> => - hb_opts:get( - scheduler_store, - hb_opts:get(store, no_viable_store, Opts), - Opts - ) - }. +opts(Opts) -> lib_scheduler:cache_opts(Opts). %% @doc Write an assignment message into the cache. -write(RawAssignment, RawOpts) -> - Assignment = hb_cache:ensure_all_loaded(RawAssignment, RawOpts), - Opts = opts(RawOpts), - Store = hb_opts:get(store, no_viable_store, Opts), - % Write the message into the main cache - ProcID = hb_ao:get(<<"process">>, Assignment, Opts), - Slot = hb_ao:get(<<"slot">>, Assignment, Opts), - ?event( - {writing_assignment, - {proc_id, ProcID}, - {slot, Slot}, - {assignment, Assignment} - } - ), - case hb_cache:write(Assignment, Opts) of - {ok, _UnsignedID} -> - % Create symlinks from the message on the process and the - % slot on the process to the underlying data. - ok = hb_store:link( - Store, - #{ - hb_path:to_binary([ - ?SCHEDULER_CACHE_PREFIX, - <<"assignments">>, - hb_util:human_id(ProcID), - hb_ao:normalize_key(Slot) - ]) => hb_message:id(Assignment, signed, Opts) - }, - Opts - ), - ok; - {error, Reason} -> - ?event(error, {failed_to_write_assignment, {reason, Reason}}), - {error, Reason} - end. +write(Assignment, Opts) -> + lib_scheduler:write_assignment(?SCHEDULER_CACHE_PREFIX, Assignment, Opts). %% @doc Write the initial assignment message to the cache. write_spawn(RawInitMessage, Opts) -> @@ -63,53 +22,8 @@ write_spawn(RawInitMessage, Opts) -> hb_cache:write(InitMessage, opts(Opts)). %% @doc Get an assignment message from the cache. -read(ProcID, Slot, Opts) when is_integer(Slot) -> - read(ProcID, hb_util:bin(Slot), Opts); -read(ProcID, Slot, RawOpts) -> - Opts = opts(RawOpts), - Store = hb_opts:get(store, no_viable_store, Opts), - P1 = hb_path:to_binary([ - ?SCHEDULER_CACHE_PREFIX, - <<"assignments">>, - hb_util:human_id(ProcID), - Slot - ]), - ?event( - {read_assignment, - {proc_id, ProcID}, - {slot, Slot}, - {store, Store} - } - ), - case hb_store:resolve(Store, P1, Opts) of - {ok, ResolvedPath} -> - ?event({resolved_path, {p1, P1}, {p2, ResolvedPath}, {resolved, ResolvedPath}}), - case hb_cache:read(ResolvedPath, Opts) of - {ok, RawAssignment} -> - % `hb_cache:read' no longer normalizes commitments; the - % scheduler relies on each assignment carrying its unsigned - % commitment ID, so we restore it here. - Assignment = - hb_message:normalize_commitments(RawAssignment, Opts), - % If the slot key is not present, the format of the assignment is - % AOS2, so we need to convert it to the canonical format. - case hb_ao:get(<<"variant">>, Assignment, Opts) of - <<"ao.TN.1">> -> - Loaded = hb_cache:ensure_all_loaded(Assignment, Opts), - Norm = dev_scheduler_formats:aos2_to_assignment(Loaded, Opts), - ?event({normalized_aos2_assignment, Norm}), - {ok, Norm}; - <<"ao.N.1">> -> - {ok, hb_cache:ensure_all_loaded(Assignment, Opts)} - end; - {error, not_found} -> - ?event(debug_sched, {read_assignment, {res, not_found}}), - not_found - end; - {error, not_found} -> - ?event(debug_sched, {read_assignment, {res, not_found}}), - not_found - end. +read(ProcID, Slot, Opts) -> + lib_scheduler:read_assignment(?SCHEDULER_CACHE_PREFIX, ProcID, Slot, Opts). %% @doc Get the assignments for a process. list(ProcID, RawOpts) -> diff --git a/src/preloaded/process/dev_scheduler_formats.erl b/src/preloaded/process/dev_scheduler_formats.erl index 915114f87f..b126330751 100644 --- a/src/preloaded/process/dev_scheduler_formats.erl +++ b/src/preloaded/process/dev_scheduler_formats.erl @@ -238,9 +238,4 @@ aos2_normalize_types(Msg) -> %% @doc For all scheduler format operations, we do not calculate hashpaths, %% perform cache lookups, or await inprogress results. -format_opts(Opts) -> - Opts#{ - <<"hashpath">> => ignore, - <<"cache-control">> => [<<"no-cache">>, <<"no-store">>], - <<"await-inprogress">> => false - }. +format_opts(Opts) -> lib_scheduler:format_opts(Opts). diff --git a/src/preloaded/process/dev_scheduler_server.erl b/src/preloaded/process/dev_scheduler_server.erl index 3a4f4ea95c..45b94438b9 100644 --- a/src/preloaded/process/dev_scheduler_server.erl +++ b/src/preloaded/process/dev_scheduler_server.erl @@ -212,27 +212,23 @@ do_assign(State, Message, ReplyPID) -> BaseStateHashpath = base_state(State), NextSlot = maps:get(current, State) + 1, {Timestamp, Height, Hash} = ar_timestamp:get(), + BaseAssignment = + lib_scheduler:base_assignment( + hb_util:id(maps:get(id, State)), + NextSlot, + Message, + Opts + ), Assignment = commit_assignment( - #{ - <<"path">> => - case hb_path:from_message(request, Message, Opts) of - undefined -> <<"compute">>; - Path -> hb_path:to_binary(Path) - end, - <<"data-protocol">> => <<"ao">>, - <<"variant">> => <<"ao.N.1">>, - <<"process">> => hb_util:id(maps:get(id, State)), - <<"epoch">> => <<"0">>, - <<"slot">> => NextSlot, + BaseAssignment#{ <<"block-height">> => Height, <<"block-hash">> => hb_util:human_id(Hash), <<"block-timestamp">> => Timestamp, % Note: Local time on the SU, not Arweave <<"timestamp">> => scheduler_time(), <<"base-hashpath">> => BaseStateHashpath, - <<"body">> => OnlyAttested, - <<"type">> => <<"Assignment">> + <<"body">> => OnlyAttested }, State ), diff --git a/src/preloaded/process/lib_scheduler.erl b/src/preloaded/process/lib_scheduler.erl new file mode 100644 index 0000000000..ad0fa4b838 --- /dev/null +++ b/src/preloaded/process/lib_scheduler.erl @@ -0,0 +1,322 @@ +%%% @doc A library of functions shared by the scheduler devices +%%% (`~scheduler@1.0' and `~arweave-scheduler@1.0'). Both present the same +%%% consumer interface, so the logic for interpreting a scheduling request -- +%%% locating the target process and the message to schedule -- is identical +%%% regardless of how the schedule is sourced. Reading a contiguous range of +%%% assignments from a scheduler cache is likewise shared, parameterized by +%%% the cache module. +-module(lib_scheduler). +-include("include/hb.hrl"). +-export([find_target_id/4, find_target_id/3, at_slot/2]). +-export([find_message_to_schedule/3, load_message_to_schedule/3]). +-export([only_committed/2, base_assignment/4, slot_unavailable/0]). +-export([parse_slot_range/2, read_assignment_range/5, read_local_assignments/5]). +-export([cache_opts/1, write_assignment/3, read_assignment/4, format_opts/1]). +-export([max_assignment_query_len/0]). + +%%% The maximum number of assignments that a schedule request returns at a +%%% time. +-define(MAX_ASSIGNMENT_QUERY_LEN, 1000). + +%% @doc The maximum number of assignments that a schedule request returns at a +%% time, for callers that need the limit itself (for example, to bound a +%% remote schedule request). +max_assignment_query_len() -> ?MAX_ASSIGNMENT_QUERY_LEN. + +%% @doc Find the schedule ID from a given request. The precedence order for +%% search is as follows: +%% 1. `ToSched/id' when `ToSched' has `type: Process' +%% 2. `ToSched/target' when `ToSched' has a `target' key +%% 3. `Req/target' +%% 4. `Req/id' when `Req' has `type: Process' +%% 5. `Base/process/id' +%% 6. `Base/id' when `Base' has `type: Process' +%% 7. `Req/id' +find_target_id(Base, Req, ToSched, Opts) -> + case hb_ao:get(<<"type">>, ToSched, not_found, Opts) of + <<"Process">> -> + lib_process:process_id(ToSched, #{}, Opts); + _ -> + case hb_ao:get(<<"target">>, ToSched, not_found, Opts) of + not_found -> find_target_id(Base, Req, Opts); + Target -> hb_util:human_id(Target) + end + end. +find_target_id(Base, Req, Opts) -> + TempOpts = Opts#{ <<"hashpath">> => ignore }, + Res = case hb_ao:resolve(Req, <<"target">>, TempOpts) of + {ok, Target} -> + % ID found at Req/target + Target; + _ -> + case hb_ao:resolve(Req, <<"type">>, TempOpts) of + {ok, <<"Process">>} -> + % Req is a Process, so the ID is at Req/id + lib_process:process_id(Req, #{}, Opts); + _ -> + case hb_ao:resolve(Base, <<"process">>, TempOpts) of + {ok, _Process} -> + lib_process:process_id(Base, #{}, Opts); + _ -> + % Does the message have a type of process? + case hb_ao:get(<<"type">>, Base, TempOpts) of + <<"Process">> -> + % Yes: Base is the process. + lib_process:process_id(Base, #{}, Opts); + _ -> + % No: Req is the target process. + lib_process:process_id(Req, #{}, Opts) + end + end + end + end, + ?event({found_id, {id, Res}, {base, Base}, {req, Req}}), + Res. + +%% @doc Search the given base and request message pair to find the message to +%% schedule. The precedence order for search is as follows: +%% 1. A key in `Req' with the value `self', indicating that the entire message +%% is the subject. +%% 2. A key in `Req' with another value, present in that message. +%% 3. The body of the message. +%% 4. The message itself. +find_message_to_schedule(Base, Req, Opts) -> + Subject = + hb_ao:get( + <<"subject">>, + Req, + not_found, + Opts#{ <<"hashpath">> => ignore } + ), + case Subject of + <<"base">> -> Base; + <<"self">> -> Req; + not_found -> + hb_ao:get(<<"body">>, Req, Req, Opts#{ <<"hashpath">> => ignore }); + Subject -> + hb_ao:get(Subject, Req, Opts#{ <<"hashpath">> => ignore }) + end. + +%% @doc The slot that the given (process-shaped) base message has been +%% computed to. +at_slot(Base, Opts) -> + hb_util:int( + hb_ao:get( + <<"at-slot">>, + Base, + Opts#{ <<"hashpath">> => ignore } + ) + ). + +%% @doc Reduce a message to its committed components, returning a 400 error +%% message if they cannot be validated. +only_committed(Msg, Opts) -> + case hb_message:with_only_committed(Msg, Opts) of + {ok, OnlyCommitted} -> {ok, OnlyCommitted}; + {error, Err} -> + {error, + #{ + <<"status">> => 400, + <<"body">> => <<"Message invalid: ", + "Committed components cannot be validated.">>, + <<"reason">> => Err + } + } + end. + +%% @doc The keys of an assignment that are shared by every scheduler device: +%% everything except the fields that describe the assignment's position in the +%% scheduler's specific sequencing space (block info, weave offset, et al), +%% which the caller merges in. The `path' is taken from `PathMsg' -- normally +%% the assigned message itself. +base_assignment(ProcID, Slot, PathMsg, Opts) -> + #{ + <<"path">> => + case hb_path:from_message(request, PathMsg, Opts) of + undefined -> <<"compute">>; + Path -> hb_path:to_binary(Path) + end, + <<"data-protocol">> => <<"ao">>, + <<"variant">> => <<"ao.N.1">>, + <<"process">> => ProcID, + <<"epoch">> => <<"0">>, + <<"slot">> => Slot, + <<"body">> => PathMsg, + <<"type">> => <<"Assignment">> + }. + +%% @doc The error returned when a requested slot is not yet present in the +%% schedule. +slot_unavailable() -> + {error, + #{ + <<"status">> => 404, + <<"reason">> => + <<"Requested slot not yet available in schedule.">> + } + }. + +%% @doc Find and fully load the message to schedule from the given base and +%% request pair. If the message cannot be completely loaded, a 404 error +%% message is returned instead. +load_message_to_schedule(Base, Req, Opts) -> + RawToSched = find_message_to_schedule(Base, Req, Opts), + try {ok, hb_cache:ensure_all_loaded(RawToSched, Opts)} + catch + error:{necessary_message_not_found, _, _} -> + {error, + #{ + <<"status">> => 404, + <<"body">> => <<"Cannot fully load message to schedule.">> + } + } + end. + +%% @doc Parse the requested slot range -- `from' and `to' -- from a schedule +%% request. `from' defaults to 0 and is clamped to be non-negative; `to' +%% defaults to `undefined' (the latest slot known to the caller). +parse_slot_range(Req, Opts) -> + From = + case hb_ao:get(<<"from">>, Req, not_found, Opts) of + not_found -> 0; + FromRes -> max(0, hb_util:int(FromRes)) + end, + To = + case hb_ao:get(<<"to">>, Req, not_found, Opts) of + not_found -> undefined; + ToRes -> hb_util:int(ToRes) + end, + {From, To}. + +%% @doc Read the assignments for slots `From'..`RequestedTo' from a scheduler +%% cache, truncating the range to at most `?MAX_ASSIGNMENT_QUERY_LEN' slots. +%% Returns the assignments and whether the request was truncated. +read_assignment_range(CacheMod, ProcID, From, RequestedTo, Opts) -> + ComputedTo = min(RequestedTo, From + ?MAX_ASSIGNMENT_QUERY_LEN), + { + read_local_assignments(CacheMod, ProcID, From, ComputedTo, Opts), + ComputedTo < RequestedTo + }. + +%% @doc Read a contiguous range of assignments (slots `From'..`To', inclusive) +%% from a scheduler cache, stopping at the first slot that is not present. The +%% cache module is passed in so both scheduler devices can share the traversal: +%% `CacheMod:read(ProcID, Slot, Opts)' must return `{ok, Assignment}' or +%% `not_found'. +read_local_assignments(_CacheMod, _ProcID, From, To, _Opts) when From > To -> + []; +read_local_assignments(CacheMod, ProcID, From, To, Opts) -> + case CacheMod:read(ProcID, From, Opts) of + not_found -> []; + {ok, Assignment} -> + [ + Assignment + | read_local_assignments(CacheMod, ProcID, From + 1, To, Opts) + ] + end. + +%% @doc Options for generating (and reading messages for) schedule responses: +%% formatting is deterministic, so its resolutions are neither cached nor +%% awaited. +format_opts(Opts) -> + Opts#{ + <<"hashpath">> => ignore, + <<"cache-control">> => [<<"no-cache">>, <<"no-store">>], + <<"await-inprogress">> => false + }. + +%% @doc Merge the scheduler store with the main store. Used before reading +%% from or writing to a scheduler cache. +cache_opts(Opts) -> + Opts#{ + <<"store">> => + hb_opts:get( + scheduler_store, + hb_opts:get(store, no_viable_store, Opts), + Opts + ) + }. + +%% @doc Write an assignment message into a scheduler cache: the message goes +%% into the main cache, and `/assignments//' is linked to +%% its signed ID. The pseudo-path prefix distinguishes the caches of the +%% scheduler devices that share this helper. +write_assignment(Prefix, RawAssignment, RawOpts) -> + Assignment = hb_cache:ensure_all_loaded(RawAssignment, RawOpts), + Opts = cache_opts(RawOpts), + Store = hb_opts:get(store, no_viable_store, Opts), + ProcID = hb_ao:get(<<"process">>, Assignment, Opts), + Slot = hb_ao:get(<<"slot">>, Assignment, Opts), + ?event( + {writing_assignment, + {prefix, Prefix}, + {proc_id, ProcID}, + {slot, Slot} + } + ), + case hb_cache:write(Assignment, Opts) of + {ok, _UnsignedID} -> + ok = hb_store:link( + Store, + #{ + assignment_path(Prefix, ProcID, Slot) => + hb_message:id(Assignment, signed, Opts) + }, + Opts + ), + ok; + {error, Reason} -> + ?event(error, {failed_to_write_assignment, {reason, Reason}}), + {error, Reason} + end. + +%% @doc Get an assignment message from a scheduler cache. Restores the +%% assignment's unsigned commitment ID (`hb_cache:read' does not normalize +%% commitments) and converts legacy `ao.TN.1' (AOS2) assignments to the +%% canonical format via the scheduler package's formats module. +read_assignment(Prefix, ProcID, Slot, Opts) when is_integer(Slot) -> + read_assignment(Prefix, ProcID, hb_util:bin(Slot), Opts); +read_assignment(Prefix, ProcID, Slot, RawOpts) -> + Opts = cache_opts(RawOpts), + Store = hb_opts:get(store, no_viable_store, Opts), + Path = assignment_path(Prefix, ProcID, Slot), + ?event( + {read_assignment, + {prefix, Prefix}, + {proc_id, ProcID}, + {slot, Slot} + } + ), + case hb_store:resolve(Store, Path, Opts) of + {ok, ResolvedPath} -> + case hb_cache:read(ResolvedPath, Opts) of + {ok, RawAssignment} -> + Assignment = + hb_message:normalize_commitments(RawAssignment, Opts), + case hb_ao:get(<<"variant">>, Assignment, Opts) of + <<"ao.TN.1">> -> + Loaded = + hb_cache:ensure_all_loaded(Assignment, Opts), + {ok, + dev_scheduler_formats:aos2_to_assignment( + Loaded, + Opts + ) + }; + <<"ao.N.1">> -> + {ok, hb_cache:ensure_all_loaded(Assignment, Opts)} + end; + {error, not_found} -> not_found + end; + {error, not_found} -> not_found + end. + +assignment_path(Prefix, ProcID, Slot) -> + hb_path:to_binary( + [ + Prefix, + <<"assignments">>, + hb_util:human_id(ProcID), + hb_ao:normalize_key(Slot) + ] + ). From 458aff8daeebc7af1b4696390125a96be1997390 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Thu, 16 Jul 2026 14:11:27 -0400 Subject: [PATCH 05/27] refactor(arweave): share best_response via lib_arweave_common `dev_arweave' and the `~arweave-scheduler@1.0' device both broadcast a request to several Arweave nodes and take the most successful reply. Move `dev_arweave''s status-sorting `best_response/1' (with its `response_status/1' helper) into `lib_arweave_common', which both devices already depend on, and drop the local copy. No behavioural change for `dev_arweave'. --- src/preloaded/arweave/dev_arweave.erl | 28 ++-------------------- src/preloaded/codec/lib_arweave_common.erl | 27 +++++++++++++++++++++ 2 files changed, 29 insertions(+), 26 deletions(-) diff --git a/src/preloaded/arweave/dev_arweave.erl b/src/preloaded/arweave/dev_arweave.erl index 580b069cf2..7dcec5ece5 100644 --- a/src/preloaded/arweave/dev_arweave.erl +++ b/src/preloaded/arweave/dev_arweave.erl @@ -885,31 +885,7 @@ request(Method, Path, Extra, LogExtra, Opts) -> <<"cache-control">> => [<<"no-cache">>, <<"no-store">>] } ), - to_message(Path, Method, best_response(Res), LogExtra, Opts). - -%% @doc Select the best response from a list of responses by sorting them -%% ascending by HTTP status code. Returns the first (best) response tuple. -best_response({error, {no_viable_responses, Responses}}) -> - best_response(Responses); -best_response([]) -> - {error, no_viable_responses}; -best_response(Responses) when is_list(Responses) -> - Sorted = lists:sort( - fun({_, ResponseA}, {_, ResponseB}) -> - StatusA = response_status(ResponseA), - StatusB = response_status(ResponseB), - StatusA =< StatusB - end, - Responses - ), - hd(Sorted); -best_response(Response) -> - Response. - -response_status(Response) when is_map(Response) -> - maps:get(<<"status">>, Response, 999); -response_status(_Response) -> - 999. + to_message(Path, Method, lib_arweave_common:best_response(Res), LogExtra, Opts). %% @doc Transform a response from the Arweave node into an AO-Core message. to_message(Path, Method, {error, #{ <<"status">> := 404 }}, LogExtra, _Opts) -> @@ -1206,7 +1182,7 @@ best_response_handles_failed_connect_entries_test_parallel() -> ], ?assertEqual( {ok, #{ <<"status">> => 200, <<"body">> => <<"OK-2">> }}, - best_response(Responses) + lib_arweave_common:best_response(Responses) ). best_response_non_map_error_round_trips_test_parallel() -> diff --git a/src/preloaded/codec/lib_arweave_common.erl b/src/preloaded/codec/lib_arweave_common.erl index 755b9492cc..6430ad2deb 100644 --- a/src/preloaded/codec/lib_arweave_common.erl +++ b/src/preloaded/codec/lib_arweave_common.erl @@ -6,6 +6,7 @@ -export([bundle_hint/4, data/3, tags/5, excluded_tags/3]). -export([to/3, to/6, siginfo/4, fields_to_tx/4]). -export([bundle_header/2, bundle_header/3]). +-export([best_response/1]). -include("include/hb.hrl"). -define(ANS104_BASE_FIELDS, [<<"anchor">>, <<"target">>]). @@ -770,3 +771,29 @@ read_bundle_header(BundleStartOffset, HeaderSize, FirstChunk, Opts) -> Error -> Error end. + +%% @doc Select the best response from a (potentially multi-node) request result +%% by sorting the responses ascending by HTTP status code and returning the +%% first (best) one. Shared by the Arweave devices, which broadcast requests to +%% several nodes and take the most successful reply. +best_response({error, {no_viable_responses, Responses}}) -> + best_response(Responses); +best_response([]) -> + {error, no_viable_responses}; +best_response(Responses) when is_list(Responses) -> + Sorted = lists:sort( + fun({_, ResponseA}, {_, ResponseB}) -> + StatusA = response_status(ResponseA), + StatusB = response_status(ResponseB), + StatusA =< StatusB + end, + Responses + ), + hd(Sorted); +best_response(Response) -> + Response. + +response_status(Response) when is_map(Response) -> + maps:get(<<"status">>, Response, 999); +response_status(_Response) -> + 999. From c2d67c5d8ac05eefdb7bd5ddb72f5a2c526af383 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Thu, 16 Jul 2026 14:11:27 -0400 Subject: [PATCH 06/27] fix(query): resolve `block` on ~query@1.0 transaction nodes A transaction node returned from the local ~query@1.0 GraphQL endpoint resolved `block' to null, because a cached tx carries no block metadata of its own. Carry each match's weave offset onto its node and resolve `block' by locating the cached block whose byte range contains that offset, distinguished from the top-level block(id/height) query by the offset key on the node. Resolution is local but scans the cached blocks, so callers that only need on-chain ordering should use the offset directly. The cached-heights enumeration and the block byte-bounds arithmetic are shared as `cached_heights/1' and `block_bounds/2' with the module's pre-existing `latest_cached_block/1' and `block_range_to_offset_range/2', which previously inlined the same code separately. --- src/preloaded/query/dev_query_arweave.erl | 80 +++++++++++++++++------ 1 file changed, 60 insertions(+), 20 deletions(-) diff --git a/src/preloaded/query/dev_query_arweave.erl b/src/preloaded/query/dev_query_arweave.erl index 5ffb05f3c9..6b91e847db 100644 --- a/src/preloaded/query/dev_query_arweave.erl +++ b/src/preloaded/query/dev_query_arweave.erl @@ -87,6 +87,13 @@ query(Obj, <<"transactions">>, Args, Opts) -> ), {ok, connection([], Args, Opts)} end; +query(#{ <<"offset">> := Offset }, <<"block">>, _Args, Opts) + when is_integer(Offset) -> + % Resolve the `block' field of a transaction node: a cached transaction + % holds no block metadata of its own, so the block is found by the weave + % offset carried on the node (see `node_with_offset/3'), which also + % distinguishes this from the top-level `block(id/height)' query. + block_at_offset(Offset, Opts); query(Obj, <<"block">>, Args, Opts) -> case query(Obj, <<"blocks">>, Args, Opts) of {ok, []} -> {ok, null}; @@ -264,11 +271,21 @@ read_ids(_, 0, _Opts) -> []; read_ids([AnnotatedID = #{ <<"id">> := ID } | Rest], Count, Opts) -> case hb_cache:read(ID, Opts) of {ok, Msg} -> - [AnnotatedID#{ <<"node">> => Msg } | read_ids(Rest, Count - 1, Opts)]; + [ + AnnotatedID#{ <<"node">> => node_with_offset(Msg, AnnotatedID, Opts) } + | read_ids(Rest, Count - 1, Opts) + ]; _ -> read_ids(Rest, Count, Opts) end. +%% @doc Carry the transaction's weave offset onto its node, so that the `block' +%% field resolver can locate the block that includes it (see `query/4'). +node_with_offset(Msg, #{ <<"offset">> := Offset }, _Opts) + when is_map(Msg), is_integer(Offset) -> + Msg#{ <<"offset">> => Offset }; +node_with_offset(Msg, _Annotated, _Opts) -> Msg. + %% @doc Drop to the cursor position, returning the list of items after the cursor. drop_to_cursor(Args, Ordered, Opts) -> drop_to_cursor( @@ -380,14 +397,8 @@ block_range_to_offset_range(Heights, Opts) -> RawMin -> case read_block(hb_util:int(RawMin), Opts) of {ok, MinBlock} -> - % The `weave_size` is the size at the _end_ of the block, - % so we must subtract the start from it to find the - % starting byte of the block. - WeaveSize = hb_util:int( - hb_maps:get(<<"weave_size">>, MinBlock, 0, Opts)), - BlockSize = hb_util:int( - hb_maps:get(<<"block_size">>, MinBlock, 0, Opts)), - WeaveSize - BlockSize; + {BlockStart, _} = block_bounds(MinBlock, Opts), + BlockStart; {error, not_found} -> 0 end end, @@ -456,18 +467,47 @@ read_cached_block(Height, Opts) -> %% @doc Return the latest block height indexed in the Arweave pseudo-path cache. latest_cached_block(Opts) -> - Blocks = - hb_cache:list_numbered( - hb_path:to_binary([ - <<"~arweave@2.9">>, - <<"block">>, - <<"height">> - ]), - Opts - ), - case Blocks of + case cached_heights(Opts) of [] -> not_found; - _ -> {ok, lists:max(Blocks)} + Blocks -> {ok, lists:max(Blocks)} + end. + +%% @doc List the block heights present in the Arweave pseudo-path cache. +cached_heights(Opts) -> + hb_cache:list_numbered( + hb_path:to_binary([ + <<"~arweave@2.9">>, + <<"block">>, + <<"height">> + ]), + Opts + ). + +%% @doc The weave byte range `{StartOffset, EndOffset}' that a block covers. +%% The block's `weave_size' is the size of the weave at the block's _end_, so +%% its starting byte is that size less the block's own size. +block_bounds(Block, Opts) -> + WeaveSize = hb_util:int(hb_maps:get(<<"weave_size">>, Block, 0, Opts)), + BlockSize = hb_util:int(hb_maps:get(<<"block_size">>, Block, 0, Opts)), + {WeaveSize - BlockSize, WeaveSize}. + +%% @doc Find the cached block that includes the given weave offset -- the block +%% whose byte range `[weave_size - block_size, weave_size)' contains it. Scans +%% the locally cached blocks, so it is only reached when the `block' field is +%% explicitly selected on a transaction. Returns `{ok, null}' when no cached +%% block covers the offset. +block_at_offset(Offset, Opts) -> + block_covering(Offset, lists:sort(cached_heights(Opts)), Opts). + +block_covering(_Offset, [], _Opts) -> {ok, null}; +block_covering(Offset, [Height | Rest], Opts) -> + case read_cached_block(Height, Opts) of + {ok, Block} -> + case block_bounds(Block, Opts) of + {Start, End} when Start =< Offset, Offset < End -> {ok, Block}; + _ -> block_covering(Offset, Rest, Opts) + end; + _ -> block_covering(Offset, Rest, Opts) end. %%% Match argument processing From 9f77c707ae2d9e1ba2fdfed65c2ab22fa50fdda4 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Thu, 16 Jul 2026 14:11:47 -0400 Subject: [PATCH 07/27] feat: add ~arweave-scheduler@1.0, an Arweave-L1-sequenced scheduler A drop-in ~scheduler@1.0 replacement that sources a process's schedule directly from Arweave layer-1 transactions. Slot 0 is the process message (its tx id is the process id); slot N is the Nth confirmed L1 tx whose `target' is the process id, in ascending weave-offset order. Assignments are deterministic and uncommitted -- Arweave itself is the authority -- so every node converges on identical assignment ids without trusting a scheduler wallet. The on-chain position recorded on each assignment is the weave offset. Discovery is local-first, with no gateway attribution: a single `~copycat@1.0/arweave' shallow pass over the range records each tx's weave offset and caches its (data-free) header, which populates the local `field-target' match; the device then resolves the node's own ~query@1.0 (via hb_ao:resolve, not a remote call) for the recipient set, ordered by that offset index. `arweave_scheduler_query_source => remote' opts into a gateway query instead. The block index is node-shared and `ensure_offsets' passes `reindex=false', so N processes on one node index each block once rather than once per process. The device is header-only end to end. Bodies are read as tx headers, so the schedule never depends on data availability; the header verifies and its committed id is the tx id. POST /schedule likewise relays the header only, rejecting a message that carries data (422) rather than uploading chunks, and rejecting any non-tx@1.0 commitment (422). The misleading tip-derived timestamp/block-height/block-hash that the standard formatter stamps are omitted from a chain-derived schedule. Synchronization tracks a `synced-to' block high-water mark: the attestable range `[spawn-height, synced-to]' the node can vouch for. A read with no new blocks does no network work; otherwise only the delta is indexed, in `arweave_scheduler_sync_chunk'-block passes whose progress is persisted per chunk (resumable first sync). /status reports each tracked process's range. The device is built on `lib_scheduler': request parsing, the assignment cache, the range read, and the base assignment schema are the same code paths ~scheduler@1.0 uses. The full consumer interface (status/next/schedule/slot/init/checkpoint + router) is implemented so ~process@1.0 drives it unchanged. Includes a permanent mainnet test fixture (a lua@5.3a process whose three scheduled messages transform state 1000 -> 1337, one posted directly to arweave.net to prove foreign-tx indexing) and device unit tests. --- .../process/dev_arweave_scheduler.erl | 1042 +++++++++++++++++ .../process/dev_arweave_scheduler_cache.erl | 111 ++ test/arweave-scheduler-test.lua | 34 + 3 files changed, 1187 insertions(+) create mode 100644 src/preloaded/process/dev_arweave_scheduler.erl create mode 100644 src/preloaded/process/dev_arweave_scheduler_cache.erl create mode 100644 test/arweave-scheduler-test.lua diff --git a/src/preloaded/process/dev_arweave_scheduler.erl b/src/preloaded/process/dev_arweave_scheduler.erl new file mode 100644 index 0000000000..37a32ea45b --- /dev/null +++ b/src/preloaded/process/dev_arweave_scheduler.erl @@ -0,0 +1,1042 @@ +%%% @doc A scheduler for AO processes that uses the Arweave network itself +%%% as the sequencing layer. The device implements the same interface as +%%% `~scheduler@1.0' from the perspective of consumers (`~process@1.0' et +%%% al), but assignments are not minted by a scheduling authority: they are +%%% implicit in the order that Arweave blocks include L1 transactions that +%%% target the process. +%%% +%%% The model is as follows: +%%%
    +%%%
  • A process is spawned by uploading its process message to Arweave +%%% as a `tx@1.0'-committed L1 transaction. The transaction ID is the +%%% process ID, and the process message occupies slot 0 of its own +%%% schedule.
  • +%%%
  • Every L1 transaction whose `target' field is the process ID is a +%%% message in the process's schedule. Slots follow the canonical weave +%%% order (ascending weave offset, which is ascending block order). The +%%% transaction's weave `offset' -- not a scheduler-assigned nonce -- is +%%% its on-chain position, and it is recorded on the assignment.
  • +%%%
  • Discovery is local-first: the node indexes the relevant +%%% blocks itself with `~copycat@1.0' and then queries its own +%%% `~query@1.0' GraphQL endpoint (`transactions(recipients: +%%% [ProcessID], sort: HEIGHT_ASC)') for the base-layer transactions +%%% addressed to the process, ordered by the local weave-offset index. +%%% A node may instead be configured to query a remote gateway +%%% (`arweave_scheduler_query_source => remote'). There is no bespoke +%%% store index -- discovery uses the node's existing Arweave index and +%%% the device's own schedule cache.
  • +%%%
+%%% +%%% Assignment bodies are transaction headers only: each is the +%%% data-free header that `~copycat@1.0/arweave' cached while indexing the +%%% range (read from the node's stores), so the schedule never carries (nor +%%% depends on the availability of) a transaction's data. A header still +%%% carries the tx@1.0 signature, so its committed ID is the transaction ID +%%% and it verifies independently. Assignments are +%%% deterministic derivations of chain data and are left uncommitted: every +%%% node that reads the same blocks converges on an identical schedule +%%% without trusting a scheduler wallet. Only `tx@1.0' commitments are +%%% accepted for dispatch -- ANS-104 data items and HTTPSig messages cannot +%%% be sequenced by the base layer. +%%% +%%% `POST /schedule' dispatches a user's presigned transaction to the +%%% network on their behalf. Only its header is relayed: because the schedule +%%% is built from headers, a message carrying data is rejected rather than +%%% dispatched, so its slot can never depend on chunk availability. No slot is +%%% returned at dispatch time: the transaction receives its slot once it is +%%% included in a block. Note +%%% that Arweave refuses to create accounts with a zero balance +%%% (`validate_overspend'), and a process ID is a fresh account: the first +%%% message addressed to a process must therefore carry a `quantity' of at +%%% least 1 winston (and, until the account exists, the sender's `reward' +%%% must cover the network's new-account fee). +%%% +%%% Each process's synchronization tracks the contiguous block range +%%% `[spawn-height, synced-to]' it has indexed -- the range the node can +%%% attest to. A sync only indexes the blocks above `synced-to' (up to the +%%% confirmed tip), so a repeated read with no new blocks does no network +%%% work, and `/status' reports each tracked process's range. The device +%%% honors the following node options: +%%%
    +%%%
  • `arweave_scheduler_confirmation_depth': blocks under the network +%%% tip that must elapse before a block is considered final +%%% (default: 10).
  • +%%%
  • `arweave_scheduler_sync_chunk': blocks indexed per synchronization +%%% pass, bounding the work redone if a pass is interrupted +%%% (default: 1000).
  • +%%%
  • `arweave_scheduler_max_height': optional hard upper bound on the +%%% synced height, pinning a schedule to an immutable block range +%%% (used primarily in tests).
  • +%%%
+-module(dev_arweave_scheduler). +-implements(<<"arweave-scheduler@1.0">>). +-device_libraries([lib_process, lib_scheduler, lib_arweave_common]). +%%% AO-Core API functions: +-export([info/0]). +%%% Scheduling functions: +-export([schedule/3, router/4]). +%%% CU-flow functions: +-export([slot/3, status/3, next/3]). +-export([checkpoint/1]). +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +%%% The default number of blocks beneath the network tip at which we +%%% consider a block final. +-define(DEFAULT_CONFIRMATION_DEPTH, 10). +%%% The default number of blocks indexed per synchronization pass. A sync +%%% persists its progress after each chunk, so this bounds the work redone if +%%% a pass is interrupted and keeps the first sync of a long-lived process +%%% resumable. Overridable with the `arweave_scheduler_sync_chunk' option. +-define(DEFAULT_SYNC_CHUNK_BLOCKS, 1000). +%%% The number of transactions requested per query page. +-define(QUERY_PAGE_SIZE, 100). +%%% The Arweave device that we resolve chain data through. +-define(ARWEAVE_DEVICE, <<"~arweave@2.9">>). + +%% @doc This device uses a default handler to route requests to the correct +%% function. +info() -> + #{ + exports => + [ + <<"status">>, + <<"next">>, + <<"schedule">>, + <<"slot">>, + <<"init">>, + <<"checkpoint">> + ], + excludes => [set, keys], + default => fun router/4 + }. + +%% @doc The default handler for the device: route all unrecognized requests +%% to `schedule'. +router(_, Base, Req, Opts) -> + ?event({arweave_scheduler_router_called, {req, Req}}), + schedule(Base, Req, Opts). + +%% @doc Return the next assignment for a process. Assumes that `Base' is a +%% `dev_process' or similar message, having an `at-slot' key. If the next +%% slot is not present in the local cache, the schedule is synchronized +%% from Arweave before the read is retried. +next(Base, Req, Opts) -> + ProcID = lib_process:process_id(Base, Req, Opts), + LastProcessed = lib_scheduler:at_slot(Base, Opts), + ?event(next, {arweave_next, {proc_id, ProcID}, {last, LastProcessed}}), + maybe + {ok, Assignment} ?= find_assignment(ProcID, LastProcessed + 1, Opts), + {ok, #{ <<"body">> => Assignment, <<"state">> => Base }} + end. + +%% @doc Read an assignment from the local cache, synchronizing the schedule +%% from Arweave if it is not yet present. +find_assignment(ProcID, Slot, Opts) -> + case dev_arweave_scheduler_cache:read(ProcID, Slot, Opts) of + {ok, Assignment} -> {ok, Assignment}; + not_found -> + maybe + {ok, _} ?= sync(ProcID, Opts), + case dev_arweave_scheduler_cache:read(ProcID, Slot, Opts) of + {ok, Assignment} -> {ok, Assignment}; + not_found -> lib_scheduler:slot_unavailable() + end + end + end. + +%% @doc A router for choosing between getting the existing schedule, or +%% dispatching a new message to Arweave. +schedule(Base, Req, Opts) -> + ?event({resolving_arweave_schedule_request, {req, Req}}), + case hb_util:key_to_atom(hb_ao:get(<<"method">>, Req, <<"GET">>, Opts)) of + post -> post_schedule(Base, Req, Opts); + get -> get_schedule(Base, Req, Opts) + end. + +%% @doc Generate and return the schedule for a process, optionally between +%% two slots -- labelled as `from' and `to'. +get_schedule(Base, Req, Opts) -> + ProcID = hb_util:human_id(lib_scheduler:find_target_id(Base, Req, Opts)), + {From, To} = lib_scheduler:parse_slot_range(Req, Opts), + maybe + {ok, #{ <<"next-slot">> := NextSlot }} ?= sync(ProcID, Opts), + Latest = NextSlot - 1, + RequestedTo = + case To of + undefined -> Latest; + _ -> min(To, Latest) + end, + {Assignments, More} = + lib_scheduler:read_assignment_range( + dev_arweave_scheduler_cache, ProcID, From, RequestedTo, Opts), + dev_arweave_scheduler_cache:assignments_to_bundle( + ProcID, + Assignments, + More, + Opts + ) + end. + +%% @doc Dispatch a new message to Arweave. The message must carry a signed +%% `tx@1.0' commitment: the caller signs the L1 transaction themselves +%% (including its `anchor' and `reward'), and this device relays it to the +%% network. The message is also written to the local cache, so that it is +%% servable by ID before the network has propagated and indexed it. +post_schedule(Base, Req, Opts) -> + maybe + {ok, ToSched} ?= lib_scheduler:load_message_to_schedule(Base, Req, Opts), + do_post_schedule(Base, Req, ToSched, Opts) + end. + +do_post_schedule(Base, Req, ToSched, Opts) -> + ProcID = lib_scheduler:find_target_id(Base, Req, ToSched, Opts), + ?event({arweave_post_schedule, {proc_id, ProcID}}), + maybe + {ok, OnlyCommitted} ?= lib_scheduler:only_committed(ToSched, Opts), + ok ?= ensure_tx_committed(OnlyCommitted, Opts), + dispatch(ProcID, OnlyCommitted, Opts) + end. + +%% @doc Ensure that the given message carries a signed, valid `tx@1.0' +%% commitment. The Arweave scheduler accepts only L1 transactions: ANS-104 +%% and HTTPSig commitments cannot be sequenced by the base layer. +ensure_tx_committed(Msg, Opts) -> + Devices = hb_message:commitment_devices(Msg, Opts), + Signers = hb_message:signers(Msg, Opts), + case {lists:member(<<"tx@1.0">>, Devices), Signers} of + {false, _} -> + {error, + #{ + <<"status">> => 422, + <<"body">> => + <<"The Arweave scheduler only accepts messages ", + "committed with tx@1.0.">> + } + }; + {true, []} -> + {error, + #{ + <<"status">> => 422, + <<"body">> => <<"Message must be signed.">> + } + }; + {true, _} -> + case hb_message:verify(Msg, signers, Opts) of + true -> ok; + false -> + {error, + #{ + <<"status">> => 400, + <<"body">> => <<"Message is not valid.">> + } + } + end + end. + +%% @doc Relay a committed `tx@1.0' message to the Arweave network. Only the +%% transaction header is dispatched: this scheduler sequences headers, never +%% data. A message carrying data is therefore rejected rather than uploaded -- +%% its chunks might be unavailable when the schedule is later read. The same +%% header-only rule governs synchronization, so a data-carrying transaction +%% posted directly to Arweave is still sequenced safely, by its header alone. +dispatch(ProcID, Msg, Opts) -> + TX = hb_message:convert(Msg, <<"tx@1.0">>, Opts), + case TX#tx.data of + <<>> -> dispatch_header(ProcID, TX, Msg, Opts); + _ -> + {error, + #{ + <<"status">> => 422, + <<"body">> => + <<"The Arweave scheduler sequences transaction ", + "headers only; a message carrying data cannot ", + "be scheduled.">> + } + } + end. + +dispatch_header(ProcID, TX, Msg, Opts) -> + TXID = hb_util:human_id(TX#tx.id), + ?event({dispatching_tx, {id, {explicit, TXID}}, {proc_id, ProcID}}), + maybe + {ok, _} ?= post_tx_header(TX, Opts), + {ok, _} = hb_cache:write(Msg, Opts), + % `tx-id' rather than `id': the reserved `id' key would be shadowed + % by the receipt message's own ID when read by consumers. + {ok, + #{ + <<"status">> => 202, + <<"tx-id">> => TXID, + <<"process">> => hb_util:human_id(ProcID), + <<"body">> => + <<"Transaction dispatched to Arweave. It will receive ", + "a slot in the schedule once mined.">> + } + } + end. + +%% @doc Post a transaction header to the network, via the node's `/arweave' +%% route. The route may broadcast to multiple nodes: acceptance by any one +%% of them places the transaction in the network's mempool, so the best +%% response wins, mirroring `~arweave@2.9's handling. +post_tx_header(TX, Opts) -> + Res = + hb_http:request( + #{ + <<"method">> => <<"POST">>, + <<"path">> => <<"/arweave/tx">>, + <<"body">> => hb_json:encode(ar_tx:tx_to_json_struct(TX)) + }, + no_result_cache(Opts) + ), + lib_arweave_common:best_response(Res). + +%% @doc Returns information about the current slot for a process. +slot(Base, Req, Opts) -> + ProcID = hb_util:human_id(lib_scheduler:find_target_id(Base, Req, Opts)), + ?event({getting_current_slot, {proc_id, ProcID}}), + maybe + {ok, #{ <<"next-slot">> := NextSlot }} ?= sync(ProcID, Opts), + {ok, + #{ + <<"process">> => ProcID, + <<"current">> => NextSlot - 1, + <<"cache-control">> => <<"no-store">> + } + } + end. + +%% @doc Returns information about the scheduler, including -- for each process +%% it tracks -- the contiguous block range it has indexed and can attest to. +status(_Base, _Req, Opts) -> + Wallet = hb_opts:get(priv_wallet, hb:wallet(), Opts), + ProcIDs = dev_arweave_scheduler_cache:list_processes(Opts), + {ok, + #{ + <<"address">> => hb_util:human_id(ar_wallet:to_address(Wallet)), + <<"processes">> => + hb_maps:from_list([ {P, attestable_range(P, Opts)} || P <- ProcIDs ]), + <<"cache-control">> => <<"no-store">> + } + }. + +%% @doc The block range a process has been indexed over, as it currently stands +%% in the cache (no synchronization is triggered). `from' is the spawn block and +%% `to' is the confirmed height up to which the schedule is complete; `current' +%% is the latest materialized slot. +attestable_range(ProcID, Opts) -> + case dev_arweave_scheduler_cache:read_state(ProcID, Opts) of + {ok, #{ + <<"spawn-height">> := Spawn, + <<"synced-to">> := SyncedTo, + <<"next-slot">> := NextSlot + }} -> + #{ + <<"from">> => Spawn, + <<"to">> => SyncedTo, + <<"current">> => NextSlot - 1 + }; + _ -> #{} + end. + +%% @doc Returns the current state of the scheduler. +checkpoint(State) -> {ok, State}. + +%%% Synchronization + +%% @doc Synchronize the schedule of a process from Arweave, extending the +%% contiguously-indexed block range `[spawn-height, synced-to]' up to the +%% confirmed network tip. Only the blocks above `synced-to' are indexed; a +%% process already synced to the tip does no network work at all. +sync(ProcID, Opts) -> + maybe + {ok, State} ?= ensure_initialized(ProcID, Opts), + {ok, Upper} ?= confirmed_tip(Opts), + do_sync(ProcID, State, Upper, Opts) + end. + +%% @doc Extend the synced range up to `Upper', one bounded chunk at a time. The +%% range grows contiguously from `synced-to + 1', and `{next-slot, synced-to}' +%% are persisted together after each chunk, so an interrupted sync resumes at +%% the start of the unfinished chunk (re-materializing is idempotent) rather +%% than re-deriving the whole history. +do_sync(_ProcID, State = #{ <<"synced-to">> := SyncedTo }, Upper, _Opts) + when SyncedTo >= Upper -> + {ok, State}; +do_sync(ProcID, State = #{ <<"synced-to">> := SyncedTo }, Upper, Opts) -> + From = SyncedTo + 1, + To = min(SyncedTo + sync_chunk_blocks(Opts), Upper), + ?event( + {arweave_scheduler_sync, + {proc_id, ProcID}, + {from, From}, + {to, To}, + {target, Upper} + } + ), + maybe + {ok, Ordered} ?= discover(ProcID, From, To, Opts), + {ok, NewState} ?= materialize(ProcID, State, Ordered, To, Opts), + do_sync(ProcID, NewState, Upper, Opts) + end. + +%% @doc The number of blocks to index per synchronization pass. +sync_chunk_blocks(Opts) -> + hb_util:int( + hb_opts:get(arweave_scheduler_sync_chunk, ?DEFAULT_SYNC_CHUNK_BLOCKS, Opts) + ). + +%% @doc Side-effecting and mutable internal resolutions must never be served +%% from (or written to) the resolution cache: nodes default to caching every +%% HTTP resolution, which would otherwise freeze the indexing runs, queries +%% and tip lookups at their first result. +no_result_cache(Opts) -> + Opts#{ + <<"hashpath">> => ignore, + <<"cache-control">> => [<<"no-cache">>, <<"no-store">>] + }. + +%% @doc Discover the base-layer transactions addressed to a process within a +%% block range, in ascending weave order, entirely from the node's own index. +%% Indexing the range with `~copycat@1.0/arweave' both records each +%% transaction's weave offset and caches its header (so its `target' is locally +%% matchable); the recipient match is then served from the node's own +%% `~query@1.0' GraphQL endpoint, and each match is annotated with its offset -- +%% the sort key and the `offset' recorded on the assignment. No gateway is +%% queried by default. Returns `{Offset, TXID}' pairs in ascending offset order; +%% bundled data items are excluded, as this scheduler sequences the base layer +%% only. +discover(ProcID, From, To, Opts) -> + maybe + ok ?= ensure_offsets(From, To, Opts), + {ok, IDs} ?= query_recipients(ProcID, From, To, Opts), + {ok, base_layer_offsets(IDs, Opts)} + end. + +%% @doc Ensure the node's local Arweave index covers the block range, so that +%% every transaction in it has both a weave offset and a locally-cached header. +%% `~copycat@1.0/arweave' in `shallow' mode records an ID->offset entry for each +%% transaction and caches its (data-free) header -- populating the `field-target' +%% match index that `query_recipients' reads. `reindex=false' skips blocks the +%% node has already indexed (for any process), so the run is idempotent, +%% incremental, and shared: overlapping ranges across processes are not +%% re-fetched. +ensure_offsets(From, To, _Opts) when From > To -> ok; +ensure_offsets(From, To, Opts) -> + maybe + {ok, _} ?= + hb_ao:resolve( + << + "~copycat@1.0/arweave&mode=shallow&reindex=false", + "&from=", (hb_util:bin(To))/binary, + "&to=", (hb_util:bin(From))/binary + >>, + no_result_cache(Opts) + ), + ok + end. + +%% @doc Query for the transactions addressed to a process within a block +%% range, ordered by weave position, returning their IDs. By default the query +%% is served by the node's own `~query@1.0' index (`local'), whose `field-target' +%% matches are populated by `~copycat@1.0/arweave' as it indexes the range (see +%% `ensure_offsets'); a node may instead be configured +%% (`arweave_scheduler_query_source => remote') to query a remote gateway. +query_recipients(ProcID, From, To, Opts) -> + Source = hb_opts:get(arweave_scheduler_query_source, local, Opts), + Query = + << + "query($after: String) { transactions(", + "recipients: [\"", (hb_util:human_id(ProcID))/binary, "\"], ", + "block: { min: ", (hb_util:bin(From))/binary, + ", max: ", (hb_util:bin(To))/binary, " }, ", + "sort: HEIGHT_ASC, ", + "first: ", (hb_util:bin(?QUERY_PAGE_SIZE))/binary, + ", after: $after", + ") { pageInfo { hasNextPage } edges { cursor node { id } } } }" + >>, + query_pages(Source, Query, undefined, [], Opts). + +query_pages(Source, Query, After, Acc, Opts) -> + Variables = + case After of + undefined -> #{}; + _ -> #{ <<"after">> => After } + end, + maybe + {ok, Transactions} ?= run_query(Source, Query, Variables, Opts), + Edges = hb_maps:get(<<"edges">>, Transactions, [], Opts), + IDs = Acc ++ [ edge_id(E, Opts) || E <- Edges ], + HasNext = + hb_util:atom( + hb_ao:get( + <<"pageInfo/hasNextPage">>, + Transactions, + false, + Opts#{ <<"hashpath">> => ignore } + ) + ), + case {HasNext, Edges} of + {true, [_ | _]} -> + Cursor = + hb_maps:get(<<"cursor">>, lists:last(Edges), undefined, Opts), + query_pages(Source, Query, Cursor, IDs, Opts); + _ -> + {ok, [ ID || ID <- IDs, is_binary(ID) ]} + end + end. + +edge_id(Edge, Opts) -> + hb_maps:get(<<"id">>, hb_maps:get(<<"node">>, Edge, #{}, Opts), undefined, Opts). + +%% @doc Run a GraphQL `transactions' query, returning its connection. `local' +%% resolves the node's own `~query@1.0/graphql' endpoint; `remote' posts to +%% the configured `gateway' directly. The remote path deliberately does not use +%% `hb_client_gateway:query': its admissibility gate rejects legitimately-empty +%% ranges, and the racing `/graphql' route lets AO-search gateways that do not +%% index arbitrary L1 transactions win with empty-but-200 responses. Here an +%% empty range is an authoritative answer. +run_query(local, Query, Variables, Opts) -> + to_transactions( + hb_ao:resolve( + #{ <<"device">> => <<"query@1.0">> }, + #{ + <<"path">> => <<"graphql">>, + <<"method">> => <<"POST">>, + <<"query">> => Query, + <<"variables">> => Variables + }, + no_result_cache(Opts) + ), + Opts + ); +run_query(remote, Query, Variables, Opts) -> + Gateway = hb_opts:get(gateway, <<"https://arweave.net">>, Opts), + to_transactions( + hb_http:post( + Gateway, + #{ + <<"path">> => <<"/graphql">>, + <<"content-type">> => <<"application/json">>, + <<"body">> => + hb_json:encode( + #{ <<"query">> => Query, <<"variables">> => Variables } + ) + }, + no_result_cache(Opts) + ), + Opts + ). + +to_transactions({ok, Response}, Opts) -> + Decoded = + hb_json:decode(hb_ao:get(<<"body">>, Response, <<"{}">>, Opts)), + {ok, + hb_ao:get( + <<"data/transactions">>, + Decoded, + #{}, + Opts#{ <<"hashpath">> => ignore } + ) + }; +to_transactions({error, Reason}, _Opts) -> + {error, + #{ + <<"status">> => 502, + <<"reason">> => <<"Arweave query failed.">>, + <<"detail">> => Reason + } + }. + +%% @doc Annotate each matched transaction with its weave offset from the +%% local index, keeping only the base-layer (`tx@1.0') transactions and +%% dropping bundled data items (indexed under the `ans104@1.0' codec) and any +%% that are not yet locally indexed. The result is sorted by offset, which is +%% the canonical weave order. +base_layer_offsets(IDs, Opts) -> + Store = hb_store_arweave:store_from_opts(Opts), + lists:keysort( + 1, + lists:filtermap( + fun(ID) -> base_layer_offset(Store, ID, Opts) end, + IDs + ) + ). + +base_layer_offset(Store, ID, Opts) -> + case offset_entry(Store, ID, Opts) of + {ok, <<"tx@1.0">>, Offset} -> {true, {Offset, ID}}; + _ -> false + end. + +%% @doc Read a transaction's local index entry, returning its codec device and +%% weave offset. +offset_entry(Store, ID, Opts) -> + case hb_store_arweave:read_offset(Store, ID, Opts) of + {ok, + #{ + <<"codec-device">> := Codec, + <<"start-offset">> := Offset + }} when is_integer(Offset) -> + {ok, Codec, Offset}; + _ -> not_found + end. + +%% @doc Materialize a chunk's assignments and record that the range is synced +%% to `To'. The discovered list holds only the messages in blocks above the +%% already-synced range, so they are exactly the next slots -- appended from +%% `next-slot' with no de-duplication. `next-slot' and `synced-to' are then +%% persisted together, so the two never diverge. +materialize(ProcID, State = #{ <<"next-slot">> := NextSlot }, Ordered, To, Opts) -> + assign(ProcID, State, NextSlot, Ordered, To, Opts). + +assign(ProcID, State, Slot, [], To, Opts) -> + NewState = State#{ <<"next-slot">> => Slot, <<"synced-to">> => To }, + ok = dev_arweave_scheduler_cache:write_state(ProcID, NewState, Opts), + {ok, NewState}; +assign(ProcID, State, Slot, [{Offset, TXID} | Rest], To, Opts) -> + case read_tx_header(TXID, Opts) of + {ok, Msg} -> + ok = write_assignment(ProcID, Slot, Offset, Msg, Opts), + assign(ProcID, State, Slot + 1, Rest, To, Opts); + {error, Err} -> + ?event( + error, + {arweave_scheduler_tx_unavailable, + {proc_id, ProcID}, + {tx, {explicit, TXID}} + } + ), + {error, Err} + end. + +%% @doc Generate and store the synthetic assignment for a message. Mirrors the +%% assignments minted by `~scheduler@1.0', but the on-chain position is the +%% transaction's weave `offset' rather than a scheduler-assigned nonce. Every +%% field derives from chain data, so the assignment is deterministic across +%% nodes and is left uncommitted. +write_assignment(ProcID, Slot, Offset, Msg, Opts) -> + BaseAssignment = + lib_scheduler:base_assignment( + hb_util:human_id(ProcID), + Slot, + Msg, + Opts + ), + Assignment = BaseAssignment#{ <<"offset">> => Offset }, + ?event( + {minting_assignment, + {proc_id, ProcID}, + {slot, Slot}, + {offset, Offset} + } + ), + dev_arweave_scheduler_cache:write(Assignment, Opts). + +%% @doc Find or create the persisted synchronization state for a process. +ensure_initialized(ProcID, Opts) -> + case dev_arweave_scheduler_cache:read_state(ProcID, Opts) of + {ok, State} -> {ok, State}; + _ -> initialize(ProcID, Opts) + end. + +%% @doc First contact with a process: locate its spawn block, index it, read +%% the process header, and mint the slot 0 assignment from the process +%% message itself. The canonical process header is also written to the cache +%% so that `~process@1.0' resolves to the verifying tx@1.0 form +%% ahead of any lossier gateway-derived copy. If the spawn is not yet +%% confirmed, initialization fails and is retried on the next synchronization. +initialize(ProcID, Opts) -> + ?event({initializing_arweave_schedule, {proc_id, ProcID}}), + maybe + {ok, SpawnHeight} ?= spawn_height(ProcID, Opts), + ok ?= ensure_offsets(SpawnHeight, SpawnHeight, Opts), + {ok, Offset} ?= tx_offset(ProcID, Opts), + {ok, Process} ?= read_tx_header(ProcID, Opts), + {ok, _} = hb_cache:write(Process, Opts), + ok = write_assignment(ProcID, 0, Offset, Process, Opts), + % `synced-to' starts one below the spawn block: slot 0 is the process + % itself, and no message-bearing block has been indexed yet. The first + % sync begins its range at the spawn block, catching any messages mined + % alongside the process. + State = + #{ + <<"next-slot">> => 1, + <<"spawn-height">> => SpawnHeight, + <<"synced-to">> => SpawnHeight - 1 + }, + ok = dev_arweave_scheduler_cache:write_state(ProcID, State, Opts), + {ok, State} + end. + +%% @doc Read an L1 transaction as a header-only message from the node's stores. +%% `~copycat@1.0/arweave' caches the (data-free) header locally while indexing, +%% so the read is normally served straight from the local store; a miss falls +%% through the rest of the store chain (which may reach the header faster than a +%% specific Arweave host). The header carries the transaction's tags and its +%% tx@1.0 signature -- so its committed ID is the transaction ID and it verifies +%% -- and no data is attached at this layer, so the schedule never depends on +%% the availability of any transaction's data. +read_tx_header(TXID, Opts) -> + case hb_cache:read(TXID, Opts) of + {ok, Msg} -> {ok, hb_cache:ensure_all_loaded(Msg, Opts)}; + _ -> + {error, + #{ + <<"status">> => 503, + <<"reason">> => + <<"Transaction header is not yet retrievable ", + "from Arweave.">>, + <<"tx">> => TXID + } + } + end. + +%% @doc Read a transaction's weave offset from the local index. A transaction +%% that has not yet been indexed has no offset. +tx_offset(TXID, Opts) -> + Store = hb_store_arweave:store_from_opts(Opts), + case offset_entry(Store, TXID, Opts) of + {ok, _Codec, Offset} -> {ok, Offset}; + not_found -> + {error, + #{ + <<"status">> => 503, + <<"reason">> => + <<"Transaction is not yet indexed locally.">>, + <<"tx">> => TXID + } + } + end. + +%% @doc Locate the block height that includes a transaction, via the Arweave +%% `tx status' API. An unconfirmed transaction has no block height: a process +%% cannot be scheduled against until its spawn is confirmed. +spawn_height(TXID, Opts) -> + Res = + hb_http:request( + #{ + <<"method">> => <<"GET">>, + <<"path">> => <<"/arweave/tx/", TXID/binary, "/status">> + }, + no_result_cache(Opts) + ), + case lib_arweave_common:best_response(Res) of + {ok, Response} -> + Status = + hb_json:decode(hb_ao:get(<<"body">>, Response, <<"{}">>, Opts)), + case hb_maps:get(<<"block_height">>, Status, undefined, Opts) of + undefined -> {error, unconfirmed_process(TXID)}; + Height -> {ok, hb_util:int(Height)} + end; + _ -> {error, unconfirmed_process(TXID)} + end. + +unconfirmed_process(TXID) -> + #{ + <<"status">> => 404, + <<"reason">> => <<"Process is not yet confirmed on Arweave.">>, + <<"process">> => TXID + }. + +%% @doc The height up to which the schedule may safely be extended: the +%% network tip less the configured confirmation depth, optionally capped by +%% the `arweave_scheduler_max_height' node option. +confirmed_tip(Opts) -> + maybe + {ok, Height} ?= + hb_ao:resolve( + <>, + no_result_cache(Opts) + ), + Depth = + hb_util:int( + hb_opts:get( + arweave_scheduler_confirmation_depth, + ?DEFAULT_CONFIRMATION_DEPTH, + Opts + ) + ), + Confirmed = hb_util:int(Height) - Depth, + case hb_opts:get(arweave_scheduler_max_height, undefined, Opts) of + undefined -> {ok, Confirmed}; + Max -> {ok, min(Confirmed, hb_util:int(Max))} + end + end. + +%%% Tests + +%%% The permanent test fixture: a `lua@5.3a' process scheduled by this +%%% device, seeded onto Arweave with three state-transformation messages +%%% (`test/arweave-scheduler-test.lua'; all transactions carry +%%% `test-suite: arweave-scheduler' tags). `?FIXTURE_MSG2' was posted +%%% directly to arweave.net, never passing through a HyperBEAM node: its +%%% presence in the schedule proves foreign-message indexing. Arweave is +%%% permanent, so these tests are repeatable against the live network. +%%% Spawn block 1958986; messages at 1958993, 1958994 and 1958995. +-define(FIXTURE_MODULE, <<"_GeSyZbQkmqWk6YzL-tjIqJ-2hakIkg-9k127DpVfO8">>). +-define(FIXTURE_PROCESS, <<"q3SycbYpO1lz-S6V2kd7FG3DIZ3AU0NrKKxWw4C3yos">>). +-define(FIXTURE_MSG1, <<"Y8GnKytC57VUk7W7FlGnD1oH4OAwFHyP-pJPGCJBa3Y">>). +-define(FIXTURE_MSG2, <<"luf1fFmhi0RMIZNnFmv1QgK2SJU5plAFQ3ZxndUsLpk">>). +-define(FIXTURE_MSG3, <<"UThcMtT6zy0pJ7iXUsMTFUMu3Xhv51EAjn5BzqHMhR4">>). +-define(FIXTURE_MAX_HEIGHT, 1958995). + +test_opts() -> + TestStore = hb_test_utils:test_store(), + IndexStore = hb_test_utils:test_store(), + % The full default config is merged in so the node has the Arweave routes + % that `~copycat@1.0' and `~arweave@2.9' resolve through. + (hb_opts:default_message())#{ + <<"store">> => [ + TestStore, + % The local Arweave offset index that discovery orders by. + #{ + <<"store-module">> => hb_store_arweave, + <<"name">> => <<"cache-arweave">>, + <<"index-store">> => [IndexStore] + }, + % Serves the Lua module transaction's data (its source) and the + % transaction headers `~copycat@1.0/arweave' fetches while indexing. + #{ + <<"store-module">> => hb_store_gateway, + <<"local-store">> => [TestStore] + } + ], + <<"arweave-index-store">> => #{ <<"index-store">> => [IndexStore] }, + <<"arweave-index-workers">> => 8, + <<"arweave-scheduler-confirmation-depth">> => 1, + <<"arweave-scheduler-max-height">> => ?FIXTURE_MAX_HEIGHT, + <<"priv-wallet">> => ar_wallet:new() + }. + +%% @doc Scheduling a message that does not carry a `tx@1.0' commitment must +%% be rejected. +reject_non_tx_message_test() -> + Opts = #{ <<"priv-wallet">> => ar_wallet:new() }, + Msg = + hb_message:commit( + #{ + <<"target">> => hb_util:human_id(crypto:strong_rand_bytes(32)), + <<"test-key">> => <<"test-value">> + }, + Opts + ), + ?assertMatch( + {error, #{ <<"status">> := 422 }}, + hb_ao:resolve( + #{ <<"device">> => <<"arweave-scheduler@1.0">> }, + #{ + <<"path">> => <<"schedule">>, + <<"method">> => <<"POST">>, + <<"body">> => Msg + }, + Opts + ) + ). + +%% @doc A validly `tx@1.0'-committed message that carries data must also be +%% rejected: this scheduler sequences transaction headers only, so it will not +%% dispatch (nor later depend on the chunk availability of) a message's data. +reject_data_message_test() -> + Opts = #{ <<"priv-wallet">> => ar_wallet:new() }, + Msg = + hb_message:commit( + #{ + <<"target">> => hb_util:human_id(crypto:strong_rand_bytes(32)), + <<"data">> => <<"unschedulable-data">> + }, + Opts, + #{ <<"commitment-device">> => <<"tx@1.0">> } + ), + ?assertMatch( + {error, #{ <<"status">> := 422 }}, + hb_ao:resolve( + #{ <<"device">> => <<"arweave-scheduler@1.0">> }, + #{ + <<"path">> => <<"schedule">>, + <<"method">> => <<"POST">>, + <<"body">> => Msg + }, + Opts + ) + ). + +%% @doc Base-layer annotation keeps only tx@1.0 offsets and sorts by offset. +base_layer_offsets_test() -> + Store = hb_test_utils:test_store(), + hb_store:start(Store), + ArwStore = #{ <<"index-store">> => [Store] }, + Opts = #{ <<"arweave-index-store">> => ArwStore }, + Late = hb_util:human_id(crypto:strong_rand_bytes(32)), + Early = hb_util:human_id(crypto:strong_rand_bytes(32)), + Bundled = hb_util:human_id(crypto:strong_rand_bytes(32)), + Unindexed = hb_util:human_id(crypto:strong_rand_bytes(32)), + ok = hb_store_arweave:write_offset(ArwStore, Late, <<"tx@1.0">>, 200, 1), + ok = hb_store_arweave:write_offset(ArwStore, Early, <<"tx@1.0">>, 100, 1), + ok = hb_store_arweave:write_offset(ArwStore, Bundled, <<"ans104@1.0">>, 150, 1), + ?assertEqual( + [{100, Early}, {200, Late}], + base_layer_offsets([Late, Bundled, Early, Unindexed], Opts) + ). + +%% @doc `/status' reports each tracked process's contiguously-indexed block +%% range straight from the cache, without triggering a synchronization. +status_attestable_range_test() -> + Store = hb_test_utils:test_store(), + hb_store:start(Store), + Opts = #{ <<"store">> => [Store], <<"priv-wallet">> => ar_wallet:new() }, + ProcID = hb_util:human_id(crypto:strong_rand_bytes(32)), + ok = + dev_arweave_scheduler_cache:write_state( + ProcID, + #{ + <<"next-slot">> => 4, + <<"spawn-height">> => 100, + <<"synced-to">> => 150 + }, + Opts + ), + {ok, #{ <<"processes">> := Processes }} = status(#{}, #{}, Opts), + ?assertEqual( + #{ <<"from">> => 100, <<"to">> => 150, <<"current">> => 3 }, + hb_maps:get(ProcID, Processes, not_found, Opts) + ). + +%% @doc Synchronize the fixture schedule from the live network, retrying on +%% transient indexing failures (for example, gateway rate limits while +%% fetching block transaction headers). +fixture_sync(_Opts, 0) -> {error, fixture_sync_failed}; +fixture_sync(Opts, Attempts) -> + Res = + hb_ao:resolve( + #{ <<"device">> => <<"arweave-scheduler@1.0">> }, + #{ + <<"path">> => <<"schedule">>, + <<"method">> => <<"GET">>, + <<"target">> => ?FIXTURE_PROCESS + }, + Opts + ), + case Res of + {ok, Schedule} -> {ok, Schedule}; + {error, _} -> + timer:sleep(5000), + fixture_sync(Opts, Attempts - 1) + end. + +%% @doc Read the fixture schedule from the live network and check that the +%% assignments arrive in seeded order, including the message that never +%% passed through a HyperBEAM node. +fixture_schedule_test_() -> + {timeout, 1200, fun fixture_schedule/0}. +fixture_schedule() -> + Opts = test_opts(), + {ok, Schedule} = fixture_sync(Opts, 5), + Assignments = + hb_ao:normalize_keys( + hb_ao:get(<<"assignments">>, Schedule, Opts), + Opts + ), + SlotIDs = + lists:map( + fun(Slot) -> + Assignment = + hb_maps:get(hb_util:bin(Slot), Assignments, not_found, Opts), + ?assertEqual( + Slot, + hb_util:int(hb_ao:get(<<"slot">>, Assignment, Opts)) + ), + Body = hb_ao:get(<<"body">>, Assignment, Opts), + % The body is a header: it verifies and its committed ID is + % the transaction ID, but it carries no data payload -- the + % schedule never depends on data availability. + ?assert(hb_message:verify(Body, all, Opts)), + BodyKeys = hb_maps:keys(hb_message:uncommitted(Body, Opts), Opts), + ?assertNot(lists:member(<<"data">>, BodyKeys)), + ?assertNot(lists:member(<<"body">>, BodyKeys)), + hb_message:id(Body, signed, Opts) + end, + [0, 1, 2, 3] + ), + ?assertEqual( + [?FIXTURE_PROCESS, ?FIXTURE_MSG1, ?FIXTURE_MSG2, ?FIXTURE_MSG3], + SlotIDs + ). + +%% @doc Compute the fixture process from its Arweave schedule and check the +%% state transformations applied in order: `setstate 1000', `addstate 337', +%% then `querystate' reporting the result. +fixture_lua_e2e_test_() -> + {timeout, 1200, fun fixture_lua_e2e/0}. +fixture_lua_e2e() -> + Opts = test_opts(), + % Prime the schedule first: synchronization indexes the spawn block, so + % the process message is read back as its canonical tx@1.0 decoding + % (rather than the gateway store's lossy representation). + {ok, _} = fixture_sync(Opts, 5), + {ok, RawProcess} = hb_cache:read(?FIXTURE_PROCESS, Opts), + Process = hb_cache:ensure_all_loaded(RawProcess, Opts), + % Luerl represents Lua numbers as floats. + ?assertEqual( + {ok, 1000.0}, + hb_ao:resolve( + Process, + #{ <<"path">> => <<"compute/state">>, <<"slot">> => 1 }, + Opts + ) + ), + ?assertEqual( + {ok, 1337.0}, + hb_ao:resolve( + Process, + #{ <<"path">> => <<"compute/state">>, <<"slot">> => 2 }, + Opts + ) + ), + ?assertEqual( + {ok, <<"state=1337.0">>}, + hb_ao:resolve( + Process, + #{ + <<"path">> => <<"compute/results/output/body">>, + <<"slot">> => 3 + }, + Opts + ) + ), + ?assertEqual({ok, 1337.0}, hb_ao:resolve(Process, <<"now/state">>, Opts)). + +%% @doc A first sync indexes the range in bounded chunks, persisting its +%% progress after each. With a chunk size smaller than the fixture's span -- +%% so the messages fall in different chunks -- the schedule is still assembled +%% in order, and `/status' reports the range synced all the way to the tip. +chunked_sync_test_() -> + {timeout, 1200, fun chunked_sync/0}. +chunked_sync() -> + Opts = (test_opts())#{ <<"arweave-scheduler-sync-chunk">> => 4 }, + {ok, Schedule} = fixture_sync(Opts, 5), + Assignments = + hb_ao:normalize_keys( + hb_ao:get(<<"assignments">>, Schedule, Opts), + Opts + ), + SlotIDs = + lists:map( + fun(Slot) -> + Assignment = + hb_maps:get(hb_util:bin(Slot), Assignments, not_found, Opts), + hb_message:id(hb_ao:get(<<"body">>, Assignment, Opts), signed, Opts) + end, + [0, 1, 2, 3] + ), + ?assertEqual( + [?FIXTURE_PROCESS, ?FIXTURE_MSG1, ?FIXTURE_MSG2, ?FIXTURE_MSG3], + SlotIDs + ), + % A completed sync makes the whole fixture range attestable: the spawn + % block through the tip. + {ok, #{ <<"processes">> := Processes }} = status(#{}, #{}, Opts), + Range = hb_maps:get(?FIXTURE_PROCESS, Processes, not_found, Opts), + ?assertEqual(?FIXTURE_MAX_HEIGHT, hb_util:int(hb_ao:get(<<"to">>, Range, Opts))), + ?assertEqual(3, hb_util:int(hb_ao:get(<<"current">>, Range, Opts))). diff --git a/src/preloaded/process/dev_arweave_scheduler_cache.erl b/src/preloaded/process/dev_arweave_scheduler_cache.erl new file mode 100644 index 0000000000..a8af374249 --- /dev/null +++ b/src/preloaded/process/dev_arweave_scheduler_cache.erl @@ -0,0 +1,111 @@ +%%% @doc A cache for the `~arweave-scheduler@1.0' device: stores the +%%% synthetic assignments derived from Arweave L1 transactions, as well as +%%% the per-process synchronization state of the indexer. +-module(dev_arweave_scheduler_cache). +-export([write/2, read/3, list_processes/1]). +-export([read_state/2, write_state/3]). +-export([assignments_to_bundle/4]). +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +%%% The pseudo-path prefix which the arweave-scheduler cache should use. +-define(CACHE_PREFIX, <<"~arweave-scheduler@1.0">>). + +%% @doc Merge the scheduler store with the main store. Used before writing +%% to the cache. +opts(Opts) -> lib_scheduler:cache_opts(Opts). + +%% @doc Write an assignment message into the cache. Assignments are +%% deterministic derivations of chain data, so writes are idempotent: +%% concurrent synchronizations of the same process converge on identical +%% messages at identical paths. +write(Assignment, Opts) -> + lib_scheduler:write_assignment(?CACHE_PREFIX, Assignment, Opts). + +%% @doc Get an assignment message from the cache. +read(ProcID, Slot, Opts) -> + lib_scheduler:read_assignment(?CACHE_PREFIX, ProcID, Slot, Opts). + +%% @doc List the processes that the device holds synchronization state for. +list_processes(RawOpts) -> + Opts = opts(RawOpts), + Store = hb_opts:get(store, no_viable_store, Opts), + case hb_store:list(Store, <>, Opts) of + {ok, Names} -> Names; + _ -> [] + end. + +%% @doc Read the persisted synchronization state for a process. Returns +%% `{ok, State}' with the `next-slot' to be assigned, the process's +%% `spawn-height', and `synced-to' (the highest block whose messages have been +%% contiguously indexed and materialized), or propagates the store's +%% `not_found'. +read_state(ProcID, RawOpts) -> + Opts = opts(RawOpts), + Store = hb_opts:get(store, no_viable_store, Opts), + maybe + {ok, NextSlot} ?= + hb_store:read(Store, state_path(ProcID, <<"next-slot">>), Opts), + {ok, SpawnHeight} ?= + hb_store:read(Store, state_path(ProcID, <<"spawn-height">>), Opts), + {ok, SyncedTo} ?= + hb_store:read(Store, state_path(ProcID, <<"synced-to">>), Opts), + {ok, + #{ + <<"next-slot">> => hb_util:int(NextSlot), + <<"spawn-height">> => hb_util:int(SpawnHeight), + <<"synced-to">> => hb_util:int(SyncedTo) + } + } + end. + +%% @doc Persist the synchronization state for a process. +write_state(ProcID, State, RawOpts) -> + Opts = opts(RawOpts), + Store = hb_opts:get(store, no_viable_store, Opts), + #{ + <<"next-slot">> := NextSlot, + <<"spawn-height">> := SpawnHeight, + <<"synced-to">> := SyncedTo + } = State, + hb_store:write( + Store, + #{ + state_path(ProcID, <<"next-slot">>) => hb_util:bin(NextSlot), + state_path(ProcID, <<"spawn-height">>) => hb_util:bin(SpawnHeight), + state_path(ProcID, <<"synced-to">>) => hb_util:bin(SyncedTo) + }, + Opts + ). + +state_path(ProcID, Key) -> + hb_path:to_binary( + [ + ?CACHE_PREFIX, + <<"state">>, + hb_util:human_id(ProcID), + Key + ] + ). + +%% @doc Generate a `GET /schedule' response for a process. Mirrors +%% `dev_scheduler_formats:assignments_to_bundle/4' (which cannot be called +%% across device namespaces), less the bundle-level `timestamp', +%% `block-height' and `block-hash'. Those describe the current weave tip +%% rather than the blocks that sequenced these assignments, so they would be +%% misleading on a schedule that is a deterministic read of historical chain +%% data. The on-chain position of each message is its assignment's `offset'. +assignments_to_bundle(ProcID, Assignments, More, RawOpts) -> + Opts = lib_scheduler:format_opts(RawOpts), + {ok, #{ + <<"type">> => <<"schedule">>, + <<"process">> => hb_util:human_id(ProcID), + <<"continues">> => hb_util:atom(More), + <<"assignments">> => + hb_message:normalize_commitments( + hb_maps:from_list( + [ {hb_ao:get(<<"slot">>, A, Opts), A} || A <- Assignments ] + ), + Opts + ) + }}. diff --git a/test/arweave-scheduler-test.lua b/test/arweave-scheduler-test.lua new file mode 100644 index 0000000000..ec816ccb47 --- /dev/null +++ b/test/arweave-scheduler-test.lua @@ -0,0 +1,34 @@ +--- @module arweave-scheduler-test +--- State-transformation test module for the `~arweave-scheduler@1.0` device. +--- Messages scheduled as Arweave L1 transactions invoke these functions via +--- their `path` tag. The function names deliberately avoid the keys that +--- `~lua@5.3a` excludes from its default handler (`set`, `path`, ...) and +--- the keys present on the process message itself. + +--- Executed for assignments without a `path` tag -- most notably slot 0, +--- the process message itself. +function compute(base, req, opts) + return base +end + +--- Overwrite the process state with the `value` tag of the message. +function setstate(base, req, opts) + base.state = tonumber(req.body.value) + return base +end + +--- Add the `value` tag of the message to the process state. +function addstate(base, req, opts) + base.state = (base.state or 0) + tonumber(req.body.value) + return base +end + +--- Report the process state in the results of the slot. +function querystate(base, req, opts) + base.results = { + output = { + body = "state=" .. tostring(base.state) + } + } + return base +end From df13f2d27868ca9092b6127d9128b7e48d11cd8e Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Fri, 24 Jul 2026 23:00:09 -0400 Subject: [PATCH 08/27] feat(arweave-scheduler): sequence a process by every L1 transaction A process message may now carry `scheduler-mode: all', and its schedule becomes every base-layer transaction on the network rather than only those whose `target' is the process. This is what lets a process observe value moving between two other addresses -- a payment it is owed but is not a party to -- which no recipient-filtered schedule can see. The mode is read from the process message itself at first contact and pinned into the persisted sync state. The process message is an L1 transaction, so the mode is chain data like the rest of the schedule, and every node derives the same one; a node option would have made slot -> body a function of node config, producing two indistinguishable, equally `valid' schedules for one process id. An unrecognized mode falls back to `target' rather than erroring: a spawn tag cannot be corrected once it is on-chain. `all' mode does not query. Dropping `recipients:' from the GraphQL would have silently returned nothing -- `dev_query_arweave' only builds candidate sets from height/id/ids/tags/owners/recipients, and `block:' is a post-filter over a set that must already exist -- so the range's block headers are walked directly instead, taking each block's own `txs' list. That is the same source `~copycat@1.0/arweave' enumerates, and `ensure_offsets' has already cached those blocks, so it is a local read. Nor does it re-sort by weave offset. Offset is not a total order: `TXStartOffset = TXEndOffset - data_size' and the accumulator only advances by `data_size', so every data-free transaction in a block shares the offset of the one before it -- and a data-free header is exactly what an AO message is. Block order (heights ascending, then each block's own transaction order) is a total order and is already canonical. The process's own transaction is skipped, as the first sync deliberately re-scans the spawn block and slot 0 is already the process. Assignments in this mode also record the `block-height' that sequenced them, so a process reading the whole chain has a deterministic clock; until now an assignment carried only its offset, and a block-denominated deadline was inexpressible. `target'-mode assignments are untouched, so existing schedules keep their assignment ids. Two things this mode cannot do the way `target' mode does. It must not drop a transaction the block lists but the local index does not hold: indexing a block is all-or-nothing and reports success either way, so dropping them would silently shorten the schedule and -- slots being positional -- shift every later slot on that node alone, with both nodes serving 200s and claiming the same attestable range. The range fails instead, leaving `synced-to' for the next pass to retry. And it sequences bundle transactions, whose headers the shallow indexing pass deliberately does not cache, so a header that is not held locally is fetched data-free on demand rather than stopping the sync. Tested: mode selection and its fallback, slot-0 detail per mode, block enumeration order and process-tx exclusion, offset annotation without re-sorting, failing closed on an unindexed transaction, the assignment cache round trip carrying block-height, and the mode's round trip through the sync state. The existing live-weave fixture tests still pass unchanged. --- .../process/dev_arweave_scheduler.erl | 363 ++++++++++++++++-- .../process/dev_arweave_scheduler_cache.erl | 19 +- 2 files changed, 354 insertions(+), 28 deletions(-) diff --git a/src/preloaded/process/dev_arweave_scheduler.erl b/src/preloaded/process/dev_arweave_scheduler.erl index 37a32ea45b..3977ef3a0f 100644 --- a/src/preloaded/process/dev_arweave_scheduler.erl +++ b/src/preloaded/process/dev_arweave_scheduler.erl @@ -27,6 +27,26 @@ %%% the device's own schedule cache. %%% %%% +%%% A process message may widen what it is sequenced by, with a +%%% `scheduler-mode' key: +%%%
    +%%%
  • `target' (the default): the process's messages are the transactions +%%% addressed to it, as above.
  • +%%%
  • `all': every base-layer transaction is a message in the +%%% process's schedule, whoever it is addressed to. This is what lets a +%%% process observe value moving between two other addresses -- a +%%% payment it is owed but is not a party to -- at the cost of a slot +%%% for every transaction on the network. Its schedule is enumerated +%%% from the block headers rather than by query, in canonical chain +%%% order (blocks ascending, then each block's own transaction order), +%%% and each assignment records the `block-height' that sequenced it, so +%%% a process in this mode has a clock. The process's own transaction is +%%% not re-assigned: it remains slot 0 alone.
  • +%%%
+%%% The mode is read from the process message itself -- an L1 transaction -- +%%% so it is chain data like the rest of the schedule, and is fixed for the +%%% life of the process. +%%% %%% Assignment bodies are transaction headers only: each is the %%% data-free header that `~copycat@1.0/arweave' cached while indexing the %%% range (read from the node's stores), so the schedule never carries (nor @@ -93,6 +113,9 @@ -define(QUERY_PAGE_SIZE, 100). %%% The Arweave device that we resolve chain data through. -define(ARWEAVE_DEVICE, <<"~arweave@2.9">>). +%%% The sequencing mode of a process whose message does not name one: only the +%%% transactions addressed to it are its messages. +-define(DEFAULT_MODE, <<"target">>). %% @doc This device uses a default handler to route requests to the correct %% function. @@ -364,19 +387,21 @@ sync(ProcID, Opts) -> do_sync(_ProcID, State = #{ <<"synced-to">> := SyncedTo }, Upper, _Opts) when SyncedTo >= Upper -> {ok, State}; -do_sync(ProcID, State = #{ <<"synced-to">> := SyncedTo }, Upper, Opts) -> +do_sync(ProcID, State = #{ <<"synced-to">> := SyncedTo, <<"mode">> := Mode }, + Upper, Opts) -> From = SyncedTo + 1, To = min(SyncedTo + sync_chunk_blocks(Opts), Upper), ?event( {arweave_scheduler_sync, {proc_id, ProcID}, + {mode, Mode}, {from, From}, {to, To}, {target, Upper} } ), maybe - {ok, Ordered} ?= discover(ProcID, From, To, Opts), + {ok, Ordered} ?= discover(ProcID, Mode, From, To, Opts), {ok, NewState} ?= materialize(ProcID, State, Ordered, To, Opts), do_sync(ProcID, NewState, Upper, Opts) end. @@ -397,21 +422,68 @@ no_result_cache(Opts) -> <<"cache-control">> => [<<"no-cache">>, <<"no-store">>] }. -%% @doc Discover the base-layer transactions addressed to a process within a -%% block range, in ascending weave order, entirely from the node's own index. -%% Indexing the range with `~copycat@1.0/arweave' both records each +%% @doc Discover the base-layer transactions that a process is sequenced by +%% within a block range, in canonical weave order, entirely from the node's own +%% index. Indexing the range with `~copycat@1.0/arweave' both records each %% transaction's weave offset and caches its header (so its `target' is locally -%% matchable); the recipient match is then served from the node's own +%% matchable), whichever mode the process is in. +%% +%% In `target' mode the recipient match is served from the node's own %% `~query@1.0' GraphQL endpoint, and each match is annotated with its offset -- %% the sort key and the `offset' recorded on the assignment. No gateway is -%% queried by default. Returns `{Offset, TXID}' pairs in ascending offset order; -%% bundled data items are excluded, as this scheduler sequences the base layer -%% only. -discover(ProcID, From, To, Opts) -> +%% queried by default. In `all' mode there is nothing to match, so no query is +%% run: the range's block headers are walked directly (see +%% `enumerate_blocks/4'). +%% +%% Returns `{Extra, TXID}' pairs in the order they are to be assigned, where +%% `Extra' is the sequencing detail recorded on each assignment. Bundled data +%% items are excluded, as this scheduler sequences the base layer only. +discover(ProcID, <<"all">>, From, To, Opts) -> + maybe + ok ?= ensure_offsets(From, To, Opts), + {ok, Located} ?= enumerate_blocks(ProcID, From, To, Opts), + base_layer_blocks(Located, Opts) + end; +discover(ProcID, _Mode, From, To, Opts) -> maybe ok ?= ensure_offsets(From, To, Opts), {ok, IDs} ?= query_recipients(ProcID, From, To, Opts), - {ok, base_layer_offsets(IDs, Opts)} + {ok, + [ + {#{ <<"offset">> => Offset }, TXID} + || + {Offset, TXID} <- base_layer_offsets(IDs, Opts) + ] + } + end. + +%% @doc Enumerate every transaction in a block range from the block headers +%% themselves, in canonical chain order: blocks ascending by height, then each +%% block's transactions in the order that block lists them. This is the same +%% source `~copycat@1.0/arweave' walks while indexing, and `ensure_offsets' has +%% already cached the blocks locally, so the enumeration is normally a local +%% read. The process's own transaction is skipped: it is already slot 0. +%% +%% Returns `{Height, TXID}' pairs. Unlike a query result these are ordered by +%% construction, and that order is total -- weave offset is not, because every +%% transaction that carries no data shares the offset of the one before it. +enumerate_blocks(_ProcID, From, To, _Opts) when From > To -> {ok, []}; +enumerate_blocks(ProcID, From, To, Opts) -> + maybe + {ok, Block} ?= + hb_ao:resolve( + <>, + no_result_cache(Opts) + ), + {ok, Rest} ?= enumerate_blocks(ProcID, From + 1, To, Opts), + {ok, + [ + {From, TXID} + || + TXID <- hb_maps:get(<<"txs">>, Block, [], Opts), + TXID =/= hb_util:human_id(ProcID) + ] ++ Rest + } end. %% @doc Ensure the node's local Arweave index covers the block range, so that @@ -570,6 +642,60 @@ base_layer_offset(Store, ID, Opts) -> _ -> false end. +%% @doc Annotate each block-enumerated transaction with its weave offset, +%% keeping only the base-layer ones. The enumeration is already in canonical +%% order, so -- unlike `base_layer_offsets/2' -- it is not re-sorted. Each +%% assignment records the height of the block that sequenced it as well as its +%% offset: in this mode the schedule follows the chain rather than the process, +%% so the block height is the only clock a process has. +%% +%% A transaction the block lists but the local index does not hold is a failure +%% to index, not an absence -- `~copycat@1.0/arweave' skips a whole block if any +%% one of its transaction headers could not be fetched, while still reporting +%% success. Dropping those would silently shorten the schedule and, because +%% slots are positional, shift every later slot on this node alone. So the range +%% fails here instead, leaving `synced-to' where it was for the next pass to +%% retry. +base_layer_blocks(Located, Opts) -> + Store = hb_store_arweave:store_from_opts(Opts), + lists:foldr( + fun(_, {error, Err}) -> {error, Err}; + ({Height, ID}, {ok, Acc}) -> + case offset_entry(Store, ID, Opts) of + {ok, <<"tx@1.0">>, Offset} -> + {ok, + [ + { + #{ + <<"offset">> => Offset, + <<"block-height">> => Height + }, + ID + } + | + Acc + ] + }; + {ok, _Codec, _Offset} -> + % A bundled data item: this scheduler sequences the base + % layer only. + {ok, Acc}; + not_found -> + {error, + #{ + <<"status">> => 503, + <<"reason">> => + <<"Block range is not fully indexed locally.">>, + <<"tx">> => ID, + <<"block-height">> => Height + } + } + end + end, + {ok, []}, + Located + ). + %% @doc Read a transaction's local index entry, returning its codec device and %% weave offset. offset_entry(Store, ID, Opts) -> @@ -595,10 +721,10 @@ assign(ProcID, State, Slot, [], To, Opts) -> NewState = State#{ <<"next-slot">> => Slot, <<"synced-to">> => To }, ok = dev_arweave_scheduler_cache:write_state(ProcID, NewState, Opts), {ok, NewState}; -assign(ProcID, State, Slot, [{Offset, TXID} | Rest], To, Opts) -> +assign(ProcID, State, Slot, [{Extra, TXID} | Rest], To, Opts) -> case read_tx_header(TXID, Opts) of {ok, Msg} -> - ok = write_assignment(ProcID, Slot, Offset, Msg, Opts), + ok = write_assignment(ProcID, Slot, Extra, Msg, Opts), assign(ProcID, State, Slot + 1, Rest, To, Opts); {error, Err} -> ?event( @@ -613,10 +739,11 @@ assign(ProcID, State, Slot, [{Offset, TXID} | Rest], To, Opts) -> %% @doc Generate and store the synthetic assignment for a message. Mirrors the %% assignments minted by `~scheduler@1.0', but the on-chain position is the -%% transaction's weave `offset' rather than a scheduler-assigned nonce. Every -%% field derives from chain data, so the assignment is deterministic across -%% nodes and is left uncommitted. -write_assignment(ProcID, Slot, Offset, Msg, Opts) -> +%% transaction's weave `offset' rather than a scheduler-assigned nonce (joined, +%% in `all' mode, by the `block-height' that sequenced it). Every field derives +%% from chain data, so the assignment is deterministic across nodes and is left +%% uncommitted. +write_assignment(ProcID, Slot, Extra, Msg, Opts) -> BaseAssignment = lib_scheduler:base_assignment( hb_util:human_id(ProcID), @@ -624,12 +751,12 @@ write_assignment(ProcID, Slot, Offset, Msg, Opts) -> Msg, Opts ), - Assignment = BaseAssignment#{ <<"offset">> => Offset }, + Assignment = hb_maps:merge(BaseAssignment, Extra, Opts), ?event( {minting_assignment, {proc_id, ProcID}, {slot, Slot}, - {offset, Offset} + {extra, Extra} } ), dev_arweave_scheduler_cache:write(Assignment, Opts). @@ -655,7 +782,15 @@ initialize(ProcID, Opts) -> {ok, Offset} ?= tx_offset(ProcID, Opts), {ok, Process} ?= read_tx_header(ProcID, Opts), {ok, _} = hb_cache:write(Process, Opts), - ok = write_assignment(ProcID, 0, Offset, Process, Opts), + Mode = mode(Process, Opts), + ok = + write_assignment( + ProcID, + 0, + slot_zero(Mode, Offset, SpawnHeight), + Process, + Opts + ), % `synced-to' starts one below the spawn block: slot 0 is the process % itself, and no message-bearing block has been indexed yet. The first % sync begins its range at the spawn block, catching any messages mined @@ -664,12 +799,31 @@ initialize(ProcID, Opts) -> #{ <<"next-slot">> => 1, <<"spawn-height">> => SpawnHeight, - <<"synced-to">> => SpawnHeight - 1 + <<"synced-to">> => SpawnHeight - 1, + <<"mode">> => Mode }, ok = dev_arweave_scheduler_cache:write_state(ProcID, State, Opts), {ok, State} end. +%% @doc The sequencing mode a process message asks for. A mode the device does +%% not implement falls back to `target' rather than erroring: the process +%% message is immutable, so a typo in a spawn tag would otherwise wedge the +%% process permanently. +mode(Process, Opts) -> + case hb_ao:get(<<"scheduler-mode">>, Process, ?DEFAULT_MODE, Opts) of + <<"all">> -> <<"all">>; + _ -> ?DEFAULT_MODE + end. + +%% @doc The sequencing detail recorded on slot 0. The process message is its +%% own first message, so in `all' mode it carries the height of its spawn block +%% -- the process's clock starts at the block it was created in. +slot_zero(<<"all">>, Offset, SpawnHeight) -> + #{ <<"offset">> => Offset, <<"block-height">> => SpawnHeight }; +slot_zero(_Mode, Offset, _SpawnHeight) -> + #{ <<"offset">> => Offset }. + %% @doc Read an L1 transaction as a header-only message from the node's stores. %% `~copycat@1.0/arweave' caches the (data-free) header locally while indexing, %% so the read is normally served straight from the local store; a miss falls @@ -680,6 +834,27 @@ initialize(ProcID, Opts) -> %% the availability of any transaction's data. read_tx_header(TXID, Opts) -> case hb_cache:read(TXID, Opts) of + {ok, Msg} -> {ok, hb_cache:ensure_all_loaded(Msg, Opts)}; + _ -> fetch_tx_header(TXID, Opts) + end. + +%% @doc Fetch a transaction's data-free header from Arweave. The indexing pass +%% caches the headers of plain L1 transactions as it goes, but deliberately not +%% those of bundles -- and an `all'-mode schedule sequences every transaction in +%% a block, bundles included. Their headers are fetched on demand rather than +%% failing the whole range; `exclude-data' keeps the schedule header-only, so it +%% still never depends on the availability of a transaction's data. +fetch_tx_header(TXID, Opts) -> + Res = + hb_ao:resolve( + << + ?ARWEAVE_DEVICE/binary, "/tx", + "&tx=", TXID/binary, + "&exclude-data=true" + >>, + no_result_cache(Opts) + ), + case Res of {ok, Msg} -> {ok, hb_cache:ensure_all_loaded(Msg, Opts)}; _ -> {error, @@ -880,6 +1055,149 @@ base_layer_offsets_test() -> base_layer_offsets([Late, Bundled, Early, Unindexed], Opts) ). +%% @doc The sequencing mode is read from the process message, and anything the +%% device does not implement leaves the process sequenced as normal rather than +%% wedging it: the process message cannot be corrected once it is on-chain. +mode_test() -> + ?assertEqual(<<"all">>, mode(#{ <<"scheduler-mode">> => <<"all">> }, #{})), + ?assertEqual(?DEFAULT_MODE, mode(#{}, #{})), + ?assertEqual( + ?DEFAULT_MODE, + mode(#{ <<"scheduler-mode">> => <<"sideways">> }, #{}) + ). + +%% @doc A process sequenced by the whole chain has a clock from slot 0; one +%% sequenced by its own messages keeps the assignment shape it always had. +slot_zero_test() -> + ?assertEqual( + #{ <<"offset">> => 7, <<"block-height">> => 3 }, + slot_zero(<<"all">>, 7, 3) + ), + ?assertEqual(#{ <<"offset">> => 7 }, slot_zero(?DEFAULT_MODE, 7, 3)). + +%% @doc Write a block into the node's block cache at the height pseudo-path +%% `~copycat@1.0/arweave' caches it under, so the enumerator reads it locally +%% rather than from the network. +write_test_block(Height, TXs, Opts) -> + {ok, MsgID} = + hb_cache:write(#{ <<"height">> => Height, <<"txs">> => TXs }, Opts), + hb_cache:link( + MsgID, + hb_path:to_binary( + [?ARWEAVE_DEVICE, <<"block">>, <<"height">>, hb_util:bin(Height)] + ), + Opts + ), + ok. + +%% @doc In `all' mode the schedule is enumerated from the block headers, in +%% chain order, and the process's own transaction is not re-assigned: it is +%% already slot 0. +enumerate_blocks_test() -> + Store = hb_test_utils:test_store(), + hb_store:start(Store), + Opts = #{ <<"store">> => [Store] }, + ProcID = hb_util:human_id(crypto:strong_rand_bytes(32)), + First = hb_util:human_id(crypto:strong_rand_bytes(32)), + Second = hb_util:human_id(crypto:strong_rand_bytes(32)), + Third = hb_util:human_id(crypto:strong_rand_bytes(32)), + ok = write_test_block(10, [First, ProcID], Opts), + ok = write_test_block(11, [Second, Third], Opts), + ?assertEqual( + {ok, [{10, First}, {11, Second}, {11, Third}]}, + enumerate_blocks(ProcID, 10, 11, Opts) + ). + +%% @doc Block-enumerated transactions keep chain order rather than being sorted +%% by offset -- offsets tie for every transaction that carries no data, which +%% is what an AO message is -- and each records the height that sequenced it. +%% Bundled data items are still dropped. +base_layer_blocks_test() -> + Store = hb_test_utils:test_store(), + hb_store:start(Store), + ArwStore = #{ <<"index-store">> => [Store] }, + Opts = #{ <<"arweave-index-store">> => ArwStore }, + Early = hb_util:human_id(crypto:strong_rand_bytes(32)), + Late = hb_util:human_id(crypto:strong_rand_bytes(32)), + Bundled = hb_util:human_id(crypto:strong_rand_bytes(32)), + ok = hb_store_arweave:write_offset(ArwStore, Late, <<"tx@1.0">>, 100, 0), + ok = hb_store_arweave:write_offset(ArwStore, Bundled, <<"ans104@1.0">>, 150, 0), + ok = hb_store_arweave:write_offset(ArwStore, Early, <<"tx@1.0">>, 200, 0), + ?assertEqual( + {ok, + [ + {#{ <<"offset">> => 100, <<"block-height">> => 10 }, Late}, + {#{ <<"offset">> => 200, <<"block-height">> => 11 }, Early} + ] + }, + base_layer_blocks([{10, Late}, {11, Bundled}, {11, Early}], Opts) + ). + +%% @doc A transaction the block lists but the index does not hold means the +%% range was not fully indexed. Dropping it would shorten the schedule and, as +%% slots are positional, shift every later slot on this node alone -- so the +%% range fails and `synced-to' stays where it was. +base_layer_blocks_unindexed_test() -> + Store = hb_test_utils:test_store(), + hb_store:start(Store), + ArwStore = #{ <<"index-store">> => [Store] }, + Opts = #{ <<"arweave-index-store">> => ArwStore }, + Indexed = hb_util:human_id(crypto:strong_rand_bytes(32)), + Missing = hb_util:human_id(crypto:strong_rand_bytes(32)), + ok = hb_store_arweave:write_offset(ArwStore, Indexed, <<"tx@1.0">>, 100, 0), + ?assertMatch( + {error, #{ <<"status">> := 503 }}, + base_layer_blocks([{10, Indexed}, {10, Missing}], Opts) + ). + +%% @doc An `all'-mode assignment records the height that sequenced it as well +%% as its weave offset, and both survive the cache round trip that a process +%% reads its schedule back through. This is the contract +%% `~arweave-swap@1.0' reads its clock from. +all_mode_assignment_test() -> + Store = hb_test_utils:test_store(), + hb_store:start(Store), + Opts = #{ <<"store">> => [Store], <<"priv-wallet">> => ar_wallet:new() }, + ProcID = hb_util:human_id(crypto:strong_rand_bytes(32)), + Msg = + hb_message:commit( + #{ <<"target">> => ProcID }, + Opts, + #{ <<"commitment-device">> => <<"tx@1.0">> } + ), + ok = + write_assignment( + ProcID, + 1, + #{ <<"offset">> => 42, <<"block-height">> => 1958986 }, + Msg, + Opts + ), + {ok, Assignment} = dev_arweave_scheduler_cache:read(ProcID, 1, Opts), + ?assertEqual(1, hb_util:int(hb_ao:get(<<"slot">>, Assignment, Opts))), + ?assertEqual(42, hb_util:int(hb_ao:get(<<"offset">>, Assignment, Opts))), + ?assertEqual( + 1958986, + hb_util:int(hb_ao:get(<<"block-height">>, Assignment, Opts)) + ). + +%% @doc The mode is pinned in the persisted state, so a schedule can never be +%% half-derived in each mode. +state_mode_round_trip_test() -> + Store = hb_test_utils:test_store(), + hb_store:start(Store), + Opts = #{ <<"store">> => [Store] }, + ProcID = hb_util:human_id(crypto:strong_rand_bytes(32)), + State = + #{ + <<"next-slot">> => 1, + <<"spawn-height">> => 10, + <<"synced-to">> => 9, + <<"mode">> => <<"all">> + }, + ok = dev_arweave_scheduler_cache:write_state(ProcID, State, Opts), + ?assertEqual({ok, State}, dev_arweave_scheduler_cache:read_state(ProcID, Opts)). + %% @doc `/status' reports each tracked process's contiguously-indexed block %% range straight from the cache, without triggering a synchronization. status_attestable_range_test() -> @@ -893,7 +1211,8 @@ status_attestable_range_test() -> #{ <<"next-slot">> => 4, <<"spawn-height">> => 100, - <<"synced-to">> => 150 + <<"synced-to">> => 150, + <<"mode">> => ?DEFAULT_MODE }, Opts ), diff --git a/src/preloaded/process/dev_arweave_scheduler_cache.erl b/src/preloaded/process/dev_arweave_scheduler_cache.erl index a8af374249..e94e930848 100644 --- a/src/preloaded/process/dev_arweave_scheduler_cache.erl +++ b/src/preloaded/process/dev_arweave_scheduler_cache.erl @@ -37,9 +37,11 @@ list_processes(RawOpts) -> %% @doc Read the persisted synchronization state for a process. Returns %% `{ok, State}' with the `next-slot' to be assigned, the process's -%% `spawn-height', and `synced-to' (the highest block whose messages have been -%% contiguously indexed and materialized), or propagates the store's -%% `not_found'. +%% `spawn-height', `synced-to' (the highest block whose messages have been +%% contiguously indexed and materialized), and the `mode' the process is +%% sequenced in, or propagates the store's `not_found'. The mode is pinned here +%% at first contact: it is read from the process message, which cannot change, +%% and a schedule half-derived in each mode would be undetectable. read_state(ProcID, RawOpts) -> Opts = opts(RawOpts), Store = hb_opts:get(store, no_viable_store, Opts), @@ -50,11 +52,14 @@ read_state(ProcID, RawOpts) -> hb_store:read(Store, state_path(ProcID, <<"spawn-height">>), Opts), {ok, SyncedTo} ?= hb_store:read(Store, state_path(ProcID, <<"synced-to">>), Opts), + {ok, Mode} ?= + hb_store:read(Store, state_path(ProcID, <<"mode">>), Opts), {ok, #{ <<"next-slot">> => hb_util:int(NextSlot), <<"spawn-height">> => hb_util:int(SpawnHeight), - <<"synced-to">> => hb_util:int(SyncedTo) + <<"synced-to">> => hb_util:int(SyncedTo), + <<"mode">> => Mode } } end. @@ -66,14 +71,16 @@ write_state(ProcID, State, RawOpts) -> #{ <<"next-slot">> := NextSlot, <<"spawn-height">> := SpawnHeight, - <<"synced-to">> := SyncedTo + <<"synced-to">> := SyncedTo, + <<"mode">> := Mode } = State, hb_store:write( Store, #{ state_path(ProcID, <<"next-slot">>) => hb_util:bin(NextSlot), state_path(ProcID, <<"spawn-height">>) => hb_util:bin(SpawnHeight), - state_path(ProcID, <<"synced-to">>) => hb_util:bin(SyncedTo) + state_path(ProcID, <<"synced-to">>) => hb_util:bin(SyncedTo), + state_path(ProcID, <<"mode">>) => Mode }, Opts ). From bd4c8195d513895ec0eae9f24d10420f7a4d524b Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Sat, 25 Jul 2026 00:47:29 -0400 Subject: [PATCH 09/27] fix(arweave-scheduler): read a header without reaching for its data The device promises that assignment bodies are transaction headers only, so that a schedule never depends on the availability of anybody's data. Reading one through the whole store chain broke that promise: the read falls through to a gateway, which answers with the entire transaction -- its data, or, for a bundle, its items decoded into submessages whose contents this node does not hold -- and `lib_scheduler:write_assignment' then forces the assignment to load. The slot dies on a link to a chunk nobody fetched. In `target' mode this never came up, because the only messages in a schedule were the header-only ones addressed to the process. In `all' mode every transaction on the network is a message and most of them carry data, so it came up immediately: replaying a five-block range failed on a stranger's bundle. Headers are now read from the node's own stores, where the indexing pass has cached a data-free header for every plain transaction it walked, and anything else is fetched as a header in its own right with `exclude-data'. Loading whatever happened to be available locally would have been worse than either: two nodes would mint different assignments for the same transaction. Found by replaying a real sale off mainnet; the live-weave fixture tests for `target' mode are unchanged and still pass. --- .../process/dev_arweave_scheduler.erl | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/preloaded/process/dev_arweave_scheduler.erl b/src/preloaded/process/dev_arweave_scheduler.erl index 3977ef3a0f..93c5d0efb6 100644 --- a/src/preloaded/process/dev_arweave_scheduler.erl +++ b/src/preloaded/process/dev_arweave_scheduler.erl @@ -832,9 +832,22 @@ slot_zero(_Mode, Offset, _SpawnHeight) -> %% tx@1.0 signature -- so its committed ID is the transaction ID and it verifies %% -- and no data is attached at this layer, so the schedule never depends on %% the availability of any transaction's data. +%% The header is taken as it is read, links and all. Forcing it to load in full +%% would demand a transaction's data -- the very thing this schedule promises +%% never to depend on -- and any transaction on the network can be a message +%% here, so most of them carry some. Loading only what happens to be available +%% locally would be worse than either: two nodes would mint different +%% assignments for the same transaction. +%% Only the node's own stores are consulted, because only they are known to +%% hold what this schedule wants: the indexing pass caches a data-free header +%% for every plain transaction it walks. Letting the read fall through to a +%% gateway would answer with the whole transaction instead -- data, or a bundle +%% decoded into items whose contents are not on this node -- and the schedule +%% would then depend on the availability of data it promises never to touch. +%% Anything not cached locally is fetched as a header in its own right. read_tx_header(TXID, Opts) -> - case hb_cache:read(TXID, Opts) of - {ok, Msg} -> {ok, hb_cache:ensure_all_loaded(Msg, Opts)}; + case hb_cache:read(TXID, hb_store:scope(Opts, local)) of + {ok, Msg} -> {ok, Msg}; _ -> fetch_tx_header(TXID, Opts) end. @@ -855,7 +868,7 @@ fetch_tx_header(TXID, Opts) -> no_result_cache(Opts) ), case Res of - {ok, Msg} -> {ok, hb_cache:ensure_all_loaded(Msg, Opts)}; + {ok, Msg} -> {ok, Msg}; _ -> {error, #{ From 37256467b916f2875c55db091de7c9f99481d480 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Fri, 24 Jul 2026 23:00:33 -0400 Subject: [PATCH 10/27] feat: add ~arweave-swap@1.0, selling process tokens for native AR A process device that sells its own token for Arweave's. The two halves of the trade live in different places, and everything here follows from that asymmetry: the token half is in the process, where it can be held in escrow and paid out with certainty, and the AR half is not -- it moves directly between two user addresses, and no process can hold, redirect or refund it. So the process is sequenced by `~arweave-scheduler@1.0' in its `all' mode. The payment leg is an ordinary transfer to the seller, tagged with an `order-id'; the process is not addressed by it and sees it only because it is sequenced by every transaction on the network. The protocol is four messages. `make-offer' escrows the seller's `offer-quantity + deposit' out of the `balances' submessage that a token implementation keeps in the same base, recording an `asking' price in winston and a `deadline' in blocks. The offered amount cannot be called `quantity': that is a transaction's own value field, so the codec would carry it as winston of AR sent to the process -- an address with no key, which would destroy it. `register-interest' costs nothing and moves nothing, but makes the order exclusively one buyer's and freezes the seller out of cancelling it -- that window is the whole point, since without it a buyer races the seller's cancellation having already sent AR that nobody can claw back. `cancel-order' returns the goods. The payment settles the order, crediting the goods to the payer and returning the bond to the seller. The `deposit' is the seller's bond against a payment the protocol cannot honour, so it outlives the goods: cancelling or expiring an order releases them at once but holds the `deposit' until `deadline + swap-cancel-grace'. A payment landing in that window against goods that are gone or spoken for is paid the bond instead. A seller who strands nobody always gets it back. Neither the order's creator nor its recipient may pay it: paying oneself costs only a network fee, and would otherwise let a seller settle their own order, taking back the goods and the bond and leaving a real payment to arrive against an order already spent. Deadlines are read from the `block-height' the assignment carries, which is the device's only clock: reading the chain tip during a compute would be non-deterministic, and a slot result is cached forever. Expiry is swept lazily before each dispatch, guarded by a single `next-deadline' integer, because in this mode the device runs on the order of a thousand times per block and almost every slot is unrelated traffic. Being sequenced by the whole chain also means every message it computes is a stranger's, and a slot that cannot be computed can never be skipped: the schedule is immutable chain data, so one bad slot stops the process on every node forever. The device therefore exports no key list -- the key a slot resolves comes from the sender's own `path' tag, and a key outside an export list would answer `not_found' and fail the slot -- and excludes none either, since an excluded key is handed to `~message@1.0', whose `set' would let a passer-by write the process's own balances and whose `keys' would replace the state with a list of key names. `info' is arity 0 for the same reason: a device's `info' is always exported, so at arity 1 it would answer the `info' key and hand back its own map as the process state. Figures a stranger wrote are read with `hb_util:safe_int/1', because a tag like `deadline: tomorrow' would otherwise raise out of the enclosing `maybe', which catches mismatches and not exceptions. An `order-id' is looked up as a key of the order book rather than resolved as a path, or `/creator' would reach an address and be read as an order. And the state is read as a message, because during a compute it carries this device, and a plain read of one of its keys would resolve that key back through `compute'. Tested: whole trades played out against the device -- settlement, underpayment, payment to the wrong address, cancellation and bond retirement, cancellation by a stranger, reservation blocking cancellation, reservation exclusivity and lapse, expiry, a late payment taking the bond, the bond paying out only once, double settlement, a seller unable to buy or compensate their own order, unknown and hostile order ids, a non-numeric figure, an offer that misnames its amount `quantity', stray and reserved `path' tags, numbers that came back from the cache encoded as binaries, and unrelated traffic moving only the clock. --- src/preloaded/process/dev_arweave_swap.erl | 1253 ++++++++++++++++++++ 1 file changed, 1253 insertions(+) create mode 100644 src/preloaded/process/dev_arweave_swap.erl diff --git a/src/preloaded/process/dev_arweave_swap.erl b/src/preloaded/process/dev_arweave_swap.erl new file mode 100644 index 0000000000..3afbe8c547 --- /dev/null +++ b/src/preloaded/process/dev_arweave_swap.erl @@ -0,0 +1,1253 @@ +%%% @doc A device for selling a process's tokens for native Arweave value. +%%% +%%% The two halves of the trade live in different places, and that asymmetry +%%% is the whole design. The token half lives here: this device debits and +%%% credits the `balances' submessage that a `~token@1.0' implementation keeps +%%% in the same process's base, so it can hold tokens in escrow and pay them +%%% out with certainty. The Arweave half does not live here at all -- AR moves +%%% directly from buyer to seller as an ordinary layer-1 transfer, which no +%%% process can hold, redirect or refund. +%%% +%%% A process using this device is therefore sequenced by +%%% `~arweave-scheduler@1.0' in its `all' mode: every base-layer transaction +%%% becomes a slot, so a payment between two addresses that the process is not +%%% a party to is nonetheless something the process sees, and can settle +%%% against. Its process message reads: +%%%
+%%%     scheduler-device: arweave-scheduler@1.0
+%%%     scheduler-mode:   all
+%%%     execution-device: arweave-swap@1.0
+%%% 
+%%% +%%% The protocol is four messages: +%%%
    +%%%
  • `make-offer' (to the process, from the seller), carrying +%%% `offer-quantity' in token units, `asking' in winston, `deposit' in +%%% token units and a `deadline' block height. The seller's +%%% `offer-quantity + deposit' moves into escrow at once, so delivery is +%%% never in doubt: the goods are already held before any buyer commits +%%% anything. The offered amount cannot be called `quantity': that is a +%%% transaction's own value field, so the codec would carry it as winston +%%% of AR sent to the process -- an address with no key, which would +%%% destroy it.
  • +%%%
  • `register-interest' (to the process, from a buyer), naming an +%%% `order-id'. It moves no value. It buys exclusivity: for +%%% `swap-reservation-blocks' the order is that buyer's alone and the +%%% seller cannot cancel it. That window is what makes paying safe -- +%%% without it a buyer races the seller's cancellation, having already +%%% sent AR that nobody can claw back.
  • +%%%
  • `cancel-order' (to the process, from the seller), naming an +%%% `order-id'. Returns the goods, but not the deposit (see below).
  • +%%%
  • The payment itself: an ordinary transfer whose `target' is the +%%% order's `recipient', whose `quantity' is at least the `asking' +%%% winston, tagged with the `order-id'. This is the message the `all' +%%% mode exists to deliver.
  • +%%%
+%%% +%%% The `deposit' is the seller's bond against a payment the protocol cannot +%%% honour. Once AR has been sent it cannot be returned, so the only lever +%%% remaining is a token-denominated bond, and it outlives the goods: cancelling +%%% or expiring an order releases the goods immediately but holds the `deposit' +%%% until `deadline + swap-cancel-grace'. A payment that lands in that window +%%% against goods that are gone or spoken for is paid the deposit instead. A +%%% seller who strands nobody always gets it back. Neither the order's creator +%%% nor its recipient may pay it: a self-transfer costs only a network fee, and +%%% would otherwise let a seller settle their own order to escape the bond. +%%% Because any other payer may claim that bond, a seller should not post a +%%% deposit worth more than the price they are asking, or paying for it becomes +%%% a trade in itself. +%%% +%%% Deadlines are Arweave block heights, read from the `block-height' that +%%% `all'-mode assignments carry. That is the only clock the device has, and +%%% deliberately so: reading the chain tip during a compute would be +%%% non-deterministic, and `~process@1.0' caches every slot result forever. +-module(dev_arweave_swap). +-implements(<<"arweave-swap@1.0">>). +%%% AO-Core API functions: +-export([info/0, compute/3, set/3, keys/3]). +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +%%% The balances submessage this device settles in. Owned by the process's +%%% token implementation; the swap only moves value inside it. +-define(BALANCES, <<"balances">>). +%%% The number of blocks an order is reserved for by `register-interest'. +-define(DEFAULT_RESERVATION_BLOCKS, 10). +%%% The number of blocks after an order's deadline during which a payment may +%%% still be compensated from the deposit. +-define(DEFAULT_CANCEL_GRACE, 20). + +%% @doc Every state transition in this device is driven by the schedule, never +%% by a direct request, so every key routes to `compute'. +%% +%% The key a slot resolves is chosen by the scheduled transaction's own `path' +%% tag, and this process is sequenced by every transaction on Arweave: whatever +%% a stranger writes there must be applied like any other message. So there is +%% no `exports' list -- a key outside it would fall through to `~message@1.0', +%% answer `not_found', fail its slot and wedge the process permanently -- and no +%% `excludes' list either, since an excluded key is handed to `~message@1.0' +%% instead, whose `set' would let a passer-by write the process's own balances +%% and whose `keys' would replace the state with a list of key names. `set' and +%% `keys' are therefore implemented here. +%% +%% `info' is deliberately arity 0. A device's `info' is always exported, and had +%% it taken the base as an argument it would also answer the `info' key, so a +%% transaction tagged `path: info' would replace the process state with this +%% map. +info() -> + #{ default => fun router/4 }. + +%% @doc Apply any scheduled message, whatever it asked to be routed to. +router(_Key, Base, Assignment, Opts) -> + compute(Base, Assignment, Opts). + +%% @doc Setting the device is honoured, and nothing else is. `lib_process' puts +%% the process's own device back with a `set' after every slot, and reading the +%% state as a message is itself a device set, so refusing those would leave the +%% device unable to read itself. A scheduled message that asks to be routed to +%% `set' is applied like any other message instead of being allowed to write the +%% state directly. +%% +%% The device is written here rather than delegated to `~message@1.0', because +%% delegating means viewing this state as a message, which is another device +%% set: it would arrive back here forever. +set(Base, Req, Opts) -> + case hb_maps:keys(Req, Opts) -- [<<"path">>, <<"set-mode">>] of + [<<"device">>] -> + {ok, + Base#{ + <<"device">> => + hb_maps:get(<<"device">>, Req, undefined, Opts) + } + }; + _ -> compute(Base, Req, Opts) + end. + +%% @doc Listing the state's keys is not a state transition, so a scheduled +%% message asking for it is applied as one. +keys(Base, Req, Opts) -> compute(Base, Req, Opts). + +%% @doc Apply one assignment to the swap's state. +%% +%% In `all' mode the overwhelming majority of slots are unrelated Arweave +%% traffic, so the classification below is ordered by cost: advance the clock, +%% then compare a single field -- the transaction's `target' -- against the +%% process, and only then look at tags. A transaction that is neither addressed +%% to the process nor a payment against a live order leaves the state exactly as +%% it was. +%% The process is identified from the assignment rather than from the process +%% message, which `lib_process:process_id/3' would re-verify the signature of on +%% every one of the network's transactions. +compute(Base, Assignment, Opts) -> + Height = hb_util:int(hb_ao:get(<<"block-height">>, Assignment, 0, Opts)), + ProcID = hb_ao:get(<<"process">>, Assignment, <<>>, Opts), + Body = hb_ao:get(<<"body">>, Assignment, #{}, Opts), + Advanced = advance(Base, Height, Opts), + case hb_ao:get(<<"target">>, Body, <<>>, Opts) of + ProcID -> {ok, control(Advanced, Body, Height, Opts)}; + Target -> {ok, payment(Advanced, Body, Target, Height, Opts)} + end. + +%%% Order lifecycle + +%% @doc Route a transaction addressed to the process by its `action'. An +%% unknown action is not an error: in this mode anyone may send the process +%% anything, and a process that could be wedged by a stranger's transaction +%% would not survive its own schedule. +control(Base, Body, Height, Opts) -> + case hb_ao:get(<<"action">>, Body, <<>>, Opts) of + <<"make-offer">> -> make_offer(Base, Body, Height, Opts); + <<"cancel-order">> -> cancel_order(Base, Body, Opts); + <<"register-interest">> -> register_interest(Base, Body, Height, Opts); + _ -> Base + end. + +%% @doc Open an order, moving the goods and the seller's bond into escrow. +%% Nothing is written unless the whole offer is admissible, so a rejected offer +%% is indistinguishable from a transaction that was never sent. +make_offer(Base, Body, Height, Opts) -> + maybe + {ok, Seller} ?= signer(Body, Opts), + {ok, Quantity} ?= amount(<<"offer-quantity">>, Body, Opts), + {ok, Asking} ?= amount(<<"asking">>, Body, Opts), + {ok, Deposit} ?= amount(<<"deposit">>, Body, Opts), + {ok, Deadline} ?= amount(<<"deadline">>, Body, Opts), + Recipient = hb_ao:get(<<"recipient">>, Body, Seller, Opts), + true ?= Quantity >= 1, + true ?= Asking >= 1, + true ?= Deposit >= 0, + true ?= Deadline > Height, + true ?= balance(Base, Seller, Opts) >= Quantity + Deposit, + OrderID = hb_util:human_id(hb_message:id(Body, signed, Opts)), + Order = + #{ + <<"order-id">> => OrderID, + <<"creator">> => Seller, + <<"recipient">> => Recipient, + <<"quantity">> => Quantity, + <<"asking">> => Asking, + <<"deposit">> => Deposit, + <<"deadline">> => Deadline, + <<"created-at">> => Height, + <<"status">> => <<"open">> + }, + ?event( + {swap_order_opened, + {order, OrderID}, + {seller, Seller}, + {quantity, Quantity}, + {asking, Asking} + } + ), + note( + deadlines( + put_order( + debit(Base, Seller, Quantity + Deposit, Opts), + Order, + Opts + ), + Opts + ), + <<"order-opened">>, + OrderID, + Opts + ) + else + _ -> Base + end. + +%% @doc Withdraw an order that nobody has reserved, returning the goods. The +%% deposit stays escrowed until the grace period ends: a payment may already be +%% in flight against this order, and it is the deposit that compensates it. +cancel_order(Base, Body, Opts) -> + maybe + {ok, Signer} ?= signer(Body, Opts), + {ok, Order} ?= find_order(Base, Body, Opts), + #{ <<"creator">> := Creator, <<"status">> := Status } = Order, + true ?= Signer =:= Creator, + true ?= Status =:= <<"open">>, + ?event({swap_order_cancelled, {order, order_id(Order)}}), + note( + deadlines( + release_goods(Base, Order, Creator, <<"cancelled">>, Opts), + Opts + ), + <<"order-cancelled">>, + order_id(Order), + Opts + ) + else + _ -> Base + end. + +%% @doc Reserve an open order for the sender. The reservation is exclusive and +%% freezes the seller out of cancelling, which is precisely what makes it safe +%% for the sender to part with their AR. +register_interest(Base, Body, Height, Opts) -> + maybe + {ok, Buyer} ?= signer(Body, Opts), + {ok, Order} ?= find_order(Base, Body, Opts), + #{ <<"status">> := Status, <<"deadline">> := Deadline } = Order, + true ?= Status =:= <<"open">>, + true ?= Height < Deadline, + Until = min(Deadline, Height + reservation_blocks(Base, Opts)), + ?event( + {swap_interest_registered, + {order, order_id(Order)}, + {buyer, Buyer}, + {until, Until} + } + ), + note( + deadlines( + put_order( + Base, + Order#{ + <<"status">> => <<"reserved">>, + <<"buyer">> => Buyer, + <<"reserved-until">> => Until + }, + Opts + ), + Opts + ), + <<"interest-registered">>, + order_id(Order), + Opts + ) + else + _ -> Base + end. + +%%% Settlement + +%% @doc Settle against a layer-1 payment. The transaction is not addressed to +%% the process at all: it is a transfer between two user addresses that names an +%% `order-id', and the process sees it only because it is sequenced by every +%% transaction on the network. It counts as payment for an order when it is +%% addressed to that order's `recipient' and carries at least the asking +%% winston. +%% +%% Underpayment is ignored rather than partially filled -- the value never +%% passed through the process, so there is nothing to refund the difference +%% from. +payment(Base, Body, Target, Height, Opts) -> + maybe + {ok, Buyer} ?= signer(Body, Opts), + {ok, Order} ?= find_order(Base, Body, Opts), + #{ + <<"creator">> := Creator, + <<"recipient">> := Recipient, + <<"asking">> := Asking, + <<"deadline">> := Deadline + } = Order, + true ?= Target =:= Recipient, + % A seller paying themselves costs nothing but a network fee, and would + % otherwise let them settle their own order -- taking the goods back + % along with the bond, and leaving a real buyer's payment to arrive + % against an order that is already spent. + false ?= Buyer =:= Creator, + false ?= Buyer =:= Recipient, + {ok, Paid} ?= amount(<<"quantity">>, Body, Opts), + true ?= Paid >= Asking, + true ?= Height =< Deadline + cancel_grace(Base, Opts), + settle(Base, Order, Buyer, Height, Body, Opts) + else + _ -> Base + end. + +%% @doc Pay a matched payment. The goods go to the buyer and the bond returns +%% to the seller when the order was theirs to buy; otherwise the buyer takes the +%% bond in compensation, because they have paid for something they cannot +%% receive. +settle(Base, Order, Buyer, Height, Body, Opts) -> + PaymentID = hb_util:human_id(hb_message:id(Body, signed, Opts)), + Settled = + Order#{ + <<"status">> => <<"settled">>, + <<"settled-at">> => Height, + <<"payment-tx">> => PaymentID + }, + case claimable(Order, Buyer, Height) of + true -> + #{ <<"creator">> := Creator, <<"quantity">> := Quantity } = Order, + ?event( + {swap_order_settled, + {order, order_id(Order)}, + {buyer, Buyer}, + {quantity, Quantity} + } + ), + note( + deadlines( + put_order( + credit( + credit(Base, Buyer, Quantity, Opts), + Creator, + deposit(Order), + Opts + ), + Settled#{ <<"quantity">> => 0, <<"deposit">> => 0 }, + Opts + ), + Opts + ), + <<"order-settled">>, + order_id(Order), + Opts + ); + false -> + % The goods are gone or promised to somebody else, but the buyer + % has already paid. The bond is what they get instead, and it can + % only be paid out once. + compensate(Base, Order, Buyer, PaymentID, Opts) + end. + +%% @doc Pay a stranded buyer the seller's bond. +compensate(Base, Order, Buyer, PaymentID, Opts) -> + case deposit(Order) of + 0 -> Base; + Deposit -> + ?event( + {swap_payment_compensated, + {order, order_id(Order)}, + {buyer, Buyer}, + {deposit, Deposit} + } + ), + note( + deadlines( + put_order( + credit(Base, Buyer, Deposit, Opts), + Order#{ + <<"deposit">> => 0, + <<"payment-tx">> => PaymentID + }, + Opts + ), + Opts + ), + <<"payment-compensated">>, + order_id(Order), + Opts + ) + end. + +%% @doc Whether an order's goods are the payer's to take: an order nobody has +%% reserved is first-come, and a reserved one is its buyer's until the +%% reservation lapses. +claimable(#{ <<"quantity">> := 0 }, _Buyer, _Height) -> false; +claimable(#{ <<"status">> := <<"open">> }, _Buyer, _Height) -> true; +claimable(Order = #{ <<"status">> := <<"reserved">> }, Buyer, Height) -> + maps:get(<<"buyer">>, Order, <<>>) =:= Buyer + andalso Height =< maps:get(<<"reserved-until">>, Order, 0); +claimable(_Order, _Buyer, _Height) -> false. + +%%% The clock + +%% @doc Advance the process's notion of the chain to the height of the +%% assignment being applied, retiring whatever fell due in between. +%% +%% Every transaction on the network is a slot here, so this runs on the order of +%% a thousand times per block: the common path is one integer comparison against +%% the next height at which anything at all happens, and only crossing it walks +%% the orders. +advance(Base, Height, Opts) -> + case hb_util:int(state(<<"next-deadline">>, Base, 0, Opts)) of + Next when Next > 0, Height >= Next -> + deadlines(sweep(Base, Height, Opts), Opts); + _ -> + Base#{ <<"swap-height">> => Height } + end. + +%% @doc Apply every deadline that the given height has reached: reservations +%% lapse, unsold orders return their goods, and orders past their grace period +%% return the residual bond. +sweep(Base, Height, Opts) -> + lists:foldl( + fun(Order, Acc) -> expire(Acc, Order, Height, Opts) end, + Base#{ <<"swap-height">> => Height }, + orders(Base, Opts) + ). + +expire(Base, Order = #{ <<"status">> := <<"reserved">> }, Height, Opts) -> + case Height > maps:get(<<"reserved-until">>, Order, 0) of + true -> + % The reservation has lapsed, so the order is open to anyone again + % -- and being open, it may be due to expire in this same sweep. + Reopened = + maps:without( + [<<"buyer">>, <<"reserved-until">>], + Order#{ <<"status">> => <<"open">> } + ), + expire(put_order(Base, Reopened, Opts), Reopened, Height, Opts); + false -> Base + end; +expire(Base, Order = #{ <<"status">> := <<"open">>, <<"deadline">> := Deadline }, + Height, Opts) when Height >= Deadline -> + ?event({swap_order_expired, {order, order_id(Order)}}), + release_goods(Base, Order, maps:get(<<"creator">>, Order), <<"expired">>, Opts); +expire(Base, Order, Height, Opts) -> + % Only the bond is left outstanding, and the window in which a payment + % could still claim it has closed. + Deadline = maps:get(<<"deadline">>, Order, 0), + case + deposit(Order) > 0 andalso Height > Deadline + cancel_grace(Base, Opts) + of + true -> + ?event({swap_deposit_retired, {order, order_id(Order)}}), + put_order( + credit( + Base, + maps:get(<<"creator">>, Order), + deposit(Order), + Opts + ), + Order#{ <<"deposit">> => 0 }, + Opts + ); + false -> Base + end. + +%% @doc Record the next height at which any order needs attention, so that the +%% slots in between cost a single comparison. Zero means nothing is pending. +deadlines(Base, Opts) -> + Grace = cancel_grace(Base, Opts), + Heights = + lists:flatten( + [ order_deadlines(Order, Grace) || Order <- orders(Base, Opts) ] + ), + Next = + case Heights of + [] -> 0; + _ -> lists:min(Heights) + end, + Base#{ <<"next-deadline">> => Next }. + +order_deadlines(Order = #{ <<"status">> := <<"reserved">> }, Grace) -> + [maps:get(<<"reserved-until">>, Order, 0) + 1 + | order_deadlines(Order#{ <<"status">> => <<"open">> }, Grace)]; +order_deadlines(Order = #{ <<"status">> := <<"open">>, <<"deadline">> := D }, Grace) -> + case quantity(Order) of + 0 -> retirement(Order, Grace); + _ -> [D | retirement(Order, Grace)] + end; +order_deadlines(Order, Grace) -> + retirement(Order, Grace). + +retirement(Order, Grace) -> + case deposit(Order) of + 0 -> []; + _ -> [maps:get(<<"deadline">>, Order, 0) + Grace + 1] + end. + +%%% State helpers + +%% @doc Return an order's goods to its creator, leaving the bond escrowed for +%% whatever grace remains. +release_goods(Base, Order, Creator, Status, Opts) -> + put_order( + credit(Base, Creator, quantity(Order), Opts), + Order#{ <<"quantity">> => 0, <<"status">> => Status }, + Opts + ). + +%% @doc Read a key of the process's own state. +%% +%% While a slot is being computed the state carries this device, so a plain read +%% of one of its keys would resolve that key *through this device* and land back +%% in `compute'. Every read of the state is therefore taken as a message. Writes +%% have the same hazard one level down -- setting a nested path resolves the +%% keys above it on the way -- so the device only ever writes whole top-level +%% keys. +state(Key, Base, Default, Opts) -> + hb_ao:get(Key, {as, <<"message@1.0">>, Base}, Default, Opts). + +%% @doc Read the orders currently held, as plain maps. The state may have been +%% written to the process cache and read back since it was last touched, so it +%% is loaded through the link layer, and anything that is not an order is +%% ignored rather than assumed away. +orders(Base, Opts) -> + Orders = hb_cache:ensure_all_loaded(order_book(Base, Opts), Opts), + [ + order(Held) + || + Held <- + [ hb_maps:get(ID, Orders, #{}, Opts) || ID <- hb_ao:keys(Orders, Opts) ], + is_map(Held), + maps:is_key(<<"order-id">>, Held) + ]. + +order_book(Base, Opts) -> state(<<"orders">>, Base, #{}, Opts). + +%% @doc Read an order with its numbers as numbers. Between slots the state is +%% written to the process cache and read back, so nothing that a comparison or +%% a sum depends on is assumed to have survived as an integer term -- a height +%% that came back as a binary would sort above every integer, and a deadline +%% would then simply never fall due. +order(Held) -> + lists:foldl( + fun(Key, Order) -> + case maps:find(Key, Order) of + {ok, Value} -> Order#{ Key => hb_util:int(Value) }; + error -> Order + end + end, + Held, + [ + <<"quantity">>, + <<"deposit">>, + <<"asking">>, + <<"deadline">>, + <<"created-at">>, + <<"reserved-until">>, + <<"settled-at">> + ] + ). + +%% @doc Read the order a message names, if the process holds it. +%% +%% The name comes from a stranger's transaction, so it is looked up as a key of +%% the order book rather than resolved as a path: a path would let +%% `order-id: /creator' reach an address, and a reserved name like `keys' +%% reach a list, either of which would then be read as an order and fail the +%% slot. Whatever comes back must look like an order before it is treated as +%% one. +find_order(Base, Body, Opts) -> + Held = + hb_maps:get( + hb_ao:get(<<"order-id">>, Body, <<>>, Opts), + order_book(Base, Opts), + not_found, + Opts + ), + case hb_cache:ensure_all_loaded(Held, Opts) of + Order when is_map(Order) -> + case maps:is_key(<<"order-id">>, Order) of + true -> {ok, order(Order)}; + false -> not_found + end; + _ -> not_found + end. + +%% @doc Write an order back, replacing the one held rather than merging over +%% it, so that a lapsed reservation leaves no buyer behind. +put_order(Base, Order, Opts) -> + Base#{ + <<"orders">> => + hb_maps:put( + order_id(Order), + Order, + order_book(Base, Opts), + Opts + ) + }. + +order_id(Order) -> maps:get(<<"order-id">>, Order). + +quantity(Order) -> hb_util:int(maps:get(<<"quantity">>, Order, 0)). + +deposit(Order) -> hb_util:int(maps:get(<<"deposit">>, Order, 0)). + +%% @doc Read an address's token balance from the ledger this process shares. +%% Only the one entry is read: the ledger may be large, and the rest of it is +%% none of this device's business. +balance(Base, Address, Opts) -> + hb_util:int(state([?BALANCES, Address], Base, 0, Opts)). + +credit(Base, _Address, 0, _Opts) -> Base; +credit(Base, Address, Amount, Opts) -> + settle_balance(Base, Address, balance(Base, Address, Opts) + Amount, Opts). + +debit(Base, Address, Amount, Opts) -> + settle_balance(Base, Address, balance(Base, Address, Opts) - Amount, Opts). + +settle_balance(Base, Address, Value, Opts) -> + Base#{ + ?BALANCES => + hb_maps:put( + Address, + Value, + state(?BALANCES, Base, #{}, Opts), + Opts + ) + }. + +%% @doc Report what the slot did, in the results of the slot itself. +note(Base, Event, OrderID, _Opts) -> + Base#{ + <<"results">> => #{ <<"event">> => Event, <<"order-id">> => OrderID } + }. + +%% @doc Read a number a stranger wrote. Every figure in the protocol arrives as +%% a tag on somebody else's transaction, and this process is sequenced by all of +%% them: coercing `deadline: tomorrow' with `hb_util:int/1' would raise out of +%% the enclosing `maybe' -- which catches mismatches, not exceptions -- and fail +%% that slot on every node, for good. A value that is not a number is simply not +%% an admissible message. +amount(Key, Body, Opts) -> + hb_util:safe_int(hb_ao:get(Key, Body, 0, Opts)). + +%% @doc The single signer of a message. A message with any other number of +%% signers is not attributable to one party, so it cannot open, cancel, reserve +%% or pay for anything. +signer(Body, Opts) -> + case hb_message:signers(Body, Opts) of + [Signer] -> {ok, hb_util:human_id(Signer)}; + _ -> not_found + end. + +reservation_blocks(Base, Opts) -> + hb_util:int( + state( + <<"swap-reservation-blocks">>, + Base, + ?DEFAULT_RESERVATION_BLOCKS, + Opts + ) + ). + +cancel_grace(Base, Opts) -> + hb_util:int( + state(<<"swap-cancel-grace">>, Base, ?DEFAULT_CANCEL_GRACE, Opts) + ). + +%%% Tests + +%%% The tests drive `compute/3' directly with synthetic assignments, exactly as +%%% `~process@1.0' would: the device reads chain data but performs no I/O, so a +%%% whole trade can be played out without a weave. + +-define(PROCESS, <<"pRoCeSs00000000000000000000000000000000000">>). + +test_opts() -> #{ <<"priv-wallet">> => ar_wallet:new() }. + +%% @doc A party to a trade: a wallet and the address it signs as. +party() -> + Wallet = ar_wallet:new(), + {Wallet, hb_util:human_id(ar_wallet:to_address(Wallet))}. + +%% @doc A process base holding the given balances and nothing else. +base(Balances) -> + #{ ?BALANCES => Balances }. + +%% @doc An L1 transaction, committed as the base layer commits them. +tx(Wallet, Fields) -> + hb_message:commit( + Fields, + #{ <<"priv-wallet">> => Wallet }, + #{ <<"commitment-device">> => <<"tx@1.0">> } + ). + +%% @doc Sequence a transaction into the process at a block height, as +%% `~arweave-scheduler@1.0' in `all' mode does. +apply_tx(Base, Body, Height, Opts) -> + {ok, New} = + compute( + Base, + #{ + <<"process">> => ?PROCESS, + <<"slot">> => 1, + <<"block-height">> => Height, + <<"body">> => Body + }, + Opts + ), + New. + +%% @doc Advance the process to a height without anything happening, which is +%% what the network's unrelated traffic does. +tick(Base, Height, Opts) -> + apply_tx(Base, #{ <<"target">> => <<"someone-else">> }, Height, Opts). + +offer(Wallet, Quantity, Asking, Deposit, Deadline) -> + tx( + Wallet, + #{ + <<"target">> => ?PROCESS, + <<"action">> => <<"make-offer">>, + <<"offer-quantity">> => hb_util:bin(Quantity), + <<"asking">> => hb_util:bin(Asking), + <<"deposit">> => hb_util:bin(Deposit), + <<"deadline">> => hb_util:bin(Deadline) + } + ). + +order_action(Wallet, Action, OrderID) -> + tx( + Wallet, + #{ + <<"target">> => ?PROCESS, + <<"action">> => Action, + <<"order-id">> => OrderID + } + ). + +%% @doc The payment leg: an ordinary transfer to the seller that the process is +%% not addressed by, naming the order it settles. +pay(Wallet, To, Winston, OrderID) -> + tx( + Wallet, + #{ + <<"target">> => To, + <<"quantity">> => hb_util:bin(Winston), + <<"order-id">> => OrderID + } + ). + +only_order(Base, Opts) -> + [Order] = orders(Base, Opts), + Order. + +balance_of(Base, Address, Opts) -> balance(Base, Address, Opts). + +%% @doc Opening an offer moves the goods and the bond into escrow, and leaves +%% the order open. +make_offer_escrows_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + Base = base(#{ SellerAddr => 100 }), + Opened = apply_tx(Base, offer(Seller, 10, 500, 5, 200), 100, Opts), + Order = only_order(Opened, Opts), + ?assertEqual(85, balance_of(Opened, SellerAddr, Opts)), + ?assertEqual(<<"open">>, maps:get(<<"status">>, Order)), + ?assertEqual(10, quantity(Order)), + ?assertEqual(5, deposit(Order)), + ?assertEqual(SellerAddr, maps:get(<<"recipient">>, Order)). + +%% @doc An offer for more than the seller holds changes nothing at all. +make_offer_insufficient_balance_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + Base = base(#{ SellerAddr => 4 }), + Result = apply_tx(Base, offer(Seller, 10, 500, 5, 200), 100, Opts), + ?assertEqual([], orders(Result, Opts)), + ?assertEqual(4, balance_of(Result, SellerAddr, Opts)). + +%% @doc A deadline that has already passed is not an offer. +make_offer_stale_deadline_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + Base = base(#{ SellerAddr => 100 }), + Result = apply_tx(Base, offer(Seller, 10, 500, 5, 100), 100, Opts), + ?assertEqual([], orders(Result, Opts)). + +%% @doc The whole trade: the buyer pays the seller directly on layer one, and +%% the process -- which is not a party to that payment -- settles it. +settlement_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + {Buyer, BuyerAddr} = party(), + Base = base(#{ SellerAddr => 100 }), + Opened = apply_tx(Base, offer(Seller, 10, 500, 5, 200), 100, Opts), + OrderID = order_id(only_order(Opened, Opts)), + Settled = + apply_tx( + Opened, + pay(Buyer, SellerAddr, 500, OrderID), + 120, + Opts + ), + Order = only_order(Settled, Opts), + ?assertEqual(<<"settled">>, maps:get(<<"status">>, Order)), + % The buyer has the goods; the seller has their bond back and is out the + % tokens they sold. + ?assertEqual(10, balance_of(Settled, BuyerAddr, Opts)), + ?assertEqual(90, balance_of(Settled, SellerAddr, Opts)). + +%% @doc Paying less than the asking price settles nothing: the process never +%% held the value, so it cannot refund a partial fill. +underpayment_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + {Buyer, BuyerAddr} = party(), + Opened = + apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 200), 100, Opts), + OrderID = order_id(only_order(Opened, Opts)), + Result = apply_tx(Opened, pay(Buyer, SellerAddr, 499, OrderID), 120, Opts), + ?assertEqual(<<"open">>, maps:get(<<"status">>, only_order(Result, Opts))), + ?assertEqual(0, balance_of(Result, BuyerAddr, Opts)). + +%% @doc A payment that names the order but is addressed to somebody else is not +%% a payment for it. +payment_to_wrong_address_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + {Buyer, BuyerAddr} = party(), + {_, Stranger} = party(), + Opened = + apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 200), 100, Opts), + OrderID = order_id(only_order(Opened, Opts)), + Result = apply_tx(Opened, pay(Buyer, Stranger, 500, OrderID), 120, Opts), + ?assertEqual(<<"open">>, maps:get(<<"status">>, only_order(Result, Opts))), + ?assertEqual(0, balance_of(Result, BuyerAddr, Opts)). + +%% @doc Cancelling returns the goods at once, but holds the bond for as long as +%% a payment could still be in flight. +cancel_returns_goods_but_holds_bond_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + Opened = + apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 200), 100, Opts), + OrderID = order_id(only_order(Opened, Opts)), + Cancelled = + apply_tx(Opened, order_action(Seller, <<"cancel-order">>, OrderID), 110, Opts), + Order = only_order(Cancelled, Opts), + ?assertEqual(<<"cancelled">>, maps:get(<<"status">>, Order)), + ?assertEqual(0, quantity(Order)), + ?assertEqual(5, deposit(Order)), + ?assertEqual(95, balance_of(Cancelled, SellerAddr, Opts)), + % Once the grace period has passed with no payment, the bond comes back. + Retired = tick(Cancelled, 200 + ?DEFAULT_CANCEL_GRACE + 1, Opts), + ?assertEqual(0, deposit(only_order(Retired, Opts))), + ?assertEqual(100, balance_of(Retired, SellerAddr, Opts)). + +%% @doc Only the seller may cancel their order. +cancel_by_stranger_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + {Stranger, _} = party(), + Opened = + apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 200), 100, Opts), + OrderID = order_id(only_order(Opened, Opts)), + Result = + apply_tx(Opened, order_action(Stranger, <<"cancel-order">>, OrderID), 110, Opts), + ?assertEqual(<<"open">>, maps:get(<<"status">>, only_order(Result, Opts))). + +%% @doc A reserved order cannot be pulled out from under the buyer who reserved +%% it. This is the guarantee that makes paying safe. +reservation_blocks_cancellation_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + {Buyer, _} = party(), + Opened = + apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 200), 100, Opts), + OrderID = order_id(only_order(Opened, Opts)), + Reserved = + apply_tx( + Opened, + order_action(Buyer, <<"register-interest">>, OrderID), + 110, + Opts + ), + ?assertEqual(<<"reserved">>, maps:get(<<"status">>, only_order(Reserved, Opts))), + Attempted = + apply_tx( + Reserved, + order_action(Seller, <<"cancel-order">>, OrderID), + 111, + Opts + ), + ?assertEqual(<<"reserved">>, maps:get(<<"status">>, only_order(Attempted, Opts))). + +%% @doc A reservation is exclusive while it lasts: somebody else's payment does +%% not take the goods, it takes the bond. +reservation_is_exclusive_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + {Buyer, _} = party(), + {Interloper, InterloperAddr} = party(), + Opened = + apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 200), 100, Opts), + OrderID = order_id(only_order(Opened, Opts)), + Reserved = + apply_tx(Opened, order_action(Buyer, <<"register-interest">>, OrderID), 110, Opts), + Result = apply_tx(Reserved, pay(Interloper, SellerAddr, 500, OrderID), 111, Opts), + Order = only_order(Result, Opts), + ?assertEqual(<<"reserved">>, maps:get(<<"status">>, Order)), + ?assertEqual(10, quantity(Order)), + ?assertEqual(0, deposit(Order)), + ?assertEqual(5, balance_of(Result, InterloperAddr, Opts)). + +%% @doc A reservation lapses on its own, without anybody sending anything: the +%% network's own traffic carries the clock forward. +reservation_lapses_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + {Buyer, _} = party(), + Opened = + apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 200), 100, Opts), + OrderID = order_id(only_order(Opened, Opts)), + Reserved = + apply_tx(Opened, order_action(Buyer, <<"register-interest">>, OrderID), 110, Opts), + ?assertEqual( + 110 + ?DEFAULT_RESERVATION_BLOCKS, + maps:get(<<"reserved-until">>, only_order(Reserved, Opts)) + ), + Lapsed = tick(Reserved, 110 + ?DEFAULT_RESERVATION_BLOCKS + 1, Opts), + ?assertEqual(<<"open">>, maps:get(<<"status">>, only_order(Lapsed, Opts))). + +%% @doc An unsold order returns its goods when its deadline passes. +expiry_returns_goods_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + Opened = + apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 200), 100, Opts), + ?assertEqual(85, balance_of(Opened, SellerAddr, Opts)), + Expired = tick(Opened, 200, Opts), + Order = only_order(Expired, Opts), + ?assertEqual(<<"expired">>, maps:get(<<"status">>, Order)), + ?assertEqual(95, balance_of(Expired, SellerAddr, Opts)), + ?assertEqual(5, deposit(Order)). + +%% @doc A buyer who pays for an expired order within the grace period is paid +%% the seller's bond: they parted with value for goods that were gone. +late_payment_takes_the_bond_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + {Buyer, BuyerAddr} = party(), + Opened = + apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 200), 100, Opts), + OrderID = order_id(only_order(Opened, Opts)), + Expired = tick(Opened, 201, Opts), + Paid = apply_tx(Expired, pay(Buyer, SellerAddr, 500, OrderID), 202, Opts), + ?assertEqual(5, balance_of(Paid, BuyerAddr, Opts)), + ?assertEqual(0, deposit(only_order(Paid, Opts))), + ?assertEqual(95, balance_of(Paid, SellerAddr, Opts)). + +%% @doc The bond is paid out once. A second late payment gets nothing, because +%% there is nothing left to compensate it with. +bond_pays_out_once_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + {First, FirstAddr} = party(), + {Second, SecondAddr} = party(), + Opened = + apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 200), 100, Opts), + OrderID = order_id(only_order(Opened, Opts)), + Expired = tick(Opened, 201, Opts), + Once = apply_tx(Expired, pay(First, SellerAddr, 500, OrderID), 202, Opts), + Twice = apply_tx(Once, pay(Second, SellerAddr, 500, OrderID), 203, Opts), + ?assertEqual(5, balance_of(Twice, FirstAddr, Opts)), + ?assertEqual(0, balance_of(Twice, SecondAddr, Opts)). + +%% @doc Settling twice is not possible: the goods left with the first payment. +double_settlement_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + {Buyer, BuyerAddr} = party(), + {Late, LateAddr} = party(), + Opened = + apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 200), 100, Opts), + OrderID = order_id(only_order(Opened, Opts)), + Settled = apply_tx(Opened, pay(Buyer, SellerAddr, 500, OrderID), 120, Opts), + Again = apply_tx(Settled, pay(Late, SellerAddr, 500, OrderID), 121, Opts), + ?assertEqual(10, balance_of(Again, BuyerAddr, Opts)), + ?assertEqual(0, balance_of(Again, LateAddr, Opts)), + ?assertEqual(90, balance_of(Again, SellerAddr, Opts)). + +%% @doc A payment naming an order the process has never heard of is ordinary +%% Arweave traffic, and is ignored. +unknown_order_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + {Buyer, _} = party(), + Opened = + apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 200), 100, Opts), + Result = + apply_tx( + Opened, + pay(Buyer, SellerAddr, 500, <<"nOtAnOrDeR0000000000000000000000000000000000">>), + 120, + Opts + ), + ?assertEqual(<<"open">>, maps:get(<<"status">>, only_order(Result, Opts))). + +%% @doc Unrelated traffic -- the overwhelming majority of what this process is +%% sequenced by -- moves the clock and nothing else. +unrelated_traffic_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + Opened = + apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 200), 100, Opts), + Ticked = tick(Opened, 150, Opts), + ?assertEqual(150, hb_ao:get(<<"swap-height">>, Ticked, 0, Opts)), + ?assertEqual(<<"open">>, maps:get(<<"status">>, only_order(Ticked, Opts))), + ?assertEqual(85, balance_of(Ticked, SellerAddr, Opts)). + +%% @doc A stranger's transaction may ask to be routed anywhere -- the key a slot +%% resolves comes from the sender's own `path' tag, and this process is +%% sequenced by every transaction on Arweave. It must be applied like any other, +%% not fail its slot. +stray_path_is_applied_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + {Stranger, _} = party(), + Opened = + apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 200), 100, Opts), + Strayed = + hb_ao:resolve( + Opened#{ <<"device">> => <<"arweave-swap@1.0">> }, + #{ + <<"path">> => <<"withdraw-everything">>, + <<"process">> => ?PROCESS, + <<"slot">> => 2, + <<"block-height">> => 150, + <<"body">> => + tx(Stranger, #{ <<"target">> => <<"somebody-else">> }) + }, + Opts + ), + ?assertMatch({ok, _}, Strayed), + {ok, State} = Strayed, + % The state still carries this device here, because the test resolves it + % directly rather than through `lib_process:run_as', which puts the + % process's own device back afterwards. Read it as a message. + ?assertEqual(150, hb_util:int(state(<<"swap-height">>, State, 0, Opts))), + ?assertEqual(85, balance_of(State, SellerAddr, Opts)). + +%% @doc An order read back with its numbers encoded as binaries -- which is how +%% it returns from the process cache -- still falls due. Comparing a height +%% against a binary would silently never fire. +encoded_order_still_expires_test() -> + Opts = test_opts(), + {_, SellerAddr} = party(), + Encoded = + #{ + ?BALANCES => #{ SellerAddr => 85 }, + <<"next-deadline">> => <<"200">>, + <<"orders">> => + #{ + <<"order-1">> => + #{ + <<"order-id">> => <<"order-1">>, + <<"creator">> => SellerAddr, + <<"recipient">> => SellerAddr, + <<"quantity">> => <<"10">>, + <<"asking">> => <<"500">>, + <<"deposit">> => <<"5">>, + <<"deadline">> => <<"200">>, + <<"status">> => <<"open">> + } + } + }, + Expired = tick(Encoded, 200, Opts), + ?assertEqual(<<"expired">>, maps:get(<<"status">>, only_order(Expired, Opts))), + ?assertEqual(95, balance_of(Expired, SellerAddr, Opts)). + +%% @doc A reservation that lapses in the same sweep that its order expires is +%% resolved in order, and leaves no stale buyer behind. +lapse_and_expire_together_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + {Buyer, _} = party(), + Opened = + apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 195), 100, Opts), + OrderID = order_id(only_order(Opened, Opts)), + Reserved = + apply_tx(Opened, order_action(Buyer, <<"register-interest">>, OrderID), 190, Opts), + % Reserved to 195 (capped at the deadline); at 196 both are due. + Swept = tick(Reserved, 196, Opts), + Order = only_order(Swept, Opts), + ?assertEqual(<<"expired">>, maps:get(<<"status">>, Order)), + ?assertEqual(false, maps:is_key(<<"buyer">>, Order)), + ?assertEqual(95, balance_of(Swept, SellerAddr, Opts)). + +%% @doc The offered amount is not carried as `quantity'. That key is the +%% transaction's own value field, so the codec would send it as winston of AR to +%% the process -- an address with no key. An offer that uses it opens nothing. +offer_quantity_is_not_winston_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + Wrong = + tx( + Seller, + #{ + <<"target">> => ?PROCESS, + <<"action">> => <<"make-offer">>, + <<"quantity">> => <<"10">>, + <<"asking">> => <<"500">>, + <<"deposit">> => <<"5">>, + <<"deadline">> => <<"200">> + } + ), + Result = apply_tx(base(#{ SellerAddr => 100 }), Wrong, 100, Opts), + ?assertEqual([], orders(Result, Opts)), + ?assertEqual(100, balance_of(Result, SellerAddr, Opts)). + +%% @doc A figure that is not a number is an inadmissible message, not a failed +%% slot: anyone may send this process anything, and a slot that raises can never +%% be recomputed. +non_numeric_tag_is_ignored_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + Nonsense = + tx( + Seller, + #{ + <<"target">> => ?PROCESS, + <<"action">> => <<"make-offer">>, + <<"offer-quantity">> => <<"10">>, + <<"asking">> => <<"500">>, + <<"deposit">> => <<"5">>, + <<"deadline">> => <<"tomorrow">> + } + ), + Result = apply_tx(base(#{ SellerAddr => 100 }), Nonsense, 100, Opts), + ?assertEqual([], orders(Result, Opts)), + ?assertEqual(100, balance_of(Result, SellerAddr, Opts)). + +%% @doc An `order-id' is a stranger's text. Naming a path into the order book, +%% or one of a message's own reserved keys, must not reach anything that is then +%% read as an order. +hostile_order_id_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + {Buyer, _} = party(), + Opened = + apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 200), 100, Opts), + OrderID = order_id(only_order(Opened, Opts)), + Hostile = + [ + <>, + <<"keys">>, + <<"id">>, + <<"commitments">> + ], + lists:foreach( + fun(Name) -> + Result = apply_tx(Opened, pay(Buyer, SellerAddr, 500, Name), 120, Opts), + ?assertEqual( + <<"open">>, + maps:get(<<"status">>, only_order(Result, Opts)) + ) + end, + Hostile + ). + +%% @doc A seller cannot buy their own order. Paying oneself costs only a network +%% fee, so it would otherwise take back the goods and the bond, leaving a real +%% buyer to pay for an order that is already spent. +seller_cannot_settle_own_order_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + Opened = + apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 200), 100, Opts), + OrderID = order_id(only_order(Opened, Opts)), + Result = apply_tx(Opened, pay(Seller, SellerAddr, 500, OrderID), 120, Opts), + ?assertEqual(<<"open">>, maps:get(<<"status">>, only_order(Result, Opts))), + ?assertEqual(85, balance_of(Result, SellerAddr, Opts)). + +%% @doc Nor can they claim their own bond once the order has expired. +seller_cannot_claim_own_bond_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + Opened = + apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 200), 100, Opts), + OrderID = order_id(only_order(Opened, Opts)), + Expired = tick(Opened, 201, Opts), + Result = apply_tx(Expired, pay(Seller, SellerAddr, 500, OrderID), 202, Opts), + ?assertEqual(5, deposit(only_order(Result, Opts))), + ?assertEqual(95, balance_of(Result, SellerAddr, Opts)). + +%% @doc A scheduled message routed to `set' or `keys' is applied like any other. +%% Handing either to `~message@1.0' would let a passer-by write the process's own +%% balances, or replace its state with a list of key names. +reserved_paths_are_applied_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + {Stranger, _} = party(), + Opened = + apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 200), 100, Opts), + lists:foreach( + fun(Path) -> + {ok, State} = + hb_ao:resolve( + Opened#{ <<"device">> => <<"arweave-swap@1.0">> }, + #{ + <<"path">> => Path, + <<"process">> => ?PROCESS, + <<"slot">> => 2, + <<"block-height">> => 150, + <<"balances">> => #{ SellerAddr => 10000000 }, + <<"body">> => + tx(Stranger, #{ <<"target">> => <<"somebody-else">> }) + }, + Opts + ), + ?assertEqual(85, balance_of(State, SellerAddr, Opts)), + ?assertEqual( + <<"open">>, + maps:get(<<"status">>, only_order(State, Opts)) + ) + end, + [<<"set">>, <<"keys">>, <<"info">>] + ). + +%% @doc The next height at which anything happens is recorded, so that the +%% slots in between cost a single comparison rather than a walk of the orders. +next_deadline_tracked_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + {Buyer, _} = party(), + Opened = + apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 200), 100, Opts), + ?assertEqual(200, hb_ao:get(<<"next-deadline">>, Opened, 0, Opts)), + OrderID = order_id(only_order(Opened, Opts)), + Reserved = + apply_tx(Opened, order_action(Buyer, <<"register-interest">>, OrderID), 110, Opts), + % The reservation lapses before the deadline, so it is the next event. + ?assertEqual( + 110 + ?DEFAULT_RESERVATION_BLOCKS + 1, + hb_ao:get(<<"next-deadline">>, Reserved, 0, Opts) + ). From 37d514e67a914496c0c90591dab66208803286f1 Mon Sep 17 00:00:00 2001 From: Rani Elhusseini Date: Sat, 25 Jul 2026 12:41:13 +0200 Subject: [PATCH 11/27] fix: read payment from L1 tx fields --- src/preloaded/process/dev_arweave_swap.erl | 87 ++++++++++++++++++++-- 1 file changed, 79 insertions(+), 8 deletions(-) diff --git a/src/preloaded/process/dev_arweave_swap.erl b/src/preloaded/process/dev_arweave_swap.erl index 3afbe8c547..25c2c4ac3a 100644 --- a/src/preloaded/process/dev_arweave_swap.erl +++ b/src/preloaded/process/dev_arweave_swap.erl @@ -143,7 +143,7 @@ compute(Base, Assignment, Opts) -> ProcID = hb_ao:get(<<"process">>, Assignment, <<>>, Opts), Body = hb_ao:get(<<"body">>, Assignment, #{}, Opts), Advanced = advance(Base, Height, Opts), - case hb_ao:get(<<"target">>, Body, <<>>, Opts) of + case tx_field_target(Body, Opts) of ProcID -> {ok, control(Advanced, Body, Height, Opts)}; Target -> {ok, payment(Advanced, Body, Target, Height, Opts)} end. @@ -308,7 +308,7 @@ payment(Base, Body, Target, Height, Opts) -> % against an order that is already spent. false ?= Buyer =:= Creator, false ?= Buyer =:= Recipient, - {ok, Paid} ?= amount(<<"quantity">>, Body, Opts), + {ok, Paid} ?= tx_field_quantity(Body, Opts), true ?= Paid >= Asking, true ?= Height =< Deadline + cancel_grace(Base, Opts), settle(Base, Order, Buyer, Height, Body, Opts) @@ -648,6 +648,23 @@ note(Base, Event, OrderID, _Opts) -> amount(Key, Body, Opts) -> hb_util:safe_int(hb_ao:get(Key, Body, 0, Opts)). +%% @doc Read a value from the real L1 transaction fields recorded in the +%% `tx@1.0' commitment. Top-level keys may come from tags with the same names, so +%% payment routing and amount checks must not use them. +tx_field(Body, Field, Default, Opts) -> + case hb_message:commitment(#{ <<"commitment-device">> => <<"tx@1.0">> }, Body, Opts) of + {ok, _ID, Commitment} -> + hb_maps:get(<<"field-", Field/binary>>, Commitment, Default, Opts); + _ -> + Default + end. + +tx_field_target(Body, Opts) -> + tx_field(Body, <<"target">>, <<>>, Opts). + +tx_field_quantity(Body, Opts) -> + hb_util:safe_int(tx_field(Body, <<"quantity">>, 0, Opts)). + %% @doc The single signer of a message. A message with any other number of %% signers is not attributable to one party, so it cannot open, cancel, reserve %% or pay for anything. @@ -699,6 +716,11 @@ tx(Wallet, Fields) -> #{ <<"commitment-device">> => <<"tx@1.0">> } ). +%% @doc A transaction that carries trade keys as tags only. +tag_only_tx(Wallet, Tags) -> + Signed = ar_tx:sign(#tx{ format = 2, reward = 1, tags = Tags }, Wallet), + hb_message:convert(Signed, <<"structured@1.0">>, <<"tx@1.0">>, #{}). + %% @doc Sequence a transaction into the process at a block height, as %% `~arweave-scheduler@1.0' in `all' mode does. apply_tx(Base, Body, Height, Opts) -> @@ -755,6 +777,16 @@ pay(Wallet, To, Winston, OrderID) -> } ). +tag_only_transfer(Wallet, To, Winston, OrderID) -> + tag_only_tx( + Wallet, + [ + {<<"target">>, To}, + {<<"quantity">>, hb_util:bin(Winston)}, + {<<"order-id">>, OrderID} + ] + ). + only_order(Base, Opts) -> [Order] = orders(Base, Opts), Order. @@ -792,6 +824,28 @@ make_offer_stale_deadline_test() -> Result = apply_tx(Base, offer(Seller, 10, 500, 5, 100), 100, Opts), ?assertEqual([], orders(Result, Opts)). +%% @doc Tag-only trade keys are metadata and do not route control slots. +tag_only_target_is_metadata_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + Tagged = + tag_only_tx( + Seller, + [ + {<<"target">>, ?PROCESS}, + {<<"action">>, <<"make-offer">>}, + {<<"offer-quantity">>, <<"10">>}, + {<<"asking">>, <<"500">>}, + {<<"deposit">>, <<"5">>}, + {<<"deadline">>, <<"200">>} + ] + ), + ?assertEqual(?PROCESS, hb_ao:get(<<"target">>, Tagged, not_found, Opts)), + ?assertEqual(<<>>, tx_field_target(Tagged, Opts)), + Result = apply_tx(base(#{ SellerAddr => 100 }), Tagged, 100, Opts), + ?assertEqual([], orders(Result, Opts)), + ?assertEqual(100, balance_of(Result, SellerAddr, Opts)). + %% @doc The whole trade: the buyer pays the seller directly on layer one, and %% the process -- which is not a party to that payment -- settles it. settlement_test() -> @@ -842,6 +896,24 @@ payment_to_wrong_address_test() -> ?assertEqual(<<"open">>, maps:get(<<"status">>, only_order(Result, Opts))), ?assertEqual(0, balance_of(Result, BuyerAddr, Opts)). +%% @doc Tag-only trade keys are metadata and do not count as the payment leg. +tag_only_transfer_is_metadata_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + {Buyer, BuyerAddr} = party(), + Opened = + apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 200), 100, Opts), + OrderID = order_id(only_order(Opened, Opts)), + Tagged = tag_only_transfer(Buyer, SellerAddr, 500, OrderID), + ?assertEqual(SellerAddr, hb_ao:get(<<"target">>, Tagged, not_found, Opts)), + ?assertEqual(<<"500">>, hb_ao:get(<<"quantity">>, Tagged, not_found, Opts)), + ?assertEqual(<<>>, tx_field_target(Tagged, Opts)), + ?assertEqual({ok, 0}, tx_field_quantity(Tagged, Opts)), + Result = apply_tx(Opened, Tagged, 120, Opts), + ?assertEqual(<<"open">>, maps:get(<<"status">>, only_order(Result, Opts))), + ?assertEqual(0, balance_of(Result, BuyerAddr, Opts)), + ?assertEqual(85, balance_of(Result, SellerAddr, Opts)). + %% @doc Cancelling returns the goods at once, but holds the bond for as long as %% a payment could still be in flight. cancel_returns_goods_but_holds_bond_test() -> @@ -1147,17 +1219,16 @@ non_numeric_tag_is_ignored_test() -> ?assertEqual([], orders(Result, Opts)), ?assertEqual(100, balance_of(Result, SellerAddr, Opts)). -%% @doc An `order-id' is a stranger's text. Naming a path into the order book, -%% or one of a message's own reserved keys, must not reach anything that is then -%% read as an order. -hostile_order_id_test() -> +%% @doc An `order-id' is caller-supplied text. Path-like values and reserved +%% keys must not reach anything that is then read as an order. +reserved_order_id_names_test() -> Opts = test_opts(), {Seller, SellerAddr} = party(), {Buyer, _} = party(), Opened = apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 200), 100, Opts), OrderID = order_id(only_order(Opened, Opts)), - Hostile = + Names = [ <>, <<"keys">>, @@ -1172,7 +1243,7 @@ hostile_order_id_test() -> maps:get(<<"status">>, only_order(Result, Opts)) ) end, - Hostile + Names ). %% @doc A seller cannot buy their own order. Paying oneself costs only a network From 4798e3bd4829a89cf33c9bffff136484585ece56 Mon Sep 17 00:00:00 2001 From: Rani Elhusseini Date: Sat, 25 Jul 2026 12:45:09 +0200 Subject: [PATCH 12/27] fix: 43 chars test ?PROCESS --- src/preloaded/process/dev_arweave_swap.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/preloaded/process/dev_arweave_swap.erl b/src/preloaded/process/dev_arweave_swap.erl index 25c2c4ac3a..570de7d1de 100644 --- a/src/preloaded/process/dev_arweave_swap.erl +++ b/src/preloaded/process/dev_arweave_swap.erl @@ -695,7 +695,7 @@ cancel_grace(Base, Opts) -> %%% `~process@1.0' would: the device reads chain data but performs no I/O, so a %%% whole trade can be played out without a weave. --define(PROCESS, <<"pRoCeSs00000000000000000000000000000000000">>). +-define(PROCESS, <<"pRoCeSs000000000000000000000000000000000000">>). test_opts() -> #{ <<"priv-wallet">> => ar_wallet:new() }. From b3094ccc12f71c68845ce1fd38f13e38ca63a2dc Mon Sep 17 00:00:00 2001 From: Rani Elhusseini Date: Sat, 25 Jul 2026 15:04:44 +0200 Subject: [PATCH 13/27] chore: defensive ignoring of duplicate OrderID make_offer/4 --- src/preloaded/process/dev_arweave_swap.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/preloaded/process/dev_arweave_swap.erl b/src/preloaded/process/dev_arweave_swap.erl index 570de7d1de..d964c3405c 100644 --- a/src/preloaded/process/dev_arweave_swap.erl +++ b/src/preloaded/process/dev_arweave_swap.erl @@ -179,6 +179,7 @@ make_offer(Base, Body, Height, Opts) -> true ?= Deadline > Height, true ?= balance(Base, Seller, Opts) >= Quantity + Deposit, OrderID = hb_util:human_id(hb_message:id(Body, signed, Opts)), + not_found ?= hb_maps:get(OrderID, order_book(Base, Opts), not_found, Opts), Order = #{ <<"order-id">> => OrderID, From b320bd74eace20c0414016ba54d42299828d97fb Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Sat, 25 Jul 2026 01:18:08 -0400 Subject: [PATCH 14/27] feat(arweave-swap): let an order charge to register interest Reserving an order costs the seller something: it freezes their bond and their right to cancel behind somebody who may never pay. The only thing making that costly to abuse was the seller's `deposit', which protects the buyer, not the seller -- and an order with no bond had no protection at all. An order may now declare a `minimum-fee' in winston, which a `register-interest' transaction pays as its own transaction `reward' -- the fee the network is already charging to accept it. Overpaying the reward sends the difference where Arweave sends rewards, to miners and the endowment, so the deterrent funds the weave. The alternative was to have the registration send value to the process, but a process address is a transaction id with no key behind it: that would stranded the fee there forever. Value sent to the process is therefore explicitly not the fee and does not clear it. Being denominated in AR, the fee asks nothing of a buyer who holds none of the token they are trying to buy. That is what makes a name sellable: a name's whole supply is the single unit being offered, so its seller has nothing left to bond with, and a zero-deposit order is exactly the case that needed defending. `make-offer' asks such an order only for the goods. Tested: an order that charges turns away a registration that pays nothing, one that underpays, and one that sends value to the process instead; it admits a registration whose reward covers the fee. An order that charges nothing still registers free, and the holder of a single unit can offer it. --- src/preloaded/process/dev_arweave_swap.erl | 147 ++++++++++++++++++++- 1 file changed, 142 insertions(+), 5 deletions(-) diff --git a/src/preloaded/process/dev_arweave_swap.erl b/src/preloaded/process/dev_arweave_swap.erl index d964c3405c..3b4471f810 100644 --- a/src/preloaded/process/dev_arweave_swap.erl +++ b/src/preloaded/process/dev_arweave_swap.erl @@ -31,11 +31,20 @@ %%% of AR sent to the process -- an address with no key, which would %%% destroy it. %%%
  • `register-interest' (to the process, from a buyer), naming an -%%% `order-id'. It moves no value. It buys exclusivity: for -%%% `swap-reservation-blocks' the order is that buyer's alone and the -%%% seller cannot cancel it. That window is what makes paying safe -- -%%% without it a buyer races the seller's cancellation, having already -%%% sent AR that nobody can claw back.
  • +%%% `order-id'. It buys exclusivity: for `swap-reservation-blocks' the +%%% order is that buyer's alone and the seller cannot cancel it. That +%%% window is what makes paying safe -- without it a buyer races the +%%% seller's cancellation, having already sent AR that nobody can claw +%%% back. An order may charge a `minimum-fee' in winston to register, +%%% which the registration pays as its own transaction `reward': +%%% reserving an order costs the seller their bond and their right to +%%% cancel, so making it free makes it worth abusing. Paying it as the +%%% reward sends it where Arweave sends rewards -- to miners and the +%%% endowment -- rather than stranding it at the process's address, which +%%% is a transaction id with no key behind it. The fee is the protection +%%% an order can offer when it has no bond to post -- a name, whose whole +%%% supply is the single unit being sold -- and being denominated in AR it +%%% asks nothing of a buyer who holds none of the token. %%%
  • `cancel-order' (to the process, from the seller), naming an %%% `order-id'. Returns the goods, but not the deposit (see below).
  • %%%
  • The payment itself: an ordinary transfer whose `target' is the @@ -171,12 +180,18 @@ make_offer(Base, Body, Height, Opts) -> {ok, Quantity} ?= amount(<<"offer-quantity">>, Body, Opts), {ok, Asking} ?= amount(<<"asking">>, Body, Opts), {ok, Deposit} ?= amount(<<"deposit">>, Body, Opts), + {ok, Fee} ?= amount(<<"minimum-fee">>, Body, Opts), {ok, Deadline} ?= amount(<<"deadline">>, Body, Opts), Recipient = hb_ao:get(<<"recipient">>, Body, Seller, Opts), true ?= Quantity >= 1, true ?= Asking >= 1, true ?= Deposit >= 0, + true ?= Fee >= 0, true ?= Deadline > Height, + % An order with no bond asks only that the seller hold what they are + % offering. A seller whose whole holding is the thing being sold -- the + % single unit of a name -- has nothing left to bond with, and requiring + % otherwise would put a name beyond sale. true ?= balance(Base, Seller, Opts) >= Quantity + Deposit, OrderID = hb_util:human_id(hb_message:id(Body, signed, Opts)), not_found ?= hb_maps:get(OrderID, order_book(Base, Opts), not_found, Opts), @@ -188,6 +203,7 @@ make_offer(Base, Body, Height, Opts) -> <<"quantity">> => Quantity, <<"asking">> => Asking, <<"deposit">> => Deposit, + <<"minimum-fee">> => Fee, <<"deadline">> => Deadline, <<"created-at">> => Height, <<"status">> => <<"open">> @@ -251,6 +267,17 @@ register_interest(Base, Body, Height, Opts) -> #{ <<"status">> := Status, <<"deadline">> := Deadline } = Order, true ?= Status =:= <<"open">>, true ?= Height < Deadline, + % Reserving an order costs the seller: it freezes their bond and their + % right to cancel behind somebody who may never pay. An order may + % therefore demand a fee to register, and the registration pays it as + % its own transaction `reward' -- the fee the network is already + % charging to accept it. Overpaying that goes where Arweave sends + % rewards, which is to miners and the endowment; sending it to the + % process instead would strand it at an address with no key behind it. + % Being denominated in AR, it asks nothing of a buyer who holds none of + % the token they are trying to buy. + {ok, Paid} ?= amount(<<"reward">>, Body, Opts), + true ?= Paid >= minimum_fee(Order), Until = min(Deadline, Height + reservation_blocks(Base, Opts)), ?event( {swap_interest_registered, @@ -558,6 +585,7 @@ order(Held) -> [ <<"quantity">>, <<"deposit">>, + <<"minimum-fee">>, <<"asking">>, <<"deadline">>, <<"created-at">>, @@ -610,6 +638,8 @@ quantity(Order) -> hb_util:int(maps:get(<<"quantity">>, Order, 0)). deposit(Order) -> hb_util:int(maps:get(<<"deposit">>, Order, 0)). +minimum_fee(Order) -> hb_util:int(maps:get(<<"minimum-fee">>, Order, 0)). + %% @doc Read an address's token balance from the ledger this process shares. %% Only the one entry is read: the ledger may be large, and the rest of it is %% none of this device's business. @@ -1306,6 +1336,113 @@ reserved_paths_are_applied_test() -> [<<"set">>, <<"keys">>, <<"info">>] ). +%% @doc An offer that charges to register turns away a registration that does +%% not pay the fee, and lets one that does through. The fee is the +%% registration's own reward, so it is paid to the network rather than to an +%% address nobody holds. +minimum_fee_gates_registration_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + {Buyer, _} = party(), + Charged = + tx( + Seller, + #{ + <<"target">> => ?PROCESS, + <<"action">> => <<"make-offer">>, + <<"offer-quantity">> => <<"10">>, + <<"asking">> => <<"500">>, + <<"deposit">> => <<"0">>, + <<"minimum-fee">> => <<"1000">>, + <<"deadline">> => <<"200">> + } + ), + Opened = apply_tx(base(#{ SellerAddr => 100 }), Charged, 100, Opts), + OrderID = order_id(only_order(Opened, Opts)), + Free = + apply_tx( + Opened, + order_action(Buyer, <<"register-interest">>, OrderID), + 110, + Opts + ), + ?assertEqual(<<"open">>, maps:get(<<"status">>, only_order(Free, Opts))), + % Value sent to the process is not the fee: it would be stranded there, and + % it is the reward that the order asks for. + Stranded = + apply_tx( + Opened, + tx( + Buyer, + #{ + <<"target">> => ?PROCESS, + <<"action">> => <<"register-interest">>, + <<"order-id">> => OrderID, + <<"quantity">> => <<"100000">> + } + ), + 110, + Opts + ), + ?assertEqual(<<"open">>, maps:get(<<"status">>, only_order(Stranded, Opts))), + Underpaid = + apply_tx( + Opened, + registration(Buyer, OrderID, 999), + 110, + Opts + ), + ?assertEqual(<<"open">>, maps:get(<<"status">>, only_order(Underpaid, Opts))), + Reserved = apply_tx(Opened, registration(Buyer, OrderID, 1000), 110, Opts), + ?assertEqual( + <<"reserved">>, + maps:get(<<"status">>, only_order(Reserved, Opts)) + ). + +%% @doc An order that charges nothing is still free to register on. +no_minimum_fee_registers_free_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + {Buyer, _} = party(), + Opened = + apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 200), 100, Opts), + OrderID = order_id(only_order(Opened, Opts)), + Reserved = + apply_tx( + Opened, + order_action(Buyer, <<"register-interest">>, OrderID), + 110, + Opts + ), + ?assertEqual( + <<"reserved">>, + maps:get(<<"status">>, only_order(Reserved, Opts)) + ). + +%% @doc A seller whose whole holding is the single unit they are selling can +%% still offer it, because an order with no bond asks only for the goods. +single_unit_offer_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + Opened = + apply_tx(base(#{ SellerAddr => 1 }), offer(Seller, 1, 500, 0, 200), 100, Opts), + Order = only_order(Opened, Opts), + ?assertEqual(1, quantity(Order)), + ?assertEqual(0, deposit(Order)), + ?assertEqual(0, balance_of(Opened, SellerAddr, Opts)). + +%% @doc A registration paying a fee, as the reward on its own transaction. +registration(Wallet, OrderID, Winston) -> + tx( + Wallet, + #{ + <<"target">> => ?PROCESS, + <<"action">> => <<"register-interest">>, + <<"order-id">> => OrderID, + <<"reward">> => hb_util:bin(Winston) + } + ). + %% @doc The next height at which anything happens is recorded, so that the %% slots in between cost a single comparison rather than a walk of the orders. next_deadline_tracked_test() -> From e6bb63d0eeaff69d925c477eef6623ededcd6c97 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Sat, 25 Jul 2026 09:38:49 -0400 Subject: [PATCH 15/27] feat: add ~name-token@1.0, a name whose holder says what it means A name is a token with a single unit and a message attached. Whoever holds the unit decides what that message says; transferring the unit hands over that right with it. Selling the unit -- with `~arweave-swap@1.0', which settles in the same balances -- sells the name. The device is `token-1.0''s shape carrying `~reference@1.0''s payload. From the token: the state layout, the `transfer' action and its `Debit-Notice' / `Credit-Notice' pair, and the rule for who may `set' -- the holder of the whole supply, which `token-1.0' already calls `supply-threshold-owner'. From the reference: what `set' writes, a linked message the name resolves through to, given as a `reference-value' or inherited from the `set' message's own keys. There is no mint path, so a name cannot be diluted into existing twice. Ordering is simply the schedule's. `~reference@1.0' must reconstruct which of several competing `set's won, from a signer-declared timestamp tie-broken by weave position, because it is assembled out of loose layer-1 messages. A process is handed that order already. Two things are shaped by the chain rather than by taste. A process spawned as an Arweave transaction may only carry scalars -- the codec turns a submessage or a list into a `+link' to content the weave does not hold, and keys are lowercased on the way, which would quietly rewrite an address. So the single unit is not written into the spawn: the spawn names its first holder in a value, `initial-holder', and the unit appears the first time the process computes. And the selling device is named by a scalar, `swap-device', with this device handing each message to it before applying its own actions -- `~stack@1.0' written out by hand, because `device-stack' is a list and a list does not survive the spawn. Reads are paths into the state, as `token-1.0' reads too. The device exports no key list and answers every key with `compute', because a name that is for sale must be sequenced by every transaction on Arweave, which means a stranger's `path' tag chooses the key each slot resolves. Tested against three names that were really sold, transferred and withdrawn on mainnet, covering every path and refusal: * a name that had to be paid for -- a registration paying only what the network charges is refused, so is one that sends the fee to the process instead of paying it as its reward, and the one that overpays the reward reserves it; a payment from somebody who did not reserve it buys nothing; the registrant's payment settles; the seller can no longer speak for the name and the buyer can; a stranger routing a slot at `info' breaks nothing and an offer whose deadline is `tomorrow' opens nothing. * an offer withdrawn -- the unit comes back, a later registration paying the fee in full is refused, a later payment moves nothing in either direction, and a stranger cannot withdraw somebody else's order. * a name handed over -- a plain transfer, its notices, and the authority moving with the unit. Each story pins its transaction ids, resolves each to the slot it landed at, and reads the state forward one slot at a time (`~process@1.0/compute&slot=/orders//status'), so what the schedule did is visible in the order it happened. The driver that seeded them is here too, gated behind HB_LIVE_SUITE because it spends real AR, reporting through `?event'. --- src/preloaded/process/dev_name_token.erl | 1816 ++++++++++++++++++++++ 1 file changed, 1816 insertions(+) create mode 100644 src/preloaded/process/dev_name_token.erl diff --git a/src/preloaded/process/dev_name_token.erl b/src/preloaded/process/dev_name_token.erl new file mode 100644 index 0000000000..cb827d1478 --- /dev/null +++ b/src/preloaded/process/dev_name_token.erl @@ -0,0 +1,1816 @@ +%%% @doc A name: a token with a single unit, whose holder decides what the +%%% name resolves to. +%%% +%%% The device is `token-1.0''s shape carrying `~reference@1.0''s payload. From +%%% the token it takes the state layout -- `name', `ticker', `denomination', +%%% `total-supply' and `balances' on a flat `~process@1.0' message -- the +%%% `transfer' action and its notices, and the rule that decides who may `set': +%%% the holder of the whole supply. From the reference it takes what `set' +%%% actually writes -- a linked message whose keys the name then resolves +%%% through to. +%%% +%%% Because the supply is one indivisible unit, those two things compose into a +%%% name: exactly one address holds it, that address alone may say what the name +%%% points at, and transferring the unit hands over that right with it. Selling +%%% the unit -- with `~arweave-swap@1.0', which settles against the same +%%% `balances' -- sells the name. +%%% +%%% The protocol is two messages: +%%%
      +%%%
    • `transfer', carrying `recipient' and `quantity'. The sender is the +%%% message's signer. A `Debit-Notice' and a `Credit-Notice' are emitted +%%% with the same keys `token-1.0' uses.
    • +%%%
    • `set', carrying either a `reference-value' -- the message the name is +%%% to resolve to -- or nothing, in which case the name inherits the keys +%%% of the `set' message itself, as `~reference@1.0' defines it. Only the +%%% holder of the whole supply may send it, judged against the balances as +%%% they stand at that slot.
    • +%%%
    +%%% +%%% There is no mint path, so the supply is fixed at spawn: a name cannot be +%%% diluted into existing twice. +%%% +%%% Ordering is the schedule's. `~reference@1.0' has to reconstruct which of +%%% several competing `set's won -- by a signer-declared timestamp, tie-broken +%%% by weave position -- because it is assembled from loose layer-1 messages. A +%%% process is handed that order already: the latest `set' in slot order wins, +%%% and there is nothing to declare and nothing to tie-break. +%%% +%%% Reads are paths into the state -- `/now/balances/
    ', +%%% `/now/total-supply', `/now/value/' -- and not device keys, which is how +%%% `token-1.0' reads too. The device deliberately exports no key list: a name +%%% is meant to be sold by `~arweave-swap@1.0', which requires its process to be +%%% sequenced by every transaction on Arweave, so the key a slot resolves is +%%% chosen by a stranger's `path' tag. An exported `balance' key would let a +%%% passer-by's transaction hand back a balance as the new process state. +-module(dev_name_token). +-implements(<<"name-token@1.0">>). +%%% AO-Core API functions: +-export([info/0, compute/3, set/3, keys/3]). +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +%%% The balances submessage, shared with any device that settles in this token. +-define(BALANCES, <<"balances">>). +%%% The linked message whose keys the name resolves through to. +-define(VALUE, <<"value">>). +%%% The share of the supply a signer must hold to set the name, in basis +%%% points. The whole supply, unless the token says otherwise. +-define(DEFAULT_THRESHOLD_BPS, 10000). + +%% @doc Every key routes to `compute': the schedule drives this device, and +%% under `~arweave-scheduler@1.0''s `all' mode the key a slot resolves comes +%% from a stranger's `path' tag. See `dev_arweave_swap:info/0', which carries +%% the same reasoning at length. +info() -> + #{ default => fun router/4 }. + +%% @doc Apply any scheduled message, whatever it asked to be routed to. +router(_Key, Base, Assignment, Opts) -> + compute(Base, Assignment, Opts). + +%% @doc Apply one assignment to the name's state. +%% +%% A name that is for sale is settled in the same balances it keeps, so every +%% message is offered to the selling device first -- including the ones this +%% device would otherwise ignore, since the payment that buys a name is a +%% transfer between two other addresses. Which device that is, is a scalar key +%% on the process; see `swap/3' for why it is not a `~stack@1.0'. +compute(Base, Assignment, Opts) -> + Seeded = seed(Base, Opts), + Sold = swap(Seeded, Assignment, Opts), + Body = hb_ao:get(<<"body">>, Assignment, #{}, Opts), + ProcID = hb_ao:get(<<"process">>, Assignment, <<>>, Opts), + case hb_ao:get(<<"target">>, Body, <<>>, Opts) of + ProcID -> {ok, action(Sold, Body, Opts)}; + _ -> + % Not addressed to this name. Under `all' mode that is almost every + % transaction on the network. + {ok, Sold} + end. + +%% @doc Hand the message to the device that sells this name, if it has one, and +%% take back the state it produces. +%% +%% This is a `~stack@1.0' written out by hand, and deliberately so. A process +%% spawned as an Arweave transaction can only carry flat, scalar tags: the +%% codec turns a submessage or a list into a `+link' to content that is not on +%% the weave, so nothing else can read it back. `device-stack' is a list, so a +%% stack cannot survive the spawn -- but `swap-device' is one word. +swap(Base, Assignment, Opts) -> + case state(<<"swap-device">>, Base, not_found, Opts) of + not_found -> Base; + Device -> + case hb_ao:resolve(Base#{ <<"device">> => Device }, Assignment, Opts) of + {ok, Settled} -> Settled#{ <<"device">> => <<"name-token@1.0">> }; + _ -> Base + end + end. + +%% @doc Give the name its single unit the first time it computes. +%% +%% The holding cannot be written into the spawn: `balances' is a submessage, and +%% a submessage does not cross the chain. What does cross is the address itself, +%% as the value of `initial-holder' -- values keep their case, where keys are +%% lowercased and a lowercased address is a different address. +seed(Base, Opts) -> + case state(<<"initial-holder">>, Base, not_found, Opts) of + not_found -> Base; + Holder -> + case state(?BALANCES, Base, not_found, Opts) of + not_found -> + Supply = hb_util:int(state(<<"total-supply">>, Base, 1, Opts)), + ?event({name_token_seeded, {holder, Holder}, {supply, Supply}}), + Base#{ ?BALANCES => #{ Holder => Supply } }; + _ -> Base + end + end. + +%% @doc Route a message addressed to the name by its `action'. Matching is +%% case-insensitive, as `token-1.0' matches. An unknown action leaves the state +%% untouched rather than failing the slot, which would stop the process on every +%% node for good. +action(Base, Body, Opts) -> + case hb_util:to_lower(hb_ao:get(<<"action">>, Body, <<>>, Opts)) of + <<"transfer">> -> transfer(Base, Body, Opts); + <<"set">> -> set_value(Base, Body, Opts); + _ -> Base + end. + +%%% The token + +%% @doc Move units between balances. Nothing is written unless the whole +%% transfer is admissible, so a rejected one is indistinguishable from a +%% message that was never sent. +transfer(Base, Body, Opts) -> + maybe + {ok, Sender} ?= signer(Body, Opts), + Recipient = hb_ao:get(<<"recipient">>, Body, not_found, Opts), + true ?= is_binary(Recipient), + {ok, Quantity} ?= hb_util:safe_int(hb_ao:get(<<"quantity">>, Body, 0, Opts)), + true ?= Quantity >= 1, + true ?= balance(Base, Sender, Opts) >= Quantity, + ?event( + {name_token_transfer, + {from, Sender}, + {to, Recipient}, + {quantity, Quantity} + } + ), + notices( + credit( + debit(Base, Sender, Quantity, Opts), + Recipient, + Quantity, + Opts + ), + Sender, + Recipient, + Quantity, + Opts + ) + else + _ -> Base + end. + +%% @doc Emit the pair of notices that `token-1.0' emits for a transfer, with +%% the same keys. They are the slot's results; whether anything delivers them +%% is the process's business, not this device's. +notices(Base, Sender, Recipient, Quantity, _Opts) -> + Base#{ + <<"results">> => + #{ + <<"outbox">> => + #{ + <<"1">> => + #{ + <<"target">> => Sender, + <<"action">> => <<"Debit-Notice">>, + <<"recipient">> => Recipient, + <<"quantity">> => Quantity + }, + <<"2">> => + #{ + <<"target">> => Recipient, + <<"action">> => <<"Credit-Notice">>, + <<"sender">> => Sender, + <<"quantity">> => Quantity + } + } + } + }. + +%%% The name + +%% @doc Write the message the name resolves to. The value is the message's +%% `reference-value' if it carries one, and otherwise the message itself, less +%% the keys that carried it here -- exactly the choice `~reference@1.0' offers. +set_value(Base, Body, Opts) -> + maybe + {ok, Signer} ?= signer(Body, Opts), + true ?= owns_supply(Base, Signer, Opts), + Value = value_of(Body, Opts), + ?event({name_token_set, {by, Signer}}), + Base#{ ?VALUE => Value } + else + _ -> Base + end. + +%% @doc The message a `set' is asking the name to resolve to. +value_of(Body, Opts) -> + case hb_ao:get(<<"reference-value">>, Body, not_found, Opts) of + not_found -> + % The set message itself is the value. The keys that addressed it to + % this name are not part of what the name says. + hb_maps:without( + [ + <<"action">>, + <<"target">>, + <<"quantity">>, + <<"anchor">>, + <<"reward">>, + <<"last_tx">>, + <<"owner">>, + <<"signature">>, + <<"commitments">>, + <<"priv">> + ], + hb_cache:ensure_all_loaded(Body, Opts), + Opts + ); + Value -> hb_cache:ensure_all_loaded(Value, Opts) + end. + +%% @doc Whether an address may speak for the name: it must hold the share of the +%% supply the token requires, which is all of it unless the token says +%% otherwise. This is `token-1.0''s `supply-threshold-owner' rule, and it is +%% evaluated against the balances as they stand -- so the authority moves with +%% the unit, with no separate owner field to keep in step. +owns_supply(Base, Address, Opts) -> + Supply = hb_util:int(state(<<"total-supply">>, Base, 1, Opts)), + Threshold = + hb_util:int( + state( + <<"set-authority-threshold-bps">>, + Base, + ?DEFAULT_THRESHOLD_BPS, + Opts + ) + ), + balance(Base, Address, Opts) * 10000 >= Supply * Threshold. + +%%% State + +%% @doc Read a key of the process's own state. While a slot is being computed +%% the state carries this device, so a plain read would resolve the key back +%% through `compute'. See `dev_arweave_swap:state/4'. +state(Key, Base, Default, Opts) -> + hb_ao:get(Key, {as, <<"message@1.0">>, Base}, Default, Opts). + +balance(Base, Address, Opts) -> + hb_util:int(state([?BALANCES, Address], Base, 0, Opts)). + +credit(Base, Address, Amount, Opts) -> + write_balance(Base, Address, balance(Base, Address, Opts) + Amount, Opts). + +debit(Base, Address, Amount, Opts) -> + write_balance(Base, Address, balance(Base, Address, Opts) - Amount, Opts). + +%% @doc Write one balance back. Only whole top-level keys are written: setting a +%% nested path would resolve the keys above it through this device on the way +%% down. +write_balance(Base, Address, Value, Opts) -> + Base#{ + ?BALANCES => + hb_maps:put( + Address, + Value, + state(?BALANCES, Base, #{}, Opts), + Opts + ) + }. + +%% @doc Setting the device is honoured, and nothing else is: `lib_process' puts +%% the process's own device back after every slot, and reading this state as a +%% message is itself a device set. Anything else is a scheduled message and is +%% applied as one. See `dev_arweave_swap:set/3'. +set(Base, Req, Opts) -> + case hb_maps:keys(Req, Opts) -- [<<"path">>, <<"set-mode">>] of + [<<"device">>] -> + {ok, + Base#{ + <<"device">> => + hb_maps:get(<<"device">>, Req, undefined, Opts) + } + }; + _ -> compute(Base, Req, Opts) + end. + +%% @doc Listing the state's keys is not a state transition, so a scheduled +%% message asking for it is applied as one. +keys(Base, Req, Opts) -> compute(Base, Req, Opts). + +%% @doc The single signer of a message. A message with any other number of +%% signers is not attributable to one party, so it can neither move the unit nor +%% speak for the name. +signer(Body, Opts) -> + case hb_message:signers(Body, Opts) of + [Signer] -> {ok, hb_util:human_id(Signer)}; + _ -> not_found + end. + +%%% Tests + +%%% The tests drive `compute/3' directly with synthetic assignments, exactly as +%%% `~process@1.0' would. Below them is the live-network driver that seeded the +%%% permanent fixture, and the fixture test that replays it. + +-define(PROCESS, <<"nAmEtOkEn000000000000000000000000000000000">>). + +test_opts() -> #{ <<"priv-wallet">> => ar_wallet:new() }. + +party() -> + Wallet = ar_wallet:new(), + {Wallet, hb_util:human_id(ar_wallet:to_address(Wallet))}. + +%% @doc A name held by one address, as it stands at spawn. +name_held_by(Address) -> + #{ + <<"name">> => <<"test-name">>, + <<"ticker">> => <<"NAME">>, + <<"denomination">> => 0, + <<"total-supply">> => 1, + ?BALANCES => #{ Address => 1 } + }. + +tx(Wallet, Fields) -> + hb_message:commit( + Fields, + #{ <<"priv-wallet">> => Wallet }, + #{ <<"commitment-device">> => <<"tx@1.0">> } + ). + +apply_tx(Base, Body, Opts) -> + {ok, New} = + compute( + Base, + #{ + <<"process">> => ?PROCESS, + <<"slot">> => 1, + <<"body">> => Body + }, + Opts + ), + New. + +transfer_tx(Wallet, Recipient, Quantity) -> + tx( + Wallet, + #{ + <<"target">> => ?PROCESS, + <<"action">> => <<"transfer">>, + <<"recipient">> => Recipient, + <<"quantity">> => hb_util:bin(Quantity) + } + ). + +set_tx(Wallet, Fields) -> + tx( + Wallet, + Fields#{ <<"target">> => ?PROCESS, <<"action">> => <<"set">> } + ). + +held_by(Base, Address, Opts) -> balance(Base, Address, Opts). + +value(Base, Opts) -> + hb_cache:ensure_all_loaded(state(?VALUE, Base, #{}, Opts), Opts). + +%% @doc The unit moves, and the pair of notices `token-1.0' emits go with it. +transfer_moves_the_unit_test() -> + Opts = test_opts(), + {Owner, OwnerAddr} = party(), + {_, BuyerAddr} = party(), + Moved = + apply_tx(name_held_by(OwnerAddr), transfer_tx(Owner, BuyerAddr, 1), Opts), + ?assertEqual(0, held_by(Moved, OwnerAddr, Opts)), + ?assertEqual(1, held_by(Moved, BuyerAddr, Opts)), + Outbox = hb_ao:get(<<"results/outbox">>, {as, <<"message@1.0">>, Moved}, #{}, Opts), + ?assertEqual( + <<"Debit-Notice">>, + hb_ao:get(<<"1/action">>, Outbox, not_found, Opts) + ), + ?assertEqual( + <<"Credit-Notice">>, + hb_ao:get(<<"2/action">>, Outbox, not_found, Opts) + ), + ?assertEqual( + BuyerAddr, + hb_ao:get(<<"1/recipient">>, Outbox, not_found, Opts) + ), + ?assertEqual( + OwnerAddr, + hb_ao:get(<<"2/sender">>, Outbox, not_found, Opts) + ). + +%% @doc Nobody can send what they do not hold. +transfer_beyond_balance_test() -> + Opts = test_opts(), + {Owner, OwnerAddr} = party(), + {Stranger, _} = party(), + {_, ElsewhereAddr} = party(), + Base = name_held_by(OwnerAddr), + ?assertEqual( + 1, + held_by(apply_tx(Base, transfer_tx(Owner, ElsewhereAddr, 2), Opts), OwnerAddr, Opts) + ), + ?assertEqual( + 0, + held_by( + apply_tx(Base, transfer_tx(Stranger, ElsewhereAddr, 1), Opts), + ElsewhereAddr, + Opts + ) + ). + +%% @doc The holder says what the name resolves to, by handing over a message. +set_writes_the_linked_message_test() -> + Opts = test_opts(), + {Owner, OwnerAddr} = party(), + Set = + apply_tx( + name_held_by(OwnerAddr), + set_tx( + Owner, + #{ + <<"reference-value">> => + #{ + <<"content-type">> => <<"text/html">>, + <<"body">> => <<"

    hello

    ">> + } + } + ), + Opts + ), + ?assertEqual( + <<"text/html">>, + hb_ao:get(<<"content-type">>, value(Set, Opts), not_found, Opts) + ), + ?assertEqual( + <<"

    hello

    ">>, + hb_ao:get(<<"body">>, value(Set, Opts), not_found, Opts) + ). + +%% @doc With no `reference-value', the name inherits the keys of the `set' +%% itself -- less the keys that carried it here. +set_without_value_inherits_own_keys_test() -> + Opts = test_opts(), + {Owner, OwnerAddr} = party(), + Set = + apply_tx( + name_held_by(OwnerAddr), + set_tx(Owner, #{ <<"greeting">> => <<"ahoy">> }), + Opts + ), + Value = value(Set, Opts), + ?assertEqual(<<"ahoy">>, hb_ao:get(<<"greeting">>, Value, not_found, Opts)), + ?assertEqual(not_found, hb_ao:get(<<"action">>, Value, not_found, Opts)), + ?assertEqual(not_found, hb_ao:get(<<"target">>, Value, not_found, Opts)). + +%% @doc Anybody who does not hold the name cannot speak for it. +set_by_stranger_test() -> + Opts = test_opts(), + {_, OwnerAddr} = party(), + {Stranger, _} = party(), + Result = + apply_tx( + name_held_by(OwnerAddr), + set_tx(Stranger, #{ <<"greeting">> => <<"mine now">> }), + Opts + ), + ?assertEqual(#{}, value(Result, Opts)). + +%% @doc The authority is the holding, not a recorded owner: it goes with the +%% unit and needs nothing kept in step. This is what makes a name sellable. +authority_follows_the_unit_test() -> + Opts = test_opts(), + {Owner, OwnerAddr} = party(), + {Buyer, BuyerAddr} = party(), + Base = name_held_by(OwnerAddr), + Before = apply_tx(Base, set_tx(Owner, #{ <<"points-at">> => <<"seller">> }), Opts), + ?assertEqual( + <<"seller">>, + hb_ao:get(<<"points-at">>, value(Before, Opts), not_found, Opts) + ), + Sold = apply_tx(Before, transfer_tx(Owner, BuyerAddr, 1), Opts), + % The seller has handed over the unit, and with it the right to speak. + Stale = apply_tx(Sold, set_tx(Owner, #{ <<"points-at">> => <<"seller again">> }), Opts), + ?assertEqual( + <<"seller">>, + hb_ao:get(<<"points-at">>, value(Stale, Opts), not_found, Opts) + ), + Fresh = apply_tx(Sold, set_tx(Buyer, #{ <<"points-at">> => <<"buyer">> }), Opts), + ?assertEqual( + <<"buyer">>, + hb_ao:get(<<"points-at">>, value(Fresh, Opts), not_found, Opts) + ), + ?assertEqual(1, held_by(Fresh, BuyerAddr, Opts)), + ?assertEqual(0, held_by(Fresh, OwnerAddr, Opts)). + +%% @doc A partial holding is not the whole supply, so it does not carry the +%% authority. (A name is indivisible, but the rule is the supply threshold.) +partial_holding_cannot_set_test() -> + Opts = test_opts(), + {Owner, OwnerAddr} = party(), + {_, OtherAddr} = party(), + Split = + (name_held_by(OwnerAddr))#{ + <<"total-supply">> => 2, + ?BALANCES => #{ OwnerAddr => 1, OtherAddr => 1 } + }, + Result = apply_tx(Split, set_tx(Owner, #{ <<"greeting">> => <<"half mine">> }), Opts), + ?assertEqual(#{}, value(Result, Opts)). + +%% @doc There is no mint path, so the supply cannot grow -- an unknown action +%% is ignored rather than failing the slot. +supply_is_fixed_test() -> + Opts = test_opts(), + {Owner, OwnerAddr} = party(), + Result = + apply_tx( + name_held_by(OwnerAddr), + tx( + Owner, + #{ + <<"target">> => ?PROCESS, + <<"action">> => <<"mint">>, + <<"quantity">> => <<"1000">> + } + ), + Opts + ), + ?assertEqual(1, held_by(Result, OwnerAddr, Opts)), + ?assertEqual(1, hb_util:int(state(<<"total-supply">>, Result, 0, Opts))). + +%% @doc Actions are matched however they are cased, as `token-1.0' matches. +action_case_is_ignored_test() -> + Opts = test_opts(), + {Owner, OwnerAddr} = party(), + {_, BuyerAddr} = party(), + Moved = + apply_tx( + name_held_by(OwnerAddr), + tx( + Owner, + #{ + <<"target">> => ?PROCESS, + <<"action">> => <<"Transfer">>, + <<"recipient">> => BuyerAddr, + <<"quantity">> => <<"1">> + } + ), + Opts + ), + ?assertEqual(1, held_by(Moved, BuyerAddr, Opts)). + +%% @doc A transaction that is not addressed to this name is left alone. Under +%% `all' mode that is almost every transaction on the network. +unrelated_traffic_test() -> + Opts = test_opts(), + {Stranger, _} = party(), + {_, OwnerAddr} = party(), + Base = name_held_by(OwnerAddr), + Result = + apply_tx( + Base, + tx(Stranger, #{ <<"target">> => <<"somebody-else">>, <<"quantity">> => <<"1">> }), + Opts + ), + ?assertEqual(1, held_by(Result, OwnerAddr, Opts)). + +%% @doc A name spawned as an Arweave transaction carries only scalars, so it +%% names its first holder rather than holding a balances submessage. The unit +%% appears the first time it computes. +seeded_from_initial_holder_test() -> + Opts = test_opts(), + {Owner, OwnerAddr} = party(), + Spawned = + #{ + <<"name">> => <<"test-name">>, + <<"total-supply">> => 1, + <<"initial-holder">> => OwnerAddr + }, + % Any message at all brings the name to life, including one that is not + % addressed to it. + Alive = + apply_tx(Spawned, tx(Owner, #{ <<"target">> => <<"elsewhere">> }), Opts), + ?assertEqual(1, held_by(Alive, OwnerAddr, Opts)), + % And the holder can immediately speak for it. + Set = apply_tx(Alive, set_tx(Owner, #{ <<"greeting">> => <<"mine">> }), Opts), + ?assertEqual( + <<"mine">>, + hb_ao:get(<<"greeting">>, value(Set, Opts), not_found, Opts) + ). + +%% @doc Seeding happens once. A name whose unit has moved on is not handed a +%% fresh one by the next message that arrives. +seeding_happens_once_test() -> + Opts = test_opts(), + {Owner, OwnerAddr} = party(), + {Stranger, StrangerAddr} = party(), + Spawned = + #{ + <<"name">> => <<"test-name">>, + <<"total-supply">> => 1, + <<"initial-holder">> => OwnerAddr + }, + Alive = apply_tx(Spawned, tx(Owner, #{ <<"target">> => <<"elsewhere">> }), Opts), + Moved = apply_tx(Alive, transfer_tx(Owner, StrangerAddr, 1), Opts), + Later = + apply_tx(Moved, tx(Stranger, #{ <<"target">> => <<"elsewhere">> }), Opts), + ?assertEqual(0, held_by(Later, OwnerAddr, Opts)), + ?assertEqual(1, held_by(Later, StrangerAddr, Opts)). + +%% @doc A name with no selling device is simply a name: messages the swap would +%% have handled do nothing, and the name still works. +without_swap_device_test() -> + Opts = test_opts(), + {Owner, OwnerAddr} = party(), + Base = name_held_by(OwnerAddr), + Set = apply_tx(Base, set_tx(Owner, #{ <<"greeting">> => <<"no swap here">> }), Opts), + ?assertEqual( + <<"no swap here">>, + hb_ao:get(<<"greeting">>, value(Set, Opts), not_found, Opts) + ), + ?assertEqual(not_found, state(<<"orders">>, Set, not_found, Opts)). + +%% @doc A stranger's transaction may ask to be routed anywhere, including to +%% keys `~message@1.0' would otherwise answer. Each is applied like any other +%% message rather than handing back the state, a list of key names, or an info +%% map as the new state. +reserved_paths_are_applied_test() -> + Opts = test_opts(), + {_, OwnerAddr} = party(), + {Stranger, _} = party(), + Base = name_held_by(OwnerAddr), + lists:foreach( + fun(Path) -> + {ok, State} = + hb_ao:resolve( + Base#{ <<"device">> => <<"name-token@1.0">> }, + #{ + <<"path">> => Path, + <<"process">> => ?PROCESS, + <<"slot">> => 2, + ?BALANCES => #{ OwnerAddr => 1000000 }, + <<"body">> => + tx(Stranger, #{ <<"target">> => <<"somebody-else">> }) + }, + Opts + ), + ?assertEqual(1, held_by(State, OwnerAddr, Opts)) + end, + [<<"set">>, <<"keys">>, <<"info">>, <<"balances">>, <<"anything-else">>] + ). + +%%% The live network suite +%%% +%%% Three stories played out on mainnet, against the real weave, with every +%%% path and refusal a name sale can take. It is deliberately NOT a `_test' +%%% function: it posts real transactions and spends real AR. Invoke it by name: +%%% +%%% HB_LIVE_SUITE=1 HB_PRINT=name_token_live \\ +%%% rebar3 device test --devices dev_name_token \\ +%%% --test dev_name_token:live_suite_test_ +%%% +%%% It reports every transaction id and the slot it landed at, which is what +%%% the pinned fixtures below are made of. Each message waits for the previous +%%% one to be mined: the schedule is block order, and two messages in one block +%%% would be ordered by the block's own transaction list rather than by intent. + +%% @doc Everything live is reachable only when asked for by name in the +%% environment, so it contributes nothing to the battery. +live_suite_test_() -> live_gated("HB_LIVE_SUITE", fun live_suite/0, 21600). + +live_gated(Variable, Fun, Timeout) -> + case os:getenv(Variable) of + false -> []; + _ -> {timeout, Timeout, Fun} + end. + +%%% What the network charges, and what the orders ask for. The fee an order +%%% demands to register is set well above the price of an ordinary transaction, +%%% so that clearing it means deliberately overpaying rather than paying what +%%% the network was going to charge anyway. +-define(LIVE_MINIMUM_FEE, 100000000). +-define(LIVE_PAID_FEE, 150000000). +-define(LIVE_ASKING, 1000). + +%% @doc The wallet that pays for the live run. +live_wallet() -> + ar_wallet:load_keyfile(<<"/Users/sam/Documents/hyperbeam-key.json">>). + +%% @doc A counterparty, kept beside the node's own key so that a rerun reuses +%% it rather than paying to create another account. +live_party(Path) -> + case file:read_file(Path) of + {ok, Json} -> ar_wallet:from_json(Json); + _ -> + Wallet = ar_wallet:new(), + ok = file:write_file(Path, hb_util:bin(ar_wallet:to_json(Wallet))), + Wallet + end. + +live_address(Wallet) -> hb_util:human_id(ar_wallet:to_address(Wallet)). + +%% @doc Node options for talking to the live network, mirroring the scheduler's +%% own fixture options. +live_opts(Wallet) -> + TestStore = hb_test_utils:test_store(), + IndexStore = hb_test_utils:test_store(), + (hb_opts:default_message())#{ + <<"store">> => [ + TestStore, + #{ + <<"store-module">> => hb_store_arweave, + <<"name">> => <<"cache-arweave">>, + <<"index-store">> => [IndexStore] + }, + #{ + <<"store-module">> => hb_store_gateway, + <<"local-store">> => [TestStore] + } + ], + <<"arweave-index-store">> => #{ <<"index-store">> => [IndexStore] }, + <<"arweave-index-workers">> => 8, + <<"arweave-scheduler-confirmation-depth">> => 1, + <<"priv-wallet">> => Wallet + }. + +%% @doc What the network will charge to send a transaction of no size to an +%% address. An address that has never held AR costs thousands of times more to +%% send to, because the transaction creates the account. +live_price(Target, Opts) -> + Path = + case Target of + <<>> -> <<"/price/0">>; + _ -> <<"/price/0/", Target/binary>> + end, + {ok, Res} = hb_http:get(<<"https://arweave.net">>, Path, Opts), + hb_util:int(hb_ao:get(<<"body">>, Res, <<"0">>, Opts)). + +live_anchor(Opts) -> + {ok, Res} = hb_http:get(<<"https://arweave.net">>, <<"/tx_anchor">>, Opts), + hb_ao:get(<<"body">>, Res, <<>>, Opts). + +live_balance(Address, Opts) -> + {ok, Res} = + hb_http:get( + <<"https://arweave.net">>, + <<"/wallet/", Address/binary, "/balance">>, + Opts + ), + hb_util:int(hb_ao:get(<<"body">>, Res, <<"0">>, Opts)). + +%% @doc Sign a layer-1 transaction and hand it to the network, through the +%% scheduler's own dispatch path, then wait for it to be mined. +%% +%% A `reward' given in the fields is a floor, not a replacement: the network's +%% own price still has to be met. That is how a registration pays an order's +%% `minimum-fee' -- by overpaying the reward, which goes to miners and the +%% endowment rather than to an address with no key behind it. +live_post(Label, Fields, Wallet, Opts) -> + Target = hb_maps:get(<<"target">>, Fields, <<>>, Opts), + Floor = hb_util:int(hb_maps:get(<<"reward">>, Fields, <<"0">>, Opts)), + Reward = max(live_price(Target, Opts), Floor), + Msg = + hb_message:commit( + Fields#{ + <<"anchor">> => live_anchor(Opts), + <<"reward">> => hb_util:bin(Reward) + }, + Opts#{ <<"priv-wallet">> => Wallet }, + #{ <<"commitment-device">> => <<"tx@1.0">> } + ), + ID = hb_util:human_id(hb_message:id(Msg, signed, Opts)), + {ok, Res} = + hb_ao:resolve( + #{ <<"device">> => <<"arweave-scheduler@1.0">> }, + #{ + <<"path">> => <<"schedule">>, + <<"method">> => <<"POST">>, + <<"body">> => Msg + }, + Opts + ), + Height = live_await(ID, Opts), + ?event(name_token_live, + {posted, + {label, {string, Label}}, + {tx, {string, ID}}, + {signer, {string, live_address(Wallet)}}, + {reward, Reward}, + {status, hb_ao:get(<<"status">>, Res, none, Opts)}, + {height, Height} + } + ), + ID. + +%% @doc Wait for a transaction to be mined, returning the height it landed at. +%% While a transaction is pending the gateway answers in prose -- `Pending', +%% `Accepted' -- and only once it is in a block does it answer with the JSON +%% that carries the height. +live_await(ID, Opts) -> live_await(ID, 240, Opts). +live_await(ID, 0, _Opts) -> error({not_mined, ID}); +live_await(ID, Attempts, Opts) -> + case live_height(ID, Opts) of + not_found -> + timer:sleep(15000), + live_await(ID, Attempts - 1, Opts); + Height -> Height + end. + +live_height(ID, Opts) -> + case hb_http:get(<<"https://arweave.net">>, <<"/tx/", ID/binary, "/status">>, Opts) of + {ok, Status} -> + Body = hb_ao:get(<<"body">>, Status, <<"">>, Opts), + try hb_json:decode(Body) of + Decoded -> + case hb_ao:get(<<"block_height">>, Decoded, not_found, Opts) of + not_found -> not_found; + Height -> hb_util:int(Height) + end + catch + _:_ -> not_found + end; + _ -> not_found + end. + +%% @doc Top an account up only if it cannot pay its own way. Creating an +%% account is by far the most expensive part of a run, so a funded counterparty +%% is left alone and a rerun costs nothing here. +live_fund(Address, Need, Seller, Opts) -> + case live_balance(Address, Opts) of + Balance when Balance >= Need -> + ?event(name_token_live, + {already_funded, {address, {string, Address}}, {balance, Balance}} + ), + Balance; + Balance -> + ?event(name_token_live, + {funding, {address, {string, Address}}, {balance, Balance}} + ), + live_post( + <<"fund">>, + #{ + <<"target">> => Address, + <<"quantity">> => hb_util:bin(Need * 2) + }, + Seller, + Opts + ) + end. + +%% @doc Spawn a name on Arweave. Every key is a scalar: a submessage or a list +%% would be written to the weave as a `+link' to content the weave does not +%% hold, and no node could read the process back. +live_spawn(Name, Holder, Wallet, Opts) -> + live_post( + <<"spawn:", Name/binary>>, + #{ + <<"device">> => <<"process@1.0">>, + <<"type">> => <<"Process">>, + <<"scheduler-device">> => <<"arweave-scheduler@1.0">>, + <<"scheduler-mode">> => <<"all">>, + <<"execution-device">> => <<"name-token@1.0">>, + <<"swap-device">> => <<"arweave-swap@1.0">>, + <<"name">> => Name, + <<"ticker">> => <<"NAME">>, + <<"denomination">> => <<"0">>, + <<"total-supply">> => <<"1">>, + <<"initial-holder">> => Holder, + <<"test-suite">> => <<"name-token">> + }, + Wallet, + Opts + ). + +%% @doc Address a message to a name. The first message to a process pays to +%% create its account and must carry at least a winston to do so. +live_send(Label, ProcID, Fields, Wallet, Opts) -> + live_post( + Label, + Fields#{ <<"target">> => ProcID, <<"quantity">> => <<"1">> }, + Wallet, + Opts + ). + +%% @doc Report which slot each of a run's transactions landed at, by reading the +%% schedule back and matching each assignment's body against the ids we sent. +%% These are the numbers the fixture tests read state at. +live_slots(ProcID, Named, Opts) -> + {ok, Schedule} = + hb_ao:resolve( + #{ <<"device">> => <<"arweave-scheduler@1.0">> }, + #{ + <<"path">> => <<"schedule">>, + <<"method">> => <<"GET">>, + <<"target">> => ProcID + }, + Opts + ), + Assignments = + hb_ao:normalize_keys(hb_ao:get(<<"assignments">>, Schedule, Opts), Opts), + % A schedule's assignments are keyed by slot, alongside the keys any + % committed message carries; only the numbered ones are slots. + BySlot = + [ + { + Slot, + hb_util:human_id( + hb_message:id( + hb_ao:get(<<"body">>, Assignment, Opts), + signed, + Opts + ) + ) + } + || + {Key, Assignment} <- hb_maps:to_list(Assignments, Opts), + {ok, Slot} <- [hb_util:safe_int(Key)], + is_map(Assignment) + ], + lists:foreach( + fun({Label, ID}) -> + Slot = + case [S || {S, Body} <- BySlot, Body =:= ID] of + [Found | _] -> Found; + [] -> not_assigned + end, + ?event(name_token_live, + {slot, + {process, {string, ProcID}}, + {label, {string, Label}}, + {tx, {string, ID}}, + {slot, Slot} + } + ) + end, + Named + ), + ?event(name_token_live, + {schedule_length, {process, {string, ProcID}}, {slots, length(BySlot)}} + ), + ok. + +%%% Story one: a name that had to be paid for. +%%% +%%% An offer that charges to register turns away a buyer who will not pay the +%%% fee, and turns away a payment from somebody who never reserved it. The buyer +%%% who does both gets the name -- and with it the right to say what it means, +%%% which the seller loses in the same breath. +live_story_sale(Seller, Poor, Rich, Opts) -> + SellerAddr = live_address(Seller), + ProcID = live_spawn(<<"paid-for">>, SellerAddr, Seller, Opts), + Offer = + live_send( + <<"make-offer">>, + ProcID, + #{ + <<"action">> => <<"make-offer">>, + <<"offer-quantity">> => <<"1">>, + <<"asking">> => hb_util:bin(?LIVE_ASKING), + <<"deposit">> => <<"0">>, + <<"minimum-fee">> => hb_util:bin(?LIVE_MINIMUM_FEE), + <<"deadline">> => <<"99999999">> + }, + Seller, + Opts + ), + % A registration that pays only what the network charges anyway does not + % clear a fee set above it. + Underpaid = + live_send( + <<"register:underpaid">>, + ProcID, + #{ <<"action">> => <<"register-interest">>, <<"order-id">> => Offer }, + Poor, + Opts + ), + % Nor does value sent to the process, which would only be stranded there. + Stranded = + live_post( + <<"register:stranded">>, + #{ + <<"target">> => ProcID, + <<"quantity">> => hb_util:bin(?LIVE_PAID_FEE), + <<"action">> => <<"register-interest">>, + <<"order-id">> => Offer + }, + Poor, + Opts + ), + % Overpaying the reward does. + Registered = + live_post( + <<"register:paid">>, + #{ + <<"target">> => ProcID, + <<"quantity">> => <<"1">>, + <<"reward">> => hb_util:bin(?LIVE_PAID_FEE), + <<"action">> => <<"register-interest">>, + <<"order-id">> => Offer + }, + Rich, + Opts + ), + % The order is now the registrant's alone: somebody else's payment buys + % nothing, and with no bond posted there is nothing to compensate them with. + Interloper = + live_post( + <<"payment:interloper">>, + #{ + <<"target">> => SellerAddr, + <<"quantity">> => hb_util:bin(?LIVE_ASKING), + <<"order-id">> => Offer + }, + Poor, + Opts + ), + Payment = + live_post( + <<"payment:buyer">>, + #{ + <<"target">> => SellerAddr, + <<"quantity">> => hb_util:bin(?LIVE_ASKING), + <<"order-id">> => Offer + }, + Rich, + Opts + ), + % The seller no longer holds the name, so no longer speaks for it. + StaleSet = + live_send( + <<"set:former-owner">>, + ProcID, + #{ <<"action">> => <<"set">>, <<"greeting">> => <<"still mine">> }, + Seller, + Opts + ), + OwnerSet = + live_send( + <<"set:owner">>, + ProcID, + #{ + <<"action">> => <<"set">>, + <<"content-type">> => <<"text/plain">>, + <<"greeting">> => <<"hello from the new owner">> + }, + Rich, + Opts + ), + % A stranger may ask a slot to resolve anything at all. It must be applied + % like any other message: a slot that failed could never be recomputed, and + % the process would stop on every node for good. + Stray = + live_post( + <<"stray:path-info">>, + #{ + <<"target">> => ProcID, + <<"quantity">> => <<"1">>, + <<"path">> => <<"info">> + }, + Poor, + Opts + ), + % And a figure that is not a number is an inadmissible message, not a + % failed slot. + Nonsense = + live_send( + <<"make-offer:nonsense">>, + ProcID, + #{ + <<"action">> => <<"make-offer">>, + <<"offer-quantity">> => <<"1">>, + <<"asking">> => <<"1000">>, + <<"deposit">> => <<"0">>, + <<"deadline">> => <<"tomorrow">> + }, + Seller, + Opts + ), + live_slots( + ProcID, + [ + {<<"make-offer">>, Offer}, + {<<"register:underpaid">>, Underpaid}, + {<<"register:stranded">>, Stranded}, + {<<"register:paid">>, Registered}, + {<<"payment:interloper">>, Interloper}, + {<<"payment:buyer">>, Payment}, + {<<"set:former-owner">>, StaleSet}, + {<<"set:owner">>, OwnerSet}, + {<<"stray:path-info">>, Stray}, + {<<"make-offer:nonsense">>, Nonsense} + ], + Opts + ), + ProcID. + +%%% Story two: an offer withdrawn. +%%% +%%% A seller may take an unreserved order back, and once they have, nothing +%%% more can be done with it: registering is refused and a payment against it +%%% buys nothing. A stranger may not withdraw somebody else's order. +live_story_withdrawn(Seller, Poor, Rich, Opts) -> + SellerAddr = live_address(Seller), + ProcID = live_spawn(<<"withdrawn">>, SellerAddr, Seller, Opts), + Offer = + live_send( + <<"make-offer">>, + ProcID, + #{ + <<"action">> => <<"make-offer">>, + <<"offer-quantity">> => <<"1">>, + <<"asking">> => hb_util:bin(?LIVE_ASKING), + <<"deposit">> => <<"0">>, + <<"minimum-fee">> => hb_util:bin(?LIVE_MINIMUM_FEE), + <<"deadline">> => <<"99999999">> + }, + Seller, + Opts + ), + Cancelled = + live_send( + <<"cancel">>, + ProcID, + #{ <<"action">> => <<"cancel-order">>, <<"order-id">> => Offer }, + Seller, + Opts + ), + LateRegister = + live_post( + <<"register:after-cancel">>, + #{ + <<"target">> => ProcID, + <<"quantity">> => <<"1">>, + <<"reward">> => hb_util:bin(?LIVE_PAID_FEE), + <<"action">> => <<"register-interest">>, + <<"order-id">> => Offer + }, + Rich, + Opts + ), + LatePayment = + live_post( + <<"payment:after-cancel">>, + #{ + <<"target">> => SellerAddr, + <<"quantity">> => hb_util:bin(?LIVE_ASKING), + <<"order-id">> => Offer + }, + Rich, + Opts + ), + Second = + live_send( + <<"make-offer:second">>, + ProcID, + #{ + <<"action">> => <<"make-offer">>, + <<"offer-quantity">> => <<"1">>, + <<"asking">> => hb_util:bin(?LIVE_ASKING), + <<"deposit">> => <<"0">>, + <<"minimum-fee">> => hb_util:bin(?LIVE_MINIMUM_FEE), + <<"deadline">> => <<"99999999">> + }, + Seller, + Opts + ), + StrangerCancel = + live_send( + <<"cancel:stranger">>, + ProcID, + #{ <<"action">> => <<"cancel-order">>, <<"order-id">> => Second }, + Poor, + Opts + ), + live_slots( + ProcID, + [ + {<<"make-offer">>, Offer}, + {<<"cancel">>, Cancelled}, + {<<"register:after-cancel">>, LateRegister}, + {<<"payment:after-cancel">>, LatePayment}, + {<<"make-offer:second">>, Second}, + {<<"cancel:stranger">>, StrangerCancel} + ], + Opts + ), + ProcID. + +%%% Story three: a name handed over directly. +%%% +%%% No sale at all -- just the token half. The unit moves, and the authority +%%% moves with it: the former holder's word stops counting the moment it does. +live_story_handover(Seller, Rich, Opts) -> + SellerAddr = live_address(Seller), + RichAddr = live_address(Rich), + ProcID = live_spawn(<<"handed-over">>, SellerAddr, Seller, Opts), + FirstSet = + live_send( + <<"set:before">>, + ProcID, + #{ <<"action">> => <<"set">>, <<"points-at">> => <<"seller">> }, + Seller, + Opts + ), + Transfer = + live_send( + <<"transfer">>, + ProcID, + #{ + <<"action">> => <<"transfer">>, + <<"recipient">> => RichAddr, + <<"quantity">> => <<"1">> + }, + Seller, + Opts + ), + StaleSet = + live_send( + <<"set:former-owner">>, + ProcID, + #{ <<"action">> => <<"set">>, <<"points-at">> => <<"seller again">> }, + Seller, + Opts + ), + NewSet = + live_send( + <<"set:new-owner">>, + ProcID, + #{ <<"action">> => <<"set">>, <<"points-at">> => <<"buyer">> }, + Rich, + Opts + ), + live_slots( + ProcID, + [ + {<<"set:before">>, FirstSet}, + {<<"transfer">>, Transfer}, + {<<"set:former-owner">>, StaleSet}, + {<<"set:new-owner">>, NewSet} + ], + Opts + ), + ProcID. + +%% @doc The stories a run was asked for, named in the environment as +%% `HB_LIVE_STORIES=sale,withdrawn' or left unset for all of them. +live_stories() -> + case os:getenv("HB_LIVE_STORIES") of + false -> all; + Names -> + [ + hb_util:atom(string:trim(Name)) + || + Name <- string:split(Names, ",", all) + ] + end. + +live_story(Name, all, Fun) -> live_story(Name, [Name], Fun); +live_story(Name, Wanted, Fun) -> + case lists:member(Name, Wanted) of + true -> Fun(); + false -> + ?event(name_token_live, {story_skipped, {name, Name}}), + skipped + end. + +%% @doc Play all three stories out on mainnet. +live_suite() -> + Seller = live_wallet(), + SellerAddr = live_address(Seller), + Opts = live_opts(Seller), + Poor = live_party(<<"name-token-poor.json">>), + Rich = live_party(<<"name-token-buyer.json">>), + PoorAddr = live_address(Poor), + RichAddr = live_address(Rich), + ?event(name_token_live, + {parties, + {seller, {string, SellerAddr}}, + {underpayer, {string, PoorAddr}}, + {buyer, {string, RichAddr}}, + {seller_balance, live_balance(SellerAddr, Opts)} + } + ), + live_fund(PoorAddr, 500000000, Seller, Opts), + live_fund(RichAddr, 500000000, Seller, Opts), + % Each story stands alone on its own process, so a rerun can name just the + % ones it needs rather than paying to create every account again. + Wanted = live_stories(), + Sale = live_story(sale, Wanted, fun() -> live_story_sale(Seller, Poor, Rich, Opts) end), + Withdrawn = + live_story( + withdrawn, + Wanted, + fun() -> live_story_withdrawn(Seller, Poor, Rich, Opts) end + ), + Handover = + live_story(handover, Wanted, fun() -> live_story_handover(Seller, Rich, Opts) end), + ?event(name_token_live, + {suite_complete, + {sale, {string, Sale}}, + {withdrawn, {string, Withdrawn}}, + {handover, {string, Handover}}, + {buyer, {string, RichAddr}}, + {underpayer, {string, PoorAddr}}, + {seller, {string, SellerAddr}}, + {seller_balance, live_balance(SellerAddr, Opts)} + } + ), + ok. + +%%% The permanent fixture +%%% +%%% A name that was really sold on Arweave, by the driver above. Everything +%%% below is a deterministic read of blocks 1966039-1966044 of the weave, so it +%%% is repeatable forever: the seller spawned `test-name' holding its single +%%% unit, offered it for 1000 winston with no bond and a 1000 winston fee to +%%% register, the buyer registered (paying that fee), paid, and then -- owning +%%% the name -- pointed it at a message of their own. +%%% +%%% One transaction per block, so the schedule's order is unambiguous: +%%% +%%% 1966039 the name yWRe7v4S... +%%% 1966041 make-offer 3BApJHea... (the order id) +%%% 1966042 register-interest GGPH2lA8... +%%% 1966043 payment KROLsGpr... (to the seller, not the process) +%%% 1966044 set QjmNGlIi... +%%% +%%% The payment is the point: it is an ordinary transfer between two addresses, +%%% the process is not a party to it, and the process sees it only because +%%% `~arweave-scheduler@1.0' is sequencing it by every transaction on the +%%% network. Every transaction in that range is a slot of this +%%% process, not just these five. +-define(FIXTURE_PROCESS, <<"yWRe7v4SZ4_NKV6LkYyNPrFdzEaGh0ckblu-CaGXqG4">>). +-define(FIXTURE_SELLER, <<"ggltHF0Cnv9ylH3vM1p7amR2vXLMoPLQIUQmAEwLP-k">>). +-define(FIXTURE_BUYER, <<"LW0myHWuv7XcLec19OCDzFJW0P6jXPG_Ao49kfy9Slc">>). +-define(FIXTURE_ORDER, <<"3BApJHeatc9pVuLgjZ_P-HT5hZgRE1Q3I1bdTESgRDM">>). +-define(FIXTURE_MAX_HEIGHT, 1966044). + +%% @doc Read the fixture's state as of the pinned height. The height cap makes +%% the answer immutable: no block after 1966044 can reach this process. +fixture_opts() -> + TestStore = hb_test_utils:test_store(), + IndexStore = hb_test_utils:test_store(), + (hb_opts:default_message())#{ + <<"store">> => [ + TestStore, + #{ + <<"store-module">> => hb_store_arweave, + <<"name">> => <<"cache-arweave">>, + <<"index-store">> => [IndexStore] + }, + #{ + <<"store-module">> => hb_store_gateway, + <<"local-store">> => [TestStore] + } + ], + <<"arweave-index-store">> => #{ <<"index-store">> => [IndexStore] }, + <<"arweave-index-workers">> => 8, + <<"arweave-scheduler-confirmation-depth">> => 1, + <<"arweave-scheduler-max-height">> => ?FIXTURE_MAX_HEIGHT, + <<"name-resolvers">> => [#{ <<"test-name">> => ?FIXTURE_PROCESS }], + <<"node-host">> => <<"host">>, + <<"priv-wallet">> => ar_wallet:new() + }. + +%% @doc Synchronize the fixture's schedule from the network, retrying while the +%% gateway rate-limits us -- the same allowance the scheduler's own fixture +%% tests make. +fixture_sync(_Opts, 0) -> {error, fixture_sync_failed}; +fixture_sync(Opts, Attempts) -> + case + hb_ao:resolve( + #{ <<"device">> => <<"arweave-scheduler@1.0">> }, + #{ + <<"path">> => <<"schedule">>, + <<"method">> => <<"GET">>, + <<"target">> => ?FIXTURE_PROCESS + }, + Opts + ) + of + {ok, Schedule} -> {ok, Schedule}; + _ -> + timer:sleep(5000), + fixture_sync(Opts, Attempts - 1) + end. + +%% @doc Compute the fixture to its latest slot. The schedule is primed first, so +%% that the process message is read back as its canonical `tx@1.0' decoding +%% rather than a gateway store's lossier one. +fixture_state(Opts, Attempts) -> + {ok, _} = fixture_sync(Opts, Attempts), + {ok, Raw} = hb_cache:read(?FIXTURE_PROCESS, Opts), + Process = hb_cache:ensure_all_loaded(Raw, Opts), + hb_ao:resolve(Process, <<"now">>, Opts). + +%% @doc The whole story, read back off the weave: a name that changed hands for +%% AR that never touched the process, and then said something new. +fixture_sale_test_() -> + {timeout, 1800, fun fixture_sale/0}. +fixture_sale() -> + Opts = fixture_opts(), + {ok, State} = fixture_state(Opts, 5), + Read = fun(Path) -> hb_ao:get(Path, {as, <<"message@1.0">>, State}, not_found, Opts) end, + % The name is the buyer's: the swap settled a payment it was not paid. + ?assertEqual(1, hb_util:int(Read([?BALANCES, ?FIXTURE_BUYER]))), + ?assertEqual(0, hb_util:int(Read([?BALANCES, ?FIXTURE_SELLER]))), + ?assertEqual(1, hb_util:int(Read(<<"total-supply">>))), + % The order it went through is settled, and the buyer is recorded as the + % one who paid. + ?assertEqual( + <<"settled">>, + Read([<<"orders">>, ?FIXTURE_ORDER, <<"status">>]) + ), + ?assertEqual( + ?FIXTURE_BUYER, + Read([<<"orders">>, ?FIXTURE_ORDER, <<"buyer">>]) + ), + % And the new owner has said what the name points at. + ?assertEqual(<<"hello from the new owner">>, Read([?VALUE, <<"greeting">>])), + ?assertEqual(<<"text/plain">>, Read([?VALUE, <<"content-type">>])). + +%% @doc The name resolves: `test-name' reaches this instance, both as a bare +%% name and as the label of a host. +fixture_name_resolution_test() -> + Opts = fixture_opts(), + % A node that serves a name holds it. Priming the schedule puts the process + % in the node's own cache, which is what the resolver then loads. + {ok, _} = fixture_sync(Opts, 5), + ?assertEqual( + {ok, ?FIXTURE_PROCESS}, + hb_ao:resolve_many( + [ + #{ <<"device">> => <<"name@1.0">> }, + #{ <<"path">> => <<"test-name">>, <<"load">> => false } + ], + Opts + ) + ), + % `test-name.host' is the same lookup: the node's own host is stripped from + % the request's host, leaving the label to resolve. + {ok, Resolved} = + hb_ao:resolve( + #{ <<"device">> => <<"name@1.0">> }, + #{ + <<"path">> => <<"request">>, + <<"request">> => #{ <<"host">> => <<"test-name.host">> }, + <<"body">> => [#{ <<"path">> => <<"now">> }] + }, + Opts + ), + [Named | _] = hb_ao:get(<<"body">>, Resolved, [], Opts), + Loaded = hb_cache:ensure_all_loaded(Named, Opts), + % The message the host resolved to is this name: it carries the name's own + % spawn keys, and nothing else on the weave does. + ?assertEqual(<<"test-name">>, hb_ao:get(<<"name">>, Loaded, not_found, Opts)), + ?assertEqual( + ?FIXTURE_SELLER, + hb_ao:get(<<"initial-holder">>, Loaded, not_found, Opts) + ), + ?assertEqual( + <<"name-token@1.0">>, + hb_ao:get(<<"execution-device">>, Loaded, not_found, Opts) + ). + +%%% Story one, replayed: a name that had to be paid for +%%% +%%% Every transaction below is on mainnet. The reads walk the process forward +%%% one slot at a time, so what the schedule did to the state is visible in the +%%% order it happened, rather than only at the end. +%%% +%%% make-offer the seller's unit goes into escrow, the order opens +%%% register:underpaid a reward of 33,039,920 does not clear a 100,000,000 +%%% fee -- the order stays open +%%% register:stranded 150,000,000 sent *to the process* is not the fee +%%% either: it would only be stranded there +%%% register:paid a reward of 150,000,000 does clear it -- reserved +%%% payment:interloper the underpayer pays anyway, but the order is not +%%% theirs and there is no bond to compensate them with +%%% payment:buyer the registrant pays -- settled, and the name moves +%%% set:former-owner the seller no longer holds it, so is no longer heard +%%% set:owner the buyer says what the name means +%%% stray:path-info a stranger routes a slot at `info'; nothing breaks +%%% make-offer:nonsense a deadline of `tomorrow' opens no order +-define(SALE_PROCESS, <<"an95oAK9MlahZI_tKKeG4ykzNN01qfMi2WfJO58o_UU">>). +-define(SALE_SELLER, <<"ggltHF0Cnv9ylH3vM1p7amR2vXLMoPLQIUQmAEwLP-k">>). +-define(SALE_UNDERPAYER, <<"2yvAwMDrF62hpH_kKTfguatzB9mKVzcM2edAn8KauTQ">>). +-define(SALE_BUYER, <<"LW0myHWuv7XcLec19OCDzFJW0P6jXPG_Ao49kfy9Slc">>). +-define(SALE_OFFER, <<"r6lleOybw5_Pz-3EDHjLwAYv8XNGTlLMjT_8pA9e6o0">>). +-define(SALE_UNDERPAID, <<"060IcKkUJ4ggdojenpcvSpTQjbuGDsoDz_jEhs-C4E8">>). +-define(SALE_STRANDED, <<"f31_VyEl5NumgmKUPeokDgPAxbOZj32bFBf3D5wwf_A">>). +-define(SALE_PAID, <<"RJXzg_GbIo7mUs3oNXfG4DI2-1EK7rWipN5YlvFxZFI">>). +-define(SALE_INTERLOPER, <<"L1mrhwV8JY0-n7B2XZu2R9Ox5MjCsdnsSohpNl_UHAg">>). +-define(SALE_PAYMENT, <<"B4F3TSTcLBuivjQ9Rzf-2IyY2svHUIMnfQjHGbegn6c">>). +-define(SALE_STALE_SET, <<"ndgQ_1zZCm3cMlmC42jhLFZHFwo60wVs5znQ14Mip4Q">>). +-define(SALE_OWNER_SET, <<"vc3eQypdoGc4--NMQkah5tYYnG62KkYtpme1JVTpeAU">>). +-define(SALE_STRAY, <<"bwEotEyzbH4AYP_-5WZCZB7TgTRI4bFjVdJl6fjmuhE">>). +-define(SALE_NONSENSE, <<"aYIqfkDstZTyewfRdqNcKHYdfCeC2jqyICD0juRCoC0">>). +-define(SALE_MAX_HEIGHT, 1966084). + +%% @doc Node options pinned to a story's last block, so its answer is +%% immutable: no block after it can reach the process. +story_opts(MaxHeight, Process) -> + TestStore = hb_test_utils:test_store(), + IndexStore = hb_test_utils:test_store(), + (hb_opts:default_message())#{ + <<"store">> => [ + TestStore, + #{ + <<"store-module">> => hb_store_arweave, + <<"name">> => <<"cache-arweave">>, + <<"index-store">> => [IndexStore] + }, + #{ + <<"store-module">> => hb_store_gateway, + <<"local-store">> => [TestStore] + } + ], + <<"arweave-index-store">> => #{ <<"index-store">> => [IndexStore] }, + <<"arweave-index-workers">> => 8, + <<"arweave-scheduler-confirmation-depth">> => 1, + <<"arweave-scheduler-max-height">> => MaxHeight, + <<"name-resolvers">> => [#{ <<"test-name">> => Process }], + <<"node-host">> => <<"host">>, + <<"priv-wallet">> => ar_wallet:new() + }. + +%% @doc Synchronize a story's schedule, retrying while the gateway rate-limits +%% us -- the same allowance the scheduler's own fixture tests make. +story_sync(_Process, _Opts, 0) -> {error, sync_failed}; +story_sync(Process, Opts, Attempts) -> + case + hb_ao:resolve( + #{ <<"device">> => <<"arweave-scheduler@1.0">> }, + #{ + <<"path">> => <<"schedule">>, + <<"method">> => <<"GET">>, + <<"target">> => Process + }, + Opts + ) + of + {ok, Schedule} -> {ok, Schedule}; + _ -> + timer:sleep(5000), + story_sync(Process, Opts, Attempts - 1) + end. + +%% @doc The slot a transaction was given. The stories pin transaction ids +%% rather than slot numbers, because a slot number is a fact about the weave -- +%% every transaction on the network takes one -- while the id is the message +%% itself. +slot_of(Schedule, TXID, Opts) -> + Assignments = + hb_ao:normalize_keys(hb_ao:get(<<"assignments">>, Schedule, Opts), Opts), + Found = + [ + Slot + || + {Key, Assignment} <- hb_maps:to_list(Assignments, Opts), + {ok, Slot} <- [hb_util:safe_int(Key)], + is_map(Assignment), + hb_util:human_id( + hb_message:id(hb_ao:get(<<"body">>, Assignment, Opts), signed, Opts) + ) =:= TXID + ], + case Found of + [Slot | _] -> Slot; + [] -> error({not_scheduled, TXID}) + end. + +%% @doc Read a process's state as it stood at the end of a given slot, in the +%% form a reader can follow: `~process@1.0/compute&slot=/'. +at(Process, Slot, Path, Opts) -> + hb_ao:resolve( + << + Process/binary, + "~process@1.0/compute&slot=", + (hb_util:bin(Slot))/binary, + "/", + Path/binary + >>, + Opts#{ <<"hashpath">> => ignore } + ). + +sale_story_test_() -> {timeout, 3600, fun sale_story/0}. +sale_story() -> + Process = ?SALE_PROCESS, + Opts = story_opts(?SALE_MAX_HEIGHT, Process), + {ok, Schedule} = story_sync(Process, Opts, 5), + Slot = fun(TXID) -> slot_of(Schedule, TXID, Opts) end, + Seller = ?SALE_SELLER, + Buyer = ?SALE_BUYER, + Poor = ?SALE_UNDERPAYER, + Order = ?SALE_OFFER, + % The offer escrows the seller's only unit, and opens the order. + Offered = Slot(?SALE_OFFER), + ?assertEqual({ok, 0}, at(Process, Offered, <<"balances/", Seller/binary>>, Opts)), + ?assertEqual( + {ok, <<"open">>}, + at(Process, Offered, <<"orders/", Order/binary, "/status">>, Opts) + ), + ?assertEqual( + {ok, 100000000}, + at(Process, Offered, <<"orders/", Order/binary, "/minimum-fee">>, Opts) + ), + % A registration paying only what the network charges does not clear a fee + % set above it, and neither does value sent to the process. + Underpaid = Slot(?SALE_UNDERPAID), + ?assertEqual( + {ok, <<"open">>}, + at(Process, Underpaid, <<"orders/", Order/binary, "/status">>, Opts) + ), + Stranded = Slot(?SALE_STRANDED), + ?assertEqual( + {ok, <<"open">>}, + at(Process, Stranded, <<"orders/", Order/binary, "/status">>, Opts) + ), + % Overpaying the reward does. + Paid = Slot(?SALE_PAID), + ?assertEqual( + {ok, <<"reserved">>}, + at(Process, Paid, <<"orders/", Order/binary, "/status">>, Opts) + ), + ?assertEqual( + {ok, Buyer}, + at(Process, Paid, <<"orders/", Order/binary, "/buyer">>, Opts) + ), + % Somebody else's payment buys nothing while it is reserved, and with no + % bond posted there is nothing to compensate them with either. + Interloped = Slot(?SALE_INTERLOPER), + ?assertEqual( + {ok, <<"reserved">>}, + at(Process, Interloped, <<"orders/", Order/binary, "/status">>, Opts) + ), + ?assertMatch( + {error, not_found}, + at(Process, Interloped, <<"balances/", Poor/binary>>, Opts) + ), + % The registrant's payment settles it, and the name moves. + Settled = Slot(?SALE_PAYMENT), + ?assertEqual( + {ok, <<"settled">>}, + at(Process, Settled, <<"orders/", Order/binary, "/status">>, Opts) + ), + ?assertEqual({ok, 1}, at(Process, Settled, <<"balances/", Buyer/binary>>, Opts)), + ?assertEqual({ok, 0}, at(Process, Settled, <<"balances/", Seller/binary>>, Opts)), + % The seller no longer holds the name, so is no longer heard. + Stale = Slot(?SALE_STALE_SET), + ?assertMatch({error, not_found}, at(Process, Stale, <<"value/greeting">>, Opts)), + % The buyer is. + Spoken = Slot(?SALE_OWNER_SET), + ?assertEqual( + {ok, <<"hello from the new owner">>}, + at(Process, Spoken, <<"value/greeting">>, Opts) + ), + % A stranger routing a slot at `info' changes nothing and breaks nothing: + % the process still computes, and still holds what it held. + Stray = Slot(?SALE_STRAY), + ?assertEqual({ok, 1}, at(Process, Stray, <<"balances/", Buyer/binary>>, Opts)), + ?assertEqual( + {ok, <<"hello from the new owner">>}, + at(Process, Stray, <<"value/greeting">>, Opts) + ), + % And an offer whose deadline is `tomorrow' opens nothing. + Nonsense = Slot(?SALE_NONSENSE), + ?assertMatch( + {error, not_found}, + at(Process, Nonsense, <<"orders/", (?SALE_NONSENSE)/binary, "/status">>, Opts) + ), + ?assertEqual({ok, 1}, at(Process, Nonsense, <<"balances/", Buyer/binary>>, Opts)). + +%%% Story two, replayed: an offer withdrawn +%%% +%%% A seller may take back an order nobody has reserved. Once they have, the +%%% order is spent: registering against it is refused however much is paid, and +%%% a payment against it buys nothing and -- there being no bond -- compensates +%%% nobody. A stranger may not withdraw somebody else's order. +-define(WITHDRAWN_PROCESS, <<"petNFJyilEh0YvFb39FqoL7CeBUvmnuQ73eHSlGLYxI">>). +-define(WITHDRAWN_OFFER, <<"H1aRe1UoXqSW1IA1H4fZf8oUkHRFzgfzv2cw6hsRYGE">>). +-define(WITHDRAWN_CANCEL, <<"zr8xLGjdXJWvWa3VYMjigEk4c47mteGRfUE0cZzX_hc">>). +-define(WITHDRAWN_LATE_REGISTER, <<"m0KwPLpX1zPiR-k9H7Hx16xwGQHRM82O_WgS8E3H1X8">>). +-define(WITHDRAWN_LATE_PAYMENT, <<"nUalA_4J6M-1fV9mx0dDaT4V1GQHGWmcW09G7ZuEOjA">>). +-define(WITHDRAWN_SECOND, <<"0--Z1ngGMHPCrT5Wj9wxFXGapoXoDCq97u2jCuIlZkQ">>). +-define(WITHDRAWN_STRANGER, <<"o6N4lmA5tV318cOEoXTnN5pmBRYGXtIB8K3VaSPCONw">>). +-define(WITHDRAWN_MAX_HEIGHT, 1966093). + +withdrawn_story_test_() -> {timeout, 3600, fun withdrawn_story/0}. +withdrawn_story() -> + Process = ?WITHDRAWN_PROCESS, + Opts = story_opts(?WITHDRAWN_MAX_HEIGHT, Process), + {ok, Schedule} = story_sync(Process, Opts, 5), + Slot = fun(TXID) -> slot_of(Schedule, TXID, Opts) end, + Seller = ?SALE_SELLER, + Buyer = ?SALE_BUYER, + First = ?WITHDRAWN_OFFER, + Second = ?WITHDRAWN_SECOND, + % The offer escrows the unit. + Offered = Slot(?WITHDRAWN_OFFER), + ?assertEqual({ok, 0}, at(Process, Offered, <<"balances/", Seller/binary>>, Opts)), + ?assertEqual( + {ok, <<"open">>}, + at(Process, Offered, <<"orders/", First/binary, "/status">>, Opts) + ), + % Withdrawing it gives the unit back. + Cancelled = Slot(?WITHDRAWN_CANCEL), + ?assertEqual( + {ok, <<"cancelled">>}, + at(Process, Cancelled, <<"orders/", First/binary, "/status">>, Opts) + ), + ?assertEqual({ok, 1}, at(Process, Cancelled, <<"balances/", Seller/binary>>, Opts)), + % A registration that pays the fee in full is still refused: the order is + % no longer open. + LateRegister = Slot(?WITHDRAWN_LATE_REGISTER), + ?assertEqual( + {ok, <<"cancelled">>}, + at(Process, LateRegister, <<"orders/", First/binary, "/status">>, Opts) + ), + ?assertMatch( + {error, not_found}, + at(Process, LateRegister, <<"orders/", First/binary, "/buyer">>, Opts) + ), + % And a payment against it moves nothing, in either direction: the goods + % are back with the seller and there was no bond to compensate anyone from. + LatePayment = Slot(?WITHDRAWN_LATE_PAYMENT), + ?assertEqual( + {ok, <<"cancelled">>}, + at(Process, LatePayment, <<"orders/", First/binary, "/status">>, Opts) + ), + ?assertEqual( + {ok, 1}, + at(Process, LatePayment, <<"balances/", Seller/binary>>, Opts) + ), + ?assertMatch( + {error, not_found}, + at(Process, LatePayment, <<"balances/", Buyer/binary>>, Opts) + ), + % The seller offers it again, and a stranger tries to withdraw it. + Reoffered = Slot(?WITHDRAWN_SECOND), + ?assertEqual( + {ok, <<"open">>}, + at(Process, Reoffered, <<"orders/", Second/binary, "/status">>, Opts) + ), + Meddled = Slot(?WITHDRAWN_STRANGER), + ?assertEqual( + {ok, <<"open">>}, + at(Process, Meddled, <<"orders/", Second/binary, "/status">>, Opts) + ), + ?assertEqual({ok, 0}, at(Process, Meddled, <<"balances/", Seller/binary>>, Opts)). + +%%% Story three, replayed: a name handed over +%%% +%%% No sale at all -- the token half on its own. The unit moves by `transfer', +%%% and the authority moves with it: the former holder's word stops counting +%%% the moment it does, and the new holder's starts. +-define(HANDOVER_PROCESS, <<"D4uhF_nO_vyPoIhPDZ0kFMyfOnk1ZCFJFkmnxVw7vSs">>). +-define(HANDOVER_FIRST_SET, <<"dx_Dmvnp2FDZdb3wlDpfVvc4k3UCbQRWLXIEwwgGxIA">>). +-define(HANDOVER_TRANSFER, <<"WgX3Ih8Ef7aDzQ_8Ziio_qYSA5z_-OUcCOVjmdC8WV0">>). +-define(HANDOVER_STALE_SET, <<"-epGyJadxPyQK_bibsic_btn133f_J8UCX1AndWIWRs">>). +-define(HANDOVER_NEW_SET, <<"mqpK8TsoWFUJW345gcYLrFmw4nxqyGqpavt7xUaniC0">>). +-define(HANDOVER_MAX_HEIGHT, 1966100). + +handover_story_test_() -> {timeout, 3600, fun handover_story/0}. +handover_story() -> + Process = ?HANDOVER_PROCESS, + Opts = story_opts(?HANDOVER_MAX_HEIGHT, Process), + {ok, Schedule} = story_sync(Process, Opts, 5), + Slot = fun(TXID) -> slot_of(Schedule, TXID, Opts) end, + Seller = ?SALE_SELLER, + Buyer = ?SALE_BUYER, + % While the seller holds it, the seller speaks for it. + Spoke = Slot(?HANDOVER_FIRST_SET), + ?assertEqual({ok, 1}, at(Process, Spoke, <<"balances/", Seller/binary>>, Opts)), + ?assertEqual({ok, <<"seller">>}, at(Process, Spoke, <<"value/points-at">>, Opts)), + % The unit moves. + Moved = Slot(?HANDOVER_TRANSFER), + ?assertEqual({ok, 0}, at(Process, Moved, <<"balances/", Seller/binary>>, Opts)), + ?assertEqual({ok, 1}, at(Process, Moved, <<"balances/", Buyer/binary>>, Opts)), + % The notices `token-1.0' emits for a transfer are the slot's results. + ?assertEqual( + {ok, <<"Debit-Notice">>}, + at(Process, Moved, <<"results/outbox/1/action">>, Opts) + ), + ?assertEqual( + {ok, <<"Credit-Notice">>}, + at(Process, Moved, <<"results/outbox/2/action">>, Opts) + ), + % The former holder is no longer heard: the name still says what it said. + Stale = Slot(?HANDOVER_STALE_SET), + ?assertEqual({ok, <<"seller">>}, at(Process, Stale, <<"value/points-at">>, Opts)), + % The new holder is. + Spoken = Slot(?HANDOVER_NEW_SET), + ?assertEqual({ok, <<"buyer">>}, at(Process, Spoken, <<"value/points-at">>, Opts)), + ?assertEqual({ok, 1}, at(Process, Spoken, <<"balances/", Buyer/binary>>, Opts)). From fb74556d7f4a8f16b72e455667d296e12e0535c4 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Sat, 25 Jul 2026 20:01:29 -0400 Subject: [PATCH 16/27] feat(name-token): seed a name's holder and value at spawn A name minted with nothing in it says nothing until somebody sends it a `set`, and the first message addressed to a process pays Arweave's new-account fee -- so every name would cost that before it could resolve at all. Neither the holder nor the value can be written into the spawn directly: `balances` and the linked message are submessages, and a submessage does not cross the chain. What does cross is a scalar. `initial-holder` is an address and `initial-value` the id of whatever the name should resolve to, and the device turns them into the balance and the value the first time it computes. Also here: the live suite that exercises this against mainnet, reading state slot by slot through `hb_ao:resolve/2` so the reader can see how each message moved it. --- src/preloaded/process/dev_name_token.erl | 158 ++++++++++++++++++++++- 1 file changed, 154 insertions(+), 4 deletions(-) diff --git a/src/preloaded/process/dev_name_token.erl b/src/preloaded/process/dev_name_token.erl index cb827d1478..ae85339eb4 100644 --- a/src/preloaded/process/dev_name_token.erl +++ b/src/preloaded/process/dev_name_token.erl @@ -107,13 +107,23 @@ swap(Base, Assignment, Opts) -> end end. -%% @doc Give the name its single unit the first time it computes. +%% @doc Give the name its single unit, and whatever it was minted pointing at, +%% the first time it computes. %% -%% The holding cannot be written into the spawn: `balances' is a submessage, and -%% a submessage does not cross the chain. What does cross is the address itself, -%% as the value of `initial-holder' -- values keep their case, where keys are +%% Neither can be written into the spawn directly: `balances' and the linked +%% message are submessages, and a submessage does not cross the chain. What does +%% cross is a scalar -- `initial-holder', an address, and `initial-value', the id +%% of whatever the name should resolve to. Values keep their case, where keys are %% lowercased and a lowercased address is a different address. +%% +%% Seeding a value at spawn is what makes a name cheap to mint. Without it, a +%% freshly spawned name says nothing until somebody sends it a `set', and the +%% first message addressed to a process pays Arweave's new-account fee -- so +%% every name would cost that before it could resolve at all. seed(Base, Opts) -> + seed_value(seed_holding(Base, Opts), Opts). + +seed_holding(Base, Opts) -> case state(<<"initial-holder">>, Base, not_found, Opts) of not_found -> Base; Holder -> @@ -126,6 +136,22 @@ seed(Base, Opts) -> end end. +%% @doc A name minted pointing somewhere resolves there from its first slot. The +%% value is a message of its own -- `{ target: }' -- so that a `set' can +%% later replace it with anything at all without the shape changing underneath +%% whatever is reading it. +seed_value(Base, Opts) -> + case state(<<"initial-value">>, Base, not_found, Opts) of + not_found -> Base; + Target -> + case state(?VALUE, Base, not_found, Opts) of + not_found -> + ?event({name_token_seeded_value, {target, Target}}), + Base#{ ?VALUE => #{ <<"target">> => Target } }; + _ -> Base + end + end. + %% @doc Route a message addressed to the name by its `action'. Matching is %% case-insensitive, as `token-1.0' matches. An unknown action leaves the state %% untouched rather than failing the slot, which would stop the process on every @@ -611,6 +637,32 @@ seeded_from_initial_holder_test() -> hb_ao:get(<<"greeting">>, value(Set, Opts), not_found, Opts) ). +%% @doc A name minted pointing somewhere resolves there immediately, without +%% anybody having to send it a message -- which matters because the first message +%% addressed to a process pays to create its account. +seeded_value_test() -> + Opts = test_opts(), + {Owner, OwnerAddr} = party(), + Spawned = + #{ + <<"name">> => <<"pn-test-1">>, + <<"total-supply">> => 1, + <<"initial-holder">> => OwnerAddr, + <<"initial-value">> => <<"aBcDeFgHiJkLmNoPqRsTuVwXyZ0123456789_-aBcDe">> + }, + Alive = apply_tx(Spawned, tx(Owner, #{ <<"target">> => <<"elsewhere">> }), Opts), + ?assertEqual( + <<"aBcDeFgHiJkLmNoPqRsTuVwXyZ0123456789_-aBcDe">>, + hb_ao:get(<<"target">>, value(Alive, Opts), not_found, Opts) + ), + % And the holder can still point it somewhere else afterwards. + Reset = apply_tx(Alive, set_tx(Owner, #{ <<"greeting">> => <<"moved">> }), Opts), + ?assertEqual( + <<"moved">>, + hb_ao:get(<<"greeting">>, value(Reset, Opts), not_found, Opts) + ), + ?assertEqual(not_found, hb_ao:get(<<"target">>, value(Reset, Opts), not_found, Opts)). + %% @doc Seeding happens once. A name whose unit has moved on is not handed a %% fresh one by the next message that arrives. seeding_happens_once_test() -> @@ -1286,6 +1338,104 @@ live_story(Name, Wanted, Fun) -> skipped end. +%% @doc Mint the five `pn-test-N' names the site's test namespace resolves +%% through, and report what to put in the manifest. +%% +%% Each is spawned already holding its unit and already pointing somewhere, so +%% none of them needs a message sent to it -- which is the whole point of +%% `initial-value', since the first message addressed to a process pays Arweave's +%% new-account fee. Targets alternate between a manifest and another reference, +%% because a name that points at a reference is the shape a person actually wants: +%% a reference can be updated with a bundled data item in seconds, where a name +%% token needs consensus. +%% +%% HB_LIVE_NAMES=1 HB_PRINT=name_token_live \\ +%% rebar3 device test --devices dev_name_token \\ +%% --test dev_name_token:live_names_test_ +live_names_test_() -> live_gated("HB_LIVE_NAMES", fun live_names/0, 7200). + +live_names() -> + Seller = live_wallet(), + SellerAddr = live_address(Seller), + Opts = live_opts(Seller), + Targets = live_name_targets(), + Minted = + lists:map( + fun({Index, Kind, Target}) -> + Name = <<"pn-test-", (hb_util:bin(Index))/binary>>, + ProcID = + live_post( + <<"mint:", Name/binary>>, + #{ + <<"device">> => <<"process@1.0">>, + <<"type">> => <<"Process">>, + <<"scheduler-device">> => <<"arweave-scheduler@1.0">>, + <<"scheduler-mode">> => <<"all">>, + <<"execution-device">> => <<"name-token@1.0">>, + <<"swap-device">> => <<"arweave-swap@1.0">>, + <<"name">> => Name, + <<"ticker">> => <<"NAME">>, + <<"denomination">> => <<"0">>, + <<"total-supply">> => <<"1">>, + <<"initial-holder">> => SellerAddr, + <<"initial-value">> => Target, + <<"test-suite">> => <<"name-token">> + }, + Seller, + Opts + ), + ?event(name_token_live, + {minted, + {name, {string, Name}}, + {process, {string, ProcID}}, + {points_at, {string, Target}}, + {kind, Kind} + } + ), + {Name, ProcID, Kind, Target} + end, + Targets + ), + lists:foreach(fun({_, ProcID, _, _}) -> live_await(ProcID, Opts) end, Minted), + ?event(name_token_live, + {namespace_entries, + {holder, {string, SellerAddr}}, + {entries, + {string, + hb_util:bin( + lists:flatten( + [ + io_lib:format("~s=~s ", [Name, ProcID]) + || + {Name, ProcID, _, _} <- Minted + ] + ) + ) + } + } + } + ), + {ok, Minted}. + +%% @doc What each test name points at. The manifest is the AO site's own, so a +%% resolved name actually renders something; the references are real +%% `~reference@1.0' inits, so the deeper chain is exercised rather than mocked. +live_name_targets() -> + % A real Arweave path manifest that a gateway serves today, so a resolved + % name renders something rather than 404ing. + Manifest = <<"6oMvmlBUUltTDz_T9pZrEP2QkzpCBGk83Br8XXbqy20">>, + % Replaced with the test namespace's own reference once it is published; a + % name pointing at a reference is the shape an owner actually wants, because + % a reference can be repointed in seconds. + Reference = <<"6oMvmlBUUltTDz_T9pZrEP2QkzpCBGk83Br8XXbqy20">>, + [ + {1, manifest, Manifest}, + {2, reference, Reference}, + {3, manifest, Manifest}, + {4, reference, Reference}, + {5, manifest, Manifest} + ]. + %% @doc Play all three stories out on mainnet. live_suite() -> Seller = live_wallet(), From f15d4da9b6120676ce6a32b5933006cb86fa5874 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Sat, 25 Jul 2026 21:13:54 -0400 Subject: [PATCH 17/27] fix: read a stranger's message as a message, not as their device `~arweave-scheduler@1.0' in `all' mode hands a process every transaction on Arweave. That is the point -- it is how a swap sees a plain transfer between two other addresses as the payment for a name -- but it means every message these devices read was written by somebody else, and the `device' key of each is the author's to choose. `hb_ao:get/4' dispatches on that key. A device answers only for the keys a message does not carry, so the reads that could be attacked are exactly the ones with a default: leave the key out, and the answer comes from whatever device the author named. `lua@5.3a' answers with anything, and both its `device' and `module' tags survive a signed Arweave transaction intact, so the payload is deliverable by publishing it. Both devices already had the pinned idiom for reading their own state, for the mirror-image reason -- a plain read of the state would resolve back into `compute'. `field/4' is the same move pointed outwards, and the seven reads in `~name-token@1.0' and eleven in `~arweave-swap@1.0' that took untrusted input now go through it. Two regressions, one per device, both signed transactions carrying a real `tx@1.0' commitment and a real Lua module. Unpinned, the name-token moves a unit on a message addressed to nobody, and the swap pays a seller's escrow to an address the seller never named. Pinned, neither does. Identity was never at risk: `hb_message:signers/2' resolves through `hb_ao:raw/5', which forces the message device, so the signer of a message cannot be forged. What this closes is the gate that decides whether a message was addressed to the process at all, the values the actions then run on, and the execution of a stranger's code inside every slot -- which is also a way to wedge any `all'-mode process for good. --- src/preloaded/process/dev_arweave_swap.erl | 77 ++++++++++++-- src/preloaded/process/dev_name_token.erl | 118 +++++++++++++++++++-- 2 files changed, 180 insertions(+), 15 deletions(-) diff --git a/src/preloaded/process/dev_arweave_swap.erl b/src/preloaded/process/dev_arweave_swap.erl index 3b4471f810..193155a357 100644 --- a/src/preloaded/process/dev_arweave_swap.erl +++ b/src/preloaded/process/dev_arweave_swap.erl @@ -148,9 +148,9 @@ keys(Base, Req, Opts) -> compute(Base, Req, Opts). %% message, which `lib_process:process_id/3' would re-verify the signature of on %% every one of the network's transactions. compute(Base, Assignment, Opts) -> - Height = hb_util:int(hb_ao:get(<<"block-height">>, Assignment, 0, Opts)), - ProcID = hb_ao:get(<<"process">>, Assignment, <<>>, Opts), - Body = hb_ao:get(<<"body">>, Assignment, #{}, Opts), + Height = hb_util:int(field(<<"block-height">>, Assignment, 0, Opts)), + ProcID = field(<<"process">>, Assignment, <<>>, Opts), + Body = field(<<"body">>, Assignment, #{}, Opts), Advanced = advance(Base, Height, Opts), case tx_field_target(Body, Opts) of ProcID -> {ok, control(Advanced, Body, Height, Opts)}; @@ -164,7 +164,7 @@ compute(Base, Assignment, Opts) -> %% anything, and a process that could be wedged by a stranger's transaction %% would not survive its own schedule. control(Base, Body, Height, Opts) -> - case hb_ao:get(<<"action">>, Body, <<>>, Opts) of + case field(<<"action">>, Body, <<>>, Opts) of <<"make-offer">> -> make_offer(Base, Body, Height, Opts); <<"cancel-order">> -> cancel_order(Base, Body, Opts); <<"register-interest">> -> register_interest(Base, Body, Height, Opts); @@ -182,7 +182,7 @@ make_offer(Base, Body, Height, Opts) -> {ok, Deposit} ?= amount(<<"deposit">>, Body, Opts), {ok, Fee} ?= amount(<<"minimum-fee">>, Body, Opts), {ok, Deadline} ?= amount(<<"deadline">>, Body, Opts), - Recipient = hb_ao:get(<<"recipient">>, Body, Seller, Opts), + Recipient = field(<<"recipient">>, Body, Seller, Opts), true ?= Quantity >= 1, true ?= Asking >= 1, true ?= Deposit >= 0, @@ -551,6 +551,22 @@ release_goods(Base, Order, Creator, Status, Opts) -> state(Key, Base, Default, Opts) -> hb_ao:get(Key, {as, <<"message@1.0">>, Base}, Default, Opts). +%% @doc Read a key of a message this device did not write. +%% +%% This device sees every transaction on Arweave -- that is what `all' mode is +%% for, and how a payment between two other addresses is observed at all. The +%% `device' key of each of those is chosen by whoever authored it, and a plain +%% read dispatches on it, so the author would choose what code answers. A device +%% answers only for keys the message does not carry, which makes the reads with +%% a default the ones worth attacking: leave the key out, and the answer is +%% whatever the chosen device says. `lua@5.3a' says anything, and its `module' +%% tag survives a signed Arweave transaction. +%% +%% Reading it as a message leaves the answer to the signer, or to the default +%% this device intended. +field(Key, Msg, Default, Opts) -> + hb_ao:get(Key, {as, <<"message@1.0">>, Msg}, Default, Opts). + %% @doc Read the orders currently held, as plain maps. The state may have been %% written to the process cache and read back since it was last touched, so it %% is loaded through the link layer, and anything that is not an order is @@ -561,7 +577,11 @@ orders(Base, Opts) -> order(Held) || Held <- - [ hb_maps:get(ID, Orders, #{}, Opts) || ID <- hb_ao:keys(Orders, Opts) ], + [ + hb_maps:get(ID, Orders, #{}, Opts) + || + ID <- hb_ao:keys({as, <<"message@1.0">>, Orders}, Opts) + ], is_map(Held), maps:is_key(<<"order-id">>, Held) ]. @@ -605,7 +625,7 @@ order(Held) -> find_order(Base, Body, Opts) -> Held = hb_maps:get( - hb_ao:get(<<"order-id">>, Body, <<>>, Opts), + field(<<"order-id">>, Body, <<>>, Opts), order_book(Base, Opts), not_found, Opts @@ -677,7 +697,7 @@ note(Base, Event, OrderID, _Opts) -> %% that slot on every node, for good. A value that is not a number is simply not %% an admissible message. amount(Key, Body, Opts) -> - hb_util:safe_int(hb_ao:get(Key, Body, 0, Opts)). + hb_util:safe_int(field(Key, Body, 0, Opts)). %% @doc Read a value from the real L1 transaction fields recorded in the %% `tx@1.0' commitment. Top-level keys may come from tags with the same names, so @@ -747,6 +767,47 @@ tx(Wallet, Fields) -> #{ <<"commitment-device">> => <<"tx@1.0">> } ). +%% @doc A message this device reads is a stranger's, and a key it does not carry +%% must be answered by this device's default, not by code the stranger picked. +%% +%% `all' mode hands every transaction on Arweave to this process, so the `device' +%% key of each is the author's to choose. Reading plainly dispatches on it, and a +%% device answers only for absent keys -- so the attack is to omit one. Here +%% `recipient' is omitted: the seller of an order is meant to be paid by the +%% default, and a chosen device answering that key would redirect the goods. +strangers_device_cannot_answer_for_an_absent_key_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + {_, ThiefAddr} = party(), + Hostile = + tx( + Seller, + #{ + <<"target">> => ?PROCESS, + <<"action">> => <<"make-offer">>, + <<"offer-quantity">> => <<"1">>, + <<"asking">> => <<"10">>, + <<"deposit">> => <<"0">>, + <<"deadline">> => <<"100">>, + %% No `recipient'. The script answers for it. + <<"device">> => <<"lua@5.3a">>, + <<"module">> => + #{ + <<"content-type">> => <<"text/x-lua">>, + <<"body">> => + <<"function recipient(base, req)\n" + " return \"ok\", \"", ThiefAddr/binary, "\"\n" + "end\n">> + } + } + ), + Opened = apply_tx(base(#{ SellerAddr => 1 }), Hostile, 10, Opts), + [Order] = orders(Opened, Opts), + %% The goods come back to the seller, because that is what this device + %% decided in the absence of an instruction. + ?assertEqual(SellerAddr, maps:get(<<"recipient">>, Order, undefined)), + ?assertNotEqual(ThiefAddr, maps:get(<<"recipient">>, Order, undefined)). + %% @doc A transaction that carries trade keys as tags only. tag_only_tx(Wallet, Tags) -> Signed = ar_tx:sign(#tx{ format = 2, reward = 1, tags = Tags }, Wallet), diff --git a/src/preloaded/process/dev_name_token.erl b/src/preloaded/process/dev_name_token.erl index ae85339eb4..5fbea8a555 100644 --- a/src/preloaded/process/dev_name_token.erl +++ b/src/preloaded/process/dev_name_token.erl @@ -79,9 +79,9 @@ router(_Key, Base, Assignment, Opts) -> compute(Base, Assignment, Opts) -> Seeded = seed(Base, Opts), Sold = swap(Seeded, Assignment, Opts), - Body = hb_ao:get(<<"body">>, Assignment, #{}, Opts), - ProcID = hb_ao:get(<<"process">>, Assignment, <<>>, Opts), - case hb_ao:get(<<"target">>, Body, <<>>, Opts) of + Body = field(<<"body">>, Assignment, #{}, Opts), + ProcID = field(<<"process">>, Assignment, <<>>, Opts), + case field(<<"target">>, Body, <<>>, Opts) of ProcID -> {ok, action(Sold, Body, Opts)}; _ -> % Not addressed to this name. Under `all' mode that is almost every @@ -157,7 +157,7 @@ seed_value(Base, Opts) -> %% untouched rather than failing the slot, which would stop the process on every %% node for good. action(Base, Body, Opts) -> - case hb_util:to_lower(hb_ao:get(<<"action">>, Body, <<>>, Opts)) of + case hb_util:to_lower(field(<<"action">>, Body, <<>>, Opts)) of <<"transfer">> -> transfer(Base, Body, Opts); <<"set">> -> set_value(Base, Body, Opts); _ -> Base @@ -171,9 +171,9 @@ action(Base, Body, Opts) -> transfer(Base, Body, Opts) -> maybe {ok, Sender} ?= signer(Body, Opts), - Recipient = hb_ao:get(<<"recipient">>, Body, not_found, Opts), + Recipient = field(<<"recipient">>, Body, not_found, Opts), true ?= is_binary(Recipient), - {ok, Quantity} ?= hb_util:safe_int(hb_ao:get(<<"quantity">>, Body, 0, Opts)), + {ok, Quantity} ?= hb_util:safe_int(field(<<"quantity">>, Body, 0, Opts)), true ?= Quantity >= 1, true ?= balance(Base, Sender, Opts) >= Quantity, ?event( @@ -244,7 +244,7 @@ set_value(Base, Body, Opts) -> %% @doc The message a `set' is asking the name to resolve to. value_of(Body, Opts) -> - case hb_ao:get(<<"reference-value">>, Body, not_found, Opts) of + case field(<<"reference-value">>, Body, not_found, Opts) of not_found -> % The set message itself is the value. The keys that addressed it to % this name are not part of what the name says. @@ -293,6 +293,21 @@ owns_supply(Base, Address, Opts) -> state(Key, Base, Default, Opts) -> hb_ao:get(Key, {as, <<"message@1.0">>, Base}, Default, Opts). +%% @doc Read a key of a message this device did not write. +%% +%% Every message reaching `compute' is a stranger's: `~arweave-scheduler@1.0' +%% assigns each of them under `all' mode, so the `device' key of the thing being +%% read is chosen by whoever authored it. A plain read dispatches on that key, +%% which hands the author the choice of what code answers -- `lua@5.3a' with a +%% `module' tag will answer anything, and both tags survive a signed Arweave +%% transaction. A device answers only for keys the message does not carry, so +%% the reads that matter are precisely the ones with a default. +%% +%% Taking the message as a message removes the choice: the answer is the key the +%% signer signed, or the default this device intended. +field(Key, Msg, Default, Opts) -> + hb_ao:get(Key, {as, <<"message@1.0">>, Msg}, Default, Opts). + balance(Base, Address, Opts) -> hb_util:int(state([?BALANCES, Address], Base, 0, Opts)). @@ -389,6 +404,55 @@ apply_tx(Base, Body, Opts) -> ), New. +%% @doc A `target' tag is not an address. `~arweave-swap@1.0' states the rule +%% this device shares -- "top-level keys may come from tags with the same +%% names" -- and the addressed-to-me gate here reads that top-level key, so a +%% transaction sent to nobody can reach the actions with a tag alone. +%% +%% It buys nothing, and this records why: every action is gated again on what +%% the signer holds, so reaching `transfer' with an empty balance moves nothing +%% and reaching `set' without the supply writes nothing. The gate decides who is +%% being spoken to; the holdings decide who may speak. +tag_only_target_reaches_the_actions_but_holds_nothing_test() -> + Opts = test_opts(), + {_, OwnerAddr} = party(), + {Stranger, StrangerAddr} = party(), + Held = name_held_by(OwnerAddr), + Steal = + tag_only_tx( + Stranger, + [ + {<<"target">>, ?PROCESS}, + {<<"action">>, <<"transfer">>}, + {<<"recipient">>, StrangerAddr}, + {<<"quantity">>, <<"1">>} + ] + ), + %% The tag does put the process's own id at the key the gate reads. + ?assertEqual(?PROCESS, hb_ao:get(<<"target">>, Steal, not_found, Opts)), + Tried = apply_tx(Held, Steal, Opts), + ?assertEqual(1, held_by(Tried, OwnerAddr, Opts)), + ?assertEqual(0, held_by(Tried, StrangerAddr, Opts)), + Speak = + tag_only_tx( + Stranger, + [ + {<<"target">>, ?PROCESS}, + {<<"action">>, <<"set">>}, + {<<"reference-value">>, <<"a-value-the-name-never-agreed-to">>} + ] + ), + Said = apply_tx(Tried, Speak, Opts), + ?assertEqual( + hb_ao:get(?VALUE, {as, <<"message@1.0">>, Tried}, not_found, Opts), + hb_ao:get(?VALUE, {as, <<"message@1.0">>, Said}, not_found, Opts) + ). + +%% @doc A transaction that carries its keys as tags only, addressed to nobody. +tag_only_tx(Wallet, Tags) -> + Signed = ar_tx:sign(#tx{ format = 2, reward = 1, tags = Tags }, Wallet), + hb_message:convert(Signed, <<"structured@1.0">>, <<"tx@1.0">>, #{}). + transfer_tx(Wallet, Recipient, Quantity) -> tx( Wallet, @@ -411,6 +475,46 @@ held_by(Base, Address, Opts) -> balance(Base, Address, Opts). value(Base, Opts) -> hb_cache:ensure_all_loaded(state(?VALUE, Base, #{}, Opts), Opts). +%% @doc A scheduled message is a stranger's document, and reading a key of it +%% must not run code the stranger chose. +%% +%% Under `~arweave-scheduler@1.0''s `all' mode every transaction on Arweave is +%% assigned to this process, so the `device' of the message being read is set by +%% whoever wrote it. A plain read dispatches on it, and a device only answers for +%% keys the message does not carry -- so the attack is to leave a key out and let +%% the chosen device supply it. `lua@5.3a' will supply anything, and both the +%% `device' and `module' tags survive a signed Arweave transaction. +%% +%% Here the message carries no `target' at all. The gate that decides whether it +%% is addressed to this name reads that key, so answering it is enough to make a +%% transaction addressed elsewhere move the name's unit. +strangers_device_cannot_answer_for_an_absent_key_test() -> + Opts = test_opts(), + {Owner, OwnerAddr} = party(), + {_, ElsewhereAddr} = party(), + Hostile = + hb_message:commit( + #{ + %% No `target'. The script below answers for it. + <<"action">> => <<"transfer">>, + <<"recipient">> => ElsewhereAddr, + <<"quantity">> => <<"1">>, + <<"device">> => <<"lua@5.3a">>, + <<"module">> => + #{ + <<"content-type">> => <<"text/x-lua">>, + <<"body">> => + <<"function target(base, req)\n" + " return \"ok\", \"", ?PROCESS/binary, "\"\n" + "end\n">> + } + }, + #{ <<"priv-wallet">> => Owner } + ), + Untouched = apply_tx(name_held_by(OwnerAddr), Hostile, Opts), + ?assertEqual(1, held_by(Untouched, OwnerAddr, Opts)), + ?assertEqual(0, held_by(Untouched, ElsewhereAddr, Opts)). + %% @doc The unit moves, and the pair of notices `token-1.0' emits go with it. transfer_moves_the_unit_test() -> Opts = test_opts(), From 07bad761d715cba81fa24187acb2f2edf3d8a5c2 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Sat, 25 Jul 2026 20:40:32 -0400 Subject: [PATCH 18/27] fix: read scheduled transaction fields as data --- src/preloaded/process/dev_arweave_swap.erl | 32 +++++++----- src/preloaded/process/dev_name_token.erl | 58 ++++------------------ 2 files changed, 30 insertions(+), 60 deletions(-) diff --git a/src/preloaded/process/dev_arweave_swap.erl b/src/preloaded/process/dev_arweave_swap.erl index 193155a357..fe596ce0b7 100644 --- a/src/preloaded/process/dev_arweave_swap.erl +++ b/src/preloaded/process/dev_arweave_swap.erl @@ -551,19 +551,7 @@ release_goods(Base, Order, Creator, Status, Opts) -> state(Key, Base, Default, Opts) -> hb_ao:get(Key, {as, <<"message@1.0">>, Base}, Default, Opts). -%% @doc Read a key of a message this device did not write. -%% -%% This device sees every transaction on Arweave -- that is what `all' mode is -%% for, and how a payment between two other addresses is observed at all. The -%% `device' key of each of those is chosen by whoever authored it, and a plain -%% read dispatches on it, so the author would choose what code answers. A device -%% answers only for keys the message does not carry, which makes the reads with -%% a default the ones worth attacking: leave the key out, and the answer is -%% whatever the chosen device says. `lua@5.3a' says anything, and its `module' -%% tag survives a signed Arweave transaction. -%% -%% Reading it as a message leaves the answer to the signer, or to the default -%% this device intended. +%% @doc Read a field from an untrusted scheduled message as plain data. field(Key, Msg, Default, Opts) -> hb_ao:get(Key, {as, <<"message@1.0">>, Msg}, Default, Opts). @@ -885,6 +873,24 @@ only_order(Base, Opts) -> balance_of(Base, Address, Opts) -> balance(Base, Address, Opts). +%% @doc A foreign transaction's device cannot interpret absent control fields. +foreign_device_is_data_test() -> + Opts = test_opts(), + {Sender, SenderAddr} = party(), + Base = base(#{ SenderAddr => 1 }), + Foreign = + tx( + Sender, + #{ + <<"target">> => ?PROCESS, + <<"device">> => <<"manifest@1.0">>, + <<"manifest">> => #{} + } + ), + Untouched = apply_tx(Base, Foreign, 100, Opts), + ?assertEqual(1, balance_of(Untouched, SenderAddr, Opts)), + ?assertEqual([], orders(Untouched, Opts)). + %% @doc Opening an offer moves the goods and the bond into escrow, and leaves %% the order open. make_offer_escrows_test() -> diff --git a/src/preloaded/process/dev_name_token.erl b/src/preloaded/process/dev_name_token.erl index 5fbea8a555..4804c982b0 100644 --- a/src/preloaded/process/dev_name_token.erl +++ b/src/preloaded/process/dev_name_token.erl @@ -293,18 +293,7 @@ owns_supply(Base, Address, Opts) -> state(Key, Base, Default, Opts) -> hb_ao:get(Key, {as, <<"message@1.0">>, Base}, Default, Opts). -%% @doc Read a key of a message this device did not write. -%% -%% Every message reaching `compute' is a stranger's: `~arweave-scheduler@1.0' -%% assigns each of them under `all' mode, so the `device' key of the thing being -%% read is chosen by whoever authored it. A plain read dispatches on that key, -%% which hands the author the choice of what code answers -- `lua@5.3a' with a -%% `module' tag will answer anything, and both tags survive a signed Arweave -%% transaction. A device answers only for keys the message does not carry, so -%% the reads that matter are precisely the ones with a default. -%% -%% Taking the message as a message removes the choice: the answer is the key the -%% signer signed, or the default this device intended. +%% @doc Read a field from an untrusted scheduled message as plain data. field(Key, Msg, Default, Opts) -> hb_ao:get(Key, {as, <<"message@1.0">>, Msg}, Default, Opts). @@ -475,45 +464,20 @@ held_by(Base, Address, Opts) -> balance(Base, Address, Opts). value(Base, Opts) -> hb_cache:ensure_all_loaded(state(?VALUE, Base, #{}, Opts), Opts). -%% @doc A scheduled message is a stranger's document, and reading a key of it -%% must not run code the stranger chose. -%% -%% Under `~arweave-scheduler@1.0''s `all' mode every transaction on Arweave is -%% assigned to this process, so the `device' of the message being read is set by -%% whoever wrote it. A plain read dispatches on it, and a device only answers for -%% keys the message does not carry -- so the attack is to leave a key out and let -%% the chosen device supply it. `lua@5.3a' will supply anything, and both the -%% `device' and `module' tags survive a signed Arweave transaction. -%% -%% Here the message carries no `target' at all. The gate that decides whether it -%% is addressed to this name reads that key, so answering it is enough to make a -%% transaction addressed elsewhere move the name's unit. -strangers_device_cannot_answer_for_an_absent_key_test() -> +%% @doc A foreign transaction's device cannot interpret absent envelope fields. +foreign_device_is_data_test() -> Opts = test_opts(), {Owner, OwnerAddr} = party(), - {_, ElsewhereAddr} = party(), - Hostile = - hb_message:commit( + Foreign = + tx( + Owner, #{ - %% No `target'. The script below answers for it. - <<"action">> => <<"transfer">>, - <<"recipient">> => ElsewhereAddr, - <<"quantity">> => <<"1">>, - <<"device">> => <<"lua@5.3a">>, - <<"module">> => - #{ - <<"content-type">> => <<"text/x-lua">>, - <<"body">> => - <<"function target(base, req)\n" - " return \"ok\", \"", ?PROCESS/binary, "\"\n" - "end\n">> - } - }, - #{ <<"priv-wallet">> => Owner } + <<"device">> => <<"manifest@1.0">>, + <<"manifest">> => #{} + } ), - Untouched = apply_tx(name_held_by(OwnerAddr), Hostile, Opts), - ?assertEqual(1, held_by(Untouched, OwnerAddr, Opts)), - ?assertEqual(0, held_by(Untouched, ElsewhereAddr, Opts)). + Untouched = apply_tx(name_held_by(OwnerAddr), Foreign, Opts), + ?assertEqual(1, held_by(Untouched, OwnerAddr, Opts)). %% @doc The unit moves, and the pair of notices `token-1.0' emits go with it. transfer_moves_the_unit_test() -> From 8d55279e1595b534a306fff3366eff1aff43b202 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Sat, 25 Jul 2026 20:57:11 -0400 Subject: [PATCH 19/27] fix: read foreign transaction metadata passively --- src/preloaded/process/dev_arweave_swap.erl | 8 ++++---- src/preloaded/process/dev_name_token.erl | 5 ++--- src/preloaded/process/dev_process.erl | 8 ++++---- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/preloaded/process/dev_arweave_swap.erl b/src/preloaded/process/dev_arweave_swap.erl index fe596ce0b7..150c99ca9d 100644 --- a/src/preloaded/process/dev_arweave_swap.erl +++ b/src/preloaded/process/dev_arweave_swap.erl @@ -553,7 +553,7 @@ state(Key, Base, Default, Opts) -> %% @doc Read a field from an untrusted scheduled message as plain data. field(Key, Msg, Default, Opts) -> - hb_ao:get(Key, {as, <<"message@1.0">>, Msg}, Default, Opts). + hb_maps:get(Key, Msg, Default, Opts). %% @doc Read the orders currently held, as plain maps. The state may have been %% written to the process cache and read back since it was last touched, so it @@ -877,14 +877,14 @@ balance_of(Base, Address, Opts) -> balance(Base, Address, Opts). foreign_device_is_data_test() -> Opts = test_opts(), {Sender, SenderAddr} = party(), + {_, Recipient} = party(), Base = base(#{ SenderAddr => 1 }), Foreign = tx( Sender, #{ - <<"target">> => ?PROCESS, - <<"device">> => <<"manifest@1.0">>, - <<"manifest">> => #{} + <<"target">> => Recipient, + <<"device">> => <<"reference@1.0">> } ), Untouched = apply_tx(Base, Foreign, 100, Opts), diff --git a/src/preloaded/process/dev_name_token.erl b/src/preloaded/process/dev_name_token.erl index 4804c982b0..40b987027b 100644 --- a/src/preloaded/process/dev_name_token.erl +++ b/src/preloaded/process/dev_name_token.erl @@ -295,7 +295,7 @@ state(Key, Base, Default, Opts) -> %% @doc Read a field from an untrusted scheduled message as plain data. field(Key, Msg, Default, Opts) -> - hb_ao:get(Key, {as, <<"message@1.0">>, Msg}, Default, Opts). + hb_maps:get(Key, Msg, Default, Opts). balance(Base, Address, Opts) -> hb_util:int(state([?BALANCES, Address], Base, 0, Opts)). @@ -472,8 +472,7 @@ foreign_device_is_data_test() -> tx( Owner, #{ - <<"device">> => <<"manifest@1.0">>, - <<"manifest">> => #{} + <<"device">> => <<"reference@1.0">> } ), Untouched = apply_tx(name_held_by(OwnerAddr), Foreign, Opts), diff --git a/src/preloaded/process/dev_process.erl b/src/preloaded/process/dev_process.erl index 55eea8feb2..0e89957584 100644 --- a/src/preloaded/process/dev_process.erl +++ b/src/preloaded/process/dev_process.erl @@ -387,11 +387,11 @@ compute_slot(ProcID, State, RawInputMsg, InitReq, TargetSlot, Opts) -> {store_ms, StoreTimeMicroSecs div 1000}, {computed_slot_size, erlang:external_size(NewProcStateMsgWithSlot)}, {action, - hb_ao:get( - <<"body/action">>, - Req, + hb_maps:get( + <<"action">>, + hb_maps:get(<<"body">>, Req, #{}, Opts), no_action_set, - Opts#{ <<"hashpath">> => ignore } + Opts ) } } From cdc4c7432873e207a726c16bf2a9b059558979d5 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Sun, 26 Jul 2026 20:02:24 -0400 Subject: [PATCH 20/27] feat(arweave-swap): the buyer buys the exclusive right, and stakes for it The collateral was the seller's, posted at `make-offer' and paid out to a buyer whose payment could not be honoured. That is backwards. The party who might fail to perform is the buyer -- they take exclusivity and may then abandon it -- so the pledge is theirs, and it belongs in the process's own token because a pledge is only honourable while there is an offer to honour it against. A registration naming an offer that has gone takes nothing. AR could never serve: it settles on the weave whatever this device decides. * `make-offer' escrows the goods and nothing else. `deposit', `minimum-fee' and `deadline' are what the seller asks of a buyer; any may be zero. * `deadline' is how long a reservation lasts, in blocks, counted from the buyer's own registration -- not an absolute height at which the offer dies. Default 20. * `register-interest' checks the offer is open, that the transaction's `reward' covers the `minimum-fee', and that the buyer holds the `deposit'; then pledges it and reserves the order to them. * Paying returns the pledge with the goods; letting the reservation lapse forfeits it to the seller and reopens the offer. An offer no longer expires: it stands until its seller withdraws it or a buyer completes it. The book holds live offers only, so opening an offer and withdrawing it leaves the process exactly as it was -- with a test asserting that round trip, and another asserting escrow, pledges and balances always sum to the supply. Gone with the old model, and with nothing standing in for it: the grace period, the bond-retirement sweep, `compensate/5', order tombstones, `swap-cancel-grace' and `swap-reservation-blocks'. The name token's mainnet fixtures follow: they replay the same chain data and assert the order is gone rather than marked. Production is 122 lines shorter than it was, and what remains does one thing each: * `order/1' re-coerced every numeric key of every order on every read, on the stated grounds that the cache returns numbers as binaries. It does not -- there is now a test proving a round trip preserves them -- so the function did nothing and is gone, along with the coercion of `next-deadline' beside it. * `orders/2' enumerated keys through `hb_ao' to look each one up again; it takes the values. Both it and `find_order/3' now filter by matching the order's shape rather than by testing for it. * `credit/4', `debit/4' and `settle_balance/4' were one read-modify- write written twice, with a zero guard on one direction only; they are `adjust/4' and two names for its sign. * `quantity/1', `deposit/1', `minimum_fee/1', `balance_of/3', `release_goods/4', `sweep/3', `tx_field_target/2' and `tx_field_quantity/2' each stood between a caller and a single expression. The callers say what they mean instead. * `claimable/3' and `expire/4' match what they need in their heads. * The clock has one writer, and `deadlines/2' is recomputed only where a reservation can have changed -- not when opening or withdrawing an offer, neither of which can make one. --- src/preloaded/process/dev_arweave_swap.erl | 1088 ++++++++++---------- src/preloaded/process/dev_name_token.erl | 25 +- 2 files changed, 526 insertions(+), 587 deletions(-) diff --git a/src/preloaded/process/dev_arweave_swap.erl b/src/preloaded/process/dev_arweave_swap.erl index 150c99ca9d..e94e8feee7 100644 --- a/src/preloaded/process/dev_arweave_swap.erl +++ b/src/preloaded/process/dev_arweave_swap.erl @@ -22,54 +22,59 @@ %%% The protocol is four messages: %%%
      %%%
    • `make-offer' (to the process, from the seller), carrying -%%% `offer-quantity' in token units, `asking' in winston, `deposit' in -%%% token units and a `deadline' block height. The seller's -%%% `offer-quantity + deposit' moves into escrow at once, so delivery is -%%% never in doubt: the goods are already held before any buyer commits -%%% anything. The offered amount cannot be called `quantity': that is a -%%% transaction's own value field, so the codec would carry it as winston -%%% of AR sent to the process -- an address with no key, which would -%%% destroy it.
    • +%%% `offer-quantity' in token units, `asking' in winston, and what the +%%% seller asks of a buyer to reserve it: a `minimum-fee' in winston, a +%%% `deposit' in token units, and a `deadline' in blocks. Any of the +%%% three may be zero. The `offer-quantity' moves into escrow at once, so +%%% delivery is never in doubt: the goods are held before any buyer +%%% commits anything, and the seller stakes nothing further. The offered +%%% amount cannot be called `quantity': that is a transaction's own value +%%% field, so the codec would carry it as winston of AR sent to the +%%% process -- an address with no key, which would destroy it. %%%
    • `register-interest' (to the process, from a buyer), naming an -%%% `order-id'. It buys exclusivity: for `swap-reservation-blocks' the -%%% order is that buyer's alone and the seller cannot cancel it. That -%%% window is what makes paying safe -- without it a buyer races the -%%% seller's cancellation, having already sent AR that nobody can claw -%%% back. An order may charge a `minimum-fee' in winston to register, -%%% which the registration pays as its own transaction `reward': -%%% reserving an order costs the seller their bond and their right to -%%% cancel, so making it free makes it worth abusing. Paying it as the -%%% reward sends it where Arweave sends rewards -- to miners and the -%%% endowment -- rather than stranding it at the process's address, which -%%% is a transaction id with no key behind it. The fee is the protection -%%% an order can offer when it has no bond to post -- a name, whose whole -%%% supply is the single unit being sold -- and being denominated in AR it -%%% asks nothing of a buyer who holds none of the token.
    • +%%% `order-id'. It buys the exclusive right to complete the sale: for +%%% `deadline' blocks the order is that buyer's alone and the seller +%%% cannot withdraw it. That window is what makes paying safe -- without +%%% it a buyer races the seller's withdrawal, having already sent AR that +%%% nobody can claw back. Registering pays the order's `minimum-fee' as +%%% the registration's own transaction `reward', which sends it where +%%% Arweave sends rewards -- to miners and the endowment -- rather than +%%% stranding it at the process's address, which is a transaction id with +%%% no key behind it. Being denominated in AR it asks nothing of a buyer +%%% who holds none of the token. %%%
    • `cancel-order' (to the process, from the seller), naming an -%%% `order-id'. Returns the goods, but not the deposit (see below).
    • +%%% `order-id'. Returns the escrowed goods and removes the offer. It is +%%% refused while a reservation stands. %%%
    • The payment itself: an ordinary transfer whose `target' is the %%% order's `recipient', whose `quantity' is at least the `asking' %%% winston, tagged with the `order-id'. This is the message the `all' %%% mode exists to deliver.
    • %%%
    %%% -%%% The `deposit' is the seller's bond against a payment the protocol cannot -%%% honour. Once AR has been sent it cannot be returned, so the only lever -%%% remaining is a token-denominated bond, and it outlives the goods: cancelling -%%% or expiring an order releases the goods immediately but holds the `deposit' -%%% until `deadline + swap-cancel-grace'. A payment that lands in that window -%%% against goods that are gone or spoken for is paid the deposit instead. A -%%% seller who strands nobody always gets it back. Neither the order's creator -%%% nor its recipient may pay it: a self-transfer costs only a network fee, and -%%% would otherwise let a seller settle their own order to escape the bond. -%%% Because any other payer may claim that bond, a seller should not post a -%%% deposit worth more than the price they are asking, or paying for it becomes -%%% a trade in itself. +%%% An offer does not expire: it stands until its seller withdraws it or a +%%% buyer completes it. %%% -%%% Deadlines are Arweave block heights, read from the `block-height' that -%%% `all'-mode assignments carry. That is the only clock the device has, and -%%% deliberately so: reading the chain tip during a compute would be -%%% non-deterministic, and `~process@1.0' caches every slot result forever. +%%% The `deposit' is the buyer's collateral against taking exclusivity +%%% and then abandoning it -- pledged when they register, returned when they +%%% pay, forfeit to the seller when their reservation lapses unpaid. It is +%%% denominated in the process's own token because that is the only thing this +%%% device can decline to honour: a registration naming an offer that is +%%% already gone takes nothing. AR could not serve, because AR settles on the +%%% weave whatever any process decides. +%%% +%%% Paying an open offer without registering first is permitted and unwise: +%%% the seller may withdraw, or another buyer complete it, while the payment is +%%% in flight. +%%% +%%% The book holds live offers only, so opening an offer and withdrawing it +%%% leaves the process exactly as it was. Escrowed goods, pledged collateral +%%% and balances always sum to the supply. +%%% +%%% Reservations are measured in Arweave block heights, read from the +%%% `block-height' that `all'-mode assignments carry. That is the only clock +%%% the device has, and deliberately so: reading the chain tip during a compute +%%% would be non-deterministic, and `~process@1.0' caches every slot result +%%% forever. -module(dev_arweave_swap). -implements(<<"arweave-swap@1.0">>). %%% AO-Core API functions: @@ -80,11 +85,9 @@ %%% The balances submessage this device settles in. Owned by the process's %%% token implementation; the swap only moves value inside it. -define(BALANCES, <<"balances">>). -%%% The number of blocks an order is reserved for by `register-interest'. --define(DEFAULT_RESERVATION_BLOCKS, 10). -%%% The number of blocks after an order's deadline during which a payment may -%%% still be compensated from the deposit. --define(DEFAULT_CANCEL_GRACE, 20). +%%% How long a reservation lasts when an offer does not say, counted in blocks +%%% from the buyer's own `register-interest'. +-define(DEFAULT_DEADLINE, 20). %% @doc Every state transition in this device is driven by the schedule, never %% by a direct request, so every key routes to `compute'. @@ -152,7 +155,7 @@ compute(Base, Assignment, Opts) -> ProcID = field(<<"process">>, Assignment, <<>>, Opts), Body = field(<<"body">>, Assignment, #{}, Opts), Advanced = advance(Base, Height, Opts), - case tx_field_target(Body, Opts) of + case tx_field(Body, <<"target">>, <<>>, Opts) of ProcID -> {ok, control(Advanced, Body, Height, Opts)}; Target -> {ok, payment(Advanced, Body, Target, Height, Opts)} end. @@ -171,7 +174,7 @@ control(Base, Body, Height, Opts) -> _ -> Base end. -%% @doc Open an order, moving the goods and the seller's bond into escrow. +%% @doc Open an offer, moving the goods into escrow. %% Nothing is written unless the whole offer is admissible, so a rejected offer %% is indistinguishable from a transaction that was never sent. make_offer(Base, Body, Height, Opts) -> @@ -181,18 +184,18 @@ make_offer(Base, Body, Height, Opts) -> {ok, Asking} ?= amount(<<"asking">>, Body, Opts), {ok, Deposit} ?= amount(<<"deposit">>, Body, Opts), {ok, Fee} ?= amount(<<"minimum-fee">>, Body, Opts), - {ok, Deadline} ?= amount(<<"deadline">>, Body, Opts), + {ok, Deadline} ?= amount(<<"deadline">>, Body, ?DEFAULT_DEADLINE, Opts), Recipient = field(<<"recipient">>, Body, Seller, Opts), true ?= Quantity >= 1, true ?= Asking >= 1, true ?= Deposit >= 0, true ?= Fee >= 0, - true ?= Deadline > Height, - % An order with no bond asks only that the seller hold what they are - % offering. A seller whose whole holding is the thing being sold -- the - % single unit of a name -- has nothing left to bond with, and requiring - % otherwise would put a name beyond sale. - true ?= balance(Base, Seller, Opts) >= Quantity + Deposit, + true ?= Deadline >= 1, + % The seller escrows the goods and nothing else: the deposit is asked + % of the buyer, not staked by the seller. A seller whose whole holding + % is the thing being sold -- the single unit of a name -- can therefore + % still make an offer that asks collateral. + true ?= balance(Base, Seller, Opts) >= Quantity, OrderID = hb_util:human_id(hb_message:id(Body, signed, Opts)), not_found ?= hb_maps:get(OrderID, order_book(Base, Opts), not_found, Opts), Order = @@ -217,79 +220,78 @@ make_offer(Base, Body, Height, Opts) -> } ), note( - deadlines( - put_order( - debit(Base, Seller, Quantity + Deposit, Opts), - Order, - Opts - ), - Opts - ), + put_order(debit(Base, Seller, Quantity, Opts), Order, Opts), <<"order-opened">>, - OrderID, - Opts + OrderID ) else _ -> Base end. -%% @doc Withdraw an order that nobody has reserved, returning the goods. The -%% deposit stays escrowed until the grace period ends: a payment may already be -%% in flight against this order, and it is the deposit that compensates it. +%% @doc Withdraw an offer nobody has reserved, returning the escrowed goods and +%% removing it from the book. A reservation blocks it: that is what the buyer's +%% fee and collateral bought. cancel_order(Base, Body, Opts) -> maybe {ok, Signer} ?= signer(Body, Opts), {ok, Order} ?= find_order(Base, Body, Opts), - #{ <<"creator">> := Creator, <<"status">> := Status } = Order, - true ?= Signer =:= Creator, - true ?= Status =:= <<"open">>, - ?event({swap_order_cancelled, {order, order_id(Order)}}), + #{ + <<"order-id">> := OrderID, + <<"status">> := <<"open">>, + <<"creator">> := Signer, + <<"quantity">> := Quantity + } ?= Order, + ?event({swap_order_cancelled, {order, OrderID}}), note( - deadlines( - release_goods(Base, Order, Creator, <<"cancelled">>, Opts), - Opts - ), + drop_order(credit(Base, Signer, Quantity, Opts), Order, Opts), <<"order-cancelled">>, - order_id(Order), - Opts + OrderID ) else _ -> Base end. -%% @doc Reserve an open order for the sender. The reservation is exclusive and -%% freezes the seller out of cancelling, which is precisely what makes it safe -%% for the sender to part with their AR. +%% @doc Buy the exclusive right to complete an order. +%% +%% For `deadline' blocks the order is this buyer's alone: the seller cannot +%% withdraw it and nobody else can complete it. That is what makes it safe to +%% send AR, which no process can hold, redirect or refund. +%% +%% Two things are asked, either of which an offer may set to zero. The +%% `minimum-fee' is paid as this transaction's own `reward' -- burned to miners +%% and the endowment rather than collected, so it costs a buyer only what they +%% were spending anyway and makes idle registrations expensive. The `deposit' +%% is collateral, pledged from the buyer's own balance and forfeit to the +%% seller if the reservation lapses unpaid. It is denominated in the token +%% because a pledge is only honoured while there is an offer to honour it +%% against: a registration naming an order that has gone takes nothing at all. register_interest(Base, Body, Height, Opts) -> maybe {ok, Buyer} ?= signer(Body, Opts), {ok, Order} ?= find_order(Base, Body, Opts), - #{ <<"status">> := Status, <<"deadline">> := Deadline } = Order, - true ?= Status =:= <<"open">>, - true ?= Height < Deadline, - % Reserving an order costs the seller: it freezes their bond and their - % right to cancel behind somebody who may never pay. An order may - % therefore demand a fee to register, and the registration pays it as - % its own transaction `reward' -- the fee the network is already - % charging to accept it. Overpaying that goes where Arweave sends - % rewards, which is to miners and the endowment; sending it to the - % process instead would strand it at an address with no key behind it. - % Being denominated in AR, it asks nothing of a buyer who holds none of - % the token they are trying to buy. + #{ + <<"order-id">> := OrderID, + <<"status">> := <<"open">>, + <<"minimum-fee">> := Fee, + <<"deposit">> := Deposit, + <<"deadline">> := Deadline + } ?= Order, {ok, Paid} ?= amount(<<"reward">>, Body, Opts), - true ?= Paid >= minimum_fee(Order), - Until = min(Deadline, Height + reservation_blocks(Base, Opts)), + true ?= Paid >= Fee, + true ?= balance(Base, Buyer, Opts) >= Deposit, + Until = Height + Deadline, ?event( {swap_interest_registered, - {order, order_id(Order)}, + {order, OrderID}, {buyer, Buyer}, + {deposit, Deposit}, {until, Until} } ), note( deadlines( put_order( - Base, + debit(Base, Buyer, Deposit, Opts), Order#{ <<"status">> => <<"reserved">>, <<"buyer">> => Buyer, @@ -300,15 +302,12 @@ register_interest(Base, Body, Height, Opts) -> Opts ), <<"interest-registered">>, - order_id(Order), - Opts + OrderID ) else _ -> Base end. -%%% Settlement - %% @doc Settle against a layer-1 payment. The transaction is not addressed to %% the process at all: it is a transfer between two user addresses that names an %% `order-id', and the process sees it only because it is sequenced by every @@ -326,109 +325,78 @@ payment(Base, Body, Target, Height, Opts) -> #{ <<"creator">> := Creator, <<"recipient">> := Recipient, - <<"asking">> := Asking, - <<"deadline">> := Deadline + <<"asking">> := Asking } = Order, true ?= Target =:= Recipient, - % A seller paying themselves costs nothing but a network fee, and would - % otherwise let them settle their own order -- taking the goods back - % along with the bond, and leaving a real buyer's payment to arrive - % against an order that is already spent. + % A seller completing their own order from the address that made it + % would take the goods straight back. These two comparisons close that; + % they cannot close a second wallet, and nothing here can. false ?= Buyer =:= Creator, false ?= Buyer =:= Recipient, - {ok, Paid} ?= tx_field_quantity(Body, Opts), + {ok, Paid} ?= hb_util:safe_int(tx_field(Body, <<"quantity">>, 0, Opts)), true ?= Paid >= Asking, - true ?= Height =< Deadline + cancel_grace(Base, Opts), - settle(Base, Order, Buyer, Height, Body, Opts) + % An open order is first-come; a reserved one is its buyer's alone + % until the reservation lapses. + true ?= claimable(Order, Buyer, Height), + settle(Base, Order, Buyer, Opts) else _ -> Base end. -%% @doc Pay a matched payment. The goods go to the buyer and the bond returns -%% to the seller when the order was theirs to buy; otherwise the buyer takes the -%% bond in compensation, because they have paid for something they cannot -%% receive. -settle(Base, Order, Buyer, Height, Body, Opts) -> - PaymentID = hb_util:human_id(hb_message:id(Body, signed, Opts)), - Settled = - Order#{ - <<"status">> => <<"settled">>, - <<"settled-at">> => Height, - <<"payment-tx">> => PaymentID - }, - case claimable(Order, Buyer, Height) of - true -> - #{ <<"creator">> := Creator, <<"quantity">> := Quantity } = Order, - ?event( - {swap_order_settled, - {order, order_id(Order)}, - {buyer, Buyer}, - {quantity, Quantity} - } - ), - note( - deadlines( - put_order( - credit( - credit(Base, Buyer, Quantity, Opts), - Creator, - deposit(Order), - Opts - ), - Settled#{ <<"quantity">> => 0, <<"deposit">> => 0 }, - Opts - ), - Opts - ), - <<"order-settled">>, - order_id(Order), +%% @doc Complete an order: the goods pass to the buyer, and the collateral they +%% pledged to reserve it comes back to them. The AR was always the seller's +%% directly -- no process ever held it, and none could return it. +%% +%% The order then leaves the book, so a later payment naming it finds nothing +%% and does nothing. That is the only honest answer available: the goods are no +%% longer there to give, and the AR is not there to refund. +settle(Base, Order, Buyer, Opts) -> + #{ + <<"order-id">> := OrderID, + <<"status">> := Status, + <<"quantity">> := Quantity, + <<"deposit">> := Deposit + } = Order, + % A buyer who reserved the offer pledged its deposit; one who paid an open + % offer outright pledged nothing, and has nothing to get back. + Pledged = + case Status of + <<"reserved">> -> Deposit; + _ -> 0 + end, + ?event( + {swap_order_settled, + {order, OrderID}, + {buyer, Buyer}, + {quantity, Quantity} + } + ), + note( + deadlines( + drop_order( + credit(credit(Base, Buyer, Quantity, Opts), Buyer, Pledged, Opts), + Order, Opts - ); - false -> - % The goods are gone or promised to somebody else, but the buyer - % has already paid. The bond is what they get instead, and it can - % only be paid out once. - compensate(Base, Order, Buyer, PaymentID, Opts) - end. - -%% @doc Pay a stranded buyer the seller's bond. -compensate(Base, Order, Buyer, PaymentID, Opts) -> - case deposit(Order) of - 0 -> Base; - Deposit -> - ?event( - {swap_payment_compensated, - {order, order_id(Order)}, - {buyer, Buyer}, - {deposit, Deposit} - } ), - note( - deadlines( - put_order( - credit(Base, Buyer, Deposit, Opts), - Order#{ - <<"deposit">> => 0, - <<"payment-tx">> => PaymentID - }, - Opts - ), - Opts - ), - <<"payment-compensated">>, - order_id(Order), - Opts - ) - end. + Opts + ), + <<"order-settled">>, + OrderID + ). %% @doc Whether an order's goods are the payer's to take: an order nobody has %% reserved is first-come, and a reserved one is its buyer's until the %% reservation lapses. -claimable(#{ <<"quantity">> := 0 }, _Buyer, _Height) -> false; claimable(#{ <<"status">> := <<"open">> }, _Buyer, _Height) -> true; -claimable(Order = #{ <<"status">> := <<"reserved">> }, Buyer, Height) -> - maps:get(<<"buyer">>, Order, <<>>) =:= Buyer - andalso Height =< maps:get(<<"reserved-until">>, Order, 0); +claimable( + #{ + <<"status">> := <<"reserved">>, + <<"buyer">> := Buyer, + <<"reserved-until">> := Until + }, + Buyer, + Height) -> + Height =< Until; claimable(_Order, _Buyer, _Height) -> false. %%% The clock @@ -441,104 +409,67 @@ claimable(_Order, _Buyer, _Height) -> false. %% the next height at which anything at all happens, and only crossing it walks %% the orders. advance(Base, Height, Opts) -> - case hb_util:int(state(<<"next-deadline">>, Base, 0, Opts)) of + Dated = Base#{ <<"swap-height">> => Height }, + case state(<<"next-deadline">>, Base, 0, Opts) of Next when Next > 0, Height >= Next -> - deadlines(sweep(Base, Height, Opts), Opts); - _ -> - Base#{ <<"swap-height">> => Height } - end. - -%% @doc Apply every deadline that the given height has reached: reservations -%% lapse, unsold orders return their goods, and orders past their grace period -%% return the residual bond. -sweep(Base, Height, Opts) -> - lists:foldl( - fun(Order, Acc) -> expire(Acc, Order, Height, Opts) end, - Base#{ <<"swap-height">> => Height }, - orders(Base, Opts) - ). - -expire(Base, Order = #{ <<"status">> := <<"reserved">> }, Height, Opts) -> - case Height > maps:get(<<"reserved-until">>, Order, 0) of - true -> - % The reservation has lapsed, so the order is open to anyone again - % -- and being open, it may be due to expire in this same sweep. - Reopened = - maps:without( - [<<"buyer">>, <<"reserved-until">>], - Order#{ <<"status">> => <<"open">> } - ), - expire(put_order(Base, Reopened, Opts), Reopened, Height, Opts); - false -> Base - end; -expire(Base, Order = #{ <<"status">> := <<"open">>, <<"deadline">> := Deadline }, - Height, Opts) when Height >= Deadline -> - ?event({swap_order_expired, {order, order_id(Order)}}), - release_goods(Base, Order, maps:get(<<"creator">>, Order), <<"expired">>, Opts); -expire(Base, Order, Height, Opts) -> - % Only the bond is left outstanding, and the window in which a payment - % could still claim it has closed. - Deadline = maps:get(<<"deadline">>, Order, 0), - case - deposit(Order) > 0 andalso Height > Deadline + cancel_grace(Base, Opts) - of - true -> - ?event({swap_deposit_retired, {order, order_id(Order)}}), - put_order( - credit( - Base, - maps:get(<<"creator">>, Order), - deposit(Order), - Opts + deadlines( + lists:foldl( + fun(Order, Acc) -> expire(Acc, Order, Height, Opts) end, + Dated, + orders(Base, Opts) ), - Order#{ <<"deposit">> => 0 }, Opts ); - false -> Base - end. - -%% @doc Record the next height at which any order needs attention, so that the -%% slots in between cost a single comparison. Zero means nothing is pending. -deadlines(Base, Opts) -> - Grace = cancel_grace(Base, Opts), - Heights = - lists:flatten( - [ order_deadlines(Order, Grace) || Order <- orders(Base, Opts) ] - ), - Next = - case Heights of - [] -> 0; - _ -> lists:min(Heights) - end, - Base#{ <<"next-deadline">> => Next }. - -order_deadlines(Order = #{ <<"status">> := <<"reserved">> }, Grace) -> - [maps:get(<<"reserved-until">>, Order, 0) + 1 - | order_deadlines(Order#{ <<"status">> => <<"open">> }, Grace)]; -order_deadlines(Order = #{ <<"status">> := <<"open">>, <<"deadline">> := D }, Grace) -> - case quantity(Order) of - 0 -> retirement(Order, Grace); - _ -> [D | retirement(Order, Grace)] - end; -order_deadlines(Order, Grace) -> - retirement(Order, Grace). - -retirement(Order, Grace) -> - case deposit(Order) of - 0 -> []; - _ -> [maps:get(<<"deadline">>, Order, 0) + Grace + 1] + _ -> Dated end. -%%% State helpers +%% @doc Lapse a reservation the height has outlived: the buyer's collateral goes +%% to the seller, who has been held all this time, and the offer opens again. An +%% offer never expires, so this is the only thing the clock does here. -%% @doc Return an order's goods to its creator, leaving the bond escrowed for -%% whatever grace remains. -release_goods(Base, Order, Creator, Status, Opts) -> +expire( + Base, + Order = + #{ + <<"order-id">> := OrderID, + <<"status">> := <<"reserved">>, + <<"creator">> := Seller, + <<"deposit">> := Deposit, + <<"reserved-until">> := Until + }, + Height, + Opts) when Height > Until -> + ?event({swap_reservation_lapsed, {order, OrderID}, {forfeit, Deposit}}), put_order( - credit(Base, Creator, quantity(Order), Opts), - Order#{ <<"quantity">> => 0, <<"status">> => Status }, + credit(Base, Seller, Deposit, Opts), + hb_maps:without( + [<<"buyer">>, <<"reserved-until">>], + Order#{ <<"status">> => <<"open">> }, + Opts + ), Opts - ). + ); +expire(Base, _Order, _Height, _Opts) -> Base. + +%% @doc Record the next height at which a reservation lapses, so that the slots +%% in between cost a single comparison. Zero means nothing is pending. +deadlines(Base, Opts) -> + Lapses = + [ + Until + 1 + || + #{ <<"status">> := <<"reserved">>, <<"reserved-until">> := Until } + <- orders(Base, Opts) + ], + Base#{ + <<"next-deadline">> => + case Lapses of + [] -> 0; + _ -> lists:min(Lapses) + end + }. + +%%% State helpers %% @doc Read a key of the process's own state. %% @@ -560,48 +491,18 @@ field(Key, Msg, Default, Opts) -> %% is loaded through the link layer, and anything that is not an order is %% ignored rather than assumed away. orders(Base, Opts) -> - Orders = hb_cache:ensure_all_loaded(order_book(Base, Opts), Opts), [ - order(Held) + Order || - Held <- - [ - hb_maps:get(ID, Orders, #{}, Opts) - || - ID <- hb_ao:keys({as, <<"message@1.0">>, Orders}, Opts) - ], - is_map(Held), - maps:is_key(<<"order-id">>, Held) + Order = #{ <<"order-id">> := _ } <- + hb_maps:values( + hb_cache:ensure_all_loaded(order_book(Base, Opts), Opts), + Opts + ) ]. order_book(Base, Opts) -> state(<<"orders">>, Base, #{}, Opts). -%% @doc Read an order with its numbers as numbers. Between slots the state is -%% written to the process cache and read back, so nothing that a comparison or -%% a sum depends on is assumed to have survived as an integer term -- a height -%% that came back as a binary would sort above every integer, and a deadline -%% would then simply never fall due. -order(Held) -> - lists:foldl( - fun(Key, Order) -> - case maps:find(Key, Order) of - {ok, Value} -> Order#{ Key => hb_util:int(Value) }; - error -> Order - end - end, - Held, - [ - <<"quantity">>, - <<"deposit">>, - <<"minimum-fee">>, - <<"asking">>, - <<"deadline">>, - <<"created-at">>, - <<"reserved-until">>, - <<"settled-at">> - ] - ). - %% @doc Read the order a message names, if the process holds it. %% %% The name comes from a stranger's transaction, so it is looked up as a key of @@ -611,42 +512,42 @@ order(Held) -> %% slot. Whatever comes back must look like an order before it is treated as %% one. find_order(Base, Body, Opts) -> - Held = - hb_maps:get( - field(<<"order-id">>, Body, <<>>, Opts), - order_book(Base, Opts), - not_found, + case + hb_cache:ensure_all_loaded( + hb_maps:get( + field(<<"order-id">>, Body, <<>>, Opts), + order_book(Base, Opts), + not_found, + Opts + ), Opts - ), - case hb_cache:ensure_all_loaded(Held, Opts) of - Order when is_map(Order) -> - case maps:is_key(<<"order-id">>, Order) of - true -> {ok, order(Order)}; - false -> not_found - end; + ) + of + Order = #{ <<"order-id">> := _ } -> {ok, Order}; _ -> not_found end. %% @doc Write an order back, replacing the one held rather than merging over %% it, so that a lapsed reservation leaves no buyer behind. -put_order(Base, Order, Opts) -> +put_order(Base, Order = #{ <<"order-id">> := OrderID }, Opts) -> Base#{ <<"orders">> => hb_maps:put( - order_id(Order), + OrderID, Order, order_book(Base, Opts), Opts ) }. -order_id(Order) -> maps:get(<<"order-id">>, Order). - -quantity(Order) -> hb_util:int(maps:get(<<"quantity">>, Order, 0)). - -deposit(Order) -> hb_util:int(maps:get(<<"deposit">>, Order, 0)). - -minimum_fee(Order) -> hb_util:int(maps:get(<<"minimum-fee">>, Order, 0)). +%% @doc Forget an order. What the book holds is the offers still open to be +%% taken, so a completed or withdrawn one leaves no trace: opening an offer and +%% withdrawing it again returns the process to exactly where it was. +drop_order(Base, #{ <<"order-id">> := OrderID }, Opts) -> + Base#{ + <<"orders">> => + hb_maps:without([OrderID], order_book(Base, Opts), Opts) + }. %% @doc Read an address's token balance from the ledger this process shares. %% Only the one entry is read: the ledger may be large, and the rest of it is @@ -656,24 +557,20 @@ balance(Base, Address, Opts) -> credit(Base, _Address, 0, _Opts) -> Base; credit(Base, Address, Amount, Opts) -> - settle_balance(Base, Address, balance(Base, Address, Opts) + Amount, Opts). - -debit(Base, Address, Amount, Opts) -> - settle_balance(Base, Address, balance(Base, Address, Opts) - Amount, Opts). - -settle_balance(Base, Address, Value, Opts) -> Base#{ ?BALANCES => hb_maps:put( Address, - Value, + balance(Base, Address, Opts) + Amount, state(?BALANCES, Base, #{}, Opts), Opts ) }. +debit(Base, Address, Amount, Opts) -> credit(Base, Address, -Amount, Opts). + %% @doc Report what the slot did, in the results of the slot itself. -note(Base, Event, OrderID, _Opts) -> +note(Base, Event, OrderID) -> Base#{ <<"results">> => #{ <<"event">> => Event, <<"order-id">> => OrderID } }. @@ -684,8 +581,10 @@ note(Base, Event, OrderID, _Opts) -> %% the enclosing `maybe' -- which catches mismatches, not exceptions -- and fail %% that slot on every node, for good. A value that is not a number is simply not %% an admissible message. -amount(Key, Body, Opts) -> - hb_util:safe_int(field(Key, Body, 0, Opts)). +amount(Key, Body, Opts) -> amount(Key, Body, 0, Opts). + +amount(Key, Body, Default, Opts) -> + hb_util:safe_int(field(Key, Body, Default, Opts)). %% @doc Read a value from the real L1 transaction fields recorded in the %% `tx@1.0' commitment. Top-level keys may come from tags with the same names, so @@ -698,12 +597,6 @@ tx_field(Body, Field, Default, Opts) -> Default end. -tx_field_target(Body, Opts) -> - tx_field(Body, <<"target">>, <<>>, Opts). - -tx_field_quantity(Body, Opts) -> - hb_util:safe_int(tx_field(Body, <<"quantity">>, 0, Opts)). - %% @doc The single signer of a message. A message with any other number of %% signers is not attributable to one party, so it cannot open, cancel, reserve %% or pay for anything. @@ -713,21 +606,6 @@ signer(Body, Opts) -> _ -> not_found end. -reservation_blocks(Base, Opts) -> - hb_util:int( - state( - <<"swap-reservation-blocks">>, - Base, - ?DEFAULT_RESERVATION_BLOCKS, - Opts - ) - ). - -cancel_grace(Base, Opts) -> - hb_util:int( - state(<<"swap-cancel-grace">>, Base, ?DEFAULT_CANCEL_GRACE, Opts) - ). - %%% Tests %%% The tests drive `compute/3' directly with synthetic assignments, exactly as @@ -755,6 +633,27 @@ tx(Wallet, Fields) -> #{ <<"commitment-device">> => <<"tx@1.0">> } ). +%% @doc Do an order's numbers survive the process cache as numbers? +%% +%% `order/1' re-coerces every numeric key on the way out, on the stated grounds +%% that a value written between slots may come back as a binary. If that is not +%% so, the coercion is doing nothing at all. +order_numbers_survive_the_cache_test() -> + Opts = test_opts(), + Written = #{ <<"orders">> => #{ <<"o">> => #{ + <<"order-id">> => <<"o">>, + <<"quantity">> => 10, + <<"deadline">> => 20, + <<"reserved-until">> => 1966680 + } } }, + {ok, ID} = hb_cache:write(Written, Opts), + {ok, Read} = hb_cache:read(ID, Opts), + Loaded = hb_cache:ensure_all_loaded(Read, Opts), + Order = hb_maps:get(<<"o">>, hb_maps:get(<<"orders">>, Loaded, #{}, Opts), #{}, Opts), + ?event({round_tripped, {order, Order}}), + ?assertEqual(10, maps:get(<<"quantity">>, Order)), + ?assertEqual(1966680, maps:get(<<"reserved-until">>, Order)). + %% @doc A message this device reads is a stranger's, and a key it does not carry %% must be answered by this device's default, not by code the stranger picked. %% @@ -871,8 +770,6 @@ only_order(Base, Opts) -> [Order] = orders(Base, Opts), Order. -balance_of(Base, Address, Opts) -> balance(Base, Address, Opts). - %% @doc A foreign transaction's device cannot interpret absent control fields. foreign_device_is_data_test() -> Opts = test_opts(), @@ -888,21 +785,21 @@ foreign_device_is_data_test() -> } ), Untouched = apply_tx(Base, Foreign, 100, Opts), - ?assertEqual(1, balance_of(Untouched, SenderAddr, Opts)), + ?assertEqual(1, balance(Untouched, SenderAddr, Opts)), ?assertEqual([], orders(Untouched, Opts)). -%% @doc Opening an offer moves the goods and the bond into escrow, and leaves +%% @doc Opening an offer moves the goods into escrow, and leaves %% the order open. make_offer_escrows_test() -> Opts = test_opts(), {Seller, SellerAddr} = party(), Base = base(#{ SellerAddr => 100 }), - Opened = apply_tx(Base, offer(Seller, 10, 500, 5, 200), 100, Opts), + Opened = apply_tx(Base, offer(Seller, 10, 500, 5, 20), 100, Opts), Order = only_order(Opened, Opts), - ?assertEqual(85, balance_of(Opened, SellerAddr, Opts)), + ?assertEqual(90, balance(Opened, SellerAddr, Opts)), ?assertEqual(<<"open">>, maps:get(<<"status">>, Order)), - ?assertEqual(10, quantity(Order)), - ?assertEqual(5, deposit(Order)), + ?assertEqual(10, maps:get(<<"quantity">>, Order)), + ?assertEqual(5, maps:get(<<"deposit">>, Order)), ?assertEqual(SellerAddr, maps:get(<<"recipient">>, Order)). %% @doc An offer for more than the seller holds changes nothing at all. @@ -910,17 +807,9 @@ make_offer_insufficient_balance_test() -> Opts = test_opts(), {Seller, SellerAddr} = party(), Base = base(#{ SellerAddr => 4 }), - Result = apply_tx(Base, offer(Seller, 10, 500, 5, 200), 100, Opts), + Result = apply_tx(Base, offer(Seller, 10, 500, 5, 20), 100, Opts), ?assertEqual([], orders(Result, Opts)), - ?assertEqual(4, balance_of(Result, SellerAddr, Opts)). - -%% @doc A deadline that has already passed is not an offer. -make_offer_stale_deadline_test() -> - Opts = test_opts(), - {Seller, SellerAddr} = party(), - Base = base(#{ SellerAddr => 100 }), - Result = apply_tx(Base, offer(Seller, 10, 500, 5, 100), 100, Opts), - ?assertEqual([], orders(Result, Opts)). + ?assertEqual(4, balance(Result, SellerAddr, Opts)). %% @doc Tag-only trade keys are metadata and do not route control slots. tag_only_target_is_metadata_test() -> @@ -939,10 +828,10 @@ tag_only_target_is_metadata_test() -> ] ), ?assertEqual(?PROCESS, hb_ao:get(<<"target">>, Tagged, not_found, Opts)), - ?assertEqual(<<>>, tx_field_target(Tagged, Opts)), + ?assertEqual(<<>>, tx_field(Tagged, <<"target">>, <<>>, Opts)), Result = apply_tx(base(#{ SellerAddr => 100 }), Tagged, 100, Opts), ?assertEqual([], orders(Result, Opts)), - ?assertEqual(100, balance_of(Result, SellerAddr, Opts)). + ?assertEqual(100, balance(Result, SellerAddr, Opts)). %% @doc The whole trade: the buyer pays the seller directly on layer one, and %% the process -- which is not a party to that payment -- settles it. @@ -951,8 +840,8 @@ settlement_test() -> {Seller, SellerAddr} = party(), {Buyer, BuyerAddr} = party(), Base = base(#{ SellerAddr => 100 }), - Opened = apply_tx(Base, offer(Seller, 10, 500, 5, 200), 100, Opts), - OrderID = order_id(only_order(Opened, Opts)), + Opened = apply_tx(Base, offer(Seller, 10, 500, 5, 20), 100, Opts), + OrderID = maps:get(<<"order-id">>, only_order(Opened, Opts)), Settled = apply_tx( Opened, @@ -960,12 +849,10 @@ settlement_test() -> 120, Opts ), - Order = only_order(Settled, Opts), - ?assertEqual(<<"settled">>, maps:get(<<"status">>, Order)), - % The buyer has the goods; the seller has their bond back and is out the - % tokens they sold. - ?assertEqual(10, balance_of(Settled, BuyerAddr, Opts)), - ?assertEqual(90, balance_of(Settled, SellerAddr, Opts)). + % The offer is complete, so it is no longer an offer. + ?assertEqual([], orders(Settled, Opts)), + ?assertEqual(10, balance(Settled, BuyerAddr, Opts)), + ?assertEqual(90, balance(Settled, SellerAddr, Opts)). %% @doc Paying less than the asking price settles nothing: the process never %% held the value, so it cannot refund a partial fill. @@ -974,11 +861,11 @@ underpayment_test() -> {Seller, SellerAddr} = party(), {Buyer, BuyerAddr} = party(), Opened = - apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 200), 100, Opts), - OrderID = order_id(only_order(Opened, Opts)), + apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 20), 100, Opts), + OrderID = maps:get(<<"order-id">>, only_order(Opened, Opts)), Result = apply_tx(Opened, pay(Buyer, SellerAddr, 499, OrderID), 120, Opts), ?assertEqual(<<"open">>, maps:get(<<"status">>, only_order(Result, Opts))), - ?assertEqual(0, balance_of(Result, BuyerAddr, Opts)). + ?assertEqual(0, balance(Result, BuyerAddr, Opts)). %% @doc A payment that names the order but is addressed to somebody else is not %% a payment for it. @@ -988,11 +875,11 @@ payment_to_wrong_address_test() -> {Buyer, BuyerAddr} = party(), {_, Stranger} = party(), Opened = - apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 200), 100, Opts), - OrderID = order_id(only_order(Opened, Opts)), + apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 20), 100, Opts), + OrderID = maps:get(<<"order-id">>, only_order(Opened, Opts)), Result = apply_tx(Opened, pay(Buyer, Stranger, 500, OrderID), 120, Opts), ?assertEqual(<<"open">>, maps:get(<<"status">>, only_order(Result, Opts))), - ?assertEqual(0, balance_of(Result, BuyerAddr, Opts)). + ?assertEqual(0, balance(Result, BuyerAddr, Opts)). %% @doc Tag-only trade keys are metadata and do not count as the payment leg. tag_only_transfer_is_metadata_test() -> @@ -1000,37 +887,17 @@ tag_only_transfer_is_metadata_test() -> {Seller, SellerAddr} = party(), {Buyer, BuyerAddr} = party(), Opened = - apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 200), 100, Opts), - OrderID = order_id(only_order(Opened, Opts)), + apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 20), 100, Opts), + OrderID = maps:get(<<"order-id">>, only_order(Opened, Opts)), Tagged = tag_only_transfer(Buyer, SellerAddr, 500, OrderID), ?assertEqual(SellerAddr, hb_ao:get(<<"target">>, Tagged, not_found, Opts)), ?assertEqual(<<"500">>, hb_ao:get(<<"quantity">>, Tagged, not_found, Opts)), - ?assertEqual(<<>>, tx_field_target(Tagged, Opts)), - ?assertEqual({ok, 0}, tx_field_quantity(Tagged, Opts)), + ?assertEqual(<<>>, tx_field(Tagged, <<"target">>, <<>>, Opts)), + ?assertEqual({ok, 0}, hb_util:safe_int(tx_field(Tagged, <<"quantity">>, 0, Opts))), Result = apply_tx(Opened, Tagged, 120, Opts), ?assertEqual(<<"open">>, maps:get(<<"status">>, only_order(Result, Opts))), - ?assertEqual(0, balance_of(Result, BuyerAddr, Opts)), - ?assertEqual(85, balance_of(Result, SellerAddr, Opts)). - -%% @doc Cancelling returns the goods at once, but holds the bond for as long as -%% a payment could still be in flight. -cancel_returns_goods_but_holds_bond_test() -> - Opts = test_opts(), - {Seller, SellerAddr} = party(), - Opened = - apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 200), 100, Opts), - OrderID = order_id(only_order(Opened, Opts)), - Cancelled = - apply_tx(Opened, order_action(Seller, <<"cancel-order">>, OrderID), 110, Opts), - Order = only_order(Cancelled, Opts), - ?assertEqual(<<"cancelled">>, maps:get(<<"status">>, Order)), - ?assertEqual(0, quantity(Order)), - ?assertEqual(5, deposit(Order)), - ?assertEqual(95, balance_of(Cancelled, SellerAddr, Opts)), - % Once the grace period has passed with no payment, the bond comes back. - Retired = tick(Cancelled, 200 + ?DEFAULT_CANCEL_GRACE + 1, Opts), - ?assertEqual(0, deposit(only_order(Retired, Opts))), - ?assertEqual(100, balance_of(Retired, SellerAddr, Opts)). + ?assertEqual(0, balance(Result, BuyerAddr, Opts)), + ?assertEqual(90, balance(Result, SellerAddr, Opts)). %% @doc Only the seller may cancel their order. cancel_by_stranger_test() -> @@ -1038,21 +905,26 @@ cancel_by_stranger_test() -> {Seller, SellerAddr} = party(), {Stranger, _} = party(), Opened = - apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 200), 100, Opts), - OrderID = order_id(only_order(Opened, Opts)), + apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 20), 100, Opts), + OrderID = maps:get(<<"order-id">>, only_order(Opened, Opts)), Result = apply_tx(Opened, order_action(Stranger, <<"cancel-order">>, OrderID), 110, Opts), ?assertEqual(<<"open">>, maps:get(<<"status">>, only_order(Result, Opts))). %% @doc A reserved order cannot be pulled out from under the buyer who reserved %% it. This is the guarantee that makes paying safe. -reservation_blocks_cancellation_test() -> +reservation_prevents_cancellation_test() -> Opts = test_opts(), {Seller, SellerAddr} = party(), - {Buyer, _} = party(), + {Buyer, BuyerAddr} = party(), Opened = - apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 200), 100, Opts), - OrderID = order_id(only_order(Opened, Opts)), + apply_tx( + base(#{ SellerAddr => 100, BuyerAddr => 5 }), + offer(Seller, 10, 500, 5, 20), + 100, + Opts + ), + OrderID = maps:get(<<"order-id">>, only_order(Opened, Opts)), Reserved = apply_tx( Opened, @@ -1071,86 +943,56 @@ reservation_blocks_cancellation_test() -> ?assertEqual(<<"reserved">>, maps:get(<<"status">>, only_order(Attempted, Opts))). %% @doc A reservation is exclusive while it lasts: somebody else's payment does -%% not take the goods, it takes the bond. +%% not take the goods. reservation_is_exclusive_test() -> Opts = test_opts(), {Seller, SellerAddr} = party(), - {Buyer, _} = party(), + {Buyer, BuyerAddr} = party(), {Interloper, InterloperAddr} = party(), Opened = - apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 200), 100, Opts), - OrderID = order_id(only_order(Opened, Opts)), + apply_tx( + base(#{ SellerAddr => 100, BuyerAddr => 5 }), + offer(Seller, 10, 500, 5, 20), + 100, + Opts + ), + OrderID = maps:get(<<"order-id">>, only_order(Opened, Opts)), Reserved = apply_tx(Opened, order_action(Buyer, <<"register-interest">>, OrderID), 110, Opts), + % The buyer's collateral is pledged out of their balance. + ?assertEqual(0, balance(Reserved, BuyerAddr, Opts)), Result = apply_tx(Reserved, pay(Interloper, SellerAddr, 500, OrderID), 111, Opts), Order = only_order(Result, Opts), + % The stranger's AR bought nothing: the order is not theirs to complete, and + % there is nothing here that could give it back. ?assertEqual(<<"reserved">>, maps:get(<<"status">>, Order)), - ?assertEqual(10, quantity(Order)), - ?assertEqual(0, deposit(Order)), - ?assertEqual(5, balance_of(Result, InterloperAddr, Opts)). + ?assertEqual(BuyerAddr, maps:get(<<"buyer">>, Order)), + ?assertEqual(10, maps:get(<<"quantity">>, Order)), + ?assertEqual(0, balance(Result, InterloperAddr, Opts)). %% @doc A reservation lapses on its own, without anybody sending anything: the %% network's own traffic carries the clock forward. reservation_lapses_test() -> Opts = test_opts(), {Seller, SellerAddr} = party(), - {Buyer, _} = party(), + {Buyer, BuyerAddr} = party(), Opened = - apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 200), 100, Opts), - OrderID = order_id(only_order(Opened, Opts)), + apply_tx( + base(#{ SellerAddr => 100, BuyerAddr => 5 }), + offer(Seller, 10, 500, 5, 20), + 100, + Opts + ), + OrderID = maps:get(<<"order-id">>, only_order(Opened, Opts)), Reserved = apply_tx(Opened, order_action(Buyer, <<"register-interest">>, OrderID), 110, Opts), ?assertEqual( - 110 + ?DEFAULT_RESERVATION_BLOCKS, + 110 + 20, maps:get(<<"reserved-until">>, only_order(Reserved, Opts)) ), - Lapsed = tick(Reserved, 110 + ?DEFAULT_RESERVATION_BLOCKS + 1, Opts), + Lapsed = tick(Reserved, 110 + 20 + 1, Opts), ?assertEqual(<<"open">>, maps:get(<<"status">>, only_order(Lapsed, Opts))). -%% @doc An unsold order returns its goods when its deadline passes. -expiry_returns_goods_test() -> - Opts = test_opts(), - {Seller, SellerAddr} = party(), - Opened = - apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 200), 100, Opts), - ?assertEqual(85, balance_of(Opened, SellerAddr, Opts)), - Expired = tick(Opened, 200, Opts), - Order = only_order(Expired, Opts), - ?assertEqual(<<"expired">>, maps:get(<<"status">>, Order)), - ?assertEqual(95, balance_of(Expired, SellerAddr, Opts)), - ?assertEqual(5, deposit(Order)). - -%% @doc A buyer who pays for an expired order within the grace period is paid -%% the seller's bond: they parted with value for goods that were gone. -late_payment_takes_the_bond_test() -> - Opts = test_opts(), - {Seller, SellerAddr} = party(), - {Buyer, BuyerAddr} = party(), - Opened = - apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 200), 100, Opts), - OrderID = order_id(only_order(Opened, Opts)), - Expired = tick(Opened, 201, Opts), - Paid = apply_tx(Expired, pay(Buyer, SellerAddr, 500, OrderID), 202, Opts), - ?assertEqual(5, balance_of(Paid, BuyerAddr, Opts)), - ?assertEqual(0, deposit(only_order(Paid, Opts))), - ?assertEqual(95, balance_of(Paid, SellerAddr, Opts)). - -%% @doc The bond is paid out once. A second late payment gets nothing, because -%% there is nothing left to compensate it with. -bond_pays_out_once_test() -> - Opts = test_opts(), - {Seller, SellerAddr} = party(), - {First, FirstAddr} = party(), - {Second, SecondAddr} = party(), - Opened = - apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 200), 100, Opts), - OrderID = order_id(only_order(Opened, Opts)), - Expired = tick(Opened, 201, Opts), - Once = apply_tx(Expired, pay(First, SellerAddr, 500, OrderID), 202, Opts), - Twice = apply_tx(Once, pay(Second, SellerAddr, 500, OrderID), 203, Opts), - ?assertEqual(5, balance_of(Twice, FirstAddr, Opts)), - ?assertEqual(0, balance_of(Twice, SecondAddr, Opts)). - %% @doc Settling twice is not possible: the goods left with the first payment. double_settlement_test() -> Opts = test_opts(), @@ -1158,22 +1000,27 @@ double_settlement_test() -> {Buyer, BuyerAddr} = party(), {Late, LateAddr} = party(), Opened = - apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 200), 100, Opts), - OrderID = order_id(only_order(Opened, Opts)), + apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 20), 100, Opts), + OrderID = maps:get(<<"order-id">>, only_order(Opened, Opts)), Settled = apply_tx(Opened, pay(Buyer, SellerAddr, 500, OrderID), 120, Opts), Again = apply_tx(Settled, pay(Late, SellerAddr, 500, OrderID), 121, Opts), - ?assertEqual(10, balance_of(Again, BuyerAddr, Opts)), - ?assertEqual(0, balance_of(Again, LateAddr, Opts)), - ?assertEqual(90, balance_of(Again, SellerAddr, Opts)). + ?assertEqual(10, balance(Again, BuyerAddr, Opts)), + ?assertEqual(0, balance(Again, LateAddr, Opts)), + ?assertEqual(90, balance(Again, SellerAddr, Opts)). %% @doc A payment naming an order the process has never heard of is ordinary %% Arweave traffic, and is ignored. unknown_order_test() -> Opts = test_opts(), {Seller, SellerAddr} = party(), - {Buyer, _} = party(), + {Buyer, BuyerAddr} = party(), Opened = - apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 200), 100, Opts), + apply_tx( + base(#{ SellerAddr => 100, BuyerAddr => 5 }), + offer(Seller, 10, 500, 5, 20), + 100, + Opts + ), Result = apply_tx( Opened, @@ -1189,11 +1036,11 @@ unrelated_traffic_test() -> Opts = test_opts(), {Seller, SellerAddr} = party(), Opened = - apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 200), 100, Opts), + apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 0, 20), 100, Opts), Ticked = tick(Opened, 150, Opts), ?assertEqual(150, hb_ao:get(<<"swap-height">>, Ticked, 0, Opts)), ?assertEqual(<<"open">>, maps:get(<<"status">>, only_order(Ticked, Opts))), - ?assertEqual(85, balance_of(Ticked, SellerAddr, Opts)). + ?assertEqual(90, balance(Ticked, SellerAddr, Opts)). %% @doc A stranger's transaction may ask to be routed anywhere -- the key a slot %% resolves comes from the sender's own `path' tag, and this process is @@ -1204,7 +1051,7 @@ stray_path_is_applied_test() -> {Seller, SellerAddr} = party(), {Stranger, _} = party(), Opened = - apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 200), 100, Opts), + apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 20), 100, Opts), Strayed = hb_ao:resolve( Opened#{ <<"device">> => <<"arweave-swap@1.0">> }, @@ -1224,54 +1071,147 @@ stray_path_is_applied_test() -> % directly rather than through `lib_process:run_as', which puts the % process's own device back afterwards. Read it as a message. ?assertEqual(150, hb_util:int(state(<<"swap-height">>, State, 0, Opts))), - ?assertEqual(85, balance_of(State, SellerAddr, Opts)). + ?assertEqual(90, balance(State, SellerAddr, Opts)). -%% @doc An order read back with its numbers encoded as binaries -- which is how -%% it returns from the process cache -- still falls due. Comparing a height -%% against a binary would silently never fire. -encoded_order_still_expires_test() -> +%% @doc Collateral returns to the buyer when they complete the sale. +settlement_returns_the_collateral_test() -> Opts = test_opts(), - {_, SellerAddr} = party(), - Encoded = - #{ - ?BALANCES => #{ SellerAddr => 85 }, - <<"next-deadline">> => <<"200">>, - <<"orders">> => - #{ - <<"order-1">> => - #{ - <<"order-id">> => <<"order-1">>, - <<"creator">> => SellerAddr, - <<"recipient">> => SellerAddr, - <<"quantity">> => <<"10">>, - <<"asking">> => <<"500">>, - <<"deposit">> => <<"5">>, - <<"deadline">> => <<"200">>, - <<"status">> => <<"open">> - } - } - }, - Expired = tick(Encoded, 200, Opts), - ?assertEqual(<<"expired">>, maps:get(<<"status">>, only_order(Expired, Opts))), - ?assertEqual(95, balance_of(Expired, SellerAddr, Opts)). + {Seller, SellerAddr} = party(), + {Buyer, BuyerAddr} = party(), + Opened = + apply_tx( + base(#{ SellerAddr => 100, BuyerAddr => 5 }), + offer(Seller, 10, 500, 5, 20), + 100, + Opts + ), + OrderID = maps:get(<<"order-id">>, only_order(Opened, Opts)), + Reserved = + apply_tx(Opened, order_action(Buyer, <<"register-interest">>, OrderID), 110, Opts), + ?assertEqual(0, balance(Reserved, BuyerAddr, Opts)), + Settled = apply_tx(Reserved, pay(Buyer, SellerAddr, 500, OrderID), 115, Opts), + ?assertEqual([], orders(Settled, Opts)), + % The goods, and the collateral back. + ?assertEqual(15, balance(Settled, BuyerAddr, Opts)), + ?assertEqual(90, balance(Settled, SellerAddr, Opts)). + +%% @doc A buyer who cannot cover the collateral cannot reserve. +collateral_must_be_held_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + {Buyer, BuyerAddr} = party(), + Opened = + apply_tx( + base(#{ SellerAddr => 100, BuyerAddr => 4 }), + offer(Seller, 10, 500, 5, 20), + 100, + Opts + ), + OrderID = maps:get(<<"order-id">>, only_order(Opened, Opts)), + Tried = + apply_tx(Opened, order_action(Buyer, <<"register-interest">>, OrderID), 110, Opts), + ?assertEqual(<<"open">>, maps:get(<<"status">>, only_order(Tried, Opts))), + ?assertEqual(4, balance(Tried, BuyerAddr, Opts)). -%% @doc A reservation that lapses in the same sweep that its order expires is -%% resolved in order, and leaves no stale buyer behind. -lapse_and_expire_together_test() -> +%% @doc A registration naming an offer that has gone takes nothing. +%% +%% This is why collateral is denominated in the token: the process can decline +%% to honour a pledge when there is nothing to honour it against. AR could not +%% serve, because AR settles on the weave whatever this device decides. +collateral_is_not_taken_for_a_vanished_offer_test() -> Opts = test_opts(), {Seller, SellerAddr} = party(), - {Buyer, _} = party(), + {Buyer, BuyerAddr} = party(), + Before = base(#{ SellerAddr => 100, BuyerAddr => 5 }), + Opened = apply_tx(Before, offer(Seller, 10, 500, 5, 20), 100, Opts), + OrderID = maps:get(<<"order-id">>, only_order(Opened, Opts)), + Withdrawn = + apply_tx(Opened, order_action(Seller, <<"cancel-order">>, OrderID), 105, Opts), + Late = + apply_tx( + Withdrawn, + order_action(Buyer, <<"register-interest">>, OrderID), + 110, + Opts + ), + ?assertEqual(5, balance(Late, BuyerAddr, Opts)), + ?assertEqual([], orders(Late, Opts)). + +%% @doc Opening an offer and withdrawing it leaves the process as it was. +offer_and_withdrawal_are_a_round_trip_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + Before = base(#{ SellerAddr => 100 }), + Opened = apply_tx(Before, offer(Seller, 10, 500, 5, 20), 100, Opts), + OrderID = maps:get(<<"order-id">>, only_order(Opened, Opts)), + After = + apply_tx(Opened, order_action(Seller, <<"cancel-order">>, OrderID), 105, Opts), + ?assertEqual([], orders(After, Opts)), + ?assertEqual(100, balance(After, SellerAddr, Opts)), + ?assertEqual(0, hb_util:int(state(<<"next-deadline">>, After, 0, Opts))). + +%% @doc Units are conserved across a whole trade: what is escrowed, what is +%% pledged and what is held always sum to the supply. +supply_is_conserved_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + {Buyer, BuyerAddr} = party(), + Held = + fun(State) -> + balance(State, SellerAddr, Opts) + balance(State, BuyerAddr, Opts) + + lists:sum( + [ + % Goods are escrowed by every offer; collateral only by + % one somebody has reserved. + case maps:get(<<"status">>, Order) of + <<"reserved">> -> maps:get(<<"quantity">>, Order) + maps:get(<<"deposit">>, Order); + _ -> maps:get(<<"quantity">>, Order) + end + || + Order <- orders(State, Opts) + ] + ) + end, + Before = base(#{ SellerAddr => 100, BuyerAddr => 5 }), + ?assertEqual(105, Held(Before)), + Opened = apply_tx(Before, offer(Seller, 10, 500, 5, 20), 100, Opts), + ?assertEqual(105, Held(Opened)), + OrderID = maps:get(<<"order-id">>, only_order(Opened, Opts)), + Reserved = + apply_tx(Opened, order_action(Buyer, <<"register-interest">>, OrderID), 110, Opts), + ?assertEqual(105, Held(Reserved)), + Settled = apply_tx(Reserved, pay(Buyer, SellerAddr, 500, OrderID), 115, Opts), + ?assertEqual(105, Held(Settled)), + Lapsed = tick(apply_tx(Before, offer(Seller, 10, 500, 5, 20), 100, Opts), 300, Opts), + ?assertEqual(105, Held(Lapsed)). + +%% @doc A lapsed reservation pays the seller and reopens the offer. +%% +%% The collateral is the buyer's answer for having frozen the seller out. When +%% the window passes unpaid it is the seller's, and the offer -- which never +%% expires -- is open to anyone again. +lapse_forfeits_collateral_test() -> + Opts = test_opts(), + {Seller, SellerAddr} = party(), + {Buyer, BuyerAddr} = party(), Opened = - apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 195), 100, Opts), - OrderID = order_id(only_order(Opened, Opts)), + apply_tx( + base(#{ SellerAddr => 100, BuyerAddr => 5 }), + offer(Seller, 10, 500, 5, 20), + 100, + Opts + ), + OrderID = maps:get(<<"order-id">>, only_order(Opened, Opts)), Reserved = apply_tx(Opened, order_action(Buyer, <<"register-interest">>, OrderID), 190, Opts), - % Reserved to 195 (capped at the deadline); at 196 both are due. - Swept = tick(Reserved, 196, Opts), - Order = only_order(Swept, Opts), - ?assertEqual(<<"expired">>, maps:get(<<"status">>, Order)), + ?assertEqual(0, balance(Reserved, BuyerAddr, Opts)), + Lapsed = tick(Reserved, 190 + 20 + 1, Opts), + Order = only_order(Lapsed, Opts), + ?assertEqual(<<"open">>, maps:get(<<"status">>, Order)), ?assertEqual(false, maps:is_key(<<"buyer">>, Order)), - ?assertEqual(95, balance_of(Swept, SellerAddr, Opts)). + % The seller keeps 90 escrowed-out plus the 5 forfeited to them. + ?assertEqual(95, balance(Lapsed, SellerAddr, Opts)), + ?assertEqual(0, balance(Lapsed, BuyerAddr, Opts)). %% @doc The offered amount is not carried as `quantity'. That key is the %% transaction's own value field, so the codec would send it as winston of AR to @@ -1293,7 +1233,7 @@ offer_quantity_is_not_winston_test() -> ), Result = apply_tx(base(#{ SellerAddr => 100 }), Wrong, 100, Opts), ?assertEqual([], orders(Result, Opts)), - ?assertEqual(100, balance_of(Result, SellerAddr, Opts)). + ?assertEqual(100, balance(Result, SellerAddr, Opts)). %% @doc A figure that is not a number is an inadmissible message, not a failed %% slot: anyone may send this process anything, and a slot that raises can never @@ -1315,17 +1255,22 @@ non_numeric_tag_is_ignored_test() -> ), Result = apply_tx(base(#{ SellerAddr => 100 }), Nonsense, 100, Opts), ?assertEqual([], orders(Result, Opts)), - ?assertEqual(100, balance_of(Result, SellerAddr, Opts)). + ?assertEqual(100, balance(Result, SellerAddr, Opts)). %% @doc An `order-id' is caller-supplied text. Path-like values and reserved %% keys must not reach anything that is then read as an order. reserved_order_id_names_test() -> Opts = test_opts(), {Seller, SellerAddr} = party(), - {Buyer, _} = party(), + {Buyer, BuyerAddr} = party(), Opened = - apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 200), 100, Opts), - OrderID = order_id(only_order(Opened, Opts)), + apply_tx( + base(#{ SellerAddr => 100, BuyerAddr => 5 }), + offer(Seller, 10, 500, 5, 20), + 100, + Opts + ), + OrderID = maps:get(<<"order-id">>, only_order(Opened, Opts)), Names = [ <>, @@ -1345,29 +1290,17 @@ reserved_order_id_names_test() -> ). %% @doc A seller cannot buy their own order. Paying oneself costs only a network -%% fee, so it would otherwise take back the goods and the bond, leaving a real +%% fee, so it would otherwise take back the goods, leaving a real %% buyer to pay for an order that is already spent. seller_cannot_settle_own_order_test() -> Opts = test_opts(), {Seller, SellerAddr} = party(), Opened = - apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 200), 100, Opts), - OrderID = order_id(only_order(Opened, Opts)), + apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 20), 100, Opts), + OrderID = maps:get(<<"order-id">>, only_order(Opened, Opts)), Result = apply_tx(Opened, pay(Seller, SellerAddr, 500, OrderID), 120, Opts), ?assertEqual(<<"open">>, maps:get(<<"status">>, only_order(Result, Opts))), - ?assertEqual(85, balance_of(Result, SellerAddr, Opts)). - -%% @doc Nor can they claim their own bond once the order has expired. -seller_cannot_claim_own_bond_test() -> - Opts = test_opts(), - {Seller, SellerAddr} = party(), - Opened = - apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 200), 100, Opts), - OrderID = order_id(only_order(Opened, Opts)), - Expired = tick(Opened, 201, Opts), - Result = apply_tx(Expired, pay(Seller, SellerAddr, 500, OrderID), 202, Opts), - ?assertEqual(5, deposit(only_order(Result, Opts))), - ?assertEqual(95, balance_of(Result, SellerAddr, Opts)). + ?assertEqual(90, balance(Result, SellerAddr, Opts)). %% @doc A scheduled message routed to `set' or `keys' is applied like any other. %% Handing either to `~message@1.0' would let a passer-by write the process's own @@ -1377,7 +1310,7 @@ reserved_paths_are_applied_test() -> {Seller, SellerAddr} = party(), {Stranger, _} = party(), Opened = - apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 200), 100, Opts), + apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 20), 100, Opts), lists:foreach( fun(Path) -> {ok, State} = @@ -1394,7 +1327,7 @@ reserved_paths_are_applied_test() -> }, Opts ), - ?assertEqual(85, balance_of(State, SellerAddr, Opts)), + ?assertEqual(90, balance(State, SellerAddr, Opts)), ?assertEqual( <<"open">>, maps:get(<<"status">>, only_order(State, Opts)) @@ -1425,7 +1358,7 @@ minimum_fee_gates_registration_test() -> } ), Opened = apply_tx(base(#{ SellerAddr => 100 }), Charged, 100, Opts), - OrderID = order_id(only_order(Opened, Opts)), + OrderID = maps:get(<<"order-id">>, only_order(Opened, Opts)), Free = apply_tx( Opened, @@ -1470,10 +1403,15 @@ minimum_fee_gates_registration_test() -> no_minimum_fee_registers_free_test() -> Opts = test_opts(), {Seller, SellerAddr} = party(), - {Buyer, _} = party(), + {Buyer, BuyerAddr} = party(), Opened = - apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 200), 100, Opts), - OrderID = order_id(only_order(Opened, Opts)), + apply_tx( + base(#{ SellerAddr => 100, BuyerAddr => 5 }), + offer(Seller, 10, 500, 5, 20), + 100, + Opts + ), + OrderID = maps:get(<<"order-id">>, only_order(Opened, Opts)), Reserved = apply_tx( Opened, @@ -1487,16 +1425,16 @@ no_minimum_fee_registers_free_test() -> ). %% @doc A seller whose whole holding is the single unit they are selling can -%% still offer it, because an order with no bond asks only for the goods. +%% still offer it: an offer escrows the goods and asks nothing else of them. single_unit_offer_test() -> Opts = test_opts(), {Seller, SellerAddr} = party(), Opened = - apply_tx(base(#{ SellerAddr => 1 }), offer(Seller, 1, 500, 0, 200), 100, Opts), + apply_tx(base(#{ SellerAddr => 1 }), offer(Seller, 1, 500, 0, 20), 100, Opts), Order = only_order(Opened, Opts), - ?assertEqual(1, quantity(Order)), - ?assertEqual(0, deposit(Order)), - ?assertEqual(0, balance_of(Opened, SellerAddr, Opts)). + ?assertEqual(1, maps:get(<<"quantity">>, Order)), + ?assertEqual(0, maps:get(<<"deposit">>, Order)), + ?assertEqual(0, balance(Opened, SellerAddr, Opts)). %% @doc A registration paying a fee, as the reward on its own transaction. registration(Wallet, OrderID, Winston) -> @@ -1515,15 +1453,21 @@ registration(Wallet, OrderID, Winston) -> next_deadline_tracked_test() -> Opts = test_opts(), {Seller, SellerAddr} = party(), - {Buyer, _} = party(), + {Buyer, BuyerAddr} = party(), Opened = - apply_tx(base(#{ SellerAddr => 100 }), offer(Seller, 10, 500, 5, 200), 100, Opts), - ?assertEqual(200, hb_ao:get(<<"next-deadline">>, Opened, 0, Opts)), - OrderID = order_id(only_order(Opened, Opts)), + apply_tx( + base(#{ SellerAddr => 100, BuyerAddr => 5 }), + offer(Seller, 10, 500, 5, 20), + 100, + Opts + ), + % An offer alone has nothing pending: it never expires. + ?assertEqual(0, hb_ao:get(<<"next-deadline">>, Opened, 0, Opts)), + OrderID = maps:get(<<"order-id">>, only_order(Opened, Opts)), Reserved = apply_tx(Opened, order_action(Buyer, <<"register-interest">>, OrderID), 110, Opts), - % The reservation lapses before the deadline, so it is the next event. + % A reservation is the only thing a clock waits for. ?assertEqual( - 110 + ?DEFAULT_RESERVATION_BLOCKS + 1, + 110 + 20 + 1, hb_ao:get(<<"next-deadline">>, Reserved, 0, Opts) ). diff --git a/src/preloaded/process/dev_name_token.erl b/src/preloaded/process/dev_name_token.erl index 40b987027b..d03263e462 100644 --- a/src/preloaded/process/dev_name_token.erl +++ b/src/preloaded/process/dev_name_token.erl @@ -1645,16 +1645,11 @@ fixture_sale() -> ?assertEqual(1, hb_util:int(Read([?BALANCES, ?FIXTURE_BUYER]))), ?assertEqual(0, hb_util:int(Read([?BALANCES, ?FIXTURE_SELLER]))), ?assertEqual(1, hb_util:int(Read(<<"total-supply">>))), - % The order it went through is settled, and the buyer is recorded as the - % one who paid. + % The offer is complete, so the book no longer holds it. ?assertEqual( - <<"settled">>, + not_found, Read([<<"orders">>, ?FIXTURE_ORDER, <<"status">>]) ), - ?assertEqual( - ?FIXTURE_BUYER, - Read([<<"orders">>, ?FIXTURE_ORDER, <<"buyer">>]) - ), % And the new owner has said what the name points at. ?assertEqual(<<"hello from the new owner">>, Read([?VALUE, <<"greeting">>])), ?assertEqual(<<"text/plain">>, Read([?VALUE, <<"content-type">>])). @@ -1878,8 +1873,8 @@ sale_story() -> ), % The registrant's payment settles it, and the name moves. Settled = Slot(?SALE_PAYMENT), - ?assertEqual( - {ok, <<"settled">>}, + ?assertMatch( + {error, _}, at(Process, Settled, <<"orders/", Order/binary, "/status">>, Opts) ), ?assertEqual({ok, 1}, at(Process, Settled, <<"balances/", Buyer/binary>>, Opts)), @@ -1943,16 +1938,16 @@ withdrawn_story() -> ), % Withdrawing it gives the unit back. Cancelled = Slot(?WITHDRAWN_CANCEL), - ?assertEqual( - {ok, <<"cancelled">>}, + ?assertMatch( + {error, _}, at(Process, Cancelled, <<"orders/", First/binary, "/status">>, Opts) ), ?assertEqual({ok, 1}, at(Process, Cancelled, <<"balances/", Seller/binary>>, Opts)), % A registration that pays the fee in full is still refused: the order is % no longer open. LateRegister = Slot(?WITHDRAWN_LATE_REGISTER), - ?assertEqual( - {ok, <<"cancelled">>}, + ?assertMatch( + {error, _}, at(Process, LateRegister, <<"orders/", First/binary, "/status">>, Opts) ), ?assertMatch( @@ -1962,8 +1957,8 @@ withdrawn_story() -> % And a payment against it moves nothing, in either direction: the goods % are back with the seller and there was no bond to compensate anyone from. LatePayment = Slot(?WITHDRAWN_LATE_PAYMENT), - ?assertEqual( - {ok, <<"cancelled">>}, + ?assertMatch( + {error, _}, at(Process, LatePayment, <<"orders/", First/binary, "/status">>, Opts) ), ?assertEqual( From fe7bc041457b8fe502c4d3f569269a25e445f4bd Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Sun, 26 Jul 2026 20:52:59 -0400 Subject: [PATCH 21/27] feat(copycat): add headers-only indexing mode --- src/preloaded/query/dev_copycat_arweave.erl | 382 +++++++++++++++++++- 1 file changed, 380 insertions(+), 2 deletions(-) diff --git a/src/preloaded/query/dev_copycat_arweave.erl b/src/preloaded/query/dev_copycat_arweave.erl index 5b5b6675d8..23dd9ae452 100644 --- a/src/preloaded/query/dev_copycat_arweave.erl +++ b/src/preloaded/query/dev_copycat_arweave.erl @@ -35,11 +35,12 @@ arweave(_Base, Request, Opts) -> end; {error, Mode} -> {error, <<"Unsupported mode `", (hb_util:bin(Mode))/binary, - "`. Supported modes are: shallow, deep, full, list">>} + "`. Supported modes are: headers, shallow, deep, full, list">>} end. request_mode(Request, Opts) -> case hb_maps:get(<<"mode">>, Request, <<"shallow">>, Opts) of + <<"headers">> -> {ok, headers}; <<"shallow">> -> {ok, shallow}; <<"deep">> -> {ok, deep}; <<"full">> -> {ok, full}; @@ -338,14 +339,30 @@ process_block(BlockRes, Current, To, IndexMode, Opts) -> block_indexed_path(Height) -> <<"block/", (hb_util:bin(Height))/binary, "/mode">>. +block_indexed_path(Height, headers) -> + <<"block/", (hb_util:bin(Height))/binary, "/headers">>; +block_indexed_path(Height, _IndexMode) -> + block_indexed_path(Height). + write_block_index(Height, IndexMode, Opts) -> #{ <<"index-store">> := Store } = hb_store_arweave:store_from_opts(Opts), hb_store:write( Store, - #{ block_indexed_path(Height) => mode_name(IndexMode) }, + #{ block_indexed_path(Height, IndexMode) => mode_name(IndexMode) }, Opts ). +is_block_indexed(Height, headers, Opts) -> + case hb_store_arweave:store_from_opts(Opts) of + no_store -> + false; + #{ <<"index-store">> := Store } -> + case hb_store:read( + Store, block_indexed_path(Height, headers), Opts) of + {ok, <<"headers">>} -> true; + _ -> false + end + end; is_block_indexed(Height, IndexMode, Opts) -> case hb_store_arweave:store_from_opts(Opts) of no_store -> @@ -359,6 +376,7 @@ is_block_indexed(Height, IndexMode, Opts) -> end end. +mode_name(headers) -> <<"headers">>; mode_name(shallow) -> <<"shallow">>; mode_name(deep) -> <<"deep">>; mode_name(full) -> <<"full">>. @@ -371,6 +389,24 @@ mode_rank(<<"deep">>) -> mode_rank(deep); mode_rank(<<"full">>) -> mode_rank(full); mode_rank(_Other) -> 0. +%% @doc Cache only the data-free L1 transaction headers from the block. +maybe_index_ids(Block, headers, Opts) -> + TXIDs = hb_maps:get(<<"txs">>, Block, [], Opts), + case resolve_block_tx_headers(Block, Opts) of + error -> + {block_skipped, #{ + skipped_count => length(TXIDs), + total_txs => length(TXIDs) + }}; + {ok, TXs} -> + Results = parallel_map( + TXs, + fun(TX) -> cache_data_free_header(TX, Opts) end, + Opts + ), + {block_cached, + (sum_counters(Results))#{ total_txs => length(TXIDs) }} + end; %% @doc Index the IDs of all transactions in the block if configured to do so. maybe_index_ids(Block, IndexMode, Opts) -> TXIDs = hb_maps:get(<<"txs">>, Block, [], Opts), @@ -590,6 +626,31 @@ skip_bundle(EncodedTXID, Reason) -> ), counters(0, 1, 1). +cache_data_free_header(#tx{ data_size = 0 } = TX, Opts) -> + case hb_opts:get(arweave_index_txs, true, Opts) of + false -> + counters(0, 0, 0); + true -> + try + true = ar_tx:verify_tx_id(TX#tx.id, TX), + {ok, _} = cache_item(TX#tx{ data = <<>> }, <<"tx@1.0">>, Opts), + counters(1, 0, 0) + catch + Class:Reason -> + ?event( + copycat_short, + {tx_header_cache_skipped, + {tx_id, {explicit, hb_util:encode(TX#tx.id)}}, + {class, Class}, + {reason, Reason} + } + ), + counters(0, 0, 1) + end + end; +cache_data_free_header(_TX, _Opts) -> + counters(0, 0, 0). + index_full_bundle_bytes(BundleData, BundleStartOffset, IndexMode, Store, Opts) -> case ar_bundles:decode_bundle_header(BundleData) of invalid_bundle_header -> @@ -623,6 +684,8 @@ index_pending(IndexMode, Opts) -> process_pending_tx(TXID, IndexMode, Opts) -> case resolve_pending_tx_header(TXID, Opts) of + {ok, TX} when IndexMode =:= headers -> + cache_data_free_header(TX, Opts); {ok, TX} -> Store = hb_store_arweave:store_from_opts(Opts), case hb_store_arweave:write_offset( @@ -797,12 +860,280 @@ download_bundle_header(EndOffset, Size, Opts) -> lib_arweave_common:bundle_header(EndOffset - Size, Size, Opts) end). +%% @doc Resolve every transaction header in a block, preferring the Arweave +%% node's single-request `/block2' path and falling back for uncached headers. +resolve_block_tx_headers(Block, Opts) -> + TXIDs = hb_maps:get(<<"txs">>, Block, [], Opts), + case fetch_block_tx_headers(Block, Opts) of + error -> + error; + {ok, Entries} -> + Results = parallel_map( + lists:zip(TXIDs, Entries), + fun({TXID, Entry}) -> + resolve_block_tx_header(TXID, Entry, Opts) + end, + Opts + ), + collect_tx_headers(Results) + end. + +resolve_block_tx_header(TXID, #tx{} = TX, Opts) -> + case validate_tx_header(TXID, TX) of + {ok, _} = Valid -> Valid; + error -> resolve_block_tx_header(TXID, hb_util:decode(TXID), Opts) + end; +resolve_block_tx_header(TXID, RawTXID, Opts) when is_binary(RawTXID) -> + case hb_util:decode(TXID) of + RawTXID -> + case resolve_tx_header(TXID, Opts) of + {ok, TX} -> validate_tx_header(TXID, TX); + error -> error + end; + _ -> + error + end. + +validate_tx_header(TXID, TX) -> + try + RawTXID = hb_util:decode(TXID), + case TX#tx.id =:= RawTXID + andalso ar_tx:generate_id(TX, signed) =:= RawTXID of + true -> {ok, TX#tx{ data = <<>> }}; + false -> error + end + catch + _:_ -> error + end. + +%% @doc Ask an Arweave node to inline every transaction it still has in its +%% block cache. Bare IDs in the response are resolved individually above. +fetch_block_tx_headers(Block, Opts) -> + Height = hb_maps:get(<<"height">>, Block, 0, Opts), + BlockID = hb_maps:get(<<"indep_hash">>, Block, <<>>, Opts), + Res = observe_event(<<"block_headers">>, fun() -> + hb_http:request( + #{ + <<"path">> => + <<"/arweave/block2/hash/", BlockID/binary>>, + <<"method">> => <<"GET">>, + % `/block2' accepts one selection bit for each possible TX. + <<"body">> => binary:copy(<<255>>, 125), + <<"route-by">> => Height + }, + Opts#{ + <<"cache-control">> => [<<"no-cache">>, <<"no-store">>], + <<"http-client">> => hackney + } + ) + end), + case lib_arweave_common:best_response(Res) of + {ok, #{ <<"body">> := Body }} -> + decode_block_tx_headers(Body, Block, Opts); + _ -> + error + end. + +decode_block_tx_headers(Body, Block, Opts) -> + try parse_block2_transactions(Body) of + {ok, BlockID, TXs} -> + ExpectedBlockID = hb_util:decode( + hb_maps:get(<<"indep_hash">>, Block, <<>>, Opts)), + ExpectedTXIDs = [ + hb_util:decode(TXID) + || + TXID <- hb_maps:get(<<"txs">>, Block, [], Opts) + ], + case BlockID =:= ExpectedBlockID + andalso [block2_tx_id(TX) || TX <- TXs] =:= ExpectedTXIDs of + true -> {ok, TXs}; + false -> error + end; + error -> + error + catch + _:_ -> error + end. + +block2_tx_id(#tx{ id = TXID }) -> TXID; +block2_tx_id(TXID) when is_binary(TXID) -> TXID. + +%% @doc Decode the transaction section of an Arweave `/block2' response. +parse_block2_transactions( + << + BlockID:48/binary, + PrevHashSize:8, _:PrevHashSize/binary, + TimestampSize:8, _:(TimestampSize * 8), + NonceSize:16, _:NonceSize/binary, + HeightSize:8, _:(HeightSize * 8), + DiffSize:16, _:(DiffSize * 8), + CumulativeDiffSize:16, _:(CumulativeDiffSize * 8), + LastRetargetSize:8, _:(LastRetargetSize * 8), + HashSize:8, _:HashSize/binary, + BlockSizeSize:16, _:(BlockSizeSize * 8), + WeaveSizeSize:16, _:(WeaveSizeSize * 8), + RewardAddrSize:8, _:RewardAddrSize/binary, + TXRootSize:8, _:TXRootSize/binary, + WalletListSize:8, _:WalletListSize/binary, + HashListMerkleSize:8, _:HashListMerkleSize/binary, + RewardPoolSize:8, _:(RewardPoolSize * 8), + PackingThresholdSize:8, _:(PackingThresholdSize * 8), + StrictChunkThresholdSize:8, _:(StrictChunkThresholdSize * 8), + RateDividendSize:8, _:(RateDividendSize * 8), + RateDivisorSize:8, _:(RateDivisorSize * 8), + ScheduledRateDividendSize:8, _:(ScheduledRateDividendSize * 8), + ScheduledRateDivisorSize:8, _:(ScheduledRateDivisorSize * 8), + PoAOptionSize:8, _:(PoAOptionSize * 8), + ChunkSize:24, _:ChunkSize/binary, + TXPathSize:24, _:TXPathSize/binary, + DataPathSize:24, _:DataPathSize/binary, + Rest/binary + >> +) when NonceSize =< 512 -> + parse_block2_tags(Rest, BlockID); +parse_block2_transactions(_Bin) -> + error. + +parse_block2_tags(<< Count:16, Rest/binary >>, BlockID) + when Count =< 2048 -> + parse_block2_tags(Count, Rest, BlockID, 0); +parse_block2_tags(_Bin, _BlockID) -> + error. + +parse_block2_tags(0, Rest, BlockID, _Size) -> + parse_block2_txs(Rest, BlockID); +parse_block2_tags( + Count, << Size:16, _:Size/binary, Rest/binary >>, + BlockID, TotalSize) when TotalSize + Size =< 2048 -> + parse_block2_tags(Count - 1, Rest, BlockID, TotalSize + Size); +parse_block2_tags(_Count, _Bin, _BlockID, _Size) -> + error. + +parse_block2_txs(<< Count:16, Rest/binary >>, BlockID) when Count =< 1000 -> + parse_block2_txs(Count, Rest, BlockID, []); +parse_block2_txs(_Bin, _BlockID) -> + error. + +parse_block2_txs(0, _Rest, BlockID, TXs) -> + {ok, BlockID, TXs}; +parse_block2_txs( + Count, << Size:24, TXBin:Size/binary, Rest/binary >>, + BlockID, TXs) when Count > 0 -> + case parse_block2_tx(TXBin) of + {ok, TX} -> + parse_block2_txs(Count - 1, Rest, BlockID, [TX | TXs]); + error -> + error + end; +parse_block2_txs(_Count, _Bin, _BlockID, _TXs) -> + error. + +parse_block2_tx(<< TXID:32/binary >>) -> + {ok, TXID}; +parse_block2_tx( + << + Format:8, TXID:32/binary, + AnchorSize:8, Anchor:AnchorSize/binary, + OwnerSize:16, Owner:OwnerSize/binary, + TargetSize:8, Target:TargetSize/binary, + QuantitySize:8, Quantity:(QuantitySize * 8), + DataSizeSize:16, DataSize:(DataSizeSize * 8), + DataRootSize:8, DataRoot:DataRootSize/binary, + SignatureSize:16, Signature:SignatureSize/binary, + RewardSize:8, Reward:(RewardSize * 8), + DataEncodingSize:24, Data:DataEncodingSize/binary, + Rest/binary + >> +) when Format =:= 1; Format =:= 2 -> + case parse_block2_tx_tags(Rest) of + {ok, Tags, DenominationBin} -> + case parse_block2_tx_denomination(DenominationBin) of + {ok, Denomination} -> + TX = #tx{ + format = Format, + id = TXID, + anchor = Anchor, + owner = Owner, + tags = Tags, + target = Target, + quantity = Quantity, + data = Data, + data_size = + case Format of + 1 -> byte_size(Data); + 2 -> DataSize + end, + data_root = DataRoot, + signature = Signature, + reward = Reward, + denomination = Denomination, + signature_type = + case Owner of + <<>> -> ?ECDSA_KEY_TYPE; + _ -> ?RSA_KEY_TYPE + end + }, + {ok, block2_tx_owner(TX)}; + error -> + error + end; + error -> + error + end; +parse_block2_tx(_Bin) -> + error. + +parse_block2_tx_tags(<< Count:16, Rest/binary >>) when Count =< 2048 -> + parse_block2_tx_tags(Count, Rest, []); +parse_block2_tx_tags(_Bin) -> + error. + +parse_block2_tx_tags(0, Rest, Tags) -> + {ok, Tags, Rest}; +parse_block2_tx_tags( + Count, + << + NameSize:16, ValueSize:16, + Name:NameSize/binary, Value:ValueSize/binary, + Rest/binary + >>, + Tags +) when Count > 0 -> + parse_block2_tx_tags(Count - 1, Rest, [{Name, Value} | Tags]); +parse_block2_tx_tags(_Count, _Bin, _Tags) -> + error. + +parse_block2_tx_denomination(<<>>) -> {ok, 0}; +parse_block2_tx_denomination(<< Denomination:24 >>) + when Denomination > 0 -> + {ok, Denomination}; +parse_block2_tx_denomination(_Bin) -> + error. + +block2_tx_owner(TX = #tx{ + data_size = 0, signature_type = ?ECDSA_KEY_TYPE }) -> + Owner = ar_wallet:recover_key( + ar_tx:generate_signature_data_segment(TX), + TX#tx.signature, + TX#tx.signature_type + ), + TX#tx{ + owner = Owner, + owner_address = ar_wallet:to_address(Owner, TX#tx.signature_type) + }; +block2_tx_owner(TX = #tx{ data_size = 0 }) -> + TX#tx{ owner_address = ar_tx:get_owner_address(TX) }; +block2_tx_owner(TX) -> TX. + resolve_tx_headers(TXIDs, Opts) -> Results = parallel_map( TXIDs, fun(TXID) -> resolve_tx_header(TXID, Opts) end, Opts ), + collect_tx_headers(Results). + +collect_tx_headers(Results) -> case lists:any(fun(Res) -> Res =:= error end, Results) of true -> error; false -> @@ -911,6 +1242,53 @@ observe_event(MetricName, Fun) -> %%% Tests +headers_mode_test() -> + Block = 1967269, + BlockBin = hb_util:bin(Block), + ZeroTXID = <<"enIzMI_6vVcY80ZqfiCdWq56chbft3HSZnXMj7X7BdE">>, + DataTXID = <<"NEsEjfjjawZ_fqMEnIS9aURTIvgrZ5TqFwC9hcsE2nM">>, + TestStore = hb_test_utils:test_store(), + StoreOpts = #{ <<"index-store">> => [TestStore] }, + Opts = #{ + <<"store">> => [TestStore], + <<"arweave-index-store">> => StoreOpts, + <<"arweave-index-workers">> => 2 + }, + LocalOpts = hb_store:scope(Opts, local), + ?assertEqual({error, not_found}, hb_cache:read(ZeroTXID, LocalOpts)), + {ok, Block} = + hb_ao:resolve( + << + "~copycat@1.0/arweave&" + "from=", BlockBin/binary, "&" + "to=", BlockBin/binary, "&" + "mode=headers" + >>, + Opts + ), + ?assertMatch( + {ok, _}, + hb_cache:read( + <<"~arweave@2.9/block/height/", BlockBin/binary>>, + LocalOpts + ) + ), + {ok, CachedTX} = hb_cache:read(ZeroTXID, LocalOpts), + ?assert(hb_message:verify(CachedTX, all, Opts)), + ?assertEqual(ZeroTXID, hb_message:id(CachedTX, signed, Opts)), + ?assertEqual({error, not_found}, hb_cache:read(DataTXID, LocalOpts)), + lists:foreach( + fun(TXID) -> + ?assertEqual( + not_found, + hb_store_arweave:read_offset(StoreOpts, TXID, Opts) + ) + end, + [ZeroTXID, DataTXID] + ), + ?assert(is_block_indexed(Block, headers, Opts)), + ?assertNot(is_block_indexed(Block, shallow, Opts)). + index_ids_test_parallel() -> %% Test block: https://viewblock.io/arweave/block/1827942 %% Note: this block includes a data item with an Ethereum signature. This From 3fdab35c6d2c25b9cf95390c3a02911c120160ca Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Mon, 27 Jul 2026 11:50:59 -0400 Subject: [PATCH 22/27] refactor(copycat): simplify headers-only indexing --- .../process/dev_arweave_scheduler.erl | 197 +++++------ src/preloaded/query/dev_copycat_arweave.erl | 322 +++++------------- 2 files changed, 191 insertions(+), 328 deletions(-) diff --git a/src/preloaded/process/dev_arweave_scheduler.erl b/src/preloaded/process/dev_arweave_scheduler.erl index 93c5d0efb6..66d5dfdca1 100644 --- a/src/preloaded/process/dev_arweave_scheduler.erl +++ b/src/preloaded/process/dev_arweave_scheduler.erl @@ -32,11 +32,10 @@ %%%
      %%%
    • `target' (the default): the process's messages are the transactions %%% addressed to it, as above.
    • -%%%
    • `all': every base-layer transaction is a message in the +%%%
    • `all': every data-free base-layer transaction is a message in the %%% process's schedule, whoever it is addressed to. This is what lets a %%% process observe value moving between two other addresses -- a -%%% payment it is owed but is not a party to -- at the cost of a slot -%%% for every transaction on the network. Its schedule is enumerated +%%% payment it is owed but is not a party to. Its schedule is enumerated %%% from the block headers rather than by query, in canonical chain %%% order (blocks ascending, then each block's own transaction order), %%% and each assignment records the `block-height' that sequenced it, so @@ -424,9 +423,8 @@ no_result_cache(Opts) -> %% @doc Discover the base-layer transactions that a process is sequenced by %% within a block range, in canonical weave order, entirely from the node's own -%% index. Indexing the range with `~copycat@1.0/arweave' both records each -%% transaction's weave offset and caches its header (so its `target' is locally -%% matchable), whichever mode the process is in. +%% index. `target' mode records each transaction's weave offset and caches its +%% header. `all' mode caches only data-free headers. %% %% In `target' mode the recipient match is served from the node's own %% `~query@1.0' GraphQL endpoint, and each match is annotated with its offset -- @@ -440,7 +438,7 @@ no_result_cache(Opts) -> %% items are excluded, as this scheduler sequences the base layer only. discover(ProcID, <<"all">>, From, To, Opts) -> maybe - ok ?= ensure_offsets(From, To, Opts), + ok ?= ensure_headers(From, To, Opts), {ok, Located} ?= enumerate_blocks(ProcID, From, To, Opts), base_layer_blocks(Located, Opts) end; @@ -486,6 +484,56 @@ enumerate_blocks(ProcID, From, To, Opts) -> } end. +%% @doc Ensure every data-free transaction header in a block range is cached. +ensure_headers(From, To, _Opts) when From > To -> ok; +ensure_headers(From, To, Opts) -> + maybe + {ok, _} ?= + hb_ao:resolve( + << + "~copycat@1.0/arweave&mode=headers&reindex=false", + "&from=", (hb_util:bin(To))/binary, + "&to=", (hb_util:bin(From))/binary + >>, + no_result_cache(Opts) + ), + ok ?= headers_indexed(From, To, Opts) + end. + +headers_indexed(From, To, Opts) -> + #{ <<"index-store">> := Store } = hb_store_arweave:store_from_opts(Opts), + Missing = + lists:dropwhile( + fun(Height) -> + case hb_store:read( + Store, + <<"block/", (hb_util:bin(Height))/binary, "/mode">>, + Opts + ) of + {ok, Mode} -> + lists:member( + Mode, + [<<"headers">>, <<"shallow">>, <<"deep">>, <<"full">>] + ); + _ -> + false + end + end, + lists:seq(From, To) + ), + case Missing of + [] -> ok; + [Height | _] -> + {error, + #{ + <<"status">> => 503, + <<"reason">> => + <<"Block range is not fully indexed locally.">>, + <<"block-height">> => Height + } + } + end. + %% @doc Ensure the node's local Arweave index covers the block range, so that %% every transaction in it has both a weave offset and a locally-cached header. %% `~copycat@1.0/arweave' in `shallow' mode records an ID->offset entry for each @@ -642,59 +690,30 @@ base_layer_offset(Store, ID, Opts) -> _ -> false end. -%% @doc Annotate each block-enumerated transaction with its weave offset, -%% keeping only the base-layer ones. The enumeration is already in canonical -%% order, so -- unlike `base_layer_offsets/2' -- it is not re-sorted. Each -%% assignment records the height of the block that sequenced it as well as its -%% offset: in this mode the schedule follows the chain rather than the process, -%% so the block height is the only clock a process has. -%% -%% A transaction the block lists but the local index does not hold is a failure -%% to index, not an absence -- `~copycat@1.0/arweave' skips a whole block if any -%% one of its transaction headers could not be fetched, while still reporting -%% success. Dropping those would silently shorten the schedule and, because -%% slots are positional, shift every later slot on this node alone. So the range -%% fails here instead, leaving `synced-to' where it was for the next pass to -%% retry. +%% @doc Keep the data-free headers cached by the headers-mode pass. base_layer_blocks(Located, Opts) -> - Store = hb_store_arweave:store_from_opts(Opts), - lists:foldr( - fun(_, {error, Err}) -> {error, Err}; - ({Height, ID}, {ok, Acc}) -> - case offset_entry(Store, ID, Opts) of - {ok, <<"tx@1.0">>, Offset} -> - {ok, - [ - { - #{ - <<"offset">> => Offset, - <<"block-height">> => Height - }, - ID - } - | - Acc - ] - }; - {ok, _Codec, _Offset} -> - % A bundled data item: this scheduler sequences the base - % layer only. - {ok, Acc}; - not_found -> - {error, - #{ - <<"status">> => 503, - <<"reason">> => - <<"Block range is not fully indexed locally.">>, - <<"tx">> => ID, - <<"block-height">> => Height - } - } - end - end, - {ok, []}, - Located - ). + {ok, + [ + {#{ <<"block-height">> => Height }, ID} + || + {Height, ID} <- Located, + data_free(ID, Opts) + ] + }. + +data_free(ID, Opts) -> + LocalOpts = hb_store:scope(Opts, local), + try + case hb_cache:read(ID, LocalOpts) of + {ok, Header} -> + (hb_message:convert( + Header, <<"tx@1.0">>, LocalOpts))#tx.data_size =:= 0; + _ -> + false + end + catch + _:_ -> false + end. %% @doc Read a transaction's local index entry, returning its codec device and %% weave offset. @@ -1121,46 +1140,34 @@ enumerate_blocks_test() -> enumerate_blocks(ProcID, 10, 11, Opts) ). -%% @doc Block-enumerated transactions keep chain order rather than being sorted -%% by offset -- offsets tie for every transaction that carries no data, which -%% is what an AO message is -- and each records the height that sequenced it. -%% Bundled data items are still dropped. +%% @doc Block-enumerated transactions keep chain order and only locally cached, +%% data-free transaction headers. base_layer_blocks_test() -> Store = hb_test_utils:test_store(), hb_store:start(Store), - ArwStore = #{ <<"index-store">> => [Store] }, - Opts = #{ <<"arweave-index-store">> => ArwStore }, - Early = hb_util:human_id(crypto:strong_rand_bytes(32)), - Late = hb_util:human_id(crypto:strong_rand_bytes(32)), - Bundled = hb_util:human_id(crypto:strong_rand_bytes(32)), - ok = hb_store_arweave:write_offset(ArwStore, Late, <<"tx@1.0">>, 100, 0), - ok = hb_store_arweave:write_offset(ArwStore, Bundled, <<"ans104@1.0">>, 150, 0), - ok = hb_store_arweave:write_offset(ArwStore, Early, <<"tx@1.0">>, 200, 0), + Opts = #{ <<"store">> => [Store], <<"priv-wallet">> => ar_wallet:new() }, + DataFree = + hb_message:commit( + #{ <<"target">> => hb_util:human_id(crypto:strong_rand_bytes(32)) }, + Opts, + #{ <<"commitment-device">> => <<"tx@1.0">> } + ), + DataBearing = + hb_message:commit( + #{ <<"data">> => <<"not-a-message">> }, + Opts, + #{ <<"commitment-device">> => <<"tx@1.0">> } + ), + DataFreeID = hb_message:id(DataFree, signed, Opts), + DataBearingID = hb_message:id(DataBearing, signed, Opts), + {ok, _} = hb_cache:write(DataFree, Opts), + {ok, _} = hb_cache:write(DataBearing, Opts), ?assertEqual( - {ok, - [ - {#{ <<"offset">> => 100, <<"block-height">> => 10 }, Late}, - {#{ <<"offset">> => 200, <<"block-height">> => 11 }, Early} - ] - }, - base_layer_blocks([{10, Late}, {11, Bundled}, {11, Early}], Opts) - ). - -%% @doc A transaction the block lists but the index does not hold means the -%% range was not fully indexed. Dropping it would shorten the schedule and, as -%% slots are positional, shift every later slot on this node alone -- so the -%% range fails and `synced-to' stays where it was. -base_layer_blocks_unindexed_test() -> - Store = hb_test_utils:test_store(), - hb_store:start(Store), - ArwStore = #{ <<"index-store">> => [Store] }, - Opts = #{ <<"arweave-index-store">> => ArwStore }, - Indexed = hb_util:human_id(crypto:strong_rand_bytes(32)), - Missing = hb_util:human_id(crypto:strong_rand_bytes(32)), - ok = hb_store_arweave:write_offset(ArwStore, Indexed, <<"tx@1.0">>, 100, 0), - ?assertMatch( - {error, #{ <<"status">> := 503 }}, - base_layer_blocks([{10, Indexed}, {10, Missing}], Opts) + {ok, [{#{ <<"block-height">> => 10 }, DataFreeID}]}, + base_layer_blocks( + [{10, DataFreeID}, {10, DataBearingID}, {10, <<"missing">>}], + Opts + ) ). %% @doc An `all'-mode assignment records the height that sequenced it as well diff --git a/src/preloaded/query/dev_copycat_arweave.erl b/src/preloaded/query/dev_copycat_arweave.erl index 23dd9ae452..66532b5df0 100644 --- a/src/preloaded/query/dev_copycat_arweave.erl +++ b/src/preloaded/query/dev_copycat_arweave.erl @@ -339,30 +339,25 @@ process_block(BlockRes, Current, To, IndexMode, Opts) -> block_indexed_path(Height) -> <<"block/", (hb_util:bin(Height))/binary, "/mode">>. -block_indexed_path(Height, headers) -> - <<"block/", (hb_util:bin(Height))/binary, "/headers">>; -block_indexed_path(Height, _IndexMode) -> - block_indexed_path(Height). - write_block_index(Height, IndexMode, Opts) -> #{ <<"index-store">> := Store } = hb_store_arweave:store_from_opts(Opts), + Path = block_indexed_path(Height), + Mode = + case hb_store:read(Store, Path, Opts) of + {ok, Existing} -> + case mode_rank(Existing) >= mode_rank(IndexMode) of + true -> Existing; + false -> mode_name(IndexMode) + end; + _ -> + mode_name(IndexMode) + end, hb_store:write( Store, - #{ block_indexed_path(Height, IndexMode) => mode_name(IndexMode) }, + #{ Path => Mode }, Opts ). -is_block_indexed(Height, headers, Opts) -> - case hb_store_arweave:store_from_opts(Opts) of - no_store -> - false; - #{ <<"index-store">> := Store } -> - case hb_store:read( - Store, block_indexed_path(Height, headers), Opts) of - {ok, <<"headers">>} -> true; - _ -> false - end - end; is_block_indexed(Height, IndexMode, Opts) -> case hb_store_arweave:store_from_opts(Opts) of no_store -> @@ -381,31 +376,38 @@ mode_name(shallow) -> <<"shallow">>; mode_name(deep) -> <<"deep">>; mode_name(full) -> <<"full">>. +mode_rank(headers) -> 0; mode_rank(shallow) -> 1; mode_rank(deep) -> 2; mode_rank(full) -> 3; +mode_rank(<<"headers">>) -> mode_rank(headers); mode_rank(<<"shallow">>) -> mode_rank(shallow); mode_rank(<<"deep">>) -> mode_rank(deep); mode_rank(<<"full">>) -> mode_rank(full); -mode_rank(_Other) -> 0. +mode_rank(_Other) -> -1. %% @doc Cache only the data-free L1 transaction headers from the block. maybe_index_ids(Block, headers, Opts) -> TXIDs = hb_maps:get(<<"txs">>, Block, [], Opts), - case resolve_block_tx_headers(Block, Opts) of + case fetch_block_tx_layout(Block, Opts) of error -> {block_skipped, #{ skipped_count => length(TXIDs), total_txs => length(TXIDs) }}; - {ok, TXs} -> + {ok, Layout} when length(Layout) =:= length(TXIDs) -> Results = parallel_map( - TXs, - fun(TX) -> cache_data_free_header(TX, Opts) end, + Layout, + fun(Entry) -> cache_data_free_header(Entry, Opts) end, Opts ), {block_cached, - (sum_counters(Results))#{ total_txs => length(TXIDs) }} + (sum_counters(Results))#{ total_txs => length(TXIDs) }}; + {ok, _} -> + {block_skipped, #{ + skipped_count => length(TXIDs), + total_txs => length(TXIDs) + }} end; %% @doc Index the IDs of all transactions in the block if configured to do so. maybe_index_ids(Block, IndexMode, Opts) -> @@ -626,30 +628,24 @@ skip_bundle(EncodedTXID, Reason) -> ), counters(0, 1, 1). -cache_data_free_header(#tx{ data_size = 0 } = TX, Opts) -> - case hb_opts:get(arweave_index_txs, true, Opts) of - false -> - counters(0, 0, 0); - true -> +cache_data_free_header({_TXID, DataSize}, _Opts) + when is_integer(DataSize), DataSize > 0 -> + counters(0, 0, 0); +cache_data_free_header({TXID, DataSize}, Opts) -> + EncodedTXID = hb_util:encode(TXID), + case resolve_tx_header(EncodedTXID, Opts) of + {ok, #tx{ data_size = 0 } = TX} -> try - true = ar_tx:verify_tx_id(TX#tx.id, TX), - {ok, _} = cache_item(TX#tx{ data = <<>> }, <<"tx@1.0">>, Opts), + {ok, _} = cache_item(TX, <<"tx@1.0">>, Opts), counters(1, 0, 0) catch - Class:Reason -> - ?event( - copycat_short, - {tx_header_cache_skipped, - {tx_id, {explicit, hb_util:encode(TX#tx.id)}}, - {class, Class}, - {reason, Reason} - } - ), - counters(0, 0, 1) - end - end; -cache_data_free_header(_TX, _Opts) -> - counters(0, 0, 0). + _:_ -> counters(0, 0, 1) + end; + {ok, #tx{}} when DataSize =:= unknown -> + counters(0, 0, 0); + _ -> + counters(0, 0, 1) + end. index_full_bundle_bytes(BundleData, BundleStartOffset, IndexMode, Store, Opts) -> case ar_bundles:decode_bundle_header(BundleData) of @@ -682,10 +678,10 @@ index_pending(IndexMode, Opts) -> Error end. +process_pending_tx(_TXID, headers, _Opts) -> + counters(0, 0, 0); process_pending_tx(TXID, IndexMode, Opts) -> case resolve_pending_tx_header(TXID, Opts) of - {ok, TX} when IndexMode =:= headers -> - cache_data_free_header(TX, Opts); {ok, TX} -> Store = hb_store_arweave:store_from_opts(Opts), case hb_store_arweave:write_offset( @@ -860,55 +856,8 @@ download_bundle_header(EndOffset, Size, Opts) -> lib_arweave_common:bundle_header(EndOffset - Size, Size, Opts) end). -%% @doc Resolve every transaction header in a block, preferring the Arweave -%% node's single-request `/block2' path and falling back for uncached headers. -resolve_block_tx_headers(Block, Opts) -> - TXIDs = hb_maps:get(<<"txs">>, Block, [], Opts), - case fetch_block_tx_headers(Block, Opts) of - error -> - error; - {ok, Entries} -> - Results = parallel_map( - lists:zip(TXIDs, Entries), - fun({TXID, Entry}) -> - resolve_block_tx_header(TXID, Entry, Opts) - end, - Opts - ), - collect_tx_headers(Results) - end. - -resolve_block_tx_header(TXID, #tx{} = TX, Opts) -> - case validate_tx_header(TXID, TX) of - {ok, _} = Valid -> Valid; - error -> resolve_block_tx_header(TXID, hb_util:decode(TXID), Opts) - end; -resolve_block_tx_header(TXID, RawTXID, Opts) when is_binary(RawTXID) -> - case hb_util:decode(TXID) of - RawTXID -> - case resolve_tx_header(TXID, Opts) of - {ok, TX} -> validate_tx_header(TXID, TX); - error -> error - end; - _ -> - error - end. - -validate_tx_header(TXID, TX) -> - try - RawTXID = hb_util:decode(TXID), - case TX#tx.id =:= RawTXID - andalso ar_tx:generate_id(TX, signed) =:= RawTXID of - true -> {ok, TX#tx{ data = <<>> }}; - false -> error - end - catch - _:_ -> error - end. - -%% @doc Ask an Arweave node to inline every transaction it still has in its -%% block cache. Bare IDs in the response are resolved individually above. -fetch_block_tx_headers(Block, Opts) -> +%% @doc Read transaction IDs and data sizes from one `/block2' response. +fetch_block_tx_layout(Block, Opts) -> Height = hb_maps:get(<<"height">>, Block, 0, Opts), BlockID = hb_maps:get(<<"indep_hash">>, Block, <<>>, Opts), Res = observe_event(<<"block_headers">>, fun() -> @@ -918,7 +867,12 @@ fetch_block_tx_headers(Block, Opts) -> <<"/arweave/block2/hash/", BlockID/binary>>, <<"method">> => <<"GET">>, % `/block2' accepts one selection bit for each possible TX. - <<"body">> => binary:copy(<<255>>, 125), + <<"body">> => + binary:copy( + <<255>>, + (length(hb_maps:get(<<"txs">>, Block, [], Opts)) + 7) + div 8 + ), <<"route-by">> => Height }, Opts#{ @@ -929,39 +883,15 @@ fetch_block_tx_headers(Block, Opts) -> end), case lib_arweave_common:best_response(Res) of {ok, #{ <<"body">> := Body }} -> - decode_block_tx_headers(Body, Block, Opts); + parse_block2_transactions(Body); _ -> error end. -decode_block_tx_headers(Body, Block, Opts) -> - try parse_block2_transactions(Body) of - {ok, BlockID, TXs} -> - ExpectedBlockID = hb_util:decode( - hb_maps:get(<<"indep_hash">>, Block, <<>>, Opts)), - ExpectedTXIDs = [ - hb_util:decode(TXID) - || - TXID <- hb_maps:get(<<"txs">>, Block, [], Opts) - ], - case BlockID =:= ExpectedBlockID - andalso [block2_tx_id(TX) || TX <- TXs] =:= ExpectedTXIDs of - true -> {ok, TXs}; - false -> error - end; - error -> - error - catch - _:_ -> error - end. - -block2_tx_id(#tx{ id = TXID }) -> TXID; -block2_tx_id(TXID) when is_binary(TXID) -> TXID. - %% @doc Decode the transaction section of an Arweave `/block2' response. parse_block2_transactions( << - BlockID:48/binary, + _:48/binary, PrevHashSize:8, _:PrevHashSize/binary, TimestampSize:8, _:(TimestampSize * 8), NonceSize:16, _:NonceSize/binary, @@ -989,142 +919,70 @@ parse_block2_transactions( DataPathSize:24, _:DataPathSize/binary, Rest/binary >> -) when NonceSize =< 512 -> - parse_block2_tags(Rest, BlockID); +) -> + parse_block2_tags(Rest); parse_block2_transactions(_Bin) -> error. -parse_block2_tags(<< Count:16, Rest/binary >>, BlockID) - when Count =< 2048 -> - parse_block2_tags(Count, Rest, BlockID, 0); -parse_block2_tags(_Bin, _BlockID) -> +parse_block2_tags(<< Count:16, Rest/binary >>) when Count =< 2048 -> + parse_block2_tags(Count, Rest); +parse_block2_tags(_Bin) -> error. -parse_block2_tags(0, Rest, BlockID, _Size) -> - parse_block2_txs(Rest, BlockID); -parse_block2_tags( - Count, << Size:16, _:Size/binary, Rest/binary >>, - BlockID, TotalSize) when TotalSize + Size =< 2048 -> - parse_block2_tags(Count - 1, Rest, BlockID, TotalSize + Size); -parse_block2_tags(_Count, _Bin, _BlockID, _Size) -> +parse_block2_tags(0, Rest) -> + parse_block2_txs(Rest); +parse_block2_tags(Count, << Size:16, _:Size/binary, Rest/binary >>) -> + parse_block2_tags(Count - 1, Rest); +parse_block2_tags(_Count, _Bin) -> error. -parse_block2_txs(<< Count:16, Rest/binary >>, BlockID) when Count =< 1000 -> - parse_block2_txs(Count, Rest, BlockID, []); -parse_block2_txs(_Bin, _BlockID) -> +parse_block2_txs(<< Count:16, Rest/binary >>) when Count =< 1000 -> + parse_block2_txs(Count, Rest, []); +parse_block2_txs(_Bin) -> error. -parse_block2_txs(0, _Rest, BlockID, TXs) -> - {ok, BlockID, TXs}; +parse_block2_txs(0, _Rest, TXs) -> + {ok, TXs}; parse_block2_txs( Count, << Size:24, TXBin:Size/binary, Rest/binary >>, - BlockID, TXs) when Count > 0 -> + TXs) when Count > 0 -> case parse_block2_tx(TXBin) of {ok, TX} -> - parse_block2_txs(Count - 1, Rest, BlockID, [TX | TXs]); + parse_block2_txs(Count - 1, Rest, [TX | TXs]); error -> error end; -parse_block2_txs(_Count, _Bin, _BlockID, _TXs) -> +parse_block2_txs(_Count, _Bin, _TXs) -> error. parse_block2_tx(<< TXID:32/binary >>) -> - {ok, TXID}; + {ok, {TXID, unknown}}; parse_block2_tx( << Format:8, TXID:32/binary, - AnchorSize:8, Anchor:AnchorSize/binary, - OwnerSize:16, Owner:OwnerSize/binary, - TargetSize:8, Target:TargetSize/binary, - QuantitySize:8, Quantity:(QuantitySize * 8), + AnchorSize:8, _:AnchorSize/binary, + OwnerSize:16, _:OwnerSize/binary, + TargetSize:8, _:TargetSize/binary, + QuantitySize:8, _:(QuantitySize * 8), DataSizeSize:16, DataSize:(DataSizeSize * 8), - DataRootSize:8, DataRoot:DataRootSize/binary, - SignatureSize:16, Signature:SignatureSize/binary, - RewardSize:8, Reward:(RewardSize * 8), - DataEncodingSize:24, Data:DataEncodingSize/binary, - Rest/binary + DataRootSize:8, _:DataRootSize/binary, + SignatureSize:16, _:SignatureSize/binary, + RewardSize:8, _:(RewardSize * 8), + DataEncodingSize:24, _:DataEncodingSize/binary, + _/binary >> ) when Format =:= 1; Format =:= 2 -> - case parse_block2_tx_tags(Rest) of - {ok, Tags, DenominationBin} -> - case parse_block2_tx_denomination(DenominationBin) of - {ok, Denomination} -> - TX = #tx{ - format = Format, - id = TXID, - anchor = Anchor, - owner = Owner, - tags = Tags, - target = Target, - quantity = Quantity, - data = Data, - data_size = - case Format of - 1 -> byte_size(Data); - 2 -> DataSize - end, - data_root = DataRoot, - signature = Signature, - reward = Reward, - denomination = Denomination, - signature_type = - case Owner of - <<>> -> ?ECDSA_KEY_TYPE; - _ -> ?RSA_KEY_TYPE - end - }, - {ok, block2_tx_owner(TX)}; - error -> - error - end; - error -> - error - end; + {ok, + { + TXID, + case Format of + 1 -> DataEncodingSize; + 2 -> DataSize + end + }}; parse_block2_tx(_Bin) -> error. -parse_block2_tx_tags(<< Count:16, Rest/binary >>) when Count =< 2048 -> - parse_block2_tx_tags(Count, Rest, []); -parse_block2_tx_tags(_Bin) -> - error. - -parse_block2_tx_tags(0, Rest, Tags) -> - {ok, Tags, Rest}; -parse_block2_tx_tags( - Count, - << - NameSize:16, ValueSize:16, - Name:NameSize/binary, Value:ValueSize/binary, - Rest/binary - >>, - Tags -) when Count > 0 -> - parse_block2_tx_tags(Count - 1, Rest, [{Name, Value} | Tags]); -parse_block2_tx_tags(_Count, _Bin, _Tags) -> - error. - -parse_block2_tx_denomination(<<>>) -> {ok, 0}; -parse_block2_tx_denomination(<< Denomination:24 >>) - when Denomination > 0 -> - {ok, Denomination}; -parse_block2_tx_denomination(_Bin) -> - error. - -block2_tx_owner(TX = #tx{ - data_size = 0, signature_type = ?ECDSA_KEY_TYPE }) -> - Owner = ar_wallet:recover_key( - ar_tx:generate_signature_data_segment(TX), - TX#tx.signature, - TX#tx.signature_type - ), - TX#tx{ - owner = Owner, - owner_address = ar_wallet:to_address(Owner, TX#tx.signature_type) - }; -block2_tx_owner(TX = #tx{ data_size = 0 }) -> - TX#tx{ owner_address = ar_tx:get_owner_address(TX) }; -block2_tx_owner(TX) -> TX. - resolve_tx_headers(TXIDs, Opts) -> Results = parallel_map( TXIDs, @@ -1273,9 +1131,7 @@ headers_mode_test() -> LocalOpts ) ), - {ok, CachedTX} = hb_cache:read(ZeroTXID, LocalOpts), - ?assert(hb_message:verify(CachedTX, all, Opts)), - ?assertEqual(ZeroTXID, hb_message:id(CachedTX, signed, Opts)), + ?assertMatch({ok, _}, hb_cache:read(ZeroTXID, LocalOpts)), ?assertEqual({error, not_found}, hb_cache:read(DataTXID, LocalOpts)), lists:foreach( fun(TXID) -> From d4ee9ad8529b691bb8b13c64f279801139bfdb04 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Mon, 27 Jul 2026 12:19:49 -0400 Subject: [PATCH 23/27] refactor(name-token): minimize sale stack --- .../process/dev_arweave_scheduler.erl | 146 +-- .../process/dev_arweave_scheduler_cache.erl | 2 +- src/preloaded/process/dev_arweave_swap.erl | 95 +- src/preloaded/process/dev_name_token.erl | 1149 +++-------------- 4 files changed, 259 insertions(+), 1133 deletions(-) diff --git a/src/preloaded/process/dev_arweave_scheduler.erl b/src/preloaded/process/dev_arweave_scheduler.erl index 66d5dfdca1..23a4b81e5d 100644 --- a/src/preloaded/process/dev_arweave_scheduler.erl +++ b/src/preloaded/process/dev_arweave_scheduler.erl @@ -21,10 +21,8 @@ %%% `~query@1.0' GraphQL endpoint (`transactions(recipients: %%% [ProcessID], sort: HEIGHT_ASC)') for the base-layer transactions %%% addressed to the process, ordered by the local weave-offset index. -%%% A node may instead be configured to query a remote gateway -%%% (`arweave_scheduler_query_source => remote'). There is no bespoke -%%% store index -- discovery uses the node's existing Arweave index and -%%% the device's own schedule cache.
    • +%%% There is no bespoke store index -- discovery uses the node's existing +%%% Arweave index and the device's own schedule cache. %%%
    %%% %%% A process message may widen what it is sequenced by, with a @@ -208,13 +206,8 @@ get_schedule(Base, Req, Opts) -> post_schedule(Base, Req, Opts) -> maybe {ok, ToSched} ?= lib_scheduler:load_message_to_schedule(Base, Req, Opts), - do_post_schedule(Base, Req, ToSched, Opts) - end. - -do_post_schedule(Base, Req, ToSched, Opts) -> - ProcID = lib_scheduler:find_target_id(Base, Req, ToSched, Opts), - ?event({arweave_post_schedule, {proc_id, ProcID}}), - maybe + ProcID = lib_scheduler:find_target_id(Base, Req, ToSched, Opts), + ?event({arweave_post_schedule, {proc_id, ProcID}}), {ok, OnlyCommitted} ?= lib_scheduler:only_committed(ToSched, Opts), ok ?= ensure_tx_committed(OnlyCommitted, Opts), dispatch(ProcID, OnlyCommitted, Opts) @@ -488,15 +481,7 @@ enumerate_blocks(ProcID, From, To, Opts) -> ensure_headers(From, To, _Opts) when From > To -> ok; ensure_headers(From, To, Opts) -> maybe - {ok, _} ?= - hb_ao:resolve( - << - "~copycat@1.0/arweave&mode=headers&reindex=false", - "&from=", (hb_util:bin(To))/binary, - "&to=", (hb_util:bin(From))/binary - >>, - no_result_cache(Opts) - ), + ok ?= index_range(<<"headers">>, From, To, Opts), ok ?= headers_indexed(From, To, Opts) end. @@ -542,13 +527,15 @@ headers_indexed(From, To, Opts) -> %% node has already indexed (for any process), so the run is idempotent, %% incremental, and shared: overlapping ranges across processes are not %% re-fetched. -ensure_offsets(From, To, _Opts) when From > To -> ok; -ensure_offsets(From, To, Opts) -> +ensure_offsets(From, To, Opts) -> index_range(<<"shallow">>, From, To, Opts). + +index_range(_Mode, From, To, _Opts) when From > To -> ok; +index_range(Mode, From, To, Opts) -> maybe {ok, _} ?= hb_ao:resolve( << - "~copycat@1.0/arweave&mode=shallow&reindex=false", + "~copycat@1.0/arweave&mode=", Mode/binary, "&reindex=false", "&from=", (hb_util:bin(To))/binary, "&to=", (hb_util:bin(From))/binary >>, @@ -558,13 +545,11 @@ ensure_offsets(From, To, Opts) -> end. %% @doc Query for the transactions addressed to a process within a block -%% range, ordered by weave position, returning their IDs. By default the query -%% is served by the node's own `~query@1.0' index (`local'), whose `field-target' +%% range, ordered by weave position, returning their IDs. The query is served +%% by the node's own `~query@1.0' index, whose `field-target' %% matches are populated by `~copycat@1.0/arweave' as it indexes the range (see -%% `ensure_offsets'); a node may instead be configured -%% (`arweave_scheduler_query_source => remote') to query a remote gateway. +%% `ensure_offsets'). query_recipients(ProcID, From, To, Opts) -> - Source = hb_opts:get(arweave_scheduler_query_source, local, Opts), Query = << "query($after: String) { transactions(", @@ -576,16 +561,16 @@ query_recipients(ProcID, From, To, Opts) -> ", after: $after", ") { pageInfo { hasNextPage } edges { cursor node { id } } } }" >>, - query_pages(Source, Query, undefined, [], Opts). + query_pages(Query, undefined, [], Opts). -query_pages(Source, Query, After, Acc, Opts) -> +query_pages(Query, After, Acc, Opts) -> Variables = case After of undefined -> #{}; _ -> #{ <<"after">> => After } end, maybe - {ok, Transactions} ?= run_query(Source, Query, Variables, Opts), + {ok, Transactions} ?= run_query(Query, Variables, Opts), Edges = hb_maps:get(<<"edges">>, Transactions, [], Opts), IDs = Acc ++ [ edge_id(E, Opts) || E <- Edges ], HasNext = @@ -601,7 +586,7 @@ query_pages(Source, Query, After, Acc, Opts) -> {true, [_ | _]} -> Cursor = hb_maps:get(<<"cursor">>, lists:last(Edges), undefined, Opts), - query_pages(Source, Query, Cursor, IDs, Opts); + query_pages(Query, Cursor, IDs, Opts); _ -> {ok, [ ID || ID <- IDs, is_binary(ID) ]} end @@ -610,14 +595,8 @@ query_pages(Source, Query, After, Acc, Opts) -> edge_id(Edge, Opts) -> hb_maps:get(<<"id">>, hb_maps:get(<<"node">>, Edge, #{}, Opts), undefined, Opts). -%% @doc Run a GraphQL `transactions' query, returning its connection. `local' -%% resolves the node's own `~query@1.0/graphql' endpoint; `remote' posts to -%% the configured `gateway' directly. The remote path deliberately does not use -%% `hb_client_gateway:query': its admissibility gate rejects legitimately-empty -%% ranges, and the racing `/graphql' route lets AO-search gateways that do not -%% index arbitrary L1 transactions win with empty-but-200 responses. Here an -%% empty range is an authoritative answer. -run_query(local, Query, Variables, Opts) -> +%% @doc Run a GraphQL `transactions' query against the node's local index. +run_query(Query, Variables, Opts) -> to_transactions( hb_ao:resolve( #{ <<"device">> => <<"query@1.0">> }, @@ -630,23 +609,6 @@ run_query(local, Query, Variables, Opts) -> no_result_cache(Opts) ), Opts - ); -run_query(remote, Query, Variables, Opts) -> - Gateway = hb_opts:get(gateway, <<"https://arweave.net">>, Opts), - to_transactions( - hb_http:post( - Gateway, - #{ - <<"path">> => <<"/graphql">>, - <<"content-type">> => <<"application/json">>, - <<"body">> => - hb_json:encode( - #{ <<"query">> => Query, <<"variables">> => Variables } - ) - }, - no_result_cache(Opts) - ), - Opts ). to_transactions({ok, Response}, Opts) -> @@ -757,11 +719,10 @@ assign(ProcID, State, Slot, [{Extra, TXID} | Rest], To, Opts) -> end. %% @doc Generate and store the synthetic assignment for a message. Mirrors the -%% assignments minted by `~scheduler@1.0', but the on-chain position is the -%% transaction's weave `offset' rather than a scheduler-assigned nonce (joined, -%% in `all' mode, by the `block-height' that sequenced it). Every field derives -%% from chain data, so the assignment is deterministic across nodes and is left -%% uncommitted. +%% assignments minted by `~scheduler@1.0', but records the chain position that +%% sequences its mode: weave `offset' for `target', `block-height' for `all'. +%% Every field derives from chain data, so the assignment is deterministic +%% across nodes and is left uncommitted. write_assignment(ProcID, Slot, Extra, Msg, Opts) -> BaseAssignment = lib_scheduler:base_assignment( @@ -787,29 +748,21 @@ ensure_initialized(ProcID, Opts) -> _ -> initialize(ProcID, Opts) end. -%% @doc First contact with a process: locate its spawn block, index it, read -%% the process header, and mint the slot 0 assignment from the process -%% message itself. The canonical process header is also written to the cache -%% so that `~process@1.0' resolves to the verifying tx@1.0 form -%% ahead of any lossier gateway-derived copy. If the spawn is not yet -%% confirmed, initialization fails and is retried on the next synchronization. +%% @doc First contact with a process: locate its spawn block, read the process +%% header, and mint the slot 0 assignment from the process message itself. The +%% canonical process header is also written to the cache so that +%% `~process@1.0' resolves to the verifying tx@1.0 form ahead of any +%% lossier gateway-derived copy. If the spawn is not yet confirmed, +%% initialization fails and is retried on the next synchronization. initialize(ProcID, Opts) -> ?event({initializing_arweave_schedule, {proc_id, ProcID}}), maybe {ok, SpawnHeight} ?= spawn_height(ProcID, Opts), - ok ?= ensure_offsets(SpawnHeight, SpawnHeight, Opts), - {ok, Offset} ?= tx_offset(ProcID, Opts), {ok, Process} ?= read_tx_header(ProcID, Opts), {ok, _} = hb_cache:write(Process, Opts), Mode = mode(Process, Opts), - ok = - write_assignment( - ProcID, - 0, - slot_zero(Mode, Offset, SpawnHeight), - Process, - Opts - ), + {ok, Zero} ?= slot_zero(Mode, ProcID, SpawnHeight, Opts), + ok = write_assignment(ProcID, 0, Zero, Process, Opts), % `synced-to' starts one below the spawn block: slot 0 is the process % itself, and no message-bearing block has been indexed yet. The first % sync begins its range at the spawn block, catching any messages mined @@ -837,11 +790,17 @@ mode(Process, Opts) -> %% @doc The sequencing detail recorded on slot 0. The process message is its %% own first message, so in `all' mode it carries the height of its spawn block -%% -- the process's clock starts at the block it was created in. -slot_zero(<<"all">>, Offset, SpawnHeight) -> - #{ <<"offset">> => Offset, <<"block-height">> => SpawnHeight }; -slot_zero(_Mode, Offset, _SpawnHeight) -> - #{ <<"offset">> => Offset }. +%% -- the process's clock starts at the block it was created in -- and in the +%% modes that sort by weave position, its offset. Only those modes index the +%% spawn block to read that offset. +slot_zero(<<"all">>, _ProcID, SpawnHeight, _Opts) -> + {ok, #{ <<"block-height">> => SpawnHeight }}; +slot_zero(_Mode, ProcID, SpawnHeight, Opts) -> + maybe + ok ?= ensure_offsets(SpawnHeight, SpawnHeight, Opts), + {ok, Offset} ?= tx_offset(ProcID, Opts), + {ok, #{ <<"offset">> => Offset }} + end. %% @doc Read an L1 transaction as a header-only message from the node's stores. %% `~copycat@1.0/arweave' caches the (data-free) header locally while indexing, @@ -982,7 +941,6 @@ confirmed_tip(Opts) -> %%% presence in the schedule proves foreign-message indexing. Arweave is %%% permanent, so these tests are repeatable against the live network. %%% Spawn block 1958986; messages at 1958993, 1958994 and 1958995. --define(FIXTURE_MODULE, <<"_GeSyZbQkmqWk6YzL-tjIqJ-2hakIkg-9k127DpVfO8">>). -define(FIXTURE_PROCESS, <<"q3SycbYpO1lz-S6V2kd7FG3DIZ3AU0NrKKxWw4C3yos">>). -define(FIXTURE_MSG1, <<"Y8GnKytC57VUk7W7FlGnD1oH4OAwFHyP-pJPGCJBa3Y">>). -define(FIXTURE_MSG2, <<"luf1fFmhi0RMIZNnFmv1QgK2SJU5plAFQ3ZxndUsLpk">>). @@ -1098,14 +1056,12 @@ mode_test() -> mode(#{ <<"scheduler-mode">> => <<"sideways">> }, #{}) ). -%% @doc A process sequenced by the whole chain has a clock from slot 0; one -%% sequenced by its own messages keeps the assignment shape it always had. +%% @doc A process sequenced by the whole chain has a clock from slot 0. slot_zero_test() -> ?assertEqual( - #{ <<"offset">> => 7, <<"block-height">> => 3 }, - slot_zero(<<"all">>, 7, 3) - ), - ?assertEqual(#{ <<"offset">> => 7 }, slot_zero(?DEFAULT_MODE, 7, 3)). + {ok, #{ <<"block-height">> => 3 }}, + slot_zero(<<"all">>, ignored, 3, #{}) + ). %% @doc Write a block into the node's block cache at the height pseudo-path %% `~copycat@1.0/arweave' caches it under, so the enumerator reads it locally @@ -1170,10 +1126,9 @@ base_layer_blocks_test() -> ) ). -%% @doc An `all'-mode assignment records the height that sequenced it as well -%% as its weave offset, and both survive the cache round trip that a process -%% reads its schedule back through. This is the contract -%% `~arweave-swap@1.0' reads its clock from. +%% @doc An `all'-mode assignment records the height that sequenced it, and it +%% survives the cache round trip that a process reads its schedule back through. +%% This is the contract `~arweave-swap@1.0' reads its clock from. all_mode_assignment_test() -> Store = hb_test_utils:test_store(), hb_store:start(Store), @@ -1189,13 +1144,12 @@ all_mode_assignment_test() -> write_assignment( ProcID, 1, - #{ <<"offset">> => 42, <<"block-height">> => 1958986 }, + #{ <<"block-height">> => 1958986 }, Msg, Opts ), {ok, Assignment} = dev_arweave_scheduler_cache:read(ProcID, 1, Opts), ?assertEqual(1, hb_util:int(hb_ao:get(<<"slot">>, Assignment, Opts))), - ?assertEqual(42, hb_util:int(hb_ao:get(<<"offset">>, Assignment, Opts))), ?assertEqual( 1958986, hb_util:int(hb_ao:get(<<"block-height">>, Assignment, Opts)) diff --git a/src/preloaded/process/dev_arweave_scheduler_cache.erl b/src/preloaded/process/dev_arweave_scheduler_cache.erl index e94e930848..14beb5e371 100644 --- a/src/preloaded/process/dev_arweave_scheduler_cache.erl +++ b/src/preloaded/process/dev_arweave_scheduler_cache.erl @@ -101,7 +101,7 @@ state_path(ProcID, Key) -> %% `block-height' and `block-hash'. Those describe the current weave tip %% rather than the blocks that sequenced these assignments, so they would be %% misleading on a schedule that is a deterministic read of historical chain -%% data. The on-chain position of each message is its assignment's `offset'. +%% data. Each assignment carries the position that sequences its mode. assignments_to_bundle(ProcID, Assignments, More, RawOpts) -> Opts = lib_scheduler:format_opts(RawOpts), {ok, #{ diff --git a/src/preloaded/process/dev_arweave_swap.erl b/src/preloaded/process/dev_arweave_swap.erl index e94e8feee7..7c59c7aa65 100644 --- a/src/preloaded/process/dev_arweave_swap.erl +++ b/src/preloaded/process/dev_arweave_swap.erl @@ -9,10 +9,10 @@ %%% process can hold, redirect or refund. %%% %%% A process using this device is therefore sequenced by -%%% `~arweave-scheduler@1.0' in its `all' mode: every base-layer transaction -%%% becomes a slot, so a payment between two addresses that the process is not -%%% a party to is nonetheless something the process sees, and can settle -%%% against. Its process message reads: +%%% `~arweave-scheduler@1.0' in its `all' mode: every data-free base-layer +%%% transaction becomes a slot, so a payment between two addresses that the +%%% process is not a party to is nonetheless something the process sees, and +%%% can settle against. Its process message reads: %%%
     %%%     scheduler-device: arweave-scheduler@1.0
     %%%     scheduler-mode:   all
    @@ -78,7 +78,7 @@
     -module(dev_arweave_swap).
     -implements(<<"arweave-swap@1.0">>).
     %%% AO-Core API functions:
    --export([info/0, compute/3, set/3, keys/3]).
    +-export([info/0, compute/3, set/3]).
     -include("include/hb.hrl").
     -include_lib("eunit/include/eunit.hrl").
     
    @@ -93,14 +93,14 @@
     %% by a direct request, so every key routes to `compute'.
     %%
     %% The key a slot resolves is chosen by the scheduled transaction's own `path'
    -%% tag, and this process is sequenced by every transaction on Arweave: whatever
    -%% a stranger writes there must be applied like any other message. So there is
    -%% no `exports' list -- a key outside it would fall through to `~message@1.0',
    +%% tag, and this process is sequenced by every data-free transaction on Arweave:
    +%% whatever a stranger writes there must be applied like any other message. So
    +%% there is no `exports' list -- a key outside it would fall through to `~message@1.0',
     %% answer `not_found', fail its slot and wedge the process permanently -- and no
     %% `excludes' list either, since an excluded key is handed to `~message@1.0'
     %% instead, whose `set' would let a passer-by write the process's own balances
    -%% and whose `keys' would replace the state with a list of key names. `set' and
    -%% `keys' are therefore implemented here.
    +%% and whose `keys' would replace the state with a list of key names. `set' is
    +%% therefore implemented here.
     %%
     %% `info' is deliberately arity 0. A device's `info' is always exported, and had
     %% it taken the base as an argument it would also answer the `info' key, so a
    @@ -135,10 +135,6 @@ set(Base, Req, Opts) ->
             _ -> compute(Base, Req, Opts)
         end.
     
    -%% @doc Listing the state's keys is not a state transition, so a scheduled
    -%% message asking for it is applied as one.
    -keys(Base, Req, Opts) -> compute(Base, Req, Opts).
    -
     %% @doc Apply one assignment to the swap's state.
     %%
     %% In `all' mode the overwhelming majority of slots are unrelated Arweave
    @@ -149,7 +145,7 @@ keys(Base, Req, Opts) -> compute(Base, Req, Opts).
     %% it was.
     %% The process is identified from the assignment rather than from the process
     %% message, which `lib_process:process_id/3' would re-verify the signature of on
    -%% every one of the network's transactions.
    +%% every data-free transaction.
     compute(Base, Assignment, Opts) ->
         Height = hb_util:int(field(<<"block-height">>, Assignment, 0, Opts)),
         ProcID = field(<<"process">>, Assignment, <<>>, Opts),
    @@ -219,11 +215,7 @@ make_offer(Base, Body, Height, Opts) ->
                     {asking, Asking}
                 }
             ),
    -        note(
    -            put_order(debit(Base, Seller, Quantity, Opts), Order, Opts),
    -            <<"order-opened">>,
    -            OrderID
    -        )
    +        put_order(debit(Base, Seller, Quantity, Opts), Order, Opts)
         else
             _ -> Base
         end.
    @@ -242,11 +234,7 @@ cancel_order(Base, Body, Opts) ->
                 <<"quantity">> := Quantity
             } ?= Order,
             ?event({swap_order_cancelled, {order, OrderID}}),
    -        note(
    -            drop_order(credit(Base, Signer, Quantity, Opts), Order, Opts),
    -            <<"order-cancelled">>,
    -            OrderID
    -        )
    +        drop_order(credit(Base, Signer, Quantity, Opts), Order, Opts)
         else
             _ -> Base
         end.
    @@ -288,21 +276,17 @@ register_interest(Base, Body, Height, Opts) ->
                     {until, Until}
                 }
             ),
    -        note(
    -            deadlines(
    -                put_order(
    -                    debit(Base, Buyer, Deposit, Opts),
    -                    Order#{
    -                        <<"status">> => <<"reserved">>,
    -                        <<"buyer">> => Buyer,
    -                        <<"reserved-until">> => Until
    -                    },
    -                    Opts
    -                ),
    +        deadlines(
    +            put_order(
    +                debit(Base, Buyer, Deposit, Opts),
    +                Order#{
    +                    <<"status">> => <<"reserved">>,
    +                    <<"buyer">> => Buyer,
    +                    <<"reserved-until">> => Until
    +                },
                     Opts
                 ),
    -            <<"interest-registered">>,
    -            OrderID
    +            Opts
             )
         else
             _ -> Base
    @@ -311,8 +295,8 @@ register_interest(Base, Body, Height, Opts) ->
     %% @doc Settle against a layer-1 payment. The transaction is not addressed to
     %% the process at all: it is a transfer between two user addresses that names an
     %% `order-id', and the process sees it only because it is sequenced by every
    -%% transaction on the network. It counts as payment for an order when it is
    -%% addressed to that order's `recipient' and carries at least the asking
    +%% data-free transaction on the network. It counts as payment for an order when
    +%% it is addressed to that order's `recipient' and carries at least the asking
     %% winston.
     %%
     %% Underpayment is ignored rather than partially filled -- the value never
    @@ -371,17 +355,13 @@ settle(Base, Order, Buyer, Opts) ->
                 {quantity, Quantity}
             }
         ),
    -    note(
    -        deadlines(
    -            drop_order(
    -                credit(credit(Base, Buyer, Quantity, Opts), Buyer, Pledged, Opts),
    -                Order,
    -                Opts
    -            ),
    +    deadlines(
    +        drop_order(
    +            credit(credit(Base, Buyer, Quantity, Opts), Buyer, Pledged, Opts),
    +            Order,
                 Opts
             ),
    -        <<"order-settled">>,
    -        OrderID
    +        Opts
         ).
     
     %% @doc Whether an order's goods are the payer's to take: an order nobody has
    @@ -569,12 +549,6 @@ credit(Base, Address, Amount, Opts) ->
     
     debit(Base, Address, Amount, Opts) -> credit(Base, Address, -Amount, Opts).
     
    -%% @doc Report what the slot did, in the results of the slot itself.
    -note(Base, Event, OrderID) ->
    -    Base#{
    -        <<"results">> => #{ <<"event">> => Event, <<"order-id">> => OrderID }
    -    }.
    -
     %% @doc Read a number a stranger wrote. Every figure in the protocol arrives as
     %% a tag on somebody else's transaction, and this process is sequenced by all of
     %% them: coercing `deadline: tomorrow' with `hb_util:int/1' would raise out of
    @@ -657,9 +631,10 @@ order_numbers_survive_the_cache_test() ->
     %% @doc A message this device reads is a stranger's, and a key it does not carry
     %% must be answered by this device's default, not by code the stranger picked.
     %%
    -%% `all' mode hands every transaction on Arweave to this process, so the `device'
    -%% key of each is the author's to choose. Reading plainly dispatches on it, and a
    -%% device answers only for absent keys -- so the attack is to omit one. Here
    +%% `all' mode hands every data-free transaction on Arweave to this process, so
    +%% the `device' key of each is the author's to choose. Reading plainly
    +%% dispatches on it, and a device answers only for absent keys -- so the attack
    +%% is to omit one. Here
     %% `recipient' is omitted: the seller of an order is meant to be paid by the
     %% default, and a chosen device answering that key would redirect the goods.
     strangers_device_cannot_answer_for_an_absent_key_test() ->
    @@ -1044,8 +1019,8 @@ unrelated_traffic_test() ->
     
     %% @doc A stranger's transaction may ask to be routed anywhere -- the key a slot
     %% resolves comes from the sender's own `path' tag, and this process is
    -%% sequenced by every transaction on Arweave. It must be applied like any other,
    -%% not fail its slot.
    +%% sequenced by every data-free transaction on Arweave. It must be applied like
    +%% any other, not fail its slot.
     stray_path_is_applied_test() ->
         Opts = test_opts(),
         {Seller, SellerAddr} = party(),
    diff --git a/src/preloaded/process/dev_name_token.erl b/src/preloaded/process/dev_name_token.erl
    index d03263e462..34c963208f 100644
    --- a/src/preloaded/process/dev_name_token.erl
    +++ b/src/preloaded/process/dev_name_token.erl
    @@ -40,13 +40,14 @@
     %%% `/now/total-supply', `/now/value/' -- and not device keys, which is how
     %%% `token-1.0' reads too. The device deliberately exports no key list: a name
     %%% is meant to be sold by `~arweave-swap@1.0', which requires its process to be
    -%%% sequenced by every transaction on Arweave, so the key a slot resolves is
    -%%% chosen by a stranger's `path' tag. An exported `balance' key would let a
    -%%% passer-by's transaction hand back a balance as the new process state.
    +%%% sequenced by every data-free transaction on Arweave, so the key a slot
    +%%% resolves is chosen by a stranger's `path' tag. An exported `balance' key
    +%%% would let a passer-by's transaction hand back a balance as the new process
    +%%% state.
     -module(dev_name_token).
     -implements(<<"name-token@1.0">>).
     %%% AO-Core API functions:
    --export([info/0, compute/3, set/3, keys/3]).
    +-export([info/0, compute/3, set/3]).
     -include("include/hb.hrl").
     -include_lib("eunit/include/eunit.hrl").
     
    @@ -77,15 +78,15 @@ router(_Key, Base, Assignment, Opts) ->
     %% transfer between two other addresses. Which device that is, is a scalar key
     %% on the process; see `swap/3' for why it is not a `~stack@1.0'.
     compute(Base, Assignment, Opts) ->
    -    Seeded = seed(Base, Opts),
    -    Sold = swap(Seeded, Assignment, Opts),
    -    Body = field(<<"body">>, Assignment, #{}, Opts),
    -    ProcID = field(<<"process">>, Assignment, <<>>, Opts),
    -    case field(<<"target">>, Body, <<>>, Opts) of
    +    Sold = swap(seed(Base, Opts), Assignment, Opts),
    +    Body = hb_maps:get(<<"body">>, Assignment, #{}, Opts),
    +    ProcID = hb_maps:get(<<"process">>, Assignment, <<>>, Opts),
    +    case tx_field(Body, <<"target">>, <<>>, Opts) of
    +        <<>> -> {ok, Sold};
             ProcID -> {ok, action(Sold, Body, Opts)};
             _ ->
                 % Not addressed to this name. Under `all' mode that is almost every
    -            % transaction on the network.
    +            % data-free transaction on the network.
                 {ok, Sold}
         end.
     
    @@ -101,9 +102,11 @@ swap(Base, Assignment, Opts) ->
         case state(<<"swap-device">>, Base, not_found, Opts) of
             not_found -> Base;
             Device ->
    -            case hb_ao:resolve(Base#{ <<"device">> => Device }, Assignment, Opts) of
    +            try hb_ao:resolve(Base#{ <<"device">> => Device }, Assignment, Opts) of
                     {ok, Settled} -> Settled#{ <<"device">> => <<"name-token@1.0">> };
                     _ -> Base
    +            catch
    +                _:_ -> Base
                 end
         end.
     
    @@ -129,7 +132,7 @@ seed_holding(Base, Opts) ->
             Holder ->
                 case state(?BALANCES, Base, not_found, Opts) of
                     not_found ->
    -                    Supply = hb_util:int(state(<<"total-supply">>, Base, 1, Opts)),
    +                    Supply = supply(Base, Opts),
                         ?event({name_token_seeded, {holder, Holder}, {supply, Supply}}),
                         Base#{ ?BALANCES => #{ Holder => Supply } };
                     _ -> Base
    @@ -192,8 +195,7 @@ transfer(Base, Body, Opts) ->
                 ),
                 Sender,
                 Recipient,
    -            Quantity,
    -            Opts
    +            Quantity
             )
         else
             _ -> Base
    @@ -202,7 +204,7 @@ transfer(Base, Body, Opts) ->
     %% @doc Emit the pair of notices that `token-1.0' emits for a transfer, with
     %% the same keys. They are the slot's results; whether anything delivers them
     %% is the process's business, not this device's.
    -notices(Base, Sender, Recipient, Quantity, _Opts) ->
    +notices(Base, Sender, Recipient, Quantity) ->
         Base#{
             <<"results">> =>
                 #{
    @@ -273,7 +275,7 @@ value_of(Body, Opts) ->
     %% evaluated against the balances as they stand -- so the authority moves with
     %% the unit, with no separate owner field to keep in step.
     owns_supply(Base, Address, Opts) ->
    -    Supply = hb_util:int(state(<<"total-supply">>, Base, 1, Opts)),
    +    Supply = supply(Base, Opts),
         Threshold =
             hb_util:int(
                 state(
    @@ -285,6 +287,13 @@ owns_supply(Base, Address, Opts) ->
             ),
         balance(Base, Address, Opts) * 10000 >= Supply * Threshold.
     
    +%% @doc Read the name's supply, falling back to its single unit if malformed.
    +supply(Base, Opts) ->
    +    hb_util:ok_or(
    +        hb_util:safe_int(state(<<"total-supply">>, Base, 1, Opts)),
    +        1
    +    ).
    +
     %%% State
     
     %% @doc Read a key of the process's own state. While a slot is being computed
    @@ -297,29 +306,37 @@ state(Key, Base, Default, Opts) ->
     field(Key, Msg, Default, Opts) ->
         hb_maps:get(Key, Msg, Default, Opts).
     
    +%% @doc Read a value from the real layer-1 transaction fields.
    +tx_field(Body, Field, Default, Opts) ->
    +    case
    +        hb_message:commitment(
    +            #{ <<"commitment-device">> => <<"tx@1.0">> },
    +            Body,
    +            Opts
    +        )
    +    of
    +        {ok, _ID, Commitment} ->
    +            hb_maps:get(<<"field-", Field/binary>>, Commitment, Default, Opts);
    +        _ -> Default
    +    end.
    +
     balance(Base, Address, Opts) ->
         hb_util:int(state([?BALANCES, Address], Base, 0, Opts)).
     
    +%% @doc Credit one balance by replacing the whole balances submessage.
     credit(Base, Address, Amount, Opts) ->
    -    write_balance(Base, Address, balance(Base, Address, Opts) + Amount, Opts).
    -
    -debit(Base, Address, Amount, Opts) ->
    -    write_balance(Base, Address, balance(Base, Address, Opts) - Amount, Opts).
    -
    -%% @doc Write one balance back. Only whole top-level keys are written: setting a
    -%% nested path would resolve the keys above it through this device on the way
    -%% down.
    -write_balance(Base, Address, Value, Opts) ->
         Base#{
             ?BALANCES =>
                 hb_maps:put(
                     Address,
    -                Value,
    +                balance(Base, Address, Opts) + Amount,
                     state(?BALANCES, Base, #{}, Opts),
                     Opts
                 )
         }.
     
    +debit(Base, Address, Amount, Opts) -> credit(Base, Address, -Amount, Opts).
    +
     %% @doc Setting the device is honoured, and nothing else is: `lib_process' puts
     %% the process's own device back after every slot, and reading this state as a
     %% message is itself a device set. Anything else is a scheduled message and is
    @@ -336,10 +353,6 @@ set(Base, Req, Opts) ->
             _ -> compute(Base, Req, Opts)
         end.
     
    -%% @doc Listing the state's keys is not a state transition, so a scheduled
    -%% message asking for it is applied as one.
    -keys(Base, Req, Opts) -> compute(Base, Req, Opts).
    -
     %% @doc The single signer of a message. A message with any other number of
     %% signers is not attributable to one party, so it can neither move the unit nor
     %% speak for the name.
    @@ -352,10 +365,9 @@ signer(Body, Opts) ->
     %%% Tests
     
     %%% The tests drive `compute/3' directly with synthetic assignments, exactly as
    -%%% `~process@1.0' would. Below them is the live-network driver that seeded the
    -%%% permanent fixture, and the fixture test that replays it.
    -
    --define(PROCESS, <<"nAmEtOkEn000000000000000000000000000000000">>).
    +%%% `~process@1.0' would. The process id must be a real 43-character address so
    +%%% the transaction codec can carry it as the layer-1 target.
    +-define(PROCESS, <<"nAmEtOkEn0000000000000000000000000000000000">>).
     
     test_opts() -> #{ <<"priv-wallet">> => ar_wallet:new() }.
     
    @@ -393,16 +405,10 @@ apply_tx(Base, Body, Opts) ->
             ),
         New.
     
    -%% @doc A `target' tag is not an address. `~arweave-swap@1.0' states the rule
    -%% this device shares -- "top-level keys may come from tags with the same
    -%% names" -- and the addressed-to-me gate here reads that top-level key, so a
    -%% transaction sent to nobody can reach the actions with a tag alone.
    -%%
    -%% It buys nothing, and this records why: every action is gated again on what
    -%% the signer holds, so reaching `transfer' with an empty balance moves nothing
    -%% and reaching `set' without the supply writes nothing. The gate decides who is
    -%% being spoken to; the holdings decide who may speak.
    -tag_only_target_reaches_the_actions_but_holds_nothing_test() ->
    +%% @doc A `target' tag is not an address. A transaction addressed to nobody
    +%% carries the process's own id at the key a plain read would find, so the gate
    +%% reads the field the base layer moved value to instead.
    +tags_are_not_transaction_fields_test() ->
         Opts = test_opts(),
         {_, OwnerAddr} = party(),
         {Stranger, StrangerAddr} = party(),
    @@ -417,25 +423,11 @@ tag_only_target_reaches_the_actions_but_holds_nothing_test() ->
                     {<<"quantity">>, <<"1">>}
                 ]
             ),
    -    %% The tag does put the process's own id at the key the gate reads.
         ?assertEqual(?PROCESS, hb_ao:get(<<"target">>, Steal, not_found, Opts)),
    +    ?assertEqual(<<>>, tx_field(Steal, <<"target">>, <<>>, Opts)),
         Tried = apply_tx(Held, Steal, Opts),
         ?assertEqual(1, held_by(Tried, OwnerAddr, Opts)),
    -    ?assertEqual(0, held_by(Tried, StrangerAddr, Opts)),
    -    Speak =
    -        tag_only_tx(
    -            Stranger,
    -            [
    -                {<<"target">>, ?PROCESS},
    -                {<<"action">>, <<"set">>},
    -                {<<"reference-value">>, <<"a-value-the-name-never-agreed-to">>}
    -            ]
    -        ),
    -    Said = apply_tx(Tried, Speak, Opts),
    -    ?assertEqual(
    -        hb_ao:get(?VALUE, {as, <<"message@1.0">>, Tried}, not_found, Opts),
    -        hb_ao:get(?VALUE, {as, <<"message@1.0">>, Said}, not_found, Opts)
    -    ).
    +    ?assertEqual(0, held_by(Tried, StrangerAddr, Opts)).
     
     %% @doc A transaction that carries its keys as tags only, addressed to nobody.
     tag_only_tx(Wallet, Tags) ->
    @@ -666,7 +658,7 @@ action_case_is_ignored_test() ->
         ?assertEqual(1, held_by(Moved, BuyerAddr, Opts)).
     
     %% @doc A transaction that is not addressed to this name is left alone. Under
    -%% `all' mode that is almost every transaction on the network.
    +%% `all' mode that is almost every data-free transaction on the network.
     unrelated_traffic_test() ->
         Opts = test_opts(),
         {Stranger, _} = party(),
    @@ -730,6 +722,27 @@ seeded_value_test() ->
         ),
         ?assertEqual(not_found, hb_ao:get(<<"target">>, value(Reset, Opts), not_found, Opts)).
     
    +%% @doc An unreadable supply falls back to the single unit a name represents.
    +unreadable_supply_is_one_unit_test() ->
    +    Opts = test_opts(),
    +    {Owner, OwnerAddr} = party(),
    +    Alive =
    +        apply_tx(
    +            #{
    +                <<"name">> => <<"test-name">>,
    +                <<"total-supply">> => <<"one">>,
    +                <<"initial-holder">> => OwnerAddr
    +            },
    +            tx(Owner, #{ <<"target">> => <<"elsewhere">> }),
    +            Opts
    +        ),
    +    ?assertEqual(1, held_by(Alive, OwnerAddr, Opts)),
    +    Set = apply_tx(Alive, set_tx(Owner, #{ <<"greeting">> => <<"mine">> }), Opts),
    +    ?assertEqual(
    +        <<"mine">>,
    +        hb_ao:get(<<"greeting">>, value(Set, Opts), not_found, Opts)
    +    ).
    +
     %% @doc Seeding happens once. A name whose unit has moved on is not handed a
     %% fresh one by the next message that arrives.
     seeding_happens_once_test() ->
    @@ -749,18 +762,29 @@ seeding_happens_once_test() ->
         ?assertEqual(0, held_by(Later, OwnerAddr, Opts)),
         ?assertEqual(1, held_by(Later, StrangerAddr, Opts)).
     
    -%% @doc A name with no selling device is simply a name: messages the swap would
    -%% have handled do nothing, and the name still works.
    -without_swap_device_test() ->
    +%% @doc A missing or invalid swap device cannot wedge the name.
    +without_a_working_swap_device_test() ->
         Opts = test_opts(),
         {Owner, OwnerAddr} = party(),
    -    Base = name_held_by(OwnerAddr),
    -    Set = apply_tx(Base, set_tx(Owner, #{ <<"greeting">> => <<"no swap here">> }), Opts),
    -    ?assertEqual(
    -        <<"no swap here">>,
    -        hb_ao:get(<<"greeting">>, value(Set, Opts), not_found, Opts)
    -    ),
    -    ?assertEqual(not_found, state(<<"orders">>, Set, not_found, Opts)).
    +    lists:foreach(
    +        fun(Base) ->
    +            Set =
    +                apply_tx(
    +                    Base,
    +                    set_tx(Owner, #{ <<"greeting">> => <<"no swap here">> }),
    +                    Opts
    +                ),
    +            ?assertEqual(
    +                <<"no swap here">>,
    +                hb_ao:get(<<"greeting">>, value(Set, Opts), not_found, Opts)
    +            ),
    +            ?assertEqual(not_found, state(<<"orders">>, Set, not_found, Opts))
    +        end,
    +        [
    +            name_held_by(OwnerAddr),
    +            (name_held_by(OwnerAddr))#{ <<"swap-device">> => <<"nOt-A-dEvIcE@1.0">> }
    +        ]
    +    ).
     
     %% @doc A stranger's transaction may ask to be routed anywhere, including to
     %% keys `~message@1.0' would otherwise answer. Each is applied like any other
    @@ -791,912 +815,13 @@ reserved_paths_are_applied_test() ->
             [<<"set">>, <<"keys">>, <<"info">>, <<"balances">>, <<"anything-else">>]
         ).
     
    -%%% The live network suite
    -%%%
    -%%% Three stories played out on mainnet, against the real weave, with every
    -%%% path and refusal a name sale can take. It is deliberately NOT a `_test'
    -%%% function: it posts real transactions and spends real AR. Invoke it by name:
    -%%%
    -%%%     HB_LIVE_SUITE=1 HB_PRINT=name_token_live \\
    -%%%         rebar3 device test --devices dev_name_token \\
    -%%%             --test dev_name_token:live_suite_test_
    -%%%
    -%%% It reports every transaction id and the slot it landed at, which is what
    -%%% the pinned fixtures below are made of. Each message waits for the previous
    -%%% one to be mined: the schedule is block order, and two messages in one block
    -%%% would be ordered by the block's own transaction list rather than by intent.
    -
    -%% @doc Everything live is reachable only when asked for by name in the
    -%% environment, so it contributes nothing to the battery.
    -live_suite_test_() -> live_gated("HB_LIVE_SUITE", fun live_suite/0, 21600).
    -
    -live_gated(Variable, Fun, Timeout) ->
    -    case os:getenv(Variable) of
    -        false -> [];
    -        _ -> {timeout, Timeout, Fun}
    -    end.
    -
    -%%% What the network charges, and what the orders ask for. The fee an order
    -%%% demands to register is set well above the price of an ordinary transaction,
    -%%% so that clearing it means deliberately overpaying rather than paying what
    -%%% the network was going to charge anyway.
    --define(LIVE_MINIMUM_FEE, 100000000).
    --define(LIVE_PAID_FEE, 150000000).
    --define(LIVE_ASKING, 1000).
    -
    -%% @doc The wallet that pays for the live run.
    -live_wallet() ->
    -    ar_wallet:load_keyfile(<<"/Users/sam/Documents/hyperbeam-key.json">>).
    -
    -%% @doc A counterparty, kept beside the node's own key so that a rerun reuses
    -%% it rather than paying to create another account.
    -live_party(Path) ->
    -    case file:read_file(Path) of
    -        {ok, Json} -> ar_wallet:from_json(Json);
    -        _ ->
    -            Wallet = ar_wallet:new(),
    -            ok = file:write_file(Path, hb_util:bin(ar_wallet:to_json(Wallet))),
    -            Wallet
    -    end.
    -
    -live_address(Wallet) -> hb_util:human_id(ar_wallet:to_address(Wallet)).
    -
    -%% @doc Node options for talking to the live network, mirroring the scheduler's
    -%% own fixture options.
    -live_opts(Wallet) ->
    -    TestStore = hb_test_utils:test_store(),
    -    IndexStore = hb_test_utils:test_store(),
    -    (hb_opts:default_message())#{
    -        <<"store">> => [
    -            TestStore,
    -            #{
    -                <<"store-module">> => hb_store_arweave,
    -                <<"name">> => <<"cache-arweave">>,
    -                <<"index-store">> => [IndexStore]
    -            },
    -            #{
    -                <<"store-module">> => hb_store_gateway,
    -                <<"local-store">> => [TestStore]
    -            }
    -        ],
    -        <<"arweave-index-store">> => #{ <<"index-store">> => [IndexStore] },
    -        <<"arweave-index-workers">> => 8,
    -        <<"arweave-scheduler-confirmation-depth">> => 1,
    -        <<"priv-wallet">> => Wallet
    -    }.
    -
    -%% @doc What the network will charge to send a transaction of no size to an
    -%% address. An address that has never held AR costs thousands of times more to
    -%% send to, because the transaction creates the account.
    -live_price(Target, Opts) ->
    -    Path =
    -        case Target of
    -            <<>> -> <<"/price/0">>;
    -            _ -> <<"/price/0/", Target/binary>>
    -        end,
    -    {ok, Res} = hb_http:get(<<"https://arweave.net">>, Path, Opts),
    -    hb_util:int(hb_ao:get(<<"body">>, Res, <<"0">>, Opts)).
    -
    -live_anchor(Opts) ->
    -    {ok, Res} = hb_http:get(<<"https://arweave.net">>, <<"/tx_anchor">>, Opts),
    -    hb_ao:get(<<"body">>, Res, <<>>, Opts).
    -
    -live_balance(Address, Opts) ->
    -    {ok, Res} =
    -        hb_http:get(
    -            <<"https://arweave.net">>,
    -            <<"/wallet/", Address/binary, "/balance">>,
    -            Opts
    -        ),
    -    hb_util:int(hb_ao:get(<<"body">>, Res, <<"0">>, Opts)).
    -
    -%% @doc Sign a layer-1 transaction and hand it to the network, through the
    -%% scheduler's own dispatch path, then wait for it to be mined.
    -%%
    -%% A `reward' given in the fields is a floor, not a replacement: the network's
    -%% own price still has to be met. That is how a registration pays an order's
    -%% `minimum-fee' -- by overpaying the reward, which goes to miners and the
    -%% endowment rather than to an address with no key behind it.
    -live_post(Label, Fields, Wallet, Opts) ->
    -    Target = hb_maps:get(<<"target">>, Fields, <<>>, Opts),
    -    Floor = hb_util:int(hb_maps:get(<<"reward">>, Fields, <<"0">>, Opts)),
    -    Reward = max(live_price(Target, Opts), Floor),
    -    Msg =
    -        hb_message:commit(
    -            Fields#{
    -                <<"anchor">> => live_anchor(Opts),
    -                <<"reward">> => hb_util:bin(Reward)
    -            },
    -            Opts#{ <<"priv-wallet">> => Wallet },
    -            #{ <<"commitment-device">> => <<"tx@1.0">> }
    -        ),
    -    ID = hb_util:human_id(hb_message:id(Msg, signed, Opts)),
    -    {ok, Res} =
    -        hb_ao:resolve(
    -            #{ <<"device">> => <<"arweave-scheduler@1.0">> },
    -            #{
    -                <<"path">> => <<"schedule">>,
    -                <<"method">> => <<"POST">>,
    -                <<"body">> => Msg
    -            },
    -            Opts
    -        ),
    -    Height = live_await(ID, Opts),
    -    ?event(name_token_live,
    -        {posted,
    -            {label, {string, Label}},
    -            {tx, {string, ID}},
    -            {signer, {string, live_address(Wallet)}},
    -            {reward, Reward},
    -            {status, hb_ao:get(<<"status">>, Res, none, Opts)},
    -            {height, Height}
    -        }
    -    ),
    -    ID.
    -
    -%% @doc Wait for a transaction to be mined, returning the height it landed at.
    -%% While a transaction is pending the gateway answers in prose -- `Pending',
    -%% `Accepted' -- and only once it is in a block does it answer with the JSON
    -%% that carries the height.
    -live_await(ID, Opts) -> live_await(ID, 240, Opts).
    -live_await(ID, 0, _Opts) -> error({not_mined, ID});
    -live_await(ID, Attempts, Opts) ->
    -    case live_height(ID, Opts) of
    -        not_found ->
    -            timer:sleep(15000),
    -            live_await(ID, Attempts - 1, Opts);
    -        Height -> Height
    -    end.
    -
    -live_height(ID, Opts) ->
    -    case hb_http:get(<<"https://arweave.net">>, <<"/tx/", ID/binary, "/status">>, Opts) of
    -        {ok, Status} ->
    -            Body = hb_ao:get(<<"body">>, Status, <<"">>, Opts),
    -            try hb_json:decode(Body) of
    -                Decoded ->
    -                    case hb_ao:get(<<"block_height">>, Decoded, not_found, Opts) of
    -                        not_found -> not_found;
    -                        Height -> hb_util:int(Height)
    -                    end
    -            catch
    -                _:_ -> not_found
    -            end;
    -        _ -> not_found
    -    end.
    -
    -%% @doc Top an account up only if it cannot pay its own way. Creating an
    -%% account is by far the most expensive part of a run, so a funded counterparty
    -%% is left alone and a rerun costs nothing here.
    -live_fund(Address, Need, Seller, Opts) ->
    -    case live_balance(Address, Opts) of
    -        Balance when Balance >= Need ->
    -            ?event(name_token_live,
    -                {already_funded, {address, {string, Address}}, {balance, Balance}}
    -            ),
    -            Balance;
    -        Balance ->
    -            ?event(name_token_live,
    -                {funding, {address, {string, Address}}, {balance, Balance}}
    -            ),
    -            live_post(
    -                <<"fund">>,
    -                #{
    -                    <<"target">> => Address,
    -                    <<"quantity">> => hb_util:bin(Need * 2)
    -                },
    -                Seller,
    -                Opts
    -            )
    -    end.
    -
    -%% @doc Spawn a name on Arweave. Every key is a scalar: a submessage or a list
    -%% would be written to the weave as a `+link' to content the weave does not
    -%% hold, and no node could read the process back.
    -live_spawn(Name, Holder, Wallet, Opts) ->
    -    live_post(
    -        <<"spawn:", Name/binary>>,
    -        #{
    -            <<"device">> => <<"process@1.0">>,
    -            <<"type">> => <<"Process">>,
    -            <<"scheduler-device">> => <<"arweave-scheduler@1.0">>,
    -            <<"scheduler-mode">> => <<"all">>,
    -            <<"execution-device">> => <<"name-token@1.0">>,
    -            <<"swap-device">> => <<"arweave-swap@1.0">>,
    -            <<"name">> => Name,
    -            <<"ticker">> => <<"NAME">>,
    -            <<"denomination">> => <<"0">>,
    -            <<"total-supply">> => <<"1">>,
    -            <<"initial-holder">> => Holder,
    -            <<"test-suite">> => <<"name-token">>
    -        },
    -        Wallet,
    -        Opts
    -    ).
    -
    -%% @doc Address a message to a name. The first message to a process pays to
    -%% create its account and must carry at least a winston to do so.
    -live_send(Label, ProcID, Fields, Wallet, Opts) ->
    -    live_post(
    -        Label,
    -        Fields#{ <<"target">> => ProcID, <<"quantity">> => <<"1">> },
    -        Wallet,
    -        Opts
    -    ).
    -
    -%% @doc Report which slot each of a run's transactions landed at, by reading the
    -%% schedule back and matching each assignment's body against the ids we sent.
    -%% These are the numbers the fixture tests read state at.
    -live_slots(ProcID, Named, Opts) ->
    -    {ok, Schedule} =
    -        hb_ao:resolve(
    -            #{ <<"device">> => <<"arweave-scheduler@1.0">> },
    -            #{
    -                <<"path">> => <<"schedule">>,
    -                <<"method">> => <<"GET">>,
    -                <<"target">> => ProcID
    -            },
    -            Opts
    -        ),
    -    Assignments =
    -        hb_ao:normalize_keys(hb_ao:get(<<"assignments">>, Schedule, Opts), Opts),
    -    % A schedule's assignments are keyed by slot, alongside the keys any
    -    % committed message carries; only the numbered ones are slots.
    -    BySlot =
    -        [
    -            {
    -                Slot,
    -                hb_util:human_id(
    -                    hb_message:id(
    -                        hb_ao:get(<<"body">>, Assignment, Opts),
    -                        signed,
    -                        Opts
    -                    )
    -                )
    -            }
    -        ||
    -            {Key, Assignment} <- hb_maps:to_list(Assignments, Opts),
    -            {ok, Slot} <- [hb_util:safe_int(Key)],
    -            is_map(Assignment)
    -        ],
    -    lists:foreach(
    -        fun({Label, ID}) ->
    -            Slot =
    -                case [S || {S, Body} <- BySlot, Body =:= ID] of
    -                    [Found | _] -> Found;
    -                    [] -> not_assigned
    -                end,
    -            ?event(name_token_live,
    -                {slot,
    -                    {process, {string, ProcID}},
    -                    {label, {string, Label}},
    -                    {tx, {string, ID}},
    -                    {slot, Slot}
    -                }
    -            )
    -        end,
    -        Named
    -    ),
    -    ?event(name_token_live,
    -        {schedule_length, {process, {string, ProcID}}, {slots, length(BySlot)}}
    -    ),
    -    ok.
    -
    -%%% Story one: a name that had to be paid for.
    -%%%
    -%%% An offer that charges to register turns away a buyer who will not pay the
    -%%% fee, and turns away a payment from somebody who never reserved it. The buyer
    -%%% who does both gets the name -- and with it the right to say what it means,
    -%%% which the seller loses in the same breath.
    -live_story_sale(Seller, Poor, Rich, Opts) ->
    -    SellerAddr = live_address(Seller),
    -    ProcID = live_spawn(<<"paid-for">>, SellerAddr, Seller, Opts),
    -    Offer =
    -        live_send(
    -            <<"make-offer">>,
    -            ProcID,
    -            #{
    -                <<"action">> => <<"make-offer">>,
    -                <<"offer-quantity">> => <<"1">>,
    -                <<"asking">> => hb_util:bin(?LIVE_ASKING),
    -                <<"deposit">> => <<"0">>,
    -                <<"minimum-fee">> => hb_util:bin(?LIVE_MINIMUM_FEE),
    -                <<"deadline">> => <<"99999999">>
    -            },
    -            Seller,
    -            Opts
    -        ),
    -    % A registration that pays only what the network charges anyway does not
    -    % clear a fee set above it.
    -    Underpaid =
    -        live_send(
    -            <<"register:underpaid">>,
    -            ProcID,
    -            #{ <<"action">> => <<"register-interest">>, <<"order-id">> => Offer },
    -            Poor,
    -            Opts
    -        ),
    -    % Nor does value sent to the process, which would only be stranded there.
    -    Stranded =
    -        live_post(
    -            <<"register:stranded">>,
    -            #{
    -                <<"target">> => ProcID,
    -                <<"quantity">> => hb_util:bin(?LIVE_PAID_FEE),
    -                <<"action">> => <<"register-interest">>,
    -                <<"order-id">> => Offer
    -            },
    -            Poor,
    -            Opts
    -        ),
    -    % Overpaying the reward does.
    -    Registered =
    -        live_post(
    -            <<"register:paid">>,
    -            #{
    -                <<"target">> => ProcID,
    -                <<"quantity">> => <<"1">>,
    -                <<"reward">> => hb_util:bin(?LIVE_PAID_FEE),
    -                <<"action">> => <<"register-interest">>,
    -                <<"order-id">> => Offer
    -            },
    -            Rich,
    -            Opts
    -        ),
    -    % The order is now the registrant's alone: somebody else's payment buys
    -    % nothing, and with no bond posted there is nothing to compensate them with.
    -    Interloper =
    -        live_post(
    -            <<"payment:interloper">>,
    -            #{
    -                <<"target">> => SellerAddr,
    -                <<"quantity">> => hb_util:bin(?LIVE_ASKING),
    -                <<"order-id">> => Offer
    -            },
    -            Poor,
    -            Opts
    -        ),
    -    Payment =
    -        live_post(
    -            <<"payment:buyer">>,
    -            #{
    -                <<"target">> => SellerAddr,
    -                <<"quantity">> => hb_util:bin(?LIVE_ASKING),
    -                <<"order-id">> => Offer
    -            },
    -            Rich,
    -            Opts
    -        ),
    -    % The seller no longer holds the name, so no longer speaks for it.
    -    StaleSet =
    -        live_send(
    -            <<"set:former-owner">>,
    -            ProcID,
    -            #{ <<"action">> => <<"set">>, <<"greeting">> => <<"still mine">> },
    -            Seller,
    -            Opts
    -        ),
    -    OwnerSet =
    -        live_send(
    -            <<"set:owner">>,
    -            ProcID,
    -            #{
    -                <<"action">> => <<"set">>,
    -                <<"content-type">> => <<"text/plain">>,
    -                <<"greeting">> => <<"hello from the new owner">>
    -            },
    -            Rich,
    -            Opts
    -        ),
    -    % A stranger may ask a slot to resolve anything at all. It must be applied
    -    % like any other message: a slot that failed could never be recomputed, and
    -    % the process would stop on every node for good.
    -    Stray =
    -        live_post(
    -            <<"stray:path-info">>,
    -            #{
    -                <<"target">> => ProcID,
    -                <<"quantity">> => <<"1">>,
    -                <<"path">> => <<"info">>
    -            },
    -            Poor,
    -            Opts
    -        ),
    -    % And a figure that is not a number is an inadmissible message, not a
    -    % failed slot.
    -    Nonsense =
    -        live_send(
    -            <<"make-offer:nonsense">>,
    -            ProcID,
    -            #{
    -                <<"action">> => <<"make-offer">>,
    -                <<"offer-quantity">> => <<"1">>,
    -                <<"asking">> => <<"1000">>,
    -                <<"deposit">> => <<"0">>,
    -                <<"deadline">> => <<"tomorrow">>
    -            },
    -            Seller,
    -            Opts
    -        ),
    -    live_slots(
    -        ProcID,
    -        [
    -            {<<"make-offer">>, Offer},
    -            {<<"register:underpaid">>, Underpaid},
    -            {<<"register:stranded">>, Stranded},
    -            {<<"register:paid">>, Registered},
    -            {<<"payment:interloper">>, Interloper},
    -            {<<"payment:buyer">>, Payment},
    -            {<<"set:former-owner">>, StaleSet},
    -            {<<"set:owner">>, OwnerSet},
    -            {<<"stray:path-info">>, Stray},
    -            {<<"make-offer:nonsense">>, Nonsense}
    -        ],
    -        Opts
    -    ),
    -    ProcID.
    -
    -%%% Story two: an offer withdrawn.
    +%%% The permanent fixtures
     %%%
    -%%% A seller may take an unreserved order back, and once they have, nothing
    -%%% more can be done with it: registering is refused and a payment against it
    -%%% buys nothing. A stranger may not withdraw somebody else's order.
    -live_story_withdrawn(Seller, Poor, Rich, Opts) ->
    -    SellerAddr = live_address(Seller),
    -    ProcID = live_spawn(<<"withdrawn">>, SellerAddr, Seller, Opts),
    -    Offer =
    -        live_send(
    -            <<"make-offer">>,
    -            ProcID,
    -            #{
    -                <<"action">> => <<"make-offer">>,
    -                <<"offer-quantity">> => <<"1">>,
    -                <<"asking">> => hb_util:bin(?LIVE_ASKING),
    -                <<"deposit">> => <<"0">>,
    -                <<"minimum-fee">> => hb_util:bin(?LIVE_MINIMUM_FEE),
    -                <<"deadline">> => <<"99999999">>
    -            },
    -            Seller,
    -            Opts
    -        ),
    -    Cancelled =
    -        live_send(
    -            <<"cancel">>,
    -            ProcID,
    -            #{ <<"action">> => <<"cancel-order">>, <<"order-id">> => Offer },
    -            Seller,
    -            Opts
    -        ),
    -    LateRegister =
    -        live_post(
    -            <<"register:after-cancel">>,
    -            #{
    -                <<"target">> => ProcID,
    -                <<"quantity">> => <<"1">>,
    -                <<"reward">> => hb_util:bin(?LIVE_PAID_FEE),
    -                <<"action">> => <<"register-interest">>,
    -                <<"order-id">> => Offer
    -            },
    -            Rich,
    -            Opts
    -        ),
    -    LatePayment =
    -        live_post(
    -            <<"payment:after-cancel">>,
    -            #{
    -                <<"target">> => SellerAddr,
    -                <<"quantity">> => hb_util:bin(?LIVE_ASKING),
    -                <<"order-id">> => Offer
    -            },
    -            Rich,
    -            Opts
    -        ),
    -    Second =
    -        live_send(
    -            <<"make-offer:second">>,
    -            ProcID,
    -            #{
    -                <<"action">> => <<"make-offer">>,
    -                <<"offer-quantity">> => <<"1">>,
    -                <<"asking">> => hb_util:bin(?LIVE_ASKING),
    -                <<"deposit">> => <<"0">>,
    -                <<"minimum-fee">> => hb_util:bin(?LIVE_MINIMUM_FEE),
    -                <<"deadline">> => <<"99999999">>
    -            },
    -            Seller,
    -            Opts
    -        ),
    -    StrangerCancel =
    -        live_send(
    -            <<"cancel:stranger">>,
    -            ProcID,
    -            #{ <<"action">> => <<"cancel-order">>, <<"order-id">> => Second },
    -            Poor,
    -            Opts
    -        ),
    -    live_slots(
    -        ProcID,
    -        [
    -            {<<"make-offer">>, Offer},
    -            {<<"cancel">>, Cancelled},
    -            {<<"register:after-cancel">>, LateRegister},
    -            {<<"payment:after-cancel">>, LatePayment},
    -            {<<"make-offer:second">>, Second},
    -            {<<"cancel:stranger">>, StrangerCancel}
    -        ],
    -        Opts
    -    ),
    -    ProcID.
    -
    -%%% Story three: a name handed over directly.
    +%%% The driver that posted these stories spends real AR and does not belong in
    +%%% the test battery. It remains recoverable from the commit that created them:
     %%%
    -%%% No sale at all -- just the token half. The unit moves, and the authority
    -%%% moves with it: the former holder's word stops counting the moment it does.
    -live_story_handover(Seller, Rich, Opts) ->
    -    SellerAddr = live_address(Seller),
    -    RichAddr = live_address(Rich),
    -    ProcID = live_spawn(<<"handed-over">>, SellerAddr, Seller, Opts),
    -    FirstSet =
    -        live_send(
    -            <<"set:before">>,
    -            ProcID,
    -            #{ <<"action">> => <<"set">>, <<"points-at">> => <<"seller">> },
    -            Seller,
    -            Opts
    -        ),
    -    Transfer =
    -        live_send(
    -            <<"transfer">>,
    -            ProcID,
    -            #{
    -                <<"action">> => <<"transfer">>,
    -                <<"recipient">> => RichAddr,
    -                <<"quantity">> => <<"1">>
    -            },
    -            Seller,
    -            Opts
    -        ),
    -    StaleSet =
    -        live_send(
    -            <<"set:former-owner">>,
    -            ProcID,
    -            #{ <<"action">> => <<"set">>, <<"points-at">> => <<"seller again">> },
    -            Seller,
    -            Opts
    -        ),
    -    NewSet =
    -        live_send(
    -            <<"set:new-owner">>,
    -            ProcID,
    -            #{ <<"action">> => <<"set">>, <<"points-at">> => <<"buyer">> },
    -            Rich,
    -            Opts
    -        ),
    -    live_slots(
    -        ProcID,
    -        [
    -            {<<"set:before">>, FirstSet},
    -            {<<"transfer">>, Transfer},
    -            {<<"set:former-owner">>, StaleSet},
    -            {<<"set:new-owner">>, NewSet}
    -        ],
    -        Opts
    -    ),
    -    ProcID.
    -
    -%% @doc The stories a run was asked for, named in the environment as
    -%% `HB_LIVE_STORIES=sale,withdrawn' or left unset for all of them.
    -live_stories() ->
    -    case os:getenv("HB_LIVE_STORIES") of
    -        false -> all;
    -        Names ->
    -            [
    -                hb_util:atom(string:trim(Name))
    -            ||
    -                Name <- string:split(Names, ",", all)
    -            ]
    -    end.
    -
    -live_story(Name, all, Fun) -> live_story(Name, [Name], Fun);
    -live_story(Name, Wanted, Fun) ->
    -    case lists:member(Name, Wanted) of
    -        true -> Fun();
    -        false ->
    -            ?event(name_token_live, {story_skipped, {name, Name}}),
    -            skipped
    -    end.
    -
    -%% @doc Mint the five `pn-test-N' names the site's test namespace resolves
    -%% through, and report what to put in the manifest.
    -%%
    -%% Each is spawned already holding its unit and already pointing somewhere, so
    -%% none of them needs a message sent to it -- which is the whole point of
    -%% `initial-value', since the first message addressed to a process pays Arweave's
    -%% new-account fee. Targets alternate between a manifest and another reference,
    -%% because a name that points at a reference is the shape a person actually wants:
    -%% a reference can be updated with a bundled data item in seconds, where a name
    -%% token needs consensus.
    -%%
    -%%     HB_LIVE_NAMES=1 HB_PRINT=name_token_live \\
    -%%         rebar3 device test --devices dev_name_token \\
    -%%             --test dev_name_token:live_names_test_
    -live_names_test_() -> live_gated("HB_LIVE_NAMES", fun live_names/0, 7200).
    -
    -live_names() ->
    -    Seller = live_wallet(),
    -    SellerAddr = live_address(Seller),
    -    Opts = live_opts(Seller),
    -    Targets = live_name_targets(),
    -    Minted =
    -        lists:map(
    -            fun({Index, Kind, Target}) ->
    -                Name = <<"pn-test-", (hb_util:bin(Index))/binary>>,
    -                ProcID =
    -                    live_post(
    -                        <<"mint:", Name/binary>>,
    -                        #{
    -                            <<"device">> => <<"process@1.0">>,
    -                            <<"type">> => <<"Process">>,
    -                            <<"scheduler-device">> => <<"arweave-scheduler@1.0">>,
    -                            <<"scheduler-mode">> => <<"all">>,
    -                            <<"execution-device">> => <<"name-token@1.0">>,
    -                            <<"swap-device">> => <<"arweave-swap@1.0">>,
    -                            <<"name">> => Name,
    -                            <<"ticker">> => <<"NAME">>,
    -                            <<"denomination">> => <<"0">>,
    -                            <<"total-supply">> => <<"1">>,
    -                            <<"initial-holder">> => SellerAddr,
    -                            <<"initial-value">> => Target,
    -                            <<"test-suite">> => <<"name-token">>
    -                        },
    -                        Seller,
    -                        Opts
    -                    ),
    -                ?event(name_token_live,
    -                    {minted,
    -                        {name, {string, Name}},
    -                        {process, {string, ProcID}},
    -                        {points_at, {string, Target}},
    -                        {kind, Kind}
    -                    }
    -                ),
    -                {Name, ProcID, Kind, Target}
    -            end,
    -            Targets
    -        ),
    -    lists:foreach(fun({_, ProcID, _, _}) -> live_await(ProcID, Opts) end, Minted),
    -    ?event(name_token_live,
    -        {namespace_entries,
    -            {holder, {string, SellerAddr}},
    -            {entries,
    -                {string,
    -                    hb_util:bin(
    -                        lists:flatten(
    -                            [
    -                                io_lib:format("~s=~s ", [Name, ProcID])
    -                            ||
    -                                {Name, ProcID, _, _} <- Minted
    -                            ]
    -                        )
    -                    )
    -                }
    -            }
    -        }
    -    ),
    -    {ok, Minted}.
    -
    -%% @doc What each test name points at. The manifest is the AO site's own, so a
    -%% resolved name actually renders something; the references are real
    -%% `~reference@1.0' inits, so the deeper chain is exercised rather than mocked.
    -live_name_targets() ->
    -    % A real Arweave path manifest that a gateway serves today, so a resolved
    -    % name renders something rather than 404ing.
    -    Manifest = <<"6oMvmlBUUltTDz_T9pZrEP2QkzpCBGk83Br8XXbqy20">>,
    -    % Replaced with the test namespace's own reference once it is published; a
    -    % name pointing at a reference is the shape an owner actually wants, because
    -    % a reference can be repointed in seconds.
    -    Reference = <<"6oMvmlBUUltTDz_T9pZrEP2QkzpCBGk83Br8XXbqy20">>,
    -    [
    -        {1, manifest, Manifest},
    -        {2, reference, Reference},
    -        {3, manifest, Manifest},
    -        {4, reference, Reference},
    -        {5, manifest, Manifest}
    -    ].
    -
    -%% @doc Play all three stories out on mainnet.
    -live_suite() ->
    -    Seller = live_wallet(),
    -    SellerAddr = live_address(Seller),
    -    Opts = live_opts(Seller),
    -    Poor = live_party(<<"name-token-poor.json">>),
    -    Rich = live_party(<<"name-token-buyer.json">>),
    -    PoorAddr = live_address(Poor),
    -    RichAddr = live_address(Rich),
    -    ?event(name_token_live,
    -        {parties,
    -            {seller, {string, SellerAddr}},
    -            {underpayer, {string, PoorAddr}},
    -            {buyer, {string, RichAddr}},
    -            {seller_balance, live_balance(SellerAddr, Opts)}
    -        }
    -    ),
    -    live_fund(PoorAddr, 500000000, Seller, Opts),
    -    live_fund(RichAddr, 500000000, Seller, Opts),
    -    % Each story stands alone on its own process, so a rerun can name just the
    -    % ones it needs rather than paying to create every account again.
    -    Wanted = live_stories(),
    -    Sale = live_story(sale, Wanted, fun() -> live_story_sale(Seller, Poor, Rich, Opts) end),
    -    Withdrawn =
    -        live_story(
    -            withdrawn,
    -            Wanted,
    -            fun() -> live_story_withdrawn(Seller, Poor, Rich, Opts) end
    -        ),
    -    Handover =
    -        live_story(handover, Wanted, fun() -> live_story_handover(Seller, Rich, Opts) end),
    -    ?event(name_token_live,
    -        {suite_complete,
    -            {sale, {string, Sale}},
    -            {withdrawn, {string, Withdrawn}},
    -            {handover, {string, Handover}},
    -            {buyer, {string, RichAddr}},
    -            {underpayer, {string, PoorAddr}},
    -            {seller, {string, SellerAddr}},
    -            {seller_balance, live_balance(SellerAddr, Opts)}
    -        }
    -    ),
    -    ok.
    -
    -%%% The permanent fixture
    -%%%
    -%%% A name that was really sold on Arweave, by the driver above. Everything
    -%%% below is a deterministic read of blocks 1966039-1966044 of the weave, so it
    -%%% is repeatable forever: the seller spawned `test-name' holding its single
    -%%% unit, offered it for 1000 winston with no bond and a 1000 winston fee to
    -%%% register, the buyer registered (paying that fee), paid, and then -- owning
    -%%% the name -- pointed it at a message of their own.
    +%%%     git show 824816e7c:src/preloaded/process/dev_name_token.erl
     %%%
    -%%% One transaction per block, so the schedule's order is unambiguous:
    -%%%
    -%%%     1966039  the name          yWRe7v4S...
    -%%%     1966041  make-offer        3BApJHea...  (the order id)
    -%%%     1966042  register-interest GGPH2lA8...
    -%%%     1966043  payment           KROLsGpr...  (to the seller, not the process)
    -%%%     1966044  set               QjmNGlIi...
    -%%%
    -%%% The payment is the point: it is an ordinary transfer between two addresses,
    -%%% the process is not a party to it, and the process sees it only because
    -%%% `~arweave-scheduler@1.0' is sequencing it by every transaction on the
    -%%% network. Every transaction in that range is a slot of this
    -%%% process, not just these five.
    --define(FIXTURE_PROCESS, <<"yWRe7v4SZ4_NKV6LkYyNPrFdzEaGh0ckblu-CaGXqG4">>).
    --define(FIXTURE_SELLER, <<"ggltHF0Cnv9ylH3vM1p7amR2vXLMoPLQIUQmAEwLP-k">>).
    --define(FIXTURE_BUYER, <<"LW0myHWuv7XcLec19OCDzFJW0P6jXPG_Ao49kfy9Slc">>).
    --define(FIXTURE_ORDER, <<"3BApJHeatc9pVuLgjZ_P-HT5hZgRE1Q3I1bdTESgRDM">>).
    --define(FIXTURE_MAX_HEIGHT, 1966044).
    -
    -%% @doc Read the fixture's state as of the pinned height. The height cap makes
    -%% the answer immutable: no block after 1966044 can reach this process.
    -fixture_opts() ->
    -    TestStore = hb_test_utils:test_store(),
    -    IndexStore = hb_test_utils:test_store(),
    -    (hb_opts:default_message())#{
    -        <<"store">> => [
    -            TestStore,
    -            #{
    -                <<"store-module">> => hb_store_arweave,
    -                <<"name">> => <<"cache-arweave">>,
    -                <<"index-store">> => [IndexStore]
    -            },
    -            #{
    -                <<"store-module">> => hb_store_gateway,
    -                <<"local-store">> => [TestStore]
    -            }
    -        ],
    -        <<"arweave-index-store">> => #{ <<"index-store">> => [IndexStore] },
    -        <<"arweave-index-workers">> => 8,
    -        <<"arweave-scheduler-confirmation-depth">> => 1,
    -        <<"arweave-scheduler-max-height">> => ?FIXTURE_MAX_HEIGHT,
    -        <<"name-resolvers">> => [#{ <<"test-name">> => ?FIXTURE_PROCESS }],
    -        <<"node-host">> => <<"host">>,
    -        <<"priv-wallet">> => ar_wallet:new()
    -    }.
    -
    -%% @doc Synchronize the fixture's schedule from the network, retrying while the
    -%% gateway rate-limits us -- the same allowance the scheduler's own fixture
    -%% tests make.
    -fixture_sync(_Opts, 0) -> {error, fixture_sync_failed};
    -fixture_sync(Opts, Attempts) ->
    -    case
    -        hb_ao:resolve(
    -            #{ <<"device">> => <<"arweave-scheduler@1.0">> },
    -            #{
    -                <<"path">> => <<"schedule">>,
    -                <<"method">> => <<"GET">>,
    -                <<"target">> => ?FIXTURE_PROCESS
    -            },
    -            Opts
    -        )
    -    of
    -        {ok, Schedule} -> {ok, Schedule};
    -        _ ->
    -            timer:sleep(5000),
    -            fixture_sync(Opts, Attempts - 1)
    -    end.
    -
    -%% @doc Compute the fixture to its latest slot. The schedule is primed first, so
    -%% that the process message is read back as its canonical `tx@1.0' decoding
    -%% rather than a gateway store's lossier one.
    -fixture_state(Opts, Attempts) ->
    -    {ok, _} = fixture_sync(Opts, Attempts),
    -    {ok, Raw} = hb_cache:read(?FIXTURE_PROCESS, Opts),
    -    Process = hb_cache:ensure_all_loaded(Raw, Opts),
    -    hb_ao:resolve(Process, <<"now">>, Opts).
    -
    -%% @doc The whole story, read back off the weave: a name that changed hands for
    -%% AR that never touched the process, and then said something new.
    -fixture_sale_test_() ->
    -    {timeout, 1800, fun fixture_sale/0}.
    -fixture_sale() ->
    -    Opts = fixture_opts(),
    -    {ok, State} = fixture_state(Opts, 5),
    -    Read = fun(Path) -> hb_ao:get(Path, {as, <<"message@1.0">>, State}, not_found, Opts) end,
    -    % The name is the buyer's: the swap settled a payment it was not paid.
    -    ?assertEqual(1, hb_util:int(Read([?BALANCES, ?FIXTURE_BUYER]))),
    -    ?assertEqual(0, hb_util:int(Read([?BALANCES, ?FIXTURE_SELLER]))),
    -    ?assertEqual(1, hb_util:int(Read(<<"total-supply">>))),
    -    % The offer is complete, so the book no longer holds it.
    -    ?assertEqual(
    -        not_found,
    -        Read([<<"orders">>, ?FIXTURE_ORDER, <<"status">>])
    -    ),
    -    % And the new owner has said what the name points at.
    -    ?assertEqual(<<"hello from the new owner">>, Read([?VALUE, <<"greeting">>])),
    -    ?assertEqual(<<"text/plain">>, Read([?VALUE, <<"content-type">>])).
    -
    -%% @doc The name resolves: `test-name' reaches this instance, both as a bare
    -%% name and as the label of a host.
    -fixture_name_resolution_test() ->
    -    Opts = fixture_opts(),
    -    % A node that serves a name holds it. Priming the schedule puts the process
    -    % in the node's own cache, which is what the resolver then loads.
    -    {ok, _} = fixture_sync(Opts, 5),
    -    ?assertEqual(
    -        {ok, ?FIXTURE_PROCESS},
    -        hb_ao:resolve_many(
    -            [
    -                #{ <<"device">> => <<"name@1.0">> },
    -                #{ <<"path">> => <<"test-name">>, <<"load">> => false }
    -            ],
    -            Opts
    -        )
    -    ),
    -    % `test-name.host' is the same lookup: the node's own host is stripped from
    -    % the request's host, leaving the label to resolve.
    -    {ok, Resolved} =
    -        hb_ao:resolve(
    -            #{ <<"device">> => <<"name@1.0">> },
    -            #{
    -                <<"path">> => <<"request">>,
    -                <<"request">> => #{ <<"host">> => <<"test-name.host">> },
    -                <<"body">> => [#{ <<"path">> => <<"now">> }]
    -            },
    -            Opts
    -        ),
    -    [Named | _] = hb_ao:get(<<"body">>, Resolved, [], Opts),
    -    Loaded = hb_cache:ensure_all_loaded(Named, Opts),
    -    % The message the host resolved to is this name: it carries the name's own
    -    % spawn keys, and nothing else on the weave does.
    -    ?assertEqual(<<"test-name">>, hb_ao:get(<<"name">>, Loaded, not_found, Opts)),
    -    ?assertEqual(
    -        ?FIXTURE_SELLER,
    -        hb_ao:get(<<"initial-holder">>, Loaded, not_found, Opts)
    -    ),
    -    ?assertEqual(
    -        <<"name-token@1.0">>,
    -        hb_ao:get(<<"execution-device">>, Loaded, not_found, Opts)
    -    ).
    -
     %%% Story one, replayed: a name that had to be paid for
     %%%
     %%% Every transaction below is on mainnet. The reads walk the process forward
    @@ -1782,8 +907,8 @@ story_sync(Process, Opts, Attempts) ->
     
     %% @doc The slot a transaction was given. The stories pin transaction ids
     %% rather than slot numbers, because a slot number is a fact about the weave --
    -%% every transaction on the network takes one -- while the id is the message
    -%% itself.
    +%% every data-free transaction on the network takes one -- while the id is the
    +%% message itself.
     slot_of(Schedule, TXID, Opts) ->
         Assignments =
             hb_ao:normalize_keys(hb_ao:get(<<"assignments">>, Schedule, Opts), Opts),
    @@ -2026,3 +1151,75 @@ handover_story() ->
         Spoken = Slot(?HANDOVER_NEW_SET),
         ?assertEqual({ok, <<"buyer">>}, at(Process, Spoken, <<"value/points-at">>, Opts)),
         ?assertEqual({ok, 1}, at(Process, Spoken, <<"balances/", Buyer/binary>>, Opts)).
    +
    +%%% Story four: a name, read as a name.
    +%%%
    +%%% The three stories above are read slot by slot, through
    +%%% `~process@1.0/compute&slot='. This one is read the way a person reads a
    +%%% name: `~name@1.0' resolves the label to the process, and `/now' computes the
    +%%% whole schedule at once rather than one slot at a time. The name really was
    +%%% sold -- the buyer holds the unit and has said what it means -- and the
    +%%% payment that did it was an ordinary transfer between two addresses that the
    +%%% process was never a party to.
    +-define(NAMED_PROCESS, <<"yWRe7v4SZ4_NKV6LkYyNPrFdzEaGh0ckblu-CaGXqG4">>).
    +-define(NAMED_ORDER, <<"3BApJHeatc9pVuLgjZ_P-HT5hZgRE1Q3I1bdTESgRDM">>).
    +-define(NAMED_MAX_HEIGHT, 1966044).
    +
    +name_resolution_test_() -> {timeout, 1800, fun name_resolution/0}.
    +name_resolution() ->
    +    Opts = story_opts(?NAMED_MAX_HEIGHT, ?NAMED_PROCESS),
    +    % A node that serves a name holds it: priming the schedule puts the process
    +    % in the node's own cache, which is what the resolver then loads.
    +    {ok, _} = story_sync(?NAMED_PROCESS, Opts, 5),
    +    ?assertEqual(
    +        {ok, ?NAMED_PROCESS},
    +        hb_ao:resolve_many(
    +            [
    +                #{ <<"device">> => <<"name@1.0">> },
    +                #{ <<"path">> => <<"test-name">>, <<"load">> => false }
    +            ],
    +            Opts
    +        )
    +    ),
    +    % `test-name.host' is the same lookup: the node's own host is stripped from
    +    % the request's host, leaving the label to resolve.
    +    {ok, Resolved} =
    +        hb_ao:resolve(
    +            #{ <<"device">> => <<"name@1.0">> },
    +            #{
    +                <<"path">> => <<"request">>,
    +                <<"request">> => #{ <<"host">> => <<"test-name.host">> },
    +                <<"body">> => [#{ <<"path">> => <<"now">> }]
    +            },
    +            Opts
    +        ),
    +    [Named | _] = hb_ao:get(<<"body">>, Resolved, [], Opts),
    +    Loaded = hb_cache:ensure_all_loaded(Named, Opts),
    +    % The message the host resolved to is this name: it carries the name's own
    +    % spawn keys, and nothing else on the weave does.
    +    ?assertEqual(<<"test-name">>, hb_ao:get(<<"name">>, Loaded, not_found, Opts)),
    +    ?assertEqual(
    +        ?SALE_SELLER,
    +        hb_ao:get(<<"initial-holder">>, Loaded, not_found, Opts)
    +    ),
    +    ?assertEqual(
    +        <<"name-token@1.0">>,
    +        hb_ao:get(<<"execution-device">>, Loaded, not_found, Opts)
    +    ),
    +    % And the whole schedule, computed at once: the name is the buyer's, the
    +    % order it was sold by has left the book, and the new owner has spoken. The
    +    % process is read from the node's cache, so that it is its canonical `tx@1.0'
    +    % decoding rather than a gateway store's lossier one.
    +    {ok, Raw} = hb_cache:read(?NAMED_PROCESS, Opts),
    +    {ok, State} =
    +        hb_ao:resolve(hb_cache:ensure_all_loaded(Raw, Opts), <<"now">>, Opts),
    +    Read =
    +        fun(Path) ->
    +            hb_ao:get(Path, {as, <<"message@1.0">>, State}, not_found, Opts)
    +        end,
    +    ?assertEqual(1, hb_util:int(Read([?BALANCES, ?SALE_BUYER]))),
    +    ?assertEqual(0, hb_util:int(Read([?BALANCES, ?SALE_SELLER]))),
    +    ?assertEqual(1, hb_util:int(Read(<<"total-supply">>))),
    +    ?assertEqual(not_found, Read([<<"orders">>, ?NAMED_ORDER, <<"creator">>])),
    +    ?assertEqual(<<"hello from the new owner">>, Read([?VALUE, <<"greeting">>])),
    +    ?assertEqual(<<"text/plain">>, Read([?VALUE, <<"content-type">>])).
    
    From 62140749ddfc5373cd6488fc0010eeb28a993ea2 Mon Sep 17 00:00:00 2001
    From: Jack Frain 
    Date: Mon, 27 Jul 2026 15:09:32 -0400
    Subject: [PATCH 24/27] feat: isolate process store
    
    ---
     src/preloaded/process/dev_process_cache.erl | 41 +++++++++++++++------
     src/preloaded/process/lib_process.erl       | 13 +++++++
     2 files changed, 42 insertions(+), 12 deletions(-)
    
    diff --git a/src/preloaded/process/dev_process_cache.erl b/src/preloaded/process/dev_process_cache.erl
    index b2617ee6ee..5ca9d97ed2 100644
    --- a/src/preloaded/process/dev_process_cache.erl
    +++ b/src/preloaded/process/dev_process_cache.erl
    @@ -10,13 +10,15 @@
     %% @doc Read the result of a process at a given slot.
     read(ProcID, Opts) ->
         hb_util:ok(latest(ProcID, Opts)).
    -read(ProcID, SlotRef, Opts) ->
    +read(ProcID, SlotRef, RawOpts) ->
    +    Opts = lib_process:cache_opts(RawOpts),
         ?event({reading_computed_result, ProcID, SlotRef}),
         Path = path(ProcID, SlotRef, Opts),
         hb_cache:read(Path, Opts).
     
     %% @doc Write a process computation result to the cache.
    -write(ProcID, Slot, Msg, Opts) ->
    +write(ProcID, Slot, Msg, RawOpts) ->
    +    Opts = lib_process:cache_opts(RawOpts),
         % Write the item to the cache in the root of the store.
         {ok, Root} = hb_cache:write(hb_private:reset(Msg), Opts),
         % Link the item to the path in the store by slot number.
    @@ -65,16 +67,7 @@ latest(ProcID, Opts) -> latest(ProcID, [], Opts).
     latest(ProcID, RequiredPath, Opts) ->
         latest(ProcID, RequiredPath, undefined, Opts).
     latest(ProcID, RawRequiredPath, Limit, RawOpts) ->
    -    Scope = hb_opts:get(process_cache_scope, local, RawOpts),
    -    % Normalize the store descriptor to a list of stores.
    -    UnscopedStore =
    -        case hb_opts:get(store, no_viable_store, RawOpts) of
    -            StoreMsg when is_map(StoreMsg) -> [StoreMsg];
    -            Other -> Other
    -        end,
    -    % Apply the scope to the store and update the options message.
    -    ScopedStore = hb_store:scope(UnscopedStore, Scope),
    -    Opts = RawOpts#{ <<"store">> => ScopedStore },
    +    Opts = lib_process:cache_opts(RawOpts),
         % Convert the required path to a list of _binary_ keys.
         RequiredPath =
             case RawRequiredPath of
    @@ -230,3 +223,27 @@ find_latest_outputs(Opts) ->
         ?event(read_latest_slot_with_deep_key),
         {ok, 1, ReadBase} = latest(ProcID, [], 1, Opts),
         ?assert(hb_message:match(Base, ReadBase)).
    +
    +%% @doc Process cache writes go only to `process-store' when configured.
    +isolated_process_store_test() ->
    +    MainStore = hb_test_utils:test_store(hb_store_fs, <<"process-main">>),
    +    ProcessStore = hb_test_utils:test_store(hb_store_fs, <<"process-isolated">>),
    +    Opts = #{
    +        <<"store">> => [MainStore],
    +        <<"process-store">> => [ProcessStore]
    +    },
    +    hb_store:start(MainStore),
    +    hb_store:start(ProcessStore),
    +    ProcID = hb_util:human_id(crypto:strong_rand_bytes(32)),
    +    Msg = #{ <<"results">> => #{ <<"ok">> => <<"stored">> } },
    +    {ok, Path} = write(ProcID, 1, Msg, Opts),
    +    MainOpts = #{ <<"store">> => [MainStore] },
    +    ?assertMatch({error, not_found}, hb_cache:read(Path, MainOpts)),
    +    ?assertMatch({ok, _}, read(ProcID, 1, Opts)),
    +    ?assertMatch({ok, 1, _}, latest(ProcID, Opts)),
    +    hb_store:reset(MainStore),
    +    ?assertMatch({ok, _}, read(ProcID, 1, Opts)),
    +    ?assertMatch({ok, 1, _}, latest(ProcID, Opts)),
    +    hb_store:reset(ProcessStore),
    +    ?assertMatch({error, not_found}, read(ProcID, 1, Opts)),
    +    ?assertMatch({error, not_found}, latest(ProcID, Opts)).
    diff --git a/src/preloaded/process/lib_process.erl b/src/preloaded/process/lib_process.erl
    index c86c1f6f84..1096a09cb4 100644
    --- a/src/preloaded/process/lib_process.erl
    +++ b/src/preloaded/process/lib_process.erl
    @@ -6,6 +6,7 @@
         as_process/2,
         run_as/4,
         process_id/3,
    +    cache_opts/1,
         set_results/3,
         ensure_process_key/2,
         default_device/3
    @@ -34,6 +35,18 @@ process_id(Base, Req, Opts) ->
                 end
         end.
     
    +%% @doc Merge the process store with the main store. Used before reading
    +%% from or writing to a process cache.
    +cache_opts(Opts) ->
    +    Opts#{
    +        <<"store">> =>
    +            hb_opts:get(
    +                process_store,
    +                hb_opts:get(store, no_viable_store, Opts),
    +                Opts
    +            )
    +    }.
    +
     %% @doc Run a message against Base, with the device being swapped out for
     %% the device found at `Key'. After execution, the device is swapped back
     %% to the original device if the device is the same as we left it.
    
    From 412d77afffcf99f08169d5e4287876000fac33df Mon Sep 17 00:00:00 2001
    From: Jack Frain 
    Date: Mon, 27 Jul 2026 15:32:01 -0400
    Subject: [PATCH 25/27] fix: add back process store scope
    
    ---
     src/preloaded/process/dev_process_cache.erl | 12 +++++++++++-
     1 file changed, 11 insertions(+), 1 deletion(-)
    
    diff --git a/src/preloaded/process/dev_process_cache.erl b/src/preloaded/process/dev_process_cache.erl
    index 5ca9d97ed2..3584a8f462 100644
    --- a/src/preloaded/process/dev_process_cache.erl
    +++ b/src/preloaded/process/dev_process_cache.erl
    @@ -67,7 +67,17 @@ latest(ProcID, Opts) -> latest(ProcID, [], Opts).
     latest(ProcID, RequiredPath, Opts) ->
         latest(ProcID, RequiredPath, undefined, Opts).
     latest(ProcID, RawRequiredPath, Limit, RawOpts) ->
    -    Opts = lib_process:cache_opts(RawOpts),
    +    CacheOpts = lib_process:cache_opts(RawOpts),
    +    Scope = hb_opts:get(process_cache_scope, local, CacheOpts),
    +    % Normalize the selected store descriptor to a list of stores.
    +    UnscopedStore =
    +        case hb_opts:get(store, no_viable_store, CacheOpts) of
    +            StoreMsg when is_map(StoreMsg) -> [StoreMsg];
    +            Other -> Other
    +        end,
    +    % Apply the scope to the process store and update the options message.
    +    ScopedStore = hb_store:scope(UnscopedStore, Scope),
    +    Opts = CacheOpts#{ <<"store">> => ScopedStore },
         % Convert the required path to a list of _binary_ keys.
         RequiredPath =
             case RawRequiredPath of
    
    From 6674fc265fddce6fe3b862f251bd2d981da0d1b6 Mon Sep 17 00:00:00 2001
    From: Jack Frain 
    Date: Mon, 27 Jul 2026 17:02:45 -0400
    Subject: [PATCH 26/27] fix: fix process cache test suite to run on module call
    
    ---
     src/preloaded/process/dev_process_cache.erl | 93 +++++++++++++--------
     1 file changed, 59 insertions(+), 34 deletions(-)
    
    diff --git a/src/preloaded/process/dev_process_cache.erl b/src/preloaded/process/dev_process_cache.erl
    index 3584a8f462..b7d081a8b3 100644
    --- a/src/preloaded/process/dev_process_cache.erl
    +++ b/src/preloaded/process/dev_process_cache.erl
    @@ -129,43 +129,54 @@ latest(ProcID, RawRequiredPath, Limit, RawOpts) ->
         end.
     
     %% @doc Find the latest assignment with the requested path suffix.
    -first_with_path(ProcID, RequiredPath, Slots, Opts) ->
    -    first_with_path(
    -        ProcID,
    -        RequiredPath,
    -        Slots,
    -        Opts,
    -        hb_opts:get(store, no_viable_store, Opts)
    -    ).
    -first_with_path(_ProcID, _Required, [], _Opts, _Store) ->
    +first_with_path(_ProcID, _Required, [], _Opts) ->
         not_found;
    -first_with_path(ProcID, RequiredPath, [Slot | Rest], Opts, Store) ->
    -    RawPath = path(ProcID, Slot, RequiredPath, Opts),
    +first_with_path(ProcID, RequiredPath, [Slot | Rest], Opts) ->
    +    RawPath = path(ProcID, Slot, Opts),
         ?event({trying_slot, {slot, Slot}, {path, RawPath}}),
    -    case hb_store:read(Store, RawPath, Opts) of
    +    case hb_cache:read(RawPath, Opts) of
             {error, not_found} ->
    -            first_with_path(ProcID, RequiredPath, Rest, Opts, Store);
    +            first_with_path(ProcID, RequiredPath, Rest, Opts);
             {failure, _} = Failure ->
                 Failure;
             {error, _} = Error ->
                 Error;
    -        _ ->
    -            Slot
    +        {ok, Msg} ->
    +            case path_exists(RequiredPath, hb_cache:ensure_all_loaded(Msg, Opts)) of
    +                true -> Slot;
    +                false -> first_with_path(ProcID, RequiredPath, Rest, Opts)
    +            end
         end.
     
    +path_exists([], _Msg) ->
    +    true;
    +path_exists([Key | Rest], Msg) when is_map(Msg) ->
    +    case maps:find(Key, Msg) of
    +        {ok, Next} -> path_exists(Rest, Next);
    +        error -> false
    +    end;
    +path_exists(_Path, _Msg) ->
    +    false.
    +
     %%% Tests
     
     process_cache_suite_test_() ->
         hb_store:generate_test_suite(
             [
    -            {"write and read process outputs", fun test_write_and_read_output/1},
    -            {"find latest output (with path)", fun find_latest_outputs/1}
    +            {
    +                "write and read process outputs",
    +                fun(Store) ->
    +                    test_write_and_read_output(#{ <<"store">> => [Store] })
    +                end
    +            },
    +            {
    +                "find latest output (with path)",
    +                fun(Store) ->
    +                    find_latest_outputs(#{ <<"store">> => [Store] })
    +                end
    +            }
             ],
    -        [
    -            {Name, Opts}
    -        ||
    -            {Name, Opts} <- hb_store:test_stores()
    -        ]
    +        hb_store:test_stores()
         ).
     
     %% @doc Test for writing multiple computed outputs, then getting them by
    @@ -199,20 +210,20 @@ find_latest_outputs(Opts) ->
         Store = hb_opts:get(store, no_viable_store, Opts),
         ResetRes = hb_store:reset(Store),
         ?event({reset_store, {result, ResetRes}, {store, Store}}),
    -    Proc1 = hb_process_test_vectors:aos_process(),
    -    ProcID = hb_util:human_id(hb_ao:get(id, Proc1, Opts)),
    +    ProcID = hb_util:human_id(crypto:strong_rand_bytes(32)),
    +    ProcessRef = <<"test-process-ref">>,
         % Create messages for the slots, with only the middle slot having a
         % `/Process' field, while the top slot has a `/Deep/Process' field.
         Msg0 = #{ <<"Results">> => #{ <<"Result-Number">> => 0 } },
         Base =
             #{ 
                 <<"Results">> => #{ <<"Result-Number">> => 1 }, 
    -            <<"Process">> => Proc1 
    +            <<"Process">> => ProcessRef
             },
         Req =
             #{ 
                 <<"Results">> => #{ <<"Result-Number">> => 2 }, 
    -            <<"Deep">> => #{ <<"Process">> => Proc1 } 
    +            <<"Deep">> => #{ <<"Process">> => ProcessRef }
             },
         % Write the messages to the cache.
         {ok, _} = write(ProcID, 0, Msg0, Opts),
    @@ -220,19 +231,33 @@ find_latest_outputs(Opts) ->
         {ok, _} = write(ProcID, 2, Req, Opts),
         ?event(wrote_items),
         % Read the messages with various qualifiers.
    -    {ok, 2, ReadReq} = latest(ProcID, Opts),
    +    {ok, 2, RawReadReq} = latest(ProcID, Opts),
    +    ReadReq = hb_cache:ensure_all_loaded(RawReadReq, Opts),
         ?event({read_latest, ReadReq}),
    -    ?assert(hb_message:match(Req, ReadReq)),
    +    ?assertEqual(2, maps:get(<<"Result-Number">>, maps:get(<<"Results">>, ReadReq))),
    +    ?assertEqual(ProcessRef, maps:get(<<"Process">>, maps:get(<<"Deep">>, ReadReq))),
         ?event(read_latest_slot_without_qualifiers),
    -    {ok, 1, ReadBaseRequired} = latest(ProcID, <<"Process">>, Opts),
    +    {ok, 1, RawReadBaseRequired} = latest(ProcID, <<"Process">>, Opts),
    +    ReadBaseRequired = hb_cache:ensure_all_loaded(RawReadBaseRequired, Opts),
         ?event({read_latest_with_process, ReadBaseRequired}),
    -    ?assert(hb_message:match(Base, ReadBaseRequired)),
    +    ?assertEqual(
    +        1,
    +        maps:get(<<"Result-Number">>, maps:get(<<"Results">>, ReadBaseRequired))
    +    ),
    +    ?assertEqual(ProcessRef, maps:get(<<"Process">>, ReadBaseRequired)),
         ?event(read_latest_slot_with_shallow_key),
    -    {ok, 2, ReadReqRequired} = latest(ProcID, <<"Deep/Process">>, Opts),
    -    ?assert(hb_message:match(Req, ReadReqRequired)),
    +    {ok, 2, RawReadReqRequired} = latest(ProcID, <<"Deep/Process">>, Opts),
    +    ReadReqRequired = hb_cache:ensure_all_loaded(RawReadReqRequired, Opts),
    +    ?assertEqual(
    +        2,
    +        maps:get(<<"Result-Number">>, maps:get(<<"Results">>, ReadReqRequired))
    +    ),
    +    ?assertEqual(ProcessRef, maps:get(<<"Process">>, maps:get(<<"Deep">>, ReadReqRequired))),
         ?event(read_latest_slot_with_deep_key),
    -    {ok, 1, ReadBase} = latest(ProcID, [], 1, Opts),
    -    ?assert(hb_message:match(Base, ReadBase)).
    +    {ok, 1, RawReadBase} = latest(ProcID, [], 1, Opts),
    +    ReadBase = hb_cache:ensure_all_loaded(RawReadBase, Opts),
    +    ?assertEqual(1, maps:get(<<"Result-Number">>, maps:get(<<"Results">>, ReadBase))),
    +    ?assertEqual(ProcessRef, maps:get(<<"Process">>, ReadBase)).
     
     %% @doc Process cache writes go only to `process-store' when configured.
     isolated_process_store_test() ->
    
    From 36418d4d93eda0e073c993b7295c1a6d9657d963 Mon Sep 17 00:00:00 2001
    From: Jack Frain 
    Date: Mon, 27 Jul 2026 17:56:34 -0400
    Subject: [PATCH 27/27] feat: max age for cached process now requests
    
    ---
     src/core/resolver/hb_opts.erl               |   2 +
     src/preloaded/process/dev_process.erl       |  71 +++++---
     src/preloaded/process/dev_process_cache.erl | 175 ++++++++++++++++++--
     src/preloaded/process/lib_process.erl       |  42 +++++
     4 files changed, 253 insertions(+), 37 deletions(-)
    
    diff --git a/src/core/resolver/hb_opts.erl b/src/core/resolver/hb_opts.erl
    index b1acd6be82..569ea86d66 100644
    --- a/src/core/resolver/hb_opts.erl
    +++ b/src/core/resolver/hb_opts.erl
    @@ -484,6 +484,8 @@ raw_default_message() ->
             % default_index => #{ <<"device">> => <<"hyperbuddy@1.0">> },
             % Should we use the latest cached state of a process when computing?
             <<"process-now-from-cache">> => false,
    +        % Maximum age, in seconds, for `/now' to serve from the process cache.
    +        <<"process-now-max-age">> => infinity,
             % Should we trust the GraphQL API when converting to ANS-104? Some GQL
             % services do not provide the `anchor' or `last_tx' fields, so their
             % responses are not verifiable.
    diff --git a/src/preloaded/process/dev_process.erl b/src/preloaded/process/dev_process.erl
    index 0e89957584..85b19caf36 100644
    --- a/src/preloaded/process/dev_process.erl
    +++ b/src/preloaded/process/dev_process.erl
    @@ -646,35 +646,56 @@ now(RawBase, Req, Opts) ->
                 LatestKnown = dev_process_cache:latest(ProcessID, [], Opts),
                 case LatestKnown of
                     {ok, LatestSlot, RawLatestMsg} ->
    -                    LatestMsg = without_snapshot(RawLatestMsg, Opts),
    -                    ?event(compute_cache,
    -                        {serving_latest_cached_state,
    -                            {proc_id, ProcessID},
    -                            {slot, LatestSlot}
    -                        },
    -                        Opts
    -                    ),
    -                    dev_process_worker:notify_compute(
    -                        ProcessID,
    -                        LatestSlot,
    -                        {ok, LatestMsg},
    -                        Opts
    -                    ),
    -                    {ok, LatestMsg};
    +                    case dev_process_cache:fresh(ProcessID, Req, Opts) of
    +                        true ->
    +                            LatestMsg = without_snapshot(RawLatestMsg, Opts),
    +                            ?event(compute_cache,
    +                                {serving_latest_cached_state,
    +                                    {proc_id, ProcessID},
    +                                    {slot, LatestSlot}
    +                                },
    +                                Opts
    +                            ),
    +                            dev_process_worker:notify_compute(
    +                                ProcessID,
    +                                LatestSlot,
    +                                {ok, LatestMsg},
    +                                Opts
    +                            ),
    +                            {ok, LatestMsg};
    +                        false ->
    +                            ?event(compute_cache,
    +                                {latest_cached_state_stale,
    +                                    {proc_id, ProcessID},
    +                                    {slot, LatestSlot}
    +                                },
    +                                Opts
    +                            ),
    +                            uncached_now(
    +                                CacheParam,
    +                                Base,
    +                                Req,
    +                                Opts,
    +                                <<"No fresh cached state available.">>
    +                            )
    +                    end;
                     _ ->
    -                    if CacheParam =/= always ->
    -                        % The node is configured to use the cache if possible,
    -                        % but forcing computation is also admissible. Subsequently,
    -                        % as no other option is available, we compute the state.
    -                        now(Base, Req, Opts#{ <<"process-now-from-cache">> => false });
    -                    true ->
    -                        % The node is configured to only serve the latest known
    -                        % state from the cache, so we return the latest slot.
    -                        {failure, <<"No cached state available.">>}
    -                    end
    +                    uncached_now(CacheParam, Base, Req, Opts)
                 end
         end.
     
    +uncached_now(CacheParam, Base, Req, Opts) ->
    +    uncached_now(CacheParam, Base, Req, Opts, <<"No cached state available.">>).
    +
    +uncached_now(always, _Base, _Req, _Opts, Failure) ->
    +    {failure, Failure};
    +uncached_now(<<"always">>, _Base, _Req, _Opts, Failure) ->
    +    {failure, Failure};
    +uncached_now(_CacheParam, Base, Req, Opts, _Failure) ->
    +    % The node is configured to use the cache if possible, but forcing
    +    % computation is also admissible.
    +    now(Base, Req, Opts#{ <<"process-now-from-cache">> => false }).
    +
     %% @doc Recursively push messages to the scheduler until we find a message
     %% that does not lead to any further messages being scheduled.
     push(Base, Req, Opts) ->
    diff --git a/src/preloaded/process/dev_process_cache.erl b/src/preloaded/process/dev_process_cache.erl
    index b7d081a8b3..d969e7ea20 100644
    --- a/src/preloaded/process/dev_process_cache.erl
    +++ b/src/preloaded/process/dev_process_cache.erl
    @@ -3,7 +3,7 @@
     %%% convenient interface for reading the result of a process at a given slot or
     %%% message ID.
     -module(dev_process_cache).
    --export([latest/2, latest/3, latest/4, read/2, read/3, write/4]).
    +-export([fresh/3, latest/2, latest/3, latest/4, read/2, read/3, write/4]).
     -include_lib("eunit/include/eunit.hrl").
     -include("include/hb.hrl").
     
    @@ -40,9 +40,77 @@ write(ProcID, Slot, Msg, RawOpts) ->
             }
         ),
         hb_cache:link(Root, MsgIDPath, Opts),
    +    write_refreshed_at(ProcID, Opts),
         % Return the slot number path.
         {ok, SlotNumPath}.
     
    +%% @doc Mark the process as refreshed at the current clock time.
    +write_refreshed_at(ProcID, Opts) ->
    +    Store = hb_opts:get(store, no_viable_store, Opts),
    +    ok = hb_store:write(
    +        Store,
    +        #{ refreshed_path(ProcID) => hb_util:bin(clock(Opts)) },
    +        Opts
    +    ).
    +
    +refreshed_path(ProcID) ->
    +    path(ProcID, <<"refreshed-at">>, #{}).
    +
    +%% @doc Return whether the latest cached process output is fresh enough for
    +%% `/now' to serve from cache under the effective `max-age'.
    +fresh(ProcID, Req, RawOpts) ->
    +    Opts = lib_process:scoped_opts(RawOpts),
    +    case effective_max_age(Req, Opts) of
    +        infinity ->
    +            true;
    +        MaxAge ->
    +            case read_refreshed_at(ProcID, Opts) of
    +                undefined -> false;
    +                RefreshedAt -> clock(Opts) =< RefreshedAt + MaxAge
    +            end
    +    end.
    +
    +%% @doc Read the timestamp of the last refresh of a process.
    +read_refreshed_at(ProcID, Opts) ->
    +    Store = hb_opts:get(store, no_viable_store, Opts),
    +    case hb_store:read(Store, refreshed_path(ProcID), Opts) of
    +        {ok, Timestamp} -> hb_util:int(Timestamp);
    +        _ -> undefined
    +    end.
    +
    +%% @doc Calculate the effective maximum age of a process cache entry.
    +effective_max_age(Req, Opts) ->
    +    case lib_process:only_if_cached(Req, Opts) of
    +        true ->
    +            infinity;
    +        false ->
    +            case max_age_from_request(Req, Opts) of
    +                {ok, MaxAge} ->
    +                    normalize_max_age(MaxAge);
    +                error ->
    +                    normalize_max_age(
    +                        hb_opts:get(process_now_max_age, infinity, Opts)
    +                    )
    +            end
    +    end.
    +
    +max_age_from_request(Req, Opts) when is_map(Req) ->
    +    hb_maps:find(<<"max-age">>, Req, Opts);
    +max_age_from_request(_Req, _Opts) ->
    +    error.
    +
    +normalize_max_age(infinity) -> infinity;
    +normalize_max_age(<<"infinity">>) -> infinity;
    +normalize_max_age(RawMaxAge) -> hb_util:int(RawMaxAge).
    +
    +%% @doc Return the current clock time. Allows the option to override the clock
    +%% time with a custom value for test use.
    +clock(Opts) ->
    +    case hb_opts:get(process_clock, undefined, Opts) of
    +        undefined -> erlang:system_time(second);
    +        Time -> hb_util:int(Time)
    +    end.
    +
     %% @doc Calculate the path of a result, given a process ID and a slot.
     path(ProcID, Ref, Opts) ->
         path(ProcID, Ref, [], Opts).
    @@ -67,17 +135,7 @@ latest(ProcID, Opts) -> latest(ProcID, [], Opts).
     latest(ProcID, RequiredPath, Opts) ->
         latest(ProcID, RequiredPath, undefined, Opts).
     latest(ProcID, RawRequiredPath, Limit, RawOpts) ->
    -    CacheOpts = lib_process:cache_opts(RawOpts),
    -    Scope = hb_opts:get(process_cache_scope, local, CacheOpts),
    -    % Normalize the selected store descriptor to a list of stores.
    -    UnscopedStore =
    -        case hb_opts:get(store, no_viable_store, CacheOpts) of
    -            StoreMsg when is_map(StoreMsg) -> [StoreMsg];
    -            Other -> Other
    -        end,
    -    % Apply the scope to the process store and update the options message.
    -    ScopedStore = hb_store:scope(UnscopedStore, Scope),
    -    Opts = CacheOpts#{ <<"store">> => ScopedStore },
    +    Opts = lib_process:scoped_opts(RawOpts),
         % Convert the required path to a list of _binary_ keys.
         RequiredPath =
             case RawRequiredPath of
    @@ -174,6 +232,12 @@ process_cache_suite_test_() ->
                     fun(Store) ->
                         find_latest_outputs(#{ <<"store">> => [Store] })
                     end
    +            },
    +            {
    +                "honor max-age when checking process cache freshness",
    +                fun(Store) ->
    +                    freshness_max_age(#{ <<"store">> => [Store] })
    +                end
                 }
             ],
             hb_store:test_stores()
    @@ -259,6 +323,93 @@ find_latest_outputs(Opts) ->
         ?assertEqual(1, maps:get(<<"Result-Number">>, maps:get(<<"Results">>, ReadBase))),
         ?assertEqual(ProcessRef, maps:get(<<"Process">>, ReadBase)).
     
    +%% @doc Test for serving `/now' from cache only while the cache is fresh enough.
    +freshness_max_age(Opts) ->
    +    ProcID = hb_util:human_id(crypto:strong_rand_bytes(32)),
    +    Slot = 1,
    +    SlotResult = #{
    +        <<"device">> => <<"process@1.0">>,
    +        <<"at-slot">> => Slot,
    +        <<"results">> => #{ <<"number">> => 1 }
    +    },
    +    % Assert that the process is not fresh by default.
    +    ?assertEqual(
    +        false,
    +        fresh(
    +            ProcID,
    +            #{ <<"max-age">> => 60 },
    +            Opts#{ <<"process-clock">> => 100 }
    +        )
    +    ),
    +    % Write the slot result to the cache at clock time 100.
    +    {ok, _} = write(ProcID, Slot, SlotResult, Opts#{ <<"process-clock">> => 100 }),
    +    {ok, 1, RawReadSlotResult} = latest(ProcID, Opts),
    +    ReadSlotResult = hb_cache:ensure_all_loaded(RawReadSlotResult, Opts),
    +    ?assertEqual(<<"process@1.0">>, maps:get(<<"device">>, ReadSlotResult)),
    +    ?assertEqual(1, maps:get(<<"number">>, maps:get(<<"results">>, ReadSlotResult))),
    +    % Assert that the process is fresh exactly at the max-age.
    +    ?assertEqual(
    +        true,
    +        fresh(
    +            ProcID,
    +            #{ <<"max-age">> => 60 },
    +            Opts#{ <<"process-clock">> => 160 }
    +        )
    +    ),
    +    % Assert that the process is not fresh after the max-age.
    +    ?assertEqual(
    +        false,
    +        fresh(
    +            ProcID,
    +            #{ <<"max-age">> => 60 },
    +            Opts#{ <<"process-clock">> => 161 }
    +        )
    +    ),
    +    % Assert that the process is fresh if the max-age is infinity.
    +    ?assertEqual(
    +        true,
    +        fresh(
    +            ProcID,
    +            #{ <<"max-age">> => <<"infinity">> },
    +            Opts#{ <<"process-clock">> => 1000 }
    +        )
    +    ),
    +    % Assert that the process is fresh if the only-if-cached flag is set.
    +    ?assertEqual(
    +        true,
    +        fresh(
    +            ProcID,
    +            #{
    +                <<"cache-control">> => [<<"only-if-cached">>],
    +                <<"max-age">> => 0
    +            },
    +            Opts#{ <<"process-clock">> => 1000 }
    +        )
    +    ),
    +    % Assert that the max age is read as a fallback from the node opts
    +    ?assertEqual(
    +        true,
    +        fresh(
    +            ProcID,
    +            #{},
    +            Opts#{
    +                <<"process-clock">> => 160,
    +                <<"process-now-max-age">> => 60
    +            }
    +        )
    +    ),
    +    ?assertEqual(
    +        false,
    +        fresh(
    +            ProcID,
    +            #{},
    +            Opts#{
    +                <<"process-clock">> => 161,
    +                <<"process-now-max-age">> => 60
    +            }
    +        )
    +    ).
    +
     %% @doc Process cache writes go only to `process-store' when configured.
     isolated_process_store_test() ->
         MainStore = hb_test_utils:test_store(hb_store_fs, <<"process-main">>),
    diff --git a/src/preloaded/process/lib_process.erl b/src/preloaded/process/lib_process.erl
    index 1096a09cb4..270a7a2180 100644
    --- a/src/preloaded/process/lib_process.erl
    +++ b/src/preloaded/process/lib_process.erl
    @@ -7,6 +7,10 @@
         run_as/4,
         process_id/3,
         cache_opts/1,
    +    cache_control/2,
    +    normalize_cache_control/1,
    +    only_if_cached/2,
    +    scoped_opts/1,
         set_results/3,
         ensure_process_key/2,
         default_device/3
    @@ -47,6 +51,44 @@ cache_opts(Opts) ->
                 )
         }.
     
    +%% @doc Apply the process cache store and configured store scope to opts.
    +scoped_opts(RawOpts) ->
    +    CacheOpts = cache_opts(RawOpts),
    +    Scope = hb_opts:get(process_cache_scope, local, CacheOpts),
    +    % Normalize the selected store descriptor to a list of stores.
    +    UnscopedStore =
    +        case hb_opts:get(store, no_viable_store, CacheOpts) of
    +            StoreMsg when is_map(StoreMsg) -> [StoreMsg];
    +            Other -> Other
    +        end,
    +    % Apply the scope to the process store and update the options message.
    +    CacheOpts#{ <<"store">> => hb_store:scope(UnscopedStore, Scope) }.
    +
    +%% @doc Return the effective cache-control directives from a request, falling
    +%% back to the node options when the request does not specify them.
    +cache_control(Req, Opts) when is_map(Req) ->
    +    normalize_cache_control(
    +        hb_maps:get(
    +            <<"cache-control">>,
    +            Req,
    +            hb_opts:get(cache_control, [], Opts),
    +            Opts
    +        )
    +    );
    +cache_control(_Req, Opts) ->
    +    normalize_cache_control(hb_opts:get(cache_control, [], Opts)).
    +
    +%% @doc Normalize cache-control values to the binary directive form used by
    +%% process devices.
    +normalize_cache_control(CC) when is_list(CC) ->
    +    lists:map(fun hb_ao:normalize_key/1, CC);
    +normalize_cache_control(CC) ->
    +    normalize_cache_control([CC]).
    +
    +%% @doc Return whether the request or node opts require cached results only.
    +only_if_cached(Req, Opts) ->
    +    lists:member(<<"only-if-cached">>, cache_control(Req, Opts)).
    +
     %% @doc Run a message against Base, with the device being swapped out for
     %% the device found at `Key'. After execution, the device is swapped back
     %% to the original device if the device is the same as we left it.