From 322883616359365c91378d12db43519894f40b7e Mon Sep 17 00:00:00 2001 From: speeddragon Date: Thu, 30 Jul 2026 15:15:20 +0100 Subject: [PATCH 1/4] feat: Add functionality to only copycat a specific TX ID to help debug flow --- src/preloaded/query/dev_copycat_arweave.erl | 63 ++++++++++++++++----- 1 file changed, 49 insertions(+), 14 deletions(-) diff --git a/src/preloaded/query/dev_copycat_arweave.erl b/src/preloaded/query/dev_copycat_arweave.erl index 481390829..c8c5ad125 100644 --- a/src/preloaded/query/dev_copycat_arweave.erl +++ b/src/preloaded/query/dev_copycat_arweave.erl @@ -113,7 +113,7 @@ latest_height(Opts) -> end. index_range(Request, true, From, To, IndexMode, Opts) -> - case index_pending(IndexMode, Opts) of + case index_pending(Request, IndexMode, Opts) of {ok, PendingRes} -> case block_range_empty(From, To) of true -> @@ -245,12 +245,14 @@ fetch_blocks(Req, Current, undefined, IndexMode, Opts) -> stop_at_indexed_block(Req, Current); false -> BlockRes = fetch_block_header(Current, Opts), - case IndexMode =:= shallow andalso is_already_indexed(BlockRes, Opts) of + case IndexMode =:= shallow andalso + is_already_indexed(BlockRes, Req, Opts) of true -> stop_at_indexed_block(Req, Current); false -> observe_event(<<"block_indexed">>, fun() -> - process_block(BlockRes, Current, undefined, IndexMode, Opts) + process_block( + BlockRes, Current, undefined, IndexMode, Req, Opts) end), fetch_blocks(Req, Current - 1, undefined, IndexMode, Opts) end @@ -260,7 +262,7 @@ fetch_blocks(Req, Current, To, IndexMode, Opts) -> % this mode, so overlapping ranges from different callers are not re-fetched. (reindex(Req, Opts) orelse not is_block_indexed(Current, IndexMode, Opts)) andalso observe_event(<<"block_indexed">>, fun() -> - process_block(fetch_block_header(Current, Opts), Current, To, IndexMode, Opts) + process_block(fetch_block_header(Current, Opts), Current, To, IndexMode, Req, Opts) end), fetch_blocks(Req, Current - 1, To, IndexMode, Opts). @@ -278,18 +280,21 @@ stop_at_indexed_block(Req, Current) -> ), {ok, Current}. -is_already_indexed({ok, Block}, Opts) -> +is_already_indexed({ok, Block}, Request, Opts) -> TXIDs = hb_maps:get(<<"txs">>, Block, [], Opts), - lists:any(fun(TXID) -> is_tx_indexed(TXID, Opts) end, TXIDs); -is_already_indexed({error, _}, _Opts) -> + lists:any( + fun(TXID) -> is_tx_indexed(TXID, Opts) end, + filter_txids(TXIDs, Request, Opts) + ); +is_already_indexed({error, _}, _Request, _Opts) -> false. -process_block(BlockRes, Current, To, IndexMode, Opts) -> +process_block(BlockRes, Current, To, IndexMode, Request, Opts) -> case BlockRes of {ok, Block} -> ?event(debug_copycat, {{processing_block, Current}, {indep_hash, hb_maps:get(<<"indep_hash">>, Block, <<>>)}}), - case maybe_index_ids(Block, IndexMode, Opts) of + case maybe_index_ids(Block, IndexMode, Request, Opts) of {block_skipped, Results} -> TotalTXs = maps:get(total_txs, Results, 0), ?event( @@ -367,9 +372,21 @@ mode_rank(<<"deep">>) -> mode_rank(deep); mode_rank(<<"full">>) -> mode_rank(full); mode_rank(_Other) -> 0. +%% @doc Restrict L1 transaction IDs to the request's optional `txid'. +filter_txids(TXIDs, Request, Opts) -> + case requested_txid(Request, Opts) of + undefined -> TXIDs; + TXID -> lists:filter(fun(ID) -> ID =:= TXID end, TXIDs) + end. + +%% @doc Return the request's optional L1 transaction ID filter. +requested_txid(Request, Opts) -> + hb_maps:get(<<"txid">>, Request, undefined, Opts). + %% @doc Index the IDs of all transactions in the block if configured to do so. -maybe_index_ids(Block, IndexMode, Opts) -> - TXIDs = hb_maps:get(<<"txs">>, Block, [], Opts), +maybe_index_ids(Block, IndexMode, Request, Opts) -> + TXIDs = filter_txids( + hb_maps:get(<<"txs">>, Block, [], Opts), Request, Opts), TotalTXs = length(TXIDs), case hb_opts:get(arweave_index_ids, true, Opts) of false -> @@ -604,15 +621,18 @@ index_full_bundle_bytes(BundleData, BundleStartOffset, IndexMode, Store, Opts) - end. %% @doc Index unconfirmed transactions from the Arweave mempool. -index_pending(IndexMode, Opts) -> +index_pending(Request, IndexMode, Opts) -> case hb_ao:resolve(<>, Opts) of {ok, TXIDs} when is_list(TXIDs) -> + FilteredTXIDs = filter_txids(TXIDs, Request, Opts), Results = parallel_map( - TXIDs, + FilteredTXIDs, fun(TXID) -> process_pending_tx(TXID, IndexMode, Opts) end, Opts ), - {ok, (sum_counters(Results))#{ total_txs => length(TXIDs) }}; + {ok, (sum_counters(Results))#{ + total_txs => length(FilteredTXIDs) + }}; Error -> Error end. @@ -907,11 +927,26 @@ index_ids_test_parallel() -> %% however we should still be able to index it (we just can't deserialize %% it). {_TestStore, StoreOpts, Opts} = setup_index_opts(), + FilteredTXID = <<"bXEgFm4K2b5VD64skBNAlS3I__4qxlM3Sm4Z5IXj3h8">>, + IgnoredTXID = <<"WbRAQbeyjPHgopBKyi0PLeKWvYZr3rgZvQ7QY3ASJS4">>, + {ok, 1827942} = + hb_ao:resolve( + << + "~copycat@1.0/arweave?mode=deep&from=1827942&to=1827942", + "&txid=", FilteredTXID/binary + >>, + Opts + ), + ?assert(is_tx_indexed(FilteredTXID, Opts)), + ?assertNot(is_tx_indexed(IgnoredTXID, Opts)), + ?assertNot(is_block_indexed(1827942, deep, Opts)), {ok, 1827942} = hb_ao:resolve( <<"~copycat@1.0/arweave&from=1827942&to=1827942">>, Opts ), + ?assert(is_tx_indexed(IgnoredTXID, Opts)), + ?assert(is_block_indexed(1827942, shallow, Opts)), ?assertMatch( {ok, _}, hb_store_arweave:read( From b710f3992b1b5937b2862c072ec8fde615d1b363 Mon Sep 17 00:00:00 2001 From: speeddragon Date: Thu, 30 Jul 2026 17:12:54 +0100 Subject: [PATCH 2/4] fix: Breaking index with filtering before calculate offsets with block information --- src/preloaded/query/dev_copycat_arweave.erl | 40 ++++++++++++--------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/src/preloaded/query/dev_copycat_arweave.erl b/src/preloaded/query/dev_copycat_arweave.erl index c8c5ad125..ece7c6757 100644 --- a/src/preloaded/query/dev_copycat_arweave.erl +++ b/src/preloaded/query/dev_copycat_arweave.erl @@ -246,7 +246,7 @@ fetch_blocks(Req, Current, undefined, IndexMode, Opts) -> false -> BlockRes = fetch_block_header(Current, Opts), case IndexMode =:= shallow andalso - is_already_indexed(BlockRes, Req, Opts) of + is_already_indexed(BlockRes, Opts) of true -> stop_at_indexed_block(Req, Current); false -> @@ -280,13 +280,13 @@ stop_at_indexed_block(Req, Current) -> ), {ok, Current}. -is_already_indexed({ok, Block}, Request, Opts) -> +is_already_indexed({ok, Block}, Opts) -> TXIDs = hb_maps:get(<<"txs">>, Block, [], Opts), lists:any( fun(TXID) -> is_tx_indexed(TXID, Opts) end, - filter_txids(TXIDs, Request, Opts) + TXIDs ); -is_already_indexed({error, _}, _Request, _Opts) -> +is_already_indexed({error, _}, _Opts) -> false. process_block(BlockRes, Current, To, IndexMode, Request, Opts) -> @@ -372,21 +372,16 @@ mode_rank(<<"deep">>) -> mode_rank(deep); mode_rank(<<"full">>) -> mode_rank(full); mode_rank(_Other) -> 0. -%% @doc Restrict L1 transaction IDs to the request's optional `txid'. -filter_txids(TXIDs, Request, Opts) -> - case requested_txid(Request, Opts) of - undefined -> TXIDs; - TXID -> lists:filter(fun(ID) -> ID =:= TXID end, TXIDs) - end. - %% @doc Return the request's optional L1 transaction ID filter. requested_txid(Request, Opts) -> - hb_maps:get(<<"txid">>, Request, undefined, Opts). + case hb_maps:get(<<"txid">>, Request, undefined, Opts) of + TXID when ?IS_ID(TXID) -> TXID; + _ -> undefined + end. %% @doc Index the IDs of all transactions in the block if configured to do so. maybe_index_ids(Block, IndexMode, Request, Opts) -> - TXIDs = filter_txids( - hb_maps:get(<<"txs">>, Block, [], Opts), Request, Opts), + TXIDs = hb_maps:get(<<"txs">>, Block, [], Opts), TotalTXs = length(TXIDs), case hb_opts:get(arweave_index_ids, true, Opts) of false -> @@ -412,9 +407,16 @@ maybe_index_ids(Block, IndexMode, Request, Opts) -> {ok, TXs} -> Height = hb_maps:get(<<"height">>, Block, 0, Opts), TXsWithData = ar_block:generate_size_tagged_list_from_txs(TXs, Height), - % Filter out padding entries before processing + % Filter out padding entries before processing. + % If a TXID is provided, filter by that. + RequestedTXID = requested_txid(Request, Opts), ValidTXs = lists:filter( - fun({{padding, _}, _}) -> false; (_) -> true end, + fun + ({{padding, _}, _}) -> false; + ({{_TX, _}, _}) when RequestedTXID == undefined -> true; + ({{TX, _}, _}) -> hb_util:encode(TX#tx.id) == RequestedTXID; + (_) -> true + end, TXsWithData ), TXResults = process_txs( @@ -624,7 +626,11 @@ index_full_bundle_bytes(BundleData, BundleStartOffset, IndexMode, Store, Opts) - index_pending(Request, IndexMode, Opts) -> case hb_ao:resolve(<>, Opts) of {ok, TXIDs} when is_list(TXIDs) -> - FilteredTXIDs = filter_txids(TXIDs, Request, Opts), + FilteredTXIDs = + case requested_txid(Request, Opts) of + undefined -> TXIDs; + TXID -> lists:filter(fun(ID) -> ID =:= TXID end, TXIDs) + end, Results = parallel_map( FilteredTXIDs, fun(TXID) -> process_pending_tx(TXID, IndexMode, Opts) end, From fc117d356fb1af30ed9db47f4235324e98c7508b Mon Sep 17 00:00:00 2001 From: speeddragon Date: Fri, 31 Jul 2026 17:38:56 +0100 Subject: [PATCH 3/4] fix: Tests --- src/preloaded/query/dev_copycat_arweave.erl | 113 +++++++------------- 1 file changed, 36 insertions(+), 77 deletions(-) diff --git a/src/preloaded/query/dev_copycat_arweave.erl b/src/preloaded/query/dev_copycat_arweave.erl index ece7c6757..d4864b419 100644 --- a/src/preloaded/query/dev_copycat_arweave.erl +++ b/src/preloaded/query/dev_copycat_arweave.erl @@ -310,9 +310,11 @@ process_block(BlockRes, Current, To, IndexMode, Request, Opts) -> TotalTXs = maps:get(total_txs, Results, 0), BundleTXs = maps:get(bundle_count, Results, 0), SkippedTXs = maps:get(skipped_count, Results, 0), - case SkippedTXs of - 0 -> ok = write_block_index(Current, IndexMode, Opts); - _ -> ok + case {requested_txid(Request, Opts), SkippedTXs} of + {undefined, 0} -> + ok = write_block_index(Current, IndexMode, Opts); + _ -> + ok end, ?event( copycat_short, @@ -926,92 +928,49 @@ observe_event(MetricName, Fun) -> %%% Tests -index_ids_test_parallel() -> - %% Test block: https://viewblock.io/arweave/block/1827942 - %% Note: this block includes a data item with an Ethereum signature. This - %% signature type is not yet (as of Jan 2026) supported by ar_bundles.erl, - %% however we should still be able to index it (we just can't deserialize - %% it). - {_TestStore, StoreOpts, Opts} = setup_index_opts(), - FilteredTXID = <<"bXEgFm4K2b5VD64skBNAlS3I__4qxlM3Sm4Z5IXj3h8">>, - IgnoredTXID = <<"WbRAQbeyjPHgopBKyi0PLeKWvYZr3rgZvQ7QY3ASJS4">>, - {ok, 1827942} = +filtered_txid_test_parallel() -> + {_TestStore, _StoreOpts, Opts} = setup_index_opts(), + FilteredTXID = <<"tiT3XhhgSvK39Lx40jniKD9CpbTTTDaQmGimrhFh-sw">>, + BlockHeight = 1969127, + BlockHeightBin = integer_to_binary(BlockHeight), + {ok, Block} = fetch_block_header(BlockHeight, Opts), + TXIDs = hb_maps:get(<<"txs">>, Block, [], Opts), + ?assert(lists:member(FilteredTXID, TXIDs)), + IgnoredTXIDs = lists:delete(FilteredTXID, TXIDs), + {ok, BlockHeight} = hb_ao:resolve( << - "~copycat@1.0/arweave?mode=deep&from=1827942&to=1827942", + "~copycat@1.0/arweave?mode=deep&from=", + BlockHeightBin/binary, "&to=", BlockHeightBin/binary, "&txid=", FilteredTXID/binary >>, Opts ), ?assert(is_tx_indexed(FilteredTXID, Opts)), - ?assertNot(is_tx_indexed(IgnoredTXID, Opts)), - ?assertNot(is_block_indexed(1827942, deep, Opts)), - {ok, 1827942} = - hb_ao:resolve( - <<"~copycat@1.0/arweave&from=1827942&to=1827942">>, - Opts - ), - ?assert(is_tx_indexed(IgnoredTXID, Opts)), - ?assert(is_block_indexed(1827942, shallow, Opts)), - ?assertMatch( - {ok, _}, - hb_store_arweave:read( - StoreOpts, - #{ <<"read">> => <<"WbRAQbeyjPHgopBKyi0PLeKWvYZr3rgZvQ7QY3ASJS4">> }, - Opts - ) + ?assertEqual( + [], + [TXID || TXID <- IgnoredTXIDs, is_tx_indexed(TXID, Opts)] ), - assert_item_read( - <<"0vy2Ey8bWkSDcRIvWQJjxDeVGYOrTSmYIIhBILJntY8">>, - Opts), - assert_item_read( - <<"2lmrYydmDweX2MgGH39ZEB9hKm2JqGOYmRiG3n_xh8A">>, - Opts), - assert_item_read( - <<"ATi9pQF_eqb99UK84R5rq8lGfRGpilVQOYyth7rXxh8">>, - Opts), - assert_item_read( - <<"4VSfUbhMVZQHW5VfVwQZOmC5fR3W21DZgFCyz8CA-cE">>, - Opts), - assert_item_read( - <<"ZQRHZhktk6dAtX9BlhO1teOtVlGHoyaWP25kAlhxrM4">>, - Opts), - % The T2pluNnaavL7-S2GkO_m3pASLUqMH_XQ9IiIhZKfySs can be deserialized so - % we'll verify that some of its items were index and match the version - % in the deserialized bundle. + ?assertNot(is_block_indexed(BlockHeight, deep, Opts)), assert_bundle_read( - <<"T2pluNnaavL7-S2GkO_m3pASLUqMH_XQ9IiIhZKfySs">>, - [ - {<<"54K1ehEIKZxGSusgZzgbGYaHfllwWQ09-S9-eRUJg5Y">>, <<"1">>}, - {<<"MgatoEjlO_YtdbxFi9Q7Hxbs0YQVcChddhSS7FsdeIg">>, <<"19">>}, - {<<"z-oKJfhMq5qoVFrljEfiBKgumaJmCWVxNJaavR5aPE8">>, <<"26">>} - ], + FilteredTXID, + [{<<"m8oR6EqaqhPqHZtKUCvDYWp3sPbMOQzxVf2VxpR7sWY">>, <<"1">>}], Opts ), - % Non-ans104 data transaction - assert_item_read( - <<"bXEgFm4K2b5VD64skBNAlS3I__4qxlM3Sm4Z5IXj3h8">>, - Opts), - % This bundle previously triggered the ANS-104 tag-section boundary bug: - % the decoder ran past the declared tag bytes into the JSON body and - % crashed with a badmatch on the body content (the `"address":"0x..."' - % string). With the strict tag-section boundary enforced, the item is - % decoded and indexed correctly. - ?assertMatch( - {ok, _}, - hb_store_arweave:read( - StoreOpts, - #{ <<"read">> => <<"kK67S13W_8jM9JUw2umVamo0zh9v1DeVxWrru2evNco">> }, - Opts) - ), - assert_bundle_read( - <<"c2ATDuTgwKCcHpAFZqSt13NC-tA4hdA7Aa2xBPuOzoE">>, - [ - {<<"OBKr-7UrmjxFD-h-qP-XLuvCgtyuO_IDpBMgIytvusA">>, <<"1">>} - ], - Opts + {ok, BlockHeight} = + hb_ao:resolve( + << + "~copycat@1.0/arweave&from=", BlockHeightBin/binary, + "&to=", BlockHeightBin/binary, "&mode=deep" + >>, + Opts + ), + ?assertEqual( + [], + [TXID || TXID <- TXIDs, not is_tx_indexed(TXID, Opts)] ), - ok. + ?assert(is_block_indexed(BlockHeight, deep, Opts)), + ok. %% @doc Test a bundle header that fits in a single chunk. small_bundle_header_test_parallel() -> From cd76e0bae2a3e27b404c8f255353314cfb30780e Mon Sep 17 00:00:00 2001 From: speeddragon Date: Fri, 31 Jul 2026 17:48:41 +0100 Subject: [PATCH 4/4] fix: Add previous test --- src/preloaded/query/dev_copycat_arweave.erl | 72 +++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/src/preloaded/query/dev_copycat_arweave.erl b/src/preloaded/query/dev_copycat_arweave.erl index d4864b419..0ea1616f3 100644 --- a/src/preloaded/query/dev_copycat_arweave.erl +++ b/src/preloaded/query/dev_copycat_arweave.erl @@ -928,6 +928,78 @@ observe_event(MetricName, Fun) -> %%% Tests +index_ids_test_parallel() -> + %% Test block: https://viewblock.io/arweave/block/1827942 + %% Note: this block includes a data item with an Ethereum signature. This + %% signature type is not yet (as of Jan 2026) supported by ar_bundles.erl, + %% however we should still be able to index it (we just can't deserialize + %% it). + {_TestStore, StoreOpts, Opts} = setup_index_opts(), + {ok, 1827942} = + hb_ao:resolve( + <<"~copycat@1.0/arweave&from=1827942&to=1827942">>, + Opts + ), + ?assertMatch( + {ok, _}, + hb_store_arweave:read( + StoreOpts, + #{ <<"read">> => <<"WbRAQbeyjPHgopBKyi0PLeKWvYZr3rgZvQ7QY3ASJS4">> }, + Opts + ) + ), + assert_item_read( + <<"0vy2Ey8bWkSDcRIvWQJjxDeVGYOrTSmYIIhBILJntY8">>, + Opts), + assert_item_read( + <<"2lmrYydmDweX2MgGH39ZEB9hKm2JqGOYmRiG3n_xh8A">>, + Opts), + assert_item_read( + <<"ATi9pQF_eqb99UK84R5rq8lGfRGpilVQOYyth7rXxh8">>, + Opts), + assert_item_read( + <<"4VSfUbhMVZQHW5VfVwQZOmC5fR3W21DZgFCyz8CA-cE">>, + Opts), + assert_item_read( + <<"ZQRHZhktk6dAtX9BlhO1teOtVlGHoyaWP25kAlhxrM4">>, + Opts), + % The T2pluNnaavL7-S2GkO_m3pASLUqMH_XQ9IiIhZKfySs can be deserialized so + % we'll verify that some of its items were index and match the version + % in the deserialized bundle. + assert_bundle_read( + <<"T2pluNnaavL7-S2GkO_m3pASLUqMH_XQ9IiIhZKfySs">>, + [ + {<<"54K1ehEIKZxGSusgZzgbGYaHfllwWQ09-S9-eRUJg5Y">>, <<"1">>}, + {<<"MgatoEjlO_YtdbxFi9Q7Hxbs0YQVcChddhSS7FsdeIg">>, <<"19">>}, + {<<"z-oKJfhMq5qoVFrljEfiBKgumaJmCWVxNJaavR5aPE8">>, <<"26">>} + ], + Opts + ), + % Non-ans104 data transaction + assert_item_read( + <<"bXEgFm4K2b5VD64skBNAlS3I__4qxlM3Sm4Z5IXj3h8">>, + Opts), + % This bundle previously triggered the ANS-104 tag-section boundary bug: + % the decoder ran past the declared tag bytes into the JSON body and + % crashed with a badmatch on the body content (the `"address":"0x..."' + % string). With the strict tag-section boundary enforced, the item is + % decoded and indexed correctly. + ?assertMatch( + {ok, _}, + hb_store_arweave:read( + StoreOpts, + #{ <<"read">> => <<"kK67S13W_8jM9JUw2umVamo0zh9v1DeVxWrru2evNco">> }, + Opts) + ), + assert_bundle_read( + <<"c2ATDuTgwKCcHpAFZqSt13NC-tA4hdA7Aa2xBPuOzoE">>, + [ + {<<"OBKr-7UrmjxFD-h-qP-XLuvCgtyuO_IDpBMgIytvusA">>, <<"1">>} + ], + Opts + ), + ok. + filtered_txid_test_parallel() -> {_TestStore, _StoreOpts, Opts} = setup_index_opts(), FilteredTXID = <<"tiT3XhhgSvK39Lx40jniKD9CpbTTTDaQmGimrhFh-sw">>,