diff --git a/rs/ledger_suite/common/ledger_canister_core/src/archive.rs b/rs/ledger_suite/common/ledger_canister_core/src/archive.rs index d2dfa7643c53..2df838b44fae 100644 --- a/rs/ledger_suite/common/ledger_canister_core/src/archive.rs +++ b/rs/ledger_suite/common/ledger_canister_core/src/archive.rs @@ -63,6 +63,25 @@ impl LedgerArchivingGuard { } } +impl Drop for LedgerArchivingGuard { + fn drop(&mut self) { + // Archiving that fails gracefully is counted where it fails, but + // archiving that *traps* cannot be: the trap discards everything the + // failing message did, including any attempt to record it. Destructors + // are the exception. They run while the task is being canceled, in the + // cleanup callback, whose state changes are kept — which is how the + // archiving lock below gets released as well. + // + // Without this, a ledger whose archiving keeps trapping looks exactly + // like a ledger that is not archiving because it has nothing to + // archive. Since archiving no longer holds up the reply, nobody else + // would notice either. + if ic_cdk::futures::is_recovering_from_trap() { + LA::with_ledger_mut(|ledger| ledger.increment_archiving_failure_metric()); + } + } +} + pub enum ArchivingGuardError { /// There is no archive to lock, the archiving is disabled. NoArchive, diff --git a/rs/ledger_suite/icp/ledger/BUILD.bazel b/rs/ledger_suite/icp/ledger/BUILD.bazel index dea77a03e9c1..2c796a5da216 100644 --- a/rs/ledger_suite/icp/ledger/BUILD.bazel +++ b/rs/ledger_suite/icp/ledger/BUILD.bazel @@ -121,6 +121,7 @@ rust_ic_test( ":ledger-canister-wasm-allowance-getter", ":ledger-canister-wasm-next-version", ":ledger-canister-wasm-prev-version", + "//rs/universal_canister/impl:universal_canister.wasm.gz", "@mainnet_canisters//:ledger.wasm.gz", ], env = { @@ -131,6 +132,7 @@ rust_ic_test( "LEDGER_CANISTER_ALLOWANCE_GETTER_WASM_PATH": "$(rootpath :ledger-canister-wasm-allowance-getter)", "LEDGER_CANISTER_NEXT_VERSION_WASM_PATH": "$(rootpath :ledger-canister-wasm-next-version)", "LEDGER_CANISTER_PREV_VERSION_WASM_PATH": "$(rootpath :ledger-canister-wasm-prev-version)", + "UNIVERSAL_CANISTER_WASM_PATH": "$(rootpath //rs/universal_canister/impl:universal_canister.wasm.gz)", }, exec_properties = {"cpu": "4"}, deps = [ diff --git a/rs/ledger_suite/icp/ledger/src/main.rs b/rs/ledger_suite/icp/ledger/src/main.rs index c340a3048d09..36fc27e1ef93 100644 --- a/rs/ledger_suite/icp/ledger/src/main.rs +++ b/rs/ledger_suite/icp/ledger/src/main.rs @@ -189,6 +189,25 @@ fn init( /// * `to` - The account you want to send the funds to. /// * `created_at_time`: When the transaction has been created. If not set then /// now is used. +/// Starts archiving as a background task instead of awaiting it. +/// +/// The ledger applies a transaction synchronously but can only archive by +/// calling the archive canisters, so archiving has to await. Awaiting it before +/// replying would make the reply depend on continuations that run *after* the +/// transaction was committed: an await is a commit point, so a trap in one of +/// them — the replica refusing a memory growth, for instance — cannot roll the +/// transaction back, but it does turn the reply into a reject, which a caller +/// cannot tell apart from a transaction that never happened. +/// +/// Spawning puts archiving on its own chain of messages. The reply is produced +/// in the same message that commits the transaction, and a failure while +/// archiving can no longer contradict it — the blocks simply stay in the ledger +/// until the next attempt. +fn spawn_archiving() { + let max_msg_size = *MAX_MESSAGE_SIZE_BYTES.read().unwrap(); + ic_cdk::futures::spawn(archive_blocks::(DebugOutSink, max_msg_size as u64)); +} + async fn send( memo: Memo, amount: Tokens, @@ -255,11 +274,10 @@ async fn send( }; certified_data_set(hash.into_bytes()); - // Don't put anything that could ever trap after this call or people using this - // endpoint. If something did panic the payment would appear to fail, but would - // actually succeed on chain. - let max_msg_size = *MAX_MESSAGE_SIZE_BYTES.read().unwrap(); - archive_blocks::(DebugOutSink, max_msg_size as u64).await; + // Nothing after this point may trap: the payment is already committed, so a + // trap here would make it appear to fail while it actually succeeded on + // chain. Archiving is spawned rather than awaited for exactly that reason. + spawn_archiving(); Ok(height) } @@ -388,8 +406,7 @@ async fn icrc1_send( created_at_time, )?; - let max_msg_size = *MAX_MESSAGE_SIZE_BYTES.read().unwrap(); - archive_blocks::(DebugOutSink, max_msg_size as u64).await; + spawn_archiving(); Ok(block_index) } @@ -1419,8 +1436,7 @@ fn icrc2_approve_not_async( async fn icrc2_approve(arg: ApproveArgs) -> Result { let block_index = icrc2_approve_not_async(caller(), arg, None)?; - let max_msg_size = *MAX_MESSAGE_SIZE_BYTES.read().unwrap(); - archive_blocks::(DebugOutSink, max_msg_size as u64).await; + spawn_archiving(); Ok(block_index) } @@ -1454,8 +1470,7 @@ async fn remove_approval(args: RemoveApprovalArgs) -> Result }); let block_index = icrc2_approve_not_async(caller(), approve_arg, Some(spender))?; - let max_msg_size = *MAX_MESSAGE_SIZE_BYTES.read().unwrap(); - archive_blocks::(DebugOutSink, max_msg_size as u64).await; + spawn_archiving(); Ok(block_index) } diff --git a/rs/ledger_suite/icp/ledger/tests/tests.rs b/rs/ledger_suite/icp/ledger/tests/tests.rs index d0d7f5e86724..2ee305253ae8 100644 --- a/rs/ledger_suite/icp/ledger/tests/tests.rs +++ b/rs/ledger_suite/icp/ledger/tests/tests.rs @@ -2071,6 +2071,25 @@ fn test_archiving_respects_num_blocks_to_archive_upper_limit() { ); } +#[test] +fn test_trapped_archiving_is_counted() { + ic_ledger_suite_state_machine_tests::subnet_memory::test_trapped_archiving_is_counted( + ledger_wasm(), + encode_init_args, + icp_archives, + ic_ledger_suite_state_machine_tests::archiving::query_encoded_blocks, + ); +} + +#[test] +fn test_transfer_when_subnet_is_out_of_memory() { + ic_ledger_suite_state_machine_tests::subnet_memory::test_transfer_when_subnet_is_out_of_memory( + ledger_wasm(), + encode_init_args, + ic_ledger_suite_state_machine_tests::archiving::query_encoded_blocks, + ); +} + #[test] fn test_archiving_fails_on_app_subnet_if_ledger_does_not_have_enough_cycles() { ic_ledger_suite_state_machine_tests::archiving::test_archiving_fails_on_app_subnet_if_ledger_does_not_have_enough_cycles( diff --git a/rs/ledger_suite/icrc1/ledger/BUILD.bazel b/rs/ledger_suite/icrc1/ledger/BUILD.bazel index d61f81ec561c..670a8af62416 100644 --- a/rs/ledger_suite/icrc1/ledger/BUILD.bazel +++ b/rs/ledger_suite/icrc1/ledger/BUILD.bazel @@ -462,3 +462,34 @@ rust_ic_test( "@crate_index//:candid", ], ) + +rust_ic_test( + name = "ledger_archiving_atomicity_tests", + srcs = ["tests/archiving_atomicity_tests.rs"], + crate_features = [], + data = [ + ":ledger_canister.wasm.gz", + "//rs/universal_canister/impl:universal_canister.wasm.gz", + ], + env = { + "IC_ICRC1_LEDGER_WASM_PATH": "$(rootpath :ledger_canister.wasm.gz)", + "UNIVERSAL_CANISTER_WASM_PATH": "$(rootpath //rs/universal_canister/impl:universal_canister.wasm.gz)", + }, + deps = [ + # Keep sorted. + ":ledger", + "//packages/icrc-ledger-types:icrc_ledger_types_storable", + "//rs/config", + "//rs/ledger_suite/common/ledger_canister_core", + "//rs/ledger_suite/test_utils/state_machine_helpers:ic-ledger-suite-state-machine-helpers", + "//rs/registry/subnet_type", + "//rs/state_machine_tests", + "//rs/types/base_types", + "//rs/types/cycles", + "//rs/types/management_canister_types", + "//rs/types/types", + "//rs/universal_canister/lib", + "@crate_index//:candid", + "@crate_index//:num-traits", + ], +) diff --git a/rs/ledger_suite/icrc1/ledger/src/main.rs b/rs/ledger_suite/icrc1/ledger/src/main.rs index 5ebdfe3f0361..7439d8a869cd 100644 --- a/rs/ledger_suite/icrc1/ledger/src/main.rs +++ b/rs/ledger_suite/icrc1/ledger/src/main.rs @@ -540,7 +540,26 @@ fn icrc1_total_supply() -> Nat { Access::with_ledger(|ledger| ledger.balances().total_supply().into()) } -async fn execute_transfer( +/// Starts archiving as a background task instead of awaiting it. +/// +/// The ledger applies a transaction synchronously but can only archive by +/// calling the archive canisters, so archiving has to await. Awaiting it before +/// replying would make the reply depend on continuations that run *after* the +/// transaction was committed: an await is a commit point, so a trap in one of +/// them — the replica refusing a memory growth, for instance — cannot roll the +/// transaction back, but it does turn the reply into a reject. Callers cannot +/// tell such a reject apart from one where nothing happened, and clients that +/// retry on reject (the ck minters) would mint a deposit twice. +/// +/// Spawning puts archiving on its own chain of messages. The reply is produced +/// in the same message that commits the transaction, and a failure while +/// archiving can no longer contradict it — the blocks simply stay in the ledger +/// until the next attempt. +fn spawn_archiving() { + ic_cdk::futures::spawn(archive_blocks::(&LOG, MAX_MESSAGE_SIZE)); +} + +fn execute_transfer( from_account: Account, to: Account, spender: Option, @@ -559,11 +578,11 @@ async fn execute_transfer( created_at_time, )?; - // NB. we need to set the certified data before the first async call to make sure that the + // NB. we need to set the certified data before spawning the archiving to make sure that the // blockchain state agrees with the certificate while archiving is in progress. ic_cdk::api::certified_data_set(Access::with_ledger(Ledger::root_hash)); - archive_blocks::(&LOG, MAX_MESSAGE_SIZE).await; + spawn_archiving(); Ok(Nat::from(block_idx)) } @@ -687,7 +706,6 @@ async fn icrc1_transfer(arg: TransferArg) -> Result { arg.memo, arg.created_at_time, ) - .await .map_err(convert_transfer_error) .map_err(|err| { let err: TransferError = match err.try_into() { @@ -713,7 +731,6 @@ async fn icrc2_transfer_from(arg: TransferFromArgs) -> Result Result Result { let block_idx = icrc2_approve_not_async(ic_cdk::api::msg_caller(), arg)?; - // NB. we need to set the certified data before the first async call to make sure that the + // NB. we need to set the certified data before spawning the archiving to make sure that the // blockchain state agrees with the certificate while archiving is in progress. ic_cdk::api::certified_data_set(Access::with_ledger(Ledger::root_hash)); - archive_blocks::(&LOG, MAX_MESSAGE_SIZE).await; + spawn_archiving(); Ok(Nat::from(block_idx)) } @@ -991,7 +1008,7 @@ fn icrc152_mint_not_async( async fn icrc152_mint(args: Icrc152MintArgs) -> Result { let block_idx = icrc152_mint_not_async(ic_cdk::api::msg_caller(), args)?; ic_cdk::api::certified_data_set(Access::with_ledger(Ledger::root_hash)); - archive_blocks::(&LOG, MAX_MESSAGE_SIZE).await; + spawn_archiving(); Ok(Nat::from(block_idx)) } @@ -1089,7 +1106,7 @@ fn icrc152_burn_not_async( async fn icrc152_burn(args: Icrc152BurnArgs) -> Result { let block_idx = icrc152_burn_not_async(ic_cdk::api::msg_caller(), args)?; ic_cdk::api::certified_data_set(Access::with_ledger(Ledger::root_hash)); - archive_blocks::(&LOG, MAX_MESSAGE_SIZE).await; + spawn_archiving(); Ok(Nat::from(block_idx)) } diff --git a/rs/ledger_suite/icrc1/ledger/tests/archiving_atomicity_tests.rs b/rs/ledger_suite/icrc1/ledger/tests/archiving_atomicity_tests.rs new file mode 100644 index 000000000000..1830b3d4a21f --- /dev/null +++ b/rs/ledger_suite/icrc1/ledger/tests/archiving_atomicity_tests.rs @@ -0,0 +1,544 @@ +//! Regression test for the atomicity of a ledger transfer with respect to the +//! archiving that the ledger performs before replying. +//! +//! `execute_transfer` applies the transaction synchronously — balances, total +//! supply, the block log and the certified data all change — and only then +//! awaits `archive_blocks`. The await is a commit point: everything applied +//! before it is durable even if a later continuation of the same call traps. +//! When that happens the caller receives a reject for a transfer that did take +//! effect. +//! +//! A caller cannot distinguish that reject from one where nothing happened. +//! The ck minters read it as "the mint did not happen" and retry, which mints +//! the same deposit twice (reported as ICPBB-369). +//! +//! The trap is triggered here the same way it can be triggered on mainnet: the +//! subnet is pushed above its storage-reservation threshold while the ledger is +//! suspended in the archive await, so the memory growth performed by the +//! archive continuation is rejected with `IC0534` +//! (`ReservedCyclesLimitExceededInMemoryGrow`). Unlike a trap in the +//! synchronous part of the message, that one cannot roll the transfer back. +//! +//! The subnet-wide thresholds are scaled down through the test's +//! `HypervisorConfig` rather than reproduced at their mainnet size, and the +//! filler canister only grows *logical* stable memory (it never writes to it), +//! so no meaningful amount of host storage is used. + +use candid::{Decode, Encode, Nat}; +use ic_base_types::PrincipalId; +use ic_config::{execution_environment::Config as HypervisorConfig, subnet_config::SubnetConfig}; +use ic_icrc1_ledger::{InitArgs, LedgerArgument}; +use ic_ledger_canister_core::archive::ArchiveOptions; +use ic_ledger_suite_state_machine_helpers::{ + balance_of, icrc3_get_blocks, list_archives, parse_metric, +}; +use ic_management_canister_types_private::CanisterSettingsArgsBuilder; +use ic_registry_subnet_type::SubnetType; +use ic_state_machine_tests::{ + StateMachine, StateMachineBuilder, StateMachineConfig, UserError, WasmResult, +}; +use ic_types::ingress::{IngressState, IngressStatus}; +use ic_types::{CanisterId, NumBytes}; +use ic_types_cycles::Cycles; +use ic_universal_canister::{UNIVERSAL_CANISTER_WASM, wasm}; +use icrc_ledger_types::icrc1::account::Account; +use icrc_ledger_types::icrc1::transfer::TransferArg; +use num_traits::ToPrimitive; + +const WASM_PAGE_SIZE: u64 = 64 * 1024; +const GIB: u64 = 1024 * 1024 * 1024; + +/// Storage reservation starts above this much subnet memory (750 GiB on +/// mainnet). It only has to be larger than what the canisters under test use. +const SUBNET_MEMORY_THRESHOLD: u64 = 4 * GIB; +/// A single message can only grow memory by `(capacity - usage) / +/// scheduler_cores` — both the available memory and the saturation a canister +/// sees are scaled down by the number of scheduler cores, see +/// `ExecutionEnvironment::subnet_memory_saturation`. Keeping the capacity well +/// above the threshold leaves the filler's grows plenty of room. +const SUBNET_MEMORY_CAPACITY: u64 = 64 * GIB; + +/// Logical stable memory the filler holds before the ledger call starts. +const FILLER_BASE_PAGES: u64 = 3 * GIB / WASM_PAGE_SIZE; +/// Logical stable memory the filler adds while the ledger is suspended in the +/// archive await. This crosses the threshold, leaving the subnet ~4 GiB above +/// it, where growing a single 64 KiB page reserves ~388M cycles. +const FILLER_CROSSING_PAGES: u64 = 5 * GIB / WASM_PAGE_SIZE; + +/// Small enough that the first page the archive continuation grows exceeds it. +const LEDGER_RESERVED_CYCLES_LIMIT: u128 = 100_000_000; + +/// `create_and_initialize_node_canister` requires at least 4.5T for the archive +/// and at least 10T left over in the ledger. +const CYCLES_FOR_ARCHIVE_CREATION: u64 = 20_000_000_000_000; +const LEDGER_CYCLES: u128 = 200_000_000_000_000; +const FILLER_CYCLES: u128 = 100_000_000_000_000_000; + +const ARCHIVE_TRIGGER_THRESHOLD: usize = 10; +const NUM_BLOCKS_TO_ARCHIVE: usize = 5; +const TRANSFER_FEE: u64 = 10_000; +const MINT_AMOUNT: u64 = 1_000_000; + +fn ledger_wasm() -> Vec { + std::fs::read(std::env::var("IC_ICRC1_LEDGER_WASM_PATH").unwrap()).unwrap() +} + +fn minter() -> PrincipalId { + PrincipalId::new_user_test_id(0) +} + +fn beneficiary() -> Account { + Account::from(PrincipalId::new_user_test_id(1).0) +} + +fn hypervisor_config() -> HypervisorConfig { + HypervisorConfig { + subnet_memory_threshold: NumBytes::new(SUBNET_MEMORY_THRESHOLD), + subnet_memory_capacity: NumBytes::new(SUBNET_MEMORY_CAPACITY), + // Memory held back for response callbacks. Keeping it at the mainnet + // value (10 GiB) would leave the scaled-down capacity negative for + // update calls. + subnet_memory_reservation: NumBytes::new(0), + ..HypervisorConfig::default() + } +} + +fn ledger_init_args() -> Vec { + Encode!(&LedgerArgument::Init(InitArgs { + minting_account: minter().0.into(), + fee_collector_account: None, + initial_balances: vec![], + transfer_fee: TRANSFER_FEE.into(), + token_name: "Test Token".to_string(), + decimals: Some(8), + token_symbol: "XTST".to_string(), + metadata: vec![], + archive_options: ArchiveOptions { + trigger_threshold: ARCHIVE_TRIGGER_THRESHOLD, + num_blocks_to_archive: NUM_BLOCKS_TO_ARCHIVE, + node_max_memory_size_bytes: None, + max_message_size_bytes: None, + controller_id: minter(), + more_controller_ids: None, + cycles_for_archive_creation: Some(CYCLES_FOR_ARCHIVE_CREATION), + max_transactions_per_response: None, + }, + max_memo_length: None, + feature_flags: None, + index_principal: None, + })) + .unwrap() +} + +fn mint_arg(amount: u64) -> Vec { + Encode!(&TransferArg { + from_subaccount: None, + to: beneficiary(), + fee: None, + // The ck minters do the same, which is what makes a retry after an + // ambiguous reject a second, valid mint. + created_at_time: None, + memo: None, + amount: Nat::from(amount), + }) + .unwrap() +} + +fn chain_length(env: &StateMachine, ledger: CanisterId) -> u64 { + icrc3_get_blocks(env, ledger, 0, 0) + .log_length + .0 + .to_u64() + .unwrap() +} + +/// Grows the filler's *logical* stable memory. The pages are never written to, +/// so they cost the host nothing while still counting towards the subnet's +/// memory usage, which is what drives the storage reservation. +fn grow_filler(env: &StateMachine, filler: CanisterId, pages: u64) { + let reply = env + .execute_ingress( + filler, + "update", + wasm().stable64_grow(pages).reply_int64().build(), + ) + .expect("failed to call the filler canister"); + let previous_size = u64::from_le_bytes(reply.bytes()[..8].try_into().unwrap()); + assert_ne!( + previous_size, + u64::MAX, + "stable64_grow({pages}) failed on the filler canister" + ); +} + +/// Memory usage in bytes and reserved cycles, as reported by `canister_status`. +fn memory_and_reservation(env: &StateMachine, canister_id: CanisterId) -> (u64, u128) { + let status = env + .canister_status_as(minter(), canister_id) + .expect("failed to call canister_status") + .expect("canister_status returned an error"); + (status.memory_size().get(), status.reserved_cycles()) +} + +fn canister_settings(reserved_cycles_limit: u128) -> CanisterSettingsArgsBuilder { + CanisterSettingsArgsBuilder::new() + .with_controllers(vec![minter()]) + .with_reserved_cycles_limit(reserved_cycles_limit) +} + +/// Returns the reply of the `icrc1_transfer` that triggers archiving, together +/// with the state committed by the time the reply was produced. +struct TriggeringTransfer { + result: Result, + /// Whether the ledger had already committed the block while the call was + /// still outstanding, i.e. whether the call really did reach the archive + /// await. + committed_while_outstanding: bool, + balance_after: u64, + chain_length_after: u64, +} + +fn setup() -> (StateMachine, CanisterId, CanisterId) { + let env = StateMachineBuilder::new() + .with_config(Some(StateMachineConfig::new( + SubnetConfig::new(SubnetType::Application), + hypervisor_config(), + ))) + .build(); + + let filler = env + .install_canister_with_cycles( + UNIVERSAL_CANISTER_WASM.to_vec(), + vec![], + // The filler pays a large storage reservation when it crosses the + // threshold; it must not be capped by the default 5T limit. + Some(canister_settings(FILLER_CYCLES).build()), + Cycles::new(FILLER_CYCLES), + ) + .expect("failed to install the filler canister"); + + let ledger = env + .install_canister_with_cycles( + ledger_wasm(), + ledger_init_args(), + Some(canister_settings(LEDGER_RESERVED_CYCLES_LIMIT).build()), + Cycles::new(LEDGER_CYCLES), + ) + .expect("failed to install the ledger canister"); + + // Put the subnet under storage pressure. The ledger has not grown any + // memory of its own at this point, so it has reserved nothing yet. + grow_filler(&env, filler, FILLER_BASE_PAGES); + let (filler_memory, _) = memory_and_reservation(&env, filler); + assert!( + filler_memory >= FILLER_BASE_PAGES * WASM_PAGE_SIZE, + "the filler's stable memory growth did not take effect" + ); + + (env, ledger, filler) +} + +/// Brings the ledger to one block short of the archive trigger threshold, so +/// that the next transfer is the one that awaits `archive_blocks`. +fn fill_up_to_archive_trigger(env: &StateMachine, ledger: CanisterId) { + for _ in 0..(ARCHIVE_TRIGGER_THRESHOLD - 1) { + env.execute_ingress_as(minter(), ledger, "icrc1_transfer", mint_arg(MINT_AMOUNT)) + .expect("failed to mint"); + } + assert_eq!( + chain_length(env, ledger), + ARCHIVE_TRIGGER_THRESHOLD as u64 - 1 + ); + assert!( + list_archives(env, ledger).is_empty(), + "archiving must not have been triggered yet" + ); +} + +fn submit_triggering_transfer( + env: &StateMachine, + ledger: CanisterId, + filler: CanisterId, +) -> TriggeringTransfer { + let chain_length_before = chain_length(env, ledger); + let message = env.send_ingress(minter(), ledger, "icrc1_transfer", mint_arg(MINT_AMOUNT)); + + // Let the ledger apply the transaction and suspend in the archive await. + let mut committed_while_outstanding = false; + for _ in 0..100 { + if chain_length(env, ledger) > chain_length_before { + committed_while_outstanding = matches!( + env.ingress_status(&message), + IngressStatus::Known { + state: IngressState::Processing | IngressState::Received, + .. + } + ); + break; + } + env.tick(); + } + assert_eq!( + chain_length(env, ledger), + chain_length_before + 1, + "the ledger did not commit the transfer" + ); + + // Raise the storage pressure further while the archive work is in flight, + // so that the next page the ledger grows costs more than its reservation + // limit. This is an ordinary update call to an unrelated canister. + grow_filler(env, filler, FILLER_CROSSING_PAGES); + let (filler_memory, filler_reserved) = memory_and_reservation(env, filler); + assert!( + filler_memory >= (FILLER_BASE_PAGES + FILLER_CROSSING_PAGES) * WASM_PAGE_SIZE, + "the filler's stable memory growth did not take effect" + ); + assert!( + filler_reserved > 0, + "the subnet is not above the storage reservation threshold" + ); + + let result = env.await_ingress(message, 100); + TriggeringTransfer { + result, + committed_while_outstanding, + balance_after: balance_of(env, ledger, beneficiary()), + chain_length_after: chain_length(env, ledger), + } +} + +/// Archiving must recover once the condition that killed it is gone. +/// +/// The archiving lock is taken before the first await, so a trapped +/// continuation could leave `archiving_in_progress` set and silently stop the +/// ledger from ever archiving again. It should be cleared while the task is +/// canceled. Now that a failure to archive is invisible to callers, a stuck +/// lock would be easy to miss. +#[test] +fn archiving_recovers_after_a_trapped_attempt() { + let (env, ledger, filler) = setup(); + fill_up_to_archive_trigger(&env, ledger); + + let transfer = submit_triggering_transfer(&env, ledger, filler); + assert!( + transfer.result.is_ok(), + "the transfer should have succeeded: {:?}", + transfer.result + ); + assert!( + list_archives(&env, ledger).is_empty(), + "expected the archive continuation to have been trapped" + ); + + // Raise the reservation limit, as an operator would once alerted, and let + // the ledger reach the archive trigger threshold again. + env.update_settings(&ledger, canister_settings(LEDGER_CYCLES / 2).build()) + .expect("failed to raise the ledger's reserved cycles limit"); + env.execute_ingress_as(minter(), ledger, "icrc1_transfer", mint_arg(MINT_AMOUNT)) + .expect("failed to mint"); + + for _ in 0..20 { + if !list_archives(&env, ledger).is_empty() { + return; + } + env.tick(); + } + panic!("the ledger never archived again after a trapped archiving attempt"); +} + +/// A rejected `icrc1_transfer` must not leave a committed block behind. +/// +/// Before archiving was moved off the reply path this failed: the ledger +/// replied with `IC0534` raised by its archive continuation, while the mint it +/// had already committed stayed in the ledger. +#[test] +fn transfer_reject_must_not_leave_a_committed_block() { + let (env, ledger, filler) = setup(); + fill_up_to_archive_trigger(&env, ledger); + + let balance_before = balance_of(&env, ledger, beneficiary()); + let chain_length_before = chain_length(&env, ledger); + let transfer = submit_triggering_transfer(&env, ledger, filler); + + println!( + "committed_while_outstanding={} chain_length={}->{} balance={}->{} archives={:?}", + transfer.committed_while_outstanding, + chain_length_before, + transfer.chain_length_after, + balance_before, + transfer.balance_after, + list_archives(&env, ledger), + ); + + // Preconditions. Without these the test would pass vacuously: it has to + // reach the state where the ledger has committed the transfer and its + // archive continuation has then been killed. + assert_eq!( + transfer.chain_length_after, + chain_length_before + 1, + "the ledger did not commit the transfer, so this run never reached the \ + post-commit failure the test is about" + ); + assert!( + list_archives(&env, ledger).is_empty(), + "expected the archive continuation to have been trapped, but an archive \ + was created; the test is not exercising a post-commit failure" + ); + // The caller no longer learns that archiving failed, so it has to be + // recorded. A trap discards everything the failing message did, but + // destructors still run while the task is canceled, and that is where the + // archiving guard counts the failure. + assert_eq!( + parse_metric(&env, ledger, "ledger_archiving_failures"), + 1, + "the trapped archiving attempt was not counted" + ); + + // The invariant: having committed the transfer, the ledger must not reply + // with a reject. + match transfer.result { + Err(err) => panic!( + "the ledger rejected the transfer with `{err}`, but had already committed it: \ + chain length {chain_length_before} -> {}, beneficiary balance {balance_before} -> {}. \ + A caller cannot tell this reject apart from one where nothing happened.", + transfer.chain_length_after, transfer.balance_after, + ), + Ok(reply) => { + let block_index = Decode!(&reply.bytes(), Result) + .expect("failed to decode icrc1_transfer response") + .expect("the transfer should have succeeded"); + assert_eq!(block_index, Nat::from(chain_length_before)); + assert_eq!(transfer.balance_after, balance_before + MINT_AMOUNT); + } + } +} + +/// Shows how narrow the post-commit window is: it needs the ledger to grow its +/// memory, which archiving only does when it creates an archive canister. +/// +/// A routine round hands the archive another chunk of blocks using memory the +/// allocator already has from the previous round, so it neither grows nor +/// traps, even under the storage pressure that breaks the creation round. Note +/// that this is about the *ledger's* memory: the archive canister running out +/// of memory is harmless, because it answers with a reject that the ledger +/// handles gracefully rather than trapping on. +/// +/// If this test ever fails, routine archiving has started to allocate as well, +/// and the window is wider than it is assumed to be here. +/// +/// Uses production-shaped archive options: 1000 blocks per round, and enough +/// initial balances that the first two transfers each trigger a full round — +/// the first creating the archive, the second appending to an archive that +/// already holds 1000 blocks. +#[test] +fn routine_archiving_does_not_grow_the_ledger() { + const BLOCKS_PER_ROUND: usize = 1_000; + let env = StateMachineBuilder::new() + .with_config(Some(StateMachineConfig::new( + SubnetConfig::new(SubnetType::Application), + hypervisor_config(), + ))) + .build(); + let filler = env + .install_canister_with_cycles( + UNIVERSAL_CANISTER_WASM.to_vec(), + vec![], + Some(canister_settings(FILLER_CYCLES).build()), + Cycles::new(FILLER_CYCLES), + ) + .expect("failed to install the filler canister"); + let initial_balances: Vec<_> = (0..2 * BLOCKS_PER_ROUND) + .map(|i| { + ( + Account::from(PrincipalId::new_user_test_id(1_000 + i as u64).0), + Nat::from(MINT_AMOUNT), + ) + }) + .collect(); + let ledger = env + .install_canister_with_cycles( + ledger_wasm(), + Encode!(&LedgerArgument::Init(InitArgs { + minting_account: minter().0.into(), + fee_collector_account: None, + initial_balances, + transfer_fee: TRANSFER_FEE.into(), + token_name: "Test Token".to_string(), + decimals: Some(8), + token_symbol: "XTST".to_string(), + metadata: vec![], + archive_options: ArchiveOptions { + trigger_threshold: BLOCKS_PER_ROUND, + num_blocks_to_archive: BLOCKS_PER_ROUND, + node_max_memory_size_bytes: None, + max_message_size_bytes: None, + controller_id: minter(), + more_controller_ids: None, + cycles_for_archive_creation: Some(CYCLES_FOR_ARCHIVE_CREATION), + max_transactions_per_response: None, + }, + max_memo_length: None, + feature_flags: None, + index_principal: None, + })) + .unwrap(), + Some(canister_settings(LEDGER_RESERVED_CYCLES_LIMIT).build()), + Cycles::new(LEDGER_CYCLES), + ) + .expect("failed to install the ledger canister"); + grow_filler(&env, filler, FILLER_BASE_PAGES); + + // First round: creates the archive and fills it with 1000 blocks. There is + // still memory at this point, so it succeeds. + let (memory_before, _) = memory_and_reservation(&env, ledger); + env.execute_ingress_as(minter(), ledger, "icrc1_transfer", mint_arg(MINT_AMOUNT)) + .expect("failed to mint"); + env.run_until_completion(200); + let archives = list_archives(&env, ledger); + assert_eq!(archives.len(), 1, "the archive was not created"); + let (memory_after_creation, _) = memory_and_reservation(&env, ledger); + println!( + "archive creation round: {:?}, ledger memory {memory_before}->{memory_after_creation}", + archives[0] + ); + assert!( + memory_after_creation > memory_before, + "creating the archive did not grow the ledger's memory, so the \ + comparison below says nothing" + ); + + // Second round: appends another 1000 blocks to the archive that now holds + // 1000, with the subnet pushed over the reservation threshold while the + // ledger is doing it. + let chain_length_before = chain_length(&env, ledger); + let transfer = submit_triggering_transfer(&env, ledger, filler); + let (memory_after_round, _) = memory_and_reservation(&env, ledger); + println!( + "routine round: chain_length={}->{} ledger memory {memory_after_creation}->{memory_after_round} archives={:?}", + chain_length_before, + transfer.chain_length_after, + list_archives(&env, ledger), + ); + + assert!( + transfer.result.is_ok(), + "the routine archiving round should not have failed: {:?}", + transfer.result + ); + assert_eq!( + memory_after_round, memory_after_creation, + "the routine archiving round grew the ledger's memory" + ); + assert_eq!( + parse_metric(&env, ledger, "ledger_archiving_failures"), + 0, + "the routine archiving round failed" + ); + let archives = list_archives(&env, ledger); + assert_eq!(archives.len(), 1); + assert_eq!( + archives[0].block_range_end, + Nat::from(2 * BLOCKS_PER_ROUND as u64 - 1), + "the second round did not archive its blocks" + ); +} diff --git a/rs/ledger_suite/icrc1/ledger/tests/tests.rs b/rs/ledger_suite/icrc1/ledger/tests/tests.rs index bacca4f65559..d7452a52a16e 100644 --- a/rs/ledger_suite/icrc1/ledger/tests/tests.rs +++ b/rs/ledger_suite/icrc1/ledger/tests/tests.rs @@ -11,7 +11,9 @@ use ic_ledger_canister_core::archive::ArchiveOptions; use ic_ledger_core::block::{BlockIndex, BlockType}; use ic_ledger_hash_of::{HASH_LENGTH, HashOf}; use ic_ledger_suite_in_memory_ledger::{AllowancesRecentlyPurged, verify_ledger_state}; -use ic_ledger_suite_state_machine_helpers::{AllowanceProvider, send_approval, send_transfer_from}; +use ic_ledger_suite_state_machine_helpers::{ + AllowanceProvider, await_archiving, send_approval, send_transfer_from, +}; use ic_ledger_suite_state_machine_tests::MINTER; use ic_ledger_suite_state_machine_tests::archiving::icrc_archives; use ic_ledger_suite_state_machine_tests::fee_collector::BlockRetrieval; @@ -552,6 +554,25 @@ fn test_get_blocks_returns_multiple_archive_callbacks() { ); } +#[test] +fn test_trapped_archiving_is_counted() { + ic_ledger_suite_state_machine_tests::subnet_memory::test_trapped_archiving_is_counted( + ledger_wasm(), + encode_init_args, + icrc_archives, + ic_ledger_suite_state_machine_tests::archiving::query_icrc3_get_blocks, + ); +} + +#[test] +fn test_transfer_when_subnet_is_out_of_memory() { + ic_ledger_suite_state_machine_tests::subnet_memory::test_transfer_when_subnet_is_out_of_memory( + ledger_wasm(), + encode_init_args, + ic_ledger_suite_state_machine_tests::archiving::query_icrc3_get_blocks, + ); +} + #[test] fn test_archiving_fails_on_app_subnet_if_ledger_does_not_have_enough_cycles() { ic_ledger_suite_state_machine_tests::archiving::test_archiving_fails_on_app_subnet_if_ledger_does_not_have_enough_cycles( @@ -843,12 +864,16 @@ fn transfer( .execute_ingress_as(from.owner.into(), ledger_id, "icrc1_transfer", args) .expect("Unable to perform icrc1_transfer") .bytes(); - Decode!(&res, Result) + let block_index = Decode!(&res, Result) .unwrap() .expect("Unable to decode icrc1_transfer error") .0 .to_u64() - .unwrap() + .unwrap(); + // The ledger replies before it archives, so let any archiving triggered by + // this transfer run to completion. + await_archiving(env); + block_index } #[test] diff --git a/rs/ledger_suite/test_utils/state_machine_helpers/src/lib.rs b/rs/ledger_suite/test_utils/state_machine_helpers/src/lib.rs index b3801a40449d..2701738362c5 100644 --- a/rs/ledger_suite/test_utils/state_machine_helpers/src/lib.rs +++ b/rs/ledger_suite/test_utils/state_machine_helpers/src/lib.rs @@ -576,6 +576,24 @@ pub fn icrc21_consent_message( .expect("failed to decode icrc21_canister_call_consent_message response") } +/// Generous upper bound on the rounds the ledger needs to spawn an archive +/// canister, install it and send it the blocks. +const MAX_ARCHIVING_ROUNDS: usize = 100; + +/// Waits for archiving triggered by a preceding call to finish. +/// +/// The ledger does not archive before replying — it applies the transaction and +/// replies in one message, and archives in the messages that follow, so that a +/// failure while archiving cannot turn a committed transaction into a reject. +/// Anything asserting on archives right after the call that triggered the +/// archiving therefore has to let those messages run first. +/// +/// Returns immediately when nothing is in flight, so it is cheap to call after +/// every ledger update. +pub fn await_archiving(env: &StateMachine) { + env.run_until_completion(MAX_ARCHIVING_ROUNDS); +} + pub fn list_archives(env: &StateMachine, ledger: CanisterId) -> Vec { Decode!( &env.query(ledger, "archives", Encode!().unwrap()) @@ -659,7 +677,7 @@ pub fn send_approval( from: Principal, arg: &ApproveArgs, ) -> Result { - Decode!( + let result = Decode!( &env.execute_ingress_as( PrincipalId(from), ledger, @@ -672,7 +690,9 @@ pub fn send_approval( Result ) .expect("failed to decode approve response") - .map(|n| n.0.to_u64().unwrap()) + .map(|n| n.0.to_u64().unwrap()); + await_archiving(env); + result } pub fn send_transfer( @@ -687,14 +707,16 @@ pub fn send_transfer( "icrc1_transfer", Encode!(arg).unwrap(), ); - Decode!( + let result = Decode!( &response .expect("failed to transfer funds") .bytes(), Result ) .expect("failed to decode transfer response") - .map(|n| n.0.to_u64().unwrap()) + .map(|n| n.0.to_u64().unwrap()); + await_archiving(env); + result } pub fn send_transfer_from( @@ -703,7 +725,7 @@ pub fn send_transfer_from( from: Principal, arg: &TransferFromArgs, ) -> Result { - Decode!( + let result = Decode!( &env.execute_ingress_as( PrincipalId(from), ledger, @@ -716,7 +738,9 @@ pub fn send_transfer_from( Result ) .expect("failed to decode transfer_from response") - .map(|n| n.0.to_u64().unwrap()) + .map(|n| n.0.to_u64().unwrap()); + await_archiving(env); + result } /// Upgrade a canister as its controller. The canister is stopped before the upgrade and restarted diff --git a/rs/ledger_suite/tests/sm-tests/src/lib.rs b/rs/ledger_suite/tests/sm-tests/src/lib.rs index 18192878ff67..dd62e0ce9d02 100644 --- a/rs/ledger_suite/tests/sm-tests/src/lib.rs +++ b/rs/ledger_suite/tests/sm-tests/src/lib.rs @@ -5122,8 +5122,19 @@ pub mod archiving { "icrc1_transfer", encode_transfer_args(p1.0, p2.0, 10_000), ); - let mut transfer_status = message_status(&env, &transfer_message_id).unwrap(); - assert!(transfer_status.is_none()); + // The ledger archives after replying, so the transfer completes in the + // first round while the archiving continues in the background. + let transfer_result = Decode!( + &message_status(&env, &transfer_message_id) + .unwrap() + .expect("the transfer should have completed") + .bytes(), + Result + ) + .expect("failed to decode transfer response") + .map(|n| n.0.to_u64().unwrap()) + .expect("transfer should succeed"); + assert_eq!(transfer_result, NUM_INITIAL_BALANCES); // Keep listing the archives and calling env.tick() until the ledger reports that an // archive has been created. @@ -5156,24 +5167,14 @@ pub mod archiving { // Verify that the ledger response contained no archive info. assert!(get_blocks_res.archived_ranges.is_empty()); - // Tick until the transfer completes, meaning the archiving also completes. + // Tick until the archiving completes. const MAX_TICKS: usize = 500; let mut ticks = 0; - while transfer_status.is_none() { + while env.has_inflight_messages() { env.tick(); ticks += 1; assert!(ticks < MAX_TICKS); - transfer_status = message_status(&env, &transfer_message_id).unwrap(); } - let transfer_result = Decode!( - &transfer_status.unwrap() - .bytes(), - Result - ) - .expect("failed to decode transfer response") - .map(|n| n.0.to_u64().unwrap()) - .expect("transfer should succeed"); - assert_eq!(transfer_result, NUM_INITIAL_BALANCES); // Verify that the ledger now does not return the first block, but reports that it is in // the first archive. @@ -5261,15 +5262,27 @@ pub mod archiving { &get_blocks_res )); - // Tick until the transfer completes, meaning the archiving also completes. + // The ledger archives after replying, so the transfer completes in + // the first round while the archiving continues in the background. + let transfer_result = Decode!( + &message_status(&env, &transfer_message_id) + .unwrap() + .expect("the transfer should have completed") + .bytes(), + Result + ) + .expect("failed to decode transfer response") + .map(|n| n.0.to_u64().unwrap()) + .expect("transfer should succeed"); + assert_eq!(transfer_result, NUM_INITIAL_BALANCES + i - 1); + + // Tick until the archiving completes. const MAX_TICKS: usize = 500; let mut ticks = 0; - let mut transfer_status = message_status(&env, &transfer_message_id).unwrap(); - while transfer_status.is_none() { + while env.has_inflight_messages() { env.tick(); ticks += 1; assert!(ticks < MAX_TICKS); - transfer_status = message_status(&env, &transfer_message_id).unwrap(); // Verify that block `0` is only reported to exist in one place. let get_blocks_res = get_blocks_fn(&env, ledger_id, 0, 1); assert!(!ledger_reports_first_block_in_two_places( @@ -5277,15 +5290,6 @@ pub mod archiving { &get_blocks_res )); } - let transfer_result = Decode!( - &transfer_status.unwrap() - .bytes(), - Result - ) - .expect("failed to decode transfer response") - .map(|n| n.0.to_u64().unwrap()) - .expect("transfer should succeed"); - assert_eq!(transfer_result, NUM_INITIAL_BALANCES + i - 1); // An archive should exist assert!(!get_archives(&env, ledger_id).is_empty()); @@ -5818,7 +5822,7 @@ pub mod archiving { ) } - fn encode_transfer_args( + pub(crate) fn encode_transfer_args( from: impl Into, to: impl Into, amount: u64, @@ -5922,6 +5926,301 @@ pub mod archiving { } } +/// Tests covering how the ledger behaves when the subnet it runs on has no +/// memory left. +pub mod subnet_memory { + use super::*; + use ic_ledger_suite_state_machine_helpers::{balance_of, parse_metric, total_supply}; + use ic_management_canister_types_private::CanisterSettingsArgsBuilder; + use ic_state_machine_tests::StateMachineBuilder; + use ic_types::NumBytes; + use ic_universal_canister::{UNIVERSAL_CANISTER_WASM, wasm}; + use std::fmt::Debug; + + const WASM_PAGE_SIZE: u64 = 64 * 1024; + const MIB: u64 = 1024 * 1024; + const GIB: u64 = 1024 * MIB; + + /// Deliberately small, so that the filler canister can exhaust it. The + /// reservation threshold is set to the capacity, which switches storage + /// reservation off entirely (`ResourceSaturation::reservation_factor` + /// returns 0 once the two are equal) — this test is about running out of + /// memory, not about running out of reserved cycles. + const SUBNET_MEMORY_CAPACITY: u64 = 8 * GIB; + + /// Chunk sizes, in bytes, used to fill the subnet: each is grown repeatedly + /// until it no longer fits, so the leftover headroom ends up below the + /// smallest chunk. A single message can only grow memory by + /// `(capacity - usage) / scheduler_cores`, hence the first chunk is well + /// below the capacity. + const FILLER_CHUNKS: [u64; 4] = [GIB, 64 * MIB, MIB, WASM_PAGE_SIZE]; + + /// A transaction that arrives when the subnet is out of memory must leave + /// no trace. + /// + /// The ledger applies a transaction synchronously, so a trap while applying + /// it — here because the ledger cannot allocate the memory its first block + /// needs — discards the whole message: the caller gets an error, and no + /// block, balance or supply change is committed. + /// + /// This is the counterpart to the archiving atomicity test: a failure + /// *before* the ledger's first await is atomic, whereas one after it is + /// not, which is why archiving must not be awaited before replying. + /// + /// Note what it takes to get here. A full subnet is not by itself enough to + /// make a transaction fail: the ledger allocates in large chunks and serves + /// most transactions out of memory it already holds. This test relies on a + /// freshly installed ledger, whose first transaction has to allocate the + /// memory for its block log. + pub fn test_transfer_when_subnet_is_out_of_memory( + ledger_wasm: Vec, + encode_init_args: fn(InitArgs) -> T, + get_blocks_fn: fn( + &StateMachine, + CanisterId, + u64, + usize, + ) -> archiving::GenericGetBlocksResponse, + ) where + T: CandidType, + B: Eq + Debug, + { + let p1 = PrincipalId::new_user_test_id(1); + + let env = StateMachineBuilder::new() + .with_config(Some(StateMachineConfig::new( + SubnetConfig::new(SubnetType::Application), + HypervisorConfig { + subnet_memory_threshold: NumBytes::new(SUBNET_MEMORY_CAPACITY), + subnet_memory_capacity: NumBytes::new(SUBNET_MEMORY_CAPACITY), + // Memory held back for response callbacks. At the mainnet + // value it would exceed the scaled-down capacity. + subnet_memory_reservation: NumBytes::new(0), + ..HypervisorConfig::default() + }, + ))) + .build(); + + // A ledger with no initial balances has no blocks yet, so its first + // transaction is also its first allocation. + let ledger_id = env + .install_canister_with_cycles( + ledger_wasm, + Encode!(&encode_init_args(init_args(vec![]))).unwrap(), + None, + Cycles::new(100_000_000_000_000), + ) + .expect("failed to install the ledger"); + assert_eq!(get_blocks_fn(&env, ledger_id, 0, 1).chain_length, 0); + + // Fill the subnet. The filler only grows logical stable memory, which + // it never writes to, so this costs no host storage. + let filler = env + .install_canister_with_cycles( + UNIVERSAL_CANISTER_WASM.to_vec(), + vec![], + None, + Cycles::new(100_000_000_000_000_000), + ) + .expect("failed to install the filler canister"); + for chunk in FILLER_CHUNKS { + while grow_filler(&env, filler, chunk / WASM_PAGE_SIZE) {} + } + + // The mint has to allocate, and cannot. + let mint_result = env.execute_ingress_as( + PrincipalId(MINTER.owner), + ledger_id, + "icrc1_transfer", + archiving::encode_transfer_args(MINTER, p1.0, 10_000_000), + ); + let err = mint_result.expect_err("the mint should have failed on a full subnet"); + println!("mint on a full subnet failed with `{err}`"); + assert_eq!( + get_blocks_fn(&env, ledger_id, 0, 1).chain_length, + 0, + "the ledger rejected the mint with `{err}` but recorded a block anyway" + ); + assert_eq!( + balance_of(&env, ledger_id, p1.0), + 0, + "the ledger rejected the mint with `{err}` but minted funds anyway" + ); + assert_eq!( + total_supply(&env, ledger_id), + 0, + "the ledger rejected the mint with `{err}` but changed the total supply anyway" + ); + + // Give the memory back: the very same mint now succeeds, which shows + // the ledger was only held up by the subnet being full and that the + // failed attempt left it in a usable state. + env.uninstall_code(filler) + .expect("failed to uninstall the filler canister"); + env.execute_ingress_as( + PrincipalId(MINTER.owner), + ledger_id, + "icrc1_transfer", + archiving::encode_transfer_args(MINTER, p1.0, 10_000_000), + ) + .expect("the mint should succeed once the subnet has memory again"); + assert_eq!(get_blocks_fn(&env, ledger_id, 0, 1).chain_length, 1); + assert_eq!(balance_of(&env, ledger_id, p1.0), 10_000_000); + } + + /// Storage pressure the filler holds while the ledger works normally. + const PRESSURE_BASE_PAGES: u64 = 3 * GIB / WASM_PAGE_SIZE; + /// Added while the ledger is suspended in archiving, so that the next page + /// it grows reserves more cycles than its limit allows. + const PRESSURE_CROSSING_PAGES: u64 = 5 * GIB / WASM_PAGE_SIZE; + /// Small enough that the first page the archive continuation grows exceeds + /// it. + const LEDGER_RESERVED_CYCLES_LIMIT: u128 = 100_000_000; + /// `create_and_initialize_node_canister` wants at least 4.5T for the + /// archive and at least 10T left in the ledger. + const CYCLES_FOR_ARCHIVE_CREATION: u64 = 20_000_000_000_000; + const LEDGER_CYCLES: u128 = 200_000_000_000_000; + const FILLER_CYCLES: u128 = 100_000_000_000_000_000; + + /// An archiving attempt that traps must be counted. + /// + /// Archiving does not hold up the reply, so a round that traps is invisible + /// to the caller and `ledger_archiving_failures` is what is left to notice + /// it by. A trap discards everything the failing message did, including any + /// attempt to record it, so the count comes from the archiving guard's + /// destructor, which runs while the task is being canceled. + /// + /// The trap is produced the way it can be produced on mainnet: the subnet + /// is pushed over its storage reservation threshold while the ledger is + /// suspended in archiving, so the memory its continuation needs is refused + /// with `IC0534`. + pub fn test_trapped_archiving_is_counted( + ledger_wasm: Vec, + encode_init_args: fn(InitArgs) -> T, + get_archives: fn(&StateMachine, CanisterId) -> Vec, + get_blocks_fn: fn( + &StateMachine, + CanisterId, + u64, + usize, + ) -> archiving::GenericGetBlocksResponse, + ) where + T: CandidType, + B: Eq + Debug, + { + let p1 = PrincipalId::new_user_test_id(1); + let env = StateMachineBuilder::new() + .with_config(Some(StateMachineConfig::new( + SubnetConfig::new(SubnetType::Application), + HypervisorConfig { + // Reservation starts well below the capacity, so that the + // filler can push the subnet over the threshold while + // leaving the ledger room to be installed and to run. + subnet_memory_threshold: NumBytes::new(4 * GIB), + subnet_memory_capacity: NumBytes::new(64 * GIB), + subnet_memory_reservation: NumBytes::new(0), + ..HypervisorConfig::default() + }, + ))) + .build(); + + let filler = env + .install_canister_with_cycles( + UNIVERSAL_CANISTER_WASM.to_vec(), + vec![], + Some( + CanisterSettingsArgsBuilder::new() + // The filler reserves a lot when it crosses the + // threshold; it must not be capped by the default 5T. + .with_reserved_cycles_limit(FILLER_CYCLES) + .build(), + ), + Cycles::new(FILLER_CYCLES), + ) + .expect("failed to install the filler canister"); + + let mut init = init_args(vec![]); + init.archive_options.cycles_for_archive_creation = Some(CYCLES_FOR_ARCHIVE_CREATION); + let ledger_id = env + .install_canister_with_cycles( + ledger_wasm, + Encode!(&encode_init_args(init)).unwrap(), + Some( + CanisterSettingsArgsBuilder::new() + .with_reserved_cycles_limit(LEDGER_RESERVED_CYCLES_LIMIT) + .build(), + ), + Cycles::new(LEDGER_CYCLES), + ) + .expect("failed to install the ledger"); + assert!(grow_filler(&env, filler, PRESSURE_BASE_PAGES)); + assert_eq!(parse_metric(&env, ledger_id, ARCHIVING_FAILURES_METRIC), 0); + + // One block short of the archive trigger threshold, so that the next + // transfer is the one that archives. + for _ in 0..(ARCHIVE_TRIGGER_THRESHOLD - 1) { + transfer(&env, ledger_id, MINTER, p1.0, 10_000_000).expect("mint failed"); + } + assert!( + get_archives(&env, ledger_id).is_empty(), + "archiving must not have been triggered yet" + ); + + // Let the ledger apply the transfer and suspend in archiving, then push + // the subnet over the threshold while it is waiting. + let blocks_before = get_blocks_fn(&env, ledger_id, 0, 1).chain_length; + let message = env.send_ingress( + PrincipalId(MINTER.owner), + ledger_id, + "icrc1_transfer", + archiving::encode_transfer_args(MINTER, p1.0, 10_000_000), + ); + for _ in 0..100 { + if get_blocks_fn(&env, ledger_id, 0, 1).chain_length > blocks_before { + break; + } + env.tick(); + } + assert_eq!( + get_blocks_fn(&env, ledger_id, 0, 1).chain_length, + blocks_before + 1, + "the ledger did not commit the transfer" + ); + assert!(grow_filler(&env, filler, PRESSURE_CROSSING_PAGES)); + + // The ledger replies before archiving, so the transfer succeeds even + // though the archiving that follows it traps. + env.await_ingress(message, 100) + .expect("the transfer should have succeeded"); + env.run_until_completion(200); + + assert!( + get_archives(&env, ledger_id).is_empty(), + "expected the archive continuation to have been trapped, but an \ + archive was created; the test is not exercising a trapped attempt" + ); + assert_eq!( + parse_metric(&env, ledger_id, ARCHIVING_FAILURES_METRIC), + 1, + "the trapped archiving attempt was not counted" + ); + } + + const ARCHIVING_FAILURES_METRIC: &str = "ledger_archiving_failures"; + + /// Grows the filler's logical stable memory, returning whether it fit. + fn grow_filler(env: &StateMachine, filler: CanisterId, pages: u64) -> bool { + let reply = env + .execute_ingress( + filler, + "update", + wasm().stable64_grow(pages).reply_int64().build(), + ) + .expect("failed to call the filler canister"); + u64::from_le_bytes(reply.bytes()[..8].try_into().unwrap()) != u64::MAX + } +} + pub fn test_setting_fee_collector_to_minting_account( ledger_wasm: Vec, encode_init_args: fn(InitArgs) -> T,