diff --git a/rs/execution_environment/src/execution/call_or_task.rs b/rs/execution_environment/src/execution/call_or_task.rs index d93e842ad809..fc7a2964ff18 100644 --- a/rs/execution_environment/src/execution/call_or_task.rs +++ b/rs/execution_environment/src/execution/call_or_task.rs @@ -443,8 +443,10 @@ impl CallOrTaskHelper { } /// Replays the previous update call steps on the given clean canister. - /// Returns an error if any step fails. Otherwise, it returns an instance of - /// the helper that can be used to continue the update call execution. + /// Returns an error if the cycles balance of the clean canister dropped + /// below the cycles balance at the start of the DTS execution or if any step + /// fails. Otherwise, it returns an instance of the helper that can be used + /// to continue the update call execution. fn resume( clean_canister: &CanisterState, original: &OriginalContext, @@ -453,7 +455,13 @@ impl CallOrTaskHelper { ) -> Result { let mut helper = Self::new(clean_canister, original, deallocation_sender)?; helper.executed_wasm_instructions = paused.executed_wasm_instructions; - if helper.initial_cycles_balance != paused.initial_cycles_balance { + // The cycles balance of the clean canister must not decrease during the + // DTS execution: the recorded steps are replayed on the clean canister + // state and a lower balance might no longer be able to cover them. + // An increase is safe: all cycles changes of the DTS execution are + // applied relative to the balance of the clean canister state and hence + // the additional cycles are preserved. + if helper.initial_cycles_balance < paused.initial_cycles_balance { let msg = match original.call_or_task { CanisterCallOrTask::Update(_) => { "Mismatch in cycles balance when resuming an update call".to_string() diff --git a/rs/execution_environment/src/execution/install_code.rs b/rs/execution_environment/src/execution/install_code.rs index b03b53bd523d..ac9cad2caea6 100644 --- a/rs/execution_environment/src/execution/install_code.rs +++ b/rs/execution_environment/src/execution/install_code.rs @@ -267,7 +267,7 @@ impl InstallCodeHelper { } /// Replays the previous `install_code` steps on the given clean canister. - /// Returns an error if the cycles balance of the clean canister differs from + /// Returns an error if the cycles balance of the clean canister dropped below /// the cycles balance at the start of the DTS execution or if any step /// fails. Otherwise, it returns an instance of the helper that can be used /// to continue the `install_code` execution. @@ -299,9 +299,13 @@ impl InstallCodeHelper { .saturating_sub(executed_wasm_instructions.get()), ); - // The cycles balance of the clean canister must not change during the - // DTS execution. - if helper.initial_cycles_balance != paused.initial_cycles_balance { + // The cycles balance of the clean canister must not decrease during the + // DTS execution: the recorded steps are replayed on the clean canister + // state and a lower balance might no longer be able to cover them. + // An increase is safe: all cycles changes of the DTS execution are + // applied relative to the balance of the clean canister state and hence + // the additional cycles are preserved. + if helper.initial_cycles_balance < paused.initial_cycles_balance { let msg = "Mismatch in cycles balance when resuming an install code".to_string(); let err = HypervisorError::WasmEngineError(FailedToApplySystemChanges(msg)); let err = (clean_canister.canister_id(), err).into(); diff --git a/rs/execution_environment/src/execution/install_code/tests.rs b/rs/execution_environment/src/execution/install_code/tests.rs index 0acc8a66351e..fd2301cf011d 100644 --- a/rs/execution_environment/src/execution/install_code/tests.rs +++ b/rs/execution_environment/src/execution/install_code/tests.rs @@ -23,7 +23,7 @@ use ic_test_utilities_metrics::fetch_int_counter; use ic_types::ingress::{IngressState, IngressStatus, WasmResult}; use ic_types::messages::MessageId; use ic_types::{CanisterId, ComputeAllocation, MemoryAllocation, NumBytes, NumInstructions}; -use ic_types_cycles::{Cycles, CyclesUseCase, NominalCycles}; +use ic_types_cycles::{CompoundCycles, Cycles, CyclesUseCase, Instructions, NominalCycles}; use ic_types_test_utils::ids::{canister_test_id, subnet_test_id, user_test_id}; use ic_universal_canister::{UNIVERSAL_CANISTER_WASM, call_args, wasm}; use maplit::btreemap; @@ -139,19 +139,38 @@ fn consumed_cycles_for_instructions( .unwrap_or_default() } -/// Analogously to `dts_resume_fails_due_to_cycles_decrease` for calls, -/// replicated queries, callbacks, and tasks, resuming a paused `install_code` -/// whose canister lost cycles while it was paused fails instead of replaying the -/// recorded steps on a balance that can no longer cover them. The failed -/// execution is charged for exactly the instructions it had already executed, -/// including those of the paused Wasm execution. -#[test] -fn dts_install_code_resume_fails_due_to_cycles_decrease() { - const INSTRUCTION_LIMIT: u64 = 50_000_000; - const SLICE_INSTRUCTION_LIMIT: u64 = 132_000; +/// The instruction limits of the DTS `install_code` tests that change the cycles +/// balance of the canister while its execution is paused. +const DTS_INSTALL_CODE_INSTRUCTION_LIMIT: u64 = 50_000_000; +const DTS_INSTALL_CODE_SLICE_INSTRUCTION_LIMIT: u64 = 132_000; + +/// A canister with a paused `install_code` execution, along with a snapshot of +/// the accounting counters of that canister taken before the execution started. +/// +/// Shared by the tests that decrease and increase the cycles balance of the +/// canister while its `install_code` execution is paused. +struct PausedInstallCode { + test: ExecutionTest, + canister_id: CanisterId, + /// The ingress message of the `install_code` subnet message. + ingress_id: MessageId, + /// The cycles balance before the execution cycles were prepaid. + original_balance: Cycles, + /// The cycles consumed for instructions. + original_consumed_cycles: NominalCycles, + /// The instructions executed by all the slices of the canister. + original_executed_instructions: NumInstructions, + /// The accumulated cost of the instructions executed by the canister. + original_execution_cost: CompoundCycles, +} + +/// Starts an `install_code` execution that pauses after its first slice: that +/// slice compiles the Wasm module and executes its `(start)` function, which +/// together exceed the slice instruction limit. +fn install_code_paused_after_first_slice() -> PausedInstallCode { let mut test = ExecutionTestBuilder::new() - .with_install_code_instruction_limit(INSTRUCTION_LIMIT) - .with_install_code_slice_instruction_limit(SLICE_INSTRUCTION_LIMIT) + .with_install_code_instruction_limit(DTS_INSTALL_CODE_INSTRUCTION_LIMIT) + .with_install_code_slice_instruction_limit(DTS_INSTALL_CODE_SLICE_INSTRUCTION_LIMIT) .with_create_execution_state_base_cost(0) .with_manual_execution() .build(); @@ -169,6 +188,7 @@ fn dts_install_code_resume_fails_due_to_cycles_decrease() { let original_balance = test.canister_state(canister_id).system_state.balance(); let original_consumed_cycles = consumed_cycles_for_instructions(&test, canister_id); let original_executed_instructions = test.canister_executed_instructions(canister_id); + let original_execution_cost = test.canister_execution_cost(canister_id); let ingress_id = test.dts_install_code(payload); @@ -187,13 +207,42 @@ fn dts_install_code_resume_fails_due_to_cycles_decrease() { - test .cycles_account_manager() .execution_cost( - NumInstructions::from(INSTRUCTION_LIMIT), + NumInstructions::from(DTS_INSTALL_CODE_INSTRUCTION_LIMIT), test.get_own_subnet_cycles_config(), WASM_EXECUTION_MODE, ) .real(), ); + PausedInstallCode { + test, + canister_id, + ingress_id, + original_balance, + original_consumed_cycles, + original_executed_instructions, + original_execution_cost, + } +} + +/// Analogously to `dts_resume_fails_due_to_cycles_decrease` for calls, +/// replicated queries, callbacks, and tasks, resuming a paused `install_code` +/// whose canister lost cycles while it was paused fails instead of replaying the +/// recorded steps on a balance that can no longer cover them. The failed +/// execution is charged for exactly the instructions it had already executed, +/// including those of the paused Wasm execution. +#[test] +fn dts_install_code_resume_fails_due_to_cycles_decrease() { + let PausedInstallCode { + mut test, + canister_id, + ingress_id, + original_balance, + original_consumed_cycles, + original_executed_instructions, + .. + } = install_code_paused_after_first_slice(); + // Decrease the cycles balance of the clean canister. test.canister_state_mut(canister_id) .system_state @@ -224,7 +273,10 @@ fn dts_install_code_resume_fails_due_to_cycles_decrease() { // more than the slice instruction limit. let executed_instructions = test.canister_executed_instructions(canister_id) - original_executed_instructions; - assert_gt!(executed_instructions.get(), SLICE_INSTRUCTION_LIMIT); + assert_gt!( + executed_instructions.get(), + DTS_INSTALL_CODE_SLICE_INSTRUCTION_LIMIT + ); // The canister is charged exactly the cost of those instructions, including // the instructions of the paused Wasm execution: the rest of the prepaid @@ -245,6 +297,62 @@ fn dts_install_code_resume_fails_due_to_cycles_decrease() { ); } +/// Counterpart of `dts_install_code_resume_fails_due_to_cycles_decrease`: while +/// resuming a paused `install_code` whose canister lost cycles fails, an +/// increase of the cycles balance while the execution is paused is tolerated and +/// the additional cycles are not lost when the execution completes. +#[test] +fn dts_install_code_resume_succeeds_after_cycles_increase() { + const CYCLES_ADDED_WHILE_PAUSED: Cycles = Cycles::new(1_234_567_890); + + let PausedInstallCode { + mut test, + canister_id, + ingress_id, + original_balance, + original_executed_instructions, + original_execution_cost, + .. + } = install_code_paused_after_first_slice(); + + // Increase the cycles balance of the clean canister. + test.canister_state_mut(canister_id) + .system_state + .add_cycles(CYCLES_ADDED_WHILE_PAUSED); + + // The remaining slices resume the paused execution, which completes. + while test.canister_state(canister_id).next_execution() == NextExecution::ContinueInstallCode { + test.execute_slice(canister_id); + } + assert_eq!( + test.canister_state(canister_id).next_execution(), + NextExecution::None + ); + + let result = check_ingress_status(test.ingress_status(&ingress_id)).unwrap(); + assert_eq!(result, WasmResult::Reply(EmptyBlob.encode())); + + // The code has been installed. + assert!(test.canister_state(canister_id).execution_state.is_some()); + + // The execution spanned multiple slices. + let executed_instructions = + test.canister_executed_instructions(canister_id) - original_executed_instructions; + assert_gt!( + executed_instructions.get(), + DTS_INSTALL_CODE_SLICE_INSTRUCTION_LIMIT + ); + + // The canister is charged exactly the cost of the executed instructions: the + // cycles added while the execution was paused are not lost. + assert_eq!( + test.canister_state(canister_id).system_state.balance(), + original_balance + - (test.canister_execution_cost(canister_id) - original_execution_cost).real() + + CYCLES_ADDED_WHILE_PAUSED + ); +} + #[test] fn dts_abort_works_in_install_code() { const INSTRUCTION_LIMIT: u64 = 50_000_000; diff --git a/rs/execution_environment/src/execution/response.rs b/rs/execution_environment/src/execution/response.rs index bf560cea295e..5fbd95445753 100644 --- a/rs/execution_environment/src/execution/response.rs +++ b/rs/execution_environment/src/execution/response.rs @@ -339,8 +339,8 @@ impl ResponseHelper { /// call context, and execution state because it is not possible to invoke /// the cleanup callback in such cases. /// - /// It returns an error if the cycles balance of the clean canister differs - /// from the cycles balances at the start of the DTS execution. + /// It returns an error if the cycles balance of the clean canister dropped + /// below the cycles balance at the start of the DTS execution. #[allow(clippy::result_large_err)] fn resume( paused: PausedResponseHelper, @@ -387,9 +387,13 @@ impl ResponseHelper { .validate(&call_context, original, round, round_limits) .expect("Failed to resume DTS response: validation"); - // The cycles balance of the clean canister must not change during the - // DTS execution. - if helper.initial_cycles_balance != paused.initial_cycles_balance { + // The cycles balance of the clean canister must not decrease during the + // DTS execution: the initial steps are replayed on the clean canister + // state and a lower balance might no longer be able to cover them. + // An increase is safe: all cycles changes of the DTS execution are + // applied relative to the balance of the clean canister state and hence + // the additional cycles are preserved. + if helper.initial_cycles_balance < paused.initial_cycles_balance { let msg = "Mismatch in cycles balance when resuming a response call".to_string(); let err = HypervisorError::WasmEngineError(FailedToApplySystemChanges(msg)); return Err((helper, err)); diff --git a/rs/execution_environment/src/execution_environment/tests.rs b/rs/execution_environment/src/execution_environment/tests.rs index 72aca5104858..5d8b37818b8a 100644 --- a/rs/execution_environment/src/execution_environment/tests.rs +++ b/rs/execution_environment/src/execution_environment/tests.rs @@ -1,4 +1,5 @@ use crate::units::GIB as ONE_GIB; +use assert_matches::assert_matches; use candid::{Decode, Encode}; use ic_base_types::{NumBytes, NumSeconds}; use ic_btc_interface::NetworkInRequest; @@ -39,8 +40,8 @@ use ic_types::{ consensus::idkg::{IDkgMasterPublicKeyId, PreSigId}, ingress::{IngressState, IngressStatus, WasmResult}, messages::{ - CallbackId, CanisterTask, MAX_RESPONSE_COUNT_BYTES, NO_DEADLINE, Payload, RejectContext, - RequestOrResponse, Response, + CallbackId, CanisterTask, MAX_RESPONSE_COUNT_BYTES, MessageId, NO_DEADLINE, Payload, + RejectContext, RequestOrResponse, Response, }, time::UNIX_EPOCH, }; @@ -6122,9 +6123,134 @@ fn paused_execution_fails_to_resume_after_cycles_decrease( ); } +/// A canister that is about to start a long-running execution: the first slice +/// of that execution exceeds the slice instruction limit and hence pauses. +/// +/// Shared by the tests that decrease and increase the cycles balance of the +/// canister while its execution is paused. +struct LongRunningExecution { + test: ExecutionTest, + /// The canister whose long-running execution pauses. + canister_id: CanisterId, + /// The ingress message whose status reflects the outcome of the long-running + /// execution; `None` for canister tasks, which have no ingress status. + ingress_id: Option, +} + +/// Sets up a long-running update call (`method` is `"update"`) or replicated +/// query (`method` is `"query"`). +fn long_running_call(method: &str) -> LongRunningExecution { + let mut test = ExecutionTestBuilder::new() + .with_instruction_limit(1_000_000) + .with_slice_instruction_limit(200_000) + .with_manual_execution() + .build(); + + let a_id = test.universal_canister().unwrap(); + + let a = wasm() + .instruction_counter_is_at_least(200_000) + .message_payload() + .append_and_reply() + .build(); + + let (ingress_id, _) = test.ingress_raw(a_id, method, a); + + LongRunningExecution { + test, + canister_id: a_id, + ingress_id: Some(ingress_id), + } +} + +/// Sets up a long-running response callback, or a long-running cleanup callback +/// if `cleanup` is set. In the latter case the response callback traps, so that +/// the long-running callback is the cleanup one. +fn long_running_callback(cleanup: bool) -> LongRunningExecution { + let mut test = ExecutionTestBuilder::new() + .with_instruction_limit(100_000_000) + .with_slice_instruction_limit(1_000_000) + .with_manual_execution() + .build(); + + let a_id = test.universal_canister().unwrap(); + let b_id = test.universal_canister().unwrap(); + + let b = wasm().message_payload().append_and_reply().build(); + + let long_execution = wasm() + .instruction_counter_is_at_least(1_000_000) + .message_payload() + .append_and_reply() + .build(); + let call_args = if cleanup { + call_args() + .other_side(b) + .on_reply(wasm().trap()) + .on_cleanup(long_execution) + } else { + call_args().other_side(b).on_reply(long_execution) + }; + let a = wasm().call_simple(b_id, "update", call_args).build(); + + let (ingress_id, _) = test.ingress_raw(a_id, "update", a); + + // Canister A calls canister B, which replies. + test.execute_message(a_id); + test.induct_messages(); + test.execute_message(b_id); + test.induct_messages(); + + LongRunningExecution { + test, + canister_id: a_id, + ingress_id: Some(ingress_id), + } +} + +/// Sets up a long-running canister task: the heartbeat. It grows the stable +/// memory before the execution is paused, so that its state changes can be +/// checked to be either dropped or kept, depending on whether resuming the +/// execution fails or succeeds. +fn long_running_heartbeat() -> LongRunningExecution { + let mut test = ExecutionTestBuilder::new() + .with_instruction_limit(100_000_000) + .with_slice_instruction_limit(1_000_000) + .with_manual_execution() + .build(); + + let canister_id = test.universal_canister().unwrap(); + let (ingress_id, _) = test.ingress_raw( + canister_id, + "update", + wasm() + .set_heartbeat( + wasm() + .stable_grow(1) + .instruction_counter_is_at_least(1_000_000) + .build(), + ) + .reply() + .build(), + ); + test.execute_message(canister_id); + check_ingress_status(test.ingress_status(&ingress_id)).unwrap(); + + test.canister_state_mut(canister_id) + .system_state + .task_queue + .enqueue(ExecutionTask::Heartbeat); + + LongRunningExecution { + test, + canister_id, + ingress_id: None, + } +} + /// Every kind of paused execution re-creates its helper from the current clean /// canister state when it is resumed, so it relies on the cycles balance of that -/// state not changing while the execution is paused. This test covers all of +/// state not decreasing while the execution is paused. This test covers all of /// them: update calls, replicated queries, response callbacks, cleanup /// callbacks, and canister tasks (heartbeat). The `install_code` case is covered /// by `dts_install_code_resume_fails_due_to_cycles_decrease`. @@ -6132,25 +6258,15 @@ fn paused_execution_fails_to_resume_after_cycles_decrease( fn dts_resume_fails_due_to_cycles_decrease() { // 1. Update calls and replicated queries. for method in ["update", "query"] { - let mut test = ExecutionTestBuilder::new() - .with_instruction_limit(1_000_000) - .with_slice_instruction_limit(200_000) - .with_manual_execution() - .build(); - - let a_id = test.universal_canister().unwrap(); - - let a = wasm() - .instruction_counter_is_at_least(200_000) - .message_payload() - .append_and_reply() - .build(); - - let (ingress_id, _) = test.ingress_raw(a_id, method, a); + let LongRunningExecution { + mut test, + canister_id, + ingress_id, + } = long_running_call(method); - paused_execution_fails_to_resume_after_cycles_decrease(&mut test, a_id); + paused_execution_fails_to_resume_after_cycles_decrease(&mut test, canister_id); - let err = check_ingress_status(test.ingress_status(&ingress_id)).unwrap_err(); + let err = check_ingress_status(test.ingress_status(&ingress_id.unwrap())).unwrap_err(); let message = if method == "update" { "an update call" } else { @@ -6159,54 +6275,24 @@ fn dts_resume_fails_due_to_cycles_decrease() { err.assert_contains( ErrorCode::CanisterWasmEngineError, &format!( - "Error from Canister {a_id}: Canister encountered a Wasm engine error: \ + "Error from Canister {canister_id}: Canister encountered a Wasm engine error: \ Failed to apply system changes: Mismatch in cycles \ balance when resuming {message}" ), ); } - // 2. Response and cleanup callbacks. The response callback traps in the - // cleanup case, so that the long-running callback is the cleanup one. + // 2. Response and cleanup callbacks. for cleanup in [false, true] { - let mut test = ExecutionTestBuilder::new() - .with_instruction_limit(100_000_000) - .with_slice_instruction_limit(1_000_000) - .with_manual_execution() - .build(); - - let a_id = test.universal_canister().unwrap(); - let b_id = test.universal_canister().unwrap(); - - let b = wasm().message_payload().append_and_reply().build(); - - let long_execution = wasm() - .instruction_counter_is_at_least(1_000_000) - .message_payload() - .append_and_reply() - .build(); - let call_args = if cleanup { - call_args() - .other_side(b) - .on_reply(wasm().trap()) - .on_cleanup(long_execution) - } else { - call_args().other_side(b).on_reply(long_execution) - }; - let a = wasm().call_simple(b_id, "update", call_args).build(); - - let (ingress_id, _) = test.ingress_raw(a_id, "update", a); - - // Canister A calls canister B, which replies. - test.execute_message(a_id); - test.induct_messages(); - test.execute_message(b_id); - test.induct_messages(); + let LongRunningExecution { + mut test, + canister_id, + ingress_id, + } = long_running_callback(cleanup); - // Start executing the response|cleanup callback. - paused_execution_fails_to_resume_after_cycles_decrease(&mut test, a_id); + paused_execution_fails_to_resume_after_cycles_decrease(&mut test, canister_id); - let err = check_ingress_status(test.ingress_status(&ingress_id)).unwrap_err(); + let err = check_ingress_status(test.ingress_status(&ingress_id.unwrap())).unwrap_err(); let code = if cleanup { // The error of the trapping response callback takes precedence. ErrorCode::CanisterCalledTrap @@ -6225,38 +6311,13 @@ fn dts_resume_fails_due_to_cycles_decrease() { // 3. Canister tasks, e.g. the heartbeat. { - let mut test = ExecutionTestBuilder::new() - .with_instruction_limit(100_000_000) - .with_slice_instruction_limit(1_000_000) - .with_manual_execution() - .build(); - - let canister_id = test.universal_canister().unwrap(); - let (ingress_id, _) = test.ingress_raw( + let LongRunningExecution { + mut test, canister_id, - "update", - wasm() - .set_heartbeat( - // The stable memory is grown before the execution is paused - // so that the resume can be checked to drop that change. - wasm() - .stable_grow(1) - .instruction_counter_is_at_least(1_000_000) - .build(), - ) - .reply() - .build(), - ); - test.execute_message(canister_id); - check_ingress_status(test.ingress_status(&ingress_id)).unwrap(); + .. + } = long_running_heartbeat(); let stable_memory_size = test.execution_state(canister_id).stable_memory.size; - test.canister_state_mut(canister_id) - .system_state - .task_queue - .enqueue(ExecutionTask::Heartbeat); - - // Start executing the heartbeat. paused_execution_fails_to_resume_after_cycles_decrease(&mut test, canister_id); // A canister task has no ingress status and the failure is not recorded @@ -6268,3 +6329,144 @@ fn dts_resume_fails_due_to_cycles_decrease() { ); } } + +/// The cycles added to the cycles balance of a canister while its execution is +/// paused. +const CYCLES_ADDED_WHILE_PAUSED: Cycles = Cycles::new(1_234_567_890); + +/// Executes the first slice of the next execution of the given canister, which +/// must pause, then adds `CYCLES_ADDED_WHILE_PAUSED` to the cycles balance of +/// that canister if `add_cycles` is set, and finally executes all the remaining +/// slices of that execution. +/// +/// Asserts that resuming the paused execution did not fail: a failed resume +/// aborts the paused Wasm execution without executing any further instructions, +/// so the instructions executed by the remaining slices witness that the Wasm +/// execution was resumed. +/// +/// Returns the cycles balance of the canister after the execution has finished. +fn paused_execution_resumes_after_cycles_increase( + test: &mut ExecutionTest, + canister_id: CanisterId, + add_cycles: bool, +) -> Cycles { + test.execute_slice(canister_id); + assert_eq!( + test.canister_state(canister_id).next_execution(), + NextExecution::ContinueLong, + ); + let executed_instructions_when_paused = test.canister_executed_instructions(canister_id); + + if add_cycles { + test.canister_state_mut(canister_id) + .system_state + .add_cycles(CYCLES_ADDED_WHILE_PAUSED); + } + + while test.canister_state(canister_id).next_execution() == NextExecution::ContinueLong { + test.execute_slice(canister_id); + } + assert_eq!( + test.canister_state(canister_id).next_execution(), + NextExecution::None, + ); + assert_gt!( + test.canister_executed_instructions(canister_id), + executed_instructions_when_paused + ); + + test.canister_state(canister_id).system_state.balance() +} + +/// Counterpart of `dts_resume_fails_due_to_cycles_decrease`: while resuming a +/// paused execution fails if the cycles balance of the canister decreased in the +/// meantime, an increase of that balance is tolerated and the additional cycles +/// are not lost when the execution completes. This test covers the same kinds of +/// paused executions; the `install_code` case is covered by +/// `dts_install_code_resume_succeeds_after_cycles_increase`. +/// +/// Every scenario is executed twice, once without adding any cycles and once +/// with adding `CYCLES_ADDED_WHILE_PAUSED` while the execution is paused. The +/// two runs are identical otherwise, so the final cycles balances must differ by +/// exactly the added cycles. +#[test] +fn dts_resume_succeeds_after_cycles_increase() { + // 1. Update calls and replicated queries. + for method in ["update", "query"] { + let mut balances = vec![]; + for add_cycles in [false, true] { + let LongRunningExecution { + mut test, + canister_id, + ingress_id, + } = long_running_call(method); + + balances.push(paused_execution_resumes_after_cycles_increase( + &mut test, + canister_id, + add_cycles, + )); + + // The execution completed successfully. + let result = check_ingress_status(test.ingress_status(&ingress_id.unwrap())).unwrap(); + assert_matches!(result, WasmResult::Reply(_)); + } + assert_eq!(balances[1], balances[0] + CYCLES_ADDED_WHILE_PAUSED); + } + + // 2. Response and cleanup callbacks. + for cleanup in [false, true] { + let mut balances = vec![]; + for add_cycles in [false, true] { + let LongRunningExecution { + mut test, + canister_id, + ingress_id, + } = long_running_callback(cleanup); + + balances.push(paused_execution_resumes_after_cycles_increase( + &mut test, + canister_id, + add_cycles, + )); + + let status = check_ingress_status(test.ingress_status(&ingress_id.unwrap())); + if cleanup { + // The response callback traps, but the cleanup callback resumed + // and completed successfully. + let err = status.unwrap_err(); + assert_eq!(err.code(), ErrorCode::CanisterCalledTrap); + } else { + assert_matches!(status.unwrap(), WasmResult::Reply(_)); + } + } + assert_eq!(balances[1], balances[0] + CYCLES_ADDED_WHILE_PAUSED); + } + + // 3. Canister tasks, e.g. the heartbeat. + { + let mut balances = vec![]; + for add_cycles in [false, true] { + let LongRunningExecution { + mut test, + canister_id, + .. + } = long_running_heartbeat(); + let stable_memory_size = test.execution_state(canister_id).stable_memory.size; + + balances.push(paused_execution_resumes_after_cycles_increase( + &mut test, + canister_id, + add_cycles, + )); + + // A canister task has no ingress status, but the state changes of + // the heartbeat must have been kept. + assert_gt!( + test.execution_state(canister_id).stable_memory.size, + stable_memory_size + ); + } + assert_eq!(balances[1], balances[0] + CYCLES_ADDED_WHILE_PAUSED); + } +}