diff --git a/rs/ethereum/cketh/minter/BUILD.bazel b/rs/ethereum/cketh/minter/BUILD.bazel index 48878d79f02a..986451ce9a5f 100644 --- a/rs/ethereum/cketh/minter/BUILD.bazel +++ b/rs/ethereum/cketh/minter/BUILD.bazel @@ -206,6 +206,7 @@ rust_ic_test_suite( "tests/deposit_from_cex_demo.rs", "tests/deposit_from_cex.rs", "tests/sweeper_funding.rs", + "tests/sweeper_funding_hardening.rs", ], ), data = [ @@ -321,6 +322,36 @@ rust_test( ], ) +rust_test( + name = "sweeper_funding_hardening", + # Three live scenarios, two of which watch for longer than a withdrawal-timer tick to show the + # minter does *not* act. Slower than the happy path by design. + size = "enormous", + timeout = "eternal", + srcs = ["tests/sweeper_funding_hardening.rs"], + data = [ + ":cketh_minter_debug.wasm.gz", + "//:anvil", + "//rs/ledger_suite/icrc1/ledger:ledger_canister_u256.wasm.gz", + "//rs/pocket_ic_server:pocket-ic-server", + "@evm_rpc.wasm.gz//file", + ], + env = { + "ANVIL_BIN": "$(rootpath //:anvil)", + "CARGO_MANIFEST_DIR": "rs/ethereum/cketh/minter", + "CKETH_MINTER_WASM_PATH": "$(rootpath :cketh_minter_debug.wasm.gz)", + "EVM_RPC_CANISTER_WASM_PATH": "$(rootpath @evm_rpc.wasm.gz//file)", + "LEDGER_CANISTER_WASM_PATH": "$(rootpath //rs/ledger_suite/icrc1/ledger:ledger_canister_u256.wasm.gz)", + "POCKET_IC_BIN": "$(rootpath //rs/pocket_ic_server:pocket-ic-server)", + }, + tags = ["long_test"], # ~9 minutes; two scenarios watch for 8 to show the minter does nothing. + deps = [ + # Keep sorted. + ":minter", + "//rs/ethereum/cketh/test_utils", + ], +) + rust_test( name = "sweeper_funding", # Waits on the minter's 6-minute withdrawal timer for the funding transfer to be sent. Its own diff --git a/rs/ethereum/cketh/minter/tests/sweeper_funding.rs b/rs/ethereum/cketh/minter/tests/sweeper_funding.rs index c48146d3cf7c..148863d69e78 100644 --- a/rs/ethereum/cketh/minter/tests/sweeper_funding.rs +++ b/rs/ethereum/cketh/minter/tests/sweeper_funding.rs @@ -18,7 +18,7 @@ fn should_fund_the_sweeper_address_by_burning_cketh_from_the_fee_account() { let fee_account_before = setup.cketh_balance_of(setup.fee_account()); let minter_eth_before = setup.anvil_eth_balance(&setup.minter_address()); - let sweeper = setup.await_funding_decision(Duration::from_secs(120)); + let sweeper = setup.await_sweeper_address(Duration::from_secs(180)); assert_eq!( setup.anvil_eth_balance(&sweeper), 0, diff --git a/rs/ethereum/cketh/minter/tests/sweeper_funding_hardening.rs b/rs/ethereum/cketh/minter/tests/sweeper_funding_hardening.rs new file mode 100644 index 000000000000..05ef84b8b726 --- /dev/null +++ b/rs/ethereum/cketh/minter/tests/sweeper_funding_hardening.rs @@ -0,0 +1,229 @@ +//! Adversarial end-to-end coverage of sweeper fee funding, complementing the happy path in +//! `sweeper_funding.rs`. Each test drives a way funding can go wrong through the real pipeline and +//! asserts the minter fails safe. +//! +//! Two are bounded *negative* assertions — "the minter must not do X" cannot be proven outright, so +//! they watch for longer than a withdrawal-timer tick. +//! +//! No live fee-spike test: that ceiling is pinned exactly by the unit tests, and reproducing it here +//! would mean driving anvil's base fee up and waiting out several ticks for little extra signal. + +use ic_cketh_test_utils::sweeper_funding::{FEE_ACCOUNT_BALANCE, SweeperFundingSetup}; +use std::time::Duration; + +/// Longer than a withdrawal-timer tick, so a transfer that was going to happen would have. +const OBSERVATION_WINDOW: Duration = Duration::from_secs(8 * 60); +const ABOVE_LOW_WATER_MARK: u128 = 500_000_000_000_000_000; // 0.5 ETH +/// Sending and finalizing both wait on the 6-minute withdrawal timer, so allow for two ticks. +const FINALIZATION_DEADLINE: Duration = Duration::from_secs(15 * 60); +/// How long to wait for the minter to record a funding it has already burned for. Generous for an +/// inter-canister hop, yet far below the six minutes before the transaction can finalize and clear +/// the row again. +const IN_FLIGHT_DEADLINE: Duration = Duration::from_secs(2 * 60); + +#[test] +fn should_not_fund_when_the_fee_account_is_empty() { + let setup = SweeperFundingSetup::new_live_with_empty_fee_account(); + assert_eq!(setup.cketh_balance_of(setup.fee_account()), 0); + + let sweeper = setup.await_sweeper_address(Duration::from_secs(180)); + let supply_before = setup.cketh_total_supply(); + let minter_eth_before = setup.anvil_eth_balance(&setup.minter_address()); + + setup.assert_no_eth_received(&sweeper, OBSERVATION_WINDOW); + + assert_eq!( + setup.cketh_total_supply(), + supply_before, + "nothing may be burned when the fee account cannot cover the funding" + ); + assert_eq!( + setup.anvil_eth_balance(&setup.minter_address()), + minter_eth_before, + "no ETH may leave the main address" + ); + assert!( + setup + .minter_logs() + .iter() + .any(|line| line.contains("[fund_sweeper]") && line.contains("failed to burn")), + "the minter should report why funding was skipped; logs:\n{}", + setup.minter_logs().join("\n") + ); +} + +/// A sweeper that still holds plenty of gas must not be topped up: burning ckETH for gas already in +/// place would be pure loss, and it is the failure mode a wrongly-defaulted balance read would +/// cause. +#[test] +fn should_not_fund_a_sweeper_above_the_low_water_mark() { + // Starts with an empty fee account so the install-time check cannot fund anything: it decides a + // funding is due, fails to burn, and changes nothing. That is the only window in which the + // sweeper can be arranged, since its address is undiscoverable until the minter caches its key. + let setup = SweeperFundingSetup::new_live_with_empty_fee_account(); + let sweeper = setup.await_sweeper_address(Duration::from_secs(180)); + // Waits for the install-time check to have *finished* failing, not merely for the address to + // exist: it reads the balance and attempts its burn immediately afterwards, so funding the fee + // account any earlier would let that burn succeed and a funding proceed. + setup.await_minter_log( + "[fund_sweeper]: SKIPPING: failed to burn", + Duration::from_secs(180), + ); + + setup.set_eth_balance(&sweeper, ABOVE_LOW_WATER_MARK); + // The minter reads at `finalized`, which trails `latest` by two blocks. + setup.mine(3); + setup.mint_cketh(setup.fee_account(), FEE_ACCOUNT_BALANCE); + + // Captured before the timers are re-armed, not after: the post-upgrade check runs on a + // zero-delay timer, so a minter that wrongly funded could burn before these queries returned + // and both assertions below would then compare against an already-debited state — passing + // precisely when the behaviour they reject had happened. + let supply_before = setup.cketh_total_supply(); + let fee_account_before = setup.cketh_balance_of(setup.fee_account()); + + // The next scheduled check is a whole interval away, so re-arm the timers: from here a funding + // could succeed, and the point is that it declines. + setup.upgrade_minter(); + + let start = std::time::Instant::now(); + while start.elapsed() <= OBSERVATION_WINDOW { + std::thread::sleep(Duration::from_secs(10)); + // Keeps the PocketIC instance — and the minter's timers — alive. + let _ = setup.cketh_total_supply(); + } + + assert_eq!( + setup.cketh_total_supply(), + supply_before, + "a topped-up sweeper must not trigger a burn" + ); + assert_eq!( + setup.cketh_balance_of(setup.fee_account()), + fee_account_before, + "the fee account must be untouched" + ); + assert_eq!( + setup.anvil_eth_balance(&sweeper), + ABOVE_LOW_WATER_MARK, + "the sweeper balance must be left exactly as it was" + ); + // Without this the test passes for the wrong reason: a task that never read the balance also + // produces no burn. Proving it ran and *declined* is the point. + let prepaid = setup + .dashboard_row("sweeper-prepaid-gas") + .expect("the dashboard must have a prepaid-gas row"); + assert_ne!( + prepaid, "never observed", + "the funding task must have observed the balance and declined, not merely skipped" + ); +} + +/// A funding transaction that fails on chain is never reimbursed; the burn becomes prepaid gas. +/// +/// Has to be arranged, because it is otherwise unreachable: the sweeper is a code-less EOA precisely +/// so a bare transfer cannot fail. Placing code there leaves the 21'000 base gas with nothing to run +/// it, so the transaction fails. +#[test] +fn should_not_reimburse_a_funding_transaction_that_fails_on_chain() { + // Starts with an empty fee account for the same reason as the test above: the sweeper cannot be + // arranged before the minter derives its address, and a funded fee account would let the + // funding it attempts right afterwards succeed while the sweeper is still a plain EOA. + let setup = SweeperFundingSetup::new_live_with_empty_fee_account(); + let sweeper = setup.await_sweeper_address(Duration::from_secs(180)); + setup.await_minter_log( + "[fund_sweeper]: SKIPPING: failed to burn", + Duration::from_secs(180), + ); + + // PUSH1 0, PUSH1 0, REVERT — reverts on any call, with no return data. + setup.set_code(&sweeper, &[0x60, 0x00, 0x60, 0x00, 0xfd]); + // Read back rather than assumed: an arrangement placed on the wrong account makes the whole + // test vacuous, and the transfer then simply succeeds. + assert!( + !setup.code(&sweeper).is_empty(), + "the reverting code must be at {sweeper}" + ); + + setup.mint_cketh(setup.fee_account(), FEE_ACCOUNT_BALANCE); + let supply_before = setup.cketh_total_supply(); + // The next scheduled check is a whole interval away, so re-arm the timers: from here the + // funding proceeds, and the point is what happens when its transaction fails. + setup.upgrade_minter(); + + let burned = await_burn(&setup, supply_before, Duration::from_secs(180)); + assert!(burned > 0, "funding must burn ckETH up front"); + // Polled, not read once: the minter records the funding only after the ledger call it awaited + // returns, so the supply `await_burn` watches drops before the dashboard shows the request. + // Bounded well below the time to finalization, since the row clears again once that happens. + let burn_index = await_in_flight_burn_index(&setup, IN_FLIGHT_DEADLINE); + + // Waits for the transaction to finalize rather than watching for a fixed window: without this + // the assertions below all hold while it is merely still in flight, which proves nothing about + // what happens when it fails. + setup.await_funding_finalized(FINALIZATION_DEADLINE); + let status = setup.withdrawal_status(burn_index); + // Pending reimbursement is imprecise here — nothing will ever settle it — and deliberately so: + // a status of its own meant adding a variant to `retrieve_eth_status`, which breaks every + // existing client, to describe a state mainnet cannot reach. This test reaches it only by + // placing code at an address derived from the minter's own key. The invariant that actually + // matters is asserted below: the burn is never paid back. + assert!( + status.starts_with("PendingReimbursement("), + "unexpected status for a failed funding: {status} (sweeper {sweeper}, {} bytes of code, \ + balance {})", + setup.code(&sweeper).len(), + setup.anvil_eth_balance(&sweeper), + ); + + assert_eq!( + setup.anvil_eth_balance(&sweeper), + 0, + "the transfer failed, so no ETH may have reached the sweeper" + ); + assert_eq!( + setup.cketh_total_supply(), + supply_before - burned, + "a failed funding must NOT be reimbursed — the supply must stay reduced" + ); + let surplus = setup + .dashboard_row("sweeper-burned-not-yet-spent") + .expect("the dashboard must report the unspent burn"); + assert_ne!( + surplus, "0 Wei", + "the unreimbursed burn must be tracked as prepaid gas, got {surplus}" + ); +} + +/// The burn index of the funding the minter currently has in flight, waiting for it to appear. +fn await_in_flight_burn_index(setup: &SweeperFundingSetup, deadline: Duration) -> u64 { + let start = std::time::Instant::now(); + loop { + if let Some(index) = setup.in_flight_funding_burn_index() { + return index; + } + assert!( + start.elapsed() <= deadline, + "the minter burned ckETH but recorded no in-flight funding within {deadline:?}; \ + minter logs:\n{}", + setup.minter_logs().join("\n") + ); + std::thread::sleep(Duration::from_secs(2)); + } +} + +fn await_burn(setup: &SweeperFundingSetup, supply_before: u128, deadline: Duration) -> u128 { + let start = std::time::Instant::now(); + loop { + let supply = setup.cketh_total_supply(); + if supply < supply_before { + return supply_before - supply; + } + assert!( + start.elapsed() <= deadline, + "no burn observed within {deadline:?}; minter logs:\n{}", + setup.minter_logs().join("\n") + ); + std::thread::sleep(Duration::from_secs(2)); + } +} diff --git a/rs/ethereum/cketh/test_utils/src/sweeper_funding.rs b/rs/ethereum/cketh/test_utils/src/sweeper_funding.rs index 87a4ea3436e1..02b4b575f80f 100644 --- a/rs/ethereum/cketh/test_utils/src/sweeper_funding.rs +++ b/rs/ethereum/cketh/test_utils/src/sweeper_funding.rs @@ -23,9 +23,10 @@ use candid::{Decode, Encode, Nat, Principal}; use evm_rpc_types::{InstallArgs, OverrideProvider, RegexSubstitution}; -use ic_cketh_minter::endpoints::CandidBlockTag; +use ic_cketh_minter::endpoints::{CandidBlockTag, RetrieveEthStatus}; use ic_cketh_minter::lifecycle::{EthereumNetwork, MinterArg, init::InitArg as MinterInitArgs}; use ic_ethereum_types::Address; +use ic_http_types::{HttpRequest, HttpResponse}; use ic_icrc1_ledger::{FeatureFlags, LedgerArgument}; use icrc_ledger_types::icrc1::account::Account; use icrc_ledger_types::icrc1::transfer::{TransferArg, TransferError}; @@ -210,18 +211,52 @@ impl SweeperFundingSetup { } } - /// The sweeper address the minter derived, scraped from its log line: there is no getter for it - /// yet, and it cannot be derived test-side without the master public key. + /// The sweeper address the minter derived, read from its dashboard: available as soon as the + /// master public key is, i.e. before the funding task has decided anything. pub fn sweeper_address(&self) -> Option
{ - self.minter_logs().iter().find_map(|line| { - let rest = line.split("[fund_sweeper]: ").nth(1)?; - let hex = rest.split_whitespace().next()?; - hex.parse().ok() - }) + let dashboard = self.dashboard_html(); + let row = dashboard.split(r#"id="sweeper-address""#).nth(1)?; + // Bounded to the row: until the minter has cached its key the cell holds no address at all, + // and an unbounded search then runs on into later rows and returns an unrelated one — the + // deposit helper contract's — as if it were the sweeper's, leaving a test to arrange the + // wrong account. Returning `None` is what keeps `await_sweeper_address` waiting. + let row = row.split("").next()?; + let start = row.find("0x")?; + row.get(start..start + 42)?.parse().ok() + } + + /// The minter's rendered dashboard. + pub fn dashboard_html(&self) -> String { + let request = HttpRequest { + method: "GET".to_string(), + url: "/dashboard".to_string(), + headers: vec![], + body: serde_bytes::ByteBuf::new(), + }; + let reply = self + .env + .query_call( + self.minter_id, + Principal::anonymous(), + "http_request", + Encode!(&request).unwrap(), + ) + .expect("the dashboard query was rejected"); + let response = Decode!(&reply, HttpResponse).unwrap(); + String::from_utf8_lossy(&response.body).to_string() + } + + /// The value rendered in a dashboard row, e.g. `sweeper-cketh-burned`. + pub fn dashboard_row(&self, id: &str) -> Option { + let dashboard = self.dashboard_html(); + let row = dashboard.split(&format!(r#"id="{id}""#)).nth(1)?; + let cell = row.split("").nth(1)?.split("").next()?; + Some(cell.trim().to_string()) } - /// Waits until the funding task has logged the sweeper address, polling a canister meanwhile. - pub fn await_funding_decision(&self, deadline: Duration) -> Address { + /// Waits until the minter has derived its sweeper address, which happens once the master public + /// key is cached — before the first funding check acts on it. + pub fn await_sweeper_address(&self, deadline: Duration) -> Address { let start = Instant::now(); loop { if let Some(address) = self.sweeper_address() { @@ -229,13 +264,118 @@ impl SweeperFundingSetup { } assert!( start.elapsed() <= deadline, - "the funding task did not decide to fund within {deadline:?}; minter logs:\n{}", + "the minter did not derive a sweeper address within {deadline:?}; minter logs:\n{}", + self.minter_logs().join("\n") + ); + std::thread::sleep(Duration::from_secs(2)); + } + } + + /// Credits `address` with `wei` on the owned anvil node. + pub fn set_eth_balance(&self, address: &Address, wei: u128) { + self.anvil.set_balance(address, wei); + } + + /// Waits until the minter has logged a line containing `needle`, polling its log endpoint. + pub fn await_minter_log(&self, needle: &str, deadline: Duration) { + let start = Instant::now(); + loop { + let logs = self.minter_logs(); + if logs.iter().any(|line| line.contains(needle)) { + return; + } + assert!( + start.elapsed() <= deadline, + "the minter never logged {needle:?} within {deadline:?}; logs:\n{}", + logs.join("\n") + ); + std::thread::sleep(Duration::from_secs(2)); + } + } + + /// Places code at `address`, so a plain value transfer to it no longer succeeds: with the + /// 21'000 gas of a bare transfer there is nothing left to execute it. + pub fn set_code(&self, address: &Address, code: &[u8]) { + self.anvil.set_code(address, code); + } + + /// The runtime bytecode at `address`, so a test can check that what it arranged is where it + /// meant to put it rather than assuming so. + pub fn code(&self, address: &Address) -> Vec { + self.anvil.code(address) + } + + /// Asserts that `address` receives no ETH for `window`, polling a canister throughout so the + /// PocketIC instance stays alive. A bounded negative check — the best available shape for + /// "the minter must not do this" — sized well beyond one withdrawal-timer tick. + pub fn assert_no_eth_received(&self, address: &Address, window: Duration) { + let start = Instant::now(); + while start.elapsed() <= window { + self.anvil.mine(1); + let balance = self.anvil.eth_balance(address, "latest"); + assert_eq!( + balance, + 0, + "{address} unexpectedly received {balance} wei; minter logs:\n{}", + self.minter_logs().join("\n") + ); + // Keeps the instance alive, and its timers with it. + let _ = self.cketh_total_supply(); + std::thread::sleep(Duration::from_secs(5)); + } + } + + /// Waits until the in-flight funding row clears, i.e. its transaction has finalized, mining + /// meanwhile so the minter's `finalized` view keeps advancing. + /// + /// Polls the dashboard rather than the status endpoint: that endpoint is an update call, and + /// several minutes of ingress messages at a few seconds apart is enough load to destabilise the + /// PocketIC instance. + pub fn await_funding_finalized(&self, deadline: Duration) { + let start = Instant::now(); + loop { + if self.dashboard_row("sweeper-in-flight-funding").as_deref() == Some("none") { + return; + } + assert!( + start.elapsed() <= deadline, + "the funding had not finalized after {deadline:?}; minter logs:\n{}", self.minter_logs().join("\n") ); + self.anvil.mine(1); + // Deliberately short: the PocketIC client panics on a transient HTTP failure rather + // than retrying, and a pooled connection left idle for ~10s gets closed server-side, + // which surfaces as `hyper::Error(IncompleteMessage)` on the next request. std::thread::sleep(Duration::from_secs(2)); } } + /// The burn index of the funding currently in flight, read from the dashboard. `None` once it has + /// finalized, so a test needing it must read it between the burn and the finalization. + pub fn in_flight_funding_burn_index(&self) -> Option { + let row = self.dashboard_row("sweeper-in-flight-funding")?; + let rest = row.strip_prefix("burn index ")?; + rest.split(',').next()?.trim().parse().ok() + } + + /// The minter's public status for `burn_index`, rendered. + pub fn withdrawal_status(&self, burn_index: u64) -> String { + let message_id = self + .env + .submit_call( + self.minter_id, + Principal::anonymous(), + "retrieve_eth_status", + Encode!(&burn_index).unwrap(), + ) + .expect("retrieve_eth_status submission rejected"); + let reply = self + .env + .await_call_no_ticks(message_id) + .expect("retrieve_eth_status rejected"); + Decode!(&reply, RetrieveEthStatus).unwrap().to_string() + } + fn fetch_minter_address(&self) -> Address { let message_id = self .env