diff --git a/linera-core/src/unit_tests/worker_backup_tests.rs b/linera-core/src/unit_tests/worker_backup_tests.rs index 75261ac0add1..793e70606b7b 100644 --- a/linera-core/src/unit_tests/worker_backup_tests.rs +++ b/linera-core/src/unit_tests/worker_backup_tests.rs @@ -3,8 +3,9 @@ // SPDX-License-Identifier: Apache-2.0 #![expect(clippy::large_futures)] -//! Phase 0a: empirically confirm that a WorkerState can boot from a RocksDB backup -//! produced by a different validator and then continue processing blocks. +//! Phase 0a/0b: confirm that a `WorkerState` can boot from a backup produced by a +//! different validator and continue processing blocks. Tests cover three backends: +//! `MemoryDatabase`, `RocksDbDatabase`, and `ScyllaDbDatabase`. use linera_base::{ crypto::{InMemorySigner, ValidatorKeypair, ValidatorSecretKey}, @@ -12,26 +13,286 @@ use linera_base::{ identifiers::{Account, AccountOwner}, }; use linera_storage::{DbStorage, TestClock}; +use linera_views::{memory::MemoryDatabase, random::generate_test_namespace}; +use tempfile::TempDir; + +use crate::{ + chain_worker::ChainWorkerConfig, + test_utils::{ClientOutcomeResultExt as _, MemoryStorageBuilder, TestBuilder}, + worker::WorkerState, +}; + +// ── Memory backend helpers ──────────────────────────────────────────────────── + +/// Deserializes a memory backup from `dir` into a fresh namespace and returns +/// a `DbStorage` connected to it. +async fn restore_memory_backup( + backup_dir: &TempDir, + namespace: &str, +) -> DbStorage { + use std::collections::BTreeMap; + + use linera_views::{ + batch::{Batch, WriteOperation}, + store::{KeyValueDatabase as _, TestKeyValueDatabase as _, WritableKeyValueStore as _}, + }; + + let encoded = + std::fs::read(backup_dir.path().join("memory_backup.bcs")).expect("read memory backup"); + let snapshot: BTreeMap, BTreeMap, Vec>> = + bcs::from_bytes(&encoded).expect("deserialize memory backup"); + + let config = MemoryDatabase::new_test_config() + .await + .expect("memory test config"); + + MemoryDatabase::create(&config, namespace) + .await + .expect("create memory namespace"); + { + let db = MemoryDatabase::connect(&config, namespace) + .await + .expect("connect to memory"); + for (root_key, kv_pairs) in snapshot { + let store = db.open_shared(&root_key).expect("open memory store"); + if !kv_pairs.is_empty() { + let batch = Batch { + operations: kv_pairs + .into_iter() + .map(|(key, value)| WriteOperation::Put { key, value }) + .collect(), + }; + store.write_batch(batch).await.expect("write memory batch"); + } + } + } + + DbStorage::::connect_for_testing( + config, + namespace, + None, + TestClock::new(), + ) + .await + .expect("connect memory storage for testing") +} + +/// Sets up a 4-validator memory-backed network, commits block 0 (transfer), +/// takes a backup, commits block 1, and returns `(backup_dir, cert_for_block_1)`. +async fn memory_setup_backup_and_next_cert( +) -> (TempDir, linera_chain::types::ConfirmedBlockCertificate) { + let mut builder = TestBuilder::new( + MemoryStorageBuilder::default(), + 4, + 0, + InMemorySigner::new(None), + ) + .await + .expect("test builder"); + + let chain_a = builder + .add_root_chain(0, Amount::from_tokens(10)) + .await + .expect("chain a"); + let chain_b = builder + .add_root_chain(1, Amount::ZERO) + .await + .expect("chain b"); + + chain_a + .transfer_to_account( + AccountOwner::CHAIN, + Amount::from_tokens(3), + Account::chain(chain_b.chain_id()), + ) + .await + .unwrap_ok_committed(); + + let source_storage = builder + .validator_storages + .values() + .next() + .expect("at least one validator") + .clone(); + let backup_dir = TempDir::new().expect("backup dir"); + source_storage + .backup_to(backup_dir.path()) + .await + .expect("backup"); + + let cert1 = chain_a + .transfer_to_account( + AccountOwner::CHAIN, + Amount::from_tokens(1), + Account::chain(chain_b.chain_id()), + ) + .await + .unwrap_ok_committed(); + + (backup_dir, cert1) +} + +/// Sets up a 4-validator memory-backed network, commits block 0, takes a backup +/// of validator 0 together with its key, commits blocks 1 and 2, and returns +/// `(backup_dir, validator_secret, cert1, cert2)`. +async fn memory_setup_stale_backup_and_two_certs() -> ( + TempDir, + ValidatorSecretKey, + linera_chain::types::ConfirmedBlockCertificate, + linera_chain::types::ConfirmedBlockCertificate, +) { + let mut builder = TestBuilder::new( + MemoryStorageBuilder::default(), + 4, + 0, + InMemorySigner::new(None), + ) + .await + .expect("test builder"); + + let chain_a = builder + .add_root_chain(0, Amount::from_tokens(10)) + .await + .expect("chain a"); + let chain_b = builder + .add_root_chain(1, Amount::ZERO) + .await + .expect("chain b"); + + chain_a + .transfer_to_account( + AccountOwner::CHAIN, + Amount::from_tokens(3), + Account::chain(chain_b.chain_id()), + ) + .await + .unwrap_ok_committed(); + + let (backup_validator_pub_key, source_storage) = builder + .validator_storages + .iter() + .next() + .map(|(k, v)| (*k, v.clone())) + .expect("at least one validator"); + let backup_dir = TempDir::new().expect("backup dir"); + source_storage + .backup_to(backup_dir.path()) + .await + .expect("backup"); + let validator_secret = builder + .validator_key_pairs + .get(&backup_validator_pub_key) + .expect("validator key pair") + .copy(); + + let cert1 = chain_a + .transfer_to_account( + AccountOwner::CHAIN, + Amount::from_tokens(1), + Account::chain(chain_b.chain_id()), + ) + .await + .unwrap_ok_committed(); + let cert2 = chain_a + .transfer_to_account( + AccountOwner::CHAIN, + Amount::from_tokens(1), + Account::chain(chain_b.chain_id()), + ) + .await + .unwrap_ok_committed(); + + (backup_dir, validator_secret, cert1, cert2) +} + +// ── Memory tests ────────────────────────────────────────────────────────────── + +#[tokio::test] +async fn test_memory_no_key_boots_on_cross_key_backup() { + let (backup_dir, cert1) = memory_setup_backup_and_next_cert().await; + + let namespace = generate_test_namespace(); + let restored_storage = restore_memory_backup(&backup_dir, &namespace).await; + + let config = ChainWorkerConfig { + allow_inactive_chains: true, + ..ChainWorkerConfig::default() + }; + let restored_worker = WorkerState::new(restored_storage, config, None); + + restored_worker + .fully_handle_certificate_with_notifications(cert1, &()) + .await + .expect("memory observer worker should apply the block"); +} + +#[tokio::test] +async fn test_memory_mismatched_key_boots_on_cross_key_backup() { + let (backup_dir, cert1) = memory_setup_backup_and_next_cert().await; + + let namespace = generate_test_namespace(); + let restored_storage = restore_memory_backup(&backup_dir, &namespace).await; + + let mismatched_key = ValidatorKeypair::generate().secret_key; + let config = ChainWorkerConfig { + allow_inactive_chains: true, + ..ChainWorkerConfig::default() + } + .with_key_pair(Some(mismatched_key)); + let restored_worker = WorkerState::new(restored_storage, config, None); + + restored_worker + .fully_handle_certificate_with_notifications(cert1, &()) + .await + .expect("memory mismatched-key worker should apply the block"); +} + +#[tokio::test] +async fn test_memory_stale_backup_catches_up_with_own_key() { + let (backup_dir, validator_secret, cert1, cert2) = + memory_setup_stale_backup_and_two_certs().await; + + let namespace = generate_test_namespace(); + let restored_storage = restore_memory_backup(&backup_dir, &namespace).await; + + let config = ChainWorkerConfig { + allow_inactive_chains: true, + ..ChainWorkerConfig::default() + } + .with_key_pair(Some(validator_secret)); + let restored_worker = WorkerState::new(restored_storage, config, None); + + restored_worker + .fully_handle_certificate_with_notifications(cert1, &()) + .await + .expect("memory stale-backup worker should apply block 1"); + + restored_worker + .fully_handle_certificate_with_notifications(cert2, &()) + .await + .expect("memory stale-backup worker should apply block 2"); +} + +// ── RocksDB backend helpers ──────────────────────────────────────────────────── + +#[cfg(feature = "rocksdb")] use linera_views::{ - random::generate_test_namespace, rocks_db::{PathWithGuard, RocksDbDatabase}, store::TestKeyValueDatabase as _, }; +#[cfg(feature = "rocksdb")] use rocksdb::{ backup::{BackupEngine, BackupEngineOptions, RestoreOptions}, Env, }; -use tempfile::TempDir; -use crate::{ - chain_worker::ChainWorkerConfig, - test_utils::{ClientOutcomeResultExt as _, RocksDbStorageBuilder, TestBuilder}, - worker::WorkerState, -}; +#[cfg(feature = "rocksdb")] +use crate::test_utils::RocksDbStorageBuilder; /// Restores a RocksDB backup into `restore_base/` and returns a fresh /// `DbStorage` connected to it. -async fn restore_backup( +#[cfg(feature = "rocksdb")] +async fn restore_rocksdb_backup( backup_dir: &TempDir, namespace: &str, ) -> DbStorage { @@ -64,11 +325,9 @@ async fn restore_backup( .expect("connect for testing") } -/// Sets up a 4-validator test network, commits one block (a transfer from chain 0 to -/// chain 1), takes a RocksDB backup of one validator's storage, and returns the backup -/// directory together with the certificate for a second block ready to be fed to the -/// restored worker. -async fn setup_backup_and_next_cert() -> (TempDir, linera_chain::types::ConfirmedBlockCertificate) { +#[cfg(feature = "rocksdb")] +async fn rocksdb_setup_backup_and_next_cert( +) -> (TempDir, linera_chain::types::ConfirmedBlockCertificate) { let mut builder = TestBuilder::new( RocksDbStorageBuilder::new().await, 4, @@ -78,7 +337,6 @@ async fn setup_backup_and_next_cert() -> (TempDir, linera_chain::types::Confirme .await .expect("test builder"); - // Two chains so there is a real transfer target. let chain_a = builder .add_root_chain(0, Amount::from_tokens(10)) .await @@ -88,7 +346,6 @@ async fn setup_backup_and_next_cert() -> (TempDir, linera_chain::types::Confirme .await .expect("chain b"); - // Block 0: transfer — all 4 validators propose, vote, and commit this block. chain_a .transfer_to_account( AccountOwner::CHAIN, @@ -98,7 +355,6 @@ async fn setup_backup_and_next_cert() -> (TempDir, linera_chain::types::Confirme .await .unwrap_ok_committed(); - // Back up one validator's storage after block 0 is committed. let source_storage = builder .validator_storages .values() @@ -106,9 +362,11 @@ async fn setup_backup_and_next_cert() -> (TempDir, linera_chain::types::Confirme .expect("at least one validator") .clone(); let backup_dir = TempDir::new().expect("backup dir"); - source_storage.backup_to(backup_dir.path()).expect("backup"); + source_storage + .backup_to(backup_dir.path()) + .await + .expect("backup"); - // Block 1: second transfer — all 4 original validators commit this too. let cert1 = chain_a .transfer_to_account( AccountOwner::CHAIN, @@ -121,14 +379,88 @@ async fn setup_backup_and_next_cert() -> (TempDir, linera_chain::types::Confirme (backup_dir, cert1) } +#[cfg(feature = "rocksdb")] +async fn rocksdb_setup_stale_backup_and_two_certs() -> ( + TempDir, + ValidatorSecretKey, + linera_chain::types::ConfirmedBlockCertificate, + linera_chain::types::ConfirmedBlockCertificate, +) { + let mut builder = TestBuilder::new( + RocksDbStorageBuilder::new().await, + 4, + 0, + InMemorySigner::new(None), + ) + .await + .expect("test builder"); + + let chain_a = builder + .add_root_chain(0, Amount::from_tokens(10)) + .await + .expect("chain a"); + let chain_b = builder + .add_root_chain(1, Amount::ZERO) + .await + .expect("chain b"); + + chain_a + .transfer_to_account( + AccountOwner::CHAIN, + Amount::from_tokens(3), + Account::chain(chain_b.chain_id()), + ) + .await + .unwrap_ok_committed(); + + let (backup_validator_pub_key, source_storage) = builder + .validator_storages + .iter() + .next() + .map(|(k, v)| (*k, v.clone())) + .expect("at least one validator"); + let backup_dir = TempDir::new().expect("backup dir"); + source_storage + .backup_to(backup_dir.path()) + .await + .expect("backup"); + let validator_secret = builder + .validator_key_pairs + .get(&backup_validator_pub_key) + .expect("validator key pair") + .copy(); + + let cert1 = chain_a + .transfer_to_account( + AccountOwner::CHAIN, + Amount::from_tokens(1), + Account::chain(chain_b.chain_id()), + ) + .await + .unwrap_ok_committed(); + let cert2 = chain_a + .transfer_to_account( + AccountOwner::CHAIN, + Amount::from_tokens(1), + Account::chain(chain_b.chain_id()), + ) + .await + .unwrap_ok_committed(); + + (backup_dir, validator_secret, cert1, cert2) +} + +// ── RocksDB tests ───────────────────────────────────────────────────────────── + /// A WorkerState with no key (observer mode) booted from another validator's backup can /// apply the next committed block. +#[cfg(feature = "rocksdb")] #[tokio::test] async fn test_no_key_boots_on_cross_key_backup() { - let (backup_dir, cert1) = setup_backup_and_next_cert().await; + let (backup_dir, cert1) = rocksdb_setup_backup_and_next_cert().await; let namespace = generate_test_namespace(); - let restored_storage = restore_backup(&backup_dir, &namespace).await; + let restored_storage = restore_rocksdb_backup(&backup_dir, &namespace).await; let config = ChainWorkerConfig { allow_inactive_chains: true, @@ -144,12 +476,13 @@ async fn test_no_key_boots_on_cross_key_backup() { /// A WorkerState with a mismatched keypair booted from another validator's backup can /// apply the next committed block. +#[cfg(feature = "rocksdb")] #[tokio::test] async fn test_mismatched_key_boots_on_cross_key_backup() { - let (backup_dir, cert1) = setup_backup_and_next_cert().await; + let (backup_dir, cert1) = rocksdb_setup_backup_and_next_cert().await; let namespace = generate_test_namespace(); - let restored_storage = restore_backup(&backup_dir, &namespace).await; + let restored_storage = restore_rocksdb_backup(&backup_dir, &namespace).await; let mismatched_key = ValidatorKeypair::generate().secret_key; let config = ChainWorkerConfig { @@ -165,17 +498,176 @@ async fn test_mismatched_key_boots_on_cross_key_backup() { .expect("mismatched-key worker should apply the block"); } -/// Sets up a 4-validator test network, commits one block (a transfer from chain 0 to -/// chain 1), takes a RocksDB backup of validator 0's storage together with its secret key, -/// then commits two more blocks that the backup does not contain. -async fn setup_stale_backup_and_two_certs() -> ( +/// A WorkerState restored from a stale backup using the original validator's own key can +/// sequentially apply the blocks it missed and catch up to the current chain tip. +#[cfg(feature = "rocksdb")] +#[tokio::test] +async fn test_stale_backup_catches_up_with_own_key() { + let (backup_dir, validator_secret, cert1, cert2) = + rocksdb_setup_stale_backup_and_two_certs().await; + + let namespace = generate_test_namespace(); + let restored_storage = restore_rocksdb_backup(&backup_dir, &namespace).await; + + let config = ChainWorkerConfig { + allow_inactive_chains: true, + ..ChainWorkerConfig::default() + } + .with_key_pair(Some(validator_secret)); + let restored_worker = WorkerState::new(restored_storage, config, None); + + restored_worker + .fully_handle_certificate_with_notifications(cert1, &()) + .await + .expect("stale-backup worker should apply block 1"); + + restored_worker + .fully_handle_certificate_with_notifications(cert2, &()) + .await + .expect("stale-backup worker should apply block 2"); +} + +// ── ScyllaDB backend helpers ─────────────────────────────────────────────────── + +#[cfg(feature = "scylladb")] +use linera_views::{ + backends::scylla_db::ScyllaDbDatabaseInternal, + batch::{SimpleUnorderedBatch, UnorderedBatch}, + scylla_db::ScyllaDbDatabase, + store::{DirectWritableKeyValueStore as _, KeyValueDatabase as _}, +}; + +#[cfg(feature = "scylladb")] +use crate::test_utils::ScyllaDbStorageBuilder; + +/// Deserializes a ScyllaDB backup from `dir` into a fresh namespace and returns +/// a `DbStorage` connected to it. +#[cfg(feature = "scylladb")] +async fn restore_scylladb_backup( + backup_dir: &TempDir, + namespace: &str, +) -> DbStorage { + use std::collections::BTreeMap; + + let encoded = + std::fs::read(backup_dir.path().join("scylladb_backup.bcs")).expect("read scylladb backup"); + let snapshot: BTreeMap, BTreeMap, Vec>> = + bcs::from_bytes(&encoded).expect("deserialize scylladb backup"); + + let config = ScyllaDbDatabase::new_test_config() + .await + .expect("scylladb test config"); + let inner_config = config.inner_config.clone(); + + ScyllaDbDatabaseInternal::create(&inner_config, namespace) + .await + .expect("create scylladb namespace"); + { + let db = ScyllaDbDatabaseInternal::connect(&inner_config, namespace) + .await + .expect("connect to scylladb inner"); + for (big_root_key, kv_pairs) in snapshot { + // big_root_key in the dump = [0] ++ actual_root_key (see get_big_root_key). + // open_shared re-applies get_big_root_key, so we strip the leading byte. + let actual_root_key = &big_root_key[1..]; + let store = db + .open_shared(actual_root_key) + .expect("open scylladb store"); + // The empty key is the reserved writetime sentinel; `write_batch` + // rejects zero-length keys and writes its own sentinel, so drop it. + let insertions: Vec<(Vec, Vec)> = kv_pairs + .into_iter() + .filter(|(key, _)| !key.is_empty()) + .collect(); + if !insertions.is_empty() { + let batch = UnorderedBatch { + key_prefix_deletions: vec![], + simple_unordered_batch: SimpleUnorderedBatch { + deletions: vec![], + insertions, + }, + }; + store + .write_batch(batch) + .await + .expect("write scylladb batch"); + } + } + } + + DbStorage::::connect_for_testing( + config, + namespace, + None, + TestClock::new(), + ) + .await + .expect("connect scylladb storage for testing") +} + +#[cfg(feature = "scylladb")] +async fn scylladb_setup_backup_and_next_cert( +) -> (TempDir, linera_chain::types::ConfirmedBlockCertificate) { + let mut builder = TestBuilder::new( + ScyllaDbStorageBuilder::default(), + 4, + 0, + InMemorySigner::new(None), + ) + .await + .expect("test builder"); + + let chain_a = builder + .add_root_chain(0, Amount::from_tokens(10)) + .await + .expect("chain a"); + let chain_b = builder + .add_root_chain(1, Amount::ZERO) + .await + .expect("chain b"); + + chain_a + .transfer_to_account( + AccountOwner::CHAIN, + Amount::from_tokens(3), + Account::chain(chain_b.chain_id()), + ) + .await + .unwrap_ok_committed(); + + let source_storage = builder + .validator_storages + .values() + .next() + .expect("at least one validator") + .clone(); + let backup_dir = TempDir::new().expect("backup dir"); + source_storage + .backup_to(backup_dir.path()) + .await + .expect("scylladb backup"); + + let cert1 = chain_a + .transfer_to_account( + AccountOwner::CHAIN, + Amount::from_tokens(1), + Account::chain(chain_b.chain_id()), + ) + .await + .unwrap_ok_committed(); + + (backup_dir, cert1) +} + +#[cfg(feature = "scylladb")] +async fn scylladb_setup_stale_backup_and_two_certs() -> ( TempDir, ValidatorSecretKey, linera_chain::types::ConfirmedBlockCertificate, linera_chain::types::ConfirmedBlockCertificate, ) { let mut builder = TestBuilder::new( - RocksDbStorageBuilder::new().await, + ScyllaDbStorageBuilder::default(), 4, 0, InMemorySigner::new(None), @@ -192,7 +684,6 @@ async fn setup_stale_backup_and_two_certs() -> ( .await .expect("chain b"); - // Block 0: all 4 validators sign and commit this block. chain_a .transfer_to_account( AccountOwner::CHAIN, @@ -202,7 +693,6 @@ async fn setup_stale_backup_and_two_certs() -> ( .await .unwrap_ok_committed(); - // Take the backup of validator 0 (keyed by its public key) together with its secret key. let (backup_validator_pub_key, source_storage) = builder .validator_storages .iter() @@ -210,14 +700,16 @@ async fn setup_stale_backup_and_two_certs() -> ( .map(|(k, v)| (*k, v.clone())) .expect("at least one validator"); let backup_dir = TempDir::new().expect("backup dir"); - source_storage.backup_to(backup_dir.path()).expect("backup"); + source_storage + .backup_to(backup_dir.path()) + .await + .expect("scylladb backup"); let validator_secret = builder .validator_key_pairs .get(&backup_validator_pub_key) .expect("validator key pair") .copy(); - // Blocks 1 and 2: all 4 original validators commit; the backup does not contain these. let cert1 = chain_a .transfer_to_account( AccountOwner::CHAIN, @@ -238,14 +730,58 @@ async fn setup_stale_backup_and_two_certs() -> ( (backup_dir, validator_secret, cert1, cert2) } -/// A WorkerState restored from a stale backup using the original validator's own key can -/// sequentially apply the blocks it missed and catch up to the current chain tip. +// ── ScyllaDB tests ──────────────────────────────────────────────────────────── + +#[cfg(feature = "scylladb")] #[tokio::test] -async fn test_stale_backup_catches_up_with_own_key() { - let (backup_dir, validator_secret, cert1, cert2) = setup_stale_backup_and_two_certs().await; +async fn test_scylladb_no_key_boots_on_cross_key_backup() { + let (backup_dir, cert1) = scylladb_setup_backup_and_next_cert().await; + + let namespace = generate_test_namespace(); + let restored_storage = restore_scylladb_backup(&backup_dir, &namespace).await; + + let config = ChainWorkerConfig { + allow_inactive_chains: true, + ..ChainWorkerConfig::default() + }; + let restored_worker = WorkerState::new(restored_storage, config, None); + + restored_worker + .fully_handle_certificate_with_notifications(cert1, &()) + .await + .expect("scylladb observer worker should apply the block"); +} + +#[cfg(feature = "scylladb")] +#[tokio::test] +async fn test_scylladb_mismatched_key_boots_on_cross_key_backup() { + let (backup_dir, cert1) = scylladb_setup_backup_and_next_cert().await; let namespace = generate_test_namespace(); - let restored_storage = restore_backup(&backup_dir, &namespace).await; + let restored_storage = restore_scylladb_backup(&backup_dir, &namespace).await; + + let mismatched_key = ValidatorKeypair::generate().secret_key; + let config = ChainWorkerConfig { + allow_inactive_chains: true, + ..ChainWorkerConfig::default() + } + .with_key_pair(Some(mismatched_key)); + let restored_worker = WorkerState::new(restored_storage, config, None); + + restored_worker + .fully_handle_certificate_with_notifications(cert1, &()) + .await + .expect("scylladb mismatched-key worker should apply the block"); +} + +#[cfg(feature = "scylladb")] +#[tokio::test] +async fn test_scylladb_stale_backup_catches_up_with_own_key() { + let (backup_dir, validator_secret, cert1, cert2) = + scylladb_setup_stale_backup_and_two_certs().await; + + let namespace = generate_test_namespace(); + let restored_storage = restore_scylladb_backup(&backup_dir, &namespace).await; let config = ChainWorkerConfig { allow_inactive_chains: true, @@ -257,10 +793,10 @@ async fn test_stale_backup_catches_up_with_own_key() { restored_worker .fully_handle_certificate_with_notifications(cert1, &()) .await - .expect("stale-backup worker should apply block 1"); + .expect("scylladb stale-backup worker should apply block 1"); restored_worker .fully_handle_certificate_with_notifications(cert2, &()) .await - .expect("stale-backup worker should apply block 2"); + .expect("scylladb stale-backup worker should apply block 2"); } diff --git a/linera-core/src/worker.rs b/linera-core/src/worker.rs index 75ebd32812cc..27f74690c0e3 100644 --- a/linera-core/src/worker.rs +++ b/linera-core/src/worker.rs @@ -81,7 +81,7 @@ pub const DEFAULT_EXECUTION_STATE_CACHE_SIZE: usize = 10_000; #[path = "unit_tests/worker_tests.rs"] mod worker_tests; -#[cfg(all(test, feature = "rocksdb"))] +#[cfg(test)] #[path = "unit_tests/worker_backup_tests.rs"] mod worker_backup_tests; diff --git a/linera-storage/src/db_storage.rs b/linera-storage/src/db_storage.rs index 25ec7b314184..702c37e7df9c 100644 --- a/linera-storage/src/db_storage.rs +++ b/linera-storage/src/db_storage.rs @@ -1592,8 +1592,8 @@ where Database: linera_views::backends::DatabaseBackup, { /// Backs up the underlying database to the given directory. - pub fn backup_to(&self, dir: &std::path::Path) -> anyhow::Result<()> { - self.database.backup_to(dir) + pub async fn backup_to(&self, dir: &std::path::Path) -> anyhow::Result<()> { + self.database.backup_to(dir).await } } diff --git a/linera-views/src/backends/journaling.rs b/linera-views/src/backends/journaling.rs index 3d8793f9bf8e..d0ed36532d02 100644 --- a/linera-views/src/backends/journaling.rs +++ b/linera-views/src/backends/journaling.rs @@ -460,6 +460,15 @@ where } } +#[cfg(with_testing)] +impl crate::backends::DatabaseBackup + for JournalingKeyValueDatabase +{ + async fn backup_to(&self, dir: &std::path::Path) -> anyhow::Result<()> { + self.database.backup_to(dir).await + } +} + impl JournalingKeyValueStore { /// Creates a new journaling store. pub fn new(store: S) -> Self { diff --git a/linera-views/src/backends/lru_caching.rs b/linera-views/src/backends/lru_caching.rs index 831b7e283db4..0ac1ab90e403 100644 --- a/linera-views/src/backends/lru_caching.rs +++ b/linera-views/src/backends/lru_caching.rs @@ -501,8 +501,10 @@ where } #[cfg(with_testing)] -impl crate::backends::DatabaseBackup for LruCachingDatabase { - fn backup_to(&self, dir: &std::path::Path) -> anyhow::Result<()> { - self.database.backup_to(dir) +impl crate::backends::DatabaseBackup + for LruCachingDatabase +{ + async fn backup_to(&self, dir: &std::path::Path) -> anyhow::Result<()> { + self.database.backup_to(dir).await } } diff --git a/linera-views/src/backends/memory.rs b/linera-views/src/backends/memory.rs index 75e4c9eeef4a..4b6833b903b6 100644 --- a/linera-views/src/backends/memory.rs +++ b/linera-views/src/backends/memory.rs @@ -356,6 +356,32 @@ impl TestKeyValueDatabase for MemoryDatabase { } } +#[cfg(with_testing)] +impl crate::backends::DatabaseBackup for MemoryDatabase { + async fn backup_to(&self, dir: &std::path::Path) -> anyhow::Result<()> { + use std::collections::BTreeMap; + let databases = MEMORY_DATABASES + .lock() + .expect("MEMORY_DATABASES lock should not be poisoned"); + let namespace_map = databases + .databases + .get(&self.namespace) + .ok_or_else(|| anyhow::anyhow!("namespace not found: {}", self.namespace))?; + let snapshot: BTreeMap, BTreeMap, Vec>> = namespace_map + .iter() + .map(|(root_key, store_map)| { + let store_map = store_map + .read() + .expect("MemoryStore lock should not be poisoned"); + (root_key.clone(), store_map.clone()) + }) + .collect(); + let encoded = bcs::to_bytes(&snapshot)?; + std::fs::write(dir.join("memory_backup.bcs"), encoded)?; + Ok(()) + } +} + /// The error type for [`MemoryStore`]. #[derive(Error, Debug)] pub enum MemoryStoreError { diff --git a/linera-views/src/backends/metering.rs b/linera-views/src/backends/metering.rs index 9dcf9aa81d4c..5abec2ea5972 100644 --- a/linera-views/src/backends/metering.rs +++ b/linera-views/src/backends/metering.rs @@ -585,8 +585,10 @@ where } #[cfg(with_testing)] -impl crate::backends::DatabaseBackup for MeteredDatabase { - fn backup_to(&self, dir: &std::path::Path) -> anyhow::Result<()> { - self.database.backup_to(dir) +impl crate::backends::DatabaseBackup + for MeteredDatabase +{ + async fn backup_to(&self, dir: &std::path::Path) -> anyhow::Result<()> { + self.database.backup_to(dir).await } } diff --git a/linera-views/src/backends/mod.rs b/linera-views/src/backends/mod.rs index 3abc14bc1fd3..18f619e42953 100644 --- a/linera-views/src/backends/mod.rs +++ b/linera-views/src/backends/mod.rs @@ -24,8 +24,11 @@ pub mod rocks_db; pub mod indexed_db; #[cfg(with_testing)] -/// Creates a RocksDB backup of the underlying database into a directory. +/// Serializes the contents of a database namespace to disk for test backup/restore. pub trait DatabaseBackup { - /// Writes a RocksDB backup snapshot into `dir`. - fn backup_to(&self, dir: &std::path::Path) -> anyhow::Result<()>; + /// Writes a snapshot of the namespace into `dir`. + fn backup_to( + &self, + dir: &std::path::Path, + ) -> impl std::future::Future> + Send; } diff --git a/linera-views/src/backends/rocks_db.rs b/linera-views/src/backends/rocks_db.rs index cde9000b967e..10581f70645c 100644 --- a/linera-views/src/backends/rocks_db.rs +++ b/linera-views/src/backends/rocks_db.rs @@ -794,7 +794,7 @@ pub type RocksDbDatabase = LruCachingDatabase anyhow::Result<()> { + async fn backup_to(&self, dir: &std::path::Path) -> anyhow::Result<()> { use rocksdb::{ backup::{BackupEngine, BackupEngineOptions}, Env, diff --git a/linera-views/src/backends/scylla_db.rs b/linera-views/src/backends/scylla_db.rs index 7fbd21a2bb35..1dc8f190bc6a 100644 --- a/linera-views/src/backends/scylla_db.rs +++ b/linera-views/src/backends/scylla_db.rs @@ -1304,6 +1304,39 @@ impl ScyllaDbDatabaseInternal { } } +#[cfg(with_testing)] +impl crate::backends::DatabaseBackup for ScyllaDbDatabaseInternal { + async fn backup_to(&self, dir: &std::path::Path) -> anyhow::Result<()> { + use std::collections::BTreeMap; + + use futures::StreamExt as _; + + let namespace = &self.store.namespace; + let statement = self + .store + .session + .prepare(format!( + "SELECT root_key, k, v FROM {KEYSPACE}.\"{namespace}\"" + )) + .await + .map_err(|e| anyhow::anyhow!("{e}"))?; + let rows = Box::pin(self.store.session.execute_iter(statement, &[])) + .await + .map_err(|e| anyhow::anyhow!("{e}"))?; + let mut snapshot: BTreeMap, BTreeMap, Vec>> = BTreeMap::new(); + let mut rows = rows + .rows_stream::<(Vec, Vec, Vec)>() + .map_err(|e| anyhow::anyhow!("{e}"))?; + while let Some(row) = rows.next().await { + let (root_key, k, v) = row.map_err(|e| anyhow::anyhow!("{e}"))?; + snapshot.entry(root_key).or_default().insert(k, v); + } + let encoded = bcs::to_bytes(&snapshot).map_err(|e| anyhow::anyhow!("{e}"))?; + std::fs::write(dir.join("scylladb_backup.bcs"), encoded)?; + Ok(()) + } +} + #[cfg(with_testing)] impl TestKeyValueDatabase for JournalingKeyValueDatabase { async fn new_test_config( diff --git a/linera-views/src/backends/value_splitting.rs b/linera-views/src/backends/value_splitting.rs index 18f73f425539..b35b30d142a1 100644 --- a/linera-views/src/backends/value_splitting.rs +++ b/linera-views/src/backends/value_splitting.rs @@ -361,11 +361,11 @@ where } #[cfg(with_testing)] -impl crate::backends::DatabaseBackup +impl crate::backends::DatabaseBackup for ValueSplittingDatabase { - fn backup_to(&self, dir: &std::path::Path) -> anyhow::Result<()> { - self.database.backup_to(dir) + async fn backup_to(&self, dir: &std::path::Path) -> anyhow::Result<()> { + self.database.backup_to(dir).await } }