From ef9c5eafcc9011b048a74ba5e00744b94cd2c506 Mon Sep 17 00:00:00 2001 From: ljedrz Date: Tue, 7 Jan 2025 14:08:15 +0100 Subject: [PATCH] refactor: simplify and unify the storage setup for tests Signed-off-by: ljedrz --- .circleci/config.yml | 9 +- Cargo.toml | 2 +- ledger/Cargo.toml | 4 +- ledger/benches/transaction.rs | 9 +- ledger/block/Cargo.toml | 2 +- ledger/src/tests.rs | 26 +-- ledger/store/Cargo.toml | 12 +- ledger/store/src/helpers/memory/program.rs | 12 -- ledger/store/src/helpers/rocksdb/consensus.rs | 25 +++ .../store/src/helpers/rocksdb/internal/map.rs | 46 ++--- .../store/src/helpers/rocksdb/internal/mod.rs | 170 ++++-------------- .../helpers/rocksdb/internal/nested_map.rs | 68 +++---- .../src/helpers/rocksdb/internal/tests.rs | 51 ++---- ledger/store/src/helpers/rocksdb/program.rs | 25 --- ledger/store/src/program/committee.rs | 13 -- ledger/store/src/program/finalize.rs | 24 +-- ledger/test-helpers/Cargo.toml | 3 + parameters/Cargo.toml | 1 + parameters/examples/inclusion.rs | 10 +- synthesizer/Cargo.toml | 2 +- synthesizer/process/Cargo.toml | 1 + synthesizer/process/src/tests/test_credits.rs | 20 +-- synthesizer/src/vm/execute.rs | 7 +- synthesizer/src/vm/finalize.rs | 26 +-- synthesizer/src/vm/mod.rs | 26 +-- .../tests/test_vm_execute_and_finalize.rs | 12 +- 26 files changed, 213 insertions(+), 393 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4aad608529..6253f1820c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -506,7 +506,7 @@ jobs: resource_class: << pipeline.parameters.medium >> steps: - run_serial: - flags: --features=rocks + flags: --features=rocks,test workspace_member: ledger cache_key: v1.0.0-rust-1.81.0-snarkvm-ledger-with-rocksdb-cache @@ -643,7 +643,7 @@ jobs: resource_class: << pipeline.parameters.twoxlarge >> steps: - run_serial: - flags: --features=rocks + flags: --features=rocks,test workspace_member: ledger/store cache_key: v1.0.0-rust-1.81.0-snarkvm-ledger-store-cache @@ -712,7 +712,7 @@ jobs: resource_class: << pipeline.parameters.xlarge >> steps: - run_serial: - flags: --features=rocks + flags: --features=rocks,test workspace_member: synthesizer/process cache_key: v1.0.0-rust-1.81.0-snarkvm-synthesizer-process-cache @@ -945,8 +945,7 @@ workflows: - curves - fields - ledger -# TODO (howardwu) - Implement `open_testing` on all storage, update to `CurrentConsensusStore::open_testing`, then re-enable. -# - ledger-with-rocksdb + - ledger-with-rocksdb - ledger-with-valid-solutions - ledger-authority - ledger-block diff --git a/Cargo.toml b/Cargo.toml index d722810219..855b9a1e01 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -135,7 +135,7 @@ history = [ "snarkvm-synthesizer/history" ] parameters_no_std_out = [ "snarkvm-parameters/no_std_out" ] noconfig = [ ] rocks = [ "snarkvm-ledger/rocks", "snarkvm-synthesizer/rocks" ] -test = [ "snarkvm-ledger/test" ] +test = [ "snarkvm-console/test", "snarkvm-ledger/test", "snarkvm-synthesizer/test" ] test-helpers = [ "snarkvm-ledger/test-helpers" ] timer = [ "snarkvm-ledger/timer" ] algorithms = [ "snarkvm-algorithms" ] diff --git a/ledger/Cargo.toml b/ledger/Cargo.toml index 0b6502b0ce..f4b66cc332 100644 --- a/ledger/Cargo.toml +++ b/ledger/Cargo.toml @@ -39,7 +39,7 @@ async = [ "synthesizer/async" ] metrics = [ "ledger-committee/metrics" ] -rocks = [ "ledger-store/rocks" ] +rocks = [ "ledger-store/rocks", "synthesizer/rocks" ] serial = [ "console/serial", "ledger-authority/serial", @@ -51,7 +51,7 @@ serial = [ "ledger-store/serial", "synthesizer/serial" ] -test = [ "console/test", "ledger-block/test", "ledger-store/test" ] +test = [ "console/test", "ledger-block/test", "ledger-store/test", "synthesizer/test" ] test-helpers = [ "ledger-test-helpers", "ledger-committee/test-helpers", diff --git a/ledger/benches/transaction.rs b/ledger/benches/transaction.rs index 933b0918c0..bce9abdc3b 100644 --- a/ledger/benches/transaction.rs +++ b/ledger/benches/transaction.rs @@ -24,16 +24,21 @@ use console::{ program::{Plaintext, Record, Value}, }; use ledger_block::Transition; -use ledger_store::{ConsensusStore, helpers::memory::ConsensusMemory}; +use ledger_store::ConsensusStore; use synthesizer::{VM, program::Program}; use criterion::Criterion; use indexmap::IndexMap; +#[cfg(not(feature = "rocks"))] +type LedgerType = ledger_store::helpers::memory::ConsensusMemory; +#[cfg(feature = "rocks")] +type LedgerType = ledger_store::helpers::rocksdb::ConsensusDB; + fn initialize_vm( private_key: &PrivateKey, rng: &mut R, -) -> (VM>, Vec>>) { +) -> (VM>, Vec>>) { // Initialize the VM. let vm = VM::from(ConsensusStore::open(None).unwrap()).unwrap(); diff --git a/ledger/block/Cargo.toml b/ledger/block/Cargo.toml index 59ea5bb4bf..9cfb832e83 100644 --- a/ledger/block/Cargo.toml +++ b/ledger/block/Cargo.toml @@ -34,7 +34,7 @@ wasm = [ "synthesizer-program/wasm", "synthesizer-snark/wasm" ] -test = [ ] +test = [ "console/test" ] [dependencies.console] package = "snarkvm-console" diff --git a/ledger/src/tests.rs b/ledger/src/tests.rs index cc5989c0e5..5691a41c6f 100644 --- a/ledger/src/tests.rs +++ b/ledger/src/tests.rs @@ -30,7 +30,7 @@ use ledger_authority::Authority; use ledger_block::{Block, ConfirmedTransaction, Execution, Ratify, Rejected, Transaction}; use ledger_committee::{Committee, MIN_VALIDATOR_STAKE}; use ledger_narwhal::{BatchCertificate, BatchHeader, Data, Subdag, Transmission, TransmissionID}; -use ledger_store::{ConsensusStore, helpers::memory::ConsensusMemory}; +use ledger_store::ConsensusStore; use snarkvm_utilities::try_vm_runtime; use synthesizer::{Stack, program::Program, vm::VM}; @@ -39,9 +39,14 @@ use rand::seq::SliceRandom; use std::collections::{BTreeMap, HashMap}; use time::OffsetDateTime; +#[cfg(not(feature = "rocks"))] +type LedgerType = ledger_store::helpers::memory::ConsensusMemory; +#[cfg(feature = "rocks")] +type LedgerType = ledger_store::helpers::rocksdb::ConsensusDB; + /// Initializes a sample VM. -fn sample_vm() -> VM> { - VM::from(ConsensusStore::>::open(None).unwrap()).unwrap() +fn sample_vm() -> VM> { + VM::from(ConsensusStore::>::open(None).unwrap()).unwrap() } /// Extract the transmissions from a block. @@ -71,8 +76,7 @@ fn construct_quorum_blocks( ) -> Vec> { // Initialize the ledger with the genesis block. let ledger = - Ledger::>::load(genesis.clone(), StorageMode::Production) - .unwrap(); + Ledger::>::load(genesis.clone(), StorageMode::Production).unwrap(); // Initialize the round parameters. assert!(num_blocks > 0); @@ -121,7 +125,7 @@ fn construct_quorum_blocks( // Helper function to create a quorum block. fn create_next_quorum_block( - ledger: &Ledger>, + ledger: &Ledger>, round: u64, leader_certificate: &BatchCertificate, previous_leader_certificate: Option<&BatchCertificate>, @@ -182,7 +186,7 @@ fn test_load() { // Sample the genesis private key. let private_key = PrivateKey::::new(rng).unwrap(); // Initialize the store. - let store = ConsensusStore::<_, ConsensusMemory<_>>::open(None).unwrap(); + let store = ConsensusStore::<_, LedgerType<_>>::open(None).unwrap(); // Create a genesis block. let genesis = VM::from(store).unwrap().genesis_beacon(&private_key, rng).unwrap(); @@ -1990,8 +1994,7 @@ fn test_max_committee_limit_with_bonds() { // Initialize a Ledger from the genesis block. let ledger = - Ledger::>::load(genesis_block, StorageMode::Production) - .unwrap(); + Ledger::>::load(genesis_block, StorageMode::Production).unwrap(); // Bond the first validator. let bond_first_transaction = ledger @@ -3050,7 +3053,7 @@ fn test_forged_block_subdags() { // Sample the genesis private key. let private_key = PrivateKey::::new(rng).unwrap(); // Initialize the store. - let store = ConsensusStore::<_, ConsensusMemory<_>>::open(None).unwrap(); + let store = ConsensusStore::<_, LedgerType<_>>::open(None).unwrap(); // Create a genesis block with a seeded RNG to reproduce the same genesis private keys. let seed: u64 = rng.gen(); let genesis_rng = &mut TestRng::from_seed(seed); @@ -3074,8 +3077,7 @@ fn test_forged_block_subdags() { let block_3 = quorum_blocks.remove(0); // Construct the ledger. - let ledger = - Ledger::>::load(genesis, StorageMode::Production).unwrap(); + let ledger = Ledger::>::load(genesis, StorageMode::Production).unwrap(); ledger.advance_to_next_block(&block_1).unwrap(); ledger.check_next_block(&block_2, rng).unwrap(); diff --git a/ledger/store/Cargo.toml b/ledger/store/Cargo.toml index 0b9d22a7a3..32026b6d2d 100644 --- a/ledger/store/Cargo.toml +++ b/ledger/store/Cargo.toml @@ -37,7 +37,7 @@ wasm = [ "synthesizer-program/wasm", "synthesizer-snark/wasm" ] -test = [ ] +test = [ "console/test", "ledger-block/test", "dep:tempfile" ] [dependencies.console] package = "snarkvm-console" @@ -123,6 +123,10 @@ default-features = false features = [ "write" ] optional = true +[dependencies.tempfile] +version = "3.8" +optional = true + [dependencies.tracing] version = "0.1" optional = true @@ -136,11 +140,5 @@ features = [ "test-helpers" ] package = "snarkvm-ledger-test-helpers" path = "../../ledger/test-helpers" -[dev-dependencies.serial_test] -version = "2" - -[dev-dependencies.tempfile] -version = "3.8" - [dev-dependencies.tracing-test] version = "0.2" diff --git a/ledger/store/src/helpers/memory/program.rs b/ledger/store/src/helpers/memory/program.rs index b48edf62c0..338ccd8cd9 100644 --- a/ledger/store/src/helpers/memory/program.rs +++ b/ledger/store/src/helpers/memory/program.rs @@ -62,12 +62,6 @@ impl FinalizeStorage for FinalizeMemory { }) } - /// Initializes the test-variant of the storage. - #[cfg(any(test, feature = "test"))] - fn open_testing(_: std::path::PathBuf, dev: Option) -> Result { - Self::open(dev) - } - /// Returns the committee store. fn committee_store(&self) -> &CommitteeStore { &self.committee_store @@ -118,12 +112,6 @@ impl CommitteeStorage for CommitteeMemory { }) } - /// Initializes the test-variant of the storage. - #[cfg(any(test, feature = "test"))] - fn open_testing(_: std::path::PathBuf, dev: Option) -> Result { - Self::open(dev) - } - /// Returns the current round map. fn current_round_map(&self) -> &Self::CurrentRoundMap { &self.current_round_map diff --git a/ledger/store/src/helpers/rocksdb/consensus.rs b/ledger/store/src/helpers/rocksdb/consensus.rs index 0ff5df0aff..950e17eb21 100644 --- a/ledger/store/src/helpers/rocksdb/consensus.rs +++ b/ledger/store/src/helpers/rocksdb/consensus.rs @@ -22,6 +22,8 @@ use crate::{ use console::prelude::*; use aleo_std_storage::StorageMode; +#[cfg(any(test, feature = "test"))] +use std::sync::Arc; /// An RocksDB consensus storage. #[derive(Clone)] @@ -30,6 +32,9 @@ pub struct ConsensusDB { finalize_store: FinalizeStore>, /// The block store. block_store: BlockStore>, + /// A test-only instance of TempDir which is cleaned up afterwards. + #[cfg(any(test, feature = "test"))] + _temp_dir: Arc, } #[rustfmt::skip] @@ -40,6 +45,7 @@ impl ConsensusStorage for ConsensusDB { type TransitionStorage = TransitionDB; /// Initializes the consensus storage. + #[cfg(not(any(test, feature = "test")))] fn open>(storage: S) -> Result { // Initialize the finalize store. let finalize_store = FinalizeStore::>::open(storage.clone())?; @@ -52,6 +58,25 @@ impl ConsensusStorage for ConsensusDB { }) } + /// Initializes a test-only consensus storage. + #[cfg(any(test, feature = "test"))] + fn open>(_storage: S) -> Result { + // Overwrite any given storage mode with a path to a temporary directory. + let temp_dir = Arc::new(tempfile::TempDir::with_prefix("snarkos_test_")?); + let storage = StorageMode::Custom(temp_dir.path().to_owned()); + + // Initialize the finalize store. + let finalize_store = FinalizeStore::>::open(storage.clone())?; + // Initialize the block store. + let block_store = BlockStore::>::open(storage)?; + // Return the consensus storage. + Ok(Self { + finalize_store, + block_store, + _temp_dir: temp_dir, + }) + } + /// Returns the finalize store. fn finalize_store(&self) -> &FinalizeStore { &self.finalize_store diff --git a/ledger/store/src/helpers/rocksdb/internal/map.rs b/ledger/store/src/helpers/rocksdb/internal/map.rs index c4c6c0a02b..ee7c32c31a 100644 --- a/ledger/store/src/helpers/rocksdb/internal/map.rs +++ b/ledger/store/src/helpers/rocksdb/internal/map.rs @@ -558,15 +558,14 @@ mod tests { FinalizeMode, atomic_batch_scope, atomic_finalize, - helpers::rocksdb::{MapID, TestMap, internal::tests::temp_dir}, + helpers::rocksdb::{MapID, TestMap}, }; use console::{ account::{Address, FromStr}, network::MainnetV0, }; - use anyhow::anyhow; - use serial_test::serial; + use anyhow::{anyhow, bail}; use tracing_test::traced_test; type CurrentNetwork = MainnetV0; @@ -600,7 +599,7 @@ mod tests { impl TestStorage { fn open() -> Self { // Initialize a database. - let database = RocksDB::open_testing(temp_dir(), None).expect("Failed to open a test database"); + let database = RocksDB::open(0, None).expect("Failed to open a test database"); Self { own_map: open_map_testing_from_db(database.clone(), MapID::Test(TestMap::Test)), @@ -733,7 +732,6 @@ mod tests { } #[test] - #[serial] fn test_contains_key_sanity_check() { // Initialize an address. let address = @@ -742,73 +740,67 @@ mod tests { // Initialize a map. let map: DataMap, ()> = - RocksDB::open_map_testing(temp_dir(), None, MapID::Test(TestMap::Test)).expect("Failed to open data map"); + RocksDB::open_map(0, None, MapID::Test(TestMap::Test)).expect("Failed to open data map"); map.insert(address, ()).expect("Failed to insert into data map"); assert!(map.contains_key_confirmed(&address).unwrap()); } #[test] - #[serial] #[traced_test] fn test_insert_and_get_speculative() { // Initialize a map. let map: DataMap = - RocksDB::open_map_testing(temp_dir(), None, MapID::Test(TestMap::Test)).expect("Failed to open data map"); + RocksDB::open_map(0, None, MapID::Test(TestMap::Test)).expect("Failed to open data map"); crate::helpers::test_helpers::map::check_insert_and_get_speculative(map); } #[test] - #[serial] #[traced_test] fn test_remove_and_get_speculative() { // Initialize a map. let map: DataMap = - RocksDB::open_map_testing(temp_dir(), None, MapID::Test(TestMap::Test)).expect("Failed to open data map"); + RocksDB::open_map(0, None, MapID::Test(TestMap::Test)).expect("Failed to open data map"); crate::helpers::test_helpers::map::check_remove_and_get_speculative(map); } #[test] - #[serial] #[traced_test] fn test_contains_key() { // Initialize a map. let map: DataMap = - RocksDB::open_map_testing(temp_dir(), None, MapID::Test(TestMap::Test)).expect("Failed to open data map"); + RocksDB::open_map(0, None, MapID::Test(TestMap::Test)).expect("Failed to open data map"); crate::helpers::test_helpers::map::check_contains_key(map); } #[test] - #[serial] #[traced_test] fn test_check_iterators_match() { // Initialize a map. let map: DataMap = - RocksDB::open_map_testing(temp_dir(), None, MapID::Test(TestMap::Test)).expect("Failed to open data map"); + RocksDB::open_map(0, None, MapID::Test(TestMap::Test)).expect("Failed to open data map"); crate::helpers::test_helpers::map::check_iterators_match(map); } #[test] - #[serial] #[traced_test] fn test_atomic_writes_are_batched() { // Initialize a map. let map: DataMap = - RocksDB::open_map_testing(temp_dir(), None, MapID::Test(TestMap::Test)).expect("Failed to open data map"); + RocksDB::open_map(0, None, MapID::Test(TestMap::Test)).expect("Failed to open data map"); crate::helpers::test_helpers::map::check_atomic_writes_are_batched(map); } #[test] - #[serial] #[traced_test] fn test_atomic_writes_can_be_aborted() { // Initialize a map. let map: DataMap = - RocksDB::open_map_testing(temp_dir(), None, MapID::Test(TestMap::Test)).expect("Failed to open data map"); + RocksDB::open_map(0, None, MapID::Test(TestMap::Test)).expect("Failed to open data map"); crate::helpers::test_helpers::map::check_atomic_writes_can_be_aborted(map); } @@ -820,7 +812,7 @@ mod tests { // Initialize a map. let map: DataMap = - RocksDB::open_map_testing(temp_dir(), None, MapID::Test(TestMap::Test)).expect("Failed to open data map"); + RocksDB::open_map(0, None, MapID::Test(TestMap::Test)).expect("Failed to open data map"); // Sanity check. assert!(map.iter_confirmed().next().is_none()); // Make sure the checkpoint index is None. @@ -894,7 +886,7 @@ mod tests { // Initialize a map. let map: DataMap = - RocksDB::open_map_testing(temp_dir(), None, MapID::Test(TestMap::Test)).expect("Failed to open data map"); + RocksDB::open_map(0, None, MapID::Test(TestMap::Test)).expect("Failed to open data map"); // Sanity check. assert!(map.iter_confirmed().next().is_none()); // Make sure the checkpoint index is None. @@ -956,7 +948,7 @@ mod tests { // Initialize a map. let map: DataMap = - RocksDB::open_map_testing(temp_dir(), None, MapID::Test(TestMap::Test)).expect("Failed to open data map"); + RocksDB::open_map(0, None, MapID::Test(TestMap::Test)).expect("Failed to open data map"); // Sanity check. assert!(map.iter_confirmed().next().is_none()); // Make sure the checkpoint index is None. @@ -1010,7 +1002,7 @@ mod tests { // Initialize a map. let map: DataMap = - RocksDB::open_map_testing(temp_dir(), None, MapID::Test(TestMap::Test)).expect("Failed to open data map"); + RocksDB::open_map(0, None, MapID::Test(TestMap::Test)).expect("Failed to open data map"); // Sanity check. assert!(map.iter_confirmed().next().is_none()); // Make sure the checkpoint index is None. @@ -1089,7 +1081,7 @@ mod tests { // Initialize a map. let map: DataMap = - RocksDB::open_map_testing(temp_dir(), None, MapID::Test(TestMap::Test)).expect("Failed to open data map"); + RocksDB::open_map(0, None, MapID::Test(TestMap::Test)).expect("Failed to open data map"); // Sanity check. assert!(map.iter_confirmed().next().is_none()); // Make sure the checkpoint index is None. @@ -1167,7 +1159,7 @@ mod tests { fn test_atomic_finalize_fails_to_start() { // Initialize a map. let map: DataMap = - RocksDB::open_map_testing(temp_dir(), None, MapID::Test(TestMap::Test)).expect("Failed to open data map"); + RocksDB::open_map(0, None, MapID::Test(TestMap::Test)).expect("Failed to open data map"); // Sanity check. assert!(map.iter_confirmed().next().is_none()); // Make sure the checkpoint index is None. @@ -1200,7 +1192,7 @@ mod tests { fn test_atomic_checkpoint_truncation() { // Initialize a map. let map: DataMap = - RocksDB::open_map_testing(temp_dir(), None, MapID::Test(TestMap::Test)).expect("Failed to open data map"); + RocksDB::open_map(0, None, MapID::Test(TestMap::Test)).expect("Failed to open data map"); // Sanity check. assert!(map.iter_confirmed().next().is_none()); // Make sure the checkpoint index is None. @@ -1258,7 +1250,7 @@ mod tests { fn test_atomic_finalize_with_nested_batch_scope() -> Result<()> { // Initialize a map. let map: DataMap = - RocksDB::open_map_testing(temp_dir(), None, MapID::Test(TestMap::Test)).expect("Failed to open data map"); + RocksDB::open_map(0, None, MapID::Test(TestMap::Test)).expect("Failed to open data map"); // Sanity check. assert!(map.iter_confirmed().next().is_none()); // Make sure the checkpoint index is None. @@ -1376,7 +1368,6 @@ mod tests { } #[test] - #[serial] #[traced_test] fn test_nested_atomic_write_batch_success() -> Result<()> { // Initialize a multi-layer test storage. @@ -1451,7 +1442,6 @@ mod tests { } #[test] - #[serial] #[traced_test] fn test_nested_atomic_write_batch_failure() { // We'll want to execute the atomic write batch in its own function, in order to be able to diff --git a/ledger/store/src/helpers/rocksdb/internal/mod.rs b/ledger/store/src/helpers/rocksdb/internal/mod.rs index a5c5d3c7e4..d9e07c2143 100644 --- a/ledger/store/src/helpers/rocksdb/internal/mod.rs +++ b/ledger/store/src/helpers/rocksdb/internal/mod.rs @@ -26,15 +26,17 @@ pub use nested_map::*; mod tests; use aleo_std_storage::StorageMode; -use anyhow::{Result, bail, ensure}; -use once_cell::sync::OnceCell; +use anyhow::{Result, ensure}; +use once_cell::sync::Lazy; use parking_lot::Mutex; use serde::{Serialize, de::DeserializeOwned}; use std::{ borrow::Borrow, + collections::HashMap, marker::PhantomData, mem, ops::Deref, + path::PathBuf, sync::{ Arc, atomic::{AtomicBool, AtomicUsize, Ordering}, @@ -43,6 +45,11 @@ use std::{ pub const PREFIX_LEN: usize = 4; // N::ID (u16) + DataID (u16) +// A static map of database paths to their objects; it's needed in order to facilitate concurrent +// tests involving persistent storage, but it only ever has a single member outside of them. +// TODO: remove the static in favor of improved `open` methods. +static DATABASES: Mutex>> = Mutex::new(Lazy::new(|| HashMap::new())); + pub trait Database { /// Opens the database. fn open>(network_id: u16, storage: S) -> Result @@ -93,6 +100,10 @@ pub struct RocksDB { pub(super) atomic_writes_paused: Arc, /// This is an optimization that avoids some allocations when querying the database. pub(super) default_readopts: rocksdb::ReadOptions, + /// A test-only instance of TempDir which is cleaned up afterwards; it is optional, as + /// it is not needed when creating the database via the higher-level ConsensusDB. + #[cfg(any(test, feature = "test"))] + temp_dir: Option>, } impl Clone for RocksDB { @@ -105,6 +116,8 @@ impl Clone for RocksDB { atomic_depth: self.atomic_depth.clone(), atomic_writes_paused: self.atomic_writes_paused.clone(), default_readopts: Default::default(), + #[cfg(any(test, feature = "test"))] + temp_dir: self.temp_dir.clone(), } } } @@ -123,11 +136,22 @@ impl Database for RocksDB { /// In production mode, the database opens directory `~/.aleo/storage/ledger-{network}`. /// In development mode, the database opens directory `/path/to/repo/.ledger-{network}-{id}`. fn open>(network_id: u16, storage: S) -> Result { - static DB: OnceCell = OnceCell::new(); + let mut storage = storage.into(); + #[cfg(any(test, feature = "test"))] + let mut temp_dir = None; + // If we are running tests and haven't already received a custom path indicating the + // existence of a TempDir, create one and bind its lifetime to the RocksDB object. + if cfg!(any(test, feature = "test")) && !matches!(storage, StorageMode::Custom(_)) { + temp_dir = Some(Arc::new(tempfile::TempDir::with_prefix("snarkos_test_")?)); + storage = StorageMode::Custom(temp_dir.as_ref().unwrap().path().to_owned()); + } // Retrieve the database. - let database = DB - .get_or_try_init(|| { + let db_path = aleo_std_storage::aleo_ledger_dir(network_id, storage.clone()); + let db = DATABASES + .lock() + .entry(db_path.clone()) + .or_insert_with(|| { // Customize database options. let mut options = rocksdb::Options::default(); options.set_compression_type(rocksdb::DBCompressionType::Lz4); @@ -136,33 +160,30 @@ impl Database for RocksDB { let prefix_extractor = rocksdb::SliceTransform::create_fixed_prefix(PREFIX_LEN); options.set_prefix_extractor(prefix_extractor); - let primary = aleo_std_storage::aleo_ledger_dir(network_id, storage.clone().into()); let rocksdb = { options.increase_parallelism(2); options.set_max_background_jobs(4); options.create_if_missing(true); options.set_max_open_files(8192); - Arc::new(rocksdb::DB::open(&options, primary)?) + Arc::new(rocksdb::DB::open(&options, db_path).expect("Couldn't open the database")) }; - Ok::<_, anyhow::Error>(RocksDB { + RocksDB { rocksdb, network_id, - storage_mode: storage.clone().into(), + storage_mode: storage.into(), atomic_batch: Default::default(), atomic_depth: Default::default(), atomic_writes_paused: Default::default(), default_readopts: Default::default(), - }) - })? + #[cfg(any(test, feature = "test"))] + temp_dir, + } + }) .clone(); - // Ensure the database network ID and storage mode match. - match database.network_id == network_id && database.storage_mode == storage.into() { - true => Ok(database), - false => bail!("Mismatching network ID or storage mode in the database"), - } + Ok(db) } /// Opens the map with the given `network_id`, `storage mode`, and `map_id` from storage. @@ -273,123 +294,6 @@ impl RocksDB { fn are_atomic_writes_paused(&self) -> bool { self.atomic_writes_paused.load(Ordering::SeqCst) } - - /// Opens the test database. - #[cfg(any(test, feature = "test"))] - pub fn open_testing(temp_dir: std::path::PathBuf, dev: Option) -> Result { - use console::prelude::{Rng, TestRng}; - - // Ensure the `temp_dir` is unique. - let temp_dir = temp_dir.join(Rng::gen::(&mut TestRng::default()).to_string()); - - // Construct the directory for the test database. - let primary = match dev { - Some(dev) => temp_dir.join(dev.to_string()), - None => temp_dir, - }; - - // Prepare the storage mode. - let storage_mode = StorageMode::from(primary.clone()); - - let database = { - // Customize database options. - let mut options = rocksdb::Options::default(); - options.set_compression_type(rocksdb::DBCompressionType::Lz4); - - // Register the prefix length. - let prefix_extractor = rocksdb::SliceTransform::create_fixed_prefix(PREFIX_LEN); - options.set_prefix_extractor(prefix_extractor); - - let rocksdb = { - options.increase_parallelism(2); - options.set_max_background_jobs(4); - options.create_if_missing(true); - - // Keep these around as options for configuration testing. - - // options.set_max_subcompactions(4); - // options.set_use_direct_io_for_flush_and_compaction(true); - // options.set_bytes_per_sync(1 << 28); - // options.set_compaction_readahead_size(1 << 28); - // options.set_max_write_buffer_number(16); - // options.set_min_write_buffer_number_to_merge(8); - // options.set_compression_type(rocksdb::DBCompressionType::None); - // options.set_bottommost_compression_type(rocksdb::DBCompressionType::None); - // options.set_write_buffer_size(1 << 28); - - Arc::new(rocksdb::DB::open(&options, primary)?) - }; - - Ok::<_, anyhow::Error>(RocksDB { - rocksdb, - network_id: u16::MAX, - storage_mode: storage_mode.clone(), - atomic_batch: Default::default(), - atomic_depth: Default::default(), - atomic_writes_paused: Default::default(), - default_readopts: Default::default(), - }) - }?; - - // Ensure the database storage mode match. - match database.storage_mode == storage_mode { - true => Ok(database), - false => bail!("Mismatching storage mode in the test database"), - } - } - - /// Opens the test map. - #[cfg(any(test, feature = "test"))] - pub fn open_map_testing>( - temp_dir: std::path::PathBuf, - dev: Option, - map_id: T, - ) -> Result> { - // Open the RocksDB test database. - let database = Self::open_testing(temp_dir, dev)?; - - // Combine contexts to create a new scope. - let mut context = database.network_id.to_le_bytes().to_vec(); - context.extend_from_slice(&(map_id.into()).to_le_bytes()); - - // Return the DataMap. - Ok(DataMap(Arc::new(InnerDataMap { - database, - context, - batch_in_progress: Default::default(), - atomic_batch: Default::default(), - checkpoints: Default::default(), - }))) - } - - /// Opens the test nested map. - #[cfg(any(test, feature = "test"))] - pub fn open_nested_map_testing< - M: Serialize + DeserializeOwned, - K: Serialize + DeserializeOwned, - V: Serialize + DeserializeOwned, - T: Into, - >( - temp_dir: std::path::PathBuf, - dev: Option, - map_id: T, - ) -> Result> { - // Open the RocksDB test database. - let database = Self::open_testing(temp_dir, dev)?; - - // Combine contexts to create a new scope. - let mut context = database.network_id.to_le_bytes().to_vec(); - context.extend_from_slice(&(map_id.into()).to_le_bytes()); - - // Return the DataMap. - Ok(NestedDataMap { - database, - context, - batch_in_progress: Default::default(), - atomic_batch: Default::default(), - checkpoints: Default::default(), - }) - } } // impl RocksDB { diff --git a/ledger/store/src/helpers/rocksdb/internal/nested_map.rs b/ledger/store/src/helpers/rocksdb/internal/nested_map.rs index 245b50d913..e0e106082d 100644 --- a/ledger/store/src/helpers/rocksdb/internal/nested_map.rs +++ b/ledger/store/src/helpers/rocksdb/internal/nested_map.rs @@ -766,7 +766,7 @@ mod tests { atomic_batch_scope, atomic_finalize, helpers::{ - rocksdb::{MapID, TestMap, internal::tests::temp_dir}, + rocksdb::{MapID, TestMap}, traits::Map, }, }; @@ -775,8 +775,7 @@ mod tests { network::MainnetV0, }; - use anyhow::anyhow; - use serial_test::serial; + use anyhow::{anyhow, bail}; use tracing_test::traced_test; type CurrentNetwork = MainnetV0; @@ -837,7 +836,7 @@ mod tests { impl TestStorage { fn open() -> Self { // Initialize a database. - let database = RocksDB::open_testing(temp_dir(), None).expect("Failed to open a test database"); + let database = RocksDB::open(0, None).expect("Failed to open a test database"); Self { own_map: open_map_testing_from_db(database.clone(), MapID::Test(TestMap::Test)), @@ -979,7 +978,6 @@ mod tests { } #[test] - #[serial] fn test_contains_key_sanity_check() { // Initialize 'm'. let m = 0usize; @@ -990,78 +988,66 @@ mod tests { // Initialize a map. let map: NestedDataMap, ()> = - RocksDB::open_nested_map_testing(temp_dir(), None, MapID::Test(TestMap::Test)) - .expect("Failed to open data map"); + RocksDB::open_nested_map(0, None, MapID::Test(TestMap::Test)).expect("Failed to open data map"); map.insert(m, address, ()).expect("Failed to insert into data map"); assert!(map.contains_key_confirmed(&m, &address).unwrap()); } #[test] - #[serial] #[traced_test] fn test_insert_and_get_value_speculative() { // Initialize a map. let map: NestedDataMap = - RocksDB::open_nested_map_testing(temp_dir(), None, MapID::Test(TestMap::Test)) - .expect("Failed to open data map"); + RocksDB::open_nested_map(0, None, MapID::Test(TestMap::Test)).expect("Failed to open data map"); crate::helpers::test_helpers::nested_map::check_insert_and_get_value_speculative(map); } #[test] - #[serial] #[traced_test] fn test_remove_key_and_get_value_speculative() { // Initialize a map. let map: NestedDataMap = - RocksDB::open_nested_map_testing(temp_dir(), None, MapID::Test(TestMap::Test)) - .expect("Failed to open data map"); + RocksDB::open_nested_map(0, None, MapID::Test(TestMap::Test)).expect("Failed to open data map"); crate::helpers::test_helpers::nested_map::check_remove_and_get_value_speculative(map); } #[test] - #[serial] #[traced_test] fn test_contains_key() { // Initialize a map. let map: NestedDataMap = - RocksDB::open_nested_map_testing(temp_dir(), None, MapID::Test(TestMap::Test)) - .expect("Failed to open data map"); + RocksDB::open_nested_map(0, None, MapID::Test(TestMap::Test)).expect("Failed to open data map"); crate::helpers::test_helpers::nested_map::check_contains_key(map); } #[test] - #[serial] #[traced_test] fn test_get_map() { // Initialize a map. let map: NestedDataMap = - RocksDB::open_nested_map_testing(temp_dir(), None, MapID::Test(TestMap::Test)) - .expect("Failed to open data map"); + RocksDB::open_nested_map(0, None, MapID::Test(TestMap::Test)).expect("Failed to open data map"); crate::helpers::test_helpers::nested_map::check_get_map(map); } #[test] - #[serial] #[traced_test] fn test_check_iterators_match() { // Initialize a map. let map: NestedDataMap = - RocksDB::open_nested_map_testing(temp_dir(), None, MapID::Test(TestMap::Test)) - .expect("Failed to open data map"); + RocksDB::open_nested_map(0, None, MapID::Test(TestMap::Test)).expect("Failed to open data map"); crate::helpers::test_helpers::nested_map::check_iterators_match(map); } #[test] - #[serial] #[traced_test] fn test_iter_from_nested_to_non_nested() { // Open a storage with a DataMap right after a NestedDataMap. - let database = RocksDB::open_testing(temp_dir(), None).expect("Failed to open a test database"); + let database = RocksDB::open(0, None).expect("Failed to open a test database"); let test_storage = TestStorage3::open(database); // Insert 5 (confirmed) records into a nested map 77. @@ -1080,25 +1066,21 @@ mod tests { } #[test] - #[serial] #[traced_test] fn test_atomic_writes_are_batched() { // Initialize a map. let map: NestedDataMap = - RocksDB::open_nested_map_testing(temp_dir(), None, MapID::Test(TestMap::Test)) - .expect("Failed to open data map"); + RocksDB::open_nested_map(0, None, MapID::Test(TestMap::Test)).expect("Failed to open data map"); crate::helpers::test_helpers::nested_map::check_atomic_writes_are_batched(map); } #[test] - #[serial] #[traced_test] fn test_atomic_writes_can_be_aborted() { // Initialize a map. let map: NestedDataMap = - RocksDB::open_nested_map_testing(temp_dir(), None, MapID::Test(TestMap::Test)) - .expect("Failed to open data map"); + RocksDB::open_nested_map(0, None, MapID::Test(TestMap::Test)).expect("Failed to open data map"); crate::helpers::test_helpers::nested_map::check_atomic_writes_can_be_aborted(map); } @@ -1110,8 +1092,7 @@ mod tests { // Initialize a map. let map: NestedDataMap = - RocksDB::open_nested_map_testing(temp_dir(), None, MapID::Test(TestMap::Test)) - .expect("Failed to open data map"); + RocksDB::open_nested_map(0, None, MapID::Test(TestMap::Test)).expect("Failed to open data map"); // Sanity check. assert!(map.iter_confirmed().next().is_none()); // Make sure the checkpoint index is None. @@ -1185,8 +1166,7 @@ mod tests { // Initialize a map. let map: NestedDataMap = - RocksDB::open_nested_map_testing(temp_dir(), None, MapID::Test(TestMap::Test)) - .expect("Failed to open data map"); + RocksDB::open_nested_map(0, None, MapID::Test(TestMap::Test)).expect("Failed to open data map"); // Sanity check. assert!(map.iter_confirmed().next().is_none()); // Make sure the checkpoint index is None. @@ -1248,8 +1228,7 @@ mod tests { // Initialize a map. let map: NestedDataMap = - RocksDB::open_nested_map_testing(temp_dir(), None, MapID::Test(TestMap::Test)) - .expect("Failed to open data map"); + RocksDB::open_nested_map(0, None, MapID::Test(TestMap::Test)).expect("Failed to open data map"); // Sanity check. assert!(map.iter_confirmed().next().is_none()); // Make sure the checkpoint index is None. @@ -1303,8 +1282,7 @@ mod tests { // Initialize a map. let map: NestedDataMap = - RocksDB::open_nested_map_testing(temp_dir(), None, MapID::Test(TestMap::Test)) - .expect("Failed to open data map"); + RocksDB::open_nested_map(0, None, MapID::Test(TestMap::Test)).expect("Failed to open data map"); // Sanity check. assert!(map.iter_confirmed().next().is_none()); // Make sure the checkpoint index is None. @@ -1383,8 +1361,7 @@ mod tests { // Initialize a map. let map: NestedDataMap = - RocksDB::open_nested_map_testing(temp_dir(), None, MapID::Test(TestMap::Test)) - .expect("Failed to open data map"); + RocksDB::open_nested_map(0, None, MapID::Test(TestMap::Test)).expect("Failed to open data map"); // Sanity check. assert!(map.iter_confirmed().next().is_none()); // Make sure the checkpoint index is None. @@ -1462,8 +1439,7 @@ mod tests { fn test_atomic_finalize_fails_to_start() { // Initialize a map. let map: NestedDataMap = - RocksDB::open_nested_map_testing(temp_dir(), None, MapID::Test(TestMap::Test)) - .expect("Failed to open data map"); + RocksDB::open_nested_map(0, None, MapID::Test(TestMap::Test)).expect("Failed to open data map"); // Sanity check. assert!(map.iter_confirmed().next().is_none()); // Make sure the checkpoint index is None. @@ -1496,8 +1472,7 @@ mod tests { fn test_atomic_checkpoint_truncation() { // Initialize a map. let map: NestedDataMap = - RocksDB::open_nested_map_testing(temp_dir(), None, MapID::Test(TestMap::Test)) - .expect("Failed to open data map"); + RocksDB::open_nested_map(0, None, MapID::Test(TestMap::Test)).expect("Failed to open data map"); // Sanity check. assert!(map.iter_confirmed().next().is_none()); // Make sure the checkpoint index is None. @@ -1558,8 +1533,7 @@ mod tests { fn test_atomic_finalize_with_nested_batch_scope() -> Result<()> { // Initialize a map. let map: NestedDataMap = - RocksDB::open_nested_map_testing(temp_dir(), None, MapID::Test(TestMap::Test)) - .expect("Failed to open data map"); + RocksDB::open_nested_map(0, None, MapID::Test(TestMap::Test)).expect("Failed to open data map"); // Sanity check. assert!(map.iter_confirmed().next().is_none()); // Make sure the checkpoint index is None. @@ -1677,7 +1651,6 @@ mod tests { } #[test] - #[serial] #[traced_test] fn test_nested_atomic_write_batch_success() -> Result<()> { // Initialize a multi-layer test storage. @@ -1748,7 +1721,6 @@ mod tests { } #[test] - #[serial] #[traced_test] fn test_nested_atomic_write_batch_failure() { // We'll want to execute the atomic write batch in its own function, in order to be able to diff --git a/ledger/store/src/helpers/rocksdb/internal/tests.rs b/ledger/store/src/helpers/rocksdb/internal/tests.rs index cbae47a770..c3bb50e331 100644 --- a/ledger/store/src/helpers/rocksdb/internal/tests.rs +++ b/ledger/store/src/helpers/rocksdb/internal/tests.rs @@ -16,7 +16,7 @@ use crate::helpers::{ Map, MapRead, - rocksdb::{MapID, RocksDB, TestMap as TestMapID}, + rocksdb::{Database, MapID, RocksDB, TestMap as TestMapID}, }; use console::{ network::{MainnetV0, Network}, @@ -24,34 +24,24 @@ use console::{ types::Scalar, }; -use serial_test::serial; - -pub(crate) fn temp_dir() -> std::path::PathBuf { - tempfile::tempdir().expect("Failed to open temporary directory").into_path() -} - // pub(crate) fn temp_file() -> std::path::PathBuf { // tempfile::NamedTempFile::new().expect("Failed to open temporary file").path().to_owned() // } #[test] -#[serial] fn test_open() { - let _storage = RocksDB::open_testing(temp_dir(), None).expect("Failed to open storage"); + let _storage = RocksDB::open(0, None).expect("Failed to open storage"); } #[test] -#[serial] fn test_open_map() { - let _map = RocksDB::open_map_testing::(temp_dir(), None, MapID::Test(TestMapID::Test)) - .expect("Failed to open data map"); + let _map = + RocksDB::open_map::<_, u32, String, _>(0, None, MapID::Test(TestMapID::Test)).expect("Failed to open data map"); } #[test] -#[serial] fn test_insert_and_contains_key() { - let map = - RocksDB::open_map_testing(temp_dir(), None, MapID::Test(TestMapID::Test)).expect("Failed to open data map"); + let map = RocksDB::open_map(0, None, MapID::Test(TestMapID::Test)).expect("Failed to open data map"); map.insert(123456789, "123456789".to_string()).expect("Failed to insert"); assert!(map.contains_key_confirmed(&123456789).expect("Failed to call contains key")); @@ -59,10 +49,8 @@ fn test_insert_and_contains_key() { } #[test] -#[serial] fn test_insert_and_get() { - let map = - RocksDB::open_map_testing(temp_dir(), None, MapID::Test(TestMapID::Test)).expect("Failed to open data map"); + let map = RocksDB::open_map(0, None, MapID::Test(TestMapID::Test)).expect("Failed to open data map"); map.insert(123456789, "123456789".to_string()).expect("Failed to insert"); assert_eq!( @@ -74,10 +62,8 @@ fn test_insert_and_get() { } #[test] -#[serial] fn test_insert_and_remove() { - let map = - RocksDB::open_map_testing(temp_dir(), None, MapID::Test(TestMapID::Test)).expect("Failed to open data map"); + let map = RocksDB::open_map(0, None, MapID::Test(TestMapID::Test)).expect("Failed to open data map"); map.insert(123456789, "123456789".to_string()).expect("Failed to insert"); assert_eq!( @@ -90,10 +76,8 @@ fn test_insert_and_remove() { } #[test] -#[serial] fn test_insert_and_iter() { - let map = - RocksDB::open_map_testing(temp_dir(), None, MapID::Test(TestMapID::Test)).expect("Failed to open data map"); + let map = RocksDB::open_map(0, None, MapID::Test(TestMapID::Test)).expect("Failed to open data map"); map.insert(123456789, "123456789".to_string()).expect("Failed to insert"); @@ -103,10 +87,8 @@ fn test_insert_and_iter() { } #[test] -#[serial] fn test_insert_and_keys() { - let map = - RocksDB::open_map_testing(temp_dir(), None, MapID::Test(TestMapID::Test)).expect("Failed to open data map"); + let map = RocksDB::open_map(0, None, MapID::Test(TestMapID::Test)).expect("Failed to open data map"); map.insert(123456789, "123456789".to_string()).expect("Failed to insert"); @@ -116,10 +98,8 @@ fn test_insert_and_keys() { } #[test] -#[serial] fn test_insert_and_values() { - let map = - RocksDB::open_map_testing(temp_dir(), None, MapID::Test(TestMapID::Test)).expect("Failed to open data map"); + let map = RocksDB::open_map(0, None, MapID::Test(TestMapID::Test)).expect("Failed to open data map"); map.insert(123456789, "123456789".to_string()).expect("Failed to insert"); @@ -129,11 +109,10 @@ fn test_insert_and_values() { } // #[test] -// #[serial] // fn test_export_import() { // let file = temp_file(); // { -// let mut map = RocksDB::open_map_testing(temp_dir(), None, MapID::Test(TestMapID::Test)).expect("Failed to open data map"); +// let mut map = RocksDB::open_map(0, None, MapID::Test(TestMapID::Test)).expect("Failed to open data map"); // // for i in 0..100 { // map.insert(i, i.to_string()).expect("Failed to insert"); @@ -153,7 +132,6 @@ fn test_insert_and_values() { // } #[test] -#[serial] fn test_scalar_mul() { type CurrentNetwork = MainnetV0; @@ -161,8 +139,7 @@ fn test_scalar_mul() { const ITERATIONS: u32 = 1_000_000u32; - let map = - RocksDB::open_map_testing(temp_dir(), None, MapID::Test(TestMapID::Test)).expect("Failed to open data map"); + let map = RocksDB::open_map(0, None, MapID::Test(TestMapID::Test)).expect("Failed to open data map"); // Sample `ITERATION` random field elements to store. for i in 0..ITERATIONS { @@ -182,10 +159,8 @@ fn test_scalar_mul() { } #[test] -#[serial] fn test_iterator_ordering() { - let map = - RocksDB::open_map_testing(temp_dir(), None, MapID::Test(TestMapID::Test)).expect("Failed to open data map"); + let map = RocksDB::open_map(0, None, MapID::Test(TestMapID::Test)).expect("Failed to open data map"); // Insert values into the map. map.insert(5, "d".to_string()).expect("Failed to insert"); diff --git a/ledger/store/src/helpers/rocksdb/program.rs b/ledger/store/src/helpers/rocksdb/program.rs index d57a6fb286..416dd996f4 100644 --- a/ledger/store/src/helpers/rocksdb/program.rs +++ b/ledger/store/src/helpers/rocksdb/program.rs @@ -62,20 +62,6 @@ impl FinalizeStorage for FinalizeDB { }) } - /// Initializes the test-variant of the storage. - #[cfg(any(test, feature = "test"))] - fn open_testing(temp_dir: std::path::PathBuf, dev: Option) -> Result { - // Initialize the committee store. - let committee_store = CommitteeStore::>::open_testing(temp_dir.clone(), dev)?; - // Return the finalize storage. - Ok(Self { - committee_store, - program_id_map: rocksdb::RocksDB::open_map_testing(temp_dir.clone(), dev, MapID::Program(ProgramMap::ProgramID))?, - key_value_map: rocksdb::RocksDB::open_nested_map_testing(temp_dir, dev, MapID::Program(ProgramMap::KeyValueID))?, - storage_mode: dev.into(), - }) - } - /// Returns the committee store. fn committee_store(&self) -> &CommitteeStore { &self.committee_store @@ -126,17 +112,6 @@ impl CommitteeStorage for CommitteeDB { }) } - /// Initializes the test-variant of the storage. - #[cfg(any(test, feature = "test"))] - fn open_testing(temp_dir: std::path::PathBuf, dev: Option) -> Result { - Ok(Self { - current_round_map: rocksdb::RocksDB::open_map_testing(temp_dir.clone(), dev, MapID::Committee(CommitteeMap::CurrentRound))?, - round_to_height_map: rocksdb::RocksDB::open_map_testing(temp_dir.clone(), dev, MapID::Committee(CommitteeMap::RoundToHeight))?, - committee_map: rocksdb::RocksDB::open_map_testing(temp_dir, dev, MapID::Committee(CommitteeMap::Committee))?, - storage_mode: dev.into(), - }) - } - /// Returns the current round map. fn current_round_map(&self) -> &Self::CurrentRoundMap { &self.current_round_map diff --git a/ledger/store/src/program/committee.rs b/ledger/store/src/program/committee.rs index a3261e99d1..0a0abf12ca 100644 --- a/ledger/store/src/program/committee.rs +++ b/ledger/store/src/program/committee.rs @@ -40,10 +40,6 @@ pub trait CommitteeStorage: 'static + Clone + Send + Sync { /// Initializes the committee storage. fn open>(storage: S) -> Result; - /// Initializes the test-variant of the storage. - #[cfg(any(test, feature = "test"))] - fn open_testing(temp_dir: std::path::PathBuf, dev: Option) -> Result; - /// Returns the current round map. fn current_round_map(&self) -> &Self::CurrentRoundMap; /// Returns the round to height map. @@ -308,15 +304,6 @@ impl> CommitteeStore { Ok(Self { storage, _phantom: PhantomData }) } - /// Initializes the test-variant of the storage. - #[cfg(any(test, feature = "test"))] - pub fn open_testing(temp_dir: std::path::PathBuf, dev: Option) -> Result { - // Initialize the committee storage. - let storage = C::open_testing(temp_dir, dev)?; - // Return the committee store. - Ok(Self { storage, _phantom: PhantomData }) - } - /// Initializes a committee store from storage. pub fn from(storage: C) -> Self { Self { storage, _phantom: PhantomData } diff --git a/ledger/store/src/program/finalize.rs b/ledger/store/src/program/finalize.rs index f0662a6feb..77e0d643c9 100644 --- a/ledger/store/src/program/finalize.rs +++ b/ledger/store/src/program/finalize.rs @@ -82,10 +82,6 @@ pub trait FinalizeStorage: 'static + Clone + Send + Sync { /// Initializes the program state storage. fn open>(storage: S) -> Result; - /// Initializes the test-variant of the storage. - #[cfg(any(test, feature = "test"))] - fn open_testing(temp_dir: std::path::PathBuf, dev: Option) -> Result; - /// Returns the committee storage. fn committee_store(&self) -> &CommitteeStore; /// Returns the program ID map. @@ -539,12 +535,6 @@ impl> FinalizeStore { Self::from(P::open(storage)?) } - /// Initializes the test-variant of the storage. - #[cfg(any(test, feature = "test"))] - pub fn open_testing(temp_dir: std::path::PathBuf, dev: Option) -> Result { - Self::from(P::open_testing(temp_dir, dev)?) - } - /// Initializes a finalize store from storage. pub fn from(storage: P) -> Result { // Return the finalize store. @@ -773,6 +763,8 @@ impl> FinalizeStore { mod tests { use super::*; use crate::helpers::memory::FinalizeMemory; + #[cfg(feature = "rocks")] + use crate::{ConsensusStore, consensus::ConsensusStorage}; use console::{network::MainnetV0, program::Literal, types::U64}; type CurrentNetwork = MainnetV0; @@ -1298,13 +1290,15 @@ mod tests { FinalizeStore::from(program_memory).unwrap() }; - // Initialize a new finalize store. + // Initialize a new finalize store; it is done via the consensus store, as that one + // is the primary storage object, and it has a dedicated test version. #[cfg(feature = "rocks")] - let finalize_store = { - let temp_dir = tempfile::tempdir().expect("Failed to open temporary directory").into_path(); - let program_rocksdb = crate::helpers::rocksdb::FinalizeDB::open_testing(temp_dir, None).unwrap(); - FinalizeStore::from(program_rocksdb).unwrap() + let consensus_store = { + let consensus_db = crate::helpers::rocksdb::ConsensusDB::open(None).unwrap(); + ConsensusStore::from(consensus_db) }; + #[cfg(feature = "rocks")] + let finalize_store = consensus_store.finalize_store(); // Now, initialize the mapping. let timer = std::time::Instant::now(); diff --git a/ledger/test-helpers/Cargo.toml b/ledger/test-helpers/Cargo.toml index a07898640f..1d952a6ae9 100644 --- a/ledger/test-helpers/Cargo.toml +++ b/ledger/test-helpers/Cargo.toml @@ -20,6 +20,7 @@ edition = "2021" package = "snarkvm-console" path = "../../console" version = "=1.1.0" +features = [ "test" ] [dependencies.circuit] package = "snarkvm-circuit" @@ -30,6 +31,7 @@ version = "=1.1.0" package = "snarkvm-ledger-block" path = "../block" version = "=1.1.0" +features = [ "test" ] [dependencies.ledger-query] package = "snarkvm-ledger-query" @@ -42,6 +44,7 @@ features = [ "query" ] package = "snarkvm-ledger-store" path = "../store" version = "=1.1.0" +features = [ "test" ] [dependencies.synthesizer-program] package = "snarkvm-synthesizer-program" diff --git a/parameters/Cargo.toml b/parameters/Cargo.toml index c4e6f93496..75a19e699c 100644 --- a/parameters/Cargo.toml +++ b/parameters/Cargo.toml @@ -27,6 +27,7 @@ edition = "2021" default = [ "colored", "remote" ] no_std_out = [ ] remote = [ "curl" ] +rocks = [ "snarkvm-ledger-store/rocks" ] wasm = [ "encoding", "js-sys", "web-sys" ] [dependencies.snarkvm-curves] diff --git a/parameters/examples/inclusion.rs b/parameters/examples/inclusion.rs index dc241cbd95..6f3f7c8c96 100644 --- a/parameters/examples/inclusion.rs +++ b/parameters/examples/inclusion.rs @@ -22,7 +22,13 @@ use snarkvm_console::{ program::{Plaintext, Record, StatePath}, types::Field, }; -use snarkvm_ledger_store::{ConsensusStore, helpers::memory::ConsensusMemory}; +use snarkvm_ledger_store::ConsensusStore; + +#[cfg(not(feature = "rocks"))] +type LedgerType = snarkvm_ledger_store::helpers::memory::ConsensusMemory; +#[cfg(feature = "rocks")] +type LedgerType = snarkvm_ledger_store::helpers::rocksdb::ConsensusDB; + use snarkvm_synthesizer::{VM, process::InclusionAssignment, snark::UniversalSRS}; use anyhow::{Result, anyhow}; @@ -70,7 +76,7 @@ fn write_metadata(filename: &str, metadata: &Value) -> Result<()> { #[allow(clippy::type_complexity)] pub fn sample_assignment>() -> Result<(Assignment, StatePath, Field)> { // Initialize the consensus store. - let store = ConsensusStore::>::open(None)?; + let store = ConsensusStore::>::open(None)?; // Initialize a new VM. let vm = VM::from(store)?; diff --git a/synthesizer/Cargo.toml b/synthesizer/Cargo.toml index d100b6d540..7dd520aaf3 100644 --- a/synthesizer/Cargo.toml +++ b/synthesizer/Cargo.toml @@ -44,7 +44,7 @@ serial = [ "synthesizer-snark/serial" ] setup = [ ] -test = [ "console/test" ] +test = [ "console/test", "ledger-block/test", "ledger-store/test", "synthesizer-process/test" ] timer = [ "aleo-std/timer" ] wasm = [ "process", diff --git a/synthesizer/process/Cargo.toml b/synthesizer/process/Cargo.toml index 56c8de65b6..ef8b1941c5 100644 --- a/synthesizer/process/Cargo.toml +++ b/synthesizer/process/Cargo.toml @@ -43,6 +43,7 @@ wasm = [ "synthesizer-program/wasm", "synthesizer-snark/wasm" ] +test = [ "console/test", "ledger-block/test", "ledger-store/test" ] timer = [ "aleo-std/timer" ] [[bench]] diff --git a/synthesizer/process/src/tests/test_credits.rs b/synthesizer/process/src/tests/test_credits.rs index 9de634854f..42209c5f6c 100644 --- a/synthesizer/process/src/tests/test_credits.rs +++ b/synthesizer/process/src/tests/test_credits.rs @@ -23,6 +23,8 @@ use console::{ }; use ledger_committee::{MIN_DELEGATOR_STAKE, MIN_VALIDATOR_SELF_STAKE, MIN_VALIDATOR_STAKE}; use ledger_query::Query; +#[cfg(feature = "rocks")] +use ledger_store::ConsensusStorage; use ledger_store::{ BlockStore, FinalizeMode, @@ -45,18 +47,14 @@ const TEST_COMMISSION: u8 = 5; macro_rules! sample_finalize_store { () => {{ #[cfg(feature = "rocks")] - let temp_dir = tempfile::tempdir().expect("Failed to open temporary directory"); + let (store, temp_dir) = { + let consensus_db = ledger_store::helpers::rocksdb::ConsensusDB::open(None).unwrap(); + let consensus_store = ledger_store::ConsensusStore::from(consensus_db); + // The consensus store creates and holds the temporary database directory. + (consensus_store.finalize_store().clone(), consensus_store) + }; #[cfg(not(feature = "rocks"))] - let temp_dir = (); - - #[cfg(feature = "rocks")] - let store = FinalizeStore::>::open_testing( - temp_dir.path().to_owned(), - None, - ) - .unwrap(); - #[cfg(not(feature = "rocks"))] - let store = FinalizeStore::>::open(None).unwrap(); + let (store, temp_dir) = (FinalizeStore::>::open(None).unwrap(), ()); (store, temp_dir) }}; diff --git a/synthesizer/src/vm/execute.rs b/synthesizer/src/vm/execute.rs index 97ab70d379..dc68e43ccd 100644 --- a/synthesizer/src/vm/execute.rs +++ b/synthesizer/src/vm/execute.rs @@ -220,18 +220,21 @@ mod tests { types::Field, }; use ledger_block::Transition; - use ledger_store::helpers::memory::ConsensusMemory; use synthesizer_process::{ConsensusFeeVersion, cost_per_command, execution_cost_v2}; use synthesizer_program::StackProgram; use indexmap::IndexMap; type CurrentNetwork = MainnetV0; + #[cfg(not(feature = "rocks"))] + type LedgerType = ledger_store::helpers::memory::ConsensusMemory; + #[cfg(feature = "rocks")] + type LedgerType = ledger_store::helpers::rocksdb::ConsensusDB; fn prepare_vm( rng: &mut TestRng, ) -> Result<( - VM>, + VM, IndexMap, Record>>, )> { // Initialize the genesis block. diff --git a/synthesizer/src/vm/finalize.rs b/synthesizer/src/vm/finalize.rs index 199a571060..c90319269e 100644 --- a/synthesizer/src/vm/finalize.rs +++ b/synthesizer/src/vm/finalize.rs @@ -1390,16 +1390,19 @@ mod tests { }; use ledger_block::{Block, Header, Metadata, Transaction, Transition}; use ledger_committee::{MAX_DELEGATORS, MIN_VALIDATOR_STAKE}; - use ledger_store::helpers::memory::ConsensusMemory; use synthesizer_program::Program; use rand::distributions::DistString; type CurrentNetwork = test_helpers::CurrentNetwork; + #[cfg(not(feature = "rocks"))] + type LedgerType = ledger_store::helpers::memory::ConsensusMemory; + #[cfg(feature = "rocks")] + type LedgerType = ledger_store::helpers::rocksdb::ConsensusDB; /// Sample a new program and deploy it to the VM. Returns the program name. fn new_program_deployment( - vm: &VM>, + vm: &VM, private_key: &PrivateKey, previous_block: &Block, unspent_records: &mut Vec>>, @@ -1467,7 +1470,7 @@ finalize transfer_public: /// Construct a new block based on the given transactions. fn sample_next_block( - vm: &VM>, + vm: &VM, private_key: &PrivateKey, transactions: &[Transaction], previous_block: &Block, @@ -1537,7 +1540,7 @@ finalize transfer_public: /// Generate split transactions for the unspent records. fn generate_splits( - vm: &VM>, + vm: &VM, private_key: &PrivateKey, previous_block: &Block, unspent_records: &mut Vec>>, @@ -1576,7 +1579,7 @@ finalize transfer_public: /// Create an execution transaction. fn create_execution( - vm: &VM>, + vm: &VM, caller_private_key: PrivateKey, program_id: &str, function_name: &str, @@ -1603,7 +1606,7 @@ finalize transfer_public: /// Sample a public mint transaction. fn sample_mint_public( - vm: &VM>, + vm: &VM, caller_private_key: PrivateKey, program_id: &str, recipient: Address, @@ -1621,7 +1624,7 @@ finalize transfer_public: /// Sample a public transfer transaction. fn sample_transfer_public( - vm: &VM>, + vm: &VM, caller_private_key: PrivateKey, program_id: &str, recipient: Address, @@ -2388,11 +2391,11 @@ finalize compute: let mut transactions = Vec::new(); let mut excess_transaction_ids = Vec::new(); - for _ in 0..VM::>::MAXIMUM_CONFIRMED_TRANSACTIONS + 1 { + for _ in 0..VM::::MAXIMUM_CONFIRMED_TRANSACTIONS + 1 { let transaction = sample_mint_public(&vm, caller_private_key, &program_id, caller_address, 10, &mut unspent_records, rng); // Abort the transaction if the block is full. - if transactions.len() >= VM::>::MAXIMUM_CONFIRMED_TRANSACTIONS { + if transactions.len() >= VM::::MAXIMUM_CONFIRMED_TRANSACTIONS { excess_transaction_ids.push(transaction.id()); } @@ -2406,10 +2409,7 @@ finalize compute: // Ensure that the excess transactions were aborted. assert_eq!(next_block.aborted_transaction_ids(), &excess_transaction_ids); - assert_eq!( - next_block.transactions().len(), - VM::>::MAXIMUM_CONFIRMED_TRANSACTIONS - ); + assert_eq!(next_block.transactions().len(), VM::::MAXIMUM_CONFIRMED_TRANSACTIONS); } #[test] diff --git a/synthesizer/src/vm/mod.rs b/synthesizer/src/vm/mod.rs index 72b7166b70..0ea6e0595b 100644 --- a/synthesizer/src/vm/mod.rs +++ b/synthesizer/src/vm/mod.rs @@ -471,36 +471,29 @@ pub(crate) mod test_helpers { types::Field, }; use ledger_block::{Block, Header, Metadata, Transition}; - use ledger_store::helpers::memory::ConsensusMemory; - #[cfg(feature = "rocks")] - use ledger_store::helpers::rocksdb::ConsensusDB; use ledger_test_helpers::{large_transaction_program, small_transaction_program}; use synthesizer_program::Program; use indexmap::IndexMap; use once_cell::sync::OnceCell; - #[cfg(feature = "rocks")] - use std::path::Path; use synthesizer_snark::VerifyingKey; pub(crate) type CurrentNetwork = MainnetV0; + #[cfg(not(feature = "rocks"))] + type LedgerType = ledger_store::helpers::memory::ConsensusMemory; + #[cfg(feature = "rocks")] + type LedgerType = ledger_store::helpers::rocksdb::ConsensusDB; /// Samples a new finalize state. pub(crate) fn sample_finalize_state(block_height: u32) -> FinalizeGlobalState { FinalizeGlobalState::from(block_height as u64, block_height, [0u8; 32]) } - pub(crate) fn sample_vm() -> VM> { + pub(crate) fn sample_vm() -> VM { // Initialize a new VM. VM::from(ConsensusStore::open(None).unwrap()).unwrap() } - #[cfg(feature = "rocks")] - pub(crate) fn sample_vm_rocks(path: &Path) -> VM> { - // Initialize a new VM. - VM::from(ConsensusStore::open(path.to_owned()).unwrap()).unwrap() - } - pub(crate) fn sample_genesis_private_key(rng: &mut TestRng) -> PrivateKey { static INSTANCE: OnceCell> = OnceCell::new(); *INSTANCE.get_or_init(|| { @@ -523,9 +516,7 @@ pub(crate) mod test_helpers { .clone() } - pub(crate) fn sample_vm_with_genesis_block( - rng: &mut TestRng, - ) -> VM> { + pub(crate) fn sample_vm_with_genesis_block(rng: &mut TestRng) -> VM { // Initialize the VM. let vm = crate::vm::test_helpers::sample_vm(); // Initialize the genesis block. @@ -784,7 +775,7 @@ function compute: } pub fn sample_next_block( - vm: &VM>, + vm: &VM, private_key: &PrivateKey, transactions: &[Transaction], rng: &mut R, @@ -2563,8 +2554,7 @@ finalize transfer_public_to_private: let block2 = sample_next_block(&vm, &genesis_private_key, &[], rng).unwrap(); // Create a new, rocks-based VM shadowing the 1st one. - let tempdir = tempfile::tempdir().unwrap(); - let vm = sample_vm_rocks(tempdir.path()); + let vm = sample_vm(); vm.add_next_block(&genesis).unwrap(); // This time, however, try to insert the 2nd block first, which fails due to height. assert!(vm.add_next_block(&block2).is_err()); diff --git a/synthesizer/tests/test_vm_execute_and_finalize.rs b/synthesizer/tests/test_vm_execute_and_finalize.rs index c15e42c4db..7d597e9385 100644 --- a/synthesizer/tests/test_vm_execute_and_finalize.rs +++ b/synthesizer/tests/test_vm_execute_and_finalize.rs @@ -31,7 +31,7 @@ use ledger_block::{ Transactions, Transition, }; -use ledger_store::{ConsensusStorage, ConsensusStore, helpers::memory::ConsensusMemory}; +use ledger_store::{ConsensusStorage, ConsensusStore}; use snarkvm_synthesizer::{VM, program::FinalizeOperation}; use synthesizer_program::FinalizeGlobalState; @@ -41,6 +41,11 @@ use indexmap::IndexMap; use rayon::prelude::*; use utilities::*; +#[cfg(not(feature = "rocks"))] +type LedgerType = ledger_store::helpers::memory::ConsensusMemory; +#[cfg(feature = "rocks")] +type LedgerType = ledger_store::helpers::rocksdb::ConsensusDB; + #[test] fn test_vm_execute_and_finalize() { // Load the tests. @@ -365,10 +370,9 @@ fn run_test(test: &ProgramTest) -> serde_yaml::Mapping { fn initialize_vm( private_key: &PrivateKey, rng: &mut R, -) -> (VM>, Vec>>) { +) -> (VM>, Vec>>) { // Initialize a VM. - let vm: VM> = - VM::from(ConsensusStore::open(None).unwrap()).unwrap(); + let vm: VM> = VM::from(ConsensusStore::open(None).unwrap()).unwrap(); // Initialize the genesis block. let genesis = vm.genesis_beacon(private_key, rng).unwrap();