refactor: simplify and unify the storage setup for tests

Signed-off-by: ljedrz <[email protected]>

.circleci/config.yml

@@ -506,7 +506,7 @@ jobs:
     resource_class: << pipeline.parameters.medium >>
     steps:
       - run_serial:
-          flags: --features=rocks
+          flags: --features=rocks,test
           workspace_member: ledger
           cache_key: v1.0.0-rust-1.81.0-snarkvm-ledger-with-rocksdb-cache
 
@@ -643,7 +643,7 @@ jobs:
     resource_class: << pipeline.parameters.twoxlarge >>
     steps:
       - run_serial:
-          flags: --features=rocks
+          flags: --features=rocks,test
           workspace_member: ledger/store
           cache_key: v1.0.0-rust-1.81.0-snarkvm-ledger-store-cache
 
@@ -712,7 +712,7 @@ jobs:
     resource_class: << pipeline.parameters.xlarge >>
     steps:
       - run_serial:
-          flags: --features=rocks
+          flags: --features=rocks,test
           workspace_member: synthesizer/process
           cache_key: v1.0.0-rust-1.81.0-snarkvm-synthesizer-process-cache
 
@@ -945,8 +945,7 @@ workflows:
       - curves
       - fields
       - ledger
-# TODO (howardwu) - Implement `open_testing` on all storage, update to `CurrentConsensusStore::open_testing`, then re-enable.
-#      - ledger-with-rocksdb
+      - ledger-with-rocksdb
       - ledger-with-valid-solutions
       - ledger-authority
       - ledger-block

Cargo.toml

@@ -135,7 +135,7 @@ history = [ "snarkvm-synthesizer/history" ]
 parameters_no_std_out = [ "snarkvm-parameters/no_std_out" ]
 noconfig = [ ]
 rocks = [ "snarkvm-ledger/rocks", "snarkvm-synthesizer/rocks" ]
-test = [ "snarkvm-ledger/test" ]
+test = [ "snarkvm-console/test", "snarkvm-ledger/test", "snarkvm-synthesizer/test" ]
 test-helpers = [ "snarkvm-ledger/test-helpers" ]
 timer = [ "snarkvm-ledger/timer" ]
 algorithms = [ "snarkvm-algorithms" ]

ledger/Cargo.toml

@@ -39,7 +39,7 @@ async = [
   "synthesizer/async"
 ]
 metrics = [ "ledger-committee/metrics" ]
-rocks = [ "ledger-store/rocks" ]
+rocks = [ "ledger-store/rocks", "synthesizer/rocks" ]
 serial = [
   "console/serial",
   "ledger-authority/serial",
@@ -51,7 +51,7 @@ serial = [
   "ledger-store/serial",
   "synthesizer/serial"
 ]
-test = [ "console/test", "ledger-block/test", "ledger-store/test" ]
+test = [ "console/test", "ledger-block/test", "ledger-store/test", "synthesizer/test" ]
 test-helpers = [
   "ledger-test-helpers",
   "ledger-committee/test-helpers",

ledger/benches/transaction.rs

@@ -24,16 +24,21 @@ use console::{
     program::{Plaintext, Record, Value},
 };
 use ledger_block::Transition;
-use ledger_store::{ConsensusStore, helpers::memory::ConsensusMemory};
+use ledger_store::ConsensusStore;
 use synthesizer::{VM, program::Program};
 
 use criterion::Criterion;
 use indexmap::IndexMap;
 
+#[cfg(not(feature = "rocks"))]
+type LedgerType<N> = ledger_store::helpers::memory::ConsensusMemory<N>;
+#[cfg(feature = "rocks")]
+type LedgerType<N> = ledger_store::helpers::rocksdb::ConsensusDB<N>;
+
 fn initialize_vm<R: Rng + CryptoRng>(
     private_key: &PrivateKey<MainnetV0>,
     rng: &mut R,
-) -> (VM<MainnetV0, ConsensusMemory<MainnetV0>>, Vec<Record<MainnetV0, Plaintext<MainnetV0>>>) {
+) -> (VM<MainnetV0, LedgerType<MainnetV0>>, Vec<Record<MainnetV0, Plaintext<MainnetV0>>>) {
     // Initialize the VM.
     let vm = VM::from(ConsensusStore::open(None).unwrap()).unwrap();
 

ledger/block/Cargo.toml

@@ -34,7 +34,7 @@ wasm = [
   "synthesizer-program/wasm",
   "synthesizer-snark/wasm"
 ]
-test = [ ]
+test = [ "console/test" ]
 
 [dependencies.console]
 package = "snarkvm-console"

ledger/src/tests.rs

@@ -30,7 +30,7 @@ use ledger_authority::Authority;
 use ledger_block::{Block, ConfirmedTransaction, Execution, Ratify, Rejected, Transaction};
 use ledger_committee::{Committee, MIN_VALIDATOR_STAKE};
 use ledger_narwhal::{BatchCertificate, BatchHeader, Data, Subdag, Transmission, TransmissionID};
-use ledger_store::{ConsensusStore, helpers::memory::ConsensusMemory};
+use ledger_store::ConsensusStore;
 use snarkvm_utilities::try_vm_runtime;
 use synthesizer::{Stack, program::Program, vm::VM};
 
@@ -39,9 +39,14 @@ use rand::seq::SliceRandom;
 use std::collections::{BTreeMap, HashMap};
 use time::OffsetDateTime;
 
+#[cfg(not(feature = "rocks"))]
+type LedgerType<N> = ledger_store::helpers::memory::ConsensusMemory<N>;
+#[cfg(feature = "rocks")]
+type LedgerType<N> = ledger_store::helpers::rocksdb::ConsensusDB<N>;
+
 /// Initializes a sample VM.
-fn sample_vm() -> VM<CurrentNetwork, ConsensusMemory<CurrentNetwork>> {
-    VM::from(ConsensusStore::<CurrentNetwork, ConsensusMemory<CurrentNetwork>>::open(None).unwrap()).unwrap()
+fn sample_vm() -> VM<CurrentNetwork, LedgerType<CurrentNetwork>> {
+    VM::from(ConsensusStore::<CurrentNetwork, LedgerType<CurrentNetwork>>::open(None).unwrap()).unwrap()
 }
 
 /// Extract the transmissions from a block.
@@ -71,8 +76,7 @@ fn construct_quorum_blocks(
 ) -> Vec<Block<CurrentNetwork>> {
     // Initialize the ledger with the genesis block.
     let ledger =
-        Ledger::<CurrentNetwork, ConsensusMemory<CurrentNetwork>>::load(genesis.clone(), StorageMode::Production)
-            .unwrap();
+        Ledger::<CurrentNetwork, LedgerType<CurrentNetwork>>::load(genesis.clone(), StorageMode::Production).unwrap();
 
     // Initialize the round parameters.
     assert!(num_blocks > 0);
@@ -121,7 +125,7 @@ fn construct_quorum_blocks(
 
     // Helper function to create a quorum block.
     fn create_next_quorum_block(
-        ledger: &Ledger<CurrentNetwork, ConsensusMemory<CurrentNetwork>>,
+        ledger: &Ledger<CurrentNetwork, LedgerType<CurrentNetwork>>,
         round: u64,
         leader_certificate: &BatchCertificate<CurrentNetwork>,
         previous_leader_certificate: Option<&BatchCertificate<CurrentNetwork>>,
@@ -182,7 +186,7 @@ fn test_load() {
     // Sample the genesis private key.
     let private_key = PrivateKey::<CurrentNetwork>::new(rng).unwrap();
     // Initialize the store.
-    let store = ConsensusStore::<_, ConsensusMemory<_>>::open(None).unwrap();
+    let store = ConsensusStore::<_, LedgerType<_>>::open(None).unwrap();
     // Create a genesis block.
     let genesis = VM::from(store).unwrap().genesis_beacon(&private_key, rng).unwrap();
 
@@ -1990,8 +1994,7 @@ fn test_max_committee_limit_with_bonds() {
 
     // Initialize a Ledger from the genesis block.
     let ledger =
-        Ledger::<CurrentNetwork, ConsensusMemory<CurrentNetwork>>::load(genesis_block, StorageMode::Production)
-            .unwrap();
+        Ledger::<CurrentNetwork, LedgerType<CurrentNetwork>>::load(genesis_block, StorageMode::Production).unwrap();
 
     // Bond the first validator.
     let bond_first_transaction = ledger
@@ -3050,7 +3053,7 @@ fn test_forged_block_subdags() {
     // Sample the genesis private key.
     let private_key = PrivateKey::<CurrentNetwork>::new(rng).unwrap();
     // Initialize the store.
-    let store = ConsensusStore::<_, ConsensusMemory<_>>::open(None).unwrap();
+    let store = ConsensusStore::<_, LedgerType<_>>::open(None).unwrap();
     // Create a genesis block with a seeded RNG to reproduce the same genesis private keys.
     let seed: u64 = rng.gen();
     let genesis_rng = &mut TestRng::from_seed(seed);
@@ -3074,8 +3077,7 @@ fn test_forged_block_subdags() {
     let block_3 = quorum_blocks.remove(0);
 
     // Construct the ledger.
-    let ledger =
-        Ledger::<CurrentNetwork, ConsensusMemory<CurrentNetwork>>::load(genesis, StorageMode::Production).unwrap();
+    let ledger = Ledger::<CurrentNetwork, LedgerType<CurrentNetwork>>::load(genesis, StorageMode::Production).unwrap();
     ledger.advance_to_next_block(&block_1).unwrap();
     ledger.check_next_block(&block_2, rng).unwrap();
 

ledger/store/Cargo.toml

@@ -37,7 +37,7 @@ wasm = [
   "synthesizer-program/wasm",
   "synthesizer-snark/wasm"
 ]
-test = [ ]
+test = [ "console/test", "ledger-block/test", "dep:tempfile" ]
 
 [dependencies.console]
 package = "snarkvm-console"
@@ -123,6 +123,10 @@ default-features = false
 features = [ "write" ]
 optional = true
 
+[dependencies.tempfile]
+version = "3.8"
+optional = true
+
 [dependencies.tracing]
 version = "0.1"
 optional = true
@@ -136,11 +140,5 @@ features = [ "test-helpers" ]
 package = "snarkvm-ledger-test-helpers"
 path = "../../ledger/test-helpers"
 
-[dev-dependencies.serial_test]
-version = "2"
-
-[dev-dependencies.tempfile]
-version = "3.8"
-
 [dev-dependencies.tracing-test]
 version = "0.2"

ledger/store/src/helpers/memory/program.rs

@@ -62,12 +62,6 @@ impl<N: Network> FinalizeStorage<N> for FinalizeMemory<N> {
         })
     }
 
-    /// Initializes the test-variant of the storage.
-    #[cfg(any(test, feature = "test"))]
-    fn open_testing(_: std::path::PathBuf, dev: Option<u16>) -> Result<Self> {
-        Self::open(dev)
-    }
-
     /// Returns the committee store.
     fn committee_store(&self) -> &CommitteeStore<N, Self::CommitteeStorage> {
         &self.committee_store
@@ -118,12 +112,6 @@ impl<N: Network> CommitteeStorage<N> for CommitteeMemory<N> {
         })
     }
 
-    /// Initializes the test-variant of the storage.
-    #[cfg(any(test, feature = "test"))]
-    fn open_testing(_: std::path::PathBuf, dev: Option<u16>) -> Result<Self> {
-        Self::open(dev)
-    }
-
     /// Returns the current round map.
     fn current_round_map(&self) -> &Self::CurrentRoundMap {
         &self.current_round_map

ledger/store/src/helpers/rocksdb/consensus.rs

@@ -22,6 +22,8 @@ use crate::{
 use console::prelude::*;
 
 use aleo_std_storage::StorageMode;
+#[cfg(any(test, feature = "test"))]
+use std::sync::Arc;
 
 /// An RocksDB consensus storage.
 #[derive(Clone)]
@@ -30,6 +32,9 @@ pub struct ConsensusDB<N: Network> {
     finalize_store: FinalizeStore<N, FinalizeDB<N>>,
     /// The block store.
     block_store: BlockStore<N, BlockDB<N>>,
+    /// A test-only instance of TempDir which is cleaned up afterwards.
+    #[cfg(any(test, feature = "test"))]
+    _temp_dir: Arc<tempfile::TempDir>,
 }
 
 #[rustfmt::skip]
@@ -40,6 +45,7 @@ impl<N: Network> ConsensusStorage<N> for ConsensusDB<N> {
     type TransitionStorage = TransitionDB<N>;
 
     /// Initializes the consensus storage.
+    #[cfg(not(any(test, feature = "test")))]
     fn open<S: Clone + Into<StorageMode>>(storage: S) -> Result<Self> {
         // Initialize the finalize store.
         let finalize_store = FinalizeStore::<N, FinalizeDB<N>>::open(storage.clone())?;
@@ -52,6 +58,25 @@ impl<N: Network> ConsensusStorage<N> for ConsensusDB<N> {
         })
     }
 
+    /// Initializes a test-only consensus storage.
+    #[cfg(any(test, feature = "test"))]
+    fn open<S: Clone + Into<StorageMode>>(_storage: S) -> Result<Self> {
+        // Overwrite any given storage mode with a path to a temporary directory.
+        let temp_dir = Arc::new(tempfile::TempDir::with_prefix("snarkos_test_")?);
+        let storage = StorageMode::Custom(temp_dir.path().to_owned());
+
+        // Initialize the finalize store.
+        let finalize_store = FinalizeStore::<N, FinalizeDB<N>>::open(storage.clone())?;
+        // Initialize the block store.
+        let block_store = BlockStore::<N, BlockDB<N>>::open(storage)?;
+        // Return the consensus storage.
+        Ok(Self {
+            finalize_store,
+            block_store,
+            _temp_dir: temp_dir,
+        })
+    }
+
     /// Returns the finalize store.
     fn finalize_store(&self) -> &FinalizeStore<N, Self::FinalizeStorage> {
         &self.finalize_store