Add Solidity tests JS benchmark (#506)

crates/edr_napi/index.d.ts

@@ -536,7 +536,7 @@ export class Response {
 /** Executes solidity tests. */
 export class SolidityTestRunner {
   /**Creates a new instance of the SolidityTestRunner. The callback function will be called with suite results as they finish. */
-  constructor(resultsCallback: (...args: any[]) => any)
+  constructor(gasReport: boolean, resultsCallback: (...args: any[]) => any)
   /**Runs the given test suites. */
   runTests(testSuites: Array<TestSuite>): Promise<Array<SuiteResult>>
 }

crates/edr_napi/src/solidity_tests.rs

@@ -24,14 +24,15 @@ use crate::solidity_tests::{
 pub struct SolidityTestRunner {
     /// The callback to call with the results as they become available.
     results_callback_fn: ThreadsafeFunction<SuiteResult>,
+    gas_report: bool,
 }
 
 // The callback has to be passed in the constructor because it's not `Send`.
 #[napi]
 impl SolidityTestRunner {
     #[doc = "Creates a new instance of the SolidityTestRunner. The callback function will be called with suite results as they finish."]
     #[napi(constructor)]
-    pub fn new(env: Env, results_callback: JsFunction) -> napi::Result<Self> {
+    pub fn new(env: Env, gas_report: bool, results_callback: JsFunction) -> napi::Result<Self> {
         let mut results_callback_fn: ThreadsafeFunction<_, ErrorStrategy::CalleeHandled> =
             results_callback.create_threadsafe_function(
                 // Unbounded queue size
@@ -44,6 +45,7 @@ impl SolidityTestRunner {
 
         Ok(Self {
             results_callback_fn,
+            gas_report,
         })
     }
 
@@ -54,7 +56,7 @@ impl SolidityTestRunner {
             .into_iter()
             .map(|item| Ok((item.id.try_into()?, item.contract.try_into()?)))
             .collect::<Result<Vec<_>, napi::Error>>()?;
-        let runner = build_runner(test_suites)?;
+        let runner = build_runner(test_suites, self.gas_report)?;
 
         let (tx_results, mut rx_results) =
             tokio::sync::mpsc::unbounded_channel::<(String, forge::result::SuiteResult)>();

crates/edr_napi/src/solidity_tests/runner.rs

@@ -8,23 +8,32 @@ use forge::{
     multi_runner::TestContract,
     opts::{Env as EvmEnv, EvmOpts},
     revm::primitives::SpecId,
-    MultiContractRunner, MultiContractRunnerBuilder, TestOptions, TestOptionsBuilder,
+    MultiContractRunner, MultiContractRunnerBuilder, TestOptionsBuilder,
 };
 use foundry_compilers::ArtifactId;
-use foundry_config::{
-    Config, FuzzConfig, FuzzDictionaryConfig, InvariantConfig, RpcEndpoint, RpcEndpoints,
-};
+use foundry_config::{Config, RpcEndpoint, RpcEndpoints};
 
 pub(super) fn build_runner(
     test_suites: Vec<(ArtifactId, TestContract)>,
+    gas_report: bool,
 ) -> napi::Result<MultiContractRunner> {
-    let config = foundry_config();
+    let config = foundry_config(gas_report);
+
     let mut evm_opts = evm_opts();
     evm_opts.isolate = config.isolate;
+    evm_opts.verbosity = config.verbosity;
+    evm_opts.memory_limit = config.memory_limit;
+    evm_opts.env.gas_limit = config.gas_limit.into();
+
+    let test_options = TestOptionsBuilder::default()
+        .fuzz(config.fuzz.clone())
+        .invariant(config.invariant.clone())
+        .build_hardhat()
+        .expect("Config loaded");
 
     let builder = MultiContractRunnerBuilder::new(Arc::new(config))
         .sender(evm_opts.sender)
-        .with_test_options(test_opts());
+        .with_test_options(test_options);
 
     let abis = test_suites.iter().map(|(_, contract)| &contract.abi);
     let revert_decoder = RevertDecoder::new().with_abis(abis);
@@ -55,7 +64,7 @@ fn project_root() -> PathBuf {
     ))
 }
 
-fn foundry_config() -> Config {
+fn foundry_config(gas_report: bool) -> Config {
     const TEST_PROFILE: &str = "default";
 
     // Forge project root.
@@ -77,6 +86,11 @@ fn foundry_config() -> Config {
     // no prompt testing
     config.prompt_timeout = 0;
 
+    if !gas_report {
+        config.fuzz.gas_report_samples = 0;
+        config.invariant.gas_report_samples = 0;
+    }
+
     config
 }
 
@@ -103,41 +117,3 @@ fn evm_opts() -> EvmOpts {
 fn rpc_endpoints() -> RpcEndpoints {
     RpcEndpoints::new([("alchemy", RpcEndpoint::Url("${ALCHEMY_URL}".to_string()))])
 }
-
-pub fn test_opts() -> TestOptions {
-    TestOptionsBuilder::default()
-        .fuzz(FuzzConfig {
-            runs: 256,
-            max_test_rejects: 65536,
-            seed: None,
-            dictionary: FuzzDictionaryConfig {
-                include_storage: true,
-                include_push_bytes: true,
-                dictionary_weight: 40,
-                max_fuzz_dictionary_addresses: 10_000,
-                max_fuzz_dictionary_values: 10_000,
-            },
-            gas_report_samples: 256,
-            failure_persist_dir: Some(tempfile::tempdir().unwrap().into_path()),
-            failure_persist_file: Some("testfailure".to_string()),
-        })
-        .invariant(InvariantConfig {
-            runs: 256,
-            depth: 15,
-            fail_on_revert: false,
-            call_override: false,
-            dictionary: FuzzDictionaryConfig {
-                dictionary_weight: 80,
-                include_storage: true,
-                include_push_bytes: true,
-                max_fuzz_dictionary_addresses: 10_000,
-                max_fuzz_dictionary_values: 10_000,
-            },
-            shrink_run_limit: 2usize.pow(18u32),
-            max_assume_rejects: 65536,
-            gas_report_samples: 256,
-            failure_persist_dir: Some(tempfile::tempdir().unwrap().into_path()),
-        })
-        .build_hardhat()
-        .expect("Config loaded")
-}

crates/edr_napi/test/solidity-tests.ts

@@ -1,4 +1,7 @@
 import { assert } from "chai";
+import { mkdtemp } from "fs/promises";
+import { tmpdir } from "os";
+import path from "path";
 
 import {
   SolidityTestRunner,
@@ -15,8 +18,10 @@ describe("Solidity Tests", () => {
       loadContract("./artifacts/PaymentFailureTest.json"),
     ];
 
+    const gasReport = false;
+
     const resultsFromCallback: Array<SuiteResult> = [];
-    const runner = new SolidityTestRunner((...args) => {
+    const runner = new SolidityTestRunner(gasReport, (...args) => {
       resultsFromCallback.push(args[1] as SuiteResult);
     });
 

crates/tools/js/benchmark/.gitignore

@@ -1 +1,2 @@
 node_modules
+forge-std

crates/tools/js/benchmark/index.js

@@ -13,6 +13,7 @@ const {
   createHardhatNetworkProvider,
 } = require("hardhat/internal/hardhat-network/provider/provider");
 const { HttpProvider } = require("hardhat/internal/core/providers/http");
+const { runForgeStdTests, setupForgeStdRepo } = require("./solidity-tests");
 
 const SCENARIOS_DIR = "../../scenarios/";
 const SCENARIO_SNAPSHOT_NAME = "snapshot.json";
@@ -24,7 +25,7 @@ async function main() {
     description: "Scenario benchmark runner",
   });
   parser.add_argument("command", {
-    choices: ["benchmark", "verify", "report"],
+    choices: ["benchmark", "verify", "report", "solidity-tests"],
     help: "Whether to run a benchmark, verify that there are no regressions or create a report for `github-action-benchmark`",
   });
   parser.add_argument("-g", "--grep", {
@@ -61,6 +62,9 @@ async function main() {
   } else if (args.command === "report") {
     await report(args.benchmark_output);
     await flushStdout();
+  } else if (args.command === "solidity-tests") {
+    const repoPath = await setupForgeStdRepo();
+    await runForgeStdTests(repoPath);
   }
 }
 
@@ -90,10 +94,9 @@ async function report(benchmarkResultPath) {
 async function verify(benchmarkResultPath) {
   let success = true;
   const benchmarkResult = require(benchmarkResultPath);
-  const snapshotResult = require(path.join(
-    getScenariosDir(),
-    SCENARIO_SNAPSHOT_NAME
-  ));
+  const snapshotResult = require(
+    path.join(getScenariosDir(), SCENARIO_SNAPSHOT_NAME),
+  );
 
   for (let scenarioName in snapshotResult) {
     // TODO https://github.com/NomicFoundation/edr/issues/365
@@ -107,7 +110,7 @@ async function verify(benchmarkResultPath) {
       if (ratio > NEPTUNE_MAX_MIN_FAILURES) {
         console.error(
           `Snapshot failure for ${scenarioName} with max/min failure ratio`,
-          ratio
+          ratio,
         );
         success = false;
       }
@@ -129,16 +132,16 @@ async function verify(benchmarkResultPath) {
       if (shouldFail.size > 0) {
         console.error(
           `Scenario ${scenarioName} should fail at indexes ${Array.from(
-            shouldFail
-          ).sort()}`
+            shouldFail,
+          ).sort()}`,
         );
       }
 
       if (shouldNotFail.size > 0) {
         console.error(
           `Scenario ${scenarioName} should not fail at indexes ${Array.from(
-            shouldNotFail
-          ).sort()}`
+            shouldNotFail,
+          ).sort()}`,
         );
       }
     }
@@ -207,7 +210,7 @@ async function benchmarkAllScenarios(outPath, useAnvil) {
   console.error(
     `Total time ${
       Math.round(100 * (totalTime / 1000)) / 100
-    } seconds with ${totalFailures} failures.`
+    } seconds with ${totalFailures} failures.`,
   );
 
   console.error(`Benchmark results written to ${outPath}`);
@@ -275,7 +278,7 @@ async function benchmarkScenario(scenarioFileName, useAnvil) {
   console.error(
     `${name} finished in ${
       Math.round(100 * (timeMs / 1000)) / 100
-    } seconds with ${failures.length} failures.`
+    } seconds with ${failures.length} failures.`,
   );
 
   const result = {
@@ -331,11 +334,11 @@ function preprocessConfig(config) {
   config = removeNull(config);
 
   config.providerConfig.initialDate = new Date(
-    config.providerConfig.initialDate.secsSinceEpoch * 1000
+    config.providerConfig.initialDate.secsSinceEpoch * 1000,
   );
 
   config.providerConfig.hardfork = normalizeHardfork(
-    config.providerConfig.hardfork
+    config.providerConfig.hardfork,
   );
 
   // "accounts" in EDR are "genesisAccounts" in Hardhat
@@ -345,7 +348,7 @@ function preprocessConfig(config) {
   config.providerConfig.genesisAccounts = config.providerConfig.accounts.map(
     ({ balance, secretKey }) => {
       return { balance, privateKey: secretKey };
-    }
+    },
   );
   delete config.providerConfig.accounts;
 

crates/tools/js/benchmark/package.json

@@ -7,6 +7,8 @@
   "scripts": {
     "benchmark": "node --noconcurrent_sweeping --noconcurrent_recompilation --max-old-space-size=28000 index.js benchmark",
     "prebenchmark": "cd ../../../edr_napi/ && pnpm build",
+    "soltests": "node --noconcurrent_sweeping --noconcurrent_recompilation --max-old-space-size=28000 index.js solidity-tests",
+    "presoltests": "cd ../../../edr_napi/ && pnpm build",
     "verify": "node index.js verify",
     "report": "node index.js report",
     "test": "mocha --recursive \"test/**/*.js\"",
@@ -16,11 +18,13 @@
   "author": "",
   "license": "ISC",
   "devDependencies": {
+    "@nomicfoundation/edr": "workspace:^",
     "argparse": "^2.0.1",
     "chai": "^4.2.0",
     "hardhat": "2.22.5",
     "lodash": "^4.17.11",
     "mocha": "^10.0.0",
+    "simple-git": "^3.25.0",
     "tsx": "^4.7.1"
   }
 }