From 765d33b065aa47ed4da482b0ef5e7c055b9d92d5 Mon Sep 17 00:00:00 2001 From: Jacek Glen Date: Wed, 26 Jun 2024 11:58:09 +0200 Subject: [PATCH 1/8] Add bytecode execution benchmarks --- crates/revm/Cargo.toml | 5 +++ crates/revm/README.md | 19 ++++++++++ crates/revm/benches/bytecode.rs | 62 +++++++++++++++++++++++++++++++++ 3 files changed, 86 insertions(+) create mode 100644 crates/revm/README.md create mode 100644 crates/revm/benches/bytecode.rs diff --git a/crates/revm/Cargo.toml b/crates/revm/Cargo.toml index bae8199148..ae620ffd9c 100644 --- a/crates/revm/Cargo.toml +++ b/crates/revm/Cargo.toml @@ -152,3 +152,8 @@ required-features = ["std", "serde-json"] name = "bench" path = "benches/bench.rs" harness = false + +[[bench]] +name = "bytecode_benchmark" +path = "benches/bytecode.rs" +harness = false diff --git a/crates/revm/README.md b/crates/revm/README.md new file mode 100644 index 0000000000..6db80f0bde --- /dev/null +++ b/crates/revm/README.md @@ -0,0 +1,19 @@ +# REVM benchmarks + +## Standard benchmarks + +This executes the standard benchmarks for the REVM crate. + +Execute the following command to run the standard benchmarks: + +`cargo bench --bench bench` + +## Bytecode execution benchmark +This is a command line tool to benchmark the execution of any given bytecode. + +Execute the following command to run the bytecode execution benchmark: + +`echo "60FF600052610FFF600020" | cargo bench --bench=bytecode_benchmark` + +## Debug +For debug information set environmental variable CRITERION_DEBUG=1. \ No newline at end of file diff --git a/crates/revm/benches/bytecode.rs b/crates/revm/benches/bytecode.rs new file mode 100644 index 0000000000..e746d023bf --- /dev/null +++ b/crates/revm/benches/bytecode.rs @@ -0,0 +1,62 @@ +use criterion::{criterion_group, criterion_main, Criterion}; +use revm::{ + db::BenchmarkDB, + interpreter::{analysis::to_analysed, Contract, DummyHost, Interpreter}, + primitives::{address, hex, Bytecode, Bytes, ShanghaiSpec, TxKind}, + Evm, +}; +use revm_interpreter::{opcode::make_instruction_table, SharedMemory, EMPTY_SHARED_MEMORY}; +use std::io; + +fn bytecode_bench(c: &mut Criterion) { + let mut code_input = String::new(); + let stdin = io::stdin(); + match stdin.read_line(&mut code_input) { + Ok(_n) => {} + Err(_error) => {} + } + + if code_input.trim().len() == 0 { + println!("Empty bytecode on stdin."); + return + } + + let code_input = code_input.trim(); + let bytecode = to_analysed(Bytecode::new_raw(hex::decode(code_input).unwrap().into())); + + let evm = Evm::builder() + .with_db(BenchmarkDB::new_bytecode(Bytecode::new())) + .modify_tx_env(|tx| { + tx.caller = address!("1000000000000000000000000000000000000000"); + tx.transact_to = TxKind::Call(address!("0000000000000000000000000000000000000000")); + }) + .build(); + + let mut g = c.benchmark_group("bytecode"); + + g.bench_function("bytecode-benchmark", |b| { + let contract = Contract { + input: Bytes::from(hex::decode("").unwrap()), + bytecode: to_analysed(bytecode.clone()), + ..Default::default() + }; + let mut shared_memory = SharedMemory::new(); + let mut host = DummyHost::new(*evm.context.evm.env.clone()); + let instruction_table = make_instruction_table::(); + b.iter(move || { + let temp = core::mem::replace(&mut shared_memory, EMPTY_SHARED_MEMORY); + let mut interpreter = Interpreter::new(contract.clone(), u64::MAX, false); + let res = interpreter.run(temp, &instruction_table, &mut host); + shared_memory = interpreter.take_memory(); + host.clear(); + res + }) + }); + + g.finish(); +} + +#[rustfmt::skip] +criterion_group!(benches, bytecode_bench); + +criterion_main!(benches); From b9d8070696325ee38caf7ed914c9b87084ac4d4d Mon Sep 17 00:00:00 2001 From: Jacek Glen Date: Wed, 26 Jun 2024 12:10:30 +0200 Subject: [PATCH 2/8] Fmt fix --- crates/revm/benches/bytecode.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/revm/benches/bytecode.rs b/crates/revm/benches/bytecode.rs index e746d023bf..10732783d5 100644 --- a/crates/revm/benches/bytecode.rs +++ b/crates/revm/benches/bytecode.rs @@ -18,7 +18,7 @@ fn bytecode_bench(c: &mut Criterion) { if code_input.trim().len() == 0 { println!("Empty bytecode on stdin."); - return + return; } let code_input = code_input.trim(); From 82d634483d9e216707857f6f4a00aa514452f3e5 Mon Sep 17 00:00:00 2001 From: Jacek Glen Date: Wed, 26 Jun 2024 12:15:03 +0200 Subject: [PATCH 3/8] Clippy fix --- crates/revm/benches/bytecode.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/revm/benches/bytecode.rs b/crates/revm/benches/bytecode.rs index 10732783d5..7c4014e8a5 100644 --- a/crates/revm/benches/bytecode.rs +++ b/crates/revm/benches/bytecode.rs @@ -16,7 +16,7 @@ fn bytecode_bench(c: &mut Criterion) { Err(_error) => {} } - if code_input.trim().len() == 0 { + if code_input.trim().is_empty() { println!("Empty bytecode on stdin."); return; } From 2041aa066be1d440b3a00b4c5fa0a9b773ed4b82 Mon Sep 17 00:00:00 2001 From: Jacek Glen Date: Wed, 3 Jul 2024 15:13:41 +0200 Subject: [PATCH 4/8] Migrate revme to Criterion --- Cargo.lock | 2 +- bins/revme/Cargo.toml | 2 +- bins/revme/README.md | 31 ++++++++++++++++- bins/revme/src/cmd/evmrunner.rs | 54 +++++++++++++++++++++------- crates/revm/Cargo.toml | 5 --- crates/revm/README.md | 19 ---------- crates/revm/benches/bytecode.rs | 62 --------------------------------- 7 files changed, 73 insertions(+), 102 deletions(-) delete mode 100644 crates/revm/README.md delete mode 100644 crates/revm/benches/bytecode.rs diff --git a/Cargo.lock b/Cargo.lock index 838a95ac93..1350c8ffa8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3206,12 +3206,12 @@ name = "revme" version = "0.6.0" dependencies = [ "alloy-rlp", + "criterion", "hash-db", "hashbrown", "hex", "indicatif", "k256", - "microbench", "plain_hasher", "revm", "serde", diff --git a/bins/revme/Cargo.toml b/bins/revme/Cargo.toml index 04f9b26442..2546178fe4 100644 --- a/bins/revme/Cargo.toml +++ b/bins/revme/Cargo.toml @@ -9,11 +9,11 @@ description = "Rust Ethereum Virtual Machine Executable" version = "0.6.0" [dependencies] +criterion = "0.5" hash-db = "0.15" hex = "0.4" hashbrown = "0.14" indicatif = "0.17" -microbench = "0.5" plain_hasher = "0.2" revm = { path = "../../crates/revm", version = "10.0.0", default-features = false, features = [ "ethersdb", diff --git a/bins/revme/README.md b/bins/revme/README.md index 61390f233e..fb005126df 100644 --- a/bins/revme/README.md +++ b/bins/revme/README.md @@ -2,7 +2,17 @@ `revme` is a binary crate to execute the evm in multiple ways. -Currently it is mainly used to run ethereum tests with the `statetest` subcommand. + +## Usage +You can run it directly from the command line with the following command: +```shell +cargo run -p revme +``` + +or build an optimized bin and re-use with: +```shell +cargo build -p revme --profile release +``` ## State Tests @@ -24,3 +34,22 @@ cargo run -p revme statetest tests/GeneralStateTests is ignored so it won't be checked into git.* [et]: https://github.com/ethereum/tests + +## Evm + +`evm` executes any given bytecode and returns the result, for example: + +```shell +cargo run -p revme evm 60FF60005261000F600020 +``` + +### Benchmarks + +Adding the `--bench` flag will run the benchmarks. It is important to run all the benchmarks in the release mode, as the results can be misleading in the debug mode. + +Example of running the benchmarks: +```shell +cargo run -p revme --profile release evm 60FF60005261000F600020 --bench +``` + +The detailed reports and comparisons can be found in the `target/criterion` directory. \ No newline at end of file diff --git a/bins/revme/src/cmd/evmrunner.rs b/bins/revme/src/cmd/evmrunner.rs index 50a75a8deb..8baa4c5abd 100644 --- a/bins/revme/src/cmd/evmrunner.rs +++ b/bins/revme/src/cmd/evmrunner.rs @@ -2,12 +2,15 @@ use revm::{ db::BenchmarkDB, inspector_handle_register, inspectors::TracerEip3155, - primitives::{Address, Bytecode, TxKind}, + interpreter::{ + analysis::to_analysed, opcode::make_instruction_table, Contract, DummyHost, Interpreter, + EMPTY_SHARED_MEMORY, + }, + primitives::{address, Address, Bytecode, Bytes, ShanghaiSpec, TxKind}, Evm, }; use std::io::Error as IoError; use std::path::PathBuf; -use std::time::Duration; use std::{borrow::Cow, fs}; use structopt::StructOpt; @@ -75,6 +78,12 @@ impl Cmd { let input = hex::decode(self.input.trim()) .map_err(|_| Errors::InvalidInput)? .into(); + + if self.bench { + run_benchmark(bytecode_str); + return Ok(()); + } + // BenchmarkDB is dummy state that implements Database trait. // the bytecode is deployed at zero address. let mut evm = Evm::builder() @@ -91,17 +100,6 @@ impl Cmd { }) .build(); - if self.bench { - // Microbenchmark - let bench_options = microbench::Options::default().time(Duration::from_secs(3)); - - microbench::bench(&bench_options, "Run bytecode", || { - let _ = evm.transact().unwrap(); - }); - - return Ok(()); - } - let out = if self.trace { let mut evm = evm .modify() @@ -125,3 +123,33 @@ impl Cmd { Ok(()) } } + +fn run_benchmark(bytecode_str: Cow) { + let evm = Evm::builder() + .with_db(BenchmarkDB::new_bytecode(Bytecode::new())) + .modify_tx_env(|tx| { + tx.caller = address!("1000000000000000000000000000000000000000"); + tx.transact_to = TxKind::Call(address!("0000000000000000000000000000000000000000")); + }) + .build(); + + let host = DummyHost::new(*evm.context.evm.env.clone()); + let instruction_table = make_instruction_table::(); + let contract = Contract { + input: Bytes::from(hex::decode("").unwrap()), + bytecode: to_analysed(Bytecode::new_raw( + hex::decode(bytecode_str.to_string()).unwrap().into(), + )), + ..Default::default() + }; + let mut criterion = criterion::Criterion::default(); + let mut criterion_group = criterion.benchmark_group("revme"); + criterion_group.bench_function("bytecode", |b| { + b.iter(|| { + let mut interpreter = Interpreter::new(contract.clone(), u64::MAX, false); + let res = interpreter.run(EMPTY_SHARED_MEMORY, &instruction_table, &mut host.clone()); + res + }) + }); + criterion_group.finish(); +} diff --git a/crates/revm/Cargo.toml b/crates/revm/Cargo.toml index ae620ffd9c..bae8199148 100644 --- a/crates/revm/Cargo.toml +++ b/crates/revm/Cargo.toml @@ -152,8 +152,3 @@ required-features = ["std", "serde-json"] name = "bench" path = "benches/bench.rs" harness = false - -[[bench]] -name = "bytecode_benchmark" -path = "benches/bytecode.rs" -harness = false diff --git a/crates/revm/README.md b/crates/revm/README.md deleted file mode 100644 index 6db80f0bde..0000000000 --- a/crates/revm/README.md +++ /dev/null @@ -1,19 +0,0 @@ -# REVM benchmarks - -## Standard benchmarks - -This executes the standard benchmarks for the REVM crate. - -Execute the following command to run the standard benchmarks: - -`cargo bench --bench bench` - -## Bytecode execution benchmark -This is a command line tool to benchmark the execution of any given bytecode. - -Execute the following command to run the bytecode execution benchmark: - -`echo "60FF600052610FFF600020" | cargo bench --bench=bytecode_benchmark` - -## Debug -For debug information set environmental variable CRITERION_DEBUG=1. \ No newline at end of file diff --git a/crates/revm/benches/bytecode.rs b/crates/revm/benches/bytecode.rs deleted file mode 100644 index 7c4014e8a5..0000000000 --- a/crates/revm/benches/bytecode.rs +++ /dev/null @@ -1,62 +0,0 @@ -use criterion::{criterion_group, criterion_main, Criterion}; -use revm::{ - db::BenchmarkDB, - interpreter::{analysis::to_analysed, Contract, DummyHost, Interpreter}, - primitives::{address, hex, Bytecode, Bytes, ShanghaiSpec, TxKind}, - Evm, -}; -use revm_interpreter::{opcode::make_instruction_table, SharedMemory, EMPTY_SHARED_MEMORY}; -use std::io; - -fn bytecode_bench(c: &mut Criterion) { - let mut code_input = String::new(); - let stdin = io::stdin(); - match stdin.read_line(&mut code_input) { - Ok(_n) => {} - Err(_error) => {} - } - - if code_input.trim().is_empty() { - println!("Empty bytecode on stdin."); - return; - } - - let code_input = code_input.trim(); - let bytecode = to_analysed(Bytecode::new_raw(hex::decode(code_input).unwrap().into())); - - let evm = Evm::builder() - .with_db(BenchmarkDB::new_bytecode(Bytecode::new())) - .modify_tx_env(|tx| { - tx.caller = address!("1000000000000000000000000000000000000000"); - tx.transact_to = TxKind::Call(address!("0000000000000000000000000000000000000000")); - }) - .build(); - - let mut g = c.benchmark_group("bytecode"); - - g.bench_function("bytecode-benchmark", |b| { - let contract = Contract { - input: Bytes::from(hex::decode("").unwrap()), - bytecode: to_analysed(bytecode.clone()), - ..Default::default() - }; - let mut shared_memory = SharedMemory::new(); - let mut host = DummyHost::new(*evm.context.evm.env.clone()); - let instruction_table = make_instruction_table::(); - b.iter(move || { - let temp = core::mem::replace(&mut shared_memory, EMPTY_SHARED_MEMORY); - let mut interpreter = Interpreter::new(contract.clone(), u64::MAX, false); - let res = interpreter.run(temp, &instruction_table, &mut host); - shared_memory = interpreter.take_memory(); - host.clear(); - res - }) - }); - - g.finish(); -} - -#[rustfmt::skip] -criterion_group!(benches, bytecode_bench); - -criterion_main!(benches); From a747898acaff0fe794c3a47110edc8d76c5d4a58 Mon Sep 17 00:00:00 2001 From: Jacek Glen Date: Wed, 3 Jul 2024 15:29:26 +0200 Subject: [PATCH 5/8] Clippy changes --- bins/revme/src/cmd/evmrunner.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/bins/revme/src/cmd/evmrunner.rs b/bins/revme/src/cmd/evmrunner.rs index 8baa4c5abd..ea507af1bc 100644 --- a/bins/revme/src/cmd/evmrunner.rs +++ b/bins/revme/src/cmd/evmrunner.rs @@ -143,12 +143,15 @@ fn run_benchmark(bytecode_str: Cow) { ..Default::default() }; let mut criterion = criterion::Criterion::default(); + let mut criterion_group = criterion.benchmark_group("revme"); criterion_group.bench_function("bytecode", |b| { b.iter(|| { - let mut interpreter = Interpreter::new(contract.clone(), u64::MAX, false); - let res = interpreter.run(EMPTY_SHARED_MEMORY, &instruction_table, &mut host.clone()); - res + Interpreter::new(contract.clone(), u64::MAX, false).run( + EMPTY_SHARED_MEMORY, + &instruction_table, + &mut host.clone(), + ); }) }); criterion_group.finish(); From 46ff8cb66feb78a9e0a042dc6a3caa319a6c593f Mon Sep 17 00:00:00 2001 From: Jacek Glen Date: Mon, 8 Jul 2024 16:17:52 +0200 Subject: [PATCH 6/8] Update spec --- bins/revme/src/cmd/evmrunner.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bins/revme/src/cmd/evmrunner.rs b/bins/revme/src/cmd/evmrunner.rs index ea507af1bc..72477000fe 100644 --- a/bins/revme/src/cmd/evmrunner.rs +++ b/bins/revme/src/cmd/evmrunner.rs @@ -6,7 +6,7 @@ use revm::{ analysis::to_analysed, opcode::make_instruction_table, Contract, DummyHost, Interpreter, EMPTY_SHARED_MEMORY, }, - primitives::{address, Address, Bytecode, Bytes, ShanghaiSpec, TxKind}, + primitives::{address, Address, Bytecode, Bytes, LatestSpec, TxKind}, Evm, }; use std::io::Error as IoError; @@ -134,7 +134,7 @@ fn run_benchmark(bytecode_str: Cow) { .build(); let host = DummyHost::new(*evm.context.evm.env.clone()); - let instruction_table = make_instruction_table::(); + let instruction_table = make_instruction_table::(); let contract = Contract { input: Bytes::from(hex::decode("").unwrap()), bytecode: to_analysed(Bytecode::new_raw( From 9c048494c7ec17ff80c86d57a0e5e9133e8e73e1 Mon Sep 17 00:00:00 2001 From: Jacek Glen Date: Fri, 13 Sep 2024 11:38:21 +0200 Subject: [PATCH 7/8] Update to EvmWiring --- bins/revme/src/cmd/evmrunner.rs | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/bins/revme/src/cmd/evmrunner.rs b/bins/revme/src/cmd/evmrunner.rs index 38e2df1793..cbd5ac5857 100644 --- a/bins/revme/src/cmd/evmrunner.rs +++ b/bins/revme/src/cmd/evmrunner.rs @@ -1,13 +1,15 @@ use clap::Parser; use revm::{ - db::BenchmarkDB, + db::{BenchmarkDB, EthereumBenchmarkWiring}, inspector_handle_register, inspectors::TracerEip3155, interpreter::{ analysis::to_analysed, opcode::make_instruction_table, Contract, DummyHost, Interpreter, EMPTY_SHARED_MEMORY, }, - primitives::{address, Address, Bytecode, BytecodeDecodeError, Bytes, EthereumWiring, LatestSpec, TxKind}, + primitives::{ + address, Address, Bytecode, BytecodeDecodeError, Bytes, EthereumWiring, LatestSpec, TxKind, + }, Database, Evm, }; use std::io::Error as IoError; @@ -81,7 +83,7 @@ impl Cmd { run_benchmark(bytecode_str); return Ok(()); } - + let mut db = BenchmarkDB::new_bytecode(Bytecode::new_raw_checked(bytecode.into())?); let nonce = db.basic(CALLER).unwrap().map_or(0, |account| account.nonce); @@ -97,12 +99,12 @@ impl Cmd { tx.data = input; tx.nonce = nonce; }) + .with_external_context(TracerEip3155::new(Box::new(std::io::stdout()))) .build(); let out = if self.trace { let mut evm = evm .modify() - .with_external_context(TracerEip3155::new(Box::new(std::io::stdout()))) .append_handler_register(inspector_handle_register) .build(); @@ -122,16 +124,19 @@ impl Cmd { } fn run_benchmark(bytecode_str: Cow) { - let evm = Evm::builder() + let evm = Evm::::builder() .with_db(BenchmarkDB::new_bytecode(Bytecode::new())) .modify_tx_env(|tx| { - tx.caller = address!("1000000000000000000000000000000000000000"); - tx.transact_to = TxKind::Call(address!("0000000000000000000000000000000000000000")); + tx.caller = "1000000000000000000000000000000000000000".parse().unwrap(); + tx.transact_to = + TxKind::Call("0000000000000000000000000000000000000000".parse().unwrap()); }) + .with_external_context(()) .build(); let host = DummyHost::new(*evm.context.evm.env.clone()); - let instruction_table = make_instruction_table::(); + let instruction_table = + make_instruction_table::>, LatestSpec>(); let contract = Contract { input: Bytes::from(hex::decode("").unwrap()), bytecode: to_analysed(Bytecode::new_raw( From d82c7621d7256d8067e97845411c68a3e1d6ef57 Mon Sep 17 00:00:00 2001 From: Jacek Glen Date: Sun, 15 Sep 2024 15:20:22 +0200 Subject: [PATCH 8/8] Apply changes from PR comments --- bins/revme/src/cmd/evmrunner.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/bins/revme/src/cmd/evmrunner.rs b/bins/revme/src/cmd/evmrunner.rs index cbd5ac5857..24fbd26d3b 100644 --- a/bins/revme/src/cmd/evmrunner.rs +++ b/bins/revme/src/cmd/evmrunner.rs @@ -5,7 +5,7 @@ use revm::{ inspectors::TracerEip3155, interpreter::{ analysis::to_analysed, opcode::make_instruction_table, Contract, DummyHost, Interpreter, - EMPTY_SHARED_MEMORY, + SharedMemory, }, primitives::{ address, Address, Bytecode, BytecodeDecodeError, Bytes, EthereumWiring, LatestSpec, TxKind, @@ -144,13 +144,16 @@ fn run_benchmark(bytecode_str: Cow) { )), ..Default::default() }; - let mut criterion = criterion::Criterion::default(); + let mut criterion = criterion::Criterion::default() + .warm_up_time(std::time::Duration::from_millis(100)) + .measurement_time(std::time::Duration::from_secs(1)) + .without_plots(); let mut criterion_group = criterion.benchmark_group("revme"); criterion_group.bench_function("bytecode", |b| { b.iter(|| { Interpreter::new(contract.clone(), u64::MAX, false).run( - EMPTY_SHARED_MEMORY, + SharedMemory::new(), &instruction_table, &mut host.clone(), );