diff --git a/crates/rbuilder/src/backtest/backtest_build_block.rs b/crates/rbuilder/src/backtest/backtest_build_block.rs deleted file mode 100644 index 5d0953bb..00000000 --- a/crates/rbuilder/src/backtest/backtest_build_block.rs +++ /dev/null @@ -1,398 +0,0 @@ -//! Backtest app to build a single block in a similar way as we do in live. -//! It gets the orders from a HistoricalDataStorage, simulates the orders and then runs the building algorithms. -//! It outputs the best algorithm (most profit) so we can check for improvements in our [crate::building::builders::BlockBuildingAlgorithm]s -//! BlockBuildingAlgorithm are defined on the config file but selected on the command line via "--builders" -//! Sample call: -//! backtest-build-block --config /home/happy_programmer/config.toml --builders mgp-ordering --builders mp-ordering 19380913 --show-orders --show-missing - -use ahash::HashMap; -use alloy_primitives::utils::format_ether; - -use crate::{ - backtest::{ - execute::{backtest_prepare_ctx_for_block, BacktestBlockInput}, - restore_landed_orders::{ - restore_landed_orders, sim_historical_block, ExecutedBlockTx, ExecutedTxs, - SimplifiedOrder, - }, - BlockData, HistoricalDataStorage, OrdersWithTimestamp, - }, - building::builders::BacktestSimulateBlockInput, - live_builder::{base_config::load_config_toml_and_env, cli::LiveBuilderConfig}, - primitives::{Order, OrderId, SimulatedOrder}, - utils::timestamp_as_u64, -}; -use clap::Parser; -use std::path::PathBuf; - -#[derive(Parser, Debug)] -struct Cli { - #[clap(long, help = "Config file path", env = "RBUILDER_CONFIG")] - config: PathBuf, - #[clap( - long, - help = "build block lag (ms)", - default_value = "0", - allow_hyphen_values = true - )] - block_building_time_ms: i64, - #[clap(long, help = "Show all available orders")] - show_orders: bool, - #[clap(long, help = "Show order data and top of block simulation results")] - show_sim: bool, - #[clap(long, help = "Show landed block txs values")] - sim_landed_block: bool, - #[clap(long, help = "Show missing block txs")] - show_missing: bool, - #[clap(long, help = "don't build block")] - no_block_building: bool, - #[clap( - long, - help = "builders to build block with (see config builders)", - default_value = "mp-ordering" - )] - builders: Vec, - #[clap(long, help = "use only this orders")] - only_order_ids: Vec, - #[clap(help = "Block Number")] - block: u64, -} - -pub async fn run_backtest_build_block() -> eyre::Result<()> -where - ConfigType: LiveBuilderConfig, -{ - let cli = Cli::parse(); - - let config: ConfigType = load_config_toml_and_env(cli.config)?; - config.base_config().setup_tracing_subscriber()?; - - let block_data = read_block_data( - &config.base_config().backtest_fetch_output_file, - cli.block, - cli.only_order_ids, - cli.block_building_time_ms, - cli.show_missing, - ) - .await?; - - let (orders, order_and_timestamp): (Vec, HashMap) = block_data - .available_orders - .iter() - .map(|order| (order.order.clone(), (order.order.id(), order.timestamp_ms))) - .unzip(); - - println!("Available orders: {}", orders.len()); - - if cli.show_orders { - print_order_and_timestamp(&block_data.available_orders, &block_data); - } - - let provider_factory = config.base_config().create_provider_factory()?; - let chain_spec = config.base_config().chain_spec()?; - - if cli.sim_landed_block { - let tx_sim_results = sim_historical_block( - provider_factory.clone(), - chain_spec.clone(), - block_data.onchain_block.clone(), - )?; - print_onchain_block_data(tx_sim_results, &orders, &block_data); - } - - let BacktestBlockInput { - ctx, sim_orders, .. - } = backtest_prepare_ctx_for_block( - block_data.clone(), - provider_factory.clone(), - chain_spec.clone(), - cli.block_building_time_ms, - config.base_config().blocklist()?, - &config.base_config().sbundle_mergeable_signers(), - config.base_config().coinbase_signer()?, - )?; - - if cli.show_sim { - print_simulated_orders(&sim_orders, &order_and_timestamp, &block_data); - } - - if !cli.no_block_building { - let winning_builder = cli - .builders - .iter() - .filter_map(|builder_name: &String| { - let input = BacktestSimulateBlockInput { - ctx: ctx.clone(), - builder_name: builder_name.clone(), - sim_orders: &sim_orders, - provider: provider_factory.clone(), - cached_reads: None, - }; - let build_res = config.build_backtest_block(builder_name, input); - if let Err(err) = &build_res { - println!("Error building block: {:?}", err); - return None; - } - let (block, _) = build_res.ok()?; - println!("Built block {} with builder: {:?}", cli.block, builder_name); - println!("Builder profit: {}", format_ether(block.trace.bid_value)); - println!( - "Number of used orders: {}", - block.trace.included_orders.len() - ); - - println!("Used orders:"); - for order_result in &block.trace.included_orders { - println!( - "{:>74} gas: {:>8} profit: {}", - order_result.order.id().to_string(), - order_result.gas_used, - format_ether(order_result.coinbase_profit), - ); - if let Order::Bundle(_) | Order::ShareBundle(_) = order_result.order { - for tx in &order_result.txs { - println!(" ↳ {:?}", tx.hash()); - } - - for (to, value) in &order_result.paid_kickbacks { - println!( - " - kickback to: {:?} value: {}", - to, - format_ether(*value) - ); - } - } - } - Some((builder_name.clone(), block.trace.bid_value)) - }) - .max_by_key(|(_, value)| *value); - - if let Some((builder_name, value)) = winning_builder { - println!( - "Winning builder: {} with profit: {}", - builder_name, - format_ether(value) - ); - } - } - - Ok(()) -} - -/// Reads from HistoricalDataStorage the BlockData for block. -/// only_order_ids: if not empty returns only the given order ids. -/// block_building_time_ms: If not 0, time it took to build the block. It allows us to filter out orders that arrived after we started building the block (filter_late_orders). -/// show_missing: show on-chain orders that weren't available to us at building time. -async fn read_block_data( - backtest_fetch_output_file: &PathBuf, - block: u64, - only_order_ids: Vec, - block_building_time_ms: i64, - show_missing: bool, -) -> eyre::Result { - let mut historical_data_storage = - HistoricalDataStorage::new_from_path(backtest_fetch_output_file).await?; - - let mut block_data = historical_data_storage.read_block_data(block).await?; - - if !only_order_ids.is_empty() { - block_data.filter_orders_by_ids(&only_order_ids); - } - if block_building_time_ms != 0 { - block_data.filter_late_orders(block_building_time_ms); - } - - if show_missing { - show_missing_txs(&block_data); - } - - println!( - "Block: {} {:?}", - block_data.block_number, block_data.onchain_block.header.hash - ); - println!( - "bid value: {}", - format_ether(block_data.winning_bid_trace.value) - ); - println!( - "builder pubkey: {:?}", - block_data.winning_bid_trace.builder_pubkey - ); - Ok(block_data) -} - -/// Convert a timestamp in milliseconds to the slot time relative to the given block timestamp. -fn timestamp_ms_to_slot_time(timestamp_ms: u64, block_timestamp: u64) -> i64 { - (block_timestamp * 1000) as i64 - (timestamp_ms as i64) -} - -/// Print the available orders sorted by timestamp. -fn print_order_and_timestamp(orders_with_ts: &[OrdersWithTimestamp], block_data: &BlockData) { - let mut order_by_ts = orders_with_ts.to_vec(); - order_by_ts.sort_by_key(|owt| owt.timestamp_ms); - for owt in order_by_ts { - let id = owt.order.id(); - println!( - "{:>74} ts: {}", - id.to_string(), - timestamp_ms_to_slot_time( - owt.timestamp_ms, - timestamp_as_u64(&block_data.onchain_block) - ) - ); - for (tx, optional) in owt.order.list_txs() { - println!(" {:?} {:?}", tx.hash(), optional); - println!( - " from: {:?} to: {:?} nonce: {}", - tx.signer(), - tx.to(), - tx.nonce() - ) - } - } -} - -/// Print information about transactions included on-chain but which are missing in our available orders. -fn show_missing_txs(block_data: &BlockData) { - let missing_txs = block_data.search_missing_txs_on_available_orders(); - if !missing_txs.is_empty() { - println!( - "{} of txs by hashes missing on available orders", - missing_txs.len() - ); - for missing_tx in missing_txs.iter() { - println!("Tx: {:?}", missing_tx); - } - } - let missing_nonce_txs = block_data.search_missing_account_nonce_on_available_orders(); - if !missing_nonce_txs.is_empty() { - println!( - "\n{} of txs by nonce pairs missing on available orders", - missing_nonce_txs.len() - ); - for missing_nonce_tx in missing_nonce_txs.iter() { - println!( - "Tx: {:?}, Account: {:?}, Nonce: {}", - missing_nonce_tx.0, missing_nonce_tx.1.account, missing_nonce_tx.1.nonce, - ); - } - } -} - -/// Print information about simulated orders. -fn print_simulated_orders( - sim_orders: &[SimulatedOrder], - order_and_timestamp: &HashMap, - block_data: &BlockData, -) { - println!("Simulated orders: ({} total)", sim_orders.len()); - let mut sorted_orders = sim_orders.to_owned(); - sorted_orders.sort_by_key(|order| order.sim_value.coinbase_profit); - sorted_orders.reverse(); - for order in sorted_orders { - let order_timestamp = order_and_timestamp - .get(&order.order.id()) - .copied() - .unwrap_or_default(); - - let slot_time_ms = - timestamp_ms_to_slot_time(order_timestamp, timestamp_as_u64(&block_data.onchain_block)); - - println!( - "{:>74} slot_time_ms: {:>8}, gas: {:>8} profit: {}", - order.order.id().to_string(), - slot_time_ms, - order.sim_value.gas_used, - format_ether(order.sim_value.coinbase_profit), - ); - } - println!(); -} - -fn print_onchain_block_data( - tx_sim_results: Vec, - orders: &[Order], - block_data: &BlockData, -) { - let mut executed_orders = Vec::new(); - - let txs_to_idx: HashMap<_, _> = tx_sim_results - .iter() - .enumerate() - .map(|(idx, tx)| (tx.hash(), idx)) - .collect(); - - println!("Onchain block txs:"); - for (idx, tx) in tx_sim_results.into_iter().enumerate() { - println!( - "{:>4}, {:>74} revert: {:>5} profit: {}", - idx, - tx.hash(), - !tx.receipt.success, - format_ether(tx.coinbase_profit) - ); - if !tx.conflicting_txs.is_empty() { - println!(" conflicts: "); - } - for (tx, slots) in &tx.conflicting_txs { - for slot in slots { - println!( - " {:>4} address: {:?>24}, key: {:?}", - txs_to_idx.get(tx).unwrap(), - slot.address, - slot.key - ); - } - } - executed_orders.push(ExecutedBlockTx::new( - tx.hash(), - tx.coinbase_profit, - tx.receipt.success, - )) - } - - // restored orders - let mut simplified_orders = Vec::new(); - for order in orders { - if block_data - .built_block_data - .as_ref() - .map(|bd| bd.included_orders.contains(&order.id())) - .unwrap_or(true) - { - simplified_orders.push(SimplifiedOrder::new_from_order(order)); - } - } - let restored_orders = restore_landed_orders(executed_orders, simplified_orders); - - for (id, order) in &restored_orders { - println!( - "{:>74} total_profit: {}, unique_profit: {}, error: {:?}", - id, - format_ether(order.total_coinbase_profit), - format_ether(order.unique_coinbase_profit), - order.error - ); - } - - if let Some(built_block) = &block_data.built_block_data { - println!(); - println!("Included orders:"); - for included_order in &built_block.included_orders { - if let Some(order) = restored_orders.get(included_order) { - println!( - "{:>74} total_profit: {}, unique_profit: {}, error: {:?}", - order.order, - format_ether(order.total_coinbase_profit), - format_ether(order.unique_coinbase_profit), - order.error - ); - for (other, tx) in &order.overlapping_txs { - println!(" overlap with: {:>74} tx {:?}", other, tx); - } - } else { - println!("{:>74} included order not found: ", included_order); - } - } - } -} diff --git a/crates/rbuilder/src/backtest/backtest_build_range.rs b/crates/rbuilder/src/backtest/backtest_build_range.rs index da96d7f3..5c9dec4e 100644 --- a/crates/rbuilder/src/backtest/backtest_build_range.rs +++ b/crates/rbuilder/src/backtest/backtest_build_range.rs @@ -138,6 +138,7 @@ where let mut read_blocks = spawn_block_fetcher( historical_data_storage, blocks.clone(), + cli.build_block_lag_ms as i64, cli.ignored_signers, cancel_token.clone(), ); @@ -157,7 +158,6 @@ where .map(|block_data| { ( block_data, - cli.build_block_lag_ms, provider_factory.clone(), chain_spec.clone(), builders_names.clone(), @@ -168,13 +168,12 @@ where let output = input .into_par_iter() .filter_map( - |(block_data, lag, provider_factory, chain_spec, builders_names, blocklist)| { + |(block_data, provider_factory, chain_spec, builders_names, blocklist)| { let block_number = block_data.block_number; match backtest_simulate_block( block_data, provider_factory, chain_spec, - lag as i64, builders_names, &config, blocklist, @@ -400,6 +399,7 @@ impl CSVResultWriter { fn spawn_block_fetcher( mut historical_data_storage: HistoricalDataStorage, blocks: Vec, + build_block_lag_ms: i64, ignored_signers: Vec
, cancellation_token: CancellationToken, ) -> mpsc::Receiver> { @@ -419,6 +419,7 @@ fn spawn_block_fetcher( }; for block in &mut blocks { block.filter_out_ignored_signers(&ignored_signers); + block.filter_late_orders(build_block_lag_ms); } match sender.send(blocks).await { Ok(_) => {} diff --git a/crates/rbuilder/src/backtest/build_block/backtest_build_block.rs b/crates/rbuilder/src/backtest/build_block/backtest_build_block.rs new file mode 100644 index 00000000..86d65a44 --- /dev/null +++ b/crates/rbuilder/src/backtest/build_block/backtest_build_block.rs @@ -0,0 +1,241 @@ +//! Backtest app to build a single block in a similar way as we do in live. +//! It gets the orders from a HistoricalDataStorage, simulates the orders and then runs the building algorithms. +//! It outputs the best algorithm (most profit) so we can check for improvements in our [crate::building::builders::BlockBuildingAlgorithm]s +//! BlockBuildingAlgorithm are defined on the config file but selected on the command line via "--builders" +//! Sample call: +//! backtest-build-block --config /home/happy_programmer/config.toml --builders mgp-ordering --builders mp-ordering 19380913 --show-orders --show-missing + +use ahash::HashMap; +use alloy_primitives::utils::format_ether; +use reth_db::Database; +use reth_provider::{BlockReader, DatabaseProviderFactory, StateProviderFactory}; + +use crate::{ + backtest::{ + execute::{backtest_prepare_ctx_for_block_from_building_context, BacktestBlockInput}, + OrdersWithTimestamp, + }, + building::{builders::BacktestSimulateBlockInput, BlockBuildingContext}, + live_builder::cli::LiveBuilderConfig, + primitives::{Order, OrderId, SimulatedOrder}, +}; +use clap::Parser; +use std::path::PathBuf; + +#[derive(Parser, Debug)] +pub struct BuildBlockCfg { + #[clap(long, help = "Config file path", env = "RBUILDER_CONFIG")] + pub config: PathBuf, + #[clap(long, help = "Show all available orders")] + pub show_orders: bool, + #[clap(long, help = "Show order data and top of block simulation results")] + pub show_sim: bool, + #[clap(long, help = "don't build block")] + pub no_block_building: bool, + #[clap( + long, + help = "builders to build block with (see config builders)", + default_value = "mp-ordering" + )] + pub builders: Vec, +} + +/// Provides all the orders needed to simulate the construction of a block. +/// It also provides the needed context to execute those orders. +pub trait OrdersSource +where + ConfigType: LiveBuilderConfig, + DBType: Database + Clone + 'static, + ProviderType: DatabaseProviderFactory + + StateProviderFactory + + Clone + + 'static, +{ + fn config(&self) -> &ConfigType; + /// Orders available to build blocks with their time of arrival. + fn available_orders(&self) -> Vec; + /// Start of the slot for the block. + /// Usually all the orders will arrive before block_time_as_unix_ms + 4secs (max get_header time from validator to relays). + fn block_time_as_unix_ms(&self) -> u64; + + /// ugly: it takes BaseConfig but not all implementations need it..... + fn create_provider_factory(&self) -> eyre::Result; + + fn create_block_building_context(&self) -> eyre::Result; + + /// Prints any stats specific to the particular OrdersSource implementation (eg: parameters, block simulation) + fn print_custom_stats(&self, provider: ProviderType) -> eyre::Result<()>; +} + +pub async fn run_backtest_build_block( + build_block_cfg: BuildBlockCfg, + orders_source: OrdersSourceType, +) -> eyre::Result<()> +where + ConfigType: LiveBuilderConfig, + DBType: Database + Clone + 'static, + ProviderType: DatabaseProviderFactory + + StateProviderFactory + + Clone + + 'static, + OrdersSourceType: OrdersSource, +{ + let config = orders_source.config(); + config.base_config().setup_tracing_subscriber()?; + + let available_orders = orders_source.available_orders(); + println!("Available orders: {}", available_orders.len()); + + if build_block_cfg.show_orders { + print_order_and_timestamp(&available_orders, orders_source.block_time_as_unix_ms()); + } + + let provider_factory = orders_source.create_provider_factory()?; + + orders_source.print_custom_stats(provider_factory.clone())?; + + let ctx = orders_source.create_block_building_context()?; + let BacktestBlockInput { + ctx, sim_orders, .. + } = backtest_prepare_ctx_for_block_from_building_context( + ctx, + available_orders.clone(), + provider_factory.clone(), + &config.base_config().sbundle_mergeable_signers(), + )?; + + if build_block_cfg.show_sim { + let order_and_timestamp: HashMap = available_orders + .iter() + .map(|order| (order.order.id(), order.timestamp_ms)) + .collect(); + print_simulated_orders( + &sim_orders, + &order_and_timestamp, + orders_source.block_time_as_unix_ms(), + ); + } + + if !build_block_cfg.no_block_building { + let winning_builder = build_block_cfg + .builders + .iter() + .filter_map(|builder_name: &String| { + let input = BacktestSimulateBlockInput { + ctx: ctx.clone(), + builder_name: builder_name.clone(), + sim_orders: &sim_orders, + provider: provider_factory.clone(), + cached_reads: None, + }; + let build_res = config.build_backtest_block(builder_name, input); + if let Err(err) = &build_res { + println!("Error building block: {:?}", err); + return None; + } + let (block, _) = build_res.ok()?; + println!( + "Built block {} with builder: {:?}", + ctx.block(), + builder_name + ); + println!("Builder profit: {}", format_ether(block.trace.bid_value)); + println!( + "Number of used orders: {}", + block.trace.included_orders.len() + ); + + println!("Used orders:"); + for order_result in &block.trace.included_orders { + println!( + "{:>74} gas: {:>8} profit: {}", + order_result.order.id().to_string(), + order_result.gas_used, + format_ether(order_result.coinbase_profit), + ); + if let Order::Bundle(_) | Order::ShareBundle(_) = order_result.order { + for tx in &order_result.txs { + println!(" ↳ {:?}", tx.hash()); + } + + for (to, value) in &order_result.paid_kickbacks { + println!( + " - kickback to: {:?} value: {}", + to, + format_ether(*value) + ); + } + } + } + Some((builder_name.clone(), block.trace.bid_value)) + }) + .max_by_key(|(_, value)| *value); + + if let Some((builder_name, value)) = winning_builder { + println!( + "Winning builder: {} with profit: {}", + builder_name, + format_ether(value) + ); + } + } + + Ok(()) +} + +/// Convert a timestamp in milliseconds to the slot time relative to the given block timestamp. +fn timestamp_ms_to_slot_time(timestamp_ms: u64, block_timestamp: u64) -> i64 { + (block_timestamp * 1000) as i64 - (timestamp_ms as i64) +} + +/// Print the available orders sorted by timestamp. +fn print_order_and_timestamp(orders_with_ts: &[OrdersWithTimestamp], block_time_as_unix_ms: u64) { + let mut order_by_ts = orders_with_ts.to_vec(); + order_by_ts.sort_by_key(|owt| owt.timestamp_ms); + for owt in order_by_ts { + let id = owt.order.id(); + println!( + "{:>74} ts: {}", + id.to_string(), + timestamp_ms_to_slot_time(owt.timestamp_ms, block_time_as_unix_ms) + ); + for (tx, optional) in owt.order.list_txs() { + println!(" {:?} {:?}", tx.hash(), optional); + println!( + " from: {:?} to: {:?} nonce: {}", + tx.signer(), + tx.to(), + tx.nonce() + ) + } + } +} + +/// Print information about simulated orders. +fn print_simulated_orders( + sim_orders: &[SimulatedOrder], + order_and_timestamp: &HashMap, + block_time_as_unix_ms: u64, +) { + println!("Simulated orders: ({} total)", sim_orders.len()); + let mut sorted_orders = sim_orders.to_owned(); + sorted_orders.sort_by_key(|order| order.sim_value.coinbase_profit); + sorted_orders.reverse(); + for order in sorted_orders { + let order_timestamp = order_and_timestamp + .get(&order.order.id()) + .copied() + .unwrap_or_default(); + + let slot_time_ms = timestamp_ms_to_slot_time(order_timestamp, block_time_as_unix_ms); + + println!( + "{:>74} slot_time_ms: {:>8}, gas: {:>8} profit: {}", + order.order.id().to_string(), + slot_time_ms, + order.sim_value.gas_used, + format_ether(order.sim_value.coinbase_profit), + ); + } + println!(); +} diff --git a/crates/rbuilder/src/backtest/build_block/landed_block_from_db.rs b/crates/rbuilder/src/backtest/build_block/landed_block_from_db.rs new file mode 100644 index 00000000..6adbff14 --- /dev/null +++ b/crates/rbuilder/src/backtest/build_block/landed_block_from_db.rs @@ -0,0 +1,296 @@ +//! +//! Backtest app to build a single block in a similar way as we do in live. +//! It gets the orders from a HistoricalDataStorage, simulates the orders and then runs the building algorithms. +//! It outputs the best algorithm (most profit) so we can check for improvements in our [crate::building::builders::BlockBuildingAlgorithm]s +//! BlockBuildingAlgorithm are defined on the config file but selected on the command line via "--builders" +//! Sample call: +//! backtest-build-block --config /home/happy_programmer/config.toml --builders mgp-ordering --builders mp-ordering 19380913 --show-orders --show-missing + +use ahash::HashMap; +use alloy_primitives::utils::format_ether; +use reth_db::DatabaseEnv; +use reth_node_api::NodeTypesWithDBAdapter; +use reth_node_ethereum::EthereumNode; + +use crate::{ + backtest::{ + restore_landed_orders::{ + restore_landed_orders, sim_historical_block, ExecutedBlockTx, ExecutedTxs, + SimplifiedOrder, + }, + BlockData, HistoricalDataStorage, OrdersWithTimestamp, + }, + building::BlockBuildingContext, + live_builder::{base_config::load_config_toml_and_env, cli::LiveBuilderConfig}, + utils::{timestamp_as_u64, ProviderFactoryReopener}, +}; +use clap::Parser; +use std::{path::PathBuf, sync::Arc}; + +use super::backtest_build_block::{run_backtest_build_block, BuildBlockCfg, OrdersSource}; + +#[derive(Parser, Debug)] +struct ExtraCfg { + #[clap(long, help = "Show landed block txs values")] + sim_landed_block: bool, + #[clap(long, help = "Show missing block txs")] + show_missing: bool, + #[clap(long, help = "use only this orders")] + only_order_ids: Vec, + #[clap( + long, + help = "build block lag (ms)", + default_value = "0", + allow_hyphen_values = true + )] + block_building_time_ms: i64, + #[clap(help = "Block Number")] + block: u64, +} + +#[derive(Parser, Debug)] +struct Cli { + #[command(flatten)] + pub build_block_cfg: BuildBlockCfg, + #[command(flatten)] + pub extra_cfg: ExtraCfg, +} + +/// OrdersSource that gets all the bundles from flashbot's infra. +struct LandedBlockFromDBOrdersSource { + block_data: BlockData, + sim_landed_block: bool, + config: ConfigType, +} + +impl LandedBlockFromDBOrdersSource { + async fn new(extra_cfg: ExtraCfg, config: ConfigType) -> eyre::Result { + let block_data = read_block_data( + &config.base_config().backtest_fetch_output_file, + extra_cfg.block, + extra_cfg.only_order_ids, + extra_cfg.block_building_time_ms, + extra_cfg.show_missing, + ) + .await?; + Ok(Self { + block_data, + sim_landed_block: extra_cfg.sim_landed_block, + config, + }) + } +} + +impl + OrdersSource< + ConfigType, + Arc, + ProviderFactoryReopener>>, + > for LandedBlockFromDBOrdersSource +{ + fn available_orders(&self) -> Vec { + self.block_data.available_orders.clone() + } + + fn block_time_as_unix_ms(&self) -> u64 { + timestamp_as_u64(&self.block_data.onchain_block) + } + + fn create_provider_factory( + &self, + ) -> eyre::Result>>> + { + self.config.base_config().create_provider_factory() + } + + fn create_block_building_context(&self) -> eyre::Result { + let signer = self.config.base_config().coinbase_signer()?; + Ok(BlockBuildingContext::from_onchain_block( + self.block_data.onchain_block.clone(), + self.config.base_config().chain_spec()?, + None, + self.config.base_config().blocklist()?, + signer.address, + self.block_data.winning_bid_trace.proposer_fee_recipient, + Some(signer), + )) + } + + fn print_custom_stats( + &self, + provider: ProviderFactoryReopener>>, + ) -> eyre::Result<()> { + if self.sim_landed_block { + let tx_sim_results = sim_historical_block( + provider, + self.config.base_config().chain_spec()?, + self.block_data.onchain_block.clone(), + )?; + print_onchain_block_data(tx_sim_results, &self.block_data); + } + Ok(()) + } + + fn config(&self) -> &ConfigType { + &self.config + } +} + +/// Reads from HistoricalDataStorage the BlockData for block. +/// only_order_ids: if not empty returns only the given order ids. +/// block_building_time_ms: If not 0, time it took to build the block. It allows us to filter out orders that arrived after we started building the block (filter_late_orders). +/// show_missing: show on-chain orders that weren't available to us at building time. +async fn read_block_data( + backtest_fetch_output_file: &PathBuf, + block: u64, + only_order_ids: Vec, + block_building_time_ms: i64, + show_missing: bool, +) -> eyre::Result { + let mut historical_data_storage = + HistoricalDataStorage::new_from_path(backtest_fetch_output_file).await?; + + let mut block_data = historical_data_storage.read_block_data(block).await?; + + if !only_order_ids.is_empty() { + block_data.filter_orders_by_ids(&only_order_ids); + } + + block_data.filter_late_orders(block_building_time_ms); + + if show_missing { + show_missing_txs(&block_data); + } + + println!( + "Block: {} {:?}", + block_data.block_number, block_data.onchain_block.header.hash + ); + println!( + "bid value: {}", + format_ether(block_data.winning_bid_trace.value) + ); + println!( + "builder pubkey: {:?}", + block_data.winning_bid_trace.builder_pubkey + ); + Ok(block_data) +} + +fn print_onchain_block_data(tx_sim_results: Vec, block_data: &BlockData) { + let mut executed_orders = Vec::new(); + + let txs_to_idx: HashMap<_, _> = tx_sim_results + .iter() + .enumerate() + .map(|(idx, tx)| (tx.hash(), idx)) + .collect(); + + println!("Onchain block txs:"); + for (idx, tx) in tx_sim_results.into_iter().enumerate() { + println!( + "{:>4}, {:>74} revert: {:>5} profit: {}", + idx, + tx.hash(), + !tx.receipt.success, + format_ether(tx.coinbase_profit) + ); + if !tx.conflicting_txs.is_empty() { + println!(" conflicts: "); + } + for (tx, slots) in &tx.conflicting_txs { + for slot in slots { + println!( + " {:>4} address: {:?>24}, key: {:?}", + txs_to_idx.get(tx).unwrap(), + slot.address, + slot.key + ); + } + } + executed_orders.push(ExecutedBlockTx::new( + tx.hash(), + tx.coinbase_profit, + tx.receipt.success, + )) + } + + // restored orders + let mut simplified_orders = Vec::new(); + for order in block_data.available_orders.iter().map(|os| &os.order) { + if block_data + .built_block_data + .as_ref() + .map(|bd| bd.included_orders.contains(&order.id())) + .unwrap_or(true) + { + simplified_orders.push(SimplifiedOrder::new_from_order(order)); + } + } + let restored_orders = restore_landed_orders(executed_orders, simplified_orders); + + for (id, order) in &restored_orders { + println!( + "{:>74} total_profit: {}, unique_profit: {}, error: {:?}", + id, + format_ether(order.total_coinbase_profit), + format_ether(order.unique_coinbase_profit), + order.error + ); + } + + if let Some(built_block) = &block_data.built_block_data { + println!(); + println!("Included orders:"); + for included_order in &built_block.included_orders { + if let Some(order) = restored_orders.get(included_order) { + println!( + "{:>74} total_profit: {}, unique_profit: {}, error: {:?}", + order.order, + format_ether(order.total_coinbase_profit), + format_ether(order.unique_coinbase_profit), + order.error + ); + for (other, tx) in &order.overlapping_txs { + println!(" overlap with: {:>74} tx {:?}", other, tx); + } + } else { + println!("{:>74} included order not found: ", included_order); + } + } + } +} + +/// Print information about transactions included on-chain but which are missing in our available orders. +fn show_missing_txs(block_data: &BlockData) { + let missing_txs = block_data.search_missing_txs_on_available_orders(); + if !missing_txs.is_empty() { + println!( + "{} of txs by hashes missing on available orders", + missing_txs.len() + ); + for missing_tx in missing_txs.iter() { + println!("Tx: {:?}", missing_tx); + } + } + let missing_nonce_txs = block_data.search_missing_account_nonce_on_available_orders(); + if !missing_nonce_txs.is_empty() { + println!( + "\n{} of txs by nonce pairs missing on available orders", + missing_nonce_txs.len() + ); + for missing_nonce_tx in missing_nonce_txs.iter() { + println!( + "Tx: {:?}, Account: {:?}, Nonce: {}", + missing_nonce_tx.0, missing_nonce_tx.1.account, missing_nonce_tx.1.nonce, + ); + } + } +} + +pub async fn run_backtest() -> eyre::Result<()> { + let cli = Cli::parse(); + let config: ConfigType = load_config_toml_and_env(cli.build_block_cfg.config.clone())?; + let order_source = LandedBlockFromDBOrdersSource::new(cli.extra_cfg, config).await?; + run_backtest_build_block(cli.build_block_cfg, order_source).await +} diff --git a/crates/rbuilder/src/backtest/build_block/mod.rs b/crates/rbuilder/src/backtest/build_block/mod.rs new file mode 100644 index 00000000..ecaee07b --- /dev/null +++ b/crates/rbuilder/src/backtest/build_block/mod.rs @@ -0,0 +1,3 @@ +mod backtest_build_block; +pub mod landed_block_from_db; +pub mod synthetic_orders; diff --git a/crates/rbuilder/src/backtest/build_block/synthetic_orders.rs b/crates/rbuilder/src/backtest/build_block/synthetic_orders.rs new file mode 100644 index 00000000..b976cd26 --- /dev/null +++ b/crates/rbuilder/src/backtest/build_block/synthetic_orders.rs @@ -0,0 +1,156 @@ +use std::sync::Arc; + +use clap::{command, Parser}; +use reth_db::{test_utils::TempDatabase, DatabaseEnv}; +use reth_provider::{providers::BlockchainProvider2, test_utils::MockNodeTypesWithDB}; +use revm_primitives::B256; +use uuid::Uuid; + +use crate::{ + backtest::OrdersWithTimestamp, + building::{ + testing::test_chain_state::{BlockArgs, NamedAddr, TestChainState, TxArgs}, + BlockBuildingContext, + }, + live_builder::{base_config::load_config_toml_and_env, cli::LiveBuilderConfig}, + primitives::{Bundle, MempoolTx, Metadata, Order, TransactionSignedEcRecoveredWithBlobs}, +}; + +use super::backtest_build_block::{run_backtest_build_block, BuildBlockCfg, OrdersSource}; + +#[derive(Parser, Debug)] +struct ExtraCfg { + #[clap(long, help = "Tx count")] + tx_count: u64, + #[clap(long, help = "Bundle count")] + bundle_count: u64, +} + +#[derive(Parser, Debug)] +struct Cli { + #[command(flatten)] + pub build_block_cfg: BuildBlockCfg, + #[command(flatten)] + pub extra_cfg: ExtraCfg, +} + +/// OrdersSource using a fake chain state and some synthetic orders. +/// Creates 2 types of orders: +/// - Tx: a tx from user 1 paying a small value to coinbase. +/// - Bundle: a tx from user 2 paying a small value to coinbase followed by another tx from user 3 paying a large value to coinbase. +/// +/// All the nonces are properly set so all the txs and bundles are executable (in the correct order). +struct SyntheticOrdersSource { + test_chain_state: TestChainState, + orders: Vec, + config: ConfigType, +} + +const LOW_TIP: u64 = 1_000_000; +const HIGH_TIP: u64 = 1_000_000_000; + +/// Creates a TransactionSignedEcRecoveredWithBlobs from user paying tip to coinbase. +fn create_tip_tx( + test_chain_state: &TestChainState, + user: usize, + nonce: u64, + tip: u64, +) -> TransactionSignedEcRecoveredWithBlobs { + let tx_args = TxArgs::new_send_to_coinbase(NamedAddr::User(user), nonce, tip); + let tx = test_chain_state.sign_tx(tx_args).unwrap(); + TransactionSignedEcRecoveredWithBlobs::new_no_blobs(tx).unwrap() +} + +impl SyntheticOrdersSource { + fn new(extra_cfg: ExtraCfg, config: ConfigType) -> eyre::Result { + let block_number = 1; + let test_chain_state = TestChainState::new(BlockArgs::default().number(block_number))?; + let mut orders = Vec::new(); + for i in 0..extra_cfg.tx_count { + let order = Order::Tx(MempoolTx::new(create_tip_tx( + &test_chain_state, + 1, + i, + LOW_TIP, + ))); + orders.push(OrdersWithTimestamp { + timestamp_ms: 0, + order, + }); + } + + for i in 0..extra_cfg.bundle_count { + let low_tip_tx = create_tip_tx(&test_chain_state, 2, i, LOW_TIP); + let high_tip_tx = create_tip_tx(&test_chain_state, 3, i, HIGH_TIP); + let mut bundle = Bundle { + block: block_number, + min_timestamp: None, + max_timestamp: None, + txs: vec![low_tip_tx, high_tip_tx], + reverting_tx_hashes: Default::default(), + hash: B256::ZERO, + uuid: Uuid::nil(), + replacement_data: None, + signer: None, + metadata: Metadata { + received_at_timestamp: time::OffsetDateTime::from_unix_timestamp(0).unwrap(), + }, + }; + bundle.hash_slow(); + orders.push(OrdersWithTimestamp { + timestamp_ms: 0, + order: Order::Bundle(bundle), + }); + } + + Ok(Self { + orders, + test_chain_state, + config, + }) + } +} + +impl + OrdersSource< + ConfigType, + Arc>, + BlockchainProvider2, + > for SyntheticOrdersSource +{ + fn available_orders(&self) -> Vec { + self.orders.clone() + } + + fn block_time_as_unix_ms(&self) -> u64 { + 0 + } + + fn create_provider_factory(&self) -> eyre::Result> { + Ok(BlockchainProvider2::new( + self.test_chain_state.provider_factory().clone(), + )?) + } + + fn create_block_building_context(&self) -> eyre::Result { + Ok(self.test_chain_state.block_building_context().clone()) + } + + fn print_custom_stats( + &self, + _provider: BlockchainProvider2, + ) -> eyre::Result<()> { + Ok(()) + } + + fn config(&self) -> &ConfigType { + &self.config + } +} + +pub async fn run_backtest() -> eyre::Result<()> { + let cli = Cli::parse(); + let config: ConfigType = load_config_toml_and_env(cli.build_block_cfg.config.clone())?; + let order_source = SyntheticOrdersSource::new(cli.extra_cfg, config)?; + run_backtest_build_block(cli.build_block_cfg, order_source).await +} diff --git a/crates/rbuilder/src/backtest/execute.rs b/crates/rbuilder/src/backtest/execute.rs index 4c392cdb..d69bc848 100644 --- a/crates/rbuilder/src/backtest/execute.rs +++ b/crates/rbuilder/src/backtest/execute.rs @@ -17,6 +17,8 @@ use reth_chainspec::ChainSpec; use serde::{Deserialize, Serialize}; use std::{cell::RefCell, rc::Rc, sync::Arc}; +use super::OrdersWithTimestamp; + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct BacktestBuilderOutput { pub orders_included: usize, @@ -59,7 +61,6 @@ pub fn backtest_prepare_ctx_for_block

( block_data: BlockData, provider: P, chain_spec: Arc, - build_block_lag_ms: i64, blocklist: HashSet

, sbundle_mergeabe_signers: &[Address], builder_signer: Signer, @@ -67,18 +68,6 @@ pub fn backtest_prepare_ctx_for_block

( where P: StateProviderFactory + Clone + 'static, { - let orders = block_data - .available_orders - .iter() - .filter_map(|order| { - if order.timestamp_ms as i64 + build_block_lag_ms - >= block_data.winning_bid_trace.timestamp_ms as i64 - { - return None; - } - Some(order.order.clone()) - }) - .collect::>(); let ctx = BlockBuildingContext::from_onchain_block( block_data.onchain_block, chain_spec.clone(), @@ -89,8 +78,30 @@ where Some(builder_signer), Arc::from(provider.root_hasher(block_data.winning_bid_trace.parent_hash)), ); + backtest_prepare_ctx_for_block_from_building_context( + ctx, + block_data.available_orders, + provider, + sbundle_mergeabe_signers, + ) +} + +pub fn backtest_prepare_ctx_for_block_from_building_context

( + ctx: BlockBuildingContext, + available_orders: Vec, + provider: P, + sbundle_mergeabe_signers: &[Address], +) -> eyre::Result +where + P: StateProviderFactory + Clone + 'static, +{ + let orders = available_orders + .iter() + .map(|order| order.order.clone()) + .collect::>(); + let (sim_orders, sim_errors) = - simulate_all_orders_with_sim_tree(provider.clone(), &ctx, &orders, false)?; + simulate_all_orders_with_sim_tree(provider, &ctx, &orders, false)?; // Apply bundle merging as in live building. let order_store = Rc::new(RefCell::new(SimulatedOrderStore::new())); @@ -111,7 +122,6 @@ pub fn backtest_simulate_block( block_data: BlockData, provider: P, chain_spec: Arc, - build_block_lag_ms: i64, builders_names: Vec, config: &ConfigType, blocklist: HashSet

, @@ -129,7 +139,6 @@ where block_data.clone(), provider.clone(), chain_spec.clone(), - build_block_lag_ms, blocklist, sbundle_mergeabe_signers, config.base_config().coinbase_signer()?, diff --git a/crates/rbuilder/src/backtest/mod.rs b/crates/rbuilder/src/backtest/mod.rs index 7097da50..026e8c42 100644 --- a/crates/rbuilder/src/backtest/mod.rs +++ b/crates/rbuilder/src/backtest/mod.rs @@ -1,14 +1,13 @@ -mod backtest_build_block; mod backtest_build_range; pub mod execute; pub mod fetch; +pub mod build_block; pub mod redistribute; pub mod restore_landed_orders; mod results_store; mod store; -pub use backtest_build_block::run_backtest_build_block; pub use backtest_build_range::run_backtest_build_range; use std::collections::HashSet; diff --git a/crates/rbuilder/src/backtest/redistribute/mod.rs b/crates/rbuilder/src/backtest/redistribute/mod.rs index b6a1748c..ed4f110c 100644 --- a/crates/rbuilder/src/backtest/redistribute/mod.rs +++ b/crates/rbuilder/src/backtest/redistribute/mod.rs @@ -971,16 +971,10 @@ where let base_config = config.base_config(); - // we set built_block_lag_ms to 0 here because we already prefiltered all the orders - // in built_block_data, so we essentially just disable filtering in the `backtest_simulate_block` - // but we still filter by the relay timestamp - let built_block_lag_ms = 0; - let result = backtest_simulate_block( block_data_with_excluded, provider.clone(), base_config.chain_spec()?, - built_block_lag_ms, base_config.backtest_builders.clone(), config, base_config.blocklist()?, diff --git a/crates/rbuilder/src/bin/backtest-build-block.rs b/crates/rbuilder/src/bin/backtest-build-block.rs index 56029b92..1c13fe2b 100644 --- a/crates/rbuilder/src/bin/backtest-build-block.rs +++ b/crates/rbuilder/src/bin/backtest-build-block.rs @@ -1,8 +1,10 @@ //! Instantiation of run_backtest_build_block on our sample configuration. -use rbuilder::{backtest::run_backtest_build_block, live_builder::config::Config}; +use rbuilder::{ + backtest::build_block::landed_block_from_db::run_backtest, live_builder::config::Config, +}; #[tokio::main] async fn main() -> eyre::Result<()> { - run_backtest_build_block::().await + run_backtest::().await } diff --git a/crates/rbuilder/src/bin/backtest-build-synthetic-block.rs b/crates/rbuilder/src/bin/backtest-build-synthetic-block.rs new file mode 100644 index 00000000..c6809ea7 --- /dev/null +++ b/crates/rbuilder/src/bin/backtest-build-synthetic-block.rs @@ -0,0 +1,10 @@ +//! Instantiation of run_backtest_build_block on our sample configuration. + +use rbuilder::{ + backtest::build_block::synthetic_orders::run_backtest, live_builder::config::Config, +}; + +#[tokio::main] +async fn main() -> eyre::Result<()> { + run_backtest::().await +} diff --git a/crates/rbuilder/src/building/testing/test_chain_state.rs b/crates/rbuilder/src/building/testing/test_chain_state.rs index f58e7bb0..2a5156e4 100644 --- a/crates/rbuilder/src/building/testing/test_chain_state.rs +++ b/crates/rbuilder/src/building/testing/test_chain_state.rs @@ -72,8 +72,35 @@ pub struct TestChainState { provider_factory: ProviderFactory, block_building_context: BlockBuildingContext, } +pub struct ContractData { + address: Address, + code: Bytes, + code_hash: B256, +} + +impl ContractData { + pub fn new(code: &Bytes, address: Address) -> Self { + let code_hash = keccak256(code); + ContractData { + address, + code: code.clone(), + code_hash, + } + } +} + impl TestChainState { pub fn new(block_args: BlockArgs) -> eyre::Result { + Self::new_with_balances_and_contracts(block_args, Default::default(), Default::default()) + } + + /// balances_to_increase this addresses start with that initial balances. + /// extra_contracts are deploy along with the default mev_test. + pub fn new_with_balances_and_contracts( + block_args: BlockArgs, + balances_to_increase: Vec<(Address, u128)>, + extra_contracts: Vec, + ) -> eyre::Result { let blocklisted_address = Signer::random(); let builder = Signer::random(); let fee_recipient = Signer::random(); @@ -88,7 +115,10 @@ impl TestChainState { let mev_test_address = Address::random(); let dummy_test_address = Address::random(); let test_contracts = TestContracts::load(); - let (mev_test_hash, mev_test_code) = test_contracts.mev_test(); + + let mut contracts = extra_contracts; + contracts.push(test_contracts.mev_test(mev_test_address)); + let genesis_header = chain_spec.sealed_genesis_header(); let provider_factory = create_test_provider_factory(); { @@ -122,22 +152,37 @@ impl TestChainState { }, )?; } + // Failed to map user_addresses and chain it with balances_to_increase :( + for (address, balance) in balances_to_increase { + cursor.upsert( + address, + Account { + nonce: 0, + balance: U256::from(balance), + bytecode_hash: None, + }, + )?; + } - cursor.upsert( - mev_test_address, - Account { - nonce: 0, - balance: U256::ZERO, - bytecode_hash: Some(mev_test_hash), - }, - )?; + for contract in &contracts { + cursor.upsert( + contract.address, + Account { + nonce: 0, + balance: U256::ZERO, + bytecode_hash: Some(contract.code_hash), + }, + )?; + } } { let mut cursor = provider .tx_ref() .cursor_write::() .unwrap(); - cursor.upsert(mev_test_hash, Bytecode::new_raw(mev_test_code))?; + for contract in &contracts { + cursor.upsert(contract.code_hash, Bytecode::new_raw(contract.code.clone()))?; + } } provider.commit()?; } @@ -472,6 +517,7 @@ impl TxArgs { } } +/// This contract was generated from mev-test-contract/src/MevTest.sol static TEST_CONTRACTS: &str = include_str!("./contracts.json"); #[derive(Debug, serde::Deserialize)] @@ -503,8 +549,7 @@ impl TestContracts { serde_json::from_str(TEST_CONTRACTS).expect("failed to load test contracts") } - fn mev_test(&self) -> (B256, Bytes) { - let hash = keccak256(&self.mev_test); - (hash, self.mev_test.clone()) + fn mev_test(&self, address: Address) -> ContractData { + ContractData::new(&self.mev_test, address) } } diff --git a/mev-test-contract/src/MevTest.sol b/mev-test-contract/src/MevTest.sol index 54a371e1..9b05603c 100644 --- a/mev-test-contract/src/MevTest.sol +++ b/mev-test-contract/src/MevTest.sol @@ -43,12 +43,12 @@ contract MevTest { revert(); } - /// Return sum of the contract's balance and addr's balanace, for testing evm inspector with selfbalance/balance opcode. + /// Return sum of the contract's balance and addr's balance, for testing evm inspector with selfbalance/balance opcode. function testReadBalance(address payable addr) public payable { address(this).balance + addr.balance; } - // Deploy a contract and let the contract self-destruct, for testing evm inspector on contract depoly and destruct. + // Deploy a contract and let the contract self-destruct, for testing evm inspector on contract deploy and destruct. function testEphemeralContractDestruct(address payable refund) public payable { EphemeralContractTest ephemeral_contract = new EphemeralContractTest(); ephemeral_contract.destruct{value: msg.value}(refund);