diff --git a/src/Nethermind/Nethermind.Benchmark.Runner/Program.cs b/src/Nethermind/Nethermind.Benchmark.Runner/Program.cs index 1b6a025ef45..a22f9c03bfc 100644 --- a/src/Nethermind/Nethermind.Benchmark.Runner/Program.cs +++ b/src/Nethermind/Nethermind.Benchmark.Runner/Program.cs @@ -11,6 +11,8 @@ using BenchmarkDotNet.Running; using System.Linq; using BenchmarkDotNet.Toolchains.InProcess.NoEmit; +using BenchmarkDotNet.Columns; +using Nethermind.Precompiles.Benchmark; namespace Nethermind.Benchmark.Runner { @@ -23,10 +25,10 @@ public DashboardConfig(params Job[] jobs) AddJob(job.WithToolchain(InProcessNoEmitToolchain.Instance)); } - AddColumnProvider(BenchmarkDotNet.Columns.DefaultColumnProviders.Descriptor); - AddColumnProvider(BenchmarkDotNet.Columns.DefaultColumnProviders.Statistics); - AddColumnProvider(BenchmarkDotNet.Columns.DefaultColumnProviders.Params); - AddColumnProvider(BenchmarkDotNet.Columns.DefaultColumnProviders.Metrics); + AddColumnProvider(DefaultColumnProviders.Descriptor); + AddColumnProvider(DefaultColumnProviders.Statistics); + AddColumnProvider(DefaultColumnProviders.Params); + AddColumnProvider(DefaultColumnProviders.Metrics); AddLogger(BenchmarkDotNet.Loggers.ConsoleLogger.Default); AddExporter(BenchmarkDotNet.Exporters.Json.JsonExporter.FullCompressed); AddDiagnoser(BenchmarkDotNet.Diagnosers.MemoryDiagnoser.Default); @@ -34,23 +36,28 @@ public DashboardConfig(params Job[] jobs) } } + public class PrecompileBenchmarkConfig : DashboardConfig + { + public PrecompileBenchmarkConfig() : base(Job.MediumRun.WithRuntime(CoreRuntime.Core90)) + { + AddColumnProvider(new GasColumnProvider()); + } + } + public static class Program { public static void Main(string[] args) { - List additionalJobAssemblies = new() - { - typeof(Nethermind.JsonRpc.Benchmark.EthModuleBenchmarks).Assembly, - typeof(Nethermind.Benchmarks.Core.Keccak256Benchmarks).Assembly, - typeof(Nethermind.Evm.Benchmark.EvmStackBenchmarks).Assembly, - typeof(Nethermind.Network.Benchmarks.DiscoveryBenchmarks).Assembly, - typeof(Nethermind.Precompiles.Benchmark.KeccakBenchmark).Assembly - }; + List additionalJobAssemblies = [ + typeof(JsonRpc.Benchmark.EthModuleBenchmarks).Assembly, + typeof(Benchmarks.Core.Keccak256Benchmarks).Assembly, + typeof(Evm.Benchmark.EvmStackBenchmarks).Assembly, + typeof(Network.Benchmarks.DiscoveryBenchmarks).Assembly, + ]; - List simpleJobAssemblies = new() - { - typeof(Nethermind.EthereumTests.Benchmark.EthereumTests).Assembly, - }; + List simpleJobAssemblies = [ + // typeof(EthereumTests.Benchmark.EthereumTests).Assembly, + ]; if (Debugger.IsAttached) { @@ -67,6 +74,8 @@ public static void Main(string[] args) { BenchmarkRunner.Run(assembly, new DashboardConfig(), args); } + + BenchmarkRunner.Run(typeof(KeccakBenchmark).Assembly, new PrecompileBenchmarkConfig(), args); } } } diff --git a/src/Nethermind/Nethermind.Precompiles.Benchmark/GasColumnProvider.cs b/src/Nethermind/Nethermind.Precompiles.Benchmark/GasColumnProvider.cs new file mode 100644 index 00000000000..ecfea37e041 --- /dev/null +++ b/src/Nethermind/Nethermind.Precompiles.Benchmark/GasColumnProvider.cs @@ -0,0 +1,116 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Collections.Generic; +using System.Linq; +using BenchmarkDotNet.Columns; +using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Running; +using Nethermind.Specs.Forks; +using BenchmarkDotNet.Mathematics; +using Perfolizer.Mathematics.Common; + +namespace Nethermind.Precompiles.Benchmark; + +public class GasColumnProvider : IColumnProvider +{ + private static readonly IColumn[] Columns = [ + new GasColumn(), + new GasThroughputColumn(), + new GasConfidenceIntervalColumn(true), // Lower bound + new GasConfidenceIntervalColumn(false) // Upper bound + ]; + + public IEnumerable GetColumns(Summary summary) => Columns; + + private abstract class BaseGasColumn : IColumn + { + public bool AlwaysShow => true; + public ColumnCategory Category => ColumnCategory.Custom; + public int PriorityInCategory => 0; + public bool IsNumeric => true; + public UnitType UnitType => UnitType.Size; + + public abstract string Id { get; } + public abstract string ColumnName { get; } + public abstract string Legend { get; } + + protected static (long? gas, Statistics? stats) GetBenchmarkData(Summary summary, BenchmarkCase benchmarkCase) + { + BenchmarkDotNet.Parameters.ParameterInstance? inputParam = benchmarkCase.Parameters.Items.FirstOrDefault(p => p.Name == "Input"); + var gas = ((PrecompileBenchmarkBase.Param)inputParam!.Value).Gas(Cancun.Instance); + Statistics? stats = summary.Reports.FirstOrDefault(r => r.BenchmarkCase == benchmarkCase)?.ResultStatistics; + return (gas, stats); + } + + protected static double CalculateMGasThroughput(long gas, double nanoseconds) + { + double opThroughput = 1_000_000_000.0 / nanoseconds; + return gas * opThroughput / 1_000_000.0; + } + + public abstract string GetValue(Summary summary, BenchmarkCase benchmarkCase); + + public string GetValue(Summary summary, BenchmarkCase benchmarkCase, SummaryStyle style) + => GetValue(summary, benchmarkCase); + + public bool IsDefault(Summary summary, BenchmarkCase benchmarkCase) => false; + + public bool IsAvailable(Summary summary) => true; + } + + private class GasColumn : BaseGasColumn + { + public override string Id => "Gas"; + public override string ColumnName => "Gas"; + public override string Legend => "Amount of gas used by the operation"; + + public override string GetValue(Summary summary, BenchmarkCase benchmarkCase) + { + (long? gas, Statistics? _) = GetBenchmarkData(summary, benchmarkCase); + return gas?.ToString() ?? "N/A"; + } + } + + private class GasThroughputColumn : BaseGasColumn + { + public override string Id => "GasThroughput"; + public override string ColumnName => "Throughput"; + public override string Legend => "Amount of gas processed per second"; + + public override string GetValue(Summary summary, BenchmarkCase benchmarkCase) + { + (long? gas, Statistics? stats) = GetBenchmarkData(summary, benchmarkCase); + + if (gas is null || stats?.Mean is null) + { + return "N/A"; + } + + double mgasThroughput = CalculateMGasThroughput(gas.Value, stats.Mean); + return mgasThroughput.ToString("F2") + " MGas/s"; + } + } + + private class GasConfidenceIntervalColumn(bool isLower) : BaseGasColumn + { + public override string Id => isLower ? "GasCI-Lower" : "GasCI-Upper"; + public override string ColumnName => isLower ? "Throughput CI-Lower" : "Throughput CI-Upper"; + public override string Legend => $"{(isLower ? "Lower" : "Upper")} bound of gas throughput 99% confidence interval"; + + public override string GetValue(Summary summary, BenchmarkCase benchmarkCase) + { + (long? gas, Statistics? stats) = GetBenchmarkData(summary, benchmarkCase); + + if (gas is null || stats?.Mean is null) + { + return "N/A"; + } + + ConfidenceInterval ci = stats.GetConfidenceInterval(ConfidenceLevel.L99); + double bound = isLower ? ci.Lower : ci.Upper; + double mgasThroughput = CalculateMGasThroughput(gas.Value, bound); + return mgasThroughput.ToString("F2") + " MGas/s"; + } + } +} diff --git a/src/Nethermind/Nethermind.Precompiles.Benchmark/JsonInput.cs b/src/Nethermind/Nethermind.Precompiles.Benchmark/JsonInput.cs index ffee9aa6891..23be3d6f361 100644 --- a/src/Nethermind/Nethermind.Precompiles.Benchmark/JsonInput.cs +++ b/src/Nethermind/Nethermind.Precompiles.Benchmark/JsonInput.cs @@ -1,5 +1,6 @@ -// ReSharper disable UnusedAutoPropertyAccessor.Global -// ReSharper disable UnusedMember.Global +// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + namespace Nethermind.Precompiles.Benchmark { public class JsonInput diff --git a/src/Nethermind/Nethermind.Precompiles.Benchmark/KeccakBenchmark.cs b/src/Nethermind/Nethermind.Precompiles.Benchmark/KeccakBenchmark.cs index 844c4a6dde1..4c341a6028f 100644 --- a/src/Nethermind/Nethermind.Precompiles.Benchmark/KeccakBenchmark.cs +++ b/src/Nethermind/Nethermind.Precompiles.Benchmark/KeccakBenchmark.cs @@ -12,7 +12,7 @@ public class KeccakBenchmark { public readonly struct Param { - private static Random _random = new Random(42); + private static readonly Random _random = new(42); public Param(byte[] bytes) { diff --git a/src/Nethermind/Nethermind.Precompiles.Benchmark/PrecompileBenchmarkBase.cs b/src/Nethermind/Nethermind.Precompiles.Benchmark/PrecompileBenchmarkBase.cs index 5dbaf5406d2..09e0626e142 100644 --- a/src/Nethermind/Nethermind.Precompiles.Benchmark/PrecompileBenchmarkBase.cs +++ b/src/Nethermind/Nethermind.Precompiles.Benchmark/PrecompileBenchmarkBase.cs @@ -3,48 +3,37 @@ using System; using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; using System.IO; using System.Linq; using BenchmarkDotNet.Attributes; using Nethermind.Core.Extensions; +using Nethermind.Core.Specs; using Nethermind.Evm.Precompiles; using Nethermind.Serialization.Json; using Nethermind.Specs.Forks; namespace Nethermind.Precompiles.Benchmark { - [SuppressMessage("ReSharper", "UnassignedGetOnlyAutoProperty")] - [SuppressMessage("ReSharper", "MemberCanBePrivate.Global")] - [SuppressMessage("ReSharper", "UnusedAutoPropertyAccessor.Global")] public abstract class PrecompileBenchmarkBase { protected abstract IEnumerable Precompiles { get; } protected abstract string InputsDirectory { get; } - public readonly struct Param + public readonly struct Param(IPrecompile precompile, string name, byte[] bytes, byte[]? expected) { - public IPrecompile Precompile { get; } + public IPrecompile Precompile { get; } = precompile ?? throw new ArgumentNullException(nameof(precompile)); - public Param(IPrecompile precompile, string name, byte[] bytes, byte[]? expected) - { - Precompile = precompile ?? throw new ArgumentNullException(nameof(precompile)); - Bytes = bytes; - Name = name; - ExpectedResult = expected; - } + public byte[] Bytes { get; } = bytes; - public byte[] Bytes { get; } + public byte[]? ExpectedResult { get; } = expected; - public byte[]? ExpectedResult { get; } + public string Name { get; } = name; - public string Name { get; } + public long Gas(IReleaseSpec releaseSpec) => + precompile.BaseGasCost(releaseSpec) + precompile.DataGasCost(Bytes, releaseSpec); - public override string ToString() - { - return Name; - } + public override string ToString() => Name; } public IEnumerable Inputs @@ -53,7 +42,7 @@ public IEnumerable Inputs { foreach (IPrecompile precompile in Precompiles) { - List inputs = new List(); + List inputs = []; foreach (string file in Directory.GetFiles($"{InputsDirectory}/current", "*.csv", SearchOption.TopDirectoryOnly)) { // take only first line from each file @@ -64,9 +53,9 @@ public IEnumerable Inputs foreach (string file in Directory.GetFiles($"{InputsDirectory}/current", "*.json", SearchOption.TopDirectoryOnly)) { - EthereumJsonSerializer jsonSerializer = new EthereumJsonSerializer(); - var jsonInputs = jsonSerializer.Deserialize(File.ReadAllText(file)); - var parameters = jsonInputs.Select(i => new Param(precompile, i.Name!, i.Input!, i.Expected)); + EthereumJsonSerializer jsonSerializer = new(); + JsonInput[] jsonInputs = jsonSerializer.Deserialize(File.ReadAllText(file)); + IEnumerable parameters = jsonInputs.Select(i => new Param(precompile, i.Name!, i.Input!, i.Expected)); inputs.AddRange(parameters); } @@ -82,14 +71,10 @@ public IEnumerable Inputs public Param Input { get; set; } private static byte[] LineToTestInput(string line) - { - return Bytes.FromHexString(line.Split(',')[0]); - } + => Bytes.FromHexString(line.Split(',')[0]); [Benchmark(Baseline = true)] public (ReadOnlyMemory, bool) Baseline() - { - return Input.Precompile.Run(Input.Bytes, Berlin.Instance); - } + => Input.Precompile.Run(Input.Bytes, Cancun.Instance); } }