From 19df9c3110690f7a6cde51f786565cd465a13679 Mon Sep 17 00:00:00 2001 From: Ekaterina Pavlova Date: Mon, 6 May 2024 18:11:34 +0300 Subject: [PATCH 1/4] *: adjust README.md Signed-off-by: Ekaterina Pavlova --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index a09cfca..12a10dc 100644 --- a/README.md +++ b/README.md @@ -194,6 +194,7 @@ $ ./runner.sh --nodes mixed -d "MixedGoRPC4x1" -m rate -q 50 -z 5m -t 30s Example: -t 30s --request_timeout 15s (default 30s) -i, --in Path to input file to load transactions. Example: -i ./dump.txs --in /path/to/import/transactions + --vote Vote before the bench. ```` ## Makefile usage From b675ebed378941f1b1e97699a1ef2dad1e4ec07f Mon Sep 17 00:00:00 2001 From: Ekaterina Pavlova Date: Wed, 29 May 2024 15:23:12 +0300 Subject: [PATCH 2/4] Makefile: add build.bench Building bench binary. Signed-off-by: Ekaterina Pavlova --- .github/workflows/build.yml | 2 +- .github/workflows/tests.yml | 2 +- .gitignore | 1 + Makefile | 12 ++++++++++-- README.md | 8 +++++++- 5 files changed, 20 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8f6f91f..c01aa39 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -31,7 +31,7 @@ jobs: - uses: actions/setup-go@v5 with: - go-version-file: 'cmd/go.mod' + go-version: '1.21.3' cache-dependency-path: cmd/go.sum - name: Build diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ae47958..d5f01fa 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -28,7 +28,7 @@ jobs: - uses: actions/setup-go@v5 with: - go-version-file: 'cmd/go.mod' + go-version: '1.21.3' cache-dependency-path: cmd/go.sum - name: golangci-lint diff --git a/.gitignore b/.gitignore index 1e24ec4..c960d92 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ vendor .docker/rpc/go.protocol*.yml .docker/rpc/sharp.*.json .docker/build/*.acc +cmd/bin/ diff --git a/Makefile b/Makefile index 9a6caf4..74b92c1 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ help: @echo '' @awk '/^#/{ comment = substr($$0,3) } comment && /^[a-zA-Z][a-zA-Z0-9_-]+ ?:/{ print " ", $$1, comment }' $(MAKEFILE_LIST) | column -t -s ':' | grep -v 'IGNORE' | sort | uniq -.PHONY: build prepare push gen build.node.go build.node.sharp stop start config \ +.PHONY: build prepare push gen build.node.go build.node.sharp build.bench stop start config \ start.GoSingle10wrk start.GoSingle30wrk start.GoSingle100wrk \ start.GoSingle25rate start.GoSingle50rate start.GoSingle60rate start.GoSingle300rate start.GoSingle1000rate \ start.GoFourNodes10wrk start.GoFourNodes30wrk start.GoFourNodes100wrk \ @@ -37,7 +37,15 @@ help: start.SharpFourNodesGoRPC25rate start.SharpFourNodesGoRPC50rate start.SharpFourNodesGoRPC60rate start.SharpFourNodesGoRPC300rate start.SharpFourNodesGoRPC1000rate # Build all images -build: gen build.node.bench build.node.go build.node.sharp +build: gen build.node.bench build.node.go build.node.sharp build.bench + +# Build Benchmark binary file +build.bench: + @echo "=> Building Bench binary file" + @set -x \ + && export GOGC=off \ + && export CGO_ENABLED=0 \ + && go build -C cmd -v -o bin/bench -trimpath ./bench # Push all images to registry push: diff --git a/README.md b/README.md index 12a10dc..714af87 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ ## Usage example (local benchmark + NeoGo single node) -1. Build the benchmark: +1. Build the benchmark images and binary file with the following command: ``` $ make build => Building Bench image registry.nspcc.ru/neo-bench/neo-bench:bench @@ -35,6 +35,12 @@ sha256:b08f9fd42198be6c351d725543ac1e451063d18018a738f2446678a0cdf8ee78 sha256:2bf655747dfa06b85ced1ad7f0257128e7261e6d16b2c8087bc16fd27fcb3a6d => Building Sharp Node image registry.nspcc.ru/neo-bench/neo-sharp:bench sha256:a6ed753e8f81fedf8a9be556e60c6a41e385dd1ab2c90755ab44e2ceab92bca2 +=> Building Bench binary file ++ export GOGC=off ++ GOGC=off ++ export CGO_ENABLED=0 ++ CGO_ENABLED=0 ++ go -C cmd build -v -o bin/bench -trimpath ./bench ``` 2. Run `test` target for a test run: From 1fdda2affe45bb70b64b0676b979de371e1d8854 Mon Sep 17 00:00:00 2001 From: Ekaterina Pavlova Date: Mon, 27 May 2024 11:00:07 +0300 Subject: [PATCH 3/4] bench: add --disable-stats flag for disabling docker statistics With `--disable-stats` flag the statistics from docker container won't be collected. Signed-off-by: Ekaterina Pavlova --- README.md | 1 + cmd/bench/main.go | 42 +++++++++++++++++++++------------------- cmd/internal/settings.go | 1 + 3 files changed, 24 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 714af87..fde0c91 100644 --- a/README.md +++ b/README.md @@ -201,6 +201,7 @@ $ ./runner.sh --nodes mixed -d "MixedGoRPC4x1" -m rate -q 50 -z 5m -t 30s -i, --in Path to input file to load transactions. Example: -i ./dump.txs --in /path/to/import/transactions --vote Vote before the bench. + --disable-stats Disable memory and CPU usage statistics collection. ```` ## Makefile usage diff --git a/cmd/bench/main.go b/cmd/bench/main.go index 61aec6e..4e329c1 100644 --- a/cmd/bench/main.go +++ b/cmd/bench/main.go @@ -43,6 +43,7 @@ func main() { timeLimit = v.GetDuration("timeLimit") mode = internal.BenchMode(v.GetString("mode")) client *internal.RPCClient + disableStats = v.GetBool("disable-stats") ) if mode == internal.ModeRate { @@ -92,32 +93,33 @@ func main() { log.Fatalf("could not close report: %v", err) } }() + if !disableStats { + statsPeriod := time.Second - statsPeriod := time.Second + ds, err := internal.NewStats(ctx, + internal.StatEnableLogger(), + internal.StatPeriod(statsPeriod), + internal.StatCriteria([]string{"stats"}), + internal.StatListVerifier(func(list []types.Container) error { + if len(list) == 0 { + return errors.New("containers not found by criteria") + } - ds, err := internal.NewStats(ctx, - internal.StatEnableLogger(), - internal.StatPeriod(statsPeriod), - internal.StatCriteria([]string{"stats"}), - internal.StatListVerifier(func(list []types.Container) error { - if len(list) == 0 { - return errors.New("containers not found by criteria") - } + return nil + })) - return nil - })) + if err != nil { + log.Fatalf("could not create docker stats grabber: %v", err) + } - if err != nil { - log.Fatalf("could not create docker stats grabber: %v", err) + statsStart := time.Now() + // Run stats worker: + go ds.Run(ctx, func(cpu, mem float64) { + rep.UpdateRes(statsStart, cpu, mem) + log.Printf("CPU: %0.3f%%, Mem: %0.3fMB", cpu, mem) + }) } - statsStart := time.Now() - // Run stats worker: - go ds.Run(ctx, func(cpu, mem float64) { - rep.UpdateRes(statsStart, cpu, mem) - log.Printf("CPU: %0.3f%%, Mem: %0.3fMB", cpu, mem) - }) - if in := v.GetString("in"); in != "" { dump = internal.ReadDump(in) } else { diff --git a/cmd/internal/settings.go b/cmd/internal/settings.go index 4e155d3..ab3d8dd 100644 --- a/cmd/internal/settings.go +++ b/cmd/internal/settings.go @@ -95,6 +95,7 @@ func InitSettings() *viper.Viper { "Example: -i ./dump.txs --in /path/to/import/transactions") flags.BoolP("vote", "", false, "Vote before the bench.") + flags.BoolP("disable-stats", "", false, "Disable memory and CPU usage statistics collection.") if err := v.BindPFlags(flags); err != nil { panic(err) From 007e28c6e32692b1d30cbb41a0c4e7b344c79e4e Mon Sep 17 00:00:00 2001 From: Ekaterina Pavlova Date: Thu, 30 May 2024 11:36:15 +0300 Subject: [PATCH 4/4] runner.sh: add --external flag for testing external network With --external flag bench will be run locally without docker container. The `-a` flag should be used with the `-e` flag for specifying the external RPC address. Close #160 Signed-off-by: Ekaterina Pavlova --- Makefile | 2 ++ README.md | 32 ++++++++++++++++++++++++++++++++ runner.sh | 19 +++++++++++++++---- 3 files changed, 49 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 74b92c1..d260b4e 100644 --- a/Makefile +++ b/Makefile @@ -90,6 +90,8 @@ stop: @docker compose -f $(DC_GO_IR) -f $(DC_GO_7_IR) -f $(DC_GO_RPC) -f $(DC_GO_7_RPC) \ -f $(DC_GO_IR_SINGLE) -f $(DC_SINGLE) -f $(DC_SHARP_IR) -f $(DC_SHARP_7_IR) \ -f $(DC_SHARP_RPC) -f $(DC_SHARP_7_RPC) -f $(DC_SHARP_IR_SINGLE) down --remove-orphans &> /dev/null + @echo "=> Stop Bench process" + + @killall -w -v -INT bench > /dev/null 2>&1 || : # Check that all images were built check.images: diff --git a/README.md b/README.md index fde0c91..6d48a53 100644 --- a/README.md +++ b/README.md @@ -176,6 +176,36 @@ $ ./runner.sh --validators 1 --nodes sharp -d "SharpSingle" -m rate -q 25 -z 5m $ ./runner.sh --nodes mixed -d "MixedGoRPC4x1" -m rate -q 50 -z 5m -t 30s ... ``` +## Usage example (local benchmark + external cluster) + +NeoBench can be run in a stand-alone mode (loader only) to benchmark some external network. +NeoBench expects external network to be launched with the known set of validators and committee (with wallets from `./docker/ir/`) +and not contain any transactions in the network (to successfully perform initial NEO/GAS transfers). In this setup the +loader will be launched as a system process, without Docker container. You have to provide RPS address(-es) of some +node from the external network to the loader instance on start. The loader will use the provided RPC to send +transactions to the network. + +1. Build the benchmark binary file with the following command: +``` +$ make build +=> Building Bench image registry.nspcc.ru/neo-bench/neo-bench:bench +sha256:b08f9fd42198be6c351d725543ac1e451063d18018a738f2446678a0cdf8ee78 +=> Building Go Node image registry.nspcc.ru/neo-bench/neo-go:bench +sha256:2bf655747dfa06b85ced1ad7f0257128e7261e6d16b2c8087bc16fd27fcb3a6d +=> Building Sharp Node image registry.nspcc.ru/neo-bench/neo-sharp:bench +sha256:a6ed753e8f81fedf8a9be556e60c6a41e385dd1ab2c90755ab44e2ceab92bca2 +=> Building Bench binary file ++ export GOGC=off ++ GOGC=off ++ export CGO_ENABLED=0 ++ CGO_ENABLED=0 ++ go -C cmd build -v -o bin/bench -trimpath ./bench +``` + +2. Run benchmarks using the `runner.sh` script with RPC address(-es) of the external network and `--external` flag set: +``` +$ ./runner.sh -e -d "Go4x1" -m rate -q 1000 -z 5m -t 30s -a 192.168.1.100:20331 -a 192.168.1.101:20331 +``` ## Benchmark usage @@ -307,6 +337,7 @@ The following default configurations are available: --msPerBlock Protocol setting specifying the minimal (and targeted for) time interval between blocks. Must be an integer number of milliseconds. The default value is set in configuration templates and is 1s and 5s for single node and multinode setup respectively. Example: --msPerBlock 3000 + -e, --external Use external network for benchmarking. Default is false. -a flag should be used to specify RPC addresses. ``` @@ -314,6 +345,7 @@ The following default configurations are available: By default, neo-bench uses released versions of Neo nodes to build Docker images. However, you can easily test non-released branches or even separate commits for both Go and C# Neo nodes. +`--external` flag should be used for external network benchmarks to run the loader in a standalone mode without Docker container. ### Build Go node image from sources diff --git a/runner.sh b/runner.sh index 1aeab3b..987bca2 100755 --- a/runner.sh +++ b/runner.sh @@ -3,7 +3,7 @@ source .env OUTPUT="" -ARGS=(-i "/dump.txs") +ARGS=() FILES=() MODE="" TARGET_RPS="" @@ -12,6 +12,7 @@ WORKERS_COUNT="30" IR_TYPE=go RPC_TYPE= RPC_ADDR=() +EXTERNAL_NETWORK=false export NEOBENCH_LOGGER=${NEOBENCH_LOGGER:-none} export NEOBENCH_TYPE=${NEOBENCH_TYPE:-NEO} export NEOBENCH_FROM_COUNT=${NEOBENCH_FROM_COUNT:-1} @@ -56,6 +57,7 @@ show_help() { echo " --msPerBlock Protocol setting specifying the minimal (and targeted for) time interval between blocks. Must be an integer number of milliseconds." echo " The default value is set in configuration templates and is 1s and 5s for single node and multinode setup respectively." echo " Example: --msPerBlock 1000" + echo " -e, --external Use external network for benchmarking. Default is false. -a flag should be used to specify RPC addresses." exit 0 } @@ -74,6 +76,9 @@ while test $# -gt 0; do case $_opt in -h | --help) show_help ;; + -e|--external) + EXTERNAL_NETWORK=true + ;; -l | --log) if [[ $# -gt 0 && ${1:0:1} != "-" ]]; then case "$1" in @@ -280,7 +285,13 @@ if [ -n "$NEOBENCH_VOTE" ]; then fi make prepare - -docker compose "${FILES[@]}" run bench neo-bench -o "$OUTPUT" "${ARGS[@]}" - +if [ "$EXTERNAL_NETWORK" = true ]; then + ARGS+=(-i "./.docker/build/dump.$NEOBENCH_TYPE.$NEOBENCH_FROM_COUNT.$NEOBENCH_TO_COUNT.txs" --disable-stats) + ./cmd/bin/bench -o "$OUTPUT" "${ARGS[@]}"& + pid=$! + wait $pid +else + ARGS+=(-i "/dump.txs") + docker compose "${FILES[@]}" run bench neo-bench -o "$OUTPUT" "${ARGS[@]}" +fi make stop