*: add external flag for testing external networks

.github/workflows/build.yml

@@ -31,7 +31,7 @@ jobs:
 
       - uses: actions/setup-go@v5
         with:
-          go-version-file: 'cmd/go.mod'
+          go-version: '1.21.3'
           cache-dependency-path: cmd/go.sum
 
       - name: Build

.github/workflows/tests.yml

@@ -28,7 +28,7 @@ jobs:
 
       - uses: actions/setup-go@v5
         with:
-          go-version-file: 'cmd/go.mod'
+          go-version: '1.21.3'
           cache-dependency-path: cmd/go.sum
 
       - name: golangci-lint

.gitignore

@@ -11,3 +11,4 @@ vendor
 .docker/rpc/go.protocol*.yml
 .docker/rpc/sharp.*.json
 .docker/build/*.acc
+cmd/bin/

Makefile

@@ -24,7 +24,7 @@ help:
 	@echo ''
 	@awk '/^#/{ comment = substr($$0,3) } comment && /^[a-zA-Z][a-zA-Z0-9_-]+ ?:/{ print "   ", $$1, comment }' $(MAKEFILE_LIST) | column -t -s ':' | grep -v 'IGNORE' | sort | uniq
 
-.PHONY: build prepare push gen build.node.go build.node.sharp stop start config \
+.PHONY: build prepare push gen build.node.go build.node.sharp build.bench stop start config \
 	start.GoSingle10wrk start.GoSingle30wrk start.GoSingle100wrk \
 	start.GoSingle25rate start.GoSingle50rate start.GoSingle60rate start.GoSingle300rate start.GoSingle1000rate \
 	start.GoFourNodes10wrk start.GoFourNodes30wrk start.GoFourNodes100wrk \
@@ -37,7 +37,15 @@ help:
 	start.SharpFourNodesGoRPC25rate start.SharpFourNodesGoRPC50rate start.SharpFourNodesGoRPC60rate start.SharpFourNodesGoRPC300rate start.SharpFourNodesGoRPC1000rate
 
 # Build all images
-build: gen build.node.bench build.node.go build.node.sharp
+build: gen build.node.bench build.node.go build.node.sharp build.bench
+
+# Build Benchmark binary file
+build.bench:
+	@echo "=> Building Bench binary file"
+	@set -x \
+		&& export GOGC=off \
+		&& export CGO_ENABLED=0 \
+		&& go build -C cmd -v -o bin/bench -trimpath ./bench
 
 # Push all images to registry
 push:
@@ -82,6 +90,8 @@ stop:
 	@docker compose -f $(DC_GO_IR) -f $(DC_GO_7_IR) -f $(DC_GO_RPC) -f $(DC_GO_7_RPC) \
 		-f $(DC_GO_IR_SINGLE) -f $(DC_SINGLE) -f $(DC_SHARP_IR) -f $(DC_SHARP_7_IR) \
 		-f $(DC_SHARP_RPC) -f $(DC_SHARP_7_RPC) -f $(DC_SHARP_IR_SINGLE) down --remove-orphans &> /dev/null
+	@echo "=> Stop Bench process"
+    +       @killall -w -v -INT bench > /dev/null 2>&1 || :
 
 # Check that all images were built
 check.images:

README.md

@@ -26,7 +26,7 @@
 
 ## Usage example (local benchmark + NeoGo single node)
 
-1. Build the benchmark:
+1. Build the benchmark images and binary file with the following command:
 ```
 $ make build
 => Building Bench image registry.nspcc.ru/neo-bench/neo-bench:bench
@@ -35,6 +35,12 @@ sha256:b08f9fd42198be6c351d725543ac1e451063d18018a738f2446678a0cdf8ee78
 sha256:2bf655747dfa06b85ced1ad7f0257128e7261e6d16b2c8087bc16fd27fcb3a6d
 => Building Sharp Node image registry.nspcc.ru/neo-bench/neo-sharp:bench
 sha256:a6ed753e8f81fedf8a9be556e60c6a41e385dd1ab2c90755ab44e2ceab92bca2
+=> Building Bench binary file
++ export GOGC=off
++ GOGC=off
++ export CGO_ENABLED=0
++ CGO_ENABLED=0
++ go -C cmd build -v -o bin/bench -trimpath ./bench
 ```
 
 2. Run `test` target for a test run:
@@ -170,6 +176,36 @@ $ ./runner.sh --validators 1 --nodes sharp -d "SharpSingle" -m rate -q 25 -z 5m
 $ ./runner.sh --nodes mixed -d "MixedGoRPC4x1" -m rate -q 50 -z 5m -t 30s
 ...
 ```
+## Usage example (local benchmark + external cluster)
+
+NeoBench can be run in a stand-alone mode (loader only) to benchmark some external network. 
+NeoBench expects external network to be launched with the known set of validators and committee (with wallets from `./docker/ir/`)
+and not contain any transactions in the network (to successfully perform initial NEO/GAS transfers). In this setup the 
+loader will be launched as a system process, without Docker container. You have to provide RPS address(-es) of some 
+node from the external network to the loader instance on start. The loader will use the provided RPC to send 
+transactions to the network.
+
+1. Build the benchmark binary file with the following command:
+```
+$ make build
+=> Building Bench image registry.nspcc.ru/neo-bench/neo-bench:bench
+sha256:b08f9fd42198be6c351d725543ac1e451063d18018a738f2446678a0cdf8ee78
+=> Building Go Node image registry.nspcc.ru/neo-bench/neo-go:bench
+sha256:2bf655747dfa06b85ced1ad7f0257128e7261e6d16b2c8087bc16fd27fcb3a6d
+=> Building Sharp Node image registry.nspcc.ru/neo-bench/neo-sharp:bench
+sha256:a6ed753e8f81fedf8a9be556e60c6a41e385dd1ab2c90755ab44e2ceab92bca2
+=> Building Bench binary file
++ export GOGC=off
++ GOGC=off
++ export CGO_ENABLED=0
++ CGO_ENABLED=0
++ go -C cmd build -v -o bin/bench -trimpath ./bench
+```
+
+2. Run benchmarks using the `runner.sh` script with RPC address(-es) of the external network and `--external` flag set:
+```
+$ ./runner.sh -e -d "Go4x1" -m rate -q 1000 -z 5m -t 30s -a 192.168.1.100:20331 -a 192.168.1.101:20331
+```
 
 ## Benchmark usage
 
@@ -194,6 +230,8 @@ $ ./runner.sh --nodes mixed -d "MixedGoRPC4x1" -m rate -q 50 -z 5m -t 30s
                                    Example: -t 30s --request_timeout 15s (default 30s)
   -i, --in                         Path to input file to load transactions.
                                    Example: -i ./dump.txs --in /path/to/import/transactions
+      --vote                       Vote before the bench.
+      --disable-stats              Disable memory and CPU usage statistics collection.
 ````
 
 ## Makefile usage
@@ -299,13 +337,15 @@ The following default configurations are available:
        --msPerBlock                 Protocol setting specifying the minimal (and targeted for) time interval between blocks. Must be an integer number of milliseconds.
                                     The default value is set in configuration templates and is 1s and 5s for single node and multinode setup respectively.
                                     Example: --msPerBlock 3000
+   -e, --external                   Use external network for benchmarking. Default is false. -a flag should be used to specify RPC addresses.
 
 ```
 
 ## Build options
 
 By default, neo-bench uses released versions of Neo nodes to build Docker images.
 However, you can easily test non-released branches or even separate commits for both Go and C# Neo nodes.
+`--external` flag should be used for external network benchmarks to run the loader in a standalone mode without Docker container.
 
 ### Build Go node image from sources
 

cmd/bench/main.go

@@ -43,6 +43,7 @@ func main() {
 		timeLimit       = v.GetDuration("timeLimit")
 		mode            = internal.BenchMode(v.GetString("mode"))
 		client          *internal.RPCClient
+		disableStats    = v.GetBool("disable-stats")
 	)
 
 	if mode == internal.ModeRate {
@@ -92,32 +93,33 @@ func main() {
 			log.Fatalf("could not close report: %v", err)
 		}
 	}()
+	if !disableStats {
+		statsPeriod := time.Second
 
-	statsPeriod := time.Second
+		ds, err := internal.NewStats(ctx,
+			internal.StatEnableLogger(),
+			internal.StatPeriod(statsPeriod),
+			internal.StatCriteria([]string{"stats"}),
+			internal.StatListVerifier(func(list []types.Container) error {
+				if len(list) == 0 {
+					return errors.New("containers not found by criteria")
+				}
 
-	ds, err := internal.NewStats(ctx,
-		internal.StatEnableLogger(),
-		internal.StatPeriod(statsPeriod),
-		internal.StatCriteria([]string{"stats"}),
-		internal.StatListVerifier(func(list []types.Container) error {
-			if len(list) == 0 {
-				return errors.New("containers not found by criteria")
-			}
+				return nil
+			}))
 
-			return nil
-		}))
+		if err != nil {
+			log.Fatalf("could not create docker stats grabber: %v", err)
+		}
 
-	if err != nil {
-		log.Fatalf("could not create docker stats grabber: %v", err)
+		statsStart := time.Now()
+		// Run stats worker:
+		go ds.Run(ctx, func(cpu, mem float64) {
+			rep.UpdateRes(statsStart, cpu, mem)
+			log.Printf("CPU: %0.3f%%, Mem: %0.3fMB", cpu, mem)
+		})
 	}
 
-	statsStart := time.Now()
-	// Run stats worker:
-	go ds.Run(ctx, func(cpu, mem float64) {
-		rep.UpdateRes(statsStart, cpu, mem)
-		log.Printf("CPU: %0.3f%%, Mem: %0.3fMB", cpu, mem)
-	})
-
 	if in := v.GetString("in"); in != "" {
 		dump = internal.ReadDump(in)
 	} else {

cmd/internal/settings.go

@@ -95,6 +95,7 @@ func InitSettings() *viper.Viper {
 			"Example: -i ./dump.txs --in /path/to/import/transactions")
 
 	flags.BoolP("vote", "", false, "Vote before the bench.")
+	flags.BoolP("disable-stats", "", false, "Disable memory and CPU usage statistics collection.")
 
 	if err := v.BindPFlags(flags); err != nil {
 		panic(err)

runner.sh

@@ -3,7 +3,7 @@
 source .env
 
 OUTPUT=""
-ARGS=(-i "/dump.txs")
+ARGS=()
 FILES=()
 MODE=""
 TARGET_RPS=""
@@ -12,6 +12,7 @@ WORKERS_COUNT="30"
 IR_TYPE=go
 RPC_TYPE=
 RPC_ADDR=()
+EXTERNAL_NETWORK=false
 export NEOBENCH_LOGGER=${NEOBENCH_LOGGER:-none}
 export NEOBENCH_TYPE=${NEOBENCH_TYPE:-NEO}
 export NEOBENCH_FROM_COUNT=${NEOBENCH_FROM_COUNT:-1}
@@ -56,6 +57,7 @@ show_help() {
 	echo "       --msPerBlock                 Protocol setting specifying the minimal (and targeted for) time interval between blocks. Must be an integer number of milliseconds."
 	echo "                                    The default value is set in configuration templates and is 1s and 5s for single node and multinode setup respectively."
 	echo "                                    Example: --msPerBlock 1000"
+	echo "   -e, --external                   Use external network for benchmarking. Default is false. -a flag should be used to specify RPC addresses."
 	exit 0
 }
 
@@ -74,6 +76,9 @@ while test $# -gt 0; do
 
 	case $_opt in
 	-h | --help) show_help ;;
+	-e|--external)
+        EXTERNAL_NETWORK=true
+        ;;
 	-l | --log)
 		if [[ $# -gt 0 && ${1:0:1} != "-" ]]; then
 			case "$1" in
@@ -280,7 +285,13 @@ if [ -n "$NEOBENCH_VOTE" ]; then
 fi
 
 make prepare
-
-docker compose "${FILES[@]}" run bench neo-bench -o "$OUTPUT" "${ARGS[@]}"
-
+if [ "$EXTERNAL_NETWORK" = true ]; then
+      ARGS+=(-i "./.docker/build/dump.$NEOBENCH_TYPE.$NEOBENCH_FROM_COUNT.$NEOBENCH_TO_COUNT.txs" --disable-stats)
+      ./cmd/bin/bench -o "$OUTPUT" "${ARGS[@]}"&
+      pid=$!
+      wait $pid
+else
+    ARGS+=(-i "/dump.txs")
+    docker compose "${FILES[@]}" run bench neo-bench -o "$OUTPUT" "${ARGS[@]}"
+fi
 make stop