Add online tooling for data stream (#2712) (#2715)

* online tools * udpate streamer lib * upgrade to go 1.21 * restore go.mod
0xPolygonHermez · Oct 26, 2023 · 22cd750 · 22cd750
1 parent 4225e2f
commit 22cd750
Show file tree

Hide file tree

Showing 7 changed files with 218 additions and 35 deletions.
diff --git a/go.mod b/go.mod
@@ -3,7 +3,7 @@ module github.com/0xPolygonHermez/zkevm-node
 go 1.21
 
 require (
-	github.com/0xPolygonHermez/zkevm-data-streamer v0.1.3
+	github.com/0xPolygonHermez/zkevm-data-streamer v0.1.7
 	github.com/didip/tollbooth/v6 v6.1.2
 	github.com/dop251/goja v0.0.0-20230806174421-c933cf95e127
 	github.com/ethereum/go-ethereum v1.13.2

diff --git a/go.sum b/go.sum
@@ -42,8 +42,8 @@ cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3f
 dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
 dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
-github.com/0xPolygonHermez/zkevm-data-streamer v0.1.3 h1:J4R0hBFmcPFf3SeeATgM0CDPOIhYU85DxhgRXAjorWg=
-github.com/0xPolygonHermez/zkevm-data-streamer v0.1.3/go.mod h1:853hkExOKPpMqbyEGqF8cuobSlbVd2OLVEZoRFTGRzo=
+github.com/0xPolygonHermez/zkevm-data-streamer v0.1.7 h1:OzvHbnn3Q2hJSefHOtyKjaltc2px3YHqv0YfBJsaHso=
+github.com/0xPolygonHermez/zkevm-data-streamer v0.1.7/go.mod h1:853hkExOKPpMqbyEGqF8cuobSlbVd2OLVEZoRFTGRzo=
 github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
 github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8=
 github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=

diff --git a/tools/datastreamer/Makefile b/tools/datastreamer/Makefile
@@ -7,8 +7,11 @@ check-go:
 # Targets that require the checks
 generate-file: check-go
 reprocess: check-go
+decode-entry-offline: check-go
+decode-l2block-offline: check-go
 decode-entry: check-go
 decode-l2block: check-go
+truncate: check-go
 
 arguments := $(wordlist 2,$(words $(MAKECMDGOALS)),$(MAKECMDGOALS))
 
@@ -24,6 +27,18 @@ decode-entry: ## Runs the tool to decode a given entry number
 decode-l2block: ## Runs the tool to decode a given L2 block
 	go run main.go decode-l2block -cfg config/tool.config.toml -l2block $(arguments)
 
+.PHONY: decode-entry-offline
+decode-entry-offline: ## Runs the offline tool to decode a given entry number
+	go run main.go decode-entry-offline -cfg config/tool.config.toml -entry $(arguments)
+
+.PHONY: decode-l2block-offline
+decode-l2block-offline: ## Runs the offline tool to decode a given L2 block
+	go run main.go decode-l2block-offline -cfg config/tool.config.toml -l2block $(arguments)
+
+.PHONY: truncate
+truncate: ## Runs the offline tool to truncate the stream file
+	go run main.go truncate -cfg config/tool.config.toml -entry $(arguments)
+
 # .PHONY: reprocess
 reprocess: ## Runs the tool to reprocess the information in the stream up to a given l2 block
 	go run main.go reprocess -cfg config/tool.config.toml -genesis ../test/config/test.genesis.config.json -l2block $(arguments)

diff --git a/tools/datastreamer/config/config.go b/tools/datastreamer/config/config.go
@@ -22,13 +22,20 @@ const (
 	FlagGenesis = "genesis"
 )
 
+// OnlineConfig is the configuration for the online data streamer
+type OnlineConfig struct {
+	URI        string                  `mapstructure:"URI"`
+	StreamType datastreamer.StreamType `mapstructure:"StreamType"`
+}
+
 // Config is the configuration for the tool
 type Config struct {
-	ChainID      uint64              `mapstructure:"ChainID"`
-	StreamServer datastreamer.Config `mapstructure:"StreamServer"`
-	StateDB      db.Config           `mapstructure:"StateDB"`
-	Executor     executor.Config     `mapstructure:"Executor"`
-	MerkeTree    merkletree.Config   `mapstructure:"MerkeTree"`
+	ChainID   uint64              `mapstructure:"ChainID"`
+	Online    OnlineConfig        `mapstructure:"Online"`
+	Offline   datastreamer.Config `mapstructure:"Offline"`
+	StateDB   db.Config           `mapstructure:"StateDB"`
+	Executor  executor.Config     `mapstructure:"Executor"`
+	MerkeTree merkletree.Config   `mapstructure:"MerkeTree"`
 }
 
 // Default parses the default configuration values.

diff --git a/tools/datastreamer/config/default.go b/tools/datastreamer/config/default.go
@@ -3,7 +3,10 @@ package config
 // DefaultValues is the default configuration
 const DefaultValues = `
 ChainID = 1440
-[StreamServer]
+[Online]
+URI = "stream.internal.zkevm-test.net:6900"
+StreamType = 1
+[Offline]
 Port = 6901
 Filename = "datastreamer.bin"
 	[Log]

diff --git a/tools/datastreamer/config/tool.config.toml b/tools/datastreamer/config/tool.config.toml
@@ -1,8 +1,11 @@
 ChainID = 1440
-[StreamServer]
+[Online]
+URI = "stream.internal.zkevm-test.net:6900"
+StreamType = 1
+[Offline]
 Port = 6901
 Filename = "datastream.bin"
-    [StreamServer.Log]
+    [OfflineServer.Log]
     Environment = "development"
     Level = "error"
     Outputs = ["stdout"]