Include additional information to output batches MDBX browser tool (#…

…1047)

* Populate additional details in the batch (WIP)

* Use the const for the fork id identifier

* GetBatchDataByNumbers add error handling and remove transactions collection

* Populate additional fields for the batches

* Implement output-batch-affiliation command

* Fix tests

* Fix DbDataRetriever UT

cmd/rpcdaemon/commands/zkevm_api.go

@@ -30,6 +30,7 @@ import (
 	smt "github.com/ledgerwatch/erigon/smt/pkg/smt"
 	smtUtils "github.com/ledgerwatch/erigon/smt/pkg/utils"
 	"github.com/ledgerwatch/erigon/turbo/rpchelper"
+	"github.com/ledgerwatch/erigon/zk/constants"
 	"github.com/ledgerwatch/erigon/zk/hermez_db"
 	"github.com/ledgerwatch/erigon/zk/legacy_executor_verifier"
 	types "github.com/ledgerwatch/erigon/zk/rpcdaemon"
@@ -299,6 +300,9 @@ func (api *ZkEvmAPIImpl) GetBatchDataByNumbers(ctx context.Context, batchNumbers
 	if syncing != nil && syncing != false {
 		bn := syncing.(map[string]interface{})["currentBlock"]
 		highestBatchNo, err = hermezDb.GetBatchNoByL2Block(uint64(bn.(hexutil.Uint64)))
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	bds := make([]*types.BatchDataSlim, 0, len(batchNumbers.Numbers))
@@ -342,7 +346,6 @@ func (api *ZkEvmAPIImpl) GetBatchDataByNumbers(ctx context.Context, batchNumbers
 
 		// collect blocks in batch
 		var batchBlocks []*eritypes.Block
-		var batchTxs []eritypes.Transaction
 		// handle genesis - not in the hermez tables so requires special treament
 		if batchNumber == 0 {
 			blk, err := api.ethApi.BaseAPI.blockByNumberWithSenders(tx, 0)
@@ -358,9 +361,6 @@ func (api *ZkEvmAPIImpl) GetBatchDataByNumbers(ctx context.Context, batchNumbers
 				return nil, err
 			}
 			batchBlocks = append(batchBlocks, blk)
-			for _, btx := range blk.Transactions() {
-				batchTxs = append(batchTxs, btx)
-			}
 		}
 
 		// batch l2 data - must build on the fly
@@ -369,7 +369,7 @@ func (api *ZkEvmAPIImpl) GetBatchDataByNumbers(ctx context.Context, batchNumbers
 			return nil, err
 		}
 
-		batchL2Data, err := generateBatchData(tx, hermezDb, batchBlocks, forkId)
+		batchL2Data, err := utils.GenerateBatchData(tx, hermezDb, batchBlocks, forkId)
 		if err != nil {
 			return nil, err
 		}
@@ -381,54 +381,6 @@ func (api *ZkEvmAPIImpl) GetBatchDataByNumbers(ctx context.Context, batchNumbers
 	return populateBatchDataSlimDetails(bds)
 }
 
-func generateBatchData(
-	tx kv.Tx,
-	hermezDb *hermez_db.HermezDbReader,
-	batchBlocks []*eritypes.Block,
-	forkId uint64,
-) (batchL2Data []byte, err error) {
-
-	lastBlockNoInPreviousBatch := uint64(0)
-	if batchBlocks[0].NumberU64() != 0 {
-		lastBlockNoInPreviousBatch = batchBlocks[0].NumberU64() - 1
-	}
-
-	lastBlockInPreviousBatch, err := rawdb.ReadBlockByNumber(tx, lastBlockNoInPreviousBatch)
-	if err != nil {
-		return nil, err
-	}
-
-	batchL2Data = []byte{}
-	for i := 0; i < len(batchBlocks); i++ {
-		var dTs uint32
-		if i == 0 {
-			dTs = uint32(batchBlocks[i].Time() - lastBlockInPreviousBatch.Time())
-		} else {
-			dTs = uint32(batchBlocks[i].Time() - batchBlocks[i-1].Time())
-		}
-		iti, err := hermezDb.GetBlockL1InfoTreeIndex(batchBlocks[i].NumberU64())
-		if err != nil {
-			return nil, err
-		}
-		egTx := make(map[common.Hash]uint8)
-		for _, txn := range batchBlocks[i].Transactions() {
-			eg, err := hermezDb.GetEffectiveGasPricePercentage(txn.Hash())
-			if err != nil {
-				return nil, err
-			}
-			egTx[txn.Hash()] = eg
-		}
-
-		bl2d, err := zktx.GenerateBlockBatchL2Data(uint16(forkId), dTs, uint32(iti), batchBlocks[i].Transactions(), egTx)
-		if err != nil {
-			return nil, err
-		}
-		batchL2Data = append(batchL2Data, bl2d...)
-	}
-
-	return batchL2Data, err
-}
-
 // GetBatchByNumber returns a batch from the current canonical chain. If number is nil, the
 // latest known batch is returned.
 func (api *ZkEvmAPIImpl) GetBatchByNumber(ctx context.Context, batchNumber rpc.BlockNumber, fullTx *bool) (json.RawMessage, error) {
@@ -670,7 +622,7 @@ func (api *ZkEvmAPIImpl) GetBatchByNumber(ctx context.Context, batchNumber rpc.B
 		return nil, err
 	}
 
-	batchL2Data, err := generateBatchData(tx, hermezDb, batchBlocks, forkId)
+	batchL2Data, err := utils.GenerateBatchData(tx, hermezDb, batchBlocks, forkId)
 	if err != nil {
 		return nil, err
 	}
@@ -686,7 +638,7 @@ func (api *ZkEvmAPIImpl) GetBatchByNumber(ctx context.Context, batchNumber rpc.B
 	// forkid exit roots logic
 	// if forkid < 12 then we should only set the exit roots if they have changed, otherwise 0x00..00
 	// if forkid >= 12 then we should always set the exit roots
-	if forkId < 12 {
+	if forkId < uint64(constants.ForkID12Banana) {
 		// get the previous batches exit roots
 		prevBatchNo := batchNo - 1
 		prevBatchHighestBlock, err := hermezDb.GetHighestBlockInBatch(prevBatchNo)

core/state/intra_block_state_zkevm.go

@@ -34,13 +34,19 @@ type ReadOnlyHermezDb interface {
 	GetIntermediateTxStateRoot(blockNum uint64, txhash libcommon.Hash) (libcommon.Hash, error)
 	GetReusedL1InfoTreeIndex(blockNum uint64) (bool, error)
 	GetSequenceByBatchNo(batchNo uint64) (*zktypes.L1BatchInfo, error)
+	GetSequenceByBatchNoOrHighest(batchNo uint64) (*zktypes.L1BatchInfo, error)
 	GetHighestBlockInBatch(batchNo uint64) (uint64, error)
 	GetLowestBlockInBatch(batchNo uint64) (uint64, bool, error)
 	GetL2BlockNosByBatch(batchNo uint64) ([]uint64, error)
 	GetBatchGlobalExitRoot(batchNum uint64) (*dstypes.GerUpdate, error)
 	GetVerificationByBatchNo(batchNo uint64) (*zktypes.L1BatchInfo, error)
+	GetVerificationByBatchNoOrHighest(batchNo uint64) (*zktypes.L1BatchInfo, error)
 	GetL1BatchData(batchNumber uint64) ([]byte, error)
 	GetL1InfoTreeUpdateByGer(ger libcommon.Hash) (*zktypes.L1InfoTreeUpdate, error)
+	GetBlockL1InfoTreeIndex(blockNumber uint64) (uint64, error)
+	GetBlockInfoRoot(blockNumber uint64) (libcommon.Hash, error)
+	GetLastBlockGlobalExitRoot(l2BlockNo uint64) (libcommon.Hash, uint64, error)
+	GetForkId(batchNo uint64) (uint64, error)
 }
 
 func (sdb *IntraBlockState) GetTxCount() (uint64, error) {

zk/debug_tools/mdbx-data-browser/README.md

@@ -60,6 +60,21 @@ In case `file-output` flag is provided, results are printed to a JSON file (othe
 
 **Note:** In case, `output-blocks` is ran with `verbose` flag provided, it is necessary to provide the proper chain id to the `params/chainspecs/mainnet.json`. This is the case, because CDK Erigon (for now) uses hardcoded data to recover transaction senders, and chain id information is read from the mentioned file.
 
+#### `output-batch-affiliation`
+It is used to output the batch numbers alongside with the blocks that affiliate to the certain batch.
+
+- **Name**: `output-batch-affiliation`
+- **Usage**: Outputs batch affiliation of certain blocks.
+- **Action**: `dumpBatchAffiliation`
+- **Flags**:
+  - `data-dir`: Specifies the data directory to use.
+  - `bn`: Block numbers.
+    - **Name**: `bn`
+    - **Usage**: Block numbers.
+    - **Destination**: `batchOrBlockNumbers`
+  - `verbose`: See [verbose](#verbose) flag.
+  - `file-output`: See [file-output](#file-output) flag.
+
 ### Example Usage
 
 **Pre-requisite:** Navigate to the `zk/debug_tools/mdbx-data-browser` folder and run `go build -o mdbx-data-browser`
@@ -75,3 +90,9 @@ In case `file-output` flag is provided, results are printed to a JSON file (othe
 ```sh
 ./mdbx-data-browser output-blocks --datadir chaindata/ --bn 100,101,102 [--verbose] [--file-output]
 ```
+
+#### `output-batch-affiliation` Command
+
+```sh
+./mdbx-data-browser output-batch-affiliation --datadir chaindata/ --bn 100,101,102 [--verbose] [--file-output]
+```

zk/debug_tools/mdbx-data-browser/cli.go

@@ -67,6 +67,21 @@ var (
 		},
 	}
 
+	getBatchAffiliationCmd = &cli.Command{
+		Action: dumpBatchAffiliation,
+		Name:   "output-batch-affiliation",
+		Usage:  "Outputs batch affiliation for provided block numbers",
+		Flags: []cli.Flag{
+			&utils.DataDirFlag,
+			&cli.Uint64SliceFlag{
+				Name:        "bn",
+				Usage:       "Block numbers",
+				Destination: batchOrBlockNumbers,
+			},
+			fileOutputFlag,
+		},
+	}
+
 	// parameters
 	chainDataDir        string
 	batchOrBlockNumbers *cli.Uint64Slice = cli.NewUint64Slice()
@@ -148,6 +163,36 @@ func dumpBlocksByNumbers(cliCtx *cli.Context) error {
 	return nil
 }
 
+// dumpBatchAffiliation retrieves batch numbers by given block numbers and dumps them either on standard output or to a file
+func dumpBatchAffiliation(cliCtx *cli.Context) error {
+	if !cliCtx.IsSet(utils.DataDirFlag.Name) {
+		return errors.New("chain data directory is not provided")
+	}
+
+	chainDataDir = cliCtx.String(utils.DataDirFlag.Name)
+
+	tx, cleanup, err := createDbTx(chainDataDir, cliCtx.Context)
+	if err != nil {
+		return fmt.Errorf("failed to create read-only db transaction: %w", err)
+	}
+	defer cleanup()
+
+	r := NewDbDataRetriever(tx)
+	batchInfo, err := r.GetBatchAffiliation(batchOrBlockNumbers.Value())
+	if err != nil {
+		return err
+	}
+	jsonBatchAffiliation, err := json.MarshalIndent(batchInfo, "", "  ")
+	if err != nil {
+		return fmt.Errorf("failed to serialize batch affiliation info into the JSON format: %w", err)
+	}
+
+	if err := outputResults(string(jsonBatchAffiliation)); err != nil {
+		return fmt.Errorf("failed to output results: %w", err)
+	}
+	return nil
+}
+
 // createDbTx creates a read-only database transaction, that allows querying it.
 func createDbTx(chainDataDir string, ctx context.Context) (kv.Tx, func(), error) {
 	db := mdbx.MustOpen(chainDataDir)
@@ -183,7 +228,7 @@ func outputResults(results string) error {
 			return err
 		}
 
-		fmt.Printf("results are written to the '%s'", path)
+		fmt.Printf("results are written to the '%s'\n", path)
 		return nil
 	}