add migration for the old counters maps

core/vm/zk_counters.go

@@ -70,7 +70,7 @@ func (c *Counter) AsMap() map[string]int {
 type Counters []*Counter
 
 func NewCounters() Counters {
-	array := make(Counters, counterTypesCount)
+	array := make(Counters, CounterTypesCount)
 	return array
 }
 
@@ -83,14 +83,14 @@ func NewCountersFromUsedArray(used []int) *Counters {
 }
 
 func (c Counters) UsedAsString() string {
-	res := fmt.Sprintf("[%s: %v]", SHAName, c[SHA].used)
-	res += fmt.Sprintf("[%s: %v]", AName, c[A].used)
-	res += fmt.Sprintf("[%s: %v]", BName, c[B].used)
-	res += fmt.Sprintf("[%s: %v]", KName, c[K].used)
-	res += fmt.Sprintf("[%s: %v]", MName, c[M].used)
-	res += fmt.Sprintf("[%s: %v]", PName, c[P].used)
-	res += fmt.Sprintf("[%s: %v]", SName, c[S].used)
-	res += fmt.Sprintf("[%s: %v]", DName, c[D].used)
+	res := fmt.Sprintf("[%s: %v]", CounterKeyNames[SHA], c[SHA].used)
+	res += fmt.Sprintf("[%s: %v]", CounterKeyNames[A], c[A].used)
+	res += fmt.Sprintf("[%s: %v]", CounterKeyNames[B], c[B].used)
+	res += fmt.Sprintf("[%s: %v]", CounterKeyNames[K], c[K].used)
+	res += fmt.Sprintf("[%s: %v]", CounterKeyNames[M], c[M].used)
+	res += fmt.Sprintf("[%s: %v]", CounterKeyNames[P], c[P].used)
+	res += fmt.Sprintf("[%s: %v]", CounterKeyNames[S], c[S].used)
+	res += fmt.Sprintf("[%s: %v]", CounterKeyNames[D], c[D].used)
 	return res
 }
 
@@ -106,14 +106,14 @@ func (c Counters) UsedAsArray() []int {
 
 func (c Counters) UsedAsMap() map[string]int {
 	return map[string]int{
-		string(SName):   c[S].used,
-		string(AName):   c[A].used,
-		string(BName):   c[B].used,
-		string(MName):   c[M].used,
-		string(KName):   c[K].used,
-		string(DName):   c[D].used,
-		string(PName):   c[P].used,
-		string(SHAName): c[SHA].used,
+		string(CounterKeyNames[S]):   c[S].used,
+		string(CounterKeyNames[A]):   c[A].used,
+		string(CounterKeyNames[B]):   c[B].used,
+		string(CounterKeyNames[M]):   c[M].used,
+		string(CounterKeyNames[K]):   c[K].used,
+		string(CounterKeyNames[D]):   c[D].used,
+		string(CounterKeyNames[P]):   c[P].used,
+		string(CounterKeyNames[SHA]): c[SHA].used,
 	}
 }
 
@@ -162,16 +162,7 @@ func (cc Counters) Clone() Counters {
 type CounterKey int
 type CounterName string
 
-var (
-	SName   CounterName = "S"
-	AName   CounterName = "A"
-	BName   CounterName = "B"
-	MName   CounterName = "M"
-	KName   CounterName = "K"
-	DName   CounterName = "D"
-	PName   CounterName = "P"
-	SHAName CounterName = "SHA"
-
+const (
 	S   CounterKey = 0
 	A   CounterKey = 1
 	B   CounterKey = 2
@@ -181,7 +172,12 @@ var (
 	P   CounterKey = 6
 	SHA CounterKey = 7
 
-	counterTypesCount = 8
+	CounterTypesCount = 8
+)
+
+var (
+	// important!!! must match the indexes of the keys
+	CounterKeyNames = []CounterName{"S", "A", "B", "M", "K", "D", "P", "SHA"}
 )
 
 type CounterCollector struct {

migrations/counters_to_array.go

@@ -0,0 +1,72 @@
+package migrations
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+
+	"github.com/gateway-fm/cdk-erigon-lib/common/datadir"
+	"github.com/gateway-fm/cdk-erigon-lib/kv"
+	"github.com/ledgerwatch/erigon/core/vm"
+	"github.com/ledgerwatch/erigon/zk/hermez_db"
+)
+
+var countersToArray = Migration{
+	Name: "migrate counters from map to array",
+	Up: func(db kv.RwDB, dirs datadir.Dirs, progress []byte, BeforeCommit Callback) (err error) {
+		tx, err := db.BeginRw(context.Background())
+		if err != nil {
+			return err
+		}
+		defer tx.Rollback()
+		// Migrate counters from map to array with cursor over BATCH_COUNTERS
+		c, err := tx.Cursor(kv.BATCH_COUNTERS)
+		if err != nil {
+			return err
+		}
+		defer c.Close()
+		tx.ForEach(kv.BATCH_COUNTERS, []byte{}, func(k, v []byte) error {
+			var countersMap map[string]int
+			if err := json.Unmarshal(v, &countersMap); err != nil {
+				// not a map, so pass on
+				return nil
+			}
+
+			countersArray := make([]int, vm.CounterTypesCount)
+			for counterKey, counterCount := range countersMap {
+				keyName := string(counterKey)
+				keyIndex := -1
+				for i, kn := range vm.CounterKeyNames {
+					if string(kn) == keyName {
+						keyIndex = i
+						break
+					}
+				}
+
+				if keyIndex == -1 {
+					blockNo := hermez_db.BytesToUint64(k)
+					return fmt.Errorf("unknown counter key %s for block %d", keyName, blockNo)
+				}
+				countersArray[keyIndex] = counterCount
+			}
+
+			countersArrayBytes, err := json.Marshal(countersArray)
+			if err != nil {
+				return err
+			}
+			newKey := make([]byte, len(k))
+			copy(newKey, k)
+			if err := tx.Put(kv.BATCH_COUNTERS, newKey, countersArrayBytes); err != nil {
+				return err
+			}
+
+			return nil
+		})
+
+		// This migration is no-op, but it forces the migration mechanism to apply it and thus write the DB schema version info
+		if err := BeforeCommit(tx, nil, true); err != nil {
+			return err
+		}
+		return tx.Commit()
+	},
+}

migrations/counters_to_array_test.go

@@ -0,0 +1,153 @@
+package migrations
+
+import (
+	"context"
+	"encoding/json"
+	"testing"
+
+	"github.com/gateway-fm/cdk-erigon-lib/kv"
+	"github.com/gateway-fm/cdk-erigon-lib/kv/memdb"
+	"github.com/ledgerwatch/erigon/core/vm"
+	"github.com/ledgerwatch/erigon/zk/hermez_db"
+	"github.com/stretchr/testify/require"
+)
+
+type testCases struct {
+	testName         string
+	mapCounters      map[uint64]map[string]int
+	arrayCounters    map[uint64][vm.CounterTypesCount]int
+	expectedCounters map[uint64][vm.CounterTypesCount]int
+}
+
+func TestCountersToArray(t *testing.T) {
+	testCases := []testCases{
+		{
+			testName: "only map entries",
+			mapCounters: map[uint64]map[string]int{
+				1: {string(vm.CounterKeyNames[vm.A]): 1, string(vm.CounterKeyNames[vm.B]): 2},
+				2: {string(vm.CounterKeyNames[vm.A]): 3, string(vm.CounterKeyNames[vm.B]): 4},
+			},
+			arrayCounters: map[uint64][vm.CounterTypesCount]int{},
+			expectedCounters: map[uint64][vm.CounterTypesCount]int{
+				1: {0, 1, 2, 0, 0, 0, 0, 0},
+				2: {0, 3, 4, 0, 0, 0, 0, 0},
+			},
+		},
+		{
+			testName:    "only array entries",
+			mapCounters: map[uint64]map[string]int{},
+			arrayCounters: map[uint64][vm.CounterTypesCount]int{
+				1: {0, 1, 2, 0, 0, 0, 0, 0},
+				2: {0, 3, 4, 0, 0, 0, 0, 0},
+			},
+			expectedCounters: map[uint64][vm.CounterTypesCount]int{
+				1: {0, 1, 2, 0, 0, 0, 0, 0},
+				2: {0, 3, 4, 0, 0, 0, 0, 0},
+			},
+		},
+		{
+			testName: "arrays and maps entries",
+			mapCounters: map[uint64]map[string]int{
+				1: {string(vm.CounterKeyNames[vm.A]): 1, string(vm.CounterKeyNames[vm.B]): 2},
+				2: {string(vm.CounterKeyNames[vm.A]): 3, string(vm.CounterKeyNames[vm.B]): 4},
+			},
+			arrayCounters: map[uint64][vm.CounterTypesCount]int{
+				3: {2, 1, 2, 0, 0, 0, 0, 0},
+				4: {1, 3, 4, 0, 0, 0, 0, 0},
+			},
+			expectedCounters: map[uint64][vm.CounterTypesCount]int{
+				1: {0, 1, 2, 0, 0, 0, 0, 0},
+				2: {0, 3, 4, 0, 0, 0, 0, 0},
+				3: {2, 1, 2, 0, 0, 0, 0, 0},
+				4: {1, 3, 4, 0, 0, 0, 0, 0},
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		require, tmpDir, db := require.New(t), t.TempDir(), memdb.NewTestDB(t)
+
+		err := prepareDbCounters(db, tc.mapCounters, tc.arrayCounters)
+		require.NoError(err)
+
+		migrator := NewMigrator(kv.ChainDB)
+
+		migrator.Migrations = []Migration{countersToArray}
+		err = migrator.Apply(db, tmpDir)
+		require.NoError(err)
+
+		err = assertDbCounters(t, db, tc.testName, tc.expectedCounters)
+		require.NoError(err)
+	}
+}
+
+func prepareDbCounters(db kv.RwDB, mapCounters map[uint64]map[string]int, arrayCounters map[uint64][vm.CounterTypesCount]int) error {
+	tx, err := db.BeginRw(context.Background())
+	if err != nil {
+		return err
+	}
+	defer tx.Rollback()
+
+	if err = tx.CreateBucket(kv.BATCH_COUNTERS); err != nil {
+		return err
+	}
+
+	for l2BlockNo, countersMap := range mapCounters {
+		countersMapBytes, err := json.Marshal(countersMap)
+		if err != nil {
+			return err
+		}
+
+		if err = tx.Put(kv.BATCH_COUNTERS, hermez_db.Uint64ToBytes(l2BlockNo), countersMapBytes); err != nil {
+			return err
+		}
+	}
+
+	for l2BlockNo, countersArray := range arrayCounters {
+		countersArrayBytes, err := json.Marshal(countersArray)
+		if err != nil {
+			return err
+		}
+
+		if err = tx.Put(kv.BATCH_COUNTERS, hermez_db.Uint64ToBytes(l2BlockNo), countersArrayBytes); err != nil {
+			return err
+		}
+	}
+
+	return tx.Commit()
+}
+
+func assertDbCounters(t *testing.T, db kv.RwDB, testName string, expectedCounters map[uint64][vm.CounterTypesCount]int) error {
+	tx, err := db.BeginRw(context.Background())
+	if err != nil {
+		return err
+	}
+	defer tx.Rollback()
+	c, err := tx.Cursor(kv.BATCH_COUNTERS)
+	if err != nil {
+		return err
+	}
+	defer c.Close()
+
+	actualCounters := map[uint64][vm.CounterTypesCount]int{}
+	for k, v, err := c.First(); k != nil; k, v, err = c.Next() {
+		if err != nil {
+			return err
+		}
+
+		var counters []int
+		if err = json.Unmarshal(v, &counters); err != nil {
+			return err
+		}
+
+		l2BlockNo := hermez_db.BytesToUint64(k)
+		blockCounters := [vm.CounterTypesCount]int{}
+		copy(blockCounters[:], counters)
+
+		actualCounters[l2BlockNo] = blockCounters
+	}
+
+	require.Equal(t, expectedCounters, actualCounters, testName)
+
+	return tx.Commit()
+}

migrations/migrations.go

@@ -36,6 +36,7 @@ var migrations = map[kv.Label][]Migration{
 		txsBeginEnd,
 		resetBlocks4,
 		refactorTableLastRoot,
+		countersToArray,
 	},
 	kv.TxPoolDB: {},
 	kv.SentryDB: {},

zk/hermez_db/db.go

@@ -1413,7 +1413,7 @@ func (db *HermezDb) WriteBatchCounters(blockNumber uint64, counters []int) error
 	return db.tx.Put(BATCH_COUNTERS, Uint64ToBytes(blockNumber), countersJson)
 }
 
-func (db *HermezDbReader) GetLatestBatchCounters(batchNumber uint64) (countersMap []int, found bool, err error) {
+func (db *HermezDbReader) GetLatestBatchCounters(batchNumber uint64) (countersArray []int, found bool, err error) {
 	batchBlockNumbers, err := db.GetL2BlockNosByBatch(batchNumber)
 	if err != nil {
 		return nil, false, err
@@ -1426,12 +1426,12 @@ func (db *HermezDbReader) GetLatestBatchCounters(batchNumber uint64) (countersMa
 	found = len(v) > 0
 
 	if found {
-		if err = json.Unmarshal(v, &countersMap); err != nil {
+		if err = json.Unmarshal(v, &countersArray); err != nil {
 			return nil, false, err
 		}
 	}
 
-	return countersMap, found, nil
+	return countersArray, found, nil
 }
 
 func (db *HermezDb) DeleteBatchCounters(fromBlockNum, toBlockNum uint64) error {