Merge branch 'unstable' into patch-1

status-im · Jan 13, 2025 · 0baff86 · 0baff86
2 parents c88b2be + 62e57f5
commit 0baff86
Show file tree

Hide file tree

Showing 9 changed files with 318 additions and 27 deletions.
diff --git a/AllTests-mainnet.md b/AllTests-mainnet.md
@@ -513,6 +513,11 @@ OK: 253/253 Fail: 0/253 Skip: 0/253
 OK: 56/56 Fail: 0/56 Skip: 0/56
 ## EF - PeerDAS - Networking [Preset: mainnet]
 ```diff
++ Networking - Compute Columns for Custody Group - mainnet/fulu/networking/compute_columns_f OK
++ Networking - Compute Columns for Custody Group - mainnet/fulu/networking/compute_columns_f OK
++ Networking - Compute Columns for Custody Group - mainnet/fulu/networking/compute_columns_f OK
++ Networking - Compute Columns for Custody Group - mainnet/fulu/networking/compute_columns_f OK
++ Networking - Compute Columns for Custody Group - mainnet/fulu/networking/compute_columns_f OK
 + Networking - Get Custody Groups - mainnet/fulu/networking/get_custody_groups/pyspec_tests/ OK
 + Networking - Get Custody Groups - mainnet/fulu/networking/get_custody_groups/pyspec_tests/ OK
 + Networking - Get Custody Groups - mainnet/fulu/networking/get_custody_groups/pyspec_tests/ OK
@@ -523,7 +528,7 @@ OK: 56/56 Fail: 0/56 Skip: 0/56
 + Networking - Get Custody Groups - mainnet/fulu/networking/get_custody_groups/pyspec_tests/ OK
 + Networking - Get Custody Groups - mainnet/fulu/networking/get_custody_groups/pyspec_tests/ OK
 ```
-OK: 9/9 Fail: 0/9 Skip: 0/9
+OK: 14/14 Fail: 0/14 Skip: 0/14
 ## EF - SSZ generic types
 ```diff
   Testing basic_vector inputs - invalid                                                      Skip
@@ -875,6 +880,11 @@ OK: 2/2 Fail: 0/2 Skip: 0/2
 + Accelerated shuffling computation (with epochRefState jump)                                OK
 ```
 OK: 2/2 Fail: 0/2 Skip: 0/2
+## Size bounds
+```diff
++ SignedBeaconBlockDeneb                                                                     OK
+```
+OK: 1/1 Fail: 0/1 Skip: 0/1
 ## Slashing Interchange tests  [Preset: mainnet]
 ```diff
 + Slashing test: duplicate_pubkey_not_slashable.json                                         OK
@@ -1144,4 +1154,4 @@ OK: 2/2 Fail: 0/2 Skip: 0/2
 OK: 9/9 Fail: 0/9 Skip: 0/9
 
 ---TOTAL---
-OK: 777/782 Fail: 0/782 Skip: 5/782
+OK: 783/788 Fail: 0/788 Skip: 5/788
diff --git a/beacon_chain/consensus_object_pools/block_clearance.nim b/beacon_chain/consensus_object_pools/block_clearance.nim
@@ -1,5 +1,5 @@
 # beacon_chain
-# Copyright (c) 2018-2024 Status Research & Development GmbH
+# Copyright (c) 2018-2025 Status Research & Development GmbH
 # Licensed and distributed under either of
 #   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
 #   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
@@ -509,15 +509,20 @@ proc addBackfillBlockData*(
         withBlck(parentBlock):
           forkyBlck.message.state_root
       clearanceBlock = BlockSlotId.init(parent.bid, forkyBlck.message.slot)
-      updateFlags1 = dag.updateFlags + {skipLastStateRootCalculation}
+      updateFlags1 = dag.updateFlags
+        # TODO (cheatfate): {skipLastStateRootCalculation} flag here could
+        # improve performance by 100%, but this approach needs some
+        # improvements, which is unclear.
 
     if not updateState(dag, dag.clearanceState, clearanceBlock, true, cache,
                        updateFlags1):
       error "Unable to load clearance state for parent block, " &
             "database corrupt?", clearanceBlock = shortLog(clearanceBlock)
       return err(VerifierError.MissingParent)
 
-    dag.clearanceState.setStateRoot(trustedStateRoot)
+    # dag.clearanceState.setStateRoot(trustedStateRoot)
+    # TODO (cheatfate): This is last part of previous TODO comment, we should
+    # set state's `root` to block's `state_root`.
 
     let proposerVerifyTick = Moment.now()
 

diff --git a/beacon_chain/nimbus_beacon_node.nim b/beacon_chain/nimbus_beacon_node.nim
@@ -381,11 +381,14 @@ proc initFullNode(
     else:
       dag.tail.slot
 
-  func getUntrustedBackfillSlot(): Slot =
+  proc getUntrustedBackfillSlot(): Slot =
     if clist.tail.isSome():
       clist.tail.get().blck.slot
     else:
-      dag.tail.slot
+      getLocalWallSlot()
+
+  func getUntrustedFrontfillSlot(): Slot =
+    getFirstSlotAtFinalizedEpoch()
 
   func getFrontfillSlot(): Slot =
     max(dag.frontfill.get(BlockId()).slot, dag.horizon)
@@ -528,7 +531,7 @@ proc initFullNode(
       dag.cfg.DENEB_FORK_EPOCH, dag.cfg.MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS,
       SyncQueueKind.Backward, getLocalHeadSlot,
       getLocalWallSlot, getFirstSlotAtFinalizedEpoch, getUntrustedBackfillSlot,
-      getFrontfillSlot, isWithinWeakSubjectivityPeriod,
+      getUntrustedFrontfillSlot, isWithinWeakSubjectivityPeriod,
       clistPivotSlot, untrustedBlockVerifier, maxHeadAge = 0,
       shutdownEvent = node.shutdownEvent,
       flags = syncManagerFlags)

diff --git a/beacon_chain/spec/peerdas_helpers.nim b/beacon_chain/spec/peerdas_helpers.nim
@@ -25,8 +25,8 @@ type
   ProofBytes = array[fulu.CELLS_PER_EXT_BLOB, KzgProof]
 
 # https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.10/specs/fulu/das-core.md#compute_columns_for_custody_group
-iterator compute_columns_for_custody_group(custody_group: CustodyIndex):
-                                           ColumnIndex =
+iterator compute_columns_for_custody_group*(custody_group: CustodyIndex):
+                                            ColumnIndex =
   for i in 0'u64 ..< COLUMNS_PER_GROUP:
     yield ColumnIndex(NUMBER_OF_CUSTODY_GROUPS * i + custody_group)
 

diff --git a/beacon_chain/sync/sync_overseer.nim b/beacon_chain/sync/sync_overseer.nim
@@ -1,5 +1,5 @@
 # beacon_chain
-# Copyright (c) 2018-2024 Status Research & Development GmbH
+# Copyright (c) 2018-2025 Status Research & Development GmbH
 # Licensed and distributed under either of
 #   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
 #   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
@@ -148,6 +148,22 @@ proc isWithinWeakSubjectivityPeriod(
   is_within_weak_subjectivity_period(
     dag.cfg, currentSlot, dag.headState, checkpoint)
 
+proc getLastBlockRetentionPeriodSlot(overseer: SyncOverseerRef): Slot =
+  let
+    dag = overseer.consensusManager.dag
+    currentSlot = overseer.beaconClock.now().slotOrZero()
+    slotsCount = dag.cfg.MIN_EPOCHS_FOR_BLOCK_REQUESTS * SLOTS_PER_EPOCH
+  if currentSlot < slotsCount:
+    GENESIS_SLOT
+  else:
+    currentSlot - slotsCount
+
+proc isWithinBlockRetentionPeriod(
+    overseer: SyncOverseerRef,
+    slot: Slot
+): bool =
+  slot >= overseer.getLastBlockRetentionPeriodSlot()
+
 proc isUntrustedBackfillEmpty(clist: ChainListRef): bool =
   clist.tail.isNone()
 
@@ -184,7 +200,7 @@ proc updatePerformance(overseer: SyncOverseerRef, startTick: Moment,
 
   # Update status string
   overseer.statusMsg = Opt.some(
-    timeleft.toTimeLeftString() & " (" &
+    "fill: " & timeleft.toTimeLeftString() & " (" &
     (done * 100).formatBiggestFloat(ffDecimal, 2) & "%) " &
     overseer.avgSpeed.formatBiggestFloat(ffDecimal, 4) &
     "slots/s (" & $dag.head.slot & ")")
@@ -418,6 +434,15 @@ proc mainLoop*(
     clist = overseer.clist
     currentSlot = overseer.beaconClock.now().slotOrZero()
 
+  info "Sync overseer starting",
+       wall_slot = currentSlot,
+       dag_head_slot = dag.head.slot,
+       dag_finalized_head_slot = dag.finalizedHead.slot,
+       dag_horizon = dag.horizon(),
+       dag_backfill_slot = dag.backfill.slot,
+       untrusted_tail = shortLog(clist.tail),
+       untrusted_head = shortLog(clist.head)
+
   if overseer.isWithinWeakSubjectivityPeriod(currentSlot):
     # Starting forward sync manager/monitor.
     overseer.forwardSync.start()
@@ -433,10 +458,12 @@ proc mainLoop*(
 
     if not(isUntrustedBackfillEmpty(clist)):
       let headSlot = clist.head.get().slot
-      if not(overseer.isWithinWeakSubjectivityPeriod(headSlot)):
+      if not(overseer.isWithinBlockRetentionPeriod(headSlot)):
         # Light forward sync file is too old.
-        warn "Light client sync was started too long time ago",
-             current_slot = currentSlot, backfill_data_slot = headSlot
+        warn "Light forward sync was started too long time ago",
+             current_slot = currentSlot,
+             backfill_data_slot = headSlot,
+             retention_period_slot = overseer.getLastBlockRetentionPeriodSlot()
 
     if overseer.config.longRangeSync == LongRangeSyncMode.Lenient:
       # Starting forward sync manager/monitor only.
@@ -451,13 +478,21 @@ proc mainLoop*(
               altair_start_slot = dag.cfg.ALTAIR_FORK_EPOCH.start_slot
         quit 1
 
+      if overseer.isWithinBlockRetentionPeriod(dagHead.slot):
+        fatal "Current database head slot is not in the block retention " &
+              "period range",
+              head_slot = dagHead.slot,
+              retention_period_slot = overseer.getLastBlockRetentionPeriodSlot()
+        quit 1
+
       if isUntrustedBackfillEmpty(clist):
         overseer.untrustedInProgress = true
 
         try:
           await overseer.initUntrustedSync()
         except CancelledError:
           return
+
       # We need to update pivot slot to enable timeleft calculation.
       overseer.untrustedSync.updatePivot(overseer.clist.tail.get().slot)
       # Note: We should not start forward sync manager!
@@ -486,6 +521,8 @@ proc mainLoop*(
         quit 1
 
       overseer.untrustedInProgress = false
+      # Reset status bar
+      overseer.statusMsg = Opt.none(string)
 
       # When we finished state rebuilding process - we could start forward
       # SyncManager which could perform finish sync.

diff --git a/nix/default.nix b/nix/default.nix
@@ -5,7 +5,7 @@
   # Options: nimbus_light_client, nimbus_validator_client, nimbus_signing_node, all
   targets ? ["nimbus_beacon_node"],
   # Options: 0,1,2
-  verbosity ? 0,
+  verbosity ? 1,
   # Perform 2-stage bootstrap instead of 3-stage to save time.
   quickAndDirty ? true,
   # These are the only platforms tested in CI and considered stable.
@@ -16,6 +16,10 @@
   ],
 }:
 
+# The 'or' is to handle src fallback to ../. which lack submodules attribue.
+assert pkgs.lib.assertMsg ((src.submodules or true) == true)
+  "Unable to build without submodules. Append '?submodules=1#' to the URI.";
+
 let
   inherit (pkgs) stdenv lib writeScriptBin callPackage;
 

diff --git a/tests/consensus_spec/test_fixture_networking.nim b/tests/consensus_spec/test_fixture_networking.nim
@@ -18,9 +18,28 @@ import
   ../testutil,
   ./fixtures_utils, ./os_ops
 
-from std/sequtils import mapIt
+proc runComputeForCustodyGroup(suiteName, path: string) =
+  let relativeTestPathComponent = path.relativeTestPathComponent()
+  test "Networking - Compute Columns for Custody Group - " &
+      relativeTestPathComponent:
+    type TestMetaYaml = object
+      custody_group: uint64
+      result: seq[uint64]
+    let
+      meta = block:
+        var s = openFileStream(path/"meta.yaml")
+        defer: close(s)
+        var res: TestMetaYaml
+        yaml.load(s, res)
+        res
+      custody_group = meta.custody_group
+
+    var counter = 0
+    for column in compute_columns_for_custody_group(custody_group):
+      check column == meta.result[counter]
+      inc counter
 
-proc runGetCustodyColumns(suiteName, path: string) =
+proc runGetCustodyGroups(suiteName, path: string) =
   let relativePathComponent = path.relativeTestPathComponent()
   test "Networking - Get Custody Groups - " & relativePathComponent:
     type TestMetaYaml = object
@@ -36,17 +55,22 @@ proc runGetCustodyColumns(suiteName, path: string) =
         res
       node_id = UInt256.fromDecimal(meta.node_id)
       custody_group_count = meta.custody_group_count
-      reslt = (meta.result).mapIt(it)
 
     let columns = get_custody_groups(node_id, custody_group_count)
 
     for i in 0..<columns.lenu64:
-      check columns[i] == reslt[i]
+      check columns[i] == meta.result[i]
 
 suite "EF - PeerDAS - Networking" & preset():
   const presetPath = SszTestsDir/const_preset
   # foldering to be resolved in alpha 11 release of consensus spec tests
-  let basePath =
-    presetPath/"fulu"/"networking"/"get_custody_groups"/"pyspec_tests"
-  for kind, path in walkDir(basePath, relative = true, checkDir = true):
-    runGetCustodyColumns(suiteName, basePath/path)
+  block:
+    let basePath =
+      presetPath/"fulu"/"networking"/"get_custody_groups"/"pyspec_tests"
+    for kind, path in walkDir(basePath, relative = true, checkDir = true):
+      runGetCustodyGroups(suiteName, basePath/path)
+  block:
+    let basePath =
+      presetPath/"fulu"/"networking"/"compute_columns_for_custody_group"/"pyspec_tests"
+    for kind, path in walkDir(basePath, relative = true, checkDir = true):
+      runComputeForCustodyGroup(suiteName, basePath/path)
diff --git a/tests/test_eth2_ssz_serialization.nim b/tests/test_eth2_ssz_serialization.nim
@@ -1,5 +1,5 @@
 # beacon_chain
-# Copyright (c) 2018-2024 Status Research & Development GmbH
+# Copyright (c) 2018-2025 Status Research & Development GmbH
 # Licensed and distributed under either of
 #   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
 #   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
@@ -9,9 +9,11 @@
 {.used.}
 
 import
+  std/strutils,
   unittest2,
-  ../beacon_chain/spec/datatypes/[phase0, altair],
-  ../beacon_chain/spec/eth2_ssz_serialization
+  ../beacon_chain/spec/datatypes/[phase0, altair, bellatrix, deneb],
+  ../beacon_chain/spec/eth2_ssz_serialization,
+  ./consensus_spec/os_ops
 
 static:
   doAssert isFixedSize(Slot) == true
@@ -49,3 +51,60 @@ suite "Specific field types":
     testit(phase0.TrustedSignedBeaconBlock)
     testit(altair.SignedBeaconBlock)
     testit(altair.TrustedSignedBeaconBlock)
+
+suite "Size bounds":
+  test "SignedBeaconBlockDeneb":
+    # https://gist.github.com/tbenr/a0ae19fe7496106886ec1f3cc097c208
+    template sourceDir: string = currentSourcePath.rsplit(DirSep, 1)[0]
+    let expected = os_ops.readFile(
+      sourceDir/"test_files"/"SszLengthBounds_SignedBeaconBlockDeneb.txt")
+
+    var
+      res = ""
+      loc = @["SignedBeaconBlockDeneb"]
+    func record(T: typedesc) =
+      when T is SomeSig|ValidatorPubKey:
+        res.add loc.join(".") & "[" & $T.blob.len & "]: SszLengthBounds" &
+          "{min=" & $T.minSize & ", max=" & $T.maxSize & "}\n"
+        loc[^1].add "[element]"
+        byte.record()
+      elif T is ExecutionAddress|BloomLogs:
+        res.add loc.join(".") & "[" & $T.data.len & "]: SszLengthBounds" &
+          "{min=" & $T.minSize & ", max=" & $T.maxSize & "}\n"
+        loc[^1].add "[element]"
+        byte.record()
+      elif T is KzgCommitment:
+        res.add loc.join(".") & "[" & $T.bytes.len & "]: SszLengthBounds" &
+          "{min=" & $T.minSize & ", max=" & $T.maxSize & "}\n"
+        loc[^1].add "[element]"
+        byte.record()
+      elif T is array|HashArray:
+        res.add loc.join(".") & "[" & $T.len & "]: SszLengthBounds" &
+          "{min=" & $T.minSize & ", max=" & $T.maxSize & "}\n"
+        loc[^1].add "[element]"
+        ElemType(T).record()
+      elif T is List|HashList:
+        res.add loc.join(".") & "(" & $T.maxLen & "): SszLengthBounds" &
+          "{min=" & $T.minSize & ", max=" & $T.maxSize & "}\n"
+        loc[^1].add "(element)"
+        ElemType(T).record()
+      elif T is BitArray:
+        res.add loc.join(".") & "[" & $T.bits & "]: SszLengthBounds" &
+          "{min=" & $T.minSize & ", max=" & $T.maxSize & "}\n"
+        res.add loc.join(".") & "[element]: SszLengthBounds" &
+          "{min=0(+1 bits), max=0(+1 bits)}\n"
+      elif T is BitList:
+        res.add loc.join(".") & "(" & $T.maxLen & "): SszLengthBounds" &
+          "{min=" & $T.minSize & ", max=" & $T.maxSize & "}\n"
+        res.add loc.join(".") & "(element): SszLengthBounds" &
+          "{min=0(+1 bits), max=0(+1 bits)}\n"
+      else:
+        res.add loc.join(".") & ": SszLengthBounds" &
+          "{min=" & $T.minSize & ", max=" & $T.maxSize & "}\n"
+        when T is object and T isnot Eth2Digest|UInt256:
+          T.enumAllSerializedFields():
+            loc.add fieldName
+            record(FieldType)
+            discard loc.pop()
+    record deneb.SignedBeaconBlock
+    check res.splitLines() == expected.splitLines()