introduce ForkedBlobSidecar for EIP-7688 Electra period before PeerDAS

AllTests-mainnet.md

@@ -65,7 +65,7 @@ OK: 4/4 Fail: 0/4 Skip: 0/4
 + sanity check Fulu blocks [Preset: mainnet]                                                 OK
 + sanity check Fulu states [Preset: mainnet]                                                 OK
 + sanity check Fulu states, reusing buffers [Preset: mainnet]                                OK
-+ sanity check blobs [Preset: mainnet]                                                       OK
++ sanity check blobs (Deneb) [Preset: mainnet]                                               OK
 + sanity check data columns [Preset: mainnet]                                                OK
 + sanity check genesis roundtrip [Preset: mainnet]                                           OK
 + sanity check phase 0 blocks [Preset: mainnet]                                              OK

beacon_chain/beacon_chain_db.nim

@@ -1,5 +1,5 @@
 # beacon_chain
-# Copyright (c) 2018-2024 Status Research & Development GmbH
+# Copyright (c) 2018-2025 Status Research & Development GmbH
 # Licensed and distributed under either of
 #   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
 #   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
@@ -113,7 +113,7 @@ type
     keyValues: KvStoreRef # Random stuff using DbKeyKind - suitable for small values mainly!
     blocks: array[ConsensusFork, KvStoreRef] # BlockRoot -> TrustedSignedBeaconBlock
 
-    blobs: KvStoreRef # (BlockRoot -> BlobSidecar)
+    blobs: array[BlobFork, KvStoreRef] # (BlockRoot -> BlobSidecar)
 
     columns: KvStoreRef # (BlockRoot -> DataColumnSidecar)
 
@@ -587,7 +587,9 @@ proc new*(T: type BeaconChainDB,
       sealedPeriods: "lc_sealed_periods")).expectDb()
   static: doAssert LightClientDataFork.high == LightClientDataFork.Electra
 
-  var blobs = kvStore db.openKvStore("deneb_blobs").expectDb()
+  var blobs: array[BlobFork, KvStoreRef]
+  blobs[BlobFork.Deneb] = kvStore db.openKvStore("deneb_blobs").expectDb()
+  static: doAssert BlobFork.high == BlobFork.Deneb
 
   var columns: KvStoreRef
   if cfg.FULU_FORK_EPOCH != FAR_FUTURE_EPOCH:
@@ -798,8 +800,9 @@ proc close*(db: BeaconChainDB) =
   # Close things roughly in reverse order
   if not isNil(db.columns):
     discard db.columns.close()
-  if not isNil(db.blobs):
-    discard db.blobs.close()
+  for blobFork in BlobFork:
+    if not isNil(db.blobs[blobFork]):
+      discard db.blobs[blobFork].close()
   db.lcData.close()
   db.finalizedBlocks.close()
   discard db.summaries.close()
@@ -845,16 +848,20 @@ proc putBlock*(
     db.blocks[type(value).kind].putSZSSZ(value.root.data, value)
     db.putBeaconBlockSummary(value.root, value.message.toBeaconBlockSummary())
 
-proc putBlobSidecar*(
-    db: BeaconChainDB,
-    value: BlobSidecar) =
+proc putBlobSidecar*[T: ForkyBlobSidecar](
+    db: BeaconChainDB, value: T) =
   let block_root = hash_tree_root(value.signed_block_header.message)
-  db.blobs.putSZSSZ(blobkey(block_root, value.index), value)
+  db.blobs[T.kind].putSZSSZ(blobkey(block_root, value.index), value)
 
 proc delBlobSidecar*(
     db: BeaconChainDB,
     root: Eth2Digest, index: BlobIndex): bool =
-  db.blobs.del(blobkey(root, index)).expectDb()
+  var res = false
+  for blobFork in BlobFork:
+    if db.blobs[blobFork] == nil: continue
+    if db.blobs[blobFork].del(blobkey(root, index)).expectDb():
+      res = true
+  res
 
 proc putDataColumnSidecar*(
     db: BeaconChainDB,
@@ -1120,18 +1127,22 @@ proc getBlockSSZ*(
   withConsensusFork(fork):
     getBlockSSZ(db, key, data, consensusFork.TrustedSignedBeaconBlock)
 
-proc getBlobSidecarSZ*(db: BeaconChainDB, root: Eth2Digest, index: BlobIndex,
-                       data: var seq[byte]): bool =
+proc getBlobSidecarSZ*[T: ForkyBlobSidecar](
+    db: BeaconChainDB, root: Eth2Digest, index: BlobIndex,
+    data: var seq[byte]): bool =
+  if db.blobs[T.kind] == nil: return false
   let dataPtr = addr data # Short-lived
   func decode(data: openArray[byte]) =
     assign(dataPtr[], data)
-  db.blobs.get(blobkey(root, index), decode).expectDb()
+  db.blobs[T.kind].get(blobkey(root, index), decode).expectDb()
 
-proc getBlobSidecar*(db: BeaconChainDB, root: Eth2Digest, index: BlobIndex,
-                     value: var BlobSidecar): bool =
-  db.blobs.getSZSSZ(blobkey(root, index), value) == GetResult.found
+proc getBlobSidecar*[T: ForkyBlobSidecar](
+    db: BeaconChainDB, root: Eth2Digest, index: BlobIndex,
+    value: var T): bool =
+  if db.blobs[T.kind] == nil: return false
+  db.blobs[T.kind].getSZSSZ(blobkey(root, index), value) == GetResult.found
 
-proc getDataColumnSidecarSZ*(db: BeaconChainDB, root: Eth2Digest, 
+proc getDataColumnSidecarSZ*(db: BeaconChainDB, root: Eth2Digest,
                              index: ColumnIndex, data: var seq[byte]): bool =
   let dataPtr = addr data # Short-lived
   func decode(data: openArray[byte]) =

beacon_chain/beacon_chain_file.nim

@@ -1,5 +1,5 @@
 # beacon_chain
-# Copyright (c) 2018-2024 Status Research & Development GmbH
+# Copyright (c) 2018-2025 Status Research & Development GmbH
 # Licensed and distributed under either of
 #   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
 #   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
@@ -267,6 +267,13 @@ proc getBlockConsensusFork(header: ChainFileHeader): ConsensusFork =
   else:
     raiseAssert("Should not happen")
 
+proc getBlobConsensusFork(header: ChainFileHeader): ConsensusFork =
+  let hkind = unmaskKind(header.kind)
+  if int(hkind) in BlobForkCodeRange:
+    cast[ConsensusFork](ConsensusFork.Deneb.uint64 + hkind)
+  else:
+    raiseAssert("Should not happen")
+
 template isBlock(h: ChainFileHeader | ChainFileFooter): bool =
   let hkind = unmaskKind(h.kind)
   int(hkind) in BlockForkCodeRange
@@ -291,7 +298,7 @@ proc setTail*(chandle: var ChainFileHandle, bdata: BlockData) =
   chandle.data.tail = Opt.some(bdata)
 
 proc store*(chandle: ChainFileHandle, signedBlock: ForkedSignedBeaconBlock,
-            blobs: Opt[BlobSidecars]): Result[void, string] =
+            blobs: Opt[ForkedBlobSidecars]): Result[void, string] =
   let origOffset =
     updateFilePos(chandle.handle, 0'i64, SeekPosition.SeekEnd).valueOr:
       return err(ioErrorMsg(error))
@@ -321,10 +328,11 @@ proc store*(chandle: ChainFileHandle, signedBlock: ForkedSignedBeaconBlock,
         kind =
           getBlobChunkKind(signedBlock.kind, (index + 1) == len(blobSidecars))
         (data, plainSize) =
-          block:
-            let res = SSZ.encode(blob[])
+          withForkyBlob(blob):
+            let res = SSZ.encode(forkyBlob[])
             (snappy.encode(res), len(res))
-        slot = blob[].signed_block_header.message.slot
+        slot = withForkyBlob(blob):
+          forkyBlob[].signed_block_header.message.slot
         buffer = Chunk.init(kind, uint64(slot), uint32(plainSize), data)
 
       setFilePos(chandle.handle, 0'i64, SeekPosition.SeekEnd).isOkOr:
@@ -537,21 +545,26 @@ proc decodeBlock(
 proc decodeBlob(
     header: ChainFileHeader,
     data: openArray[byte]
-): Result[BlobSidecar, string] =
+): Result[ForkedBlobSidecar, string] =
   if header.plainSize > uint32(MaxChunkSize):
     return err("Size of blob is enormously big")
 
   let
+    fork = header.getBlobConsensusFork()
+    blobFork = blobForkAtConsensusFork(fork).valueOr:
+      return err("Fork does not support blobs")
     decompressed = snappy.decode(data, uint32(header.plainSize))
     blob =
       try:
-        SSZ.decode(decompressed, BlobSidecar)
+        withBlobFork(blobFork):
+          ForkedBlobSidecar.init(newClone(
+            SSZ.decode(decompressed, blobFork.BlobSidecar)))
       except SerializationError:
         return err("Incorrect blob format")
   ok(blob)
 
 proc getChainFileTail*(handle: IoHandle): Result[Opt[BlockData], string] =
-  var sidecars: BlobSidecars
+  var sidecars: ForkedBlobSidecars
   while true:
     let chunk =
       block:
@@ -565,7 +578,7 @@ proc getChainFileTail*(handle: IoHandle): Result[Opt[BlockData], string] =
         res.get()
     if chunk.header.isBlob():
       let blob = ? decodeBlob(chunk.header, chunk.data)
-      sidecars.add(newClone blob)
+      sidecars.add(blob)
     else:
       let blck = ? decodeBlock(chunk.header, chunk.data)
       return
@@ -594,7 +607,7 @@ proc getChainFileHead*(handle: IoHandle): Result[Opt[BlockData], string] =
         ? decodeBlock(chunk.header, chunk.data)
     blob =
       block:
-        var sidecars: BlobSidecars
+        var sidecars: ForkedBlobSidecars
         block mainLoop:
           while true:
             offset = getFilePos(handle).valueOr:
@@ -609,14 +622,14 @@ proc getChainFileHead*(handle: IoHandle): Result[Opt[BlockData], string] =
                 res.get()
             if chunk.header.isBlob():
               let blob = ? decodeBlob(chunk.header, chunk.data)
-              sidecars.add(newClone blob)
+              sidecars.add(blob)
             else:
               break mainLoop
 
         if len(sidecars) > 0:
           Opt.some(sidecars)
         else:
-          Opt.none(BlobSidecars)
+          Opt.none(ForkedBlobSidecars)
 
   if not(endOfFile):
     setFilePos(handle, offset, SeekPosition.SeekBegin).isOkOr:

beacon_chain/consensus_object_pools/blob_quarantine.nim

@@ -1,5 +1,5 @@
 # beacon_chain
-# Copyright (c) 2018-2024 Status Research & Development GmbH
+# Copyright (c) 2018-2025 Status Research & Development GmbH
 # Licensed and distributed under either of
 #   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
 #   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
@@ -21,8 +21,8 @@ const
 
 type
   BlobQuarantine* = object
-    blobs*:
-      OrderedTable[(Eth2Digest, BlobIndex, KzgCommitment), ref BlobSidecar]
+    blobs*: OrderedTable[
+      (Eth2Digest, BlobIndex, KzgCommitment), ForkedBlobSidecar]
     onBlobSidecarCallback*: OnBlobSidecarCallback
 
   BlobFetchRecord* = object
@@ -38,7 +38,7 @@ func shortLog*(x: seq[BlobIndex]): string =
 func shortLog*(x: seq[BlobFetchRecord]): string =
   "[" & x.mapIt(shortLog(it.block_root) & shortLog(it.indices)).join(", ") & "]"
 
-func put*(quarantine: var BlobQuarantine, blobSidecar: ref BlobSidecar) =
+func put*(quarantine: var BlobQuarantine, blobSidecar: ForkedBlobSidecar) =
   if quarantine.blobs.lenu64 >= MaxBlobs:
     # FIFO if full. For example, sync manager and request manager can race to
     # put blobs in at the same time, so one gets blob insert -> block resolve
@@ -53,48 +53,66 @@ func put*(quarantine: var BlobQuarantine, blobSidecar: ref BlobSidecar) =
       oldest_blob_key = k
       break
     quarantine.blobs.del oldest_blob_key
-  let block_root = hash_tree_root(blobSidecar.signed_block_header.message)
-  discard quarantine.blobs.hasKeyOrPut(
-    (block_root, blobSidecar.index, blobSidecar.kzg_commitment), blobSidecar)
+  withForkyBlob(blobSidecar):
+    let block_root = hash_tree_root(forkyBlob[].signed_block_header.message)
+    discard quarantine.blobs.hasKeyOrPut(
+      (block_root, forkyBlob[].index, forkyBlob[].kzg_commitment), blobSidecar)
+
+func put*(quarantine: var BlobQuarantine, blobSidecar: ref ForkyBlobSidecar) =
+  quarantine.put(ForkedBlobSidecar.init(blobSidecar))
 
 func hasBlob*(
     quarantine: BlobQuarantine,
     slot: Slot,
     proposer_index: uint64,
     index: BlobIndex): bool =
-  for blob_sidecar in quarantine.blobs.values:
-    template block_header: untyped = blob_sidecar.signed_block_header.message
-    if block_header.slot == slot and
-        block_header.proposer_index == proposer_index and
-        blob_sidecar.index == index:
-      return true
+  for blobSidecar in quarantine.blobs.values:
+    withForkyBlob(blobSidecar):
+      template block_header: untyped = forkyBlob[].signed_block_header.message
+      if block_header.slot == slot and
+          block_header.proposer_index == proposer_index and
+          forkyBlob[].index == index:
+        return true
   false
 
 func popBlobs*(
     quarantine: var BlobQuarantine, digest: Eth2Digest,
-    blck: deneb.SignedBeaconBlock | electra.SignedBeaconBlock |
-          fulu.SignedBeaconBlock):
-    seq[ref BlobSidecar] =
-  var r: seq[ref BlobSidecar] = @[]
+    blck:
+      deneb.SignedBeaconBlock |
+      electra.SignedBeaconBlock |
+      fulu.SignedBeaconBlock): auto =
+  const blobFork = blobForkAtConsensusFork(typeof(blck).kind).expect("Blobs OK")
+  type ResultType = blobFork.BlobSidecars
+  var r: ResultType = @[]
   for idx, kzg_commitment in blck.message.body.blob_kzg_commitments:
-    var b: ref BlobSidecar
+    var b: ForkedBlobSidecar
     if quarantine.blobs.pop((digest, BlobIndex idx, kzg_commitment), b):
-      r.add(b)
+      # It was already verified that the blob is linked to `blck`.
+      # Therefore, we can assume that `BlobFork` is correct.
+      doAssert b.kind == blobFork,
+        "Must verify blob inclusion proof before `BlobQuarantine.put`"
+      r.add(b.forky(blobFork))
   r
 
-func hasBlobs*(quarantine: BlobQuarantine,
-    blck: deneb.SignedBeaconBlock | electra.SignedBeaconBlock |
-          fulu.SignedBeaconBlock): bool =
-    # Having a fulu SignedBeaconBlock is incorrect atm, but
-    # shall be fixed once data columns are rebased to fulu
+func hasBlobs*(
+    quarantine: BlobQuarantine,
+    blck:
+      deneb.SignedBeaconBlock |
+      electra.SignedBeaconBlock |
+      fulu.SignedBeaconBlock): bool =
+  # Having a fulu SignedBeaconBlock is incorrect atm, but
+  # shall be fixed once data columns are rebased to fulu
   for idx, kzg_commitment in blck.message.body.blob_kzg_commitments:
     if (blck.root, BlobIndex idx, kzg_commitment) notin quarantine.blobs:
       return false
   true
 
-func blobFetchRecord*(quarantine: BlobQuarantine,
-    blck: deneb.SignedBeaconBlock | electra.SignedBeaconBlock |
-          fulu.SignedBeaconBlock): BlobFetchRecord =
+func blobFetchRecord*(
+    quarantine: BlobQuarantine,
+    blck:
+      deneb.SignedBeaconBlock |
+      electra.SignedBeaconBlock |
+      fulu.SignedBeaconBlock): BlobFetchRecord =
   var indices: seq[BlobIndex]
   for i in 0..<len(blck.message.body.blob_kzg_commitments):
     let idx = BlobIndex(i)

beacon_chain/consensus_object_pools/block_clearance.nim

@@ -1,5 +1,5 @@
 # beacon_chain
-# Copyright (c) 2018-2024 Status Research & Development GmbH
+# Copyright (c) 2018-2025 Status Research & Development GmbH
 # Licensed and distributed under either of
 #   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
 #   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
@@ -535,7 +535,8 @@ proc addBackfillBlockData*(
 
     if bdata.blob.isSome():
       for blob in bdata.blob.get():
-        dag.db.putBlobSidecar(blob[])
+        withForkyBlob(blob):
+          dag.db.putBlobSidecar(forkyBlob[])
 
     type Trusted = typeof forkyBlck.asTrusted()
 

beacon_chain/consensus_object_pools/block_pools_types.nim

@@ -1,5 +1,5 @@
 # beacon_chain
-# Copyright (c) 2018-2024 Status Research & Development GmbH
+# Copyright (c) 2018-2025 Status Research & Development GmbH
 # Licensed and distributed under either of
 #   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
 #   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
@@ -287,7 +287,7 @@ type
 
   BlockData* = object
     blck*: ForkedSignedBeaconBlock
-    blob*: Opt[BlobSidecars]
+    blob*: Opt[ForkedBlobSidecars]
 
   OnBlockAdded*[T: ForkyTrustedSignedBeaconBlock] = proc(
     blckRef: BlockRef, blck: T, epochRef: EpochRef,