Stateless: Witness generation, clearing of witness keys and db persist and get helpers (#3458)

* Implement function to build execution witness from witness keys and proofs from database.

* Clear witness keys in ledger after processing each block.

* Add functions to persist and get witness by block hash.

* Rename/restructure witness files.

Author: bhartnett

execution_chain/core/executor/process_block.nim

@@ -198,6 +198,7 @@ proc procBlkEpilogue(
     db.persist(
       clearEmptyAccount = vmState.com.isSpuriousOrLater(header.number),
       clearCache = true,
+      clearWitness = vmState.com.statelessProviderEnabled
     )
 
   var

execution_chain/db/core_db/core_apps.nim

@@ -20,6 +20,7 @@ import
   stew/byteutils,
   results,
   "../.."/[constants],
+  "../.."/stateless/witness_types,
   ".."/[aristo, storage_types],
   "."/base
 
@@ -622,6 +623,17 @@ proc persistUncles*(db: CoreDbTxRef, uncles: openArray[Header]): Hash32 =
     warn "persistUncles()", unclesHash=result, error=($$error)
     return EMPTY_ROOT_HASH
 
+proc persistWitness*(db: CoreDbTxRef, blockHash: Hash32, witness: Witness): Result[void, string] =
+  db.put(blockHashToWitnessKey(blockHash).toOpenArray, witness.encode()).isOkOr:
+    return err("persistWitness: " & $$error)
+  ok()
+
+proc getWitness*(db: CoreDbTxRef, blockHash: Hash32): Result[Witness, string] =
+  let witnessBytes = db.get(blockHashToWitnessKey(blockHash).toOpenArray).valueOr:
+    return err("getWitness: " & $$error)
+
+  Witness.decode(witnessBytes)
+
 # ------------------------------------------------------------------------------
 # End
 # ------------------------------------------------------------------------------

execution_chain/db/ledger.nim

@@ -703,9 +703,16 @@ proc clearEmptyAccounts(ac: LedgerRef) =
     ac.deleteEmptyAccount(RIPEMD_ADDR)
     ac.ripemdSpecial = false
 
+template getWitnessKeys*(ac: LedgerRef): WitnessTable =
+  ac.witnessKeys
+
+template clearWitnessKeys*(ac: LedgerRef) =
+  ac.witnessKeys.clear()
+
 proc persist*(ac: LedgerRef,
               clearEmptyAccount: bool = false,
-              clearCache = false) =
+              clearCache = false,
+              clearWitness = false) =
   const info = "persist(): "
 
   # make sure all savepoint already committed
@@ -757,6 +764,9 @@ proc persist*(ac: LedgerRef,
 
   ac.isDirty = false
 
+  if clearWitness:
+    ac.clearWitnessKeys()
+
 iterator addresses*(ac: LedgerRef): Address =
   # make sure all savepoint already committed
   doAssert(ac.savePoint.parentSavepoint.isNil)
@@ -878,12 +888,6 @@ proc getStorageProof*(ac: LedgerRef, address: Address, slots: openArray[UInt256]
 
   storageProof
 
-func getWitnessKeys*(ac: LedgerRef): WitnessTable =
-  ac.witnessKeys
-
-proc clearWitnessKeys*(ac: LedgerRef) =
-  ac.witnessKeys.clear()
-
 # ------------------------------------------------------------------------------
 # Public virtual read-only methods
 # ------------------------------------------------------------------------------

execution_chain/db/storage_types.nim

@@ -28,6 +28,7 @@ type
     fcState = 9
     beaconHeader = 10
     wdKey = 11
+    witness = 12
 
   DbKey* = object
     # The first byte stores the key type. The rest are key-specific values
@@ -104,6 +105,11 @@ func withdrawalsKey*(h: Hash32): DbKey {.inline.} =
   result.data[1 .. 32] = h.data
   result.dataEndPos = uint8 32
 
+func blockHashToWitnessKey*(h: Hash32): DbKey {.inline.} =
+  result.data[0] = byte ord(witness)
+  result.data[1 .. 32] = h.data
+  result.dataEndPos = uint8 32
+
 template toOpenArray*(k: DbKey): openArray[byte] =
   k.data.toOpenArray(0, int(k.dataEndPos))
 

execution_chain/stateless/witness_generation.nim

@@ -0,0 +1,62 @@
+# Nimbus
+# Copyright (c) 2025 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+{.push raises: [].}
+
+import
+  std/[tables, sets],
+  eth/common,
+  ../db/ledger,
+  ./witness_types
+
+export
+  common,
+  ledger,
+  witness_types
+
+proc build*(
+    T: type Witness,
+    ledger: LedgerRef,
+    codes: var seq[seq[byte]]): T =
+  var
+    witness = Witness.init()
+    addedStateHashes = initHashSet[Hash32]()
+    addedCodeHashes = initHashSet[Hash32]()
+
+  for key, codeTouched in ledger.getWitnessKeys():
+    let (adr, maybeSlot) = key
+    if maybeSlot.isSome():
+      let slot = maybeSlot.get()
+      witness.addKey(slot.toBytesBE())
+
+      let proofs = ledger.getStorageProof(key.address, @[slot])
+      doAssert(proofs.len() == 1)
+      for trieNode in proofs[0]:
+        let nodeHash = keccak256(trieNode)
+        if nodeHash notin addedStateHashes:
+          witness.addState(trieNode)
+          addedStateHashes.incl(nodeHash)
+    else:
+      witness.addKey(key.address.data())
+
+      let proof = ledger.getAccountProof(key.address)
+      for trieNode in proof:
+        let nodeHash = keccak256(trieNode)
+        if nodeHash notin addedStateHashes:
+          witness.addState(trieNode)
+          addedStateHashes.incl(nodeHash)
+
+    if codeTouched:
+      let (codeHash, code) = ledger.getCode(key.address, returnHash = true)
+      if codeHash != EMPTY_CODE_HASH and codeHash notin addedCodeHashes:
+        codes.add(code.bytes)
+        witness.addCodeHash(codeHash)
+        addedCodeHashes.incl(codeHash)
+
+  witness

execution_chain/stateless/witness.nim renamed to execution_chain/stateless/witness_types.nim

@@ -7,6 +7,8 @@
 # This file may not be copied, modified, or distributed except according to
 # those terms.
 
+{.push raises: [].}
+
 import
   eth/common,
   eth/rlp,
@@ -16,24 +18,57 @@ export
   common,
   results
 
-{.push raises: [].}
-
 type
+  Witness* = object
+    state*: seq[seq[byte]] # MPT trie nodes accessed while executing the block.
+    keys*: seq[seq[byte]] # Ordered list of access keys (address bytes or storage slots bytes).
+    codeHashes*: seq[Hash32] # Code hashes of the bytecode required by the witness.
+    headerHashes*: seq[Hash32] # Hashes of block headers which are required by the witness.
+
   ExecutionWitness* = object
     state*: seq[seq[byte]] # MPT trie nodes accessed while executing the block.
-    codes*: seq[seq[byte]] # Contract bytecodes read while executing the block.
     keys*: seq[seq[byte]] # Ordered list of access keys (address bytes or storage slots bytes).
+    codes*: seq[seq[byte]] # Contract bytecodes read while executing the block.
     headers*: seq[Header] # Block headers required for proving correctness of stateless execution.
       # Stores the parent block headers needed to verify that the state reads are correct with respect
       # to the pre-state root.
 
+func init*(
+    T: type Witness,
+    state = newSeq[seq[byte]](),
+    keys = newSeq[seq[byte]](),
+    codeHashes = newSeq[Hash32](),
+    headerHashes = newSeq[Hash32]()): T =
+  Witness(state: state, keys: keys, headerHashes: headerHashes)
+
+template addState*(witness: var Witness, trieNode: seq[byte]) =
+  witness.state.add(trieNode)
+
+template addKey*(witness: var Witness, key: openArray[byte]) =
+  witness.keys.add(@key)
+
+template addCodeHash*(witness: var Witness, codeHash: Hash32) =
+  witness.codeHashes.add(codeHash)
+
+template addHeaderHash*(witness: var Witness, headerHash: Hash32) =
+  witness.headerHashes.add(headerHash)
+
+func encode*(witness: Witness): seq[byte] =
+  rlp.encode(witness)
+
+func decode*(T: type Witness, witnessBytes: openArray[byte]): Result[T, string] =
+  try:
+    ok(rlp.decode(witnessBytes, T))
+  except RlpError as e:
+    err(e.msg)
+
 func init*(
     T: type ExecutionWitness,
     state = newSeq[seq[byte]](),
-    codes = newSeq[seq[byte]](),
     keys = newSeq[seq[byte]](),
+    codes = newSeq[seq[byte]](),
     headers = newSeq[Header]()): T =
-  ExecutionWitness(state: state, codes: codes, keys: keys, headers: headers)
+  ExecutionWitness(state: state, keys: keys, codes: codes, headers: headers)
 
 template addState*(witness: var ExecutionWitness, trieNode: seq[byte]) =
   witness.state.add(trieNode)

tests/all_tests.nim

@@ -36,7 +36,8 @@ import
     test_txpool,
     test_networking,
     test_pooled_tx,
-    test_stateless_witness,
+    test_stateless_witness_types,
+    test_stateless_witness_generation,
     # These two suites are much slower than all the rest, so run them last
     test_blockchain_json,
     test_generalstate_json,

tests/test_ledger.nim

@@ -754,6 +754,29 @@ proc runLedgerBasicOperationsTests() =
         keysList[4][0].slot == Opt.none(UInt256)
         keysList[4][1] == false
 
+    test "Witness keys - Clear cache":
+      var
+        ac = LedgerRef.init(memDB.baseTxFrame(), false, collectWitness = true)
+        addr1 = initAddr(1)
+        addr2 = initAddr(2)
+        addr3 = initAddr(3)
+        slot1 = 1.u256
+
+      discard ac.getAccount(addr1)
+      discard ac.getCode(addr2)
+      discard ac.getCode(addr1)
+      discard ac.getStorage(addr2, slot1)
+      discard ac.getStorage(addr1, slot1)
+      discard ac.getStorage(addr2, slot1)
+      discard ac.getAccount(addr3)
+
+      check ac.getWitnessKeys().len() == 5
+
+      ac.persist() # persist should not clear the witness keys by default
+      check ac.getWitnessKeys().len() == 5
+
+      ac.persist(clearWitness = true)
+      check ac.getWitnessKeys().len() == 0
 
 # ------------------------------------------------------------------------------
 # Main function(s)