fix(ethrex): realign DB config with ethrex v23.0.0

CHANGELOG.md

@@ -17,6 +17,15 @@
   no longer written.
 
 ### Changed
+- **ethrex tracking bumped v16.0.0 → v23.0.0.** Adds the `bad_blocks`
+  column family (21 CFs; upstream added it in v22.0.0, ethrex#6948 —
+  previously ethrex created it itself on first boot) and writes
+  `schema_version: 3` to `metadata.json` (ethrex's value since v16 —
+  writing 2 sent every first boot through the migration branch, which
+  also rewrites `metadata.json`, mutating the generated datadir). Boot
+  image and golden fixture repinned to 23.0.0; every state-bearing CF
+  is byte-identical to the v16 dump, so the state root and the Go
+  trie/code codecs are unaffected.
 - **Nethermind writer runs Nethermind's own RocksDB configuration.** All
   8 databases (and every column family) now parse the verbatim DbConfig
   option strings through RocksDB's own parser instead of hand-tuned

Dockerfile.ethrex

@@ -4,7 +4,7 @@
 #
 # RocksDB strategy: build from source. Pinned to RocksDB 10.10.1 to match
 # `grocksdb v1.10.8`'s C-binding expectations (its build.sh pins this exact
-# version). ethrex uses a single RocksDB instance with 20 column families;
+# version). ethrex uses a single RocksDB instance with 21 column families;
 # the same grocksdb pairing used for besu and nethermind applies here.
 #
 # Why 10.10.1 specifically: grocksdb v1.10.8's C bindings reference
@@ -22,8 +22,8 @@
 #       grocksdb v1.10.7 → RocksDB 10.9.1
 #       grocksdb v1.10.8 → RocksDB 10.10.1  ← we use this
 #
-#   * On-disk layout: single RocksDB at <dbPath> with 20 column families.
-#     Sidecar files: metadata.json (schema_version=2) and ethrex-genesis.json
+#   * On-disk layout: single RocksDB at <dbPath> with 21 column families.
+#     Sidecar files: metadata.json (schema_version=3) and ethrex-genesis.json
 #     (full genesis JSON for `ethrex --network <path>`).
 #
 # Run:

README.md

@@ -29,7 +29,7 @@
 
 ## Why
 
-You want a pre-populated Ethereum database that a client can boot against directly — for cross-client determinism tests, devnet bootstrapping, EIP-7702 / ERC-20 fixtures, or state-bloat experiments. The alternative (run the client's `init` against a genesis with millions of `alloc` entries) is slow and client-specific. State Actor writes each client's on-disk format directly: Pebble for geth, MDBX + RocksDB + nippy-jar for reth, single RocksDB + 8 Bonsai column families for besu, seven RocksDB instances + a flat column DB for nethermind, single RocksDB + 20 column families for ethrex, Erigon v3 flat snapshot `.kv` files + a minimal MDBX for erigon.
+You want a pre-populated Ethereum database that a client can boot against directly — for cross-client determinism tests, devnet bootstrapping, EIP-7702 / ERC-20 fixtures, or state-bloat experiments. The alternative (run the client's `init` against a genesis with millions of `alloc` entries) is slow and client-specific. State Actor writes each client's on-disk format directly: Pebble for geth, MDBX + RocksDB + nippy-jar for reth, single RocksDB + 8 Bonsai column families for besu, seven RocksDB instances + a flat column DB for nethermind, single RocksDB + 21 column families for ethrex, Erigon v3 flat snapshot `.kv` files + a minimal MDBX for erigon.
 
 Three flags carry most of the weight: `--client` (which client's format to write), `--spec` (concrete entities to include, declared in YAML), `--target-size` (the DB-size budget; auto-fills mainnet-shaped 20 % / 10 % / 70 % across account-trie / bytecode / storage up to the cap). One of `--spec` or `--target-size` is required; everything else has a sane default.
 

client/ethrex/chainspec.go

@@ -32,7 +32,7 @@ const GenesisFileName = "ethrex-genesis.json"
 // MetadataFileName is the on-disk filename of the schema metadata file.
 const MetadataFileName = "metadata.json"
 
-// WriteStoreMetadata writes {"schema_version": 2} to <dbPath>/metadata.json.
+// WriteStoreMetadata writes {"schema_version": 3} to <dbPath>/metadata.json.
 // This file is REQUIRED by ethrex's Store::new: when the datadir is non-empty
 // but has no metadata.json, ethrex returns NotFoundDBVersion and refuses to
 // boot. state-actor must write it because it pre-fills the dir with SST files.

client/ethrex/chainspec_test.go

@@ -13,7 +13,7 @@ import (
 )
 
 // TestWriteStoreMetadata verifies that WriteStoreMetadata writes a valid JSON
-// file with schema_version=2.
+// file with schema_version=3.
 func TestWriteStoreMetadata(t *testing.T) {
 	dir := t.TempDir()
 	if err := ethrex.WriteStoreMetadata(dir); err != nil {
@@ -31,8 +31,8 @@ func TestWriteStoreMetadata(t *testing.T) {
 	if err := json.Unmarshal(data, &parsed); err != nil {
 		t.Fatalf("unmarshal metadata.json: %v", err)
 	}
-	if parsed.SchemaVersion != 2 {
-		t.Errorf("schema_version: got %d, want 2", parsed.SchemaVersion)
+	if parsed.SchemaVersion != 3 {
+		t.Errorf("schema_version: got %d, want 3", parsed.SchemaVersion)
 	}
 }
 

client/ethrex/dbs_cgo.go

@@ -49,9 +49,11 @@ func ethrexStateCFLevelBaseBytes() uint64 {
 
 // ethrexBlockCacheBytes is the shared LRU block cache across all CFs.
 //
-// ethrex's own crates/storage/backend/rocksdb.rs uses 4 GiB. This writer uses
-// far less BY DESIGN: a block cache only accelerates reads, and generation is
-// write-only until Close()'s CompactRange. Cache size is a process-runtime
+// ethrex (crates/storage/backend/rocksdb.rs) defaults to 12 GiB
+// (DEFAULT_ROCKSDB_BLOCK_CACHE_SIZE_BYTES, overridable with
+// --rocksdb.block-cache-size). This writer uses far less BY DESIGN: a
+// block cache only accelerates reads, and generation is write-only
+// until Close()'s CompactRange. Cache size is a process-runtime
 // knob — it does not change a single byte of the produced DB — so mirroring
 // ethrex here would buy representativeness that does not exist while costing
 // RAM that demonstrably does.
@@ -157,9 +159,10 @@ const (
 	cfIdxMiscValues           = 17
 	cfIdxExecutionWitnesses   = 18
 	cfIdxBlockAccessLists     = 19
+	cfIdxBadBlocks            = 20
 )
 
-// ethrexDB holds the open grocksdb handle and the 20 CF handles.
+// ethrexDB holds the open grocksdb handle and the 21 CF handles.
 type ethrexDB struct {
 	db     *grocksdb.DB
 	cfs    []*grocksdb.ColumnFamilyHandle
@@ -182,8 +185,9 @@ type ethrexDB struct {
 // process-runtime knobs that cannot change the compacted output: L0
 // compaction triggers (ethrex 4/20/36, maxed here to avoid stalls during
 // bulk import), state-CF memtables (256 MiB × 4 vs ethrex's 512 MiB × 6 —
-// see the state-CF case), and the block cache (512 MiB vs ethrex's 4 GiB —
-// see ethrexBlockCacheBytes). Close() runs CompactRange afterward, rewriting
+// see the state-CF case), and the block cache (512 MiB vs ethrex's
+// 12 GiB default — see ethrexBlockCacheBytes). Close() runs
+// CompactRange afterward, rewriting
 // every SST with the same compression/block/bloom options, so the final
 // on-disk shape matches ethrex regardless.
 func openEthrexDB(dbPath string) (*ethrexDB, error) {
@@ -210,17 +214,17 @@ func openEthrexDB(dbPath string) (*ethrexDB, error) {
 		return nil, fmt.Errorf("ethrex: mkdir: %w", err)
 	}
 
-	// The 20 named ethrex CFs, plus RocksDB's implicit "default" CF appended
-	// LAST (index 20). RocksDB always creates "default" on a fresh DB, and an
+	// The 21 named ethrex CFs, plus RocksDB's implicit "default" CF appended
+	// LAST (index 21). RocksDB always creates "default" on a fresh DB, and an
 	// open call must account for every existing CF or it errors with "you have
 	// to open all column families". Appending it keeps the cfIdx* constants
-	// (0..19) aligned with Tables; cfs[20] (default) is created but never
+	// (0..20) aligned with Tables; cfs[21] (default) is created but never
 	// written. Mirrors besu's explicit CFDefault inclusion.
 	cfNames := make([]string, 0, len(ethrexinternal.Tables)+1)
 	cfNames = append(cfNames, ethrexinternal.Tables...)
 	cfNames = append(cfNames, "default")
 
-	// Shared 4 GiB LRU block cache across all CFs (ethrex rocksdb.rs).
+	// Shared LRU block cache across all CFs (see ethrexBlockCacheBytes).
 	cache := grocksdb.NewLRUCache(ethrexBlockCacheBytes)
 
 	cfOpts := make([]*grocksdb.Options, len(cfNames))
@@ -318,6 +322,10 @@ func openEthrexDB(dbPath string) (*ethrexDB, error) {
 			opts.SetTargetFileSizeBase(256 << 20)
 			bbto.SetBlockSize(32 << 10)
 		default:
+			// Also covers transaction_locations, whose ethrex arm carries the
+			// same values plus a merge operator — omitted here: state-actor
+			// writes no rows there, and a CF created without one reopens
+			// cleanly with one registered.
 			opts.SetWriteBufferSize(64 << 20)
 			opts.SetMaxWriteBufferNumber(3)
 			opts.SetTargetFileSizeBase(128 << 20)

client/ethrex/doc.go

@@ -2,7 +2,7 @@
 //
 // # On-disk layout
 //
-// A single RocksDB instance at <dbPath> with 20 column families (Tables in
+// A single RocksDB instance at <dbPath> with 21 column families (Tables in
 // internal/ethrex/constants.go), all declared at open time. CFs written at genesis:
 //   - account_trie_nodes / storage_trie_nodes: MPT structural + leaf-NODE-RLP rows
 //     (storage rows are address-prefixed)
@@ -13,7 +13,7 @@
 //   - headers / bodies / block_numbers / canonical_block_hashes: genesis block
 //     (canonical_block_hashes[0] is the boot gate)
 //
-// Sidecars next to the DB: metadata.json ({"schema_version": 2}, required by
+// Sidecars next to the DB: metadata.json ({"schema_version": 3}, required by
 // ethrex Store::new) and ethrex-genesis.json (for `--network`).
 //
 // # Flat-KV (snap-synced-state) layer
@@ -33,9 +33,9 @@
 // # Pinned releases
 //
 // Golden test: byte-exact vs testdata/genesis_dump.json, regenerated at ethrex
-// v16.0.0; the state-bearing CFs are byte-identical v13–v16. E2e boot test
-// (e2e_test.go) pins the same v16.0.0 image (first release with
-// --skip-genesis-validation, lambdaclass/ethrex#6783).
+// v23.0.0; the state-bearing CFs are byte-identical v13–v23. E2e boot test
+// (e2e_test.go) pins the same v23.0.0 image. --skip-genesis-validation, which
+// the boot path requires, landed in v16.0.0 (lambdaclass/ethrex#6783).
 //
 // # Build
 //

client/ethrex/e2e_test.go

@@ -30,10 +30,11 @@ import (
 // against, digest-pinned for reproducibility. Override with
 // ETHREX_IMAGE=ghcr.io/lambdaclass/ethrex:<tag> to test a specific release.
 //
-// Official release v16.0.0 (ghcr tag 16.0.0, published 2026-06-08), the first
-// stable release >15.0.0 to include --skip-genesis-validation
-// (lambdaclass/ethrex#6783, merged 2026-06-04), which the boot phase requires.
-const pinnedEthrexImage = "ghcr.io/lambdaclass/ethrex:16.0.0@sha256:1873c36dcda955df9e5209b56e6fe47db67fb26abf00506de7695ba4683a1c5a"
+// Official release v23.0.0 (ghcr tag 23.0.0, published 2026-07-27). Boot
+// requires --skip-genesis-validation (lambdaclass/ethrex#6783, ≥v16.0.0).
+// This pin is also the source of internal/ethrex's Tables and of
+// testdata/genesis_dump.json; move all three together.
+const pinnedEthrexImage = "ghcr.io/lambdaclass/ethrex:23.0.0@sha256:1cbf2c4b498efcc71dc776a130cf5eed3f15d100896a18f05b6fa426ff0e7fc5"
 
 func ethrexImageRef() string {
 	if v := os.Getenv("ETHREX_IMAGE"); v != "" {

client/ethrex/run.go

@@ -24,7 +24,7 @@ var errNotImplemented = errors.New(
 // It delegates to the build-tag-gated runImpl:
 //
 //   - Built with `-tags cgo_ethrex` (Docker only): runImpl in run_cgo.go opens
-//     one grocksdb instance with 20 column families, drives entitygen →
+//     one grocksdb instance with 21 column families, drives entitygen →
 //     ethrex.Builder → grocksdb writes, assembles the genesis block.
 //   - Built without the tag (local default): runImpl in run_stub.go returns
 //     errNotImplemented.

docs/ARCHITECTURE.md

@@ -98,7 +98,7 @@ State Actor generates Ethereum state in three phases:
 │  │  reth: MDBX state tables + RocksDB history + nippy-jar static_files│ │
 │  │  besu: single RocksDB w/ 8 Bonsai column families + chainspec.json │ │
 │  │  nethermind: 7 RocksDB + flat column DB + parity chainspec sidecar │ │
-│  │  ethrex: single RocksDB w/ 20 CFs + metadata.json + genesis sidecar│ │
+│  │  ethrex: single RocksDB w/ 21 CFs + metadata.json + genesis sidecar│ │
 │  │  erigon: Erigon v3 flat .kv snapshots + minimal MDBX               │ │
 │  └─────────────────────────────────────────────────────────────────────┘ │
 └──────────────────────────────────────────────────────────────────────────┘
@@ -227,7 +227,7 @@ configurable worker pool / batch size at the generator level.
   under `<db>/flat` (the flat backend Nethermind >= 1.39.0 serves); periodic
   `dirSize` sample (every 100 contracts) drives the target-size stop.
 - **ethrex** (`client/ethrex/run_cgo.go`): cgo + grocksdb. Single
-  RocksDB with 20 column families. Account and storage trie nodes
+  RocksDB with 21 column families. Account and storage trie nodes
   are encoded via `internal/ethrex`'s path-keyed trie codec (two rows
   per leaf: one full-path row, one nibble-path row). Writes
   `metadata.json` + `ethrex-genesis.json` sidecars. Behind the
@@ -359,10 +359,10 @@ Today's client adapters:
   `internal/neth/flat`) that Nethermind ≥ 1.39.0 serves as its flat backend,
   and a parity-format chainspec sidecar. Behind the `cgo_neth` build tag.
 - `client/ethrex/` — cgo + grocksdb writer producing a single RocksDB
-  with 20 column families (full list in `internal/ethrex/constants.go`)
+  with 21 column families (full list in `internal/ethrex/constants.go`)
   using ethrex's own path-keyed trie codec (`internal/ethrex/`). Two
   rows written per leaf (full-path + nibble-path). Sidecars:
-  `metadata.json` (schema_version=2) and `ethrex-genesis.json` (full
+  `metadata.json` (schema_version=3) and `ethrex-genesis.json` (full
   genesis JSON for `ethrex --network <path>`). Behind the `cgo_ethrex`
   build tag.
 - `client/erigon/` — cgo + mdbx-go writer producing Erigon v3 flat
@@ -399,7 +399,7 @@ state-actor/
 │   ├── reth/                        # cgo + libmdbx writer (cgo_reth build tag)
 │   ├── besu/                        # cgo + librocksdb writer (cgo_besu build tag)
 │   ├── nethermind/                  # cgo + grocksdb writer (cgo_neth build tag)
-│   ├── ethrex/                      # cgo + grocksdb writer, 20 CFs (cgo_ethrex build tag)
+│   ├── ethrex/                      # cgo + grocksdb writer, 21 CFs (cgo_ethrex build tag)
 │   └── erigon/                      # cgo + mdbx-go writer, Erigon v3 flat .kv (cgo_erigon build tag)
 ├── generator/                       # Core generation pipeline + Writer interface
 ├── genesis/                         # Client-neutral chainspec types + builder

docs/RUNBOOK.md

@@ -304,12 +304,14 @@ docker run --rm \
 
 **On-disk layout:**
 
-- `/data/` — single RocksDB instance with 20 column families (see `internal/ethrex/constants.go` for the full list)
-- `/data/metadata.json` — `{"schema_version": 2}`, required by ethrex `Store::new`
+- `/data/` — single RocksDB instance with 21 column families (see `internal/ethrex/constants.go` for the full list)
+- `/data/metadata.json` — `{"schema_version": 3}`, required by ethrex `Store::new`
 - `/data/ethrex-genesis.json` — full genesis JSON; pass via `--network` when booting
 
 **Boot path.** ethrex's `add_initial_state` short-circuits when `canonical_block_hashes[0]` already resolves to a matching genesis header hash — state-actor writes that row, so ethrex skips state-trie recomputation at boot. Required boot flags (validated by the e2e suite, Phase 4): `--network <ethrex-genesis.json>`, `--datadir <dir>`, `--skip-genesis-validation` (trust the written stateRoot rather than recompute from the empty-alloc sidecar; needs lambdaclass/ethrex#6783, in releases ≥ v16.0.0), and `--syncmode full`. The `--syncmode full` flag is mandatory for engine-driven block production: in the default snap mode ethrex's fork-choice handler returns `SYNCING` with a null `payloadId` for every `engine_forkchoiceUpdated`, so the mock CL can never obtain a payload to build. The pattern follows the besu/nethermind Engine API approach: boot the node, then drive blocks via `engine_forkchoiceUpdated` (ethrex also mandates an authrpc JWT, signed by the driver).
 
+**Memory at boot.** ethrex's shared RocksDB block cache defaults to 12 GiB and holds index + bloom-filter blocks. The cache fills lazily, but on a large DB a memory-capped container can be OOM-killed as it fills — cap it with `--rocksdb.block-cache-size <bytes>` on memory-constrained hosts. Boot-side knob only; no bearing on the bytes state-actor writes.
+
 **Verify.**
 
 ```bash

docs/SKILL.md

@@ -101,7 +101,7 @@ See [`RUNBOOK.md#geth`](RUNBOOK.md#geth) for the boot command, and [`client/geth
 go run . --db=/tmp/sa-reth --client=reth --target-size=100MB        # MDBX + RocksDB + static files
 go run . --db=/tmp/sa-besu --client=besu --target-size=100MB        # single RocksDB, 8 Bonsai CFs
 go run . --db=/tmp/sa-neth --client=nethermind --target-size=100MB  # 7 RocksDB + flat column DB
-go run . --db=/tmp/sa-ethrex --client=ethrex --target-size=100MB   # single RocksDB, 20 CFs
+go run . --db=/tmp/sa-ethrex --client=ethrex --target-size=100MB   # single RocksDB, 21 CFs
 go run . --db=/tmp/sa-erigon --client=erigon --target-size=100MB   # Erigon v3 flat .kv snapshots + minimal MDBX
 ```
 

internal/ethrex/constants.go

@@ -1,7 +1,7 @@
 package ethrex
 
 // Column-family names for ethrex's single RocksDB instance.
-// Sourced from ethrex store.rs at pinned commit 318ec2888.
+// Sourced from ethrex crates/storage/api/tables.rs at v23.0.0.
 const (
 	CFChainData            = "chain_data"
 	CFAccountCodes         = "account_codes"
@@ -23,10 +23,13 @@ const (
 	CFMiscValues           = "misc_values"
 	CFExecutionWitnesses   = "execution_witnesses"
 	CFBlockAccessLists     = "block_access_lists"
+	CFBadBlocks            = "bad_blocks"
 )
 
-// Tables is the ordered list of all 20 ethrex column families.
-// Order matches ethrex's StorageTable enum for deterministic CF creation.
+// Tables is the ordered list of all 21 ethrex column families, matching
+// ethrex's TABLES array. It must not run ahead of the boot pin in
+// client/ethrex/e2e_test.go: ethrex silently drops any CF absent from
+// its own TABLES (drop_obsolete_cfs, warn-only).
 var Tables = []string{
 	CFChainData,
 	CFAccountCodes,
@@ -48,6 +51,7 @@ var Tables = []string{
 	CFMiscValues,
 	CFExecutionWitnesses,
 	CFBlockAccessLists,
+	CFBadBlocks,
 }
 
 // chain_data index keys (RLP of the index byte).
@@ -69,8 +73,10 @@ const EmptyCodeHashHex = "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfa
 // i.e. the root of an empty MPT — used as the default storageRoot.
 const EmptyTrieHashHex = "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421"
 
-// StoreSchemaVersion is the value written to metadata.json.
-const StoreSchemaVersion = 2
+// StoreSchemaVersion must equal ethrex's STORE_SCHEMA_VERSION
+// (crates/storage/lib.rs): lower triggers boot-time migration (which
+// rewrites metadata.json); higher is a hard MigrationFailed boot error.
+const StoreSchemaVersion = 3
 
 // misc_values keys/values controlling ethrex's flat-key-value (FKV) generator.
 // On boot the generator reads misc_values["last_written"]: empty = "not

internal/ethrex/doc.go

@@ -1,6 +1,6 @@
 // Package ethrex implements the trie codec primitives used by ethrex
-// (pinned release v15.0.0, lambdaclass/ethrex). The golden fixture is
-// byte-identical from v13.0.0 through v15.0.0.
+// (pinned release v23.0.0, lambdaclass/ethrex). The golden fixture's
+// state-bearing CFs are byte-identical from v13.0.0 through v23.0.0.
 //
 // # Two-rows-per-leaf model
 //