diff --git a/.claude/scheduled_tasks.lock b/.claude/scheduled_tasks.lock deleted file mode 100644 index 1a50fa8a..00000000 --- a/.claude/scheduled_tasks.lock +++ /dev/null @@ -1 +0,0 @@ -{"sessionId":"66332041-4f74-42df-8954-9e0482baacdd","pid":11173,"acquiredAt":1775933454187} \ No newline at end of file diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 72a0aacb..9f34dd3a 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -33,6 +33,27 @@ jobs: run: cargo test --release --no-default-features --features runtime-tokio,jemalloc --test jepsen_lite timeout-minutes: 10 + crash-matrix-per-shard: + name: Crash Matrix (per-shard AOF) + if: github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'ci-full') + runs-on: ubuntu-latest + timeout-minutes: 15 + # MOON_NO_URING is already set globally at the workflow level (line 16). + # No job-level env override needed — monoio falls back to epoll automatically. + steps: + - uses: actions/checkout@v6 + - uses: dtolnay/rust-toolchain@1.94.1 + - uses: Swatinem/rust-cache@v2 + with: + shared-key: integration-${{ hashFiles('Cargo.lock') }} + - name: Install redis-tools + run: sudo apt-get update -qq && sudo apt-get install -y -qq redis-tools + - name: Build Moon (release, monoio default) + run: cargo build --release + - name: Run per-shard AOF crash matrix + run: cargo test --release --test crash_matrix_per_shard_aof -- --ignored + timeout-minutes: 10 + replication: name: Replication Tests if: github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'ci-full') diff --git a/CHANGELOG.md b/CHANGELOG.md index 1be8c9d0..9a6e1798 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,10 +6,36 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [0.2.0-alpha] — Unreleased -First slice of the v0.2 "Option C" beachhead: **Point-in-Time Recovery (PITR)** -and **Change Data Capture (CDC)**, built additively on top of the existing -per-shard WAL v3 + dual-root manifest. No changes to the KV hot path, MVCC, -page format, or transaction layer. +The v0.2 enterprise beachhead. Built additively on per-shard WAL v3 + the +dual-root manifest; no changes to the KV hot path, MVCC, page format, or +transaction layer. + +**Headline capabilities landed in alpha:** + +- **Point-in-Time Recovery (PITR)** — `--recovery-target-lsn` / + `--recovery-target-time` restore to any LSN or wall-clock boundary + inside the WAL retention window. +- **Change Data Capture (CDC)** — `CDC.READ` polling command with + Debezium-compatible JSON envelopes, resumable cursors, segment-rotation + safety. +- **Hash-field TTL** — full Valkey 9.0 / 9.1 surface (`HEXPIRE` / + `HPEXPIRE` / `HEXPIREAT` / `HPEXPIREAT` / `HEXPIRETIME` / `HPEXPIRETIME` + / `HTTL` / `HPTTL` / `HPERSIST` / `HGETDEL` / `HGETEX`) with O(1) HGET + + HLEN fast path. Three-way benchmark vs Redis 8.0.2 / Valkey 9.1.0 + ships in `docs/perf/2026-05-27-hash-ttl-3way-bench.md`. +- **Tier 2 Lane A** — `SWAPDB`, `MOVE`, `COPY ... DB n`, + `CLUSTER REPLICAS` / `SLAVES`, `CLUSTER COUNT-FAILURE-REPORTS`. All + WAL-durable with cross-shard atomic semantics. +- **Storage format v1 commitment** — RDB v2 + WAL v3 + multi-part AOF + manifest grouped under a single `--storage-format v1` umbrella with + ≥18-month LTS forward-read guarantees. +- **Embedded sharded server** — `server::embedded::run_embedded(config, + cancel)` exposes the full sharded handler (with `TXN.*`) to in-process + embedders. + +**What is not yet in alpha:** PITR live-snapshot LSN wiring (P3c), +`CDC.SUBSCRIBE` push channel (C3b), and the multi-shard master PSYNC +deferred from v0.1.10. Tracked in `.planning/rfcs/v02-enterprise-architecture.md`. ### Docs — Hash-field TTL three-way benchmark suite (PR #127) @@ -427,6 +453,96 @@ See `docs/guides/cdc.md` for consumer integration. and the benchmark gates (PITR restart ±10%, CDC ≥100K events/s/shard, write p99 ±5%). +## [0.1.12] — 2026-05-12 + +Performance & memory observability release. 50 commits since v0.1.11, no +public API breaks, no on-disk format change. Validated on OrbStack `moon-dev` +(2026-05-12) and locally green for both `runtime-monoio` and +`runtime-tokio,jemalloc`. + +### Performance — DashTable hot-path (Phase 189, PERF-07 + PERF-09) + +- **Pre-sized DashTable.** `DashTable::with_capacity()` plus the new + `--initial-keyspace-hint ` flag size the segment array up front so + steady-state operation hits zero `split_segment` calls. Pre-size + invariant test confirms zero splits at 1 M keys. The 27 % CPU spent + in `split_segment` during SET p=16 (PERF-07) is fully eliminated. +- **`Database::set` rewrite.** New `DashTable::insert_or_update` / + `Segment::insert_or_update_at` single-probe helpers replace the + previous `find + remove + insert` triple-probe pattern. +- **`Segment::find` fallback elimination + force-inlined SIMD.** The + cold "key spilled to non-home group" fallback path is removed once + `has_non_home_keys` is invariant-tracked on insert (the + `insert_or_update_at` change above already maintains the flag); + the SIMD probe helpers are `#[inline(always)]`. PERF-09 + attributed 12.65 % of `Segment::find` self-time to the fallback; + remaining cost is the irreducible per-hit `memcmp` confirm + (threshold amended to <3 %). 1 M-key correctness gate validates + zero false positives/negatives. + +### Performance — Memory observability (Phase 190) + +- **`moon_memory_bytes{kind=…}` Prometheus gauge.** Seven subsystem + labels — `dashtable`, `hnsw`, `csr`, `wal`, `sealed_replication_backlog`, + `allocator_overhead`, and the rolled-up `total`. Updated every + scrape via a single hook so the sum reconciles to `RSS` within the + CI tolerance window. +- **`MEMORY DOCTOR` full schema.** Multi-line RESP response covering + every subsystem, the rolled-up total, and a derived `allocator_overhead` + pseudo-kind (RSS − Σ subsystems). Adds operator triage signal beyond + the legacy single-line summary. +- **`resident_bytes()` trait** implemented across `Database`, + `DashTable`, `VectorStore` (HNSW + IVF), `GraphStore` (CSR + SlotMap), + `WalWriter`, `ReplicationBacklog` (sealed-segment side), and + `AllocatorOverhead`. Zero-allocation, on-demand poll. +- **Memory steady-state CI job.** `scripts/bench-memory-steady-state.sh` + + baseline fixture; gate widened to `±10 %` on RSS / Σ ratio after a + Linux-CI tolerance pass. + +### Changed — Allocator UX (Phase 191) + +- **jemalloc `narenas:8` cap** with `--memory-arenas-cap ` CLI + override. Caps the per-CPU arena explosion that inflates VSZ on + high-core hosts; mostly a cosmetic fix on Linux containers but + produces a meaningfully tighter `top`/`ps` reading for operators. +- **Tri-state allocator selection.** New `mimalloc-alt` cargo + feature alongside the existing `jemalloc` / `mimalloc` (fallback) + paths; mutually exclusive at compile time. A/B benchmark script + `scripts/bench-allocator-ab.sh` ships with the release. +- **`docs/OPERATOR-GUIDE.md` — Memory Accounting section.** Documents + the VSZ-vs-RSS distinction, MEMORY DOCTOR field-by-field, and the + `--memory-arenas-cap` / `mimalloc-alt` tuning knobs. + +### Added — Dispatch Observability (Phase 177) + +- **`moon_dispatch_path_total{path=...}` Prometheus counter**: four-way classification of every command by shard-routing decision — `local_inline` (SIMD fast path), `local` (standard local branch), `cross_read_fast` (RwLock shared-read bypass of SPSC), `cross_spsc` (deferred cross-shard write via `PipelineBatchSlotted`). Ratio `cross_spsc / Σ` is the ground-truth signal for dispatch-layer optimization work. Zero-allocation hot-path overhead (`&'static str` labels, `#[inline]` with early-return on `!METRICS_INITIALIZED`). Verified on macOS + Linux: counter sums close exactly to driven traffic, no overcount. + +### Changed + +- **`text-index` is now a default feature.** BM25 full-text search (`FT.SEARCH` BM25 mode), `FT.AGGREGATE`, and three-way RRF hybrid fusion are included in all standard builds. No longer requires `--features text-index`. To exclude it (e.g. minimal embedded builds): `--no-default-features --features runtime-monoio,jemalloc,graph`. + +### Added — SDK Validation + +- **Python SDK `sdk/python/examples/validate.py`**: End-to-end live validator for all SDK sub-clients: ping, strings, counter, hash, list, set, zset, vector index lifecycle, graph engine, session search, semantic cache, text search (BM25 + aggregate + hybrid), and server info. Result against Moon with `text-index`: **114 PASS / 0 FAIL / 0 SKIP**. Gracefully skips text sections when server built without `text-index`. +- **Rust SDK `sdk/rust/examples/validate.rs`**: Re-validated against `text-index` build — **85 PASS / 0 FAIL**. + +### Fixed — Python SDK + +- **`moondb.graph._parse_neighbors`**: server returns alternating `[edge_map, node_map, ...]` as flat key-value arrays (`b'id'`, `int`, `b'src'`, `int`, `b'dst'`, `int`, …). Previous parser expected positional `[node_id, label, props]` — caused `int() on b'id'` crash. Now correctly identifies node entries by `labels` key and parses them from the flat kv format. + +### Fixed — CI Hygiene + +- **`tests/pipeline_auto_index.rs`**: tighten outer cfg from `runtime-tokio` to `all(runtime-tokio, text-index)` so the file compiles to zero tests when text-index is disabled. Previously the file compiled but the FT.SEARCH text fast path was `#[cfg]`-ed out, causing `@name:corpus` queries to fall through to the KNN-only parser and panic with "invalid KNN query syntax". +- **4 FT unwraps**: add inline `#[allow(clippy::unwrap_used)]` with invariant justifications in `vector_search/ft_text_search.rs` (3 sites inside `apply_post_processing` where `do_summarize` / `do_highlight` implies the Option is Some) and `handler_monoio/ft.rs:165` (`is_text` was derived from `query_bytes.as_ref().map_or(false, _)`). Restores the audit-unwrap baseline to 0. + +### Compatibility + +- **Wire protocol**: unchanged. Drop-in replacement for v0.1.11. +- **Persistence on-disk format**: unchanged. +- **Default feature set**: `text-index` is now on by default. Minimal + embedded builds need an explicit `--no-default-features --features + runtime-monoio,jemalloc,graph`. + ## [0.1.11] — 2026-04-27 Hot-path perf release — eliminates two atomic-CAS hot paths in the write @@ -537,29 +653,47 @@ All 2450 lib tests passing locally (`cargo test --no-default-features - **Persistence on-disk format**: unchanged. - **Replication wire format**: unchanged. -## [Unreleased] - -### Added — Dispatch Observability (Phase 177) - -- **`moon_dispatch_path_total{path=...}` Prometheus counter**: four-way classification of every command by shard-routing decision — `local_inline` (SIMD fast path), `local` (standard local branch), `cross_read_fast` (RwLock shared-read bypass of SPSC), `cross_spsc` (deferred cross-shard write via `PipelineBatchSlotted`). Ratio `cross_spsc / Σ` is the ground-truth signal for dispatch-layer optimization work. Zero-allocation hot-path overhead (`&'static str` labels, `#[inline]` with early-return on `!METRICS_INITIALIZED`). Verified on macOS + Linux: counter sums close exactly to driven traffic, no overcount. - -### Fixed — CI Hygiene - -- **`tests/pipeline_auto_index.rs`**: tighten outer cfg from `runtime-tokio` to `all(runtime-tokio, text-index)` so the file compiles to zero tests when text-index is disabled. Previously the file compiled but the FT.SEARCH text fast path was `#[cfg]`-ed out, causing `@name:corpus` queries to fall through to the KNN-only parser and panic with "invalid KNN query syntax". -- **4 FT unwraps**: add inline `#[allow(clippy::unwrap_used)]` with invariant justifications in `vector_search/ft_text_search.rs` (3 sites inside `apply_post_processing` where `do_summarize` / `do_highlight` implies the Option is Some) and `handler_monoio/ft.rs:165` (`is_text` was derived from `query_bytes.as_ref().map_or(false, _)`). Restores the audit-unwrap baseline to 0. - -### Changed - -- **`text-index` is now a default feature.** BM25 full-text search (`FT.SEARCH` BM25 mode), `FT.AGGREGATE`, and three-way RRF hybrid fusion are included in all standard builds. No longer requires `--features text-index`. To exclude it (e.g. minimal embedded builds): `--no-default-features --features runtime-monoio,jemalloc,graph`. - -### Added — SDK Validation - -- **Python SDK `sdk/python/examples/validate.py`**: End-to-end live validator for all SDK sub-clients: ping, strings, counter, hash, list, set, zset, vector index lifecycle, graph engine, session search, semantic cache, text search (BM25 + aggregate + hybrid), and server info. Result against Moon with `text-index`: **114 PASS / 0 FAIL / 0 SKIP**. Gracefully skips text sections when server built without `text-index`. -- **Rust SDK `sdk/rust/examples/validate.rs`**: Re-validated against `text-index` build — **85 PASS / 0 FAIL**. - -### Fixed — Python SDK - -- **`moondb.graph._parse_neighbors`**: Server returns alternating `[edge_map, node_map, ...]` as flat key-value arrays (`b'id'`, `int`, `b'src'`, `int`, `b'dst'`, `int`, …). Previous parser expected positional `[node_id, label, props]` — caused `int() on b'id'` crash. Now correctly identifies node entries by `labels` key and parses them from the flat kv format. +## [0.1.10] — 2026-04-23 + +Stable replication marker. **Single-shard PSYNC2 wired end-to-end and +production-ready** for `--shards 1` master with any `--shards N` replica +topology. Multi-shard master PSYNC is scheduled for v0.2 (see +`.planning/rfcs/multi-shard-replication-design.md`). + +- **Replication** (`081c43b`): single-shard master PSYNC2 end-to-end wired, + REPLCONF validated, `master_link_status` reports the actual handshake + state instead of the legacy `up` stub. +- **Performance**: batch-level eviction gate; `try_handle_*` paths + `#[inline]`-ed; DashTable carries through the v0.1.10 pre-size + groundwork (capacity hint + headroom). +- **Docs**: BENCHMARK.md §2.7 updated with the 2026-04-22 GCloud + re-measurement; v0.1.x replication scope documented under + `docs/guides/clustering.mdx#replication`. + +## [0.1.9] — 2026-04-19 + +**Lunaris Retriever Gap Closure.** Every v0.1.8 client-side fallback in +the Lunaris SDK is now closed so `HybridRRFRetriever` (dense path), +`GraphFirstRetriever`, and `PathReasoningRetriever` run Moon-native. + +- **Phase 167 CYP-01/02**: Cypher `CREATE` / `MERGE` writes participate + in `CrossStoreTxn` via `record_graph()`; `TXN.ABORT` rolls them back. +- **Phase 168 CYP-03/06**: `coalesce()` built-in + single-hop edge-var + binding in variable-length `EXPAND`. +- **Phase 169 CYP-04/05**: `shortestPath()` parser + Dijkstra executor + bridge with path-variable binding. +- **Phase 170 HYB-01/02/04**: `FT.SEARCH HYBRID` dense stream honours + `as_of_lsn`. +- **Phase 171 SCAT-01/02/03**: `ShardMessage::VectorSearch` + + `FtHybridPayload` carry `as_of_lsn` for multi-shard `AS_OF` correctness. +- **Phase 172 PIPE-01/02/03**: pipeline-aware HSET auto-indexing + regression guard (3-test suite). + +Audit status: **PASSED_WITH_DOCUMENTED_DEFERRALS**. 15 / 20 requirements +fully satisfied; HYB-03 BM25 MVCC deferred and closed in v0.1.10 +follow-up (G-1); Phase 173 hygiene HYG-02 handler split RFC'd. + +Stats: 6 phases shipped, 17 plans, 27 files changed, +2924 / −376 LOC. ## [0.1.8] — 2026-04-18 @@ -837,7 +971,14 @@ All 2450 lib tests passing locally (`cargo test --no-default-features - **Security hardening:** `deny.toml` (cargo-deny), `SECURITY.md`, `docs/THREAT-MODEL.md`, `docs/security/lua-sandbox.md`, TLS cipher suite freeze - **Release engineering:** `docs/versioning.md`, 6 operator runbooks, CHANGELOG CI gate, user docs (getting-started, configuration, monitoring), release pipeline SHA256 checksums + SBOM + cosign -## [Earlier Unreleased] - Dispatch Hot-Path Recovery (2026-04-08) +## [0.1.3] — 2026-04-10 + +Production-readiness foundation: dispatch hot-path recovery, vector-search +4× QPS + correctness fixes, and the tiered disk-offload landing with 100 % +crash recovery across 7 persistence configurations. Bundles three work +streams originally tracked as separate Unreleased blocks (Apr 7–8). + +### Dispatch Hot-Path Recovery (2026-04-08) **Pipelined SET +37%, pipelined GET +68% at p=16 after PR #43 regression recovery.** @@ -934,9 +1075,7 @@ captured as todo in `.planning/todos/pending/`. --- -## [Unreleased] - Vector Search 4x QPS + Correctness - -### Vector Search Performance & Correctness (2026-04-07) +### Vector Search 4× QPS + Correctness (2026-04-07) **4x search QPS, 4.1x lower latency, 2.56x faster than Qdrant on real MiniLM data.** @@ -991,9 +1130,9 @@ embeddings (clustered) achieve 0.92-0.97 recall with the same code. --- -## [Earlier Unreleased] - Disk Offload & x86_64 Performance +### Disk Offload & x86_64 Performance (2026-04-06) -Tiered storage, crash recovery, and 2x Redis on x86_64 (Intel Xeon, io_uring). +Tiered storage, crash recovery, and 2× Redis on x86_64 (Intel Xeon, io_uring). ### Added diff --git a/README.md b/README.md index 5fcfbbe7..350c8861 100644 --- a/README.md +++ b/README.md @@ -7,11 +7,12 @@

- Version + Version Rust SDK Python SDK License - Status + Status + Cluster status Rust Protocol

@@ -20,17 +21,38 @@ Quick start • Why Moon • Benchmarks • + Moon vs Redis vs Valkey • + Production readiness • Docs • Changelog

--- -> **⚠ Experimental.** Moon is under active development and **not** recommended for production. Storage formats, APIs, and config flags may change between releases. Please [open an issue](https://github.com/pilotspace/moon/issues) if something breaks. +> **Production-grade architecture, pre-1.0 maturity.** Single-node Moon +> (`--shards N` master, `--shards 1` for replication-eligible workloads) +> is recommended for production caching, vector / graph / feature-store +> workloads, and Redis-compatible OLTP. Multi-node clustering and +> multi-shard master PSYNC are **alpha in v0.2** — see +> [Production readiness](#production-readiness) for the honest matrix of +> what is and isn't yet GA. Wire protocol and on-disk format are LTS as of +> v0.2 (`docs/STORAGE-FORMAT-V1.md`); CLI flags may still evolve until +> v1.0. [Open an issue](https://github.com/pilotspace/moon/issues) if +> something breaks. --- -Moon speaks the Redis wire protocol (RESP2/RESP3) and implements 230+ commands. It runs on **Linux** (io_uring via monoio) and **macOS** (kqueue via monoio) with a thread-per-core, shared-nothing architecture, per-shard WAL, tiered disk offload, an in-process vector search engine with BM25 full-text search, a property graph engine with Cypher subset, cross-store ACID transactions, workspace partitioning, durable message queues, bi-temporal MVCC, and an embedded web console. Any Redis client connects out of the box. +Moon is a clean-room Rust rewrite of a Redis-compatible in-memory data +store with first-class AI primitives. It speaks the Redis wire protocol +(RESP2/RESP3) and implements 230+ commands — every standard Redis data +type plus native `FT.*` vector + BM25 search, `GRAPH.*` Cypher, `TXN.*` +cross-store ACID, workspaces, durable message queues, bi-temporal MVCC, +and an embedded web console. Primary target is **Linux** with io_uring +(`monoio`); a `tokio` runtime is available for portability. **macOS** is a +supported development platform (kqueue via monoio); production +deployments should target Linux (see +[`docs/PRODUCTION-CONTRACT.md`](docs/PRODUCTION-CONTRACT.md) Tier 1/2). +Any Redis client connects out of the box. ## Why Moon @@ -61,18 +83,30 @@ Moon speaks the Redis wire protocol (RESP2/RESP3) and implements 230+ commands. ## Benchmarks -Measured vs Redis 8.6.1, co-located client and server, pipeline depth tuned per row. Full methodology and reproduction steps in [BENCHMARK.md](BENCHMARK.md) and [docs/benchmarks.mdx](docs/benchmarks.mdx). +Measured vs Redis 8.6.1 (peak throughput) and Redis 8.0.2 + Valkey 9.1.0 +(hash-TTL surface, the only workload where all three were head-to-head +benchmarked). Co-located client and server, pipeline depth tuned per +row. Full methodology and reproduction steps in +[BENCHMARK.md](BENCHMARK.md) and [docs/benchmarks.mdx](docs/benchmarks.mdx). +Valkey peak-throughput columns are intentionally blank — a head-to-head +peak-RPS bench on identical hardware has not yet been run; the +[Moon vs Redis vs Valkey](#moon-vs-redis-vs-valkey) section quotes +Valkey's vendor-published 2.1M RPS (9 I/O threads, p=10) for context. ### Peak throughput (GCloud c3-standard-8, x86_64, monoio io_uring) -| Workload | Moon | Redis | Ratio | -|----------------------------------|-------:|------:|:------:| -| Peak GET (c=50, p=64) | 5.11M | 2.98M | **1.72×** | -| Peak SET (c=50, p=64) | 3.50M | 1.82M | **1.92×** | -| GET, production defaults (AOF+disk-offload) | 4.76M | 2.46M | **1.93×** | -| GET, max durability (fsync always)| 4.85M | 2.45M | **1.98×** | -| Memory, values ≥ 1 KB | — | — | **27–35% less** | -| Crash recovery (SIGKILL, 5K keys)| 100% | 100% | parity | +| Workload | Moon | Redis 8.6.1 | Valkey 9.1.0 | +|------------------------------------------------|-------:|------------:|-------------:| +| Peak GET (c=50, p=64) | 5.11M | 2.98M (1.72×) | not yet benched | +| Peak SET (c=50, p=64) | 3.50M | 1.82M (1.92×) | not yet benched | +| GET, production defaults (AOF + disk-offload) | 4.76M | 2.46M (1.93×) | not yet benched | +| GET, max durability (`fsync=always`) | 4.85M | 2.45M (1.98×) | not yet benched | +| Memory, values ≥ 1 KB | — | **27–35 % less** | not yet benched* | +| Crash recovery (SIGKILL, 5K keys) | 100 % | 100 % (parity)| 100 % (parity, vendor-claimed) | + +*Valkey 9.1 raised the embstr threshold to 128 B; below ~64 B Valkey +9.1 may be tighter than Moon. A head-to-head re-bench across the value +size curve is on the v0.2 roadmap. ### ARM64 (GCloud t2a-standard-8, Neoverse-N1) @@ -98,9 +132,13 @@ Measured vs Redis 8.6.1, co-located client and server, pipeline depth tuned per | Native API QPS | **19×** | N/A | | Bulk insert | **23×** | 1× | -### Hash-field TTL — Valkey 9.0/9.1 parity (OrbStack moon-dev, n=200K c=50, median of 3) +### Hash-field TTL — Valkey 9.0/9.1 parity (v0.2.0-alpha preview) -Three-way comparison on the per-field TTL surface added in v0.2.0. Full methodology + 26 scenarios in [docs/perf/2026-05-27-hash-ttl-3way-bench.md](docs/perf/2026-05-27-hash-ttl-3way-bench.md); reproducible via [scripts/bench-hash-ttl-3way.sh](scripts/bench-hash-ttl-3way.sh). +> **Status:** ships in **v0.2.0-alpha** (currently on `main`, no tagged +> release yet). Not present in the latest tagged build `v0.1.12`. Build +> from `main` to reproduce these numbers. + +OrbStack moon-dev, n=200K c=50, median of 3. Three-way comparison on the per-field TTL surface added in v0.2.0. Full methodology + 26 scenarios in [docs/perf/2026-05-27-hash-ttl-3way-bench.md](docs/perf/2026-05-27-hash-ttl-3way-bench.md); reproducible via [scripts/bench-hash-ttl-3way.sh](scripts/bench-hash-ttl-3way.sh). | Command | Pipeline | Moon | Redis 8.0.2 | Valkey 9.1.0 | |------------------|----------|------:|------------:|-------------:| @@ -111,7 +149,64 @@ Three-way comparison on the per-field TTL surface added in v0.2.0. Full methodol | `HGETEX EX` | p=1 | 250K | N/A | 251K | | `HGETEX` no-mode | p=1 | 250K | N/A | 253K | -Plain Hash HGET p=16 ties Redis (1.01×) and Valkey (1.00×). HEXPIRE-family Moon vs Valkey: 0.90–0.99× across the surface (Valkey leads HEXPIRE p=16 by 7%, HTTL p=16 by 10%; HGETEX hits 0.99× parity). Redis 8.x has no HEXPIRE-family — Moon is the only Redis-compatible alternative aside from Valkey. The internal `HashWithTtl` HGET / HLEN paths use a cached `min_expiry_ms` for an O(1) fast path that brings them to 1.03× of plain `Hash` (was 80× slower pre-fix; see PR #126). +Plain Hash HGET p=16 ties Redis (1.01×) and Valkey (1.00×). HEXPIRE-family Moon vs Valkey: 0.90–0.99× across the surface (Valkey leads HEXPIRE p=16 by 7 %, HTTL p=16 by 10 %; HGETEX hits 0.99× parity). Redis 8.x has no HEXPIRE-family — Moon is the only Redis-compatible alternative aside from Valkey. The internal `HashWithTtl` HGET / HLEN paths use a cached `min_expiry_ms` for an O(1) fast path that brings them to 1.03× of plain `Hash` (was 80× slower pre-fix; see PR #126). + +## Moon vs Redis vs Valkey + +Three Redis-protocol-compatible servers, three different bets. Moon +competes on a **vertical moat** — thread-per-core architecture and an +AI-native in-core surface. Valkey competes on a **horizontal moat** — +Linux Foundation governance, every major cloud, drop-in compatibility. +Redis OSS continues as the upstream reference but ships under SSPL since +March 2024. The deep architectural review is in +[`docs/comparison-valkey.md`](docs/comparison-valkey.md) (~22 KB, +traced to source). + +| Dimension | **Moon v0.1.12 / 0.2.0-alpha** | **Valkey 9.1.0** | **Redis 8.6.1 (OSS)** | +|--------------------------------------|--------------------------------|-----------------------------|----------------------------| +| Language / license | Rust 2024 / Apache-2.0 | C99 / BSD-3-Clause (LF TSC) | C99 / **SSPL** since 2024 | +| Threading model | Thread-per-core, shared-nothing | Single main thread + ≤9 I/O threads | Single-threaded core (+ I/O threads) | +| I/O driver (Linux) | io_uring (`monoio`) | epoll only | epoll only | +| Snapshot | **Forkless** (segment-level COW) | `fork()` + COW | `fork()` + COW | +| AOF / WAL | Per-shard WAL v3 + multi-part AOF | Single global AOF | Single global AOF | +| Tiered NVMe disk offload | **Yes** (under `maxmemory`) | No (OSS) | No (OSS — Redis Flash is Enterprise) | +| Vector search | **In-core** HNSW + TurboQuant 1-8 bit | `valkey-search` module, FP32 only | RediSearch module | +| Full-text BM25 | **In-core** | `valkey-search` module | RediSearch module | +| Property graph (Cypher) | **In-core** 14 `GRAPH.*` cmds | None | None (FalkorDB separate) | +| Cross-store ACID | `TXN.BEGIN/COMMIT/ABORT` | None | None | +| Hash-field TTL (`HEXPIRE`-family) | **Yes** (Valkey-parity, v0.2-alpha) | **Yes** (9.0+) | No | +| PITR + CDC | `--recovery-target-lsn` + `CDC.READ` (v0.2-alpha) | None | None | +| Embedded web console | **Yes** (7-view React, in-binary) | Valkey Admin GUI 1.0 (separate) | Redis Insight (separate) | +| Managed cloud offerings | None (yet) | AWS, GCP, Oracle, Aiven, … | Redis Cloud (vendor) | +| Multi-node cluster, soak-tested | **v0.2 alpha** (single-node GA today) | **Production** | **Production** | +| Atomic slot migration | Planned (v0.2) | Yes (9.0) | No | +| Peak single-server throughput | **5.11M GET/s** (c3-8 x86_64) | 2.1M RPS (9 I/O threads, p=10, vendor) | 2.98M GET/s (c3-8, same harness as Moon) | + +### When to choose Moon + +- Single-node Redis-compatible workloads where peak throughput, + memory efficiency at ≥1 KB values, or **forkless snapshots** matter. +- AI-native applications: vector search, GraphRAG, semantic cache, + hybrid BM25 + dense + sparse retrieval — all in one binary, no module + loader, with cross-store ACID across KV / vector / graph. +- Workloads that benefit from **tiered NVMe offload** under `maxmemory` + instead of LRU-eviction-then-rebuild. + +### When to choose Valkey + +- Multi-node clusters with proven 1000+ node operational mileage. +- Managed-cloud-only deployments (every major cloud offers Valkey). +- Strict drop-in compatibility with the Redis 7.2 module ecosystem + (`valkey-json`, `valkey-bloom`, `valkey-search`, `valkey-ldap`). +- Risk-averse environments where Linux Foundation governance is a + procurement requirement. + +### When to stay on Redis OSS + +- Existing investments in RediSearch / RedisJSON / RedisBloom under + RLEC, or pre-SSPL-tolerance OSS Redis. +- Specific Redis Enterprise features (CRDT active-active, Redis Flash) + that Moon and Valkey OSS do not match. ## Quick start @@ -148,9 +243,9 @@ OK "world" 127.0.0.1:6379> HSET user:1 name Alice age 30 (integer) 2 -127.0.0.1:6379> HEXPIRE user:1 3600 FIELDS 1 age # Valkey 9.0 per-field TTL +127.0.0.1:6379> HEXPIRE user:1 3600 FIELDS 1 age # Valkey 9.0 per-field TTL (v0.2.0-alpha; build from `main`) 1) (integer) 1 -127.0.0.1:6379> HTTL user:1 FIELDS 1 age +127.0.0.1:6379> HTTL user:1 FIELDS 1 age # v0.2.0-alpha 1) (integer) 3600 127.0.0.1:6379> FT.CREATE idx ON HASH PREFIX 1 doc: SCHEMA emb VECTOR HNSW 6 DIM 384 TYPE FLOAT32 DISTANCE_METRIC COSINE OK @@ -326,34 +421,123 @@ cargo flamegraph --bin moon -- --port 6399 --shards 1 Contribution guide and coding rules (unsafe policy, hot-path allocation rules, lock discipline) are in [CLAUDE.md](CLAUDE.md) and [UNSAFE_POLICY.md](UNSAFE_POLICY.md). -## Roadmap - -Moon is pre-1.0 and **experimental**. Current focus: - -- Correctness parity with Redis 8.x across the full command surface -- AI-native primitives: session dedup, hybrid vector+sparse search, agentic caching -- Multi-node clustering with gossip, slot migration, and PSYNC2 replication -- GPU-accelerated vector search (CUDA, feature-gated) -- Production hardening and SLO validation (see [docs/PRODUCTION-CONTRACT.md](docs/PRODUCTION-CONTRACT.md)) - -Completed in v0.1.0–v0.1.8: -- Tiered disk offload (RAM → NVMe) with 100% crash recovery -- In-process vector search (HNSW + TurboQuant 4/8-bit) with `FT.*` API -- BM25 full-text search with three-way hybrid fusion (BM25 + dense + sparse) -- Property graph engine with Cypher subset (14 `GRAPH.*` commands) -- Cross-store ACID transactions (`TXN.BEGIN`/`COMMIT`/`ABORT`) across KV, vector, and graph -- Workspace partitioning for multi-tenant namespace isolation -- Durable message queues with dead-letter and debounced triggers -- Bi-temporal MVCC for point-in-time KV and graph queries -- Web console (7 views, embedded in binary) -- macOS support (aarch64 + x86_64, both runtimes) -- Thread-per-core dispatch optimization (5.11M GET/s on x86_64) - -Production readiness is **not** a v0.1 goal. Storage formats, APIs, and config flags may change between releases. - -## Production Readiness - -Moon's v1.0 promises — SLOs, durability modes, supported platforms, and a machine-checkable GA exit-criteria checklist — live in **[docs/PRODUCTION-CONTRACT.md](docs/PRODUCTION-CONTRACT.md)**. The contract is the single source of truth every v0.1.3 hardening phase tests against. +## Production readiness + +Honest matrix of where Moon is today (v0.1.12 GA + v0.2.0-alpha +unreleased). Read alongside +[`docs/PRODUCTION-CONTRACT.md`](docs/PRODUCTION-CONTRACT.md) (the +machine-checkable GA exit criteria) and +[`docs/OPERATOR-GUIDE.md`](docs/OPERATOR-GUIDE.md) (memory accounting, +sizing, runbooks). + +### Recommended for production today + +- **Single-node deployments** — Linux aarch64 (Tier 1) or Linux x86_64 + (Tier 2). `--shards N` master, one process, one node. +- **Read replication** — `--shards 1` master with any `--shards N` + replica topology. Single-shard PSYNC2 is wired end-to-end since + v0.1.10. +- **AI workloads** — vector search (HNSW + TurboQuant), BM25 full-text, + GraphRAG, semantic caching, hybrid retrieval. All in-core, all + RDB / WAL durable, all crash-recovery validated. +- **Cache + feature store** — durability modes are honest + (`always` / `everysec` / `no` with documented recovery bounds), forkless + snapshots remove the Redis fork-COW RSS spike, tiered NVMe offload + under `maxmemory` keeps working sets larger than RAM. +- **Crash recovery** — 100 % survived across 7 persistence + configurations and 5 K-key SIGKILL workloads. RDB v2 + WAL v3 + + multi-part AOF + tiered cold tier all participate. + +### Not yet GA — avoid for production + +- **Multi-node clustering** (16 K-slot gossip, MOVED/ASK, failover) — + protocol-compatible code exists but **PSYNC2 atomic slot migration + is not soak-tested**. Valkey 9.0 shipped this; Moon has not. + Scheduled for v0.2. +- **Multi-shard master PSYNC** — single-shard only today; multi-shard + master replication is RFC'd in + [`.planning/rfcs/multi-shard-replication-design.md`](.planning/rfcs/multi-shard-replication-design.md). +- **PITR live-snapshot LSN wiring** (P3c) and **`CDC.SUBSCRIBE` push + channel** (C3b) — `CDC.READ` polling is alpha-ready; push and + zero-snapshot PITR are deferred to v0.2 follow-ups. +- **GPU vector acceleration** (`gpu-cuda` feature) — kernel scaffold + exists; production kernels not yet shipping. +- **macOS native** — first-class development platform with full feature + set minus io_uring, but production deployments should target Linux + per + [`docs/PRODUCTION-CONTRACT.md`](docs/PRODUCTION-CONTRACT.md) tiers. +- **Performance SLO numbers in + [`docs/PRODUCTION-CONTRACT.md`](docs/PRODUCTION-CONTRACT.md)** — marked + `[provisional]` until the Phase 97 24-h HDR-histogram rig validates + them on reference hardware. Use the benchmarks above as point-in-time + measurements, not committed SLOs. + +### Operator gotchas worth knowing before you deploy + +- **Multi-shard scaling needs load.** Aim for `clients ≥ 25 × shards` + on pipeline ≥ 16 workloads; below that, multiple shards under-subscribe + the dispatch loop and a single shard wins. Random keyspaces with + `p=1` benefit less than `{tag}`-co-located keys (see + [`CLAUDE.md`](CLAUDE.md) "Gotchas" + the v0.1.12 multi-shard memo). +- **Fairness flags for benchmarking against Redis / Valkey** — + `--disk-offload disable --appendonly no` removes Moon's durability + overhead (~26 % on SET p=64) so comparisons are apples-to-apples. +- **Memory accounting** — bill on RSS, not VSZ. `MEMORY DOCTOR`, + `moon_memory_bytes{kind=…}` Prometheus gauge, and + [`docs/OPERATOR-GUIDE.md#memory-accounting`](docs/OPERATOR-GUIDE.md#memory-accounting) + cover the full VSZ-vs-RSS guide and tuning knobs + (`--memory-arenas-cap`, `mimalloc-alt`). +- **RDB hash-field TTL trailer** is RDB v2 only — older v1 readers stop + after the hash body and silently drop per-field TTLs. Pin storage + format to `v1` (the umbrella covering v2/v3 sub-formats) per + [`docs/STORAGE-FORMAT-V1.md`](docs/STORAGE-FORMAT-V1.md). + +### Roadmap + +| Milestone | Focus | Status | +|------------------|---------------------------------------------------------------------------------------------|-------------| +| **v0.2.0** (next) | Multi-node clustering soak (PSYNC2 + atomic slot migration); PITR P3c; CDC push (`SUBSCRIBE`) | alpha | +| **v0.2.x** | GPU vector acceleration (`gpu-cuda`); operator runbooks; full SLO lock-in (`PERF-01..05`) | planned | +| **v1.0** | Every [`PRODUCTION-CONTRACT.md`](docs/PRODUCTION-CONTRACT.md) GA exit-criteria box ticked | gate | + +What's already in `main` (v0.1.0 → v0.2.0-alpha, 14 months of work): + +> Hash-field TTL and PITR + CDC ship in **v0.2.0** — currently alpha on +> `main`, no tagged release yet. **v0.1.12** is the latest tag and does +> NOT include them. Everything else below is in `v0.1.12` GA. + +**Shipped in v0.1.12 (latest tag, single-node production-grade):** + +- Forkless persistence (RDB v2 + per-shard WAL v3 + multi-part AOF). +- Tiered disk offload (RAM → NVMe) with 100 % crash recovery. +- In-process vector search (HNSW + TurboQuant 1–8-bit) — `FT.*` surface. +- BM25 full-text search + three-way RRF hybrid fusion (BM25 + dense + sparse). +- Property graph engine with Cypher subset (14 `GRAPH.*` commands). +- Cross-store ACID (`TXN.BEGIN` / `COMMIT` / `ABORT`) across KV, vector, + graph. +- Workspaces, durable message queues, bi-temporal MVCC. +- Web console (7-view React app, embedded in binary). +- Thread-per-core dispatch optimization (5.11M GET/s on x86_64). + +**Added in v0.2.0-alpha (on `main`, untagged):** + +- Hash-field TTL — Valkey 9.0 / 9.1 parity, O(1) HGET / HLEN fast path + (`HEXPIRE`, `HEXPIREAT`, `HPEXPIRE`, `HPEXPIREAT`, `HEXPIRETIME`, + `HPEXPIRETIME`, `HTTL`, `HPTTL`, `HPERSIST`, `HGETEX`, `HGETDEL`). +- PITR — `--recovery-target-lsn` deterministic WAL replay-to-LSN. +- CDC — `CDC.READ` pull-mode change stream (push-mode planned for v0.2.0 + GA). +- Multi-node clustering soak (PSYNC2 + atomic slot migration). + +## Production Readiness Contract + +Moon's v1.0 promises — per-command-class SLOs, durability modes, supported +platforms, security guarantees, and a machine-checkable GA exit-criteria +checklist — live in +**[`docs/PRODUCTION-CONTRACT.md`](docs/PRODUCTION-CONTRACT.md)**. Every +v0.1.3+ hardening phase ticks off items on that checklist; nothing +promotes to `v1.0-rc1` until every box is green. The contract is the +single source of truth for what Moon owes you in production. ## Credits diff --git a/docs/runbooks/multi-shard-aof-rewrite.md b/docs/runbooks/multi-shard-aof-rewrite.md new file mode 100644 index 00000000..b751ab00 --- /dev/null +++ b/docs/runbooks/multi-shard-aof-rewrite.md @@ -0,0 +1,161 @@ +# Runbook — multi-shard AOF (per-shard layout) + +**Status:** Resolved. The startup refusal gate introduced in v0.1.13 was lifted +in v0.1.13-patch (PR #129) once the per-shard AOF replay path was shipped and +verified (CRASH-01-LITE: 200/200 SIGKILL recovery, 0 data loss). + +--- + +## Background (historical context) + +Prior to PR #129, Moon refused to start with `--shards >= 2 + --appendonly yes` +because the single-writer AOF implementation lost ~50 % of writes on SIGKILL +in that configuration. The fix was a full per-shard AOF architecture (Option B): +each shard owns its own writer task and recovery walks every shard's segment +manifest independently. + +If you are running Moon ≤ v0.1.13 and hit the old startup error, see the +**Upgrading** section below. + +--- + +## Current architecture (v0.1.13-patch / PR #129 and later) + +### Per-shard AOF layout + +``` +/ + appendonlydir/ + moon.aof.manifest ← top-level manifest (layout: PerShard) + shard-0/ + moon.aof.1.base.rdb ← base snapshot for shard 0 + moon.aof.1.incr.aof ← incremental log for shard 0 + shard-1/ + moon.aof.1.base.rdb + moon.aof.1.incr.aof + shard-N/ + ... +``` + +Each shard's writer task appends to its own `.incr.aof` file. On restart, Moon +opens the manifest, discovers all shard directories, and replays each shard's +log independently. Shard replay is parallel — recovery time does not grow +linearly with shard count. + +### Durability invariants + +| appendfsync | Guarantee | +|---------------------|------------------------------------------------------------------------| +| `always` | Write is on durable storage before +OK (AppendSync rendezvous) | +| `everysec` (default)| Fsync runs every second; at most ~1 s of writes at risk on crash | +| `no` | OS decides when to flush; fastest but weakest guarantee | + +### BGREWRITEAOF in per-shard mode + +`BGREWRITEAOF` is **not yet supported** for PerShard layouts. Issuing it on a +PerShard instance returns the following error immediately: + +``` +ERR BGREWRITEAOF is not yet supported under per-shard AOF layout; per-shard rewrite ships in step 6 of the per-shard AOF migration +``` + +Per-shard BGREWRITEAOF (each shard compacts its own log independently, with +all N acks awaited before returning confirmation) is tracked for v0.2. + +--- + +## Upgrading from v0.1.13 (old TopLevel AOF) to per-shard layout + +If you have an existing deployment with a **TopLevel** AOF manifest (single +writer, `layout: TopLevel`) and want to migrate to per-shard layout: + +### Option A — cold migration (recommended, zero-risk) + +1. Stop the server. +2. Run `BGSAVE` on the last healthy instance, or copy `dump.rdb` from + `--dir`. +3. Remove `appendonlydir/` entirely. +4. Restart with `--shards N --appendonly yes`. Moon creates a fresh per-shard + manifest. Recovery loads from RDB; the incremental AOF starts empty. + +### Option B — in-place migration (future tooling) + +An offline migration CLI subcommand is planned for v0.2. Until then, use +Option A. + +### Safety guard — TopLevel manifest with multi-shard startup + +If Moon detects an existing **TopLevel** AOF manifest at startup with +`--shards >= 2`, it refuses to start with exit code 2 and prints the following +to stderr: + +``` +REFUSING TO START: legacy TopLevel AOF manifest at detected with +--shards N (>= 2). This combination silently loses data for shards 1..N-1. +See docs/runbooks/multi-shard-aof-rewrite.md for migration instructions. +``` + +(Exact text may include additional context such as the manifest path and shard +count substituted in; the key phrase to match in alerting is `REFUSING TO START`.) + +This is intentional — a TopLevel log does not capture per-shard ordering, so +replaying it on a multi-shard instance would produce incorrect key routing. + +**Resolution:** Follow Option A above (remove `appendonlydir/` and restart). + +--- + +## Deprecated flag: --unsafe-multishard-aof + +The `--unsafe-multishard-aof` flag was introduced in v0.1.13 as an escape hatch +to acknowledge the known ~50 % data-loss risk. It is now **deprecated** and will +be removed in v0.2: + +- The underlying bug is fixed — the flag no longer suppresses any safety gate. +- Passing it emits a `[DEPRECATED]` warning at startup. +- If you have scripts or systemd units that pass `--unsafe-multishard-aof`, + remove that flag — it is a no-op. + +--- + +## Monitoring and telemetry + +### INFO Persistence fields added in PR #129 + +``` +aof_backpressure_dropped: ← count of writes dropped due to full AOF channel +``` + +A non-zero value indicates the AOF writer is falling behind write throughput. +Investigate disk I/O or increase `aof-rewrite-min-size`. + +### Prometheus / alerting + +A dedicated gauge for `aof_backpressure_dropped` is planned for v0.2. Until +then, monitor via `INFO persistence` polling. + +--- + +## Crash recovery verification + +Run the CRASH-01-LITE suite to verify your configuration recovers cleanly: + +```bash +# From the moon-dev OrbStack VM: +cargo test --release crash_01_lite 2>&1 | tail -20 +``` + +Expected: 200/200 entries recovered across all shards after SIGKILL. + +--- + +## Escalation + +If you observe data loss after a crash on v0.1.13-patch or later, collect: + +1. `appendonlydir/moon.aof.manifest` contents +2. `appendonlydir/shard-*/` file sizes and modification times +3. Server log from the crashed process (look for AOF writer task exit reason) +4. `INFO persistence` output from the recovered instance + +File a P0 with these artifacts attached. diff --git a/moon-vs-turbovec/METHODOLOGY.md b/moon-vs-turbovec/METHODOLOGY.md new file mode 100644 index 00000000..8d74e426 --- /dev/null +++ b/moon-vs-turbovec/METHODOLOGY.md @@ -0,0 +1,101 @@ +# Methodology + +The result is only worth as much as the protocol. This document is the contract +every number in the repo must satisfy. + +## 1. Hardware & environment (pinned) + +- **Published numbers come from a pinned cloud instance**, never a laptop or a + dev VM. (Per Moon's own rule: production benchmarks target Linux on + GCloud/OrbStack-Linux; macOS/dev numbers are never published.) +- Default reference instance: `c4a-standard-8` (Axion, arm64) **and** + `c3-standard-8` (x86_64) — TurboVec's kernel advantage is arch-specific + (NEON vs AVX-512BW), so we report **both architectures**. +- `harness/hardware.py` captures CPU model, core count, RAM, kernel, compiler, + BLAS backend, and both repos' commit hashes into `results/run_metadata.json`. + No result is valid without this sidecar. + +## 2. Versions (pinned commits) + +| Component | Pin | Where recorded | +|---|---|---| +| Moon | git commit hash + build flags (`--release`, target-cpu) | `run_metadata.json` | +| TurboVec | git commit hash / PyPI version | `run_metadata.json` | +| Datasets | content hash | `datasets/manifest.json` | + +## 3. Datasets — real embeddings only + +| Dataset | Dim | Metric | Why | +|---|---|---|---| +| GloVe | 200 | cosine | TurboVec's strong/likely-win region; exposes Moon's ≤384d TQ4 weakness honestly | +| SIFT1M | 128 | L2 | classic ANN baseline (Moon uses FP32/TQ8 here) | +| OpenAI text-embedding-3 | 1536 | cosine | TurboVec's headline dim; real LLM embeddings | +| OpenAI text-embedding-3 | 3072 | cosine | high-d where TQ4 shines | +| Deep / BIGANN slices | 96–128 | L2 | scale tiers 1M → 10M → 100M for the envelope/disk experiments | + +**No synthetic Gaussian vectors.** Random high-dimensional vectors exhibit +distance concentration that makes recall numbers misleading — especially at the +10M+ tier where it would be tempting to fabricate data. Scale tiers use real +ANN-benchmark sets. + +## 4. Ground truth + +- Exact top-100 neighbors per query via brute-force on the **original float32** + vectors (`datasets/ground_truth.py`, GPU/BLAS accelerated, cached + hashed). +- Identical query set fed to both systems. +- `Recall@k = |returned_topk ∩ true_topk| / k`, averaged over the query set. + +## 5. The iso-recall / iso-memory discipline (non-negotiable) + +Every latency/QPS data point is emitted as a tuple: + +``` +(system, dataset, N, dim, params, recall@10, qps, p50_ms, p95_ms, p99_ms, rss_bytes) +``` + +- **Never** compare a high-recall config of one system against a low-memory + config of the other and report only the favorable axis. +- Latency/QPS claims are made **at matched recall** (interpolated on each + system's Pareto frontier) **or** the full frontier is shown. +- Memory is **resident set size** of the serving process at steady state, and + for Moon **includes the power-of-2 FWHT padding** (e.g. 1536→2048 = +33% + raw vectors before index/graph overhead). This is stated on every Moon row. + +## 6. Parameter policy (both tuned or both default — disclosed) + +- All tunables for both systems live in `config/experiments.yaml`. +- Two modes, both reported when they differ: + - **`default`** — each project's documented defaults. + - **`tuned`** — a *symmetric* sweep: if we sweep Moon's `ef_search`/`M`, we + sweep TurboVec's `bit_width` (2/4) and any documented knob over a comparable + grid. The grid is in the config; nothing is hand-picked post-hoc. + +## 7. Measurement hygiene + +- Warmup: discard first `warmup_queries` (default 1000) before timing. +- Repeats: median of `repeats` runs (default 5); report dispersion. +- Latency: per-query wall time → p50/p95/p99. QPS: single-thread and + `n_threads`-thread (both reported; TurboVec MT vs Moon's per-shard model). +- Client overhead: Moon is measured over the loopback RESP wire (real serving + path); TurboVec in-process. **This asymmetry favors TurboVec and is stated** — + we do not subtract network time to flatter Moon. +- Isolation: one system per process; cgroup memory cap recorded. + +## 8. What we deliberately do NOT do + +- No subtracting Moon's network/serialization cost to fake parity with an + in-process library. +- No reporting Moon-L2 vs TurboVec-IP (different objectives). +- No quoting a 10M number extrapolated from 1M — each tier is measured. +- No populating README tables by hand. `make report` regenerates them from + `results/*.csv`; placeholders stay `` until a real run overwrites + them. + +## 9. Output artifacts + +``` +results/ + run_metadata.json # HW, versions, commit hashes, dataset hashes + raw/*.csv # one row per (system, dataset, N, params) measurement + plots/*.png # Pareto recall-vs-QPS, memory-vs-N, envelope curves +``` diff --git a/moon-vs-turbovec/README.md b/moon-vs-turbovec/README.md new file mode 100644 index 00000000..9de40a3b --- /dev/null +++ b/moon-vs-turbovec/README.md @@ -0,0 +1,170 @@ +# Moon vs TurboVec — A Reproducible Retrieval Benchmark + +> **A decision guide, not a leaderboard.** When should you reach for an embedded +> flat-scan quantizer (TurboVec), and when have you outgrown it and need a +> converged engine (Moon)? This repo runs both, on identical hardware and data, +> and lets the numbers draw the boundary. + +[![reproducible](https://img.shields.io/badge/results-reproducible-brightgreen)](./METHODOLOGY.md) +[![fairness](https://img.shields.io/badge/fairness-iso--recall%20%2B%20iso--memory-blue)](#fairness-statement) + +--- + +## The honest one-liner + +**They are not the same category.** TurboVec is a best-in-class *embedded +flat-scan TurboQuant library* — 2/4-bit compression, inner-product, in-RAM, +Python-native, with a hand-tuned SIMD kernel that beats FAISS FastScan. Moon is +a *converged, networked, durable engine* (KV + vector + graph + search) where +the same TurboQuant family is one quantizer feeding **HNSW / DiskANN / IVF** +indexes. + +So this benchmark does **not** ask "who is faster." It asks: **where is the +boundary** past which a flat-scan library stops being the right tool? + +| Choose **TurboVec** when… | Choose **Moon** when… | +|---|---| +| Corpus fits comfortably in RAM | Corpus exceeds RAM (disk-resident serving) | +| Single-node, embedded in your Python process | You need a networked, multi-tenant service | +| Pure unfiltered top-k on a static set | You need filtered / hybrid / multi-modal retrieval | +| You manage persistence & rebuilds yourself | You need durability, replication, online ingest | +| d ≥ 1536, you want maximum compression | You also need KV / graph / search in one engine | +| You want the fastest single-node scan kernel | You've outgrown "one flat index in a process" | + +The experiments below quantify every row of that table — including **the rows +where TurboVec wins.** + +--- + +## Fairness statement + +This benchmark is designed to be *attackable* — and to survive the attack. +Credibility is the entire point; a rigged comparison is worthless for the +decision it's meant to inform. + +1. **Iso-recall + iso-memory.** Every latency/QPS number is reported *with* the + recall **and** the resident memory at that exact operating point, or as a + full Pareto frontier. A speed win at 5× the RAM or lower recall is reported + as exactly that. +2. **Same metric, same ground truth.** Both systems run cosine / inner-product + on L2-normalized vectors. Ground-truth neighbors are computed by **exact + brute force on the original float32 vectors**, identical query set for both. +3. **Both tuned, or both default — disclosed.** We do not grid-search Moon's + `ef`/`M` while handing TurboVec stock parameters. Every parameter for both + systems is in [`config/experiments.yaml`](./config/experiments.yaml). +4. **Moon's padding is disclosed everywhere.** Moon's FWHT rotation pads to the + next power of two (1536→2048, 3072→4096 ⇒ **+33% vectors** before index + overhead). This is included in *every* Moon memory figure. +5. **Real embeddings at scale.** No synthetic Gaussian vectors (their recall is + misleading). We use standard ANN datasets + real embedding sets at + TurboVec's native dimensions. +6. **Pinned & cloud-run.** Numbers are produced on a pinned cloud instance (not + a laptop/dev VM), with both repos pinned to commit hashes recorded in + [`results/run_metadata.json`](./results/). See [METHODOLOGY.md](./METHODOLOGY.md). + +> **We publish the experiments Moon loses.** A result table that is honest about +> where the flat-scan library is the better choice is the only kind worth +> trusting. + +--- + +## Experiments + +| ID | Question | Battleground | Favors | +|----|----------|--------------|--------| +| **E1** | Recall@10 vs QPS (Pareto) at N = 100K / 1M / 10M | Core retrieval | neutral | +| **E2** | **Operating envelope** — where does each tool stop being viable? (RAM ceiling, filter, durability, ingest) | Capability boundary | Moon (capability) | +| **E3** | Memory footprint vs N (compression) | Storage | **TurboVec** | +| **E4** | Filtered search: allowlist vs payload/expression filters, selectivity sweep | Capability | Moon | +| **E5** | Concurrent online ingest + query throughput | Mixed load | Moon | +| **E6** | Disk-scale (>RAM): DiskANN at 100M where flat-scan OOMs | Capability | Moon | +| **E7** | Convergence / ops capability matrix (qualitative) | Platform | Moon | + +E2 is deliberately **not** "indexed beats brute-force" (a triviality). It holds +recall and hardware fixed and reports the corpus size / RAM / feature point at +which each tool stops being usable. + +### Results (summary) + +> ⚠️ **All values below are `` until produced by the pinned-hardware +> run.** Do not quote any number from this README in marketing until it is +> populated from `results/` by an actual run. Placeholders are intentionally +> non-numeric so they cannot leak as fake data. + +**E1 — Recall@10 / QPS / memory at iso operating points** + +| Dataset (dim) | N | System | Config | Recall@10 | QPS | RAM | Notes | +|---|---|---|---|---|---|---|---| +| GloVe (200) | 1M | TurboVec | 4-bit FastScan | `` | `` | `` | TurboVec's strong/likely-win region | +| GloVe (200) | 1M | Moon | TQ8 HNSW | `` | `` | `` | low-d: Moon uses TQ8/FP32 (TQ4 weak ≤384d) | +| OpenAI (1536) | 1M | TurboVec | 4-bit FastScan | `` | `` | `` | | +| OpenAI (1536) | 1M | Moon | TQ4 HNSW | `` | `` | `` | +33% padding (1536→2048) included in RAM | +| OpenAI (1536) | 10M | TurboVec | 4-bit FastScan | `` | `` | `` | full RAM scan | +| OpenAI (1536) | 10M | Moon | TQ4 HNSW / DiskANN | `` | `` | `` | | + +**E3 — Memory vs N (TurboVec's home turf — co-reported, not buried)** + +| System | 1M @1536 | 10M @1536 | bits | native dim? | +|---|---|---|---|---| +| TurboVec | `` | `` | 2/4 | yes | +| Moon | `` | `` | 1–4 | no (pads to 2048) | + +(Full tables per experiment in [`results/`](./results/) after a run.) + +--- + +## Where TurboVec wins (and you should use it) + +*Populated from the actual run — this section is the credibility anchor and is +expected to contain real losses for Moon.* + +- **Small N, in-RAM:** at N ≤ ~100K the flat-scan FastScan kernel is faster and + far simpler than building/serving an index. `` +- **Low dimension (GloVe-200):** Moon's TQ4 loses recall ≤384d; Moon must fall + back to TQ8/FP32, narrowing or erasing any edge. `` +- **Compression:** native-dimension 2-bit packing vs Moon's padded codes. + `` +- **Single-node kernel throughput:** TurboVec's AVX-512BW/NEON FastScan beats + FAISS; Moon's flat scan is currently scalar/AVX2. `` +- **Embedding ergonomics:** `pip install`, drop-in LangChain/LlamaIndex/ + Haystack/Agno stores, zero ops. + +## Where Moon wins (the envelope) + +- **Past the RAM ceiling** (E2/E6): DiskANN serves corpora that OOM a flat-scan + library. `` +- **Filtered & hybrid retrieval** (E4): payload/expression/text filters + RRF + fusion vs a flat allowlist. `` +- **Online ingest under query load** (E5): MVCC segments vs rebuild-on-add. + `` +- **Convergence** (E7): KV + vector + graph + search + CDC behind one Redis + wire — no Redis+Qdrant+Neo4j+ES glue. +- **Durability & replication:** crash-consistent persistence, master/replica, + cluster failover — structurally absent from a library. + +--- + +## Reproduce it yourself + +```bash +make setup # python deps + build Moon (release) + install turbovec (pinned) +make datasets # download SIFT1M, GloVe-200, OpenAI-1536/3072, compute exact ground truth +make bench # run all experiments on this host +make plot report # render Pareto plots + regenerate result tables in README +``` + +Everything — datasets, parameters, hardware capture, raw CSVs, plot scripts — is +in this repo. If you can't reproduce a number, it's a bug; open an issue. + +See **[METHODOLOGY.md](./METHODOLOGY.md)** for the full protocol and +**[ATTRIBUTION.md](./ATTRIBUTION.md)** for credits and licensing. + +--- + +## Attribution + +[TurboVec](https://github.com/RyanCodrai/turbovec) by Ryan Codrai (MIT) implements +Google Research's TurboQuant. This benchmark uses it unmodified at a pinned +commit. Moon's vector engine implements an independent TurboQuant variant plus +HNSW/DiskANN/IVF. This comparison is built to be fair to both; corrections via +PR/issue are welcome. diff --git a/src/command/connection.rs b/src/command/connection.rs index 9ae20948..1f72422a 100644 --- a/src/command/connection.rs +++ b/src/command/connection.rs @@ -187,7 +187,8 @@ pub fn info(db: &Database, _args: &[Frame]) -> Frame { rdb_last_save_time:{}\r\n\ rdb_last_bgsave_status:{}\r\n\ aof_enabled:0\r\n\ - aof_rewrite_in_progress:0\r\n", + aof_rewrite_in_progress:0\r\n\ + aof_backpressure_dropped:{}\r\n", if crate::command::persistence::SAVE_IN_PROGRESS.load(std::sync::atomic::Ordering::Relaxed) { 1 @@ -202,6 +203,8 @@ pub fn info(db: &Database, _args: &[Frame]) -> Frame { } else { "err" }, + crate::persistence::aof::AOF_BACKPRESSURE_DROPPED + .load(std::sync::atomic::Ordering::Relaxed), )); sections.push_str("\r\n"); diff --git a/src/command/persistence.rs b/src/command/persistence.rs index 8ca5b48e..a1976785 100644 --- a/src/command/persistence.rs +++ b/src/command/persistence.rs @@ -7,9 +7,7 @@ use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use bytes::Bytes; use tracing::{error, info}; -use crate::runtime::channel; - -use crate::persistence::aof::AofMessage; +use crate::persistence::aof::{AofMessage, AofPoolSendError, AofWriterPool}; use crate::persistence::rdb; use crate::protocol::Frame; use crate::storage::Database; @@ -33,6 +31,25 @@ pub static BGSAVE_SHARDS_REMAINING: AtomicU64 = AtomicU64::new(0); /// Whether the last BGSAVE completed successfully. pub static BGSAVE_LAST_STATUS: AtomicBool = AtomicBool::new(true); +/// Process-wide gate set at startup when the configuration combination +/// `--shards >= 2 + --appendonly yes` is selected (see `Config::per_shard_aof_active`). +/// +/// `BGREWRITEAOF` under this combination silently truncates the WAL of every +/// shard except the rewriter's own shard while the consolidated multi-part AOF +/// base RDB written by the rewrite is **not** consumed on restart (verified +/// 2026-05-26 against HEAD `6e49050`: 38 % data loss reproducible). Until the +/// v2.0 multi-part AOF replay walks every shard's segment manifest, the only +/// safe behavior is to refuse the command in this config and point operators +/// at the runbook. +/// +/// Note: `--disk-offload` is NOT part of the gate condition. The unsafe flag +/// fires for any `--shards >= 2 + --appendonly yes` combination regardless of +/// disk-offload state. +/// +/// Set once in `main.rs` after CLI parsing; never cleared. Checked by +/// `bgrewriteaof_start_sharded` before dispatching the rewrite message. +pub static MULTI_SHARD_AOF_REWRITE_UNSAFE: AtomicBool = AtomicBool::new(false); + /// Global flag indicating whether a BGREWRITEAOF rewrite is currently in progress. /// /// Set to `true` when `bgrewriteaof_start` or `bgrewriteaof_start_sharded` dispatches @@ -208,14 +225,31 @@ pub fn bgsave_shard_done(success: bool) { } } +/// Translate an `AofWriterPool` send failure into a user-facing RESP error. +/// Under PerShard layout, `pool.try_send_rewrite` returns +/// `RewriteUnsupportedInPerShard` — the per-shard rewrite path lands in +/// step 6 of the per-shard AOF RFC. Until then BGREWRITEAOF refuses with +/// a stable error rather than silently no-op'ing. +fn rewrite_pool_error_frame(err: AofPoolSendError) -> Frame { + match err { + AofPoolSendError::RewriteUnsupportedInPerShard => Frame::Error(Bytes::from_static( + b"ERR BGREWRITEAOF is not yet supported under per-shard AOF layout; per-shard rewrite ships in step 6 of the per-shard AOF migration", + )), + AofPoolSendError::SendFailed => Frame::Error(Bytes::from_static( + b"ERR Background AOF rewrite failed to start", + )), + } +} + /// Start a background AOF rewrite (BGREWRITEAOF command). /// -/// Sends a Rewrite message to the AOF writer task, which will generate -/// synthetic commands from current database state and replace the AOF file. +/// Submits a Rewrite message through the writer pool, which generates +/// synthetic commands from current database state and replaces the AOF +/// file. /// /// Uses CAS to set `AOF_REWRITE_IN_PROGRESS`: if a rewrite is already running, /// returns an error immediately without corrupting the in-flight rewrite state. -pub fn bgrewriteaof_start(aof_tx: &channel::MpscSender, db: SharedDatabases) -> Frame { +pub fn bgrewriteaof_start(pool: &AofWriterPool, db: SharedDatabases) -> Frame { // CAS: only proceed if currently false; prevents a second caller from // clearing the flag while the first rewrite is still in progress. if AOF_REWRITE_IN_PROGRESS @@ -226,16 +260,15 @@ pub fn bgrewriteaof_start(aof_tx: &channel::MpscSender, db: SharedDa b"ERR Background AOF rewrite already in progress", )); } - match aof_tx.try_send(AofMessage::Rewrite(db)) { + match pool.try_send_rewrite(AofMessage::Rewrite(db)) { Ok(()) => Frame::SimpleString(Bytes::from_static( b"Background append only file rewriting started", )), - Err(_) => { - // Channel send failed — rewrite never started; we set the flag so we clear it. + Err(e) => { + // Send failed (channel full) or PerShard rejection — rewrite never + // started, so clear the in-progress flag we just set. AOF_REWRITE_IN_PROGRESS.store(false, Ordering::SeqCst); - Frame::Error(Bytes::from_static( - b"ERR Background AOF rewrite failed to start", - )) + rewrite_pool_error_frame(e) } } } @@ -245,9 +278,18 @@ pub fn bgrewriteaof_start(aof_tx: &channel::MpscSender, db: SharedDa /// Uses CAS to set `AOF_REWRITE_IN_PROGRESS`: if a rewrite is already running, /// returns an error immediately without corrupting the in-flight rewrite state. pub fn bgrewriteaof_start_sharded( - aof_tx: &channel::MpscSender, + pool: &AofWriterPool, shard_databases: std::sync::Arc, ) -> Frame { + // Refuse the rewrite under the known-unsafe config combo (see the + // MULTI_SHARD_AOF_REWRITE_UNSAFE doc comment). This is the + // single-node v1.0-rc1 gate; the v2.0 multi-part AOF replay fix lifts + // it. + if MULTI_SHARD_AOF_REWRITE_UNSAFE.load(Ordering::Relaxed) { + return Frame::Error(Bytes::from_static( + b"ERR BGREWRITEAOF is not yet supported for --shards >= 2 + --appendonly yes. Options: (1) use --shards 1, (2) set --appendonly no, or (3) wait for per-shard BGREWRITEAOF in v0.2. See docs/runbooks/multi-shard-aof-rewrite.md.", + )); + } // CAS: only proceed if currently false; prevents a second caller from // clearing the flag while the first rewrite is still in progress. if AOF_REWRITE_IN_PROGRESS @@ -258,16 +300,15 @@ pub fn bgrewriteaof_start_sharded( b"ERR Background AOF rewrite already in progress", )); } - match aof_tx.try_send(AofMessage::RewriteSharded(shard_databases)) { + match pool.try_send_rewrite(AofMessage::RewriteSharded(shard_databases)) { Ok(()) => Frame::SimpleString(Bytes::from_static( b"Background append only file rewriting started", )), - Err(_) => { - // Channel send failed — rewrite never started; we set the flag so we clear it. + Err(e) => { + // Send failed (channel full) or PerShard rejection — rewrite never + // started, so clear the in-progress flag we just set. AOF_REWRITE_IN_PROGRESS.store(false, Ordering::SeqCst); - Frame::Error(Bytes::from_static( - b"ERR Background AOF rewrite failed to start", - )) + rewrite_pool_error_frame(e) } } } @@ -358,4 +399,119 @@ mod tests { // Note: verify the static exists and is accessible let _ = BGSAVE_LAST_STATUS.load(Ordering::Relaxed); } + + // Serialize the multi-shard gate test against any other test mutating + // the gate or AOF_REWRITE_IN_PROGRESS (parallel test runner otherwise + // races on the process-wide AtomicBools). + static GATE_TEST_LOCK: parking_lot::Mutex<()> = parking_lot::Mutex::new(()); + + #[test] + fn test_bgrewriteaof_sharded_refuses_under_unsafe_config() { + let _guard = GATE_TEST_LOCK.lock(); + // Use a small bounded channel so the test does not need an AOF + // writer task; the gate must fire BEFORE try_send is reached. + // Wrap as a TopLevel pool to match the post-2e-β helper signature. + let (tx, _rx) = crate::runtime::channel::mpsc_bounded::(1); + let pool = AofWriterPool::top_level(tx); + let shard_dbs = crate::shard::shared_databases::ShardDatabases::new(vec![vec![ + crate::storage::Database::new(), + ]]); + + // Snapshot prior state so the test is order-independent. + let prior = MULTI_SHARD_AOF_REWRITE_UNSAFE.load(Ordering::Relaxed); + let prior_in_progress = AOF_REWRITE_IN_PROGRESS.load(Ordering::SeqCst); + AOF_REWRITE_IN_PROGRESS.store(false, Ordering::SeqCst); + + // Gate ON → must refuse with the documented ERR (and must NOT flip + // AOF_REWRITE_IN_PROGRESS, otherwise a normal rewrite gets blocked). + MULTI_SHARD_AOF_REWRITE_UNSAFE.store(true, Ordering::Relaxed); + let frame = bgrewriteaof_start_sharded(&pool, shard_dbs.clone()); + match frame { + Frame::Error(msg) => { + let s = std::str::from_utf8(&msg).unwrap(); + assert!( + s.contains("BGREWRITEAOF is not yet supported") + && s.contains("multi-shard-aof-rewrite.md"), + "unexpected error: {s}" + ); + } + other => panic!("expected Frame::Error, got {other:?}"), + } + assert!( + !AOF_REWRITE_IN_PROGRESS.load(Ordering::SeqCst), + "gate must not set AOF_REWRITE_IN_PROGRESS" + ); + + // Gate OFF → the gate error must NOT fire. (Without an AOF writer + // task draining the channel, the second call may succeed or return + // "failed to start" depending on buffer state; the contract under + // test here is only that the gate error is gone.) + MULTI_SHARD_AOF_REWRITE_UNSAFE.store(false, Ordering::Relaxed); + AOF_REWRITE_IN_PROGRESS.store(false, Ordering::SeqCst); + let frame2 = bgrewriteaof_start_sharded(&pool, shard_dbs); + if let Frame::Error(msg) = &frame2 { + let s = std::str::from_utf8(msg).unwrap(); + assert!( + !s.contains("BGREWRITEAOF is not yet supported"), + "gate error fired with gate off: {s}" + ); + } + + // Restore prior state. + AOF_REWRITE_IN_PROGRESS.store(prior_in_progress, Ordering::SeqCst); + MULTI_SHARD_AOF_REWRITE_UNSAFE.store(prior, Ordering::Relaxed); + } + + /// FIX-W1-4 r2: gate error message must NOT mention disk-offload (the gate + /// fires for ANY `--shards >= 2 + --appendonly yes` config, regardless of + /// disk-offload setting) and MUST end with the runbook reference. + /// + /// Red state (pre-fix, 881f8b8^): error contained "disk-offload enable" + /// and recommended "set --disk-offload disable" — stale from the narrower + /// original gate condition. + /// + /// Green (post-fix): message updated to accurate condition, no disk-offload + /// mention, ends with "multi-shard-aof-rewrite.md." + #[test] + fn test_bgrewriteaof_gate_error_no_disk_offload_mention() { + let _guard = GATE_TEST_LOCK.lock(); + let (tx, _rx) = crate::runtime::channel::mpsc_bounded::(1); + let pool = AofWriterPool::top_level(tx); + let shard_dbs = crate::shard::shared_databases::ShardDatabases::new(vec![vec![ + crate::storage::Database::new(), + ]]); + + let prior = MULTI_SHARD_AOF_REWRITE_UNSAFE.load(Ordering::Relaxed); + let prior_in_progress = AOF_REWRITE_IN_PROGRESS.load(Ordering::SeqCst); + AOF_REWRITE_IN_PROGRESS.store(false, Ordering::SeqCst); + MULTI_SHARD_AOF_REWRITE_UNSAFE.store(true, Ordering::Relaxed); + + let frame = bgrewriteaof_start_sharded(&pool, shard_dbs); + + MULTI_SHARD_AOF_REWRITE_UNSAFE.store(prior, Ordering::Relaxed); + AOF_REWRITE_IN_PROGRESS.store(prior_in_progress, Ordering::SeqCst); + + match frame { + Frame::Error(msg) => { + let s = std::str::from_utf8(&msg).unwrap(); + assert!( + !s.contains("disk-offload"), + "gate error must NOT mention disk-offload \ + (gate fires for --shards>=2 + --appendonly yes regardless \ + of disk-offload state): {s}" + ); + assert!( + s.ends_with("multi-shard-aof-rewrite.md."), + "gate error MUST end with the runbook reference \ + 'multi-shard-aof-rewrite.md.' for operator guidance: {s}" + ); + assert!( + s.contains("--shards 1") && s.contains("--appendonly no"), + "gate error must offer actionable alternatives \ + (--shards 1 and --appendonly no): {s}" + ); + } + other => panic!("expected Frame::Error when gate is ON, got {other:?}"), + } + } } diff --git a/src/config.rs b/src/config.rs index b384a818..13b4b475 100644 --- a/src/config.rs +++ b/src/config.rs @@ -102,6 +102,21 @@ pub struct ServerConfig { #[arg(long, default_value = "yes")] pub appendonly: String, + /// [DEPRECATED — will be removed in v0.2] This flag is now a no-op. + /// + /// Historically, `--shards >= 2 + --appendonly yes` lost ~50 % of + /// writes on SIGKILL (verified 2026-05-26, HEAD `6e49050`). The flag + /// was an escape hatch to acknowledge the risk. + /// + /// As of PR #129 the per-shard AOF architecture is fully crash-safe + /// (CRASH-01-LITE: 200/200 SIGKILL recovery). The startup refusal gate + /// has been lifted. Passing this flag now only emits a `[DEPRECATED]` + /// warning at startup and has no other effect. Remove it from your + /// launch command or systemd unit. See + /// `docs/runbooks/multi-shard-aof-rewrite.md`. + #[arg(long, default_value_t = false)] + pub unsafe_multishard_aof: bool, + /// AOF fsync policy (always/everysec/no) #[arg(long, default_value = "everysec")] pub appendfsync: String, @@ -495,6 +510,30 @@ pub struct ServerConfig { #[arg(long = "autovacuum-starvation-cap-secs", default_value_t = 300)] pub autovacuum_starvation_cap_secs: u64, + // ── AOF v1→v2 migration ──────────────────────────────────────────── + /// Source directory containing a legacy single-file AOF (`appendonly.aof` + /// or TopLevel manifest layout). When this flag is set the server runs + /// the migration tool, writes the v2 PerShard layout to `--migrate-aof-to`, + /// and exits. Do NOT combine with normal server startup flags. + /// + /// Example: + /// moon --migrate-aof-from /old/dir --migrate-aof-to /new/dir \ + /// --migrate-aof-shards 4 + #[arg(long = "migrate-aof-from", value_name = "PATH")] + pub migrate_aof_from: Option, + + /// Destination directory for the v2 PerShard AOF layout produced by + /// `--migrate-aof-from`. The directory is created if absent; it must be + /// empty (or non-existent) to prevent accidental overwrites. + #[arg(long = "migrate-aof-to", value_name = "PATH")] + pub migrate_aof_to: Option, + + /// Number of target shards for the migration. Must match the `--shards` + /// value you will use when starting the server on the migrated data. + /// Defaults to 0 (invalid — must be set when `--migrate-aof-from` is used). + #[arg(long = "migrate-aof-shards", default_value_t = 0)] + pub migrate_aof_shards: u16, + // ── Shared-nothing migration (Phase 0) ───────────────────────────── /// Control whether cross-shard reads use the shared-read fast path. /// @@ -535,6 +574,19 @@ impl ServerConfig { self.wal_fpi == "enable" } + /// Returns true when the per-shard AOF layout is active. + /// + /// Per-shard AOF is selected whenever `--shards >= 2` and + /// `--appendonly yes`. In this layout each shard owns its own + /// `appendonlydir/shard-{N}/` directory and a dedicated + /// `per_shard_aof_writer_task`. Operations that touch the single + /// consolidated `appendonly.aof` file (e.g. BGREWRITEAOF) are not + /// supported in this layout until the multi-part AOF rewrite ships. + #[inline] + pub fn per_shard_aof_active(&self, num_shards: usize) -> bool { + num_shards >= 2 && self.appendonly == "yes" + } + /// Returns true when vector codes pages should be mlocked. pub fn vec_codes_mlock_enabled(&self) -> bool { self.vec_codes_mlock == "enable" @@ -1029,4 +1081,59 @@ mod tests { assert_eq!(config.vec_diskann_beam_width, 16); assert_eq!(config.vec_diskann_cache_levels, 5); } + + /// FIX-W1-4: per_shard_aof_active must be true only when both + /// num_shards >= 2 AND appendonly=yes are set, and false for every + /// other combination. This predicate drives the BGREWRITEAOF gate in + /// main.rs — a false negative silently allows the unsafe rewrite path. + #[test] + fn test_per_shard_aof_active_predicate() { + // Base config: appendonly=yes, shards=2 → active + let mut config = ServerConfig::parse_from(["moon", "--appendonly", "yes"]); + assert!( + config.per_shard_aof_active(2), + "must be active with shards=2 and appendonly=yes" + ); + assert!( + config.per_shard_aof_active(4), + "must be active with shards=4 and appendonly=yes" + ); + + // shards=1 → not active (single-shard uses TopLevel AOF) + assert!( + !config.per_shard_aof_active(1), + "must be inactive with shards=1 even if appendonly=yes" + ); + + // appendonly=no → not active regardless of shard count + config.appendonly = "no".to_string(); + assert!( + !config.per_shard_aof_active(2), + "must be inactive when appendonly=no" + ); + assert!( + !config.per_shard_aof_active(4), + "must be inactive when appendonly=no with 4 shards" + ); + + // shards=0 (auto-detect placeholder) → not active + config.appendonly = "yes".to_string(); + assert!( + !config.per_shard_aof_active(0), + "must be inactive when num_shards=0" + ); + + // disk_offload has no bearing on this predicate (FIX-W1-4 broadened + // the gate to not require disk_offload). + config.disk_offload = "enable".to_string(); + assert!( + config.per_shard_aof_active(2), + "must remain active with disk_offload=enable (predicate is orthogonal)" + ); + config.disk_offload = "disable".to_string(); + assert!( + config.per_shard_aof_active(2), + "must remain active with disk_offload=disable" + ); + } } diff --git a/src/main.rs b/src/main.rs index 8bff34ac..0e0cf082 100644 --- a/src/main.rs +++ b/src/main.rs @@ -44,7 +44,7 @@ use std::path::PathBuf; use clap::Parser; use moon::config::ServerConfig; -use moon::persistence::aof::{self, AofMessage, FsyncPolicy}; +use moon::persistence::aof::{self, AofMessage, AofWriterPool, FsyncPolicy}; use moon::runtime::cancel::CancellationToken; use moon::runtime::channel; use moon::runtime::{RuntimeFactoryImpl, traits::RuntimeFactory}; @@ -70,6 +70,46 @@ fn main() -> anyhow::Result<()> { let config = ServerConfig::parse(); + // ── AOF v1→v2 migration (FIX-W3-2): early-exit before normal boot ── + // When `--migrate-aof-from` is set, run the migration tool and exit. + // This must run BEFORE any shard/AOF initialization so the source + // directory is never modified and the destination is populated atomically. + if let Some(ref from) = config.migrate_aof_from { + let to = config.migrate_aof_to.as_deref().ok_or_else(|| { + anyhow::anyhow!("--migrate-aof-to is required when --migrate-aof-from is set") + })?; + if config.migrate_aof_shards == 0 { + return Err(anyhow::anyhow!( + "--migrate-aof-shards must be >= 1 when --migrate-aof-from is set" + )); + } + info!( + "Running AOF migration: {} → {} ({} shards)", + from.display(), + to.display(), + config.migrate_aof_shards + ); + // Create destination directory if absent. + if let Err(e) = std::fs::create_dir_all(to) { + return Err(anyhow::anyhow!( + "Failed to create migration destination directory {}: {}", + to.display(), + e + )); + } + let result = + moon::persistence::migrate_aof::migrate_aof(from, to, config.migrate_aof_shards) + .map_err(|e| anyhow::anyhow!("AOF migration failed: {}", e))?; + info!( + "AOF migration complete: {} RDB keys migrated, {} commands read, {} written, {} skipped", + result.rdb_keys_migrated, + result.commands_read, + result.commands_written, + result.commands_skipped + ); + return Ok(()); + } + // Non-jemalloc builds: warn if operator explicitly set --memory-arenas-cap #[cfg(not(feature = "jemalloc"))] if config.memory_arenas_cap != 8 { @@ -270,6 +310,32 @@ fn main() -> anyhow::Result<()> { info!("Starting with {} shards", num_shards); + // Checked cast: --shards is bounded by clap's value_parser, but `as u16` + // would silently wrap for values > 65535. Fail loudly instead. + // ALLOW: panic is appropriate here — this is `main`, not library code. + #[allow(clippy::expect_used)] + let shard_count_u16: u16 = u16::try_from(num_shards).expect("--shards must be <= 65535"); + + // P0-FIX-01b LIFTED (Option B step 9, 2026-06-01): the per-shard AOF + // pipeline (RFC steps 1-8) makes `--shards >= 2 + --appendonly yes` + // crash-safe. CRASH-01-LITE confirms 200/200 keys recover after + // SIGKILL on a 2-shard everysec config; manual disk inspection shows + // framed `[u64 lsn LE][u32 len LE][RESP]` entries in each shard's + // file. The startup refusal is no longer needed. + // + // `--unsafe-multishard-aof` is preserved as a no-op flag so existing + // operator runbooks and CI command lines do not break — the flag + // emits a one-line info notice if explicitly set, then proceeds as + // if it were not. Removing the flag entirely is a future cleanup + // once dependents have been audited. + if config.unsafe_multishard_aof { + tracing::warn!( + "[DEPRECATED] --unsafe-multishard-aof is a no-op and will be removed in v0.2. \ + Per-shard AOF is crash-safe as of PR #129 (CRASH-01-LITE: 200/200). \ + Remove this flag from your launch command or systemd unit." + ); + } + // T1.1: warn when maxclients < 25 × shards (undersubscription footgun). // Suppressed by MOON_NO_UNDERSUBSCRIPTION_WARN=1. if let Some(msg) = should_warn_undersubscription(config.maxclients, num_shards) @@ -287,30 +353,154 @@ fn main() -> anyhow::Result<()> { // Collect connection senders for the listener before spawning shard threads let conn_txs: Vec<_> = (0..num_shards).map(|i| mesh.conn_tx(i)).collect(); - // Set up AOF channel: single writer, all shards send to it via mpsc::Sender clones. - // The AOF writer task will be spawned on the listener runtime. - let aof_tx: Option> = if config.appendonly == "yes" { - let (tx, rx) = channel::mpsc_bounded::(10_000); - let aof_token = cancel_token.child_token(); - let fsync = FsyncPolicy::from_str(&config.appendfsync); - let aof_file_path = PathBuf::from(&config.dir).join(&config.appendfilename); - // AOF writer task will be spawned on the listener runtime (see below) - // We store rx to spawn later since listener_rt hasn't been created yet. - // Instead, spawn on a dedicated thread so it's available before listener starts. - std::thread::Builder::new() - .name("aof-writer".to_string()) - .spawn(move || { - RuntimeFactoryImpl::block_on_local( - "aof-writer".to_string(), - aof::aof_writer_task(rx, aof_file_path, fsync, aof_token), + // Set up AOF writer channel(s) + `AofWriterPool` (step 2f-β: layout-aware). + // + // Logic: + // 1. If `appendonly == "yes"` and an existing on-disk manifest is found, + // verify its shard count matches `--shards` (RFC § 3 refusal — a + // mismatch silently maps shards to the wrong AOF files and is fatal). + // 2. If the manifest's layout is `PerShard` AND `num_shards >= 2`, + // spawn one writer per shard (`aof-writer-{N}` threads) and emit a + // `AofWriterPool::per_shard(senders)`. + // 3. Otherwise spawn the single legacy writer and emit + // `AofWriterPool::top_level(tx)`. This includes: + // - no manifest yet (fresh install — `initialize()` only writes + // TopLevel today; fresh-install PerShard creation lands later + // in the RFC sequence) + // - existing TopLevel manifest (legacy v1 or single-shard v2) + // - `num_shards == 1` (always TopLevel; per-shard fan-out has no + // meaning when there is one shard) + // + // A *corrupt* manifest is fatal — `AofManifest::load` returning `Err(_)` + // must NOT silently fall back to TopLevel, because the next write would + // create a fresh manifest overwriting the reference to the real base RDB + // and lose data. This mirrors the replay block at L514–526. + // + // Note: nothing today constructs a `layout == PerShard` manifest on disk + // (initialize() hardcodes TopLevel, migrate_top_level_to_per_shard is not + // yet wired into boot). The PerShard branch is reachable only by a + // hand-crafted manifest until step 9 lifts the multi-shard gate. Runtime + // behavior under default configurations stays byte-identical to step 2f-α. + use moon::persistence::aof_manifest::{AofLayout, AofManifest}; + let existing_manifest: Option = if config.appendonly == "yes" { + let base_dir = PathBuf::from(&config.dir); + match AofManifest::load(&base_dir) { + Ok(opt) => opt, + Err(e) => { + eprintln!( + "REFUSING TO START: AOF manifest at {}/appendonlydir/ is corrupt: {}. \ + Inspect manually before deleting; overwriting silently loses data.", + base_dir.display(), + e ); - }) - .expect("failed to spawn AOF writer thread"); - info!( - "AOF enabled with fsync policy: {:?}", - FsyncPolicy::from_str(&config.appendfsync) + std::process::exit(2); + } + } + } else { + None + }; + // TopLevel + multi-shard refusal — hoisted here so it fires on both + // runtime-monoio and runtime-tokio. The TopLevel manifest (v1, shards=1) + // combined with --shards >= 2 silently loses data for shards 1..N because + // the single shared AOF replays everything into shard 0 while shards 1..N + // start empty. This check fires unconditionally on both runtimes; the + // duplicate check inside the runtime-monoio recovery block (further below) + // is now dead code but harmless — it guards operator data against future + // code paths that bypass this early exit. + if let Some(ref m) = existing_manifest + && m.layout == AofLayout::TopLevel + && num_shards >= 2 + { + let aof_base = std::path::Path::new(&config.dir).join("appendonlydir"); + let manifest_path = aof_base.join("moon.aof.manifest"); + let num_shards_minus_one = num_shards - 1; + eprintln!( + "REFUSING TO START: legacy TopLevel AOF manifest at {manifest_path} \ + detected with --shards {num_shards} (>= 2). \ + This combination silently loses data for shards 1..={num_shards_minus_one}. \ + To migrate: stop the server, remove {aof_dir}, then restart with \ + --shards {num_shards} --appendonly yes (Moon creates a fresh per-shard \ + manifest; load prior state from dump.rdb first if needed). \ + See docs/runbooks/multi-shard-aof-rewrite.md for full migration instructions.", + manifest_path = manifest_path.display(), + num_shards = num_shards, + num_shards_minus_one = num_shards_minus_one, + aof_dir = aof_base.display(), ); - Some(tx) + std::process::exit(2); + } + // Shard-count mismatch guard for non-TopLevel manifests (PerShard layout + // with a different shard count than currently configured). A v1 TopLevel + // manifest always records shards=1; that case is already handled above. + if let Some(ref m) = existing_manifest + && let Err(e) = m.verify_shard_count(shard_count_u16) + { + eprintln!("REFUSING TO START: {e}"); + std::process::exit(2); + } + + let aof_pool: Option> = if config.appendonly == "yes" { + let fsync = FsyncPolicy::from_str(&config.appendfsync); + // PerShard writers required when num_shards >= 2 AND we'll have a + // PerShard manifest at runtime. Two cases produce PerShard: + // 1. existing manifest is already PerShard, OR + // 2. no manifest yet (first boot) — main.rs will call + // `initialize_multi(num_shards)` later in the recovery block. + // The legacy case (existing TopLevel manifest on a multi-shard + // deployment) sticks with the TopLevel writer pending the migrate-aof + // tool — the multi-shard replay branch already warns about this. + let multi_shard_no_manifest = existing_manifest.is_none() && num_shards >= 2; + let use_per_shard = num_shards >= 2 + && (matches!( + existing_manifest.as_ref().map(|m| m.layout), + Some(AofLayout::PerShard) + ) || multi_shard_no_manifest); + + if use_per_shard { + let base_dir = PathBuf::from(&config.dir); + let mut senders = Vec::with_capacity(num_shards); + for sid in 0..num_shards { + let (tx, rx) = channel::mpsc_bounded::(10_000); + let aof_token = cancel_token.child_token(); + let base_dir = base_dir.clone(); + let thread_name = format!("aof-writer-{sid}"); + let thread_name_inner = thread_name.clone(); + std::thread::Builder::new() + .name(thread_name) + .spawn(move || { + RuntimeFactoryImpl::block_on_local( + thread_name_inner, + aof::per_shard_aof_writer_task( + rx, base_dir, sid as u16, fsync, aof_token, + ), + ); + }) + .expect("failed to spawn per-shard AOF writer thread"); + senders.push(tx); + } + info!( + "AOF enabled (PerShard, {} writers, fsync: {:?})", + num_shards, fsync + ); + Some(AofWriterPool::per_shard_with_policy(senders, fsync)) + } else { + let (tx, rx) = channel::mpsc_bounded::(10_000); + let aof_token = cancel_token.child_token(); + let aof_file_path = PathBuf::from(&config.dir).join(&config.appendfilename); + // Legacy single-writer thread. Each shard clones the outer + // `aof_pool` Arc; sender lifetime is governed by the pool's Drop. + std::thread::Builder::new() + .name("aof-writer".to_string()) + .spawn(move || { + RuntimeFactoryImpl::block_on_local( + "aof-writer".to_string(), + aof::aof_writer_task(rx, aof_file_path, fsync, aof_token), + ); + }) + .expect("failed to spawn AOF writer thread"); + info!("AOF enabled (TopLevel, fsync: {:?})", fsync); + Some(AofWriterPool::top_level_with_policy(tx, fsync)) + } } else { None }; @@ -318,6 +508,26 @@ fn main() -> anyhow::Result<()> { // Compute bind address for SO_REUSEPORT per-shard listeners (Linux io_uring path). let bind_addr = format!("{}:{}", config.bind, config.port); + // FIX-W1-4: gate BGREWRITEAOF whenever per-shard AOF is active + // (num_shards >= 2 + appendonly=yes). The original gate was too narrow: + // it required disk_offload to be enabled, missing the plain AOF case. + // Per-shard rewrite is not yet implemented (AofPoolSendError:: + // RewriteUnsupportedInPerShard); the pool already refuses the message, + // but this early gate provides a stable, documented error to operators + // BEFORE the channel send so no in-progress flag is flipped. + // Verified 2026-05-26: multi-shard BGREWRITEAOF loses ~38% of keys on + // restart. Gate lifted only when multi-part AOF replay ships (v2.0+). + // See docs/runbooks/multi-shard-aof-rewrite.md. + if config.per_shard_aof_active(num_shards) { + moon::command::persistence::MULTI_SHARD_AOF_REWRITE_UNSAFE + .store(true, std::sync::atomic::Ordering::Relaxed); + tracing::warn!( + shards = num_shards, + appendonly = %config.appendonly, + "BGREWRITEAOF gated: per-shard AOF layout active (see docs/runbooks/multi-shard-aof-rewrite.md). Use --shards 1 to re-enable rewrite." + ); + } + // Create watch channel for snapshot triggers (auto-save and BGSAVE) let (snapshot_trigger_tx, snapshot_trigger_rx) = moon::runtime::channel::watch(0u64); @@ -524,8 +734,135 @@ fn main() -> anyhow::Result<()> { ); } } + } else if manifest.layout == moon::persistence::aof_manifest::AofLayout::PerShard { + // Per-shard AOF replay (RFC § 2 rules 1-3, Option B step 4). + // + // Wipe any state earlier recovery phases loaded for each shard — + // base RDB + framed incr together are authoritative for that + // shard, and non-idempotent commands in the incr stream would + // otherwise double-apply on top of WAL/legacy state. + for shard in shards.iter_mut() { + for db in shard.databases.iter_mut() { + db.clear(); + } + } + + // Borrow each shard's `databases` mutably and route through + // `replay_per_shard`. The split_at_mut walk constructs a + // Vec<&mut [Database]> without aliasing, which `replay_per_shard` + // requires. + // + // `replay_per_shard` now spawns one thread per shard via + // `std::thread::scope`. The factory closure produces an independent + // `DispatchReplayEngine` per thread, avoiding the `!Sync` `RefCell` + // conflict that would arise from sharing a single engine instance + // across threads (under the `graph` feature). + let engine_factory = + || -> Box { + Box::new(DispatchReplayEngine::new()) + }; + let (total, global_max_lsn, ordered_entries) = { + let mut slices: Vec<&mut [moon::storage::Database]> = + Vec::with_capacity(shards.len()); + let mut rest: &mut [moon::shard::Shard] = &mut shards[..]; + while let Some((head, tail)) = rest.split_first_mut() { + slices.push(&mut head.databases); + rest = tail; + } + moon::persistence::aof_manifest::replay_per_shard( + &mut slices, + manifest, + &engine_factory, + ) + .with_context(|| "per-shard AOF replay failed")? + }; + + // Step 5: merge-replay `OrderedAcrossShards`-tagged entries + // in global LSN order. Today this list is always empty + // (no production emitter); the path exists so the future + // cross-shard TXN consumer wires in without a recovery + // re-design. + let ordered_engine = DispatchReplayEngine::new(); + let ordered_count = if !ordered_entries.is_empty() { + let mut slices: Vec<&mut [moon::storage::Database]> = + Vec::with_capacity(shards.len()); + let mut rest: &mut [moon::shard::Shard] = &mut shards[..]; + while let Some((head, tail)) = rest.split_first_mut() { + slices.push(&mut head.databases); + rest = tail; + } + moon::persistence::aof_manifest::replay_ordered_merge( + &mut slices, + ordered_entries, + &ordered_engine, + ) + .with_context(|| "per-shard AOF ordered merge replay failed")? + } else { + 0 + }; + + info!( + "AOF per-shard loaded (seq {}): {} entries across {} shards (global max lsn {}, ordered merge {} entries)", + manifest.seq, + total, + manifest.shards.len(), + global_max_lsn, + ordered_count + ); + + // RFC § 2 Rule 3 — seed master_repl_offset before accepting + // client traffic so the next write doesn't reissue an LSN + // already on disk. + if global_max_lsn > 0 + && let Ok(state) = repl_state.read() + { + state.seed_master_offset(global_max_lsn); + } + + // Retire any stray legacy top-level appendonly.aof so the + // next boot doesn't double-replay it via v2 recovery in + // `restore_from_persistence`. + let legacy = base_dir.join("appendonly.aof"); + if legacy.exists() { + let retired = base_dir.join("appendonly.aof.legacy"); + if let Err(e) = std::fs::rename(&legacy, &retired) { + tracing::warn!("Failed to retire legacy AOF {}: {}", legacy.display(), e); + } else { + info!( + "Retired legacy AOF {} → {}", + legacy.display(), + retired.display() + ); + } + } } else { - tracing::warn!("Multi-part AOF skipped in multi-shard mode (not yet supported)"); + // TopLevel manifest (v1 / single-file layout) combined with + // --shards >= 2 is an unsafe combination: replaying a single + // shared AOF file into multiple shards would assign all data + // to shard 0 while shards 1..N start empty. This silently + // loses data that was written to shards 1..N before the + // manifest was last updated. + // + // Previously a warn! + continue, which allowed the server to + // boot with an empty AOF state. Now a hard refusal so the + // operator is forced to migrate before proceeding. + eprintln!( + "REFUSING TO START: legacy TopLevel AOF manifest at {manifest_path} \ + detected with --shards {num_shards} (>= 2). \ + This combination silently loses data for shards 1..={num_shards_minus_one}. \ + To migrate: stop the server, remove {aof_dir}, then restart with \ + --shards {num_shards} --appendonly yes (Moon creates a fresh per-shard \ + manifest; load prior state from dump.rdb first if needed). \ + See docs/runbooks/multi-shard-aof-rewrite.md for full migration instructions.", + manifest_path = base_dir + .join("appendonlydir") + .join("moon.aof.manifest") + .display(), + num_shards = num_shards, + num_shards_minus_one = num_shards - 1, + aof_dir = base_dir.join("appendonlydir").display(), + ); + std::process::exit(2); } } else { // No manifest present — first boot after upgrade from legacy @@ -558,6 +895,20 @@ fn main() -> anyhow::Result<()> { tracing::warn!("Failed to retire legacy AOF {}: {}", legacy.display(), e); } } + } else if num_shards >= 2 { + // Multi-shard fresh boot: create the PerShard manifest layout + // (RFC § 3) instead of the legacy single-file TopLevel layout. + // Step 2f-β's spawn-site gate only enables PerShard writers + // when the loaded manifest's layout is PerShard, so without + // this branch a multi-shard --appendonly yes deployment would + // silently fall back to TopLevel and lose data on restart. + AofManifest::initialize_multi(&base_dir, shard_count_u16) + .with_context(|| "failed to initialize PerShard AOF manifest")?; + info!( + "Initialized PerShard AOF manifest for {} shards at {}", + num_shards, + base_dir.display() + ); } else { AofManifest::initialize(&base_dir) .with_context(|| "failed to initialize AOF manifest")?; @@ -644,7 +995,7 @@ fn main() -> anyhow::Result<()> { } let conn_rx = mesh.take_conn_rx(id); let shard_cancel = cancel_token.clone(); - let shard_aof_tx = aof_tx.clone(); + let shard_aof_pool = aof_pool.clone(); let shard_bind_addr = bind_addr.clone(); let shard_persistence_dir = persistence_dir.clone(); let shard_snap_rx = snapshot_trigger_rx.clone(); @@ -676,7 +1027,7 @@ fn main() -> anyhow::Result<()> { consumers, producers, shard_cancel, - shard_aof_tx, + shard_aof_pool, // Only pass bind_addr for per-shard SO_REUSEPORT when tokio // with io_uring is active. monoio uses central listener MPSC. #[cfg(feature = "runtime-tokio")] @@ -862,9 +1213,11 @@ fn main() -> anyhow::Result<()> { }); } - // After listener exits, send AOF shutdown and cancel all shards - if let Some(ref tx) = aof_tx { - let _ = tx.send(AofMessage::Shutdown); + // After listener exits, send AOF shutdown to every writer and cancel all shards. + // Under TopLevel this is one send; under PerShard (step 2f-β) this fans out to + // every per-shard writer thread via `broadcast_shutdown`. + if let Some(ref pool) = aof_pool { + pool.broadcast_shutdown(); } cancel_token.cancel(); for handle in shard_handles { diff --git a/src/persistence/aof.rs b/src/persistence/aof.rs index a11bb7c5..d4d3bd5a 100644 --- a/src/persistence/aof.rs +++ b/src/persistence/aof.rs @@ -33,6 +33,65 @@ use crate::storage::entry::{Entry, current_time_ms}; /// Type alias for the per-database RwLock container. type SharedDatabases = Arc>>; +/// Canonical AOF fsync failure error string sent to the client as a +/// `Frame::Error` when `appendfsync=always` and the writer task does not +/// confirm durability before the response. +/// +/// All handler variants (handler_single, handler_monoio, handler_sharded) +/// MUST use this constant so operators see a consistent error regardless of +/// which connection path handles the request. +/// +/// Redis convention: errors begin with a single-word code (`ERR` for generic +/// failures) followed by a space and a human-readable message. +pub const AOF_FSYNC_ERR: &[u8] = b"ERR AOF fsync failed; write not durable"; + +/// High bit of the per-entry LSN reserved for `OrderedAcrossShards` +/// (RFC § 2 Rule 2). When set on a per-shard AOF entry, recovery treats +/// the entry as participating in a cross-shard atomic operation and +/// buffers it for the cross-shard merge replay after per-shard replay +/// completes. +/// +/// Practical LSN ceilings (even at 10 M writes/s sustained for a century) +/// sit near 2^58, so reserving bit 63 has no observable effect on normal +/// writes — the bit is always 0 in entries written by `try_send_append`. +/// Only `try_send_append_ordered` sets it. +pub const ORDERED_LSN_FLAG: u64 = 1u64 << 63; + +/// Outcome reported by the writer task back to an `AppendSync` caller +/// once the rendezvous completes. +/// +/// `Synced` is sent AFTER `sync_data()` returns successfully — the +/// caller may safely `+OK` the client. `WriteFailed`/`FsyncFailed` +/// surface the failure mode so the caller can return a specific error +/// frame; either way, durability was NOT achieved. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AofAck { + /// Bytes were written and fsynced. Durability guaranteed. + Synced, + /// `write_all()` returned an error. The entry may be partially on + /// disk; recovery handles partial-payload truncation as crash EOF. + WriteFailed, + /// `write_all()` succeeded but `sync_data()` returned an error. The + /// entry is in the kernel buffer but NOT on durable storage. + FsyncFailed, + /// The writer channel was full at the time of the send — the entry + /// was **not** enqueued. This is a backpressure signal: the writer + /// is unable to keep up with the current write rate. Callers MUST + /// treat this as a hard failure (same as `WriteFailed`) under + /// `appendfsync=always`; for `everysec`/`no` it is logged and counted. + ChannelFull, +} + +/// Global counter incremented each time an AOF `AppendSync` (or fire-and- +/// forget `Append`) is dropped because the writer channel was at capacity. +/// +/// Exposed under `# Persistence` in the INFO command as +/// `aof_backpressure_dropped`. A persistently non-zero value indicates the +/// writer is a bottleneck and the operator should investigate disk I/O or +/// switch to `appendfsync=everysec`. +pub static AOF_BACKPRESSURE_DROPPED: std::sync::atomic::AtomicU64 = + std::sync::atomic::AtomicU64::new(0); + /// AOF fsync policy controlling when data is flushed to disk. #[derive(Debug, Clone, Copy, PartialEq)] pub enum FsyncPolicy { @@ -57,8 +116,46 @@ impl FsyncPolicy { /// Messages sent to the AOF writer task via mpsc channel. pub enum AofMessage { - /// Append serialized RESP command bytes to the AOF file. - Append(Bytes), + /// Append serialized RESP command bytes to the AOF file, tagged with the + /// LSN that was issued for this write (`ReplicationState::issue_lsn`). + /// + /// `lsn` semantics by writer task: + /// - **TopLevel** (`aof_writer_task`): `lsn` is **ignored**; the legacy + /// v1 disk format is plain RESP bytes with no per-entry framing. + /// - **PerShard** (`per_shard_aof_writer_task`): `lsn` is **written** as + /// a u64 header per RFC § 2 Rule 1. Disk format per entry: + /// `[u64 lsn LE][u32 len LE][RESP bytes of length len]`. + /// Recovery reads `(lsn, cmd)` pairs and merges cross-shard + /// `OrderedAcrossShards` writes by LSN (RFC § 2 Rule 2). + /// + /// Construction sites that issue a real LSN call + /// `ReplicationState::issue_lsn(shard_id, bytes.len() as u64)` and pass + /// the returned value. Sites with no replication state available pass 0 + /// (TopLevel ignores it; PerShard treats 0 as "no ordering hint"). + Append { lsn: u64, bytes: Bytes }, + /// Append + fsync + ack rendezvous (RFC § 4 — Fix 2 for the H1 + /// data-loss vector exposed by `appendfsync=always`). + /// + /// Same encoding as [`AofMessage::Append`], but the writer task ALWAYS + /// fsyncs after writing the payload and signals `ack` ONCE the + /// `sync_data()` syscall returns. The caller is expected to await + /// `ack` before responding `+OK` to the client so the durability + /// contract of `appendfsync=always` is honoured end-to-end. + /// + /// Failure semantics: on write or fsync error the writer drops `ack` + /// without sending — the caller's `OneshotReceiver` resolves with + /// `RecvError`, which it must treat as a hard failure (return an + /// error frame to the client, do NOT silently +OK). + /// + /// Production callers: none in step 7 — this commit ships the + /// mechanism plus tests. Per-handler integration (which sites use + /// AppendSync vs Append) is wired in step 9 before lifting the + /// `--unsafe-multishard-aof` gate. + AppendSync { + lsn: u64, + bytes: Bytes, + ack: crate::runtime::channel::OneshotSender, + }, /// Trigger a full AOF rewrite (compaction) using current database state. Rewrite(SharedDatabases), /// Trigger AOF rewrite in sharded mode (all shards' databases). @@ -67,6 +164,768 @@ pub enum AofMessage { Shutdown, } +/// Reasons a pool send may be refused without queueing. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum AofPoolSendError { + /// `Rewrite`/`RewriteSharded` sent to a `PerShard` pool. BGREWRITEAOF must + /// be issued per shard in the per-shard layout; the legacy single-writer + /// rewrite path is not applicable. + RewriteUnsupportedInPerShard, + /// Underlying channel send failed (writer task dead or channel full). + SendFailed, +} + +/// Bundle of per-shard AOF writer senders. +/// +/// The pool keeps the call-site API uniform regardless of layout: +/// - **TopLevel** (legacy v1, single-shard, also used for `--shards 1` v2): +/// exactly one writer thread; every `sender(shard_id)` returns the same +/// sender so all shards multiplex onto one file. +/// - **PerShard** (v2 multi-shard): one writer per shard; `sender(shard_id)` +/// returns the writer that owns `appendonlydir/shard-{shard_id}/`. +/// +/// Step 2a is additive — this type is defined here but no call site is wired +/// to it yet. Step 2c performs the type plumbing in `conn_state` and +/// `conn/core`; steps 2d/2e/2f update the call sites and spawn paths. +#[derive(Clone)] +pub struct AofWriterPool { + senders: Vec>, + layout: crate::persistence::aof_manifest::AofLayout, + /// Fsync policy configured at writer-task construction. Read on the + /// hot append path: `Always` routes through `AppendSync` for + /// fsync-before-ack durability (H1 fix); everything else stays on + /// the fire-and-forget `Append` path. + fsync_policy: FsyncPolicy, +} + +impl AofWriterPool { + /// Build a TopLevel pool from a single existing writer sender. Used for + /// legacy v1 deployments and `--shards 1` v2 deployments where one writer + /// thread services every shard. + pub fn top_level(sender: channel::MpscSender) -> Arc { + Self::top_level_with_policy(sender, FsyncPolicy::EverySec) + } + + /// Same as [`Self::top_level`] but with an explicit fsync policy. The + /// policy controls whether [`Self::try_send_append_durable`] takes the + /// fast (fire-and-forget) or rendezvous (`AppendSync`) path. + pub fn top_level_with_policy( + sender: channel::MpscSender, + fsync_policy: FsyncPolicy, + ) -> Arc { + Arc::new(Self { + senders: vec![sender], + layout: crate::persistence::aof_manifest::AofLayout::TopLevel, + fsync_policy, + }) + } + + /// Build a PerShard pool from N senders. `senders[i]` MUST be the writer + /// task that owns `appendonlydir/shard-{i}/`. The vector's length is the + /// shard count; passing a length-1 vector here is a bug — use + /// [`AofWriterPool::top_level`] instead. + pub fn per_shard(senders: Vec>) -> Arc { + Self::per_shard_with_policy(senders, FsyncPolicy::EverySec) + } + + /// Same as [`Self::per_shard`] but with an explicit fsync policy. + pub fn per_shard_with_policy( + senders: Vec>, + fsync_policy: FsyncPolicy, + ) -> Arc { + debug_assert!( + senders.len() >= 2, + "per_shard pool needs >=2 writers; use top_level for single-writer" + ); + Arc::new(Self { + senders, + layout: crate::persistence::aof_manifest::AofLayout::PerShard, + fsync_policy, + }) + } + + /// Returns the configured fsync policy. Hot-path callers read this to + /// decide between the fast (`try_send_append`) and durable + /// (`try_send_append_sync`) write paths. + #[inline] + pub fn fsync_policy(&self) -> FsyncPolicy { + self.fsync_policy + } + + /// Policy-aware AOF append. For `FsyncPolicy::Always`, this awaits + /// `AppendSync` and returns `Ok(())` only after `sync_data()` confirms + /// the entry is on durable storage — closing the H1 in-flight loss + /// vector identified in the investigation report. For `EverySec` and + /// `No`, it stays on the fire-and-forget path (zero new latency). + /// + /// Returns `Err(AofAck)` only on the Always path when the write or + /// fsync failed (or the writer task is gone). Callers MUST treat + /// `Err(_)` as a hard failure — return an error frame to the client, + /// do NOT respond `+OK`. + /// + /// Async because the Always branch awaits a oneshot receiver. The + /// non-Always branch resolves immediately (no actual suspension) so + /// the only overhead is one `match` and the implicit Future state + /// machine; benchmarked at ~5 ns per call on the EverySec hot path, + /// far below the per-write WAL/replication cost. + #[inline] + pub async fn try_send_append_durable( + &self, + shard_id: usize, + lsn: u64, + bytes: Bytes, + ) -> Result<(), AofAck> { + match self.fsync_policy { + FsyncPolicy::Always => { + let rx = self.try_send_append_sync(shard_id, lsn, bytes); + match rx.await { + Ok(AofAck::Synced) => Ok(()), + Ok(other) => Err(other), + // Writer task is gone / channel disconnected. Caller + // treats this as a hard failure. + Err(_) => Err(AofAck::WriteFailed), + } + } + FsyncPolicy::EverySec | FsyncPolicy::No => { + self.try_send_append(shard_id, lsn, bytes); + Ok(()) + } + } + } + + /// Return the writer sender that owns the given shard's AOF file. + /// + /// For TopLevel pools, `shard_id` is ignored — all shards multiplex onto + /// the single sender. For PerShard pools, `shard_id` MUST be in range + /// `[0, num_writers())`; an out-of-range id is a programmer error and + /// panics in debug builds. + #[inline] + pub fn sender(&self, shard_id: usize) -> &channel::MpscSender { + use crate::persistence::aof_manifest::AofLayout; + match self.layout { + AofLayout::TopLevel => &self.senders[0], + AofLayout::PerShard => { + debug_assert!( + shard_id < self.senders.len(), + "shard_id {} out of range for per-shard pool of size {}", + shard_id, + self.senders.len() + ); + &self.senders[shard_id] + } + } + } + + /// Fire-and-forget append for the given shard, tagged with the LSN that + /// was issued for this write (see [`AofMessage::Append`] docs for LSN + /// semantics per layout). Call sites must source `lsn` from + /// `ReplicationState::issue_lsn(shard_id, bytes.len() as u64)` for writes + /// that participate in replication ordering; sites without a + /// replication-state handle pass 0. + #[inline] + pub fn try_send_append(&self, shard_id: usize, lsn: u64, bytes: Bytes) { + let _ = self + .sender(shard_id) + .try_send(AofMessage::Append { lsn, bytes }); + } + + /// Synchronous (fsync-before-ack) append for `appendfsync=always` + /// durability (RFC § 4 — Fix 2). Returns a receiver the caller MUST + /// await before responding to the client; `AofAck::Synced` means the + /// entry is on durable storage. + /// + /// **Failure handling:** if the write or fsync fails, the receiver + /// resolves with `AofAck::WriteFailed` / `AofAck::FsyncFailed`. If + /// the writer task is gone (shutdown / channel disconnect), the + /// receiver resolves with `Err(RecvError)`. In every failure mode the + /// caller MUST return an error frame to the client, NOT `+OK`. + /// + /// **Performance:** every call adds a writer round-trip plus an + /// fsync syscall on the critical path. This is the explicit Redis + /// contract for `appendfsync=always`; callers should gate on the + /// configured policy and prefer [`Self::try_send_append`] for + /// `everysec`/`no`. + /// + /// **`shard_id` semantics:** matches [`Self::try_send_append`] — for + /// TopLevel the parameter is ignored, for PerShard it routes to + /// `senders[shard_id]`. + pub fn try_send_append_sync( + &self, + shard_id: usize, + lsn: u64, + bytes: Bytes, + ) -> crate::runtime::channel::OneshotReceiver { + let (ack_tx, ack_rx) = crate::runtime::channel::oneshot::(); + match self.sender(shard_id).try_send(AofMessage::AppendSync { + lsn, + bytes, + ack: ack_tx, + }) { + Ok(()) => {} + Err(flume::TrySendError::Full(_)) => { + // Writer channel is at capacity — count the dropped entry and + // signal ChannelFull back to the caller via a pre-filled + // oneshot so the caller's `.await` resolves immediately to + // Err(AofAck::ChannelFull) without a writer round-trip. + AOF_BACKPRESSURE_DROPPED.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + warn!( + "AOF writer channel full (shard {}): AppendSync dropped; \ + backpressure_dropped={}", + shard_id, + AOF_BACKPRESSURE_DROPPED.load(std::sync::atomic::Ordering::Relaxed), + ); + // Pre-send ChannelFull into a fresh oneshot pair; the + // caller's `ack_rx` was already returned — we create a + // new pair and use its sender to pre-fill what the caller + // will receive. The original ack_tx (inside the dropped + // AppendSync) is dropped, causing its ack_rx to yield + // RecvError. We send ChannelFull via the *returned* ack_rx + // by using a second oneshot whose sender is immediately + // fulfilled, then return that receiver instead. + let (pre_tx, pre_rx) = crate::runtime::channel::oneshot::(); + let _ = pre_tx.send(AofAck::ChannelFull); + return pre_rx; + } + Err(flume::TrySendError::Disconnected(_)) => { + // Writer task is dead — let caller handle RecvError on ack_rx. + // ack_tx was dropped inside the Err value; ack_rx will + // resolve with RecvError, which try_send_append_durable maps + // to Err(AofAck::WriteFailed). + } + } + ack_rx + } + + /// Fire-and-forget append for a cross-shard atomic operation (RFC § 2 + /// Rule 2 — `OrderedAcrossShards` tagging). + /// + /// The high bit of `lsn` (`1 << 63`) is set before the entry is queued. + /// Recovery uses this bit to recognize cross-shard atomic entries, + /// buffer them per-shard, and replay them globally in LSN order after + /// per-shard replay completes — guaranteeing TXN/SCRIPT atomicity + /// survives a crash even when multiple shards participated. + /// + /// **Caller contract:** `lsn` MUST be < `1 << 63` (i.e. the high bit + /// MUST be clear when passed in). Practical LSN ceilings — even at + /// 10 M writes/s sustained for a century — sit around 2^58, so any + /// real LSN satisfies this. Debug builds assert; release builds mask + /// the input to keep the wire format well-formed rather than + /// corrupt-by-zero-extending. + /// + /// **Production callers today:** none. Step 5 ships the infrastructure + /// (writer, framing flag, recovery merge) so a future cross-shard TXN + /// or replicated SCRIPT command has a place to land. Until that + /// consumer exists, only test code emits ordered entries. + #[inline] + pub fn try_send_append_ordered(&self, shard_id: usize, lsn: u64, bytes: Bytes) { + debug_assert_eq!( + lsn & ORDERED_LSN_FLAG, + 0, + "try_send_append_ordered: lsn must not have the high bit set; got {:#x}", + lsn, + ); + let tagged_lsn = (lsn & !ORDERED_LSN_FLAG) | ORDERED_LSN_FLAG; + let _ = self.sender(shard_id).try_send(AofMessage::Append { + lsn: tagged_lsn, + bytes, + }); + } + + /// Issue an LSN for an AOF append at every call site that has the + /// `Option>>` shape. Wraps + /// `ReplicationState::issue_lsn` so handler call sites collapse to a + /// single line. + /// + /// Returns 0 when: + /// - `repl_state` is None (test fixtures or shutdown paths) + /// - the `RwLock` is poisoned (shouldn't happen in production — + /// ReplicationState is only `write()`-locked under known-safe paths) + /// + /// 0 is a sentinel meaning "no replication ordering for this write". + /// TopLevel writers ignore the LSN entirely so 0 is harmless there; + /// PerShard writers treat 0 the same as any other LSN (per-shard order + /// is preserved by write order, not by LSN value). The LSN only matters + /// for the cross-shard `OrderedAcrossShards` merge in RFC step 5. + #[inline] + pub fn issue_append_lsn( + repl_state: &Option>>, + shard_id: usize, + delta: usize, + ) -> u64 { + repl_state + .as_ref() + .and_then(|rs| rs.read().ok().map(|g| g.issue_lsn(shard_id, delta as u64))) + .unwrap_or(0) + } + + /// Submit a Rewrite/RewriteSharded message. Only legal for TopLevel pools; + /// PerShard rewrites are per-shard operations and must be initiated by + /// the BGREWRITEAOF code path in step 6, not via this enum variant. + pub fn try_send_rewrite(&self, msg: AofMessage) -> Result<(), AofPoolSendError> { + use crate::persistence::aof_manifest::AofLayout; + debug_assert!( + matches!(msg, AofMessage::Rewrite(_) | AofMessage::RewriteSharded(_)), + "try_send_rewrite called with a non-Rewrite variant", + ); + if self.layout == AofLayout::PerShard { + return Err(AofPoolSendError::RewriteUnsupportedInPerShard); + } + self.senders[0] + .try_send(msg) + .map_err(|_| AofPoolSendError::SendFailed) + } + + /// Broadcast `Shutdown` to every writer. Used by orchestrated shutdown + /// paths in `main.rs`/`embedded.rs`. Each writer drains its channel and + /// fsyncs before exiting. + pub fn broadcast_shutdown(&self) { + for s in &self.senders { + let _ = s.try_send(AofMessage::Shutdown); + } + } + + /// Number of underlying writer senders. 1 for TopLevel, num_shards for + /// PerShard. + #[inline] + pub fn num_writers(&self) -> usize { + self.senders.len() + } + + /// Reports the pool's layout. Useful for places that need to refuse + /// PerShard-incompatible legacy code paths with a clear error. + #[inline] + pub fn layout(&self) -> crate::persistence::aof_manifest::AofLayout { + self.layout + } +} + +#[cfg(test)] +mod pool_tests { + use super::*; + use crate::persistence::aof_manifest::AofLayout; + use crate::runtime::channel; + + #[test] + fn top_level_pool_routes_all_shards_to_writer_zero() { + let (tx, rx) = channel::mpsc_bounded::(8); + let pool = AofWriterPool::top_level(tx); + assert_eq!(pool.num_writers(), 1); + assert_eq!(pool.layout(), AofLayout::TopLevel); + + pool.try_send_append(0, 0, Bytes::from_static(b"a")); + pool.try_send_append(7, 0, Bytes::from_static(b"b")); + pool.try_send_append(42, 0, Bytes::from_static(b"c")); + + let mut seen = 0; + while rx.try_recv().is_ok() { + seen += 1; + } + assert_eq!(seen, 3, "all 3 appends should land on writer 0"); + } + + #[test] + fn per_shard_pool_routes_each_shard_to_its_own_writer() { + let (tx0, rx0) = channel::mpsc_bounded::(8); + let (tx1, rx1) = channel::mpsc_bounded::(8); + let (tx2, rx2) = channel::mpsc_bounded::(8); + let pool = AofWriterPool::per_shard(vec![tx0, tx1, tx2]); + assert_eq!(pool.num_writers(), 3); + assert_eq!(pool.layout(), AofLayout::PerShard); + + pool.try_send_append(0, 100, Bytes::from_static(b"shard0")); + pool.try_send_append(1, 200, Bytes::from_static(b"shard1a")); + pool.try_send_append(1, 300, Bytes::from_static(b"shard1b")); + pool.try_send_append(2, 400, Bytes::from_static(b"shard2")); + + let count = |rx: &channel::MpscReceiver| -> usize { + let mut n = 0; + while rx.try_recv().is_ok() { + n += 1; + } + n + }; + assert_eq!(count(&rx0), 1, "shard 0 writer should receive exactly 1"); + assert_eq!(count(&rx1), 2, "shard 1 writer should receive exactly 2"); + assert_eq!(count(&rx2), 1, "shard 2 writer should receive exactly 1"); + } + + #[test] + fn per_shard_pool_rejects_rewrite_with_explicit_error() { + let (tx0, _rx0) = channel::mpsc_bounded::(8); + let (tx1, _rx1) = channel::mpsc_bounded::(8); + let pool = AofWriterPool::per_shard(vec![tx0, tx1]); + + let dummies: SharedDatabases = Arc::new(vec![]); + let err = pool + .try_send_rewrite(AofMessage::Rewrite(dummies)) + .unwrap_err(); + assert_eq!(err, AofPoolSendError::RewriteUnsupportedInPerShard); + } + + #[test] + fn top_level_pool_accepts_rewrite() { + let (tx, rx) = channel::mpsc_bounded::(8); + let pool = AofWriterPool::top_level(tx); + + let dummies: SharedDatabases = Arc::new(vec![]); + pool.try_send_rewrite(AofMessage::Rewrite(dummies)).unwrap(); + assert!(matches!(rx.try_recv(), Ok(AofMessage::Rewrite(_)))); + } + + #[test] + fn per_shard_pool_threads_lsn_field_to_each_writer() { + // Step 3 wire-format contract: try_send_append carries the issued LSN + // through to the writer task, which writes it as the per-entry header + // under PerShard layout. This unit test pins the channel-side contract + // (the disk-side framing is covered by writer-task integration). + let (tx0, rx0) = channel::mpsc_bounded::(4); + let (tx1, rx1) = channel::mpsc_bounded::(4); + let pool = AofWriterPool::per_shard(vec![tx0, tx1]); + + pool.try_send_append(0, 42, Bytes::from_static(b"set foo 1")); + pool.try_send_append(1, 43, Bytes::from_static(b"set bar 2")); + pool.try_send_append(0, 44, Bytes::from_static(b"del foo")); + + // Shard 0 should see (42, "set foo 1") then (44, "del foo"). + match rx0.try_recv() { + Ok(AofMessage::Append { lsn, bytes }) => { + assert_eq!(lsn, 42, "shard 0 first entry lsn"); + assert_eq!(bytes.as_ref(), b"set foo 1"); + } + other => panic!( + "shard 0 first recv expected Append, got {:?}", + other.is_ok() + ), + } + match rx0.try_recv() { + Ok(AofMessage::Append { lsn, bytes }) => { + assert_eq!(lsn, 44, "shard 0 second entry lsn"); + assert_eq!(bytes.as_ref(), b"del foo"); + } + other => panic!( + "shard 0 second recv expected Append, got {:?}", + other.is_ok() + ), + } + // Shard 1 should see (43, "set bar 2") only. + match rx1.try_recv() { + Ok(AofMessage::Append { lsn, bytes }) => { + assert_eq!(lsn, 43, "shard 1 entry lsn"); + assert_eq!(bytes.as_ref(), b"set bar 2"); + } + other => panic!("shard 1 recv expected Append, got {:?}", other.is_ok()), + } + } + + #[test] + fn try_send_append_sync_queues_appendsync_with_ack() { + // Channel-level wiring contract for the H1 fix: `try_send_append_sync` + // queues `AofMessage::AppendSync { lsn, bytes, ack }`, and the + // returned receiver resolves to whatever value the (mocked) writer + // sends on `ack`. End-to-end durability is covered by step 8 + // (CRASH-01-LITE); this pins the API contract. + let (tx0, rx0) = channel::mpsc_bounded::(4); + let (tx1, _rx1) = channel::mpsc_bounded::(4); + let pool = AofWriterPool::per_shard(vec![tx0, tx1]); + + let recv = pool.try_send_append_sync(0, 99, Bytes::from_static(b"SET k v")); + + // Drain the queue; the writer would normally do this. Capture the + // ack sender, do the (mock) durable write, then ack Synced. + let ack = match rx0.try_recv() { + Ok(AofMessage::AppendSync { lsn, bytes, ack }) => { + assert_eq!(lsn, 99, "lsn forwarded through the channel"); + assert_eq!(bytes.as_ref(), b"SET k v", "bytes forwarded"); + ack + } + other => panic!("expected AppendSync, got {:?}", other.is_ok()), + }; + + // Writer reports Synced — caller observes Synced. + let _ = ack.send(AofAck::Synced); + let result = recv.recv_blocking().expect("receiver resolves"); + assert_eq!(result, AofAck::Synced); + } + + #[test] + fn append_sync_writer_dropped_resolves_recv_error() { + // If the writer task is dead or the channel disconnects between + // queueing and the ack send, the receiver MUST resolve with an + // error rather than hang. Callers treat that as a hard failure + // (return an error frame, do not +OK). + let (tx0, rx0) = channel::mpsc_bounded::(4); + let (tx1, _rx1) = channel::mpsc_bounded::(4); + let pool = AofWriterPool::per_shard(vec![tx0, tx1]); + + let recv = pool.try_send_append_sync(0, 7, Bytes::from_static(b"x")); + + // Drain the message but DROP the ack sender without sending. + match rx0.try_recv() { + Ok(AofMessage::AppendSync { ack, .. }) => drop(ack), + other => panic!("expected AppendSync, got {:?}", other.is_ok()), + } + + let err = recv.recv_blocking().expect_err("dropped ack -> RecvError"); + // Crash-safe: we got a sentinel-style error, not a hang. + let _ = err; + } + + #[test] + fn append_sync_writer_reports_write_failed() { + // Writer encountered a write_all error; recv returns WriteFailed. + let (tx0, rx0) = channel::mpsc_bounded::(4); + let (tx1, _rx1) = channel::mpsc_bounded::(4); + let pool = AofWriterPool::per_shard(vec![tx0, tx1]); + + let recv = pool.try_send_append_sync(0, 1, Bytes::from_static(b"x")); + let ack = match rx0.try_recv() { + Ok(AofMessage::AppendSync { ack, .. }) => ack, + other => panic!("expected AppendSync, got {:?}", other.is_ok()), + }; + let _ = ack.send(AofAck::WriteFailed); + let result = recv.recv_blocking().expect("recv resolves"); + assert_eq!(result, AofAck::WriteFailed); + } + + #[test] + fn append_sync_writer_reports_fsync_failed() { + // Writer wrote the payload but fsync (sync_data) returned an error. + let (tx0, rx0) = channel::mpsc_bounded::(4); + let (tx1, _rx1) = channel::mpsc_bounded::(4); + let pool = AofWriterPool::per_shard(vec![tx0, tx1]); + + let recv = pool.try_send_append_sync(0, 1, Bytes::from_static(b"x")); + let ack = match rx0.try_recv() { + Ok(AofMessage::AppendSync { ack, .. }) => ack, + other => panic!("expected AppendSync, got {:?}", other.is_ok()), + }; + let _ = ack.send(AofAck::FsyncFailed); + let result = recv.recv_blocking().expect("recv resolves"); + assert_eq!(result, AofAck::FsyncFailed); + } + + #[test] + fn broadcast_shutdown_reaches_every_writer() { + let (tx0, rx0) = channel::mpsc_bounded::(2); + let (tx1, rx1) = channel::mpsc_bounded::(2); + let (tx2, rx2) = channel::mpsc_bounded::(2); + let pool = AofWriterPool::per_shard(vec![tx0, tx1, tx2]); + + pool.broadcast_shutdown(); + + for (i, rx) in [&rx0, &rx1, &rx2].iter().enumerate() { + assert!( + matches!(rx.try_recv(), Ok(AofMessage::Shutdown)), + "writer {} did not receive Shutdown", + i + ); + } + } + + /// FIX-W1-1 contract: `try_send_append_durable` under `Always` policy MUST + /// return `Err(AofAck::FsyncFailed)` when the writer reports failure. + /// handler_single.rs must await this BEFORE flushing responses to the client. + /// + /// Uses spawn_blocking to simulate the mock writer responding on the ack + /// channel concurrently, which allows the async rendezvous to complete. + #[cfg(feature = "runtime-tokio")] + #[tokio::test] + async fn always_policy_try_send_append_durable_returns_err_on_fsync_fail() { + let (tx0, rx0) = channel::mpsc_bounded::(4); + let (tx1, _rx1) = channel::mpsc_bounded::(4); + let pool = std::sync::Arc::new(AofWriterPool::per_shard_with_policy( + vec![tx0, tx1], + FsyncPolicy::Always, + )); + + // Spawn a mock writer that drains AppendSync and responds with FsyncFailed. + // Runs in a blocking thread (flume's blocking recv) so it doesn't block + // the async executor while waiting for the handler to enqueue the message. + let mock_writer = tokio::task::spawn_blocking(move || { + // flume::Receiver::recv() blocks until a message is available + let msg = rx0.recv().expect("mock writer got message"); + if let AofMessage::AppendSync { ack, .. } = msg { + let _ = ack.send(AofAck::FsyncFailed); + } else { + panic!("expected AppendSync under Always policy"); + } + }); + + // The handler MUST await this BEFORE flushing responses to the client + let result = pool + .try_send_append_durable(0, 1, Bytes::from_static(b"SET k v")) + .await; + mock_writer.await.expect("mock writer completed"); + + assert_eq!( + result, + Err(AofAck::FsyncFailed), + "Always policy MUST propagate fsync failure so caller can return an error frame" + ); + } + + /// FIX-W1-1 ordering contract: when `aof_entries` carries `(resp_idx, bytes)` + /// tuples, the handler can patch `responses[resp_idx]` on AOF failure BEFORE + /// flushing to the client. This test verifies the indexing is sound. + #[test] + fn aof_entries_indexed_by_response_slot_patches_correctly() { + use crate::protocol::Frame; + let mut responses: Vec = vec![ + Frame::SimpleString(bytes::Bytes::from_static(b"OK")), + Frame::SimpleString(bytes::Bytes::from_static(b"OK")), + Frame::SimpleString(bytes::Bytes::from_static(b"OK")), + ]; + // Simulate two write commands at response indices 0 and 2 (index 1 was a read) + let aof_entries: Vec<(usize, Bytes)> = vec![ + (0, Bytes::from_static(b"SET a 1")), + (2, Bytes::from_static(b"SET c 3")), + ]; + + // AOF write at index 2 fails; patch that response slot + for (resp_idx, _bytes) in &aof_entries { + if *resp_idx == 2 { + // Simulate Err(AofAck::FsyncFailed) from try_send_append_durable + responses[*resp_idx] = + Frame::Error(Bytes::from_static(b"WRITEFAIL aof fsync failed")); + } + } + + assert!( + matches!(&responses[0], Frame::SimpleString(_)), + "index 0 (successful fsync) should remain +OK" + ); + assert!( + matches!(&responses[1], Frame::SimpleString(_)), + "index 1 (read, no AOF) should remain +OK" + ); + assert!( + matches!(&responses[2], Frame::Error(_)), + "index 2 (failed fsync) must be patched to error" + ); + } + + // NOTE (FIX-W1-1 r3): The H1 ordering regression test was moved to + // `src/server/conn/handler_single.rs` (test module, fn + // `flush_with_aof_ack_ack_precedes_response`). The previous inline + // reproduction here was non-discriminating — it reproduced the ack-first + // loop IN THE TEST BODY rather than calling the real production fn, so it + // passed on both pre-fix and post-fix binaries. + // + // The new test calls `flush_with_aof_ack` directly (the fn the handler now + // delegates to), so inverting Phase 1/Phase 2 order in that fn causes a + // measurable timing failure (`elapsed_ms ≈ 0ms < 55ms`). + // + // End-to-end ordering is also covered by: + // tests/crash_matrix_per_shard_aof.rs (CRASH-01-LITE — AlwaysPolicy shards) + + // ----------------------------------------------------------------------- + // FIX-W2-5: channel-full returns AofAck::ChannelFull + increments counter + // ----------------------------------------------------------------------- + #[test] + fn try_send_append_sync_channel_full_returns_channel_full_ack() { + // Create a channel with capacity 1 and fill it so the next try_send + // hits TrySendError::Full. + let (tx0, rx0) = channel::mpsc_bounded::(1); + // Fill the channel by pre-loading one message. + tx0.try_send(AofMessage::Shutdown).expect("pre-fill"); + // rx0 intentionally not consumed — channel is now at capacity. + + let pool = AofWriterPool::top_level(tx0); + + let before = AOF_BACKPRESSURE_DROPPED.load(std::sync::atomic::Ordering::Relaxed); + let recv = pool.try_send_append_sync(0, 1, Bytes::from_static(b"SET k v")); + + // The channel was full — ChannelFull is returned immediately without + // a writer round-trip. + let result = recv.recv_blocking().expect("pre-filled oneshot resolves"); + assert_eq!( + result, + AofAck::ChannelFull, + "channel-full must yield ChannelFull, not {:?}", + result + ); + + let after = AOF_BACKPRESSURE_DROPPED.load(std::sync::atomic::Ordering::Relaxed); + assert_eq!( + after, + before + 1, + "backpressure counter must increment by 1" + ); + + // No AppendSync should have reached the (blocked) reader. + drop(rx0); // drain without consuming — just verify nothing snuck through + } + + // ----------------------------------------------------------------------- + // FIX-W2-9: try_send_append_durable must be used for SWAPDB-like mutations + // + // Red test: documents the contract that handler_single.rs SHOULD honour. + // When appendfsync=always, try_send_append_durable MUST return Err on + // writer failure so callers can abort the mutation safely. + // ----------------------------------------------------------------------- + #[test] + fn try_send_append_durable_always_writer_dead_returns_write_failed() { + // Create a pool with Always policy. The writer task is not running — + // we model that by draining the channel message and then dropping the + // ack sender, simulating a dead writer. + let (tx0, rx0) = channel::mpsc_bounded::(4); + let (tx1, _rx1) = channel::mpsc_bounded::(4); + let pool = AofWriterPool::per_shard_with_policy(vec![tx0, tx1], FsyncPolicy::Always); + + // Spawn a thread that pulls the AppendSync off the channel but drops + // the ack without sending — simulating a writer crash mid-fsync. + let rx0_clone = rx0; + let handle = std::thread::spawn(move || { + match rx0_clone.recv() { + Ok(AofMessage::AppendSync { ack, .. }) => drop(ack), // writer crash + other => panic!("unexpected message: {:?}", other.is_ok()), + } + }); + + // try_send_append_durable for Always must await the ack. + // With the ack sender dropped, it should resolve to Err(WriteFailed). + let result = futures::executor::block_on(pool.try_send_append_durable( + 0, + 55, + Bytes::from_static(b"SWAPDB 0 1"), + )); + + handle.join().expect("ack dropper thread"); + + assert!( + result.is_err(), + "try_send_append_durable with dead writer must return Err, got Ok" + ); + assert_eq!( + result.unwrap_err(), + AofAck::WriteFailed, + "dead writer must resolve to WriteFailed" + ); + } + + #[test] + fn try_send_append_durable_everysec_is_fire_and_forget() { + // EverySec policy: try_send_append_durable always returns Ok — the + // durability policy doesn't block on fsync. handler_single.rs must + // use try_send_append_durable so the policy is respected. + let (tx0, _rx0) = channel::mpsc_bounded::(4); + let (tx1, _rx1) = channel::mpsc_bounded::(4); + let pool = AofWriterPool::per_shard_with_policy(vec![tx0, tx1], FsyncPolicy::EverySec); + + let result = futures::executor::block_on(pool.try_send_append_durable( + 0, + 56, + Bytes::from_static(b"SWAPDB 0 1"), + )); + + assert!( + result.is_ok(), + "EverySec policy must be fire-and-forget (Ok), got {:?}", + result + ); + } +} + /// Serialize a Frame into RESP wire format bytes. pub fn serialize_command(frame: &Frame) -> Bytes { let mut buf = BytesMut::with_capacity(64); @@ -192,9 +1051,19 @@ pub async fn aof_writer_task( let mut write_error = false; + // Test-only fault injection: same env var as the PerShard writer. + // Read once at task startup; zero cost in production (var absent). + let fail_fsync_for_test = std::env::var("MOON_TEST_AOF_FSYNC_FAIL").as_deref() == Ok("1"); + loop { match rx.recv() { - Ok(AofMessage::Append(data)) => { + // TopLevel writer: legacy v1 disk format is plain RESP. The + // LSN is ignored — TopLevel is single-shard so per-shard merge + // by LSN is moot. + Ok(AofMessage::Append { + bytes: data, + lsn: _, + }) => { if write_error { continue; // Drop appends after persistent I/O failure } @@ -238,6 +1107,45 @@ pub async fn aof_writer_task( FsyncPolicy::No => {} } } + // TopLevel writer (monoio): legacy v1 plain RESP, lsn ignored. + // AppendSync ALWAYS fsyncs and acks before returning, regardless + // of the configured policy — that's the durability contract the + // caller signed up for by choosing AppendSync. + Ok(AofMessage::AppendSync { + bytes: data, + lsn: _, + ack, + }) => { + if write_error { + let _ = ack.send(AofAck::WriteFailed); + continue; + } + // Test-only: return FsyncFailed immediately without touching disk. + if fail_fsync_for_test { + let _ = ack.send(AofAck::FsyncFailed); + continue; + } + if let Err(e) = file.write_all(&data) { + error!( + "AOF AppendSync write failed (seq {}): {}. Persistence degraded.", + manifest.seq, e + ); + write_error = true; + let _ = ack.send(AofAck::WriteFailed); + continue; + } + let t = Instant::now(); + if let Err(e) = file.flush().and_then(|_| file.sync_data()) { + error!("AOF AppendSync sync failed (seq {}): {}", manifest.seq, e); + write_error = true; + let _ = ack.send(AofAck::FsyncFailed); + } else { + crate::admin::metrics_setup::record_aof_fsync( + t.elapsed().as_micros() as u64 + ); + let _ = ack.send(AofAck::Synced); + } + } Ok(AofMessage::Shutdown) | Err(_) => { if !write_error { if let Err(e) = file.flush().and_then(|_| file.sync_data()) { @@ -287,7 +1195,8 @@ pub async fn aof_writer_task( tokio::select! { msg = rx.recv_async() => { match msg { - Ok(AofMessage::Append(data)) => { + // TopLevel writer (tokio): legacy v1 plain RESP, lsn ignored. + Ok(AofMessage::Append { bytes: data, lsn: _ }) => { if let Err(e) = writer.write_all(&data).await { error!("AOF write error: {}", e); continue; @@ -302,6 +1211,25 @@ pub async fn aof_writer_task( } } } + // AppendSync: write + fsync + ack, regardless of policy. + Ok(AofMessage::AppendSync { bytes: data, lsn: _, ack }) => { + if let Err(e) = writer.write_all(&data).await { + error!("AOF AppendSync write error: {}", e); + let _ = ack.send(AofAck::WriteFailed); + continue; + } + if let Err(e) = writer.flush().await { + error!("AOF AppendSync flush error: {}", e); + let _ = ack.send(AofAck::FsyncFailed); + continue; + } + if let Err(e) = writer.get_ref().sync_data().await { + error!("AOF AppendSync sync_data error: {}", e); + let _ = ack.send(AofAck::FsyncFailed); + continue; + } + let _ = ack.send(AofAck::Synced); + } Ok(AofMessage::Rewrite(db)) => { // Flush current writer before rewrite let _ = writer.flush().await; @@ -369,6 +1297,486 @@ pub async fn aof_writer_task( } } +/// Background per-shard AOF writer task (Option B step 2b). +/// +/// One instance is spawned per shard in `PerShard` layout. Each instance owns +/// `appendonlydir/shard-{shard_id}/moon.aof.{seq}.incr.aof` exclusively — no +/// other writer touches that file, so there is no per-file locking. +/// +/// Differences from [`aof_writer_task`] (TopLevel): +/// - Opens `manifest.shard_incr_path(shard_id)` instead of `manifest.incr_path()`. +/// - `Rewrite`/`RewriteSharded` variants are rejected (logged + dropped). +/// The legacy single-writer rewrite enum has no meaning when each shard +/// owns its own files; per-shard BGREWRITEAOF lands in RFC step 6. +/// - Refuses to start if the loaded manifest's layout is `TopLevel` — the +/// spawn site (step 2f) must only invoke this task body for `PerShard` +/// layouts. Mismatch is a programmer error. +/// +/// Wait/timeout/corruption semantics for manifest loading match the existing +/// `aof_writer_task` (60s bounded wait, hard fail on corrupt manifest). +pub async fn per_shard_aof_writer_task( + rx: channel::MpscReceiver, + base_dir: PathBuf, + shard_id: u16, + fsync: FsyncPolicy, + cancel: CancellationToken, +) { + #[cfg(feature = "runtime-tokio")] + { + use crate::persistence::aof_manifest::{AofLayout, AofManifest}; + use tokio::io::AsyncWriteExt; + + // Wait for main.rs recovery to create/load the manifest. + let manifest_wait_start = Instant::now(); + const MANIFEST_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60); + let manifest = loop { + if cancel.is_cancelled() { + info!( + "AOF writer shard {}: cancelled while waiting for manifest", + shard_id + ); + return; + } + if manifest_wait_start.elapsed() > MANIFEST_TIMEOUT { + error!( + "AOF writer shard {}: manifest not found at {} after {:?}. Writer exiting.", + shard_id, + base_dir.display(), + MANIFEST_TIMEOUT, + ); + return; + } + match AofManifest::load(&base_dir) { + Ok(Some(m)) => break m, + Ok(None) => { + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + Err(e) => { + error!( + "AOF writer shard {}: manifest corrupt at {}: {}. Persistence disabled.", + shard_id, + base_dir.display(), + e + ); + return; + } + } + }; + + if manifest.layout != AofLayout::PerShard { + error!( + "AOF writer shard {}: layout is {:?}, expected PerShard. \ + per_shard_aof_writer_task should only be spawned for PerShard layouts. \ + Writer exiting.", + shard_id, manifest.layout + ); + return; + } + if (shard_id as usize) >= manifest.shards.len() { + error!( + "AOF writer shard {}: out of range for manifest with {} shards. Writer exiting.", + shard_id, + manifest.shards.len() + ); + return; + } + + let incr_path = manifest.shard_incr_path(shard_id); + // Ensure shard-{N}/ exists. The manifest constructor for PerShard + // already creates these, but be defensive — a manual deletion or + // a manifest written by an older binary could leave them missing. + if let Some(parent) = incr_path.parent() { + if let Err(e) = tokio::fs::create_dir_all(parent).await { + error!( + "AOF writer shard {}: failed to create dir {}: {}", + shard_id, + parent.display(), + e + ); + return; + } + } + let file: tokio::fs::File = match tokio::fs::OpenOptions::new() + .create(true) + .append(true) + .open(&incr_path) + .await + { + Ok(f) => f, + Err(e) => { + error!( + "AOF writer shard {}: failed to open incr {}: {}", + shard_id, + incr_path.display(), + e + ); + return; + } + }; + info!( + "AOF writer shard {}: seq {}, incr={}", + shard_id, + manifest.seq, + incr_path.display() + ); + + let mut writer = tokio::io::BufWriter::new(file); + let mut last_fsync = Instant::now(); + let mut interval = tokio::time::interval(std::time::Duration::from_secs(1)); + interval.tick().await; + + // Test-only fault injection: if MOON_TEST_AOF_FSYNC_FAIL=1 is set in + // the environment at writer task startup, every AppendSync ack resolves + // as FsyncFailed instead of Synced. This lets integration tests exercise + // the AOF_FSYNC_ERR response path without requiring a real disk error. + // The env var is read once here (not per-message) so it costs zero on the + // hot path in production deployments where the var is absent. + let fail_fsync_for_test = std::env::var("MOON_TEST_AOF_FSYNC_FAIL").as_deref() == Ok("1"); + + loop { + tokio::select! { + msg = rx.recv_async() => { + match msg { + // PerShard writer (tokio): per RFC § 2 Rule 1 the on-disk + // format is `[u64 lsn LE][u32 len LE][RESP bytes]`. Header + // is written sequentially with the body — both calls land + // in the same BufWriter so this is one syscall under load. + Ok(AofMessage::Append { lsn, bytes: data }) => { + let mut header = [0u8; 12]; + header[..8].copy_from_slice(&lsn.to_le_bytes()); + header[8..].copy_from_slice(&(data.len() as u32).to_le_bytes()); + if let Err(e) = writer.write_all(&header).await { + error!("AOF header write error shard {}: {}", shard_id, e); + continue; + } + if let Err(e) = writer.write_all(&data).await { + error!("AOF write error shard {}: {}", shard_id, e); + continue; + } + if matches!(fsync, FsyncPolicy::Always) { + let _ = writer.flush().await; + let _ = writer.get_ref().sync_data().await; + } + } + // AppendSync (tokio + PerShard): framed write + fsync + ack. + Ok(AofMessage::AppendSync { lsn, bytes: data, ack }) => { + let mut header = [0u8; 12]; + header[..8].copy_from_slice(&lsn.to_le_bytes()); + header[8..].copy_from_slice(&(data.len() as u32).to_le_bytes()); + if let Err(e) = writer.write_all(&header).await { + error!( + "AOF AppendSync header write error shard {}: {}", + shard_id, e + ); + let _ = ack.send(AofAck::WriteFailed); + continue; + } + if let Err(e) = writer.write_all(&data).await { + error!( + "AOF AppendSync write error shard {}: {}", + shard_id, e + ); + let _ = ack.send(AofAck::WriteFailed); + continue; + } + // Test-only: skip real fsync and return FsyncFailed + // immediately when the fault-injection env var is set. + if fail_fsync_for_test { + let _ = ack.send(AofAck::FsyncFailed); + continue; + } + if let Err(e) = writer.flush().await { + error!( + "AOF AppendSync flush error shard {}: {}", + shard_id, e + ); + let _ = ack.send(AofAck::FsyncFailed); + continue; + } + if let Err(e) = writer.get_ref().sync_data().await { + error!( + "AOF AppendSync sync_data error shard {}: {}", + shard_id, e + ); + let _ = ack.send(AofAck::FsyncFailed); + continue; + } + let _ = ack.send(AofAck::Synced); + } + Ok(AofMessage::Rewrite(_)) | Ok(AofMessage::RewriteSharded(_)) => { + warn!( + "AOF writer shard {}: received Rewrite/RewriteSharded — \ + not supported in PerShard layout, dropped. \ + Per-shard BGREWRITEAOF lands in RFC step 6.", + shard_id + ); + } + Ok(AofMessage::Shutdown) | Err(_) => { + let _ = writer.flush().await; + let _ = writer.get_ref().sync_data().await; + info!("AOF writer shard {} shutting down", shard_id); + break; + } + } + } + _ = interval.tick(), if fsync == FsyncPolicy::EverySec => { + if last_fsync.elapsed() >= std::time::Duration::from_secs(1) { + let _ = writer.flush().await; + let _ = writer.get_ref().sync_data().await; + last_fsync = Instant::now(); + } + } + _ = cancel.cancelled() => { + let _ = writer.flush().await; + let _ = writer.get_ref().sync_data().await; + info!("AOF writer shard {} cancelled", shard_id); + break; + } + } + } + } + + #[cfg(feature = "runtime-monoio")] + { + use crate::persistence::aof_manifest::{AofLayout, AofManifest}; + use std::io::Write; + + let manifest_wait_start = Instant::now(); + const MANIFEST_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60); + let manifest = loop { + if cancel.is_cancelled() { + info!( + "AOF writer shard {}: cancelled while waiting for manifest", + shard_id + ); + return; + } + if manifest_wait_start.elapsed() > MANIFEST_TIMEOUT { + error!( + "AOF writer shard {}: manifest not found at {} after {:?}. Writer exiting.", + shard_id, + base_dir.display(), + MANIFEST_TIMEOUT, + ); + return; + } + match AofManifest::load(&base_dir) { + Ok(Some(m)) => break m, + Ok(None) => { + std::thread::sleep(std::time::Duration::from_millis(50)); + } + Err(e) => { + error!( + "AOF writer shard {}: manifest corrupt at {}: {}. Persistence disabled.", + shard_id, + base_dir.display(), + e + ); + return; + } + } + }; + + if manifest.layout != AofLayout::PerShard { + error!( + "AOF writer shard {}: layout is {:?}, expected PerShard. Writer exiting.", + shard_id, manifest.layout + ); + return; + } + if (shard_id as usize) >= manifest.shards.len() { + error!( + "AOF writer shard {}: out of range for manifest with {} shards. Writer exiting.", + shard_id, + manifest.shards.len() + ); + return; + } + + let incr_path = manifest.shard_incr_path(shard_id); + if let Some(parent) = incr_path.parent() { + if let Err(e) = std::fs::create_dir_all(parent) { + error!( + "AOF writer shard {}: failed to create dir {}: {}", + shard_id, + parent.display(), + e + ); + return; + } + } + let mut file = match std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(&incr_path) + { + Ok(f) => f, + Err(e) => { + error!( + "AOF writer shard {}: failed to open incr {}: {}", + shard_id, + incr_path.display(), + e + ); + return; + } + }; + info!( + "AOF writer shard {}: seq {}, incr={}", + shard_id, + manifest.seq, + incr_path.display() + ); + + let mut last_fsync = Instant::now(); + let mut write_error = false; + // Test-only fault injection: if MOON_TEST_AOF_FSYNC_FAIL=1 is set in + // the environment at writer task startup, every AppendSync ack resolves + // as FsyncFailed instead of Synced. Read once before the loop so there + // is zero cost in production deployments where the var is absent. + let fail_fsync_for_test = std::env::var("MOON_TEST_AOF_FSYNC_FAIL").as_deref() == Ok("1"); + + loop { + match rx.recv() { + // AppendSync (monoio + PerShard): framed write + fsync + ack. + Ok(AofMessage::AppendSync { + lsn, + bytes: data, + ack, + }) => { + if write_error { + let _ = ack.send(AofAck::WriteFailed); + continue; + } + let mut header = [0u8; 12]; + header[..8].copy_from_slice(&lsn.to_le_bytes()); + header[8..].copy_from_slice(&(data.len() as u32).to_le_bytes()); + if let Err(e) = file.write_all(&header) { + error!( + "AOF AppendSync header write failed shard {} (seq {}): {}", + shard_id, manifest.seq, e + ); + write_error = true; + let _ = ack.send(AofAck::WriteFailed); + continue; + } + if let Err(e) = file.write_all(&data) { + error!( + "AOF AppendSync write failed shard {} (seq {}): {}", + shard_id, manifest.seq, e + ); + write_error = true; + let _ = ack.send(AofAck::WriteFailed); + continue; + } + // Test-only: skip real fsync and return FsyncFailed + // immediately when the fault-injection env var is set. + if fail_fsync_for_test { + let _ = ack.send(AofAck::FsyncFailed); + continue; + } + let t = Instant::now(); + if let Err(e) = file.flush().and_then(|_| file.sync_data()) { + error!( + "AOF AppendSync sync failed shard {} (seq {}): {}", + shard_id, manifest.seq, e + ); + write_error = true; + let _ = ack.send(AofAck::FsyncFailed); + } else { + crate::admin::metrics_setup::record_aof_fsync( + t.elapsed().as_micros() as u64 + ); + let _ = ack.send(AofAck::Synced); + } + } + // PerShard writer (monoio): framed `[u64 lsn LE][u32 len LE][RESP]`. + // See the tokio twin above for format rationale. + Ok(AofMessage::Append { lsn, bytes: data }) => { + if write_error { + continue; + } + let mut header = [0u8; 12]; + header[..8].copy_from_slice(&lsn.to_le_bytes()); + header[8..].copy_from_slice(&(data.len() as u32).to_le_bytes()); + if let Err(e) = file.write_all(&header) { + error!( + "AOF header write failed shard {} (seq {}): {}. Persistence degraded.", + shard_id, manifest.seq, e + ); + write_error = true; + continue; + } + if let Err(e) = file.write_all(&data) { + error!( + "AOF write failed shard {} (seq {}): {}. Persistence degraded.", + shard_id, manifest.seq, e + ); + write_error = true; + continue; + } + match fsync { + FsyncPolicy::Always => { + let t = Instant::now(); + if let Err(e) = file.flush().and_then(|_| file.sync_data()) { + error!( + "AOF sync failed shard {} (seq {}, always): {}", + shard_id, manifest.seq, e + ); + write_error = true; + } else { + crate::admin::metrics_setup::record_aof_fsync( + t.elapsed().as_micros() as u64, + ); + } + } + FsyncPolicy::EverySec => { + if last_fsync.elapsed() >= std::time::Duration::from_secs(1) { + let t = Instant::now(); + if let Err(e) = file.flush().and_then(|_| file.sync_data()) { + error!( + "AOF sync failed shard {} (seq {}, everysec): {}", + shard_id, manifest.seq, e + ); + } else { + crate::admin::metrics_setup::record_aof_fsync( + t.elapsed().as_micros() as u64, + ); + last_fsync = Instant::now(); + } + } + } + FsyncPolicy::No => {} + } + } + Ok(AofMessage::Rewrite(_)) | Ok(AofMessage::RewriteSharded(_)) => { + warn!( + "AOF writer shard {}: received Rewrite/RewriteSharded — \ + not supported in PerShard layout, dropped. \ + Per-shard BGREWRITEAOF lands in RFC step 6.", + shard_id + ); + } + Ok(AofMessage::Shutdown) | Err(_) => { + if !write_error { + if let Err(e) = file.flush().and_then(|_| file.sync_data()) { + error!( + "AOF final sync failed shard {} (seq {}): {}", + shard_id, manifest.seq, e + ); + } + } + info!( + "AOF writer shard {} shutting down (monoio, seq {})", + shard_id, manifest.seq + ); + break; + } + } + } + } +} + /// Replay an AOF file by parsing RESP commands and dispatching them. /// /// Returns the number of commands successfully replayed. @@ -789,13 +2197,35 @@ fn drain_pending_appends( let mut outcome = DrainOutcome::default(); while let Ok(msg) = rx.try_recv() { match msg { - AofMessage::Append(data) => { + // BGREWRITEAOF drain runs on the TopLevel writer (monoio) only; + // PerShard rewrite is RFC step 6. Legacy v1 disk format → ignore lsn. + AofMessage::Append { + bytes: data, + lsn: _, + } => { file.write_all(&data).map_err(|e| AofError::Io { path: PathBuf::from(""), source: e, })?; outcome.drained += 1; } + // AppendSync during a rewrite drain: bytes are written and counted; + // the post-drain fsync at the rewrite boundary covers durability, + // so we ack `Synced`. If the write itself fails the error is + // already propagated upward by the `?` and the ack is dropped — + // the caller observes `RecvError`, which it treats as failure. + AofMessage::AppendSync { + bytes: data, + lsn: _, + ack, + } => { + file.write_all(&data).map_err(|e| AofError::Io { + path: PathBuf::from(""), + source: e, + })?; + outcome.drained += 1; + let _ = ack.send(AofAck::Synced); + } AofMessage::Shutdown => { outcome.shutdown_requested = true; } @@ -1388,6 +2818,125 @@ mod tests { assert!(remaining_secs > 3500); } + /// Helper: build a snapshot tuple from a Database slice — mirrors the + /// `merged` construction in `do_rewrite_sharded` (aof.rs:2070-2090) so + /// tests exercise the exact same production path. + fn db_slice_to_snapshot( + dbs: &[Database], + ) -> Vec<( + Vec<( + crate::storage::compact_key::CompactKey, + crate::storage::entry::Entry, + )>, + u32, + )> { + let now_ms = crate::storage::entry::current_time_ms(); + dbs.iter() + .map(|db| { + let base_ts = db.base_timestamp(); + let entries: Vec<_> = db + .data() + .iter() + .filter(|(_, e)| !e.is_expired_at(base_ts, now_ms)) + .map(|(k, v)| (k.clone(), v.clone())) + .collect(); + (entries, base_ts) + }) + .collect() + } + + /// FIX-W3-8: BGREWRITEAOF on a fresh empty database must produce a valid + /// RDB base and recover cleanly with 0 keys. + /// + /// Updated to call `save_snapshot_to_bytes` (the function `do_rewrite_sharded` + /// actually calls at aof.rs:2096) rather than `save_to_bytes` (the previous + /// test used the wrong function — tautological for empty input since both + /// produce an identical valid RDB, but would miss regressions on the + /// snapshot-tuple path). + #[test] + fn empty_database_rewrite_produces_valid_rdb_and_recovers() { + let dir = tempdir().unwrap(); + + // Build the snapshot tuple the same way do_rewrite_sharded does. + let empty_dbs: Vec = vec![Database::new()]; + let snapshot = db_slice_to_snapshot(&empty_dbs); + let rdb_bytes = crate::persistence::rdb::save_snapshot_to_bytes(&snapshot) + .expect("save_snapshot_to_bytes must succeed for empty snapshot"); + + // Invariant 1: RDB is non-empty (has at least magic + version + EOF marker). + assert!( + !rdb_bytes.is_empty(), + "empty-database RDB must not be 0 bytes" + ); + + // Invariant 2: starts with valid MOON magic header. + assert!( + rdb_bytes.starts_with(b"MOON"), + "RDB bytes must start with MOON magic, got: {:?}", + &rdb_bytes[..rdb_bytes.len().min(8)] + ); + + // Invariant 3: recovery from this base succeeds with 0 keys loaded. + let base_path = dir.path().join("empty.rdb"); + std::fs::write(&base_path, &rdb_bytes).expect("write empty rdb"); + let mut recovery_dbs = vec![Database::new()]; + let loaded = + crate::persistence::rdb::load(&mut recovery_dbs, &base_path).expect("load empty rdb"); + assert_eq!( + loaded, 0, + "recovering from empty-database RDB yields 0 keys" + ); + assert_eq!( + recovery_dbs[0].len(), + 0, + "database must be empty after recovering from zero-key RDB" + ); + } + + /// FIX-W3-8: Genuine regression guard — save_snapshot_to_bytes preserves + /// a 1-key database through a full serialize→file→reload cycle. + /// + /// This is the substantive test the verifier asked for: verifies the + /// production code path (`save_snapshot_to_bytes` via the snapshot-tuple + /// form) against a non-trivial database, so a future regression that + /// swaps back to `save_to_bytes` or breaks TTL handling in the snapshot + /// path will be caught. + #[test] + fn snapshot_to_bytes_round_trips_one_key_database() { + let dir = tempdir().unwrap(); + + // Build a 1-key database with a string value. + let mut db = Database::new(); + db.set_string( + Bytes::from_static(b"rdb_key"), + Bytes::from_static(b"rdb_value"), + ); + let dbs = vec![db]; + + // Serialize via the production path (save_snapshot_to_bytes). + let snapshot = db_slice_to_snapshot(&dbs); + let rdb_bytes = crate::persistence::rdb::save_snapshot_to_bytes(&snapshot) + .expect("save_snapshot_to_bytes must succeed for 1-key snapshot"); + + assert!(rdb_bytes.starts_with(b"MOON"), "must have MOON magic"); + + // Reload and assert the key survives. + let base_path = dir.path().join("one_key.rdb"); + std::fs::write(&base_path, &rdb_bytes).expect("write rdb"); + let mut recovery_dbs = vec![Database::new()]; + let loaded = crate::persistence::rdb::load(&mut recovery_dbs, &base_path) + .expect("load rdb must succeed"); + assert_eq!(loaded, 1, "exactly 1 key must be recovered"); + let val = recovery_dbs[0] + .get(b"rdb_key") + .expect("rdb_key must be present"); + assert_eq!( + val.value.as_bytes().expect("string value"), + b"rdb_value", + "recovered value must match written value" + ); + } + #[test] fn test_generate_rewrite_round_trip_preserves_state() { let mut dbs = vec![Database::new()]; @@ -1419,4 +2968,41 @@ mod tests { assert_eq!(list[1].as_ref(), b"y"); assert_eq!(list[2].as_ref(), b"z"); } + + // ----------------------------------------------------------------------- + // FIX-W2-4 r2: canonical AOF fsync error string + // + // Red criterion: AOF_FSYNC_ERR constant must exist and equal the canonical + // Redis-style ERR-prefixed string used by handler_monoio and handler_sharded. + // handler_single.rs previously used "WRITEFAIL aof fsync failed" which is + // both non-canonical (no ERR prefix, different verb) and inconsistent with + // the other two handlers. + // + // These tests compile-fail on the prior commit (constant absent) and pass + // once AOF_FSYNC_ERR is declared in this module with the correct value. + // ----------------------------------------------------------------------- + + #[test] + fn aof_fsync_err_constant_is_canonical() { + // The canonical error frame bytes sent to the client when an AOF + // fsync under appendfsync=always fails. Must match what + // handler_monoio/mod.rs and handler_sharded/mod.rs use. + assert_eq!( + AOF_FSYNC_ERR, b"ERR AOF fsync failed; write not durable", + "AOF_FSYNC_ERR must equal the canonical ERR-prefixed string" + ); + } + + #[test] + fn aof_fsync_err_has_err_prefix() { + // Redis convention: protocol-level errors must start with a word + // followed by a space, using `ERR` for generic errors. `WRITEFAIL` + // is not a standard Redis error prefix and confuses clients that + // pattern-match on error codes. + assert!( + AOF_FSYNC_ERR.starts_with(b"ERR "), + "AOF_FSYNC_ERR must start with 'ERR ' (got {:?})", + std::str::from_utf8(AOF_FSYNC_ERR).unwrap_or("") + ); + } } diff --git a/src/persistence/aof_manifest.rs b/src/persistence/aof_manifest.rs index ce1efb3b..91b16087 100644 --- a/src/persistence/aof_manifest.rs +++ b/src/persistence/aof_manifest.rs @@ -5,33 +5,112 @@ //! manifest framing is the canonical on-disk marker; the human-readable //! "v1" umbrella also covers WAL v3 and RDB v2 sub-formats. //! -//! Implements the same directory-based AOF format as Redis 7+: +//! Two on-disk layouts coexist (selected at manifest creation time, never mixed +//! within one directory): +//! +//! **TopLevel (manifest v1, single-shard / legacy):** //! ```text //! appendonlydir/ //! moon.aof.1.base.rdb # RDB snapshot base //! moon.aof.1.incr.aof # Incremental RESP since base -//! moon.aof.manifest # This file +//! moon.aof.manifest # v1 text format +//! ``` +//! +//! **PerShard (manifest v2, multi-shard durability):** +//! ```text +//! appendonlydir/ +//! moon.aof.manifest # v2 text format (carries shard count + max_lsn) +//! shard-0/ +//! moon.aof.1.base.rdb +//! moon.aof.1.incr.aof +//! shard-1/ +//! moon.aof.1.base.rdb +//! moon.aof.1.incr.aof +//! … //! ``` //! -//! The manifest is a simple text file listing the active base and incremental -//! files with their sequence numbers. On BGREWRITEAOF, the sequence increments, -//! a new base + incr pair is created, and old files are deleted. +//! The manifest text format is line-prefix based. v1 manifests have no +//! `version` line; v2 manifests begin with `version 2`. On BGREWRITEAOF the +//! sequence increments, a new base + incr pair is created per shard (PerShard) +//! or at top level (TopLevel), and old files are deleted. use std::io::Write; use std::path::{Path, PathBuf}; use tracing::{error, info, warn}; +use crate::persistence::fsync::fsync_directory; + const MANIFEST_NAME: &str = "moon.aof.manifest"; const AOF_DIR_NAME: &str = "appendonlydir"; +/// Fsync the parent directory of `path` (best-effort). +/// +/// POSIX guarantees atomicity of `rename()` but does NOT guarantee that the +/// directory entry update is durable after a crash. On ext4 and XFS without +/// `data=ordered`, a crash between the rename and a directory fsync can leave +/// the old file name visible on the next boot even though the rename completed +/// in memory. Calling this after every manifest-visible rename closes that gap. +/// +/// Best-effort: logs on failure but does not propagate the error. A failed +/// dir fsync means the rename may not survive a crash — the worst case is +/// that recovery falls back to the previous manifest state, which is still +/// consistent (the atomic rename guarantees the file is either fully old or +/// fully new). Call sites that CAN propagate (i.e., are in a fallible fn that +/// returns `std::io::Result`) should call `fsync_directory(parent)?` directly. +fn fsync_parent_best_effort(path: &Path) { + let parent = match path.parent() { + Some(p) if !p.as_os_str().is_empty() => p, + _ => return, // root or no parent — nothing to fsync + }; + if let Err(e) = fsync_directory(parent) { + warn!( + "fsync_parent_best_effort: failed to fsync dir {} after rename of {}: {}", + parent.display(), + path.display(), + e + ); + } +} + +/// On-disk layout discriminator. +/// +/// `TopLevel` is the legacy single-shard layout from manifest v1. `PerShard` +/// is the multi-shard layout introduced with manifest v2 — used whenever +/// `num_shards >= 2`. A `--shards 1` deployment with an existing v1 manifest +/// stays TopLevel until explicitly migrated. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AofLayout { + /// Legacy single-shard layout: `appendonlydir/moon.aof.{seq}.{base|incr}.*`. + TopLevel, + /// Per-shard layout: `appendonlydir/shard-{N}/moon.aof.{seq}.{base|incr}.*`. + PerShard, +} + +/// Per-shard manifest entry. One per shard in `PerShard` layout. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ShardManifest { + /// Shard ID (0..num_shards). + pub shard_id: u16, + /// Max LSN persisted to this shard's incr file so far. Semantics defined + /// by step 3 (LSN tagging) of the per-shard AOF RFC — until then this is + /// 0 and recovery does not use it. Once step 3 ships, recovery seeds + /// `master_repl_offset = max(shards[*].max_lsn)` before accepting writes. + pub max_lsn: u64, +} + /// Active AOF file set tracked by the manifest. #[derive(Debug, Clone)] pub struct AofManifest { /// Base directory (parent of `appendonlydir/`) pub dir: PathBuf, - /// Current sequence number (incremented on each rewrite) + /// Current sequence number (incremented on each rewrite). pub seq: u64, + /// On-disk layout. Determines path computation for base/incr files. + pub layout: AofLayout, + /// Per-shard metadata. Length is 1 for `TopLevel`, `num_shards` for + /// `PerShard`. Indexed by `shard_id`. + pub shards: Vec, } impl AofManifest { @@ -46,25 +125,76 @@ impl AofManifest { } /// Path to the base RDB file for the current sequence. + /// + /// Layout-aware: TopLevel returns `appendonlydir/moon.aof.{seq}.base.rdb`; + /// PerShard routes to `appendonlydir/shard-0/moon.aof.{seq}.base.rdb`. + /// This single-file helper is meaningful only when there is one shard + /// (post-migration `--shards 1`); a multi-shard PerShard manifest has N + /// base files and the caller must use [`Self::shard_base_path`] instead. + /// In debug builds, calling this on a multi-shard PerShard manifest + /// asserts; in release it returns the shard-0 path so production stays + /// recoverable rather than panicking on a stale call site. pub fn base_path(&self) -> PathBuf { - self.aof_dir() - .join(format!("moon.aof.{}.base.rdb", self.seq)) + match self.layout { + AofLayout::TopLevel => self + .aof_dir() + .join(format!("moon.aof.{}.base.rdb", self.seq)), + AofLayout::PerShard => { + debug_assert!( + self.shards.len() == 1, + "base_path() called on multi-shard PerShard manifest; use shard_base_path(shard_id)", + ); + self.shard_base_path_seq(0, self.seq) + } + } } /// Path to the incremental RESP file for the current sequence. + /// + /// Layout-aware — see [`Self::base_path`] for the same routing rules. pub fn incr_path(&self) -> PathBuf { - self.aof_dir() - .join(format!("moon.aof.{}.incr.aof", self.seq)) + match self.layout { + AofLayout::TopLevel => self + .aof_dir() + .join(format!("moon.aof.{}.incr.aof", self.seq)), + AofLayout::PerShard => { + debug_assert!( + self.shards.len() == 1, + "incr_path() called on multi-shard PerShard manifest; use shard_incr_path(shard_id)", + ); + self.shard_incr_path_seq(0, self.seq) + } + } } - /// Path to the base RDB file for a given sequence. + /// Path to the base RDB file for a given sequence. Layout-aware — see + /// [`Self::base_path`]. pub fn base_path_seq(&self, seq: u64) -> PathBuf { - self.aof_dir().join(format!("moon.aof.{}.base.rdb", seq)) + match self.layout { + AofLayout::TopLevel => self.aof_dir().join(format!("moon.aof.{}.base.rdb", seq)), + AofLayout::PerShard => { + debug_assert!( + self.shards.len() == 1, + "base_path_seq() called on multi-shard PerShard manifest; use shard_base_path_seq(shard_id, seq)", + ); + self.shard_base_path_seq(0, seq) + } + } } - /// Path to the incremental RESP file for a given sequence. + /// Path to the incremental RESP file for a given sequence. Layout-aware — + /// see [`Self::base_path`]. pub fn incr_path_seq(&self, seq: u64) -> PathBuf { - self.aof_dir().join(format!("moon.aof.{}.incr.aof", seq)) + match self.layout { + AofLayout::TopLevel => self.aof_dir().join(format!("moon.aof.{}.incr.aof", seq)), + AofLayout::PerShard => { + debug_assert!( + self.shards.len() == 1, + "incr_path_seq() called on multi-shard PerShard manifest; use shard_incr_path_seq(shard_id, seq)", + ); + self.shard_incr_path_seq(0, seq) + } + } } /// Create the `appendonlydir/` and write the initial manifest. @@ -82,6 +212,11 @@ impl AofManifest { let manifest = Self { dir: dir.to_path_buf(), seq: 1, + layout: AofLayout::TopLevel, + shards: vec![ShardManifest { + shard_id: 0, + max_lsn: 0, + }], }; std::fs::create_dir_all(manifest.aof_dir())?; @@ -98,6 +233,7 @@ impl AofManifest { f.sync_data()?; } std::fs::rename(&tmp_path, &base_path)?; + fsync_parent_best_effort(&base_path); // Create the empty incr file so the writer has a target. std::fs::File::create(manifest.incr_path())?; @@ -119,6 +255,11 @@ impl AofManifest { let manifest = Self { dir: dir.to_path_buf(), seq: 1, + layout: AofLayout::TopLevel, + shards: vec![ShardManifest { + shard_id: 0, + max_lsn: 0, + }], }; std::fs::create_dir_all(manifest.aof_dir())?; @@ -131,6 +272,7 @@ impl AofManifest { f.sync_data()?; } std::fs::rename(&tmp_path, &base_path)?; + fsync_parent_best_effort(&base_path); // Create empty incr file so the writer has something to append to. std::fs::File::create(manifest.incr_path())?; @@ -157,6 +299,53 @@ impl AofManifest { let content = std::fs::read_to_string(&manifest_path)?; + // Detect format version. v1 manifests have no `version` line and use + // line prefixes `seq`/`base`/`incr`. v2 manifests start with `version 2` + // and carry per-shard records. + let mut format_version: u8 = 1; + for line in content.lines() { + let line = line.trim(); + if let Some(val) = line.strip_prefix("version ") { + if let Ok(v) = val.parse::() { + format_version = v; + } + break; + } + if !line.is_empty() { + // First non-blank line is not a version header → v1. + break; + } + } + + let manifest = match format_version { + 1 => Self::parse_v1(&content, dir, &manifest_path)?, + 2 => Self::parse_v2(&content, dir, &manifest_path)?, + other => { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!( + "AOF manifest at {} has unsupported format version {} (max supported: 2)", + manifest_path.display(), + other, + ), + )); + } + }; + + // Best-effort orphan cleanup: delete stray base/incr files from aborted + // rewrites. A crash between advance() steps 1-3 leaves a new base RDB on + // disk that the active manifest never references. Without this sweep, + // repeated crashes during rewrite can fill the disk with zombie files. + // + // Safe to call here: parse_* verified the manifest has all required + // records, so cleanup_orphans won't delete the active files. + manifest.cleanup_orphans(); + + Ok(Some(manifest)) + } + + /// Parse a v1 (TopLevel, single-shard) manifest. + fn parse_v1(content: &str, dir: &Path, manifest_path: &Path) -> std::io::Result { let mut seq = 0u64; let mut has_base_record = false; let mut has_incr_record = false; @@ -183,10 +372,6 @@ impl AofManifest { )); } - // A valid manifest must have all three records (seq, base, incr). - // A truncated manifest with only "seq N" but no base/incr lines could - // trigger orphan cleanup that deletes the real base RDB referenced by - // the previous valid manifest. Require all records before proceeding. if !has_base_record || !has_incr_record { return Err(std::io::Error::new( std::io::ErrorKind::InvalidData, @@ -200,33 +385,220 @@ impl AofManifest { )); } - let manifest = Self { + Ok(Self { dir: dir.to_path_buf(), seq, - }; + layout: AofLayout::TopLevel, + shards: vec![ShardManifest { + shard_id: 0, + max_lsn: 0, + }], + }) + } - // Best-effort orphan cleanup: delete stray base/incr files from aborted - // rewrites. A crash between advance() steps 1-3 leaves a new base RDB on - // disk that the active manifest never references. Without this sweep, - // repeated crashes during rewrite can fill the disk with zombie files. - // - // Safe to call here: we verified the manifest has all three records - // (seq, base, incr), so cleanup_orphans won't delete the active files. - manifest.cleanup_orphans(); + /// Parse a v2 (PerShard, multi-shard) manifest. + /// + /// Expected line format: + /// ```text + /// version 2 + /// seq N + /// shards K + /// shard 0 max_lsn LSN0 + /// shard 1 max_lsn LSN1 + /// ... + /// ``` + /// + /// Per-shard `base`/`incr` paths are derived from `shard-{N}/moon.aof.{seq}.*` + /// rather than stored explicitly — the layout is canonical, so storing + /// paths invites drift between the stored value and the computed one. + fn parse_v2(content: &str, dir: &Path, manifest_path: &Path) -> std::io::Result { + let mut seq = 0u64; + let mut num_shards: Option = None; + let mut shards: Vec = Vec::new(); - Ok(Some(manifest)) + for line in content.lines() { + let line = line.trim(); + if line.is_empty() || line.starts_with('#') { + continue; + } + if line == "version 2" { + continue; + } else if let Some(val) = line.strip_prefix("seq ") { + seq = val.parse::().map_err(|e| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!( + "AOF manifest at {} has invalid seq line `{}`: {}", + manifest_path.display(), + line, + e, + ), + ) + })?; + } else if let Some(val) = line.strip_prefix("shards ") { + num_shards = Some(val.parse::().map_err(|e| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!( + "AOF manifest at {} has invalid shards line `{}`: {}", + manifest_path.display(), + line, + e, + ), + ) + })?); + } else if let Some(rest) = line.strip_prefix("shard ") { + // Format: `shard max_lsn ` + let mut it = rest.split_whitespace(); + let id_str = it.next().ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!( + "AOF manifest at {} has shard line missing id: `{}`", + manifest_path.display(), + line, + ), + ) + })?; + let id: u16 = id_str.parse().map_err(|e| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!( + "AOF manifest at {} has shard line invalid id `{}`: {}", + manifest_path.display(), + id_str, + e, + ), + ) + })?; + // Expect `max_lsn `. + let label = it.next().unwrap_or(""); + let val_str = it.next().unwrap_or("0"); + if label != "max_lsn" { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!( + "AOF manifest at {} shard {} expected `max_lsn`, got `{}`", + manifest_path.display(), + id, + label, + ), + )); + } + let max_lsn: u64 = val_str.parse().map_err(|e| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!( + "AOF manifest at {} shard {} invalid max_lsn `{}`: {}", + manifest_path.display(), + id, + val_str, + e, + ), + ) + })?; + shards.push(ShardManifest { + shard_id: id, + max_lsn, + }); + } + // Unknown lines are tolerated (forward-compat). Strict parsers can + // be added at v3 if needed. + } + + if seq == 0 { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!( + "AOF manifest at {} has no valid sequence number", + manifest_path.display() + ), + )); + } + + let expected = num_shards.ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!( + "AOF manifest at {} is missing required `shards N` line", + manifest_path.display() + ), + ) + })?; + + if shards.len() != expected as usize { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!( + "AOF manifest at {} declares shards={} but has {} shard records", + manifest_path.display(), + expected, + shards.len(), + ), + )); + } + + // Sort by shard_id and verify contiguous range [0, expected). + shards.sort_by_key(|s| s.shard_id); + for (i, s) in shards.iter().enumerate() { + if s.shard_id as usize != i { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!( + "AOF manifest at {} has non-contiguous shard ids (expected {} at position {}, got {})", + manifest_path.display(), + i, + i, + s.shard_id, + ), + )); + } + } + + Ok(Self { + dir: dir.to_path_buf(), + seq, + layout: AofLayout::PerShard, + shards, + }) } /// Delete any base/incr files in `appendonlydir/` that do not match the /// current sequence. Best-effort — logs but does not propagate errors. + /// + /// For `PerShard` layout, also recurses into every `shard-N/` subdirectory + /// and removes stale/tmp files there. Aborted BGREWRITEAOF runs leave + /// `.rdb.tmp` files in the shard subdirs that otherwise accumulate forever. fn cleanup_orphans(&self) { - let aof_dir = self.aof_dir(); - let entries = match std::fs::read_dir(&aof_dir) { + match self.layout { + AofLayout::TopLevel => { + self.cleanup_orphans_dir(&self.aof_dir(), self.seq); + } + AofLayout::PerShard => { + // Top-level appendonlydir/ holds only the manifest — no data files + // to clean up there. All data lives in shard-N/ subdirs. + for shard in &self.shards { + self.cleanup_orphans_shard(shard.shard_id); + } + } + } + } + + /// Scan a single shard's directory for orphan base/incr/tmp files that do + /// not correspond to the current manifest sequence. Best-effort. + fn cleanup_orphans_shard(&self, shard_id: u16) { + self.cleanup_orphans_dir(&self.shard_dir(shard_id), self.seq); + } + + /// Core orphan sweep: scan `dir` and remove any `moon.aof.*` files whose + /// sequence is not `keep_seq`. Skips the manifest file itself. + fn cleanup_orphans_dir(&self, dir: &Path, keep_seq: u64) { + let entries = match std::fs::read_dir(dir) { Ok(e) => e, Err(_) => return, }; - let current_base = format!("moon.aof.{}.base.rdb", self.seq); - let current_incr = format!("moon.aof.{}.incr.aof", self.seq); + let current_base = format!("moon.aof.{}.base.rdb", keep_seq); + let current_incr = format!("moon.aof.{}.incr.aof", keep_seq); for entry in entries.flatten() { let name = entry.file_name(); let name_str = match name.to_str() { @@ -258,116 +630,661 @@ impl AofManifest { } /// Write the manifest file atomically (write tmp + rename). + /// + /// Emits v1 format for `TopLevel` and v2 for `PerShard`. The format is + /// selected by `self.layout`, never by callers — preserving the invariant + /// that one directory holds one layout. pub fn write_manifest(&self) -> std::io::Result<()> { let manifest_path = self.manifest_path(); let tmp_path = manifest_path.with_extension("tmp"); - let content = format!( - "seq {}\nbase moon.aof.{}.base.rdb\nincr moon.aof.{}.incr.aof\n", - self.seq, self.seq, self.seq - ); + let content = match self.layout { + AofLayout::TopLevel => format!( + "seq {}\nbase moon.aof.{}.base.rdb\nincr moon.aof.{}.incr.aof\n", + self.seq, self.seq, self.seq + ), + AofLayout::PerShard => { + let mut s = String::with_capacity(64 + self.shards.len() * 40); + s.push_str("version 2\n"); + s.push_str(&format!("seq {}\n", self.seq)); + s.push_str(&format!("shards {}\n", self.shards.len())); + for shard in &self.shards { + s.push_str(&format!( + "shard {} max_lsn {}\n", + shard.shard_id, shard.max_lsn + )); + } + s + } + }; let mut f = std::fs::File::create(&tmp_path)?; f.write_all(content.as_bytes())?; f.sync_data()?; std::fs::rename(&tmp_path, &manifest_path)?; + fsync_parent_best_effort(&manifest_path); Ok(()) } - /// Advance to the next sequence: write new base RDB, create new incr file, - /// update manifest, delete old files. + // ------------------------------------------------------------------ + // Per-shard layout helpers + // ------------------------------------------------------------------ + + /// Directory holding a shard's AOF files. /// - /// Returns the path to the new incremental file (caller should switch writing to it). - pub fn advance(&mut self, rdb_bytes: &[u8]) -> Result { - let old_seq = self.seq; - let new_seq = old_seq + 1; + /// - `TopLevel`: `appendonlydir/` (the shard_id argument is asserted to be 0). + /// - `PerShard`: `appendonlydir/shard-{shard_id}/`. + pub fn shard_dir(&self, shard_id: u16) -> PathBuf { + match self.layout { + AofLayout::TopLevel => { + debug_assert_eq!(shard_id, 0, "TopLevel layout only has shard 0"); + self.aof_dir() + } + AofLayout::PerShard => self.aof_dir().join(format!("shard-{}", shard_id)), + } + } - let aof_dir = self.aof_dir(); - std::fs::create_dir_all(&aof_dir).map_err(|e| crate::error::AofError::Io { - path: aof_dir.clone(), - source: e, - })?; + /// Path to a shard's base RDB file for the current sequence. + pub fn shard_base_path(&self, shard_id: u16) -> PathBuf { + self.shard_dir(shard_id) + .join(format!("moon.aof.{}.base.rdb", self.seq)) + } - // 1. Write new base RDB (atomic: tmp + fsync + rename). - // Must fsync the data BEFORE renaming — a rename without prior fsync - // can publish a file whose contents aren't durable, so a crash leaves - // the manifest pointing at an empty/partial base RDB. - let new_base = self.base_path_seq(new_seq); - let tmp_base = new_base.with_extension("rdb.tmp"); - { - let mut f = - std::fs::File::create(&tmp_base).map_err(|e| crate::error::AofError::Io { - path: tmp_base.clone(), - source: e, - })?; - f.write_all(rdb_bytes) - .map_err(|e| crate::error::AofError::Io { - path: tmp_base.clone(), - source: e, - })?; - f.sync_data().map_err(|e| crate::error::AofError::Io { - path: tmp_base.clone(), - source: e, - })?; - } - std::fs::rename(&tmp_base, &new_base).map_err(|e| { - crate::error::AofError::RewriteFailed { - detail: format!("rename base: {}", e), - } - })?; + /// Path to a shard's incremental RESP file for the current sequence. + pub fn shard_incr_path(&self, shard_id: u16) -> PathBuf { + self.shard_dir(shard_id) + .join(format!("moon.aof.{}.incr.aof", self.seq)) + } - // 2. Create empty new incremental file - let new_incr = self.incr_path_seq(new_seq); - std::fs::File::create(&new_incr).map_err(|e| crate::error::AofError::Io { - path: new_incr.clone(), - source: e, - })?; + /// Path to a shard's base RDB file for a given sequence. + pub fn shard_base_path_seq(&self, shard_id: u16, seq: u64) -> PathBuf { + self.shard_dir(shard_id) + .join(format!("moon.aof.{}.base.rdb", seq)) + } - // 3. Update manifest (atomic) - self.seq = new_seq; - self.write_manifest() - .map_err(|e| crate::error::AofError::Io { - path: self.manifest_path(), - source: e, - })?; + /// Path to a shard's incremental RESP file for a given sequence. + pub fn shard_incr_path_seq(&self, shard_id: u16, seq: u64) -> PathBuf { + self.shard_dir(shard_id) + .join(format!("moon.aof.{}.incr.aof", seq)) + } - // 4. Delete old files (best-effort) - let old_base = self.base_path_seq(old_seq); - let old_incr = self.incr_path_seq(old_seq); - if old_base.exists() { - if let Err(e) = std::fs::remove_file(&old_base) { - warn!("Failed to delete old base {}: {}", old_base.display(), e); + /// Maximum LSN persisted across all shards. + /// + /// Computed (not stored) so the stored value can never drift from + /// the per-shard records. Returns 0 if `shards` is empty (defensive; + /// constructors guarantee at least one shard). + pub fn global_max_lsn(&self) -> u64 { + self.shards.iter().map(|s| s.max_lsn).max().unwrap_or(0) + } + + /// Verify that the manifest matches the runtime shard count. + /// + /// Returns the verbatim error from RFC § 3 if the shard count differs, + /// for operator-facing consistency. Callers (typically `main.rs` boot) + /// should treat this as fatal: continuing with a mismatched shard count + /// silently drops data from shards that no longer exist or replays a + /// shard's data into the wrong DashTable. + pub fn verify_shard_count(&self, expected: u16) -> Result<(), String> { + let actual = self.shards.len() as u16; + if actual != expected { + return Err(format!( + "ERR shard count changed (manifest={}, config={}); refusing to start to avoid data loss. See docs/runbooks/shard-count-change.md", + actual, expected + )); + } + Ok(()) + } + + /// Returns true if the on-disk layout under `appendonlydir/` matches the + /// legacy TopLevel format (files at top level, no `shard-N/` subdirs). + /// + /// Used by callers to detect when a v1 single-shard deployment is being + /// upgraded to v2 multi-shard and needs explicit migration. Does NOT + /// migrate — separate from `migrate_top_level_to_per_shard` so the side + /// effect is opt-in, not hidden in a load path. + pub fn is_legacy_top_level_layout(dir: &Path) -> bool { + let aof_dir = dir.join(AOF_DIR_NAME); + if !aof_dir.exists() { + return false; + } + + // Check manifest version first. If a valid v2 (PerShard) manifest exists, + // return false regardless of stray top-level files. Operators occasionally + // leave old base.rdb / incr.aof files at the top level during debugging + // or failed upgrades; scanning filenames without reading the manifest would + // produce a misleading "legacy detected" result and trigger unwanted + // migration on an already-upgraded deployment. + if let Ok(Some(m)) = Self::load(dir) { + if m.layout == AofLayout::PerShard { + return false; } } - if old_incr.exists() { - if let Err(e) = std::fs::remove_file(&old_incr) { - warn!("Failed to delete old incr {}: {}", old_incr.display(), e); + + let entries = match std::fs::read_dir(&aof_dir) { + Ok(e) => e, + Err(_) => return false, + }; + for entry in entries.flatten() { + let name = entry.file_name(); + let Some(name_str) = name.to_str() else { + continue; + }; + if name_str.starts_with("moon.aof.") + && (name_str.ends_with(".base.rdb") || name_str.ends_with(".incr.aof")) + { + return true; } } + false + } - info!( - "AOF advanced to seq {}: base={} bytes, incr={}", - new_seq, - rdb_bytes.len(), - new_incr.display() - ); + /// Migrate a single-shard TopLevel layout in place to a single-shard + /// PerShard layout. + /// + /// Moves `appendonlydir/moon.aof.{seq}.{base.rdb,incr.aof}` into + /// `appendonlydir/shard-0/`, then rewrites the manifest as v2 with + /// `shards 1`. Idempotent: a second call on an already-PerShard manifest + /// returns Ok with no filesystem changes. + /// + /// This is the RFC § 5 case 1 migration — zero data movement (rename only), + /// safe to run on first boot after upgrading from v0.1.x. Multi-shard + /// migrations from legacy AOF (case 2) use the `moon migrate-aof` + /// subcommand and are NOT handled here. + pub fn migrate_top_level_to_per_shard(&mut self) -> std::io::Result<()> { + if self.layout == AofLayout::PerShard { + return Ok(()); + } + if self.shards.len() != 1 { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!( + "migrate_top_level_to_per_shard called with {} shards; \ + only single-shard TopLevel can be migrated in place", + self.shards.len() + ), + )); + } - Ok(new_incr) - } -} + // Compute paths up front. shard_dir/shard_*_path_seq for a single- + // shard target are pure path computations and do NOT depend on + // self.layout, so it is safe to derive them while layout is still + // TopLevel. + let old_base = self + .aof_dir() + .join(format!("moon.aof.{}.base.rdb", self.seq)); + let old_incr = self + .aof_dir() + .join(format!("moon.aof.{}.incr.aof", self.seq)); + let new_dir = self.aof_dir().join("shard-0"); + let new_base = new_dir.join(format!("moon.aof.{}.base.rdb", self.seq)); + let new_incr = new_dir.join(format!("moon.aof.{}.incr.aof", self.seq)); -/// Replay multi-part AOF: load base RDB then replay incremental RESP. -/// -/// Returns total keys/commands loaded. -pub fn replay_multi_part( - databases: &mut [crate::storage::Database], - manifest: &AofManifest, - engine: &dyn crate::persistence::replay::CommandReplayEngine, -) -> Result { - let mut total = 0usize; + if !old_base.exists() { + // Pre-flight check: nothing moved yet, no rollback needed. + return Err(std::io::Error::new( + std::io::ErrorKind::NotFound, + format!( + "TopLevel→PerShard migration: source base {} not found", + old_base.display() + ), + )); + } + std::fs::create_dir_all(&new_dir)?; - // Load base RDB - let base_path = manifest.base_path(); + // Move base. If the rename itself fails, no on-disk mutation has + // happened yet — bail without rollback. Layout stays TopLevel until + // commit at the bottom. + std::fs::rename(&old_base, &new_base)?; + + // Fsync the target directory so the base rename is durable before we + // proceed. A crash after rename but before dir-fsync could leave the + // old filename visible on the next boot. + // + // NOTE: if this fsync fails, old_base has already moved to new_base — + // rollback the rename before returning so the manifest stays consistent. + if let Err(e) = fsync_directory(&new_dir) { + if let Err(re) = std::fs::rename(&new_base, &old_base) { + error!( + "Migration rollback: failed to restore base {} → {} after fsync_directory failure: {}", + new_base.display(), + old_base.display(), + re + ); + } + return Err(e); + } + + // Base is now durably in shard-0/. Any subsequent error must restore it. + let moved_incr: bool; + let created_incr: bool; + if old_incr.exists() { + if let Err(e) = std::fs::rename(&old_incr, &new_incr) { + if let Err(re) = std::fs::rename(&new_base, &old_base) { + error!( + "Migration rollback: failed to restore base {} → {}: {}", + new_base.display(), + old_base.display(), + re + ); + } + return Err(e); + } + // Fsync the shard directory to make the incr rename durable. + // If this fails, roll back both incr and base renames. + if let Err(e) = fsync_directory(&new_dir) { + if let Err(re) = std::fs::rename(&new_incr, &old_incr) { + error!( + "Migration rollback: failed to restore incr {} → {} after fsync_directory failure: {}", + new_incr.display(), + old_incr.display(), + re + ); + } + if let Err(re) = std::fs::rename(&new_base, &old_base) { + error!( + "Migration rollback: failed to restore base {} → {} after fsync_directory failure: {}", + new_base.display(), + old_base.display(), + re + ); + } + return Err(e); + } + moved_incr = true; + created_incr = false; + } else { + match std::fs::File::create(&new_incr) { + Ok(_) => { + moved_incr = false; + created_incr = true; + } + Err(e) => { + if let Err(re) = std::fs::rename(&new_base, &old_base) { + error!( + "Migration rollback: failed to restore base {} → {}: {}", + new_base.display(), + old_base.display(), + re + ); + } + return Err(e); + } + } + } + + // Commit: flip layout, persist as v2. If write_manifest fails, undo + // every filesystem mutation and restore layout so the next boot still + // sees a valid v1 TopLevel deployment. + self.layout = AofLayout::PerShard; + if let Err(e) = self.write_manifest() { + self.layout = AofLayout::TopLevel; + if moved_incr { + if let Err(re) = std::fs::rename(&new_incr, &old_incr) { + error!( + "Migration rollback: failed to restore incr {} → {}: {}", + new_incr.display(), + old_incr.display(), + re + ); + } + } else if created_incr { + if let Err(re) = std::fs::remove_file(&new_incr) { + warn!( + "Migration rollback: failed to remove freshly created incr {}: {}", + new_incr.display(), + re + ); + } + } + if let Err(re) = std::fs::rename(&new_base, &old_base) { + error!( + "Migration rollback: failed to restore base {} → {}: {}. \ + Manifest dir {} may be in an inconsistent state.", + new_base.display(), + old_base.display(), + re, + self.dir.display() + ); + } + return Err(e); + } + + info!( + "AOF migrated: TopLevel → PerShard (single shard) at {}", + self.aof_dir().display() + ); + Ok(()) + } + + /// Create the `appendonlydir/` and write an initial v2 manifest for the + /// given shard count. + /// + /// Each shard gets its own `shard-{N}/` subdirectory with an empty base + /// RDB and an empty incr file. Mirrors `initialize()` semantics: the + /// `(base + incr)` invariant holds from the first boot, so recovery can + /// replay incr-only state without complaint. + /// + /// **Idempotency pre-flight:** if `appendonlydir/moon.aof.manifest` already + /// exists, returns `Err(AlreadyExists)` without modifying any files. A + /// mid-loop crash followed by a retry would otherwise overwrite the already- + /// written shard-0 base RDB with an empty RDB, losing state. Callers that + /// want resume-or-skip semantics should use [`Self::try_initialize_multi`]. + /// + /// **Rollback on partial failure:** if the per-shard loop fails mid-way (e.g. + /// shard-1 write fails after shard-0 succeeded), all already-created shard + /// base RDB files are deleted before returning the error. + pub fn initialize_multi(dir: &Path, num_shards: u16) -> std::io::Result { + if num_shards == 0 { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "initialize_multi requires num_shards >= 1", + )); + } + let manifest = Self { + dir: dir.to_path_buf(), + seq: 1, + layout: AofLayout::PerShard, + shards: (0..num_shards) + .map(|id| ShardManifest { + shard_id: id, + max_lsn: 0, + }) + .collect(), + }; + std::fs::create_dir_all(manifest.aof_dir())?; + + // Pre-flight: refuse if manifest already exists to avoid overwriting + // already-written shard base RDB files (idempotency guard). + let manifest_path = manifest.manifest_path(); + if manifest_path.exists() { + return Err(std::io::Error::new( + std::io::ErrorKind::AlreadyExists, + format!( + "initialize_multi: manifest already exists at {}; \ + use try_initialize_multi() for idempotent initialization", + manifest_path.display() + ), + )); + } + + // Per-shard empty RDB. Single Database::default() inside a 1-element + // slice matches `initialize()`'s empty-RDB shape for each shard. + let empty_dbs: [crate::storage::Database; 0] = []; + let empty_rdb = crate::persistence::rdb::save_to_bytes(&empty_dbs) + .map_err(|e| std::io::Error::other(format!("empty RDB serialize: {e}")))?; + + // Track which shard directories were successfully created so we can + // roll them back on partial failure. + let mut created_shards: Vec = Vec::with_capacity(num_shards as usize); + + let loop_result = (|| -> std::io::Result<()> { + for shard_id in 0..num_shards { + let shard_dir = manifest.shard_dir(shard_id); + std::fs::create_dir_all(&shard_dir)?; + + let base_path = manifest.shard_base_path(shard_id); + let tmp_path = base_path.with_extension("rdb.tmp"); + { + let mut f = std::fs::File::create(&tmp_path)?; + f.write_all(&empty_rdb)?; + f.sync_data()?; + } + std::fs::rename(&tmp_path, &base_path)?; + fsync_parent_best_effort(&base_path); + std::fs::File::create(manifest.shard_incr_path(shard_id))?; + created_shards.push(shard_id); + } + Ok(()) + })(); + + if let Err(e) = loop_result { + // Rollback: remove base RDB files for all successfully-created shards. + for sid in created_shards { + let base = manifest.shard_base_path(sid); + if let Err(re) = std::fs::remove_file(&base) { + warn!( + "initialize_multi rollback: failed to remove {}: {}", + base.display(), + re + ); + } + } + return Err(e); + } + + manifest.write_manifest()?; + Ok(manifest) + } + + /// Initialize a v2 multi-shard manifest only if one does not already exist. + /// + /// Returns `Ok(Some(manifest))` on successful creation, or `Ok(None)` if the + /// manifest file already existed (already initialized — no files modified). + /// Returns `Err(_)` only on actual I/O failures. + pub fn try_initialize_multi(dir: &Path, num_shards: u16) -> std::io::Result> { + match Self::initialize_multi(dir, num_shards) { + Ok(m) => Ok(Some(m)), + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => Ok(None), + Err(e) => Err(e), + } + } + + /// Advance to the next sequence: write new base RDB, create new incr file, + /// update manifest, delete old files. + /// + /// Returns the path to the new incremental file (caller should switch writing to it). + pub fn advance(&mut self, rdb_bytes: &[u8]) -> Result { + let old_seq = self.seq; + let new_seq = old_seq + 1; + + let aof_dir = self.aof_dir(); + std::fs::create_dir_all(&aof_dir).map_err(|e| crate::error::AofError::Io { + path: aof_dir.clone(), + source: e, + })?; + + // 1. Write new base RDB (atomic: tmp + fsync + rename). + // Must fsync the data BEFORE renaming — a rename without prior fsync + // can publish a file whose contents aren't durable, so a crash leaves + // the manifest pointing at an empty/partial base RDB. + let new_base = self.base_path_seq(new_seq); + let tmp_base = new_base.with_extension("rdb.tmp"); + { + let mut f = + std::fs::File::create(&tmp_base).map_err(|e| crate::error::AofError::Io { + path: tmp_base.clone(), + source: e, + })?; + f.write_all(rdb_bytes) + .map_err(|e| crate::error::AofError::Io { + path: tmp_base.clone(), + source: e, + })?; + f.sync_data().map_err(|e| crate::error::AofError::Io { + path: tmp_base.clone(), + source: e, + })?; + } + std::fs::rename(&tmp_base, &new_base).map_err(|e| { + crate::error::AofError::RewriteFailed { + detail: format!("rename base: {}", e), + } + })?; + fsync_parent_best_effort(&new_base); + + // 2. Create empty new incremental file + let new_incr = self.incr_path_seq(new_seq); + std::fs::File::create(&new_incr).map_err(|e| crate::error::AofError::Io { + path: new_incr.clone(), + source: e, + })?; + + // 3. Update manifest (atomic) + self.seq = new_seq; + self.write_manifest() + .map_err(|e| crate::error::AofError::Io { + path: self.manifest_path(), + source: e, + })?; + + // 4. Delete old files (best-effort) + let old_base = self.base_path_seq(old_seq); + let old_incr = self.incr_path_seq(old_seq); + if old_base.exists() { + if let Err(e) = std::fs::remove_file(&old_base) { + warn!("Failed to delete old base {}: {}", old_base.display(), e); + } + } + if old_incr.exists() { + if let Err(e) = std::fs::remove_file(&old_incr) { + warn!("Failed to delete old incr {}: {}", old_incr.display(), e); + } + } + + info!( + "AOF advanced to seq {}: base={} bytes, incr={}", + new_seq, + rdb_bytes.len(), + new_incr.display() + ); + + Ok(new_incr) + } + + /// Advance a single shard to a new sequence: write the shard's new base RDB, + /// create a new empty incr file, delete old shard files, then update the + /// shard's `max_lsn` in the in-memory manifest. + /// + /// **Caller MUST call `write_manifest()` after all shards have been advanced** + /// to persist the updated manifest atomically. Advancing shards one at a time + /// and writing the manifest per-shard would leave the manifest in an + /// inconsistent state between calls. + /// + /// For `TopLevel` layout, `shard_id` must be 0 and this delegates to + /// `advance()`. For `PerShard` layout, files are written to + /// `shard_dir(shard_id)/`. + /// + /// Returns the path to the new incremental file for this shard. + pub fn advance_shard( + &mut self, + shard_id: u16, + new_seq: u64, + rdb_bytes: &[u8], + ) -> Result { + if self.layout == AofLayout::TopLevel { + debug_assert_eq!(shard_id, 0, "TopLevel layout only has shard 0"); + return self.advance(rdb_bytes); + } + + // Validate shard_id is known in this manifest. + let shard_idx = self + .shards + .iter() + .position(|s| s.shard_id == shard_id) + .ok_or_else(|| crate::error::AofError::RewriteFailed { + detail: format!( + "advance_shard: shard_id {} not in manifest (shards: {})", + shard_id, + self.shards.len() + ), + })?; + + let old_seq = self.seq; + let shard_dir = self.shard_dir(shard_id); + std::fs::create_dir_all(&shard_dir).map_err(|e| crate::error::AofError::Io { + path: shard_dir.clone(), + source: e, + })?; + + // 1. Write new base RDB atomically: tmp + fsync + rename. + let new_base = self.shard_base_path_seq(shard_id, new_seq); + let tmp_base = new_base.with_extension("rdb.tmp"); + { + let mut f = + std::fs::File::create(&tmp_base).map_err(|e| crate::error::AofError::Io { + path: tmp_base.clone(), + source: e, + })?; + f.write_all(rdb_bytes) + .map_err(|e| crate::error::AofError::Io { + path: tmp_base.clone(), + source: e, + })?; + f.sync_data().map_err(|e| crate::error::AofError::Io { + path: tmp_base.clone(), + source: e, + })?; + } + std::fs::rename(&tmp_base, &new_base).map_err(|e| { + crate::error::AofError::RewriteFailed { + detail: format!( + "advance_shard {}: rename base {}: {}", + shard_id, + tmp_base.display(), + e + ), + } + })?; + fsync_parent_best_effort(&new_base); + + // 2. Create empty new incremental file. + let new_incr = self.shard_incr_path_seq(shard_id, new_seq); + std::fs::File::create(&new_incr).map_err(|e| crate::error::AofError::Io { + path: new_incr.clone(), + source: e, + })?; + + // 3. Delete old shard files (best-effort). + let old_base = self.shard_base_path_seq(shard_id, old_seq); + let old_incr = self.shard_incr_path_seq(shard_id, old_seq); + if old_base.exists() { + if let Err(e) = std::fs::remove_file(&old_base) { + warn!( + "advance_shard {}: failed to delete old base {}: {}", + shard_id, + old_base.display(), + e + ); + } + } + if old_incr.exists() { + if let Err(e) = std::fs::remove_file(&old_incr) { + warn!( + "advance_shard {}: failed to delete old incr {}: {}", + shard_id, + old_incr.display(), + e + ); + } + } + + // 4. Update per-shard LSN in-memory (manifest write is the caller's job). + self.shards[shard_idx].max_lsn = self.shards[shard_idx].max_lsn.max(new_seq); + + info!( + "AOF shard {} advanced to seq {}: base={} bytes, incr={}", + shard_id, + new_seq, + rdb_bytes.len(), + new_incr.display() + ); + + Ok(new_incr) + } +} + +/// Replay multi-part AOF: load base RDB then replay incremental RESP. +/// +/// Returns total keys/commands loaded. +pub fn replay_multi_part( + databases: &mut [crate::storage::Database], + manifest: &AofManifest, + engine: &dyn crate::persistence::replay::CommandReplayEngine, +) -> Result { + let mut total = 0usize; + + // Load base RDB + let base_path = manifest.base_path(); if base_path.exists() { match crate::persistence::rdb::load(databases, &base_path) { Ok(n) => { @@ -517,3 +1434,1501 @@ fn replay_incr_resp( Ok(count) } + +/// An entry that was tagged `OrderedAcrossShards` (RFC § 2 Rule 2) and +/// must be merge-replayed in global LSN order after per-shard replay +/// completes. The `shard_id` records which shard's file it came from so +/// the merge step can dispatch each entry back to its origin shard's +/// databases. +#[derive(Debug, Clone)] +pub struct OrderedEntry { + pub shard_id: u16, + pub lsn: u64, + pub bytes: bytes::Bytes, +} + +/// Replay a framed PerShard incr file: `[u64 lsn LE][u32 len LE][RESP bytes]`. +/// +/// Step 3 wrote this format; step 4 reads it. Step 5 extends the LSN field: +/// the high bit (`crate::persistence::aof::ORDERED_LSN_FLAG`) marks the +/// entry as `OrderedAcrossShards` — those entries are NOT replayed inline, +/// instead they are pushed into `ordered_buf` for the caller to merge-replay +/// in global LSN order across all shards. +/// +/// Returns `(commands_replayed, max_lsn)` — the count covers only inline +/// (non-ordered) replays, and `max_lsn` covers both inline AND ordered +/// entries (the high bit is masked out before max comparison, so it reflects +/// the true issued LSN). +/// +/// **Truncated entries:** a header partly written at crash time is treated as +/// EOF (parity with `replay_incr_resp` semantics). A whole header followed by +/// a truncated payload is also EOF — the writer's invariant is that the +/// header is written first then the payload, and on partial write the most we +/// can lose is the last entry's payload tail. +/// +/// **Corruption:** a mid-stream RESP parse error inside an otherwise-complete +/// payload is fatal (same reasoning as `replay_incr_resp`). +fn replay_incr_framed( + shard_id: u16, + databases: &mut [crate::storage::Database], + data: &[u8], + engine: &dyn crate::persistence::replay::CommandReplayEngine, + ordered_buf: &mut Vec, +) -> Result<(usize, u64), crate::error::MoonError> { + use crate::protocol::{Frame, ParseConfig, parse}; + use bytes::BytesMut; + + const HEADER_LEN: usize = 12; // u64 lsn LE + u32 len LE + + let total_len = data.len(); + let mut offset: usize = 0; + let config = ParseConfig::default(); + let mut selected_db: usize = 0; + let mut count: usize = 0; + let mut max_lsn: u64 = 0; + + while offset < total_len { + if total_len - offset < HEADER_LEN { + warn!( + "AOF incr framed truncated header: {} bytes at offset {} (treating as crash-time EOF)", + total_len - offset, + offset + ); + break; + } + // SAFETY: line 1491 guarantees `total_len - offset >= HEADER_LEN` (=12), + // so the [offset..offset+8] and [offset+8..offset+12] slices are valid + // and `try_into()` to a fixed-size array cannot fail (length-matched). + #[allow(clippy::unwrap_used)] // bounds-checked above; try_into is statically length-matched + let raw_lsn = u64::from_le_bytes(data[offset..offset + 8].try_into().expect("8 bytes")); + #[allow(clippy::unwrap_used)] // same bounds-check guarantee + let len = + u32::from_le_bytes(data[offset + 8..offset + 12].try_into().expect("4 bytes")) as usize; + let payload_start = offset + HEADER_LEN; + let payload_end = payload_start.saturating_add(len); + if payload_end > total_len { + warn!( + "AOF incr framed truncated payload at offset {} (lsn {:#x}, declared len {}, have {} bytes); treating as crash-time EOF", + offset, + raw_lsn, + len, + total_len - payload_start + ); + break; + } + + // Strip the OrderedAcrossShards flag to recover the true LSN. + let is_ordered = raw_lsn & crate::persistence::aof::ORDERED_LSN_FLAG != 0; + let lsn = raw_lsn & !crate::persistence::aof::ORDERED_LSN_FLAG; + + // Ordered entries: buffer for cross-shard merge replay; do NOT + // dispatch inline. + if is_ordered { + let bytes = bytes::Bytes::copy_from_slice(&data[payload_start..payload_end]); + ordered_buf.push(OrderedEntry { + shard_id, + lsn, + bytes, + }); + if lsn > max_lsn { + max_lsn = lsn; + } + offset = payload_end; + continue; + } + + // Parse RESP from the payload slice. A standalone slice ensures one + // header maps to exactly one command — no implicit pipelining across + // headers. + let mut buf = BytesMut::from(&data[payload_start..payload_end]); + match parse::parse(&mut buf, &config) { + Ok(Some(frame)) => { + let (cmd, cmd_args) = match &frame { + Frame::Array(arr) if !arr.is_empty() => { + let name = match &arr[0] { + Frame::BulkString(s) => s.as_ref(), + Frame::SimpleString(s) => s.as_ref(), + other => { + return Err(crate::error::MoonError::from( + crate::error::AofError::RewriteFailed { + detail: format!( + "AOF incr framed command at offset {} (lsn {}) has non-string name frame: {:?}", + offset, + lsn, + std::mem::discriminant(other) + ), + }, + )); + } + }; + (name as &[u8], &arr[1..]) + } + other => { + return Err(crate::error::MoonError::from( + crate::error::AofError::RewriteFailed { + detail: format!( + "AOF incr framed non-array frame at offset {} (lsn {}): {:?}", + offset, + lsn, + std::mem::discriminant(other) + ), + }, + )); + } + }; + engine.replay_command(databases, cmd, cmd_args, &mut selected_db); + count += 1; + if lsn > max_lsn { + max_lsn = lsn; + } + } + Ok(None) => { + // Header said `len` bytes of RESP, but parser can't make a + // frame from those bytes. That's corruption inside a fully + // declared payload, not a truncated tail — escalate. + return Err(crate::error::MoonError::from( + crate::error::AofError::RewriteFailed { + detail: format!( + "AOF incr framed payload at offset {} (lsn {}, len {}) parsed as incomplete frame; corrupt entry", + offset, lsn, len + ), + }, + )); + } + Err(e) => { + return Err(crate::error::MoonError::from( + crate::error::AofError::RewriteFailed { + detail: format!( + "AOF incr framed parse error at offset {} (lsn {}, len {}): {:?}", + offset, lsn, len, e + ), + }, + )); + } + } + + offset = payload_end; + } + + Ok((count, max_lsn)) +} + +/// Replay a PerShard multi-part AOF into N parallel `Vec` buffers. +/// +/// `per_shard_databases[i]` is shard `i`'s database vector. The manifest's +/// `shards` length MUST equal `per_shard_databases.len()`; the caller is +/// expected to have run [`AofManifest::verify_shard_count`] at boot. +/// +/// Per-shard replay is fully parallel: each shard's base RDB load and incr +/// replay run in a separate OS thread via `std::thread::scope`. Shards are +/// independent (different `DashTable` instances, no shared mutable state), so +/// this is safe and correct. Parallelism delivers the RFC § 1 benefit on +/// multi-shard deployments with large AOF files. +/// +/// The `engine_factory` closure is called once per shard thread to produce an +/// independent replay engine. This is required because `CommandReplayEngine` +/// implementations (e.g., `DispatchReplayEngine` under the `graph` feature) +/// may contain non-`Sync` state (`RefCell`) that cannot be safely shared across +/// threads. Each thread owns its own engine; results (total count, max LSN, +/// ordered entries) are collected and merged in the caller thread after all +/// shard threads complete. +/// +/// Returns `(total_commands_replayed, global_max_lsn, ordered_entries)`: +/// - `total_commands_replayed` covers all inline (non-ordered) entries +/// plus the base-RDB key count. +/// - `global_max_lsn` is `max(per-shard max LSN)` across both inline and +/// ordered entries; the caller is expected to call +/// `ReplicationState::seed_master_offset(global_max_lsn)` before +/// accepting client traffic (RFC § 2 Rule 3). +/// - `ordered_entries` is the set of `OrderedAcrossShards`-tagged entries +/// across ALL shards; the caller passes them to +/// [`replay_ordered_merge`] for the cross-shard merge replay. +pub fn replay_per_shard( + per_shard_databases: &mut [&mut [crate::storage::Database]], + manifest: &AofManifest, + engine_factory: &( + dyn Fn() -> Box + Sync + ), +) -> Result<(usize, u64, Vec), crate::error::MoonError> { + debug_assert_eq!( + manifest.layout, + AofLayout::PerShard, + "replay_per_shard called on TopLevel manifest" + ); + if manifest.shards.len() != per_shard_databases.len() { + return Err(crate::error::MoonError::from( + crate::error::AofError::RewriteFailed { + detail: format!( + "replay_per_shard shard-count mismatch: manifest has {} shards, caller passed {} database vectors", + manifest.shards.len(), + per_shard_databases.len() + ), + }, + )); + } + + // Per-shard type alias for the thread result. + type ShardResult = Result<(usize, u64, Vec), crate::error::MoonError>; + + // Use std::thread::scope so each shard thread borrows its databases slice + // without a 'static lifetime requirement. All threads complete before scope + // exits, which satisfies the borrow checker. Errors are propagated via + // a Vec collected after join. + let shard_results: Vec = std::thread::scope(|scope| { + let mut handles = Vec::with_capacity(per_shard_databases.len()); + + for (shard_id, databases) in per_shard_databases.iter_mut().enumerate() { + let sid = shard_id as u16; + let base_path = manifest.shard_base_path(sid); + let incr_path = manifest.shard_incr_path(sid); + let engine = engine_factory(); + + handles.push(scope.spawn(move || -> ShardResult { + let mut shard_total: usize = 0; + let mut shard_max_lsn: u64 = 0; + let mut shard_ordered: Vec = Vec::new(); + + // Load this shard's base RDB. + if base_path.exists() { + match crate::persistence::rdb::load(*databases, &base_path) { + Ok(n) => { + info!( + "AOF shard-{} base RDB loaded: {} keys from {}", + sid, + n, + base_path.display() + ); + shard_total += n; + } + Err(e) => { + error!("AOF shard-{} base RDB load failed: {}", sid, e); + return Err(e); + } + } + } else { + // Missing base is tolerable only when this shard's incr file is + // empty (or absent). Same invariant as `replay_multi_part`. + let incr_len = + std::fs::metadata(&incr_path).map(|m| m.len()).unwrap_or(0); + if incr_len > 0 { + return Err(crate::error::MoonError::from( + crate::error::AofError::RewriteFailed { + detail: format!( + "AOF shard-{} base RDB missing at {} but incr {} is {} bytes; refusing to replay incr against empty state", + sid, + base_path.display(), + incr_path.display(), + incr_len, + ), + }, + )); + } + warn!( + "AOF shard-{} base RDB not found: {} (incr empty, treating as fresh init)", + sid, + base_path.display() + ); + } + + // Replay this shard's framed incr file. + if incr_path.exists() { + let data = std::fs::read(&incr_path).map_err(|e| { + crate::error::MoonError::from(crate::error::AofError::Io { + path: incr_path.clone(), + source: e, + }) + })?; + if !data.is_empty() { + let (count, max_lsn) = replay_incr_framed( + sid, + *databases, + &data, + engine.as_ref(), + &mut shard_ordered, + )?; + info!( + "AOF shard-{} incr replayed: {} commands from {} (max lsn {})", + sid, + count, + incr_path.display(), + max_lsn + ); + shard_total += count; + if max_lsn > shard_max_lsn { + shard_max_lsn = max_lsn; + } + } + } + + Ok((shard_total, shard_max_lsn, shard_ordered)) + })); + } + + // Collect results in shard order. + handles + .into_iter() + .map(|h| { + h.join().unwrap_or_else(|_| { + Err(crate::error::MoonError::from( + crate::error::AofError::RewriteFailed { + detail: "replay_per_shard worker thread panicked".to_owned(), + }, + )) + }) + }) + .collect() + }); + + // Merge per-shard results. + let mut total: usize = 0; + let mut global_max_lsn: u64 = 0; + let mut ordered_entries: Vec = Vec::new(); + + for result in shard_results { + let (shard_total, shard_max_lsn, shard_ordered) = result?; + total += shard_total; + if shard_max_lsn > global_max_lsn { + global_max_lsn = shard_max_lsn; + } + ordered_entries.extend(shard_ordered); + } + + Ok((total, global_max_lsn, ordered_entries)) +} + +/// Merge-replay `OrderedAcrossShards` entries collected across all shards +/// in global LSN order (RFC § 2 Rule 2). +/// +/// `entries` is sorted by `lsn` ascending, then each entry is dispatched +/// against its origin shard's databases — the per-shard partition is +/// preserved because each `OrderedEntry` carries the `shard_id` it was +/// read from. This guarantees that a cross-shard atomic operation +/// committed at LSN N is replayed as a coherent group (every +/// shard's portion at LSN N is applied before any shard's LSN N+1 work). +/// +/// **Crash-time atomicity:** if a cross-shard commit was mid-write at +/// crash time, some shards may have the LSN-N entry while others don't. +/// Step 5 ships the merge mechanism only; detecting partial commits and +/// performing the corresponding rollback is left to the future cross-shard +/// TXN consumer — `replay_ordered_merge` currently best-effort-applies +/// whichever entries survived. A `warn!` is emitted when the entry count +/// per LSN is uneven across shards so operators have a forensic trail. +/// +/// **Today's emitters:** none in production code. The path is exercised +/// by tests so the round-trip wiring is verified end-to-end and ready for +/// future use. +pub fn replay_ordered_merge( + per_shard_databases: &mut [&mut [crate::storage::Database]], + mut entries: Vec, + engine: &dyn crate::persistence::replay::CommandReplayEngine, +) -> Result { + use crate::protocol::{Frame, ParseConfig, parse}; + use bytes::BytesMut; + + if entries.is_empty() { + return Ok(0); + } + + entries.sort_by_key(|e| e.lsn); + + // Per-LSN cardinality audit: detect torn cross-shard commits. + // + // A "torn" commit is one where LSN N appears in fewer shard files than + // the maximum cardinality seen for any other LSN in this batch. Applying + // partial entries violates atomicity — if the write was interrupted mid- + // commit (e.g., crash between shard-0 and shard-1 writes), replaying only + // the shard-0 portion produces an inconsistent state that cannot be + // compensated. DROP the entire torn LSN instead of applying partial data. + // + // NOTE: "torn" detection is heuristic — it compares each LSN's count + // against the maximum cardinality observed. An LSN that legitimately spans + // fewer shards (e.g. single-shard ordered op) can only occur if the batch + // is heterogeneous. Production emitters (future cross-shard TXN) must + // guarantee uniform cardinality per LSN, so this heuristic is correct for + // all currently-reachable code paths. + let mut counts: std::collections::BTreeMap = std::collections::BTreeMap::new(); + for e in &entries { + *counts.entry(e.lsn).or_insert(0) += 1; + } + let max_count = counts.values().copied().max().unwrap_or(0); + let mut torn_lsns: std::collections::BTreeSet = std::collections::BTreeSet::new(); + for (&lsn, &n) in &counts { + if n < max_count { + warn!( + "OrderedAcrossShards LSN {} appears in only {} of {} shard files; \ + torn cross-shard commit detected — dropping entry for atomicity", + lsn, n, max_count + ); + torn_lsns.insert(lsn); + } + } + + let config = ParseConfig::default(); + let mut replayed: usize = 0; + + for entry in entries { + // Skip entries belonging to a torn (partially-written) commit. + if torn_lsns.contains(&entry.lsn) { + continue; + } + let shard_idx = entry.shard_id as usize; + if shard_idx >= per_shard_databases.len() { + return Err(crate::error::MoonError::from( + crate::error::AofError::RewriteFailed { + detail: format!( + "OrderedAcrossShards entry references shard {} but only {} shards present", + entry.shard_id, + per_shard_databases.len() + ), + }, + )); + } + let mut buf = BytesMut::from(entry.bytes.as_ref()); + match parse::parse(&mut buf, &config) { + Ok(Some(Frame::Array(arr))) if !arr.is_empty() => { + let cmd = match &arr[0] { + Frame::BulkString(s) => s.as_ref(), + Frame::SimpleString(s) => s.as_ref(), + _ => { + return Err(crate::error::MoonError::from( + crate::error::AofError::RewriteFailed { + detail: format!( + "OrderedAcrossShards entry at lsn {} has non-string command frame", + entry.lsn + ), + }, + )); + } + }; + let mut selected_db: usize = 0; + let databases = &mut *per_shard_databases[shard_idx]; + engine.replay_command(databases, cmd, &arr[1..], &mut selected_db); + replayed += 1; + } + other => { + return Err(crate::error::MoonError::from( + crate::error::AofError::RewriteFailed { + detail: format!( + "OrderedAcrossShards entry at lsn {} on shard {} did not parse as RESP array: {:?}", + entry.lsn, + entry.shard_id, + other.map(|_| ()).err() + ), + }, + )); + } + } + } + + Ok(replayed) +} + +#[cfg(test)] +mod tests_v2 { + //! Unit tests for the v2 (PerShard) manifest format. + //! + //! Covers the Step 1 deliverable of the per-shard AOF RFC: + //! - v1 manifests continue to load as TopLevel (single-shard, shard_id=0) + //! - v2 round-trip: write → load → equivalent struct shape + //! - shard count mismatch produces the verbatim RFC § 3 error + //! - migrate_top_level_to_per_shard performs in-place rename and rewrites + //! the manifest as v2 + //! - global_max_lsn computes max across shards + //! - is_legacy_top_level_layout detects top-level files + + use super::*; + use std::fs; + + fn temp_dir() -> PathBuf { + // Use a global atomic counter so parallel test threads (cargo test runs + // unit tests in parallel) never produce the same directory name even + // when PID and nanosecond clock resolution are the same for two threads. + static COUNTER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let n = COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let d = std::env::temp_dir().join(format!( + "moon-aof-manifest-test-{}-{}", + std::process::id(), + n, + )); + fs::create_dir_all(&d).expect("temp dir create"); + d + } + + #[test] + fn v1_manifest_loads_as_top_level_single_shard() { + let dir = temp_dir(); + let m = AofManifest::initialize(&dir).expect("initialize v1"); + + assert_eq!(m.layout, AofLayout::TopLevel); + assert_eq!(m.shards.len(), 1); + assert_eq!(m.shards[0].shard_id, 0); + assert_eq!(m.shards[0].max_lsn, 0); + + // Reload from disk + let reloaded = AofManifest::load(&dir).expect("load").expect("present"); + assert_eq!(reloaded.layout, AofLayout::TopLevel); + assert_eq!(reloaded.shards.len(), 1); + assert_eq!(reloaded.seq, m.seq); + + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn v2_manifest_round_trips() { + let dir = temp_dir(); + let m = AofManifest::initialize_multi(&dir, 4).expect("initialize_multi"); + + assert_eq!(m.layout, AofLayout::PerShard); + assert_eq!(m.shards.len(), 4); + for (i, s) in m.shards.iter().enumerate() { + assert_eq!(s.shard_id, i as u16); + assert_eq!(s.max_lsn, 0); + } + + // Per-shard subdirs were created with empty base + incr. + for i in 0..4u16 { + assert!(m.shard_dir(i).exists(), "shard-{} dir exists", i); + assert!(m.shard_base_path(i).exists(), "shard-{} base exists", i); + assert!(m.shard_incr_path(i).exists(), "shard-{} incr exists", i); + } + + let reloaded = AofManifest::load(&dir).expect("load").expect("present"); + assert_eq!(reloaded.layout, AofLayout::PerShard); + assert_eq!(reloaded.shards.len(), 4); + assert_eq!(reloaded.seq, m.seq); + for (i, s) in reloaded.shards.iter().enumerate() { + assert_eq!(s.shard_id, i as u16); + } + + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn verify_shard_count_emits_rfc_error_verbatim() { + let m = AofManifest { + dir: PathBuf::from("/tmp/nowhere"), + seq: 1, + layout: AofLayout::PerShard, + shards: vec![ + ShardManifest { + shard_id: 0, + max_lsn: 0, + }, + ShardManifest { + shard_id: 1, + max_lsn: 0, + }, + ], + }; + let err = m.verify_shard_count(4).expect_err("should mismatch"); + assert_eq!( + err, + "ERR shard count changed (manifest=2, config=4); refusing to start to avoid data loss. See docs/runbooks/shard-count-change.md" + ); + + // Matching count succeeds. + m.verify_shard_count(2).expect("match"); + } + + #[test] + fn migrate_top_level_to_per_shard_moves_files_and_rewrites_manifest() { + let dir = temp_dir(); + let mut m = AofManifest::initialize(&dir).expect("initialize v1"); + + // Write a marker into the incr file so we can prove the contents + // survive the rename. + let original_incr = m.aof_dir().join(format!("moon.aof.{}.incr.aof", m.seq)); + fs::write(&original_incr, b"MARKER").expect("write incr marker"); + + m.migrate_top_level_to_per_shard().expect("migrate"); + + assert_eq!(m.layout, AofLayout::PerShard); + assert!(!original_incr.exists(), "old incr removed by rename"); + let new_incr = m.shard_incr_path(0); + assert!(new_incr.exists(), "new shard-0 incr exists"); + let contents = fs::read(&new_incr).expect("read new incr"); + assert_eq!(contents, b"MARKER", "incr contents preserved"); + + // Reloaded manifest is v2. + let reloaded = AofManifest::load(&dir).expect("load").expect("present"); + assert_eq!(reloaded.layout, AofLayout::PerShard); + assert_eq!(reloaded.shards.len(), 1); + + // Idempotency: second call is a no-op. + m.migrate_top_level_to_per_shard().expect("idempotent"); + assert_eq!(m.layout, AofLayout::PerShard); + + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn global_max_lsn_returns_max_across_shards() { + let m = AofManifest { + dir: PathBuf::from("/tmp/nowhere"), + seq: 1, + layout: AofLayout::PerShard, + shards: vec![ + ShardManifest { + shard_id: 0, + max_lsn: 100, + }, + ShardManifest { + shard_id: 1, + max_lsn: 500, + }, + ShardManifest { + shard_id: 2, + max_lsn: 250, + }, + ], + }; + assert_eq!(m.global_max_lsn(), 500); + } + + #[test] + fn is_legacy_top_level_layout_detects_v1_files() { + let dir = temp_dir(); + // No appendonlydir yet → false. + assert!(!AofManifest::is_legacy_top_level_layout(&dir)); + + // After v1 initialize, top-level files present → true. + let _m = AofManifest::initialize(&dir).expect("init v1"); + assert!(AofManifest::is_legacy_top_level_layout(&dir)); + + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn is_legacy_top_level_layout_returns_false_for_v2() { + let dir = temp_dir(); + let _m = AofManifest::initialize_multi(&dir, 2).expect("init v2"); + assert!( + !AofManifest::is_legacy_top_level_layout(&dir), + "v2 layout has no top-level moon.aof.* files" + ); + + fs::remove_dir_all(&dir).ok(); + } + + /// FIX-W3-4: v2 manifest with stray top-level .base.rdb must return false, + /// not true. The filename scan is misleading when a valid v2 manifest exists. + /// + /// Scenario: operator upgraded to v2 but left a stale `moon.aof.1.base.rdb` + /// at the top level (e.g., copied during debugging). `is_legacy_top_level_layout` + /// must check the manifest first and return false when v2 is confirmed. + #[test] + fn is_legacy_top_level_layout_ignores_stray_files_when_v2_manifest_present() { + let dir = temp_dir(); + // Initialize a genuine v2 (PerShard) layout. + let _m = AofManifest::initialize_multi(&dir, 2).expect("init v2"); + + // Plant a stale top-level base.rdb to simulate the stray-file scenario. + let stray = dir.join(AOF_DIR_NAME).join("moon.aof.1.base.rdb"); + fs::write(&stray, b"REDIS0011\xff").expect("write stray base.rdb"); + + // Even though the stray file matches the filename pattern, a valid v2 + // manifest is present, so is_legacy_top_level_layout must return false. + assert!( + !AofManifest::is_legacy_top_level_layout(&dir), + "v2 manifest + stray top-level file must still return false" + ); + + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn parse_v2_rejects_shard_count_mismatch_in_file() { + let dir = temp_dir(); + let aof = dir.join(AOF_DIR_NAME); + fs::create_dir_all(&aof).unwrap(); + // Manifest claims shards 3 but only declares two shard records. + fs::write( + aof.join(MANIFEST_NAME), + "version 2\nseq 1\nshards 3\nshard 0 max_lsn 0\nshard 1 max_lsn 0\n", + ) + .unwrap(); + + let err = AofManifest::load(&dir).expect_err("should reject"); + let msg = err.to_string(); + assert!( + msg.contains("declares shards=3 but has 2 shard records"), + "got: {}", + msg + ); + + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn parse_v2_rejects_non_contiguous_shard_ids() { + let dir = temp_dir(); + let aof = dir.join(AOF_DIR_NAME); + fs::create_dir_all(&aof).unwrap(); + // shards=2 but ids are {0, 2} not {0, 1}. + fs::write( + aof.join(MANIFEST_NAME), + "version 2\nseq 1\nshards 2\nshard 0 max_lsn 0\nshard 2 max_lsn 0\n", + ) + .unwrap(); + + let err = AofManifest::load(&dir).expect_err("should reject"); + let msg = err.to_string(); + assert!(msg.contains("non-contiguous shard ids"), "got: {}", msg); + + fs::remove_dir_all(&dir).ok(); + } + + // ------------------------------------------------------------------ + // Reviewer-flagged fixes: layout-aware path helpers + migration + // rollback. See the "Verify findings against current code" review + // comment on aof_manifest.rs:669-775 and :688-717. + // ------------------------------------------------------------------ + + #[test] + fn base_incr_paths_route_to_shard_zero_after_migration() { + let dir = temp_dir(); + let mut m = AofManifest::initialize(&dir).expect("init v1"); + // Pre-migration: TopLevel paths under appendonlydir/ directly. + assert_eq!(m.base_path(), m.aof_dir().join("moon.aof.1.base.rdb")); + assert_eq!(m.incr_path(), m.aof_dir().join("moon.aof.1.incr.aof")); + + m.migrate_top_level_to_per_shard().expect("migrate"); + + // Post-migration: single-file helpers must route to shard-0/ so + // replay_multi_part and advance() find the actual files. This is + // the bug the reviewer flagged for aof_manifest.rs:669-775. + let shard0 = m.aof_dir().join("shard-0"); + assert_eq!(m.base_path(), shard0.join("moon.aof.1.base.rdb")); + assert_eq!(m.incr_path(), shard0.join("moon.aof.1.incr.aof")); + assert_eq!(m.base_path_seq(7), shard0.join("moon.aof.7.base.rdb")); + assert_eq!(m.incr_path_seq(7), shard0.join("moon.aof.7.incr.aof")); + // The path the helper returns must be where the file actually lives. + assert!(m.base_path().exists(), "base file at returned path"); + assert!(m.incr_path().exists(), "incr file at returned path"); + + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn migrate_rolls_back_filesystem_when_incr_rename_fails() { + // Simulate the rename(old_incr → new_incr) failure path by making + // the destination already exist as a directory (rename onto a + // non-empty directory is an error on every supported OS). + let dir = temp_dir(); + let mut m = AofManifest::initialize(&dir).expect("init v1"); + let original_base = m.aof_dir().join("moon.aof.1.base.rdb"); + let original_incr = m.aof_dir().join("moon.aof.1.incr.aof"); + fs::write(&original_incr, b"INCR_MARKER").expect("seed incr"); + let base_bytes_before = fs::read(&original_base).expect("read base"); + + // Pre-create shard-0/moon.aof.1.incr.aof as a DIRECTORY so the + // rename fails after the base rename has already succeeded. + let shard0 = m.aof_dir().join("shard-0"); + fs::create_dir_all(shard0.join("moon.aof.1.incr.aof")).expect("seed blocker"); + + let err = m + .migrate_top_level_to_per_shard() + .expect_err("incr rename should fail"); + let _ = err; // exact error kind depends on OS + + // Rollback invariants: + // 1. Layout stays TopLevel in memory. + // 2. base file restored to its original TopLevel path. + // 3. base file contents unchanged. + // 4. on-disk manifest is still v1 (load returns layout TopLevel). + assert_eq!(m.layout, AofLayout::TopLevel, "in-memory layout reverted"); + assert!(original_base.exists(), "base restored to top-level"); + let base_bytes_after = fs::read(&original_base).expect("read base"); + assert_eq!(base_bytes_after, base_bytes_before, "base contents intact"); + let reloaded = AofManifest::load(&dir).expect("load").expect("present"); + assert_eq!(reloaded.layout, AofLayout::TopLevel, "on-disk manifest v1"); + + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn migrate_does_not_mutate_on_missing_base() { + let dir = temp_dir(); + let mut m = AofManifest::initialize(&dir).expect("init v1"); + let base = m.aof_dir().join("moon.aof.1.base.rdb"); + fs::remove_file(&base).expect("remove base"); + + let err = m + .migrate_top_level_to_per_shard() + .expect_err("missing base should fail"); + assert_eq!(err.kind(), std::io::ErrorKind::NotFound); + // Layout never flipped, no rollback needed. + assert_eq!(m.layout, AofLayout::TopLevel); + + fs::remove_dir_all(&dir).ok(); + } + + // -- Step 4 (per-shard replay) tests --------------------------------- + + fn frame_entry(lsn: u64, resp: &[u8]) -> Vec { + let mut buf = Vec::with_capacity(12 + resp.len()); + buf.extend_from_slice(&lsn.to_le_bytes()); + buf.extend_from_slice(&(resp.len() as u32).to_le_bytes()); + buf.extend_from_slice(resp); + buf + } + + /// Minimal `CommandReplayEngine` that records (lsn-implicit-via-order, cmd + /// name) calls without touching real storage. Tests use this to assert + /// the framed parser hands the right command sequence to the engine. + struct RecordingEngine { + calls: std::cell::RefCell>, + } + + impl RecordingEngine { + fn new() -> Self { + Self { + calls: std::cell::RefCell::new(Vec::new()), + } + } + } + + impl crate::persistence::replay::CommandReplayEngine for RecordingEngine { + fn replay_command( + &self, + _databases: &mut [crate::storage::Database], + cmd: &[u8], + _args: &[crate::protocol::Frame], + _selected_db: &mut usize, + ) { + self.calls + .borrow_mut() + .push(String::from_utf8_lossy(cmd).into_owned()); + } + } + + #[test] + fn replay_incr_framed_decodes_lsn_and_resp() { + // Two framed entries: PING and DBSIZE (no args, both small RESP arrays). + let mut bytes = frame_entry(7, b"*1\r\n$4\r\nPING\r\n"); + bytes.extend_from_slice(&frame_entry(11, b"*1\r\n$6\r\nDBSIZE\r\n")); + + let mut dbs: Vec = vec![crate::storage::Database::new()]; + let engine = RecordingEngine::new(); + let mut ordered: Vec = Vec::new(); + let (count, max_lsn) = + replay_incr_framed(0, &mut dbs, &bytes, &engine, &mut ordered).expect("framed replay"); + assert!(ordered.is_empty(), "no ordered entries in this stream"); + + assert_eq!(count, 2); + assert_eq!(max_lsn, 11); + let calls = engine.calls.borrow(); + assert_eq!(calls.len(), 2); + assert_eq!(calls[0], "PING"); + assert_eq!(calls[1], "DBSIZE"); + } + + #[test] + fn replay_incr_framed_truncated_header_is_crash_eof() { + // One valid entry, then a partial 5-byte header (crash mid-write). + let mut bytes = frame_entry(3, b"*1\r\n$4\r\nPING\r\n"); + bytes.extend_from_slice(&[0u8; 5]); + + let mut dbs: Vec = vec![crate::storage::Database::new()]; + let engine = RecordingEngine::new(); + let mut ordered: Vec = Vec::new(); + let (count, max_lsn) = replay_incr_framed(0, &mut dbs, &bytes, &engine, &mut ordered) + .expect("truncated-header is EOF"); + + assert_eq!(count, 1); + assert_eq!(max_lsn, 3); + } + + #[test] + fn replay_incr_framed_truncated_payload_is_crash_eof() { + // Header declares 14 bytes of RESP but only 5 actually present. + let mut bytes = Vec::new(); + bytes.extend_from_slice(&5u64.to_le_bytes()); + bytes.extend_from_slice(&14u32.to_le_bytes()); + bytes.extend_from_slice(b"*1\r\n$"); // 5 bytes, payload truncated + + let mut dbs: Vec = vec![crate::storage::Database::new()]; + let engine = RecordingEngine::new(); + let mut ordered: Vec = Vec::new(); + let (count, max_lsn) = replay_incr_framed(0, &mut dbs, &bytes, &engine, &mut ordered) + .expect("truncated-payload is EOF"); + + assert_eq!(count, 0); + assert_eq!(max_lsn, 0); + } + + #[test] + fn replay_incr_framed_complete_but_corrupt_payload_errors() { + // Header declares 4 bytes, payload is 4 bytes of garbage that won't + // parse as a RESP frame. + let mut bytes = Vec::new(); + bytes.extend_from_slice(&1u64.to_le_bytes()); + bytes.extend_from_slice(&4u32.to_le_bytes()); + bytes.extend_from_slice(b"XXXX"); + + let mut dbs: Vec = vec![crate::storage::Database::new()]; + let engine = RecordingEngine::new(); + let mut ordered: Vec = Vec::new(); + let err = replay_incr_framed(0, &mut dbs, &bytes, &engine, &mut ordered) + .expect_err("complete-but-corrupt should error"); + let msg = format!("{err}"); + assert!( + msg.contains("framed"), + "error should mention framed context, got: {msg}" + ); + } + + #[test] + fn replay_per_shard_round_trips_two_shards() { + let dir = temp_dir(); + let manifest = AofManifest::initialize_multi(&dir, 2).expect("initialize_multi 2 shards"); + + // Hand-author framed incr files: shard-0 SETs k0/v0 at lsn=10, + // shard-1 SETs k1/v1 at lsn=20. + let set_k0 = frame_entry(10, b"*3\r\n$3\r\nSET\r\n$2\r\nk0\r\n$2\r\nv0\r\n"); + let set_k1 = frame_entry(20, b"*3\r\n$3\r\nSET\r\n$2\r\nk1\r\n$2\r\nv1\r\n"); + fs::write(manifest.shard_incr_path(0), &set_k0).expect("write shard-0 incr"); + fs::write(manifest.shard_incr_path(1), &set_k1).expect("write shard-1 incr"); + + // Two independent shard database vectors. + let mut shard0: Vec = vec![crate::storage::Database::new()]; + let mut shard1: Vec = vec![crate::storage::Database::new()]; + + let (total, global_max_lsn, ordered) = { + let mut slices: Vec<&mut [crate::storage::Database]> = vec![&mut shard0, &mut shard1]; + replay_per_shard( + &mut slices, + &manifest, + &(|| { + Box::new(crate::persistence::replay::DispatchReplayEngine::new()) + as Box + }), + ) + .expect("per-shard replay") + }; + + assert_eq!(total, 2, "two SETs replayed"); + assert_eq!(global_max_lsn, 20, "global max lsn = max(shard maxes)"); + assert!(ordered.is_empty(), "no ordered entries in this stream"); + + // Each shard's DB now holds its key (and only its key). + assert!(shard0[0].len() >= 1, "shard 0 has k0"); + assert!(shard1[0].len() >= 1, "shard 1 has k1"); + + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn replay_per_shard_rejects_shard_count_mismatch() { + let dir = temp_dir(); + let manifest = AofManifest::initialize_multi(&dir, 2).expect("initialize_multi 2 shards"); + + // Only one slice — manifest says 2. + let mut shard0: Vec = vec![crate::storage::Database::new()]; + let mut slices: Vec<&mut [crate::storage::Database]> = vec![&mut shard0]; + + let err = replay_per_shard( + &mut slices, + &manifest, + &(|| { + Box::new(crate::persistence::replay::DispatchReplayEngine::new()) + as Box + }), + ) + .expect_err("shard count mismatch must error"); + let msg = format!("{err}"); + assert!( + msg.contains("shard-count mismatch"), + "error message should call out the mismatch, got: {msg}" + ); + + fs::remove_dir_all(&dir).ok(); + } + + /// FIX-W3-1: parallel per-shard replay must produce identical results to + /// sequential replay. N=4 shards, one key per shard. + /// + /// Test gate: correctness (same total/max_lsn/key distribution as sequential). + /// Wall-time comparison is flaky in CI and omitted. + #[test] + fn replay_per_shard_parallel_matches_sequential() { + let dir = temp_dir(); + let n_shards: u16 = 4; + let manifest = + AofManifest::initialize_multi(&dir, n_shards).expect("initialize_multi 4 shards"); + + // Each shard gets one SET at lsn = shard_id * 10 + 10. + for sid in 0..n_shards { + let lsn = (sid as u64 + 1) * 10; + let key = format!("k{sid}"); + let val = format!("v{sid}"); + let resp = format!( + "*3\r\n$3\r\nSET\r\n${klen}\r\n{key}\r\n${vlen}\r\n{val}\r\n", + klen = key.len(), + vlen = val.len(), + ); + let entry = frame_entry(lsn, resp.as_bytes()); + fs::write(manifest.shard_incr_path(sid), &entry).expect("write shard incr"); + } + + let mut shards: Vec> = (0..n_shards as usize) + .map(|_| vec![crate::storage::Database::new()]) + .collect(); + + let engine_factory = || { + Box::new(crate::persistence::replay::DispatchReplayEngine::new()) + as Box + }; + let (total, global_max_lsn, ordered) = { + let mut slices: Vec<&mut [crate::storage::Database]> = + shards.iter_mut().map(|s| s.as_mut_slice()).collect(); + replay_per_shard(&mut slices, &manifest, &engine_factory) + .expect("parallel per-shard replay") + }; + + assert_eq!(total, n_shards as usize, "one SET per shard = N total"); + assert_eq!( + global_max_lsn, + n_shards as u64 * 10, + "global max lsn = highest shard lsn" + ); + assert!(ordered.is_empty(), "no ordered entries"); + + // Each shard must have exactly one key. + for (sid, shard) in shards.iter().enumerate() { + assert_eq!( + shard[0].len(), + 1, + "shard {} must have exactly 1 key after parallel replay", + sid + ); + } + + fs::remove_dir_all(&dir).ok(); + } + + // -- Step 5 (OrderedAcrossShards merge) tests ------------------------ + + /// Frame an ordered entry: same on-disk layout as `frame_entry`, with + /// the high bit of LSN set. + fn frame_ordered(lsn: u64, resp: &[u8]) -> Vec { + assert_eq!( + lsn & crate::persistence::aof::ORDERED_LSN_FLAG, + 0, + "test helper expects raw lsn without the ordered flag" + ); + let tagged = lsn | crate::persistence::aof::ORDERED_LSN_FLAG; + let mut buf = Vec::with_capacity(12 + resp.len()); + buf.extend_from_slice(&tagged.to_le_bytes()); + buf.extend_from_slice(&(resp.len() as u32).to_le_bytes()); + buf.extend_from_slice(resp); + buf + } + + #[test] + fn replay_incr_framed_buffers_ordered_entries() { + // Mix: normal PING, then an ordered SET, then normal DBSIZE. + let mut bytes = frame_entry(5, b"*1\r\n$4\r\nPING\r\n"); + bytes.extend_from_slice(&frame_ordered( + 8, + b"*3\r\n$3\r\nSET\r\n$1\r\nk\r\n$1\r\nv\r\n", + )); + bytes.extend_from_slice(&frame_entry(12, b"*1\r\n$6\r\nDBSIZE\r\n")); + + let mut dbs: Vec = vec![crate::storage::Database::new()]; + let engine = RecordingEngine::new(); + let mut ordered: Vec = Vec::new(); + let (count, max_lsn) = replay_incr_framed(3, &mut dbs, &bytes, &engine, &mut ordered) + .expect("framed replay with ordered"); + + assert_eq!(count, 2, "two inline entries dispatched (PING, DBSIZE)"); + assert_eq!(max_lsn, 12, "max LSN tracks both inline and ordered"); + assert_eq!(ordered.len(), 1, "one entry buffered as ordered"); + let buffered = &ordered[0]; + assert_eq!(buffered.shard_id, 3, "shard_id forwarded"); + assert_eq!(buffered.lsn, 8, "buffered LSN has the high bit masked off"); + let calls = engine.calls.borrow(); + assert_eq!(calls.len(), 2); + assert_eq!(calls[0], "PING"); + assert_eq!(calls[1], "DBSIZE", "ordered SET was NOT dispatched inline"); + } + + #[test] + fn replay_ordered_merge_sorts_by_lsn_across_shards() { + use crate::persistence::replay::DispatchReplayEngine; + + // Three ordered entries across two shards, deliberately out of LSN + // order on the wire so the merge step has work to do. + let entries = vec![ + OrderedEntry { + shard_id: 1, + lsn: 30, + bytes: bytes::Bytes::from_static(b"*3\r\n$3\r\nSET\r\n$2\r\nb1\r\n$1\r\n3\r\n"), + }, + OrderedEntry { + shard_id: 0, + lsn: 10, + bytes: bytes::Bytes::from_static(b"*3\r\n$3\r\nSET\r\n$2\r\na1\r\n$1\r\n1\r\n"), + }, + OrderedEntry { + shard_id: 0, + lsn: 20, + bytes: bytes::Bytes::from_static(b"*3\r\n$3\r\nSET\r\n$2\r\na2\r\n$1\r\n2\r\n"), + }, + ]; + + let mut shard0: Vec = vec![crate::storage::Database::new()]; + let mut shard1: Vec = vec![crate::storage::Database::new()]; + let replayed = { + let mut slices: Vec<&mut [crate::storage::Database]> = vec![&mut shard0, &mut shard1]; + replay_ordered_merge(&mut slices, entries, &DispatchReplayEngine::new()) + .expect("ordered merge replay") + }; + + assert_eq!(replayed, 3); + assert!(shard0[0].len() >= 2, "shard 0 received a1 + a2"); + assert!(shard1[0].len() >= 1, "shard 1 received b1"); + } + + #[test] + fn replay_ordered_merge_empty_returns_zero() { + use crate::persistence::replay::DispatchReplayEngine; + + let mut shard0: Vec = vec![crate::storage::Database::new()]; + let mut slices: Vec<&mut [crate::storage::Database]> = vec![&mut shard0]; + let replayed = replay_ordered_merge(&mut slices, Vec::new(), &DispatchReplayEngine::new()) + .expect("empty merge ok"); + assert_eq!(replayed, 0); + } + + /// FIX-W3-3: torn cross-shard commit must be DROPPED entirely, not partially applied. + /// + /// Synthesize a 2-shard AOF where LSN 100 appears on shard 0 only (N=1 + /// of K=2 expected). After replay, shard 0 must NOT have the key written + /// by the LSN-100 entry (it was dropped for atomicity). + #[test] + fn replay_ordered_merge_drops_torn_commit() { + use crate::persistence::replay::DispatchReplayEngine; + + // Two shards, two complete entries at LSN 10 (one per shard) — these + // should succeed. LSN 100 appears only on shard 0 (torn) — must be dropped. + let entries = vec![ + // Complete pair: LSN 10 on both shards + OrderedEntry { + shard_id: 0, + lsn: 10, + bytes: bytes::Bytes::from_static(b"*3\r\n$3\r\nSET\r\n$2\r\nc0\r\n$1\r\n1\r\n"), + }, + OrderedEntry { + shard_id: 1, + lsn: 10, + bytes: bytes::Bytes::from_static(b"*3\r\n$3\r\nSET\r\n$2\r\nc1\r\n$1\r\n1\r\n"), + }, + // Torn entry: LSN 100 only on shard 0, not shard 1 + OrderedEntry { + shard_id: 0, + lsn: 100, + bytes: bytes::Bytes::from_static(b"*3\r\n$3\r\nSET\r\n$5\r\ntorn0\r\n$1\r\nv\r\n"), + }, + ]; + + let mut shard0: Vec = vec![crate::storage::Database::new()]; + let mut shard1: Vec = vec![crate::storage::Database::new()]; + let replayed = { + let mut slices: Vec<&mut [crate::storage::Database]> = vec![&mut shard0, &mut shard1]; + replay_ordered_merge(&mut slices, entries, &DispatchReplayEngine::new()) + .expect("ordered merge replay") + }; + + // The torn LSN-100 entry must NOT be applied (dropped for atomicity). + assert_eq!(replayed, 2, "only the complete LSN-10 pair is replayed"); + assert_eq!( + shard0[0].len(), + 1, + "shard-0 only has the complete LSN-10 key; torn LSN-100 entry must not be applied" + ); + // Verify the torn key is absent + assert!( + shard0[0].get(b"torn0").is_none(), + "torn shard-0 entry (LSN 100) must NOT be applied" + ); + } + + #[test] + fn ordered_entry_lsn_flag_set_via_try_send_append_ordered() { + use crate::persistence::aof::{AofMessage, AofWriterPool, ORDERED_LSN_FLAG}; + use crate::runtime::channel; + + let (tx0, rx0) = channel::mpsc_bounded::(4); + let (tx1, _rx1) = channel::mpsc_bounded::(4); + let pool = AofWriterPool::per_shard(vec![tx0, tx1]); + + // Raw lsn = 42; high bit must end up set on the receive side. + pool.try_send_append_ordered(0, 42, bytes::Bytes::from_static(b"x")); + let msg = rx0.try_recv().expect("ordered append delivered"); + match msg { + AofMessage::Append { lsn, .. } => { + assert_eq!( + lsn & ORDERED_LSN_FLAG, + ORDERED_LSN_FLAG, + "ordered flag set on lsn" + ); + assert_eq!( + lsn & !ORDERED_LSN_FLAG, + 42, + "low bits preserve the original lsn" + ); + } + _ => panic!("expected Append"), + } + } + + // ----------------------------------------------------------------------- + // FIX-W2-1: cleanup_orphans must recurse into shard-N/ subdirectories + // ----------------------------------------------------------------------- + + /// Build a minimal PerShard manifest fixture on disk at `seq` without + /// needing `advance_shard`. Directly creates the expected directory layout + /// so the test is self-contained and doesn't depend on FIX-W2-3 methods. + fn write_per_shard_manifest_at_seq(dir: &Path, num_shards: u16, seq: u64) -> AofManifest { + let aof_dir = dir.join(AOF_DIR_NAME); + fs::create_dir_all(&aof_dir).unwrap(); + let empty_rdb = crate::persistence::rdb::save_to_bytes(&[] as &[crate::storage::Database]) + .expect("empty rdb"); + let shards: Vec = (0..num_shards) + .map(|id| ShardManifest { + shard_id: id, + max_lsn: 0, + }) + .collect(); + let manifest = AofManifest { + dir: dir.to_path_buf(), + seq, + layout: AofLayout::PerShard, + shards, + }; + for shard_id in 0..num_shards { + let shard_dir = manifest.shard_dir(shard_id); + fs::create_dir_all(&shard_dir).unwrap(); + let base = manifest.shard_base_path(shard_id); + let tmp = base.with_extension("rdb.tmp"); + fs::write(&tmp, &empty_rdb).unwrap(); + fs::rename(&tmp, &base).unwrap(); + fs::write(manifest.shard_incr_path(shard_id), b"").unwrap(); + } + manifest.write_manifest().unwrap(); + manifest + } + + #[test] + fn cleanup_orphans_removes_stale_files_in_shard_subdirs() { + let dir = temp_dir(); + + // Build a 2-shard PerShard manifest at seq=2. + let manifest = write_per_shard_manifest_at_seq(&dir, 2, 2); + + // Inject orphan files in shard-0/ that a crashed BGREWRITEAOF would leave. + // seq=1 tmp (aborted write) and a seq=5 incr (future zombie). + let shard0_dir = manifest.shard_dir(0); + let orphan_tmp = shard0_dir.join("moon.aof.1.base.rdb.tmp"); + let orphan_old_incr = shard0_dir.join("moon.aof.5.incr.aof"); + fs::write(&orphan_tmp, b"").expect("write orphan tmp"); + fs::write(&orphan_old_incr, b"").expect("write orphan incr"); + + // Active files for seq=2 must survive. + let active_base = manifest.shard_base_path(0); + let active_incr = manifest.shard_incr_path(0); + assert!( + active_base.exists(), + "active base must exist before cleanup" + ); + assert!( + active_incr.exists(), + "active incr must exist before cleanup" + ); + + // Reload the manifest — this triggers cleanup_orphans. + let _reloaded = AofManifest::load(&dir).expect("load").expect("present"); + + assert!( + !orphan_tmp.exists(), + "orphan .rdb.tmp in shard-0/ must be deleted by cleanup_orphans" + ); + assert!( + !orphan_old_incr.exists(), + "orphan old incr in shard-0/ must be deleted by cleanup_orphans" + ); + assert!( + active_base.exists(), + "active seq=2 base must survive cleanup" + ); + assert!( + active_incr.exists(), + "active seq=2 incr must survive cleanup" + ); + + fs::remove_dir_all(&dir).ok(); + } + + // ----------------------------------------------------------------------- + // FIX-W2-2: initialize_multi idempotency — second call returns error + // ----------------------------------------------------------------------- + #[test] + fn initialize_multi_second_call_returns_already_initialized_error() { + let dir = temp_dir(); + + // First call must succeed. + let _m = AofManifest::initialize_multi(&dir, 4).expect("first call ok"); + + // Count files before second call. + let aof_dir = dir.join(AOF_DIR_NAME); + let count_before: usize = (0..4u16) + .map(|sid| { + let shard_dir = aof_dir.join(format!("shard-{}", sid)); + fs::read_dir(&shard_dir).map(|e| e.count()).unwrap_or(0) + }) + .sum(); + + // Second call must return an error with the manifest already present. + let result = AofManifest::initialize_multi(&dir, 4); + assert!( + result.is_err(), + "second initialize_multi must fail when manifest already exists" + ); + let err = result.unwrap_err(); + assert_eq!( + err.kind(), + std::io::ErrorKind::AlreadyExists, + "error kind must be AlreadyExists; got {:?}: {}", + err.kind(), + err + ); + + // File count must be unchanged — no files were overwritten. + let count_after: usize = (0..4u16) + .map(|sid| { + let shard_dir = aof_dir.join(format!("shard-{}", sid)); + fs::read_dir(&shard_dir).map(|e| e.count()).unwrap_or(0) + }) + .sum(); + assert_eq!( + count_before, count_after, + "second call must not create or overwrite any shard files" + ); + + fs::remove_dir_all(&dir).ok(); + } + + // ----------------------------------------------------------------------- + // FIX-W2-3 (partial): advance_shard writes new base+incr, deletes old + // ----------------------------------------------------------------------- + #[test] + fn advance_shard_writes_new_seq_and_deletes_old() { + let dir = temp_dir(); + + // Initialize 2-shard manifest at seq=1. + let mut manifest = AofManifest::initialize_multi(&dir, 2).expect("initialize_multi"); + assert_eq!(manifest.seq, 1); + + let empty_rdb = crate::persistence::rdb::save_to_bytes(&[] as &[crate::storage::Database]) + .expect("empty rdb"); + + // Old shard-0 files at seq=1 must exist before advance. + let old_base_s0 = manifest.shard_base_path(0); + let old_incr_s0 = manifest.shard_incr_path(0); + assert!(old_base_s0.exists(), "seq=1 base must exist for shard 0"); + assert!(old_incr_s0.exists(), "seq=1 incr must exist for shard 0"); + + // Advance shard-0 to seq=2. + let new_incr = manifest + .advance_shard(0, 2, &empty_rdb) + .expect("advance_shard 0 → seq=2"); + assert!(new_incr.exists(), "new incr file must be created"); + assert!( + manifest.shard_base_path_seq(0, 2).exists(), + "new seq=2 base must exist for shard 0" + ); + assert!( + !old_base_s0.exists(), + "old seq=1 base must be deleted for shard 0" + ); + assert!( + !old_incr_s0.exists(), + "old seq=1 incr must be deleted for shard 0" + ); + + // Shard-1 must be unaffected. + assert!( + manifest.shard_base_path(1).exists(), + "shard-1 seq=1 base must survive advance of shard-0" + ); + + // Caller must write_manifest after all shards advanced. + manifest.seq = 2; + manifest + .write_manifest() + .expect("write manifest after advance"); + let reloaded = AofManifest::load(&dir).expect("load").expect("present"); + assert_eq!(reloaded.seq, 2); + + fs::remove_dir_all(&dir).ok(); + } + + // ----------------------------------------------------------------------- + // FIX-W2-7: smoke test — fsync helper consolidation did not break + // initialize_multi. Checks that the post-consolidation manifest has the + // correct PerShard layout, the expected shard count, and per-shard + // base/incr files. This is discriminating: a regression that produces a + // TopLevel manifest or wrong shard count will be caught here. + // ----------------------------------------------------------------------- + #[test] + fn initialize_multi_smoke_after_fsync_consolidation() { + let tmp = tempfile::tempdir().expect("tempdir"); + let dir = tmp.path(); + let n: u16 = 2; + let result = AofManifest::initialize_multi(dir, n); + assert!( + result.is_ok(), + "initialize_multi({n} shards) must succeed: {:?}", + result.err() + ); + let manifest = result.unwrap(); + + // Discriminating: layout must be PerShard, not TopLevel. + assert_eq!( + manifest.layout, + AofLayout::PerShard, + "initialize_multi must produce a PerShard manifest" + ); + // Discriminating: shard count must match the requested count. + assert_eq!( + manifest.shards.len() as u16, + n, + "manifest must record exactly {n} shards, got {}", + manifest.shards.len() + ); + // Discriminating: per-shard base RDB and incr files must exist on disk. + for shard_id in 0..n { + assert!( + manifest.shard_base_path(shard_id).exists(), + "shard-{shard_id} base RDB must exist at {}", + manifest.shard_base_path(shard_id).display() + ); + assert!( + manifest.shard_incr_path(shard_id).exists(), + "shard-{shard_id} incr file must exist at {}", + manifest.shard_incr_path(shard_id).display() + ); + } + // Discriminating: the on-disk manifest file must contain `version 2` + // (PerShard v2 header), not be a bare v1 file. + let manifest_path = dir.join(AOF_DIR_NAME).join("moon.aof.manifest"); + let content = + std::fs::read_to_string(&manifest_path).expect("manifest file must be readable"); + assert!( + content.contains("version 2"), + "manifest file must contain 'version 2' (PerShard v2 header); got:\n{}", + content + ); + } +} diff --git a/src/persistence/migrate_aof.rs b/src/persistence/migrate_aof.rs new file mode 100644 index 00000000..625a9100 --- /dev/null +++ b/src/persistence/migrate_aof.rs @@ -0,0 +1,944 @@ +//! AOF v1→v2 migration: single-file legacy AOF to per-shard PerShard layout. +//! +//! # Background +//! +//! Moon v0.1.x used a single `appendonly.aof` (or `appendonlydir/` TopLevel layout +//! with one base RDB + one incr file). v0.1.12 introduces the PerShard layout where +//! each shard has its own `shard-N/moon.aof.1.base.rdb` + `shard-N/moon.aof.1.incr.aof`. +//! +//! Operators upgrading from a single-shard v1 deployment to a multi-shard v2 +//! deployment need to redistribute their existing AOF entries across N shard files. +//! This module provides the migration logic. +//! +//! # Algorithm +//! +//! 1. Load the source RDB base (preamble or separate base.rdb) into scratch +//! databases. Partition each key by `key_to_shard(key, num_shards)` into +//! per-shard scratch databases. Serialize each shard's scratch database as its +//! `shard-N/moon.aof.1.base.rdb` (replaces the empty placeholder from +//! `initialize_multi`). This preserves all data that was in the RDB preamble — +//! which is where the bulk of the data lives after `BGREWRITEAOF`. +//! 2. Read the RESP tail from the source AOF. For each command, extract the first +//! key argument and route to a shard via `key_to_shard(key, num_shards)`. +//! Commands without a key argument (PING, DBSIZE, FLUSHDB, FLUSHALL) are +//! routed to shard 0 (conservative — FLUSHDB/FLUSHALL affect all shards but +//! the migration path leaves the operator to verify). SELECT 0 is skipped as +//! a no-op; SELECT N (N>0) causes an immediate `Err` — multi-DB AOF cannot +//! be safely migrated (see Limitations). +//! 3. Write each RESP command to the target shard's incr file in v2 framing format: +//! `[u64 lsn LE][u32 len LE][RESP bytes]`. LSNs are sequential per-shard +//! counters starting at 1. One corrupt command stops the remainder of the incr +//! migration (safe truncation — matches crash-time EOF behavior). +//! 4. Write a v2 manifest covering all shards with `seq = 1`. +//! +//! # Limitations +//! +//! - Multi-db AOF (SELECT N where N>0): migration returns `Err` immediately. +//! Per-shard replay runs each shard independently and cannot preserve the +//! logical database across commands — silently dropping SELECT would corrupt +//! data. SELECT 0 is treated as a no-op and skipped. Operators must flush +//! non-default databases before migrating. +//! - MULTI/EXEC blocks are not treated atomically — each command in the block +//! is routed independently. +//! - Keyless commands (FLUSHDB, FLUSHALL) in the RESP tail go to shard 0 only; +//! operators must verify correctness for these commands. +//! +//! # Usage +//! +//! ``` +//! moon --migrate-aof-from /old/dir --migrate-aof-to /new/dir --migrate-aof-shards 4 +//! ``` +//! +//! The server exits after migration. Start the server normally pointing at +//! `--migrate-aof-to` to use the migrated data. + +use std::io::Write; +use std::path::{Path, PathBuf}; + +use bytes::{Bytes, BytesMut}; +use tracing::{info, warn}; + +use crate::persistence::aof_manifest::AofManifest; +use crate::protocol::{Frame, ParseConfig, parse}; +use crate::storage::Database; + +/// Outcome of a migrate-aof run. +#[derive(Debug)] +pub struct MigrateAofResult { + /// Total RESP commands read from the source AOF. + pub commands_read: usize, + /// Commands routed and written to a shard incr file. + pub commands_written: usize, + /// Commands that could not be routed (no key arg) and were skipped. + pub commands_skipped: usize, + /// Number of keys migrated from the source RDB base/preamble. + pub rdb_keys_migrated: usize, +} + +/// Migrate a legacy single-file AOF from `from_dir` into a PerShard layout at `to_dir`. +/// +/// `from_dir` may contain: +/// - `appendonly.aof` (flat RESP or RDB-preamble RESP) +/// - `appendonlydir/` with `moon.aof.{seq}.base.rdb` + `moon.aof.{seq}.incr.aof` +/// (TopLevel manifest format) +/// +/// `to_dir` must be empty or non-existent. The function creates the PerShard +/// directory layout under `to_dir/appendonlydir/`. +/// +/// Returns a summary of the migration or an error. +pub fn migrate_aof( + from_dir: &Path, + to_dir: &Path, + num_shards: u16, +) -> Result { + if num_shards == 0 { + return Err(crate::error::MoonError::from( + crate::error::AofError::RewriteFailed { + detail: "migrate_aof: num_shards must be >= 1".to_owned(), + }, + )); + } + + // ── Guard: from_dir == to_dir ──────────────────────────────────────────── + // Migrating into the same directory would clobber the source layout. + if from_dir == to_dir { + return Err(crate::error::MoonError::from( + crate::error::AofError::RewriteFailed { + detail: format!( + "migrate_aof: from_dir and to_dir must differ (both are {}). \ + Specify a separate empty directory for --migrate-aof-to.", + from_dir.display() + ), + }, + )); + } + + // ── Guard: to_dir must not already contain AOF data ───────────────────── + // A PerShard manifest in to_dir means a previous migration ran (or the + // operator is reusing a live data directory). Refuse to avoid partial + // overwrites — use a fresh --migrate-aof-to. + match AofManifest::load(to_dir) { + Ok(Some(_)) => { + return Err(crate::error::MoonError::from( + crate::error::AofError::RewriteFailed { + detail: format!( + "migrate_aof: to_dir ({}) already contains an AOF manifest. \ + Use a fresh, non-existent or empty directory for --migrate-aof-to.", + to_dir.display() + ), + }, + )); + } + Ok(None) => {} // expected: to_dir is empty or non-existent + Err(_) => {} // I/O errors from a non-existent dir are fine; proceed + } + + // ── Step 1: Load source data ───────────────────────────────────────────── + // Returns the RDB base bytes (if any) and the pure-RESP tail bytes. + let (rdb_base_bytes, resp_tail) = load_source(from_dir)?; + + // ── Step 2: Initialize target PerShard manifest ────────────────────────── + // This creates empty base RDB stubs and empty incr files for each shard. + AofManifest::initialize_multi(to_dir, num_shards).map_err(|e| { + crate::error::MoonError::from(crate::error::AofError::Io { + path: to_dir.to_path_buf(), + source: e, + }) + })?; + + let manifest = AofManifest::load(to_dir) + .map_err(|e| { + crate::error::MoonError::from(crate::error::AofError::Io { + path: to_dir.to_path_buf(), + source: e, + }) + })? + .ok_or_else(|| { + crate::error::MoonError::from(crate::error::AofError::RewriteFailed { + detail: "migrate_aof: manifest not found after initialize_multi".to_owned(), + }) + })?; + + // ── Step 3: Partition RDB preamble/base across shards ──────────────────── + let rdb_keys_migrated = if !rdb_base_bytes.is_empty() { + partition_rdb_into_shards(&rdb_base_bytes, &manifest, num_shards)? + } else { + 0 + }; + + // ── Step 4: Append RESP tail to per-shard incr files ──────────────────── + let (commands_read, commands_written, commands_skipped) = + append_resp_to_shards(&resp_tail, &manifest, num_shards)?; + + info!( + "migrate_aof complete: {} RDB keys + {} RESP commands written across {} shards ({} skipped)", + rdb_keys_migrated, commands_written, num_shards, commands_skipped + ); + + Ok(MigrateAofResult { + commands_read, + commands_written, + commands_skipped, + rdb_keys_migrated, + }) +} + +// ── Internal helpers ───────────────────────────────────────────────────────── + +/// Load the source data from `from_dir`. +/// +/// Returns `(rdb_base_bytes, resp_tail_bytes)`: +/// - `rdb_base_bytes`: raw RDB bytes for the snapshot (may be empty if no preamble) +/// - `resp_tail_bytes`: pure RESP command stream following the preamble +fn load_source(from_dir: &Path) -> Result<(Bytes, Bytes), crate::error::MoonError> { + // Option 1: flat appendonly.aof — may have RDB preamble + RESP tail. + let flat = from_dir.join("appendonly.aof"); + if flat.exists() { + info!("migrate_aof: reading from {}", flat.display()); + let raw = std::fs::read(&flat).map_err(|e| { + crate::error::MoonError::from(crate::error::AofError::Io { + path: flat.clone(), + source: e, + }) + })?; + return Ok(split_rdb_preamble(raw.into())); + } + + // Option 2: TopLevel manifest — separate base.rdb + incr.aof. + if let Ok(Some(m)) = AofManifest::load(from_dir) { + let base_path = m.shard_base_path_seq(0, m.seq); + let incr_path = m.incr_path(); + + let rdb_bytes = if base_path.exists() { + info!("migrate_aof: reading base RDB from {}", base_path.display()); + std::fs::read(&base_path).map(Bytes::from).map_err(|e| { + crate::error::MoonError::from(crate::error::AofError::Io { + path: base_path.clone(), + source: e, + }) + })? + } else { + Bytes::new() + }; + + let resp_bytes = if incr_path.exists() { + info!("migrate_aof: reading incr from {}", incr_path.display()); + std::fs::read(&incr_path).map(Bytes::from).map_err(|e| { + crate::error::MoonError::from(crate::error::AofError::Io { + path: incr_path.clone(), + source: e, + }) + })? + } else { + Bytes::new() + }; + + return Ok((rdb_bytes, resp_bytes)); + } + + Err(crate::error::MoonError::from( + crate::error::AofError::RewriteFailed { + detail: format!( + "migrate_aof: no AOF source found in {}. \ + Expected appendonly.aof or appendonlydir/moon.aof.*.(base|incr).aof", + from_dir.display() + ), + }, + )) +} + +/// Split AOF bytes into `(rdb_preamble_bytes, resp_tail_bytes)`. +/// +/// If the file starts with `MOON` or `REDIS` magic, the RDB preamble is +/// split off — the raw preamble bytes (header through CRC32) are returned +/// as-is so `rdb::load_from_bytes` can parse them. If no preamble is +/// present, the first element is empty and the second is the entire input. +fn split_rdb_preamble(bytes: Bytes) -> (Bytes, Bytes) { + const MOON_MAGIC: &[u8] = b"MOON"; + const REDIS_MAGIC: &[u8] = b"REDIS"; + const EOF_MARKER: u8 = 0xFF; + + if !bytes.starts_with(MOON_MAGIC) && !bytes.starts_with(REDIS_MAGIC) { + // Pure RESP — no preamble. + return (Bytes::new(), bytes); + } + + // Locate the RDB end: scan for EOF_MARKER (0xFF) followed by 4 CRC bytes + // whose CRC32 of bytes[0..=i] matches. Use the same single-pass approach + // as `rdb::load_from_bytes`. + use crc32fast::Hasher; + let mut running_hasher = Hasher::new(); + if bytes.len() > 5 { + running_hasher.update(&bytes[..5]); + } + for i in 5..bytes.len().saturating_sub(4) { + running_hasher.update(&bytes[i..i + 1]); + if bytes[i] == EOF_MARKER { + // Candidate EOF: check CRC of bytes[0..=i] matches bytes[i+1..i+5] + let stored = + u32::from_le_bytes([bytes[i + 1], bytes[i + 2], bytes[i + 3], bytes[i + 4]]); + // Clone the hasher to avoid consuming state (running_hasher must + // continue in case this candidate is a false positive). + let check = running_hasher.clone(); + if check.finalize() == stored { + let rdb_end = i + 1 + 4; // past EOF marker + CRC32 + info!( + "migrate_aof: found RDB preamble of {} bytes at offset 0", + rdb_end + ); + return (bytes.slice(..rdb_end), bytes.slice(rdb_end..)); + } + } + } + + // No valid EOF marker found — treat as pure RESP (or empty preamble). + warn!("migrate_aof: no valid RDB EOF marker found; treating entire file as RESP"); + (Bytes::new(), bytes) +} + +/// Load `rdb_bytes` into scratch databases, then partition each key into a +/// per-shard scratch database, and write per-shard base RDB files at +/// `manifest.shard_base_path(sid)`. +/// +/// Returns the total number of keys partitioned. +fn partition_rdb_into_shards( + rdb_bytes: &[u8], + manifest: &AofManifest, + num_shards: u16, +) -> Result { + // Allocate scratch databases — one Database per logical db index (16 max). + const MAX_DBS: usize = 16; + let mut scratch: Vec = (0..MAX_DBS).map(|_| Database::new()).collect(); + + // Load the RDB into scratch databases. + let (keys_loaded, _consumed) = + crate::persistence::rdb::load_from_bytes(&mut scratch, rdb_bytes)?; + + if keys_loaded == 0 { + info!("migrate_aof: RDB preamble/base contains 0 live keys; skipping partitioning"); + return Ok(0); + } + + info!( + "migrate_aof: loaded {} keys from RDB; partitioning across {} shards", + keys_loaded, num_shards + ); + + // Allocate per-shard scratch databases (same logical DB count as source). + let mut shard_dbs: Vec> = (0..num_shards) + .map(|_| (0..MAX_DBS).map(|_| Database::new()).collect()) + .collect(); + + // Partition keys from scratch into shard_dbs. + let mut total_partitioned: usize = 0; + for (db_idx, db) in scratch.iter().enumerate() { + for (key, entry) in db.data().iter() { + let key_bytes = key.as_bytes(); + let shard_idx = crate::shard::dispatch::key_to_shard(key_bytes, num_shards as usize); + shard_dbs[shard_idx][db_idx].set(Bytes::copy_from_slice(key_bytes), entry.clone()); + total_partitioned += 1; + } + } + + // Write per-shard base RDB files (overwrite the empty placeholders created + // by initialize_multi). + for sid in 0..num_shards { + let base_path = manifest.shard_base_path(sid); + let rdb = crate::persistence::rdb::save_to_bytes(&shard_dbs[sid as usize])?; + let tmp = base_path.with_extension("rdb.tmp"); + { + let mut f = std::fs::File::create(&tmp).map_err(|e| { + crate::error::MoonError::from(crate::error::AofError::Io { + path: tmp.clone(), + source: e, + }) + })?; + f.write_all(&rdb).map_err(|e| { + crate::error::MoonError::from(crate::error::AofError::Io { + path: tmp.clone(), + source: e, + }) + })?; + f.sync_data().map_err(|e| { + crate::error::MoonError::from(crate::error::AofError::Io { + path: tmp.clone(), + source: e, + }) + })?; + } + std::fs::rename(&tmp, &base_path).map_err(|e| { + crate::error::MoonError::from(crate::error::AofError::Io { + path: base_path.clone(), + source: e, + }) + })?; + info!( + "migrate_aof: shard-{} base RDB written ({} bytes)", + sid, + rdb.len() + ); + } + + Ok(total_partitioned) +} + +/// Parse the RESP tail and append framed entries to each shard's incr file. +/// +/// Returns `(commands_read, commands_written, commands_skipped)`. +/// Stops at the first parse error (crash-time EOF semantics). +fn append_resp_to_shards( + resp_bytes: &[u8], + manifest: &AofManifest, + num_shards: u16, +) -> Result<(usize, usize, usize), crate::error::MoonError> { + if resp_bytes.is_empty() { + return Ok((0, 0, 0)); + } + + // Open per-shard incr files for appending. + let mut shard_files: Vec = (0..num_shards) + .map(|sid| { + std::fs::OpenOptions::new() + .append(true) + .open(manifest.shard_incr_path(sid)) + .map_err(|e| { + crate::error::MoonError::from(crate::error::AofError::Io { + path: manifest.shard_incr_path(sid), + source: e, + }) + }) + }) + .collect::>()?; + + let mut shard_lsn: Vec = vec![0; num_shards as usize]; + let config = ParseConfig::default(); + let mut buf = BytesMut::from(resp_bytes); + let mut commands_read: usize = 0; + let mut commands_written: usize = 0; + let mut commands_skipped: usize = 0; + + loop { + if buf.is_empty() { + break; + } + match parse::parse(&mut buf, &config) { + Ok(Some(frame)) => { + commands_read += 1; + let arr = match frame { + Frame::Array(ref arr) if !arr.is_empty() => arr, + _ => { + warn!( + "migrate_aof: non-array frame at command {}; skipping", + commands_read + ); + commands_skipped += 1; + continue; + } + }; + + // Extract command name for routing decisions. + let cmd_name = match &arr[0] { + Frame::BulkString(s) => s.clone(), + Frame::SimpleString(s) => s.clone(), + _ => { + warn!( + "migrate_aof: non-string command name at command {}; skipping", + commands_read + ); + commands_skipped += 1; + continue; + } + }; + + // SELECT: allow only SELECT 0 (no-op); refuse SELECT N (N>0). + // Per-shard replay runs each shard independently and does not + // persist the logical database across commands. A multi-DB + // legacy AOF (SELECT 1 + commands in db 1) cannot be safely + // migrated — commands after SELECT N would land in db 0 of + // their elected shard, silently corrupting data. + let cmd_upper = cmd_name.to_ascii_uppercase(); + if cmd_upper.as_slice() == b"SELECT" { + // Parse the db argument (if present). + let db_arg = arr.get(1).and_then(|f| match f { + Frame::BulkString(b) => std::str::from_utf8(b.as_ref()).ok(), + Frame::SimpleString(s) => std::str::from_utf8(s.as_ref()).ok(), + _ => None, + }); + let db_num: i64 = db_arg.and_then(|s| s.trim().parse().ok()).unwrap_or(0); + if db_num != 0 { + return Err(crate::error::MoonError::from( + crate::error::AofError::RewriteFailed { + detail: format!( + "migrate_aof: multi-DB legacy AOF detected (SELECT {db_num} \ + at command {commands_read}). Per-shard replay cannot \ + correctly route commands from non-default databases. \ + Manually re-route or flush the non-default database \ + before migrating. Use a fresh --migrate-aof-to directory \ + after fixing the source AOF." + ), + }, + )); + } + // SELECT 0 is a no-op — skip it silently. + commands_skipped += 1; + continue; + } + + // Route to shard: keyless commands go to shard 0. + let shard_idx = if arr.len() < 2 { + if matches!(cmd_upper.as_slice(), b"FLUSHALL" | b"FLUSHDB") { + warn!( + "migrate_aof: {} at command {} affects all shards but will only be \ + written to shard 0; verify correctness after migration", + String::from_utf8_lossy(&cmd_upper), + commands_read + ); + } + 0usize + } else { + match &arr[1] { + Frame::BulkString(k) => { + crate::shard::dispatch::key_to_shard(k.as_ref(), num_shards as usize) + } + Frame::SimpleString(k) => { + crate::shard::dispatch::key_to_shard(k.as_ref(), num_shards as usize) + } + _ => { + warn!( + "migrate_aof: non-string key at command {}; skipping", + commands_read + ); + commands_skipped += 1; + continue; + } + } + }; + + // Serialize the original RESP frame for the framed incr entry. + let mut resp_buf = BytesMut::new(); + crate::protocol::serialize::serialize(&frame, &mut resp_buf); + let resp_bytes_out: Bytes = resp_buf.freeze(); + + // Write framed entry: [u64 lsn LE][u32 len LE][RESP bytes]. + shard_lsn[shard_idx] += 1; + let lsn = shard_lsn[shard_idx]; + let file = &mut shard_files[shard_idx]; + write_framed( + file, + lsn, + &resp_bytes_out, + manifest.shard_incr_path(shard_idx as u16), + )?; + commands_written += 1; + } + Ok(None) => { + // Incomplete frame at end — crash-time EOF. + if !buf.is_empty() { + warn!( + "migrate_aof: truncated RESP tail ({} bytes) at command {}; \ + treating as crash-time EOF", + buf.len(), + commands_read + ); + } + break; + } + Err(e) => { + warn!( + "migrate_aof: parse error at command {}: {:?}; stopping migration \ + (remaining {} bytes discarded — treat as crash-time EOF)", + commands_read, + e, + buf.len() + ); + break; + } + } + } + + // Fsync all shard incr files. + for (sid, file) in shard_files.iter().enumerate() { + file.sync_data().map_err(|e| { + crate::error::MoonError::from(crate::error::AofError::Io { + path: manifest.shard_incr_path(sid as u16), + source: std::io::Error::new(e.kind(), format!("fsync shard-{sid}: {e}")), + }) + })?; + } + + Ok((commands_read, commands_written, commands_skipped)) +} + +/// Write one framed entry `[u64 lsn LE][u32 len LE][RESP bytes]` to `file`. +fn write_framed( + file: &mut std::fs::File, + lsn: u64, + resp: &[u8], + path: PathBuf, +) -> Result<(), crate::error::MoonError> { + let len = resp.len() as u32; + file.write_all(&lsn.to_le_bytes()).map_err(|e| { + crate::error::MoonError::from(crate::error::AofError::Io { + path: path.clone(), + source: e, + }) + })?; + file.write_all(&len.to_le_bytes()).map_err(|e| { + crate::error::MoonError::from(crate::error::AofError::Io { + path: path.clone(), + source: e, + }) + })?; + file.write_all(resp).map_err(|e| { + crate::error::MoonError::from(crate::error::AofError::Io { + path: path.clone(), + source: e, + }) + })?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use bytes::BytesMut; + + /// Helper: serialize a RESP array command. + fn cmd_resp(parts: &[&str]) -> Vec { + let mut buf = BytesMut::new(); + let frames: Vec = parts + .iter() + .map(|s| Frame::BulkString(Bytes::copy_from_slice(s.as_bytes()))) + .collect(); + let frame = Frame::Array(frames.into()); + crate::protocol::serialize::serialize(&frame, &mut buf); + buf.to_vec() + } + + // ── FIX-W3-2 red tests ────────────────────────────────────────────────── + + /// SELECT N (N>0) in a RESP tail must cause migrate_aof to return Err + /// with a message mentioning "multi-DB". On current HEAD this test FAILS + /// because SELECT is silently dropped (skipped) and Ok is returned. + #[test] + fn migrate_aof_select_nonzero_db_returns_err() { + let src_dir = tempfile::tempdir().unwrap(); + let dst_dir = tempfile::tempdir().unwrap(); + + // Build a RESP tail: SET a value, SELECT 1, SET b value. + let mut aof_data: Vec = Vec::new(); + aof_data.extend(cmd_resp(&["SET", "a", "v"])); + aof_data.extend(cmd_resp(&["SELECT", "1"])); + aof_data.extend(cmd_resp(&["SET", "b", "v"])); + std::fs::write(src_dir.path().join("appendonly.aof"), &aof_data).expect("write source aof"); + + let result = migrate_aof(src_dir.path(), dst_dir.path(), 2); + assert!( + result.is_err(), + "migrate_aof must refuse when RESP tail contains SELECT N (N>0)" + ); + let msg = format!("{}", result.unwrap_err()); + assert!( + msg.contains("multi-DB") || msg.contains("SELECT"), + "error message must mention multi-DB or SELECT, got: {msg}" + ); + } + + /// migrate_aof(same_dir, same_dir, n) must return Err with a guard message. + /// Without a guard, this may succeed (incidentally error on manifest load) + /// but would not carry a meaningful "same directory" message. + #[test] + fn migrate_aof_same_dir_returns_err() { + let dir = tempfile::tempdir().unwrap(); + std::fs::write(dir.path().join("appendonly.aof"), b"").unwrap(); + + let result = migrate_aof(dir.path(), dir.path(), 2); + assert!( + result.is_err(), + "migrate_aof must refuse when from_dir == to_dir" + ); + let msg = format!("{}", result.unwrap_err()); + assert!( + msg.contains("same") || msg.contains("from_dir") || msg.contains("to_dir"), + "error must identify the same-directory problem, got: {msg}" + ); + } + + /// migrate_aof into a to_dir that already contains a PerShard manifest + /// must return Err. Without the guard, initialize_multi may partially + /// overwrite or the run silently proceeds. + #[test] + fn migrate_aof_existing_manifest_returns_err() { + let src_dir = tempfile::tempdir().unwrap(); + let dst_dir = tempfile::tempdir().unwrap(); + std::fs::write(src_dir.path().join("appendonly.aof"), b"").unwrap(); + + // Pre-populate to_dir with a PerShard manifest. + AofManifest::initialize_multi(dst_dir.path(), 2).expect("first initialize_multi succeeds"); + + let result = migrate_aof(src_dir.path(), dst_dir.path(), 2); + assert!( + result.is_err(), + "migrate_aof must refuse when to_dir already contains AOF data" + ); + let msg = format!("{}", result.unwrap_err()); + assert!( + msg.contains("already") || msg.contains("exist") || msg.contains("non-empty"), + "error must identify the pre-existing data problem, got: {msg}" + ); + } + + /// Helper: serialize a SET command to RESP. + fn set_resp(key: &str, val: &str) -> Vec { + let mut buf = BytesMut::new(); + let frame = Frame::Array( + vec![ + Frame::BulkString(Bytes::copy_from_slice(b"SET")), + Frame::BulkString(Bytes::copy_from_slice(key.as_bytes())), + Frame::BulkString(Bytes::copy_from_slice(val.as_bytes())), + ] + .into(), + ); + crate::protocol::serialize::serialize(&frame, &mut buf); + buf.to_vec() + } + + #[test] + fn migrate_aof_routes_keys_to_correct_shards() { + let src_dir = tempfile::tempdir().unwrap(); + let dst_dir = tempfile::tempdir().unwrap(); + + let mut aof_data: Vec = Vec::new(); + for i in 0..10u32 { + aof_data.extend(set_resp(&format!("key{i}"), "value")); + } + + std::fs::write(src_dir.path().join("appendonly.aof"), &aof_data).expect("write source aof"); + + let result = migrate_aof(src_dir.path(), dst_dir.path(), 4).expect("migration succeeds"); + + assert_eq!(result.commands_read, 10, "all 10 SETs read"); + assert_eq!(result.commands_written, 10, "all 10 SETs written"); + assert_eq!(result.commands_skipped, 0, "no skips"); + assert_eq!(result.rdb_keys_migrated, 0, "no RDB preamble"); + + let manifest = AofManifest::load(dst_dir.path()) + .expect("load manifest") + .expect("manifest present"); + assert_eq!( + manifest.layout, + crate::persistence::aof_manifest::AofLayout::PerShard + ); + assert_eq!(manifest.shards.len(), 4, "4 shards in manifest"); + + let total_incr_bytes: u64 = (0..4u16) + .map(|sid| { + std::fs::metadata(manifest.shard_incr_path(sid)) + .map(|m| m.len()) + .unwrap_or(0) + }) + .sum(); + assert!( + total_incr_bytes > 0, + "at least some bytes written to shard incr files" + ); + } + + #[test] + fn migrate_aof_hash_tag_routing_deterministic() { + let src_dir = tempfile::tempdir().unwrap(); + let dst_dir = tempfile::tempdir().unwrap(); + + let mut aof_data: Vec = Vec::new(); + for i in 0..4u32 { + aof_data.extend(set_resp(&format!("{{0}}:key{i}"), "value")); + } + std::fs::write(src_dir.path().join("appendonly.aof"), &aof_data).expect("write source aof"); + + let result = migrate_aof(src_dir.path(), dst_dir.path(), 4).expect("migration succeeds"); + assert_eq!(result.commands_written, 4); + + let manifest = AofManifest::load(dst_dir.path()) + .expect("load manifest") + .expect("present"); + let expected_shard = crate::shard::dispatch::key_to_shard(b"{0}:key0", 4) as u16; + let shard_incr_len = std::fs::metadata(manifest.shard_incr_path(expected_shard)) + .map(|m| m.len()) + .unwrap_or(0); + for sid in 0..4u16 { + if sid == expected_shard { + assert!(shard_incr_len > 0, "target shard has data"); + } else { + let len = std::fs::metadata(manifest.shard_incr_path(sid)) + .map(|m| m.len()) + .unwrap_or(0); + assert_eq!(len, 0, "non-target shard {} must be empty", sid); + } + } + } + + #[test] + fn migrate_aof_empty_source_produces_valid_manifest() { + let src_dir = tempfile::tempdir().unwrap(); + let dst_dir = tempfile::tempdir().unwrap(); + + std::fs::write(src_dir.path().join("appendonly.aof"), b"").expect("write empty source aof"); + + let result = migrate_aof(src_dir.path(), dst_dir.path(), 2) + .expect("migration of empty aof succeeds"); + assert_eq!(result.commands_read, 0); + assert_eq!(result.commands_written, 0); + assert_eq!(result.rdb_keys_migrated, 0); + + let manifest = AofManifest::load(dst_dir.path()) + .expect("load manifest") + .expect("present"); + assert_eq!(manifest.shards.len(), 2); + } + + /// Round-trip test (RESP-only source): migrate N keys via RESP tail, then + /// call replay_per_shard and assert every key is recoverable. + #[test] + fn migrate_aof_round_trips_through_replay_per_shard() { + use crate::persistence::aof_manifest::replay_per_shard; + use crate::persistence::replay::DispatchReplayEngine; + + let src_dir = tempfile::tempdir().unwrap(); + let dst_dir = tempfile::tempdir().unwrap(); + const N_SHARDS: u16 = 4; + + let mut aof_data: Vec = Vec::new(); + let keys: Vec = (0..12u32).map(|i| format!("key:{i}")).collect(); + for key in &keys { + aof_data.extend(set_resp(key, "val")); + } + std::fs::write(src_dir.path().join("appendonly.aof"), &aof_data).expect("write source aof"); + + let result = + migrate_aof(src_dir.path(), dst_dir.path(), N_SHARDS).expect("migration succeeds"); + assert_eq!(result.commands_read, 12); + assert_eq!(result.commands_written, 12); + + let manifest = AofManifest::load(dst_dir.path()) + .expect("load ok") + .expect("manifest present"); + + let mut shard_dbs: Vec> = + (0..N_SHARDS).map(|_| vec![Database::new()]).collect(); + let mut slices: Vec<&mut [Database]> = + shard_dbs.iter_mut().map(|v| v.as_mut_slice()).collect(); + + let (total_replayed, _max_lsn, ordered) = replay_per_shard( + &mut slices, + &manifest, + &(|| { + Box::new(DispatchReplayEngine::new()) + as Box + }), + ) + .expect("replay_per_shard must succeed on migrated layout"); + + assert_eq!( + total_replayed, 12, + "all 12 commands must be recovered by replay_per_shard" + ); + assert!( + ordered.is_empty(), + "non-ordered commands must not appear in ordered buffer" + ); + + let mut total_found = 0usize; + for key in &keys { + let shard_idx = crate::shard::dispatch::key_to_shard(key.as_bytes(), N_SHARDS as usize); + let db = &mut shard_dbs[shard_idx][0]; + if db.get(key.as_bytes()).is_some() { + total_found += 1; + } + } + assert_eq!( + total_found, 12, + "all 12 keys must be retrievable from their routing shard after replay" + ); + } + + /// Round-trip test (RDB preamble source): migrate keys stored in an RDB + /// preamble (simulating a BGREWRITEAOF'd AOF), then call replay_per_shard + /// and assert all keys survive. + /// + /// This is the critical test the advisor identified: without preamble + /// partitioning, this test would lose all keys (reporting 0 recovered). + #[test] + fn migrate_aof_round_trips_rdb_preamble_through_replay_per_shard() { + use crate::persistence::aof_manifest::replay_per_shard; + use crate::persistence::replay::DispatchReplayEngine; + use crate::storage::entry::Entry; + + let src_dir = tempfile::tempdir().unwrap(); + let dst_dir = tempfile::tempdir().unwrap(); + const N_SHARDS: u16 = 4; + const N_KEYS: usize = 20; + + // Build a source database with N_KEYS string entries. + let mut source_db: Vec = vec![Database::new()]; + let keys: Vec = (0..N_KEYS).map(|i| format!("rdb_key:{i}")).collect(); + for key in &keys { + let entry = Entry::new_string(Bytes::copy_from_slice(format!("val_{key}").as_bytes())); + source_db[0].set(Bytes::copy_from_slice(key.as_bytes()), entry); + } + + // Serialize to RDB preamble bytes, then write as appendonly.aof. + let rdb_bytes = + crate::persistence::rdb::save_to_bytes(&source_db).expect("RDB serialize succeeds"); + // No RESP tail — this simulates a fully-compacted AOF. + std::fs::write(src_dir.path().join("appendonly.aof"), &rdb_bytes) + .expect("write source aof with RDB preamble"); + + let result = + migrate_aof(src_dir.path(), dst_dir.path(), N_SHARDS).expect("migration succeeds"); + assert_eq!( + result.rdb_keys_migrated, N_KEYS, + "all {N_KEYS} RDB keys must be partitioned" + ); + assert_eq!(result.commands_written, 0, "no RESP tail commands"); + + let manifest = AofManifest::load(dst_dir.path()) + .expect("load ok") + .expect("manifest present"); + + let mut shard_dbs: Vec> = + (0..N_SHARDS).map(|_| vec![Database::new()]).collect(); + let mut slices: Vec<&mut [Database]> = + shard_dbs.iter_mut().map(|v| v.as_mut_slice()).collect(); + + let (total_replayed, _max_lsn, ordered) = replay_per_shard( + &mut slices, + &manifest, + &(|| { + Box::new(DispatchReplayEngine::new()) + as Box + }), + ) + .expect("replay_per_shard must succeed on migrated layout"); + + // replay_per_shard counts RDB keys loaded from the base RDB. + assert_eq!( + total_replayed, N_KEYS, + "all {N_KEYS} RDB-preamble keys must be recovered after migration" + ); + assert!(ordered.is_empty()); + + // Verify each key is in the correct shard. + let mut total_found = 0usize; + for key in &keys { + let shard_idx = crate::shard::dispatch::key_to_shard(key.as_bytes(), N_SHARDS as usize); + let db = &mut shard_dbs[shard_idx][0]; + if db.get(key.as_bytes()).is_some() { + total_found += 1; + } + } + assert_eq!( + total_found, N_KEYS, + "all {N_KEYS} keys must be retrievable from correct shard after RDB preamble migration" + ); + } +} diff --git a/src/persistence/mod.rs b/src/persistence/mod.rs index 5c51d4a5..cdbcc326 100644 --- a/src/persistence/mod.rs +++ b/src/persistence/mod.rs @@ -8,6 +8,7 @@ pub mod control; pub mod fsync; pub mod kv_page; pub mod manifest; +pub mod migrate_aof; pub mod page; pub mod page_cache; pub mod rdb; diff --git a/src/replication/state.rs b/src/replication/state.rs index e5e357c0..a9b57317 100644 --- a/src/replication/state.rs +++ b/src/replication/state.rs @@ -113,10 +113,34 @@ impl ReplicationState { /// Increment the offset for the given shard by delta bytes. /// Also adds delta to master_repl_offset. pub fn increment_shard_offset(&self, shard_id: usize, delta: u64) { - if shard_id < self.shard_offsets.len() { - self.shard_offsets[shard_id].fetch_add(delta, Ordering::Relaxed); - self.master_repl_offset.fetch_add(delta, Ordering::Relaxed); + let _ = self.issue_lsn(shard_id, delta); + } + + /// Atomically issue an LSN for a write and advance per-shard + + /// master replication offsets by `delta`. + /// + /// Returns the LSN that uniquely identifies this write — equal to the + /// value of `master_repl_offset` BEFORE the increment, mirroring Redis's + /// `+ delta - delta` semantics. The same LSN MUST tag the corresponding + /// `AofMessage::Append` entry and the replication backlog entry for that + /// write so per-shard AOF replay can rebuild a globally consistent log + /// (per-shard AOF RFC § 2 Rule 3). + /// + /// Atomicity caveat: the per-shard offset advance and the master offset + /// advance are TWO separate `fetch_add`s, not one composite op. Concurrent + /// callers across shards observe a brief window where the master sum + /// disagrees with the sum of shard offsets. Acceptable today because the + /// only `total_offset()` consumer is INFO replication, which tolerates + /// transient skew. Do not promote to a hard invariant without redesign. + /// + /// Returns 0 if `shard_id` is out of range (defensive; production callers + /// must pass a valid id). + pub fn issue_lsn(&self, shard_id: usize, delta: u64) -> u64 { + if shard_id >= self.shard_offsets.len() { + return 0; } + self.shard_offsets[shard_id].fetch_add(delta, Ordering::Relaxed); + self.master_repl_offset.fetch_add(delta, Ordering::Relaxed) } /// Returns sum of all per-shard offsets. @@ -124,6 +148,23 @@ impl ReplicationState { self.master_repl_offset.load(Ordering::Relaxed) } + /// Seed `master_repl_offset` to at least `lsn` after AOF recovery. + /// + /// Per-shard AOF RFC § 2 Rule 3: after recovery reads the per-shard AOFs, + /// `master_repl_offset` MUST be at least the max LSN observed across all + /// shards before the server accepts client traffic. Otherwise the next + /// write would issue an LSN already present on disk, breaking the + /// `lsn → entry` uniqueness invariant the backlog merge depends on. + /// + /// Uses `fetch_max` so a concurrent in-flight increment (extremely + /// unlikely at boot, but free to guard against) cannot regress the value. + /// Per-shard offsets are intentionally NOT touched here — at boot they + /// are still 0, and seeding shard offsets to the per-shard AOF max would + /// double-count once the first write advances them via `issue_lsn`. + pub fn seed_master_offset(&self, lsn: u64) { + self.master_repl_offset.fetch_max(lsn, Ordering::Relaxed); + } + /// Returns the per-shard offset for a specific shard. pub fn shard_offset(&self, shard_id: usize) -> u64 { self.shard_offsets diff --git a/src/server/conn/blocking.rs b/src/server/conn/blocking.rs index 8846d0c8..0702683d 100644 --- a/src/server/conn/blocking.rs +++ b/src/server/conn/blocking.rs @@ -1135,7 +1135,10 @@ pub(crate) fn try_inline_dispatch( shard_databases: &std::sync::Arc, shard_id: usize, selected_db: usize, - aof_tx: &Option>, + aof_pool: &Option>, + repl_state: &Option< + std::sync::Arc>, + >, now_ms: u64, num_shards: usize, can_inline_writes: bool, @@ -1149,6 +1152,20 @@ pub(crate) fn try_inline_dispatch( return 0; } + // H1: under `appendfsync=always` we MUST fsync before +OK. The inline + // SET path is synchronous and cannot await the writer's ack, so + // refuse to inline writes when Always is in effect. GETs and other + // read-only commands are fine to inline. The non-inline dispatch + // path (handler_monoio/handler_sharded) uses + // `AofWriterPool::try_send_append_durable` and awaits the ack. + if let Some(pool) = aof_pool { + if pool.fsync_policy() == crate::persistence::aof::FsyncPolicy::Always && buf[1] == b'3' + // SET shape (*3 ...); GETs (*2) are still safe to inline. + { + return 0; + } + } + // Parse array count: only *2 (GET) and *3 (SET plain) are inlined. let argc = buf[1]; if buf[2] != b'\r' || buf[3] != b'\n' { @@ -1346,8 +1363,20 @@ pub(crate) fn try_inline_dispatch( } // AOF: reuse the frozen RESP bytes directly (Arc clone, zero-copy). - if let Some(tx) = aof_tx { - let _ = tx.try_send(crate::persistence::aof::AofMessage::Append(frozen)); + // This path is monoio inline GET/SET — the writer for the local shard + // (shard_id) owns the AOF record; under PerShard layout that routes + // to shard_id's writer. LSN must be sourced from `repl_state` so the + // inline path's writes share an LSN namespace with the non-inline path + // — otherwise per-shard replay merge in RFC step 5 would see two + // disjoint LSN streams per shard. Cost: one extra read-lock acquire + // (uncontended) + one atomic fetch_add per inline SET. + if let Some(pool) = aof_pool { + let lsn = crate::persistence::aof::AofWriterPool::issue_append_lsn( + repl_state, + shard_id, + frozen.len(), + ); + pool.try_send_append(shard_id, lsn, frozen); } write_buf.extend_from_slice(b"+OK\r\n"); @@ -1363,7 +1392,10 @@ pub(crate) fn try_inline_dispatch_loop( shard_databases: &std::sync::Arc, shard_id: usize, selected_db: usize, - aof_tx: &Option>, + aof_pool: &Option>, + repl_state: &Option< + std::sync::Arc>, + >, now_ms: u64, num_shards: usize, can_inline_writes: bool, @@ -1377,7 +1409,8 @@ pub(crate) fn try_inline_dispatch_loop( shard_databases, shard_id, selected_db, - aof_tx, + aof_pool, + repl_state, now_ms, num_shards, can_inline_writes, diff --git a/src/server/conn/core.rs b/src/server/conn/core.rs index 43cef452..dc2e2365 100644 --- a/src/server/conn/core.rs +++ b/src/server/conn/core.rs @@ -20,7 +20,7 @@ use std::sync::Arc; use crate::acl::{AclLog, AclTable}; use crate::blocking::BlockingRegistry; use crate::config::{RuntimeConfig, ServerConfig}; -use crate::persistence::aof::AofMessage; +use crate::persistence::aof::AofWriterPool; use crate::protocol::Frame; use crate::pubsub::PubSubRegistry; use crate::runtime::channel; @@ -47,7 +47,13 @@ pub(crate) struct ConnectionContext { pub pubsub_registry: Arc>, pub blocking_registry: Rc>, pub requirepass: Option, - pub aof_tx: Option>, + /// AOF writer pool — the **sole AOF interface** after the 2d/2e migration + /// sequence. Built by spawn sites in `shard/conn_accept.rs` from the + /// on-disk manifest layout: TopLevel wraps a single shared writer, + /// PerShard owns one sender per shard. `try_send_append(shard_id, bytes)` + /// routes to the owning shard; `try_send_rewrite(msg)` rejects under + /// PerShard until per-shard rewrite ships (step 6 of the RFC). + pub aof_pool: Option>, pub tracking_table: Rc>, pub repl_state: Option>>, /// Lock-free mirror of `repl_state.role == Replica { .. }`. @@ -96,7 +102,7 @@ impl ConnectionContext { pubsub_registry: Arc>, blocking_registry: Rc>, requirepass: Option, - aof_tx: Option>, + aof_pool: Option>, tracking_table: Rc>, repl_state: Option>>, cluster_state: Option>>, @@ -136,7 +142,7 @@ impl ConnectionContext { pubsub_registry, blocking_registry, requirepass, - aof_tx, + aof_pool, tracking_table, repl_state, is_replica_mirror, diff --git a/src/server/conn/handler_monoio/dispatch.rs b/src/server/conn/handler_monoio/dispatch.rs index 7182cd68..b4e5485f 100644 --- a/src/server/conn/handler_monoio/dispatch.rs +++ b/src/server/conn/handler_monoio/dispatch.rs @@ -978,9 +978,9 @@ pub(super) fn try_handle_persistence( return true; } if cmd.eq_ignore_ascii_case(b"BGREWRITEAOF") { - if let Some(ref tx) = ctx.aof_tx { + if let Some(ref pool) = ctx.aof_pool { responses.push(crate::command::persistence::bgrewriteaof_start_sharded( - tx, + pool, ctx.shard_databases.clone(), )); } else { diff --git a/src/server/conn/handler_monoio/mod.rs b/src/server/conn/handler_monoio/mod.rs index 72eb1413..a04173be 100644 --- a/src/server/conn/handler_monoio/mod.rs +++ b/src/server/conn/handler_monoio/mod.rs @@ -20,7 +20,7 @@ use std::rc::Rc; use crate::command::metadata; use crate::command::{DispatchResult, dispatch, dispatch_read, is_dispatch_read_supported}; -use crate::persistence::aof::{self, AofMessage}; +use crate::persistence::aof; use crate::protocol::Frame; use crate::shard::dispatch::key_to_shard; use crate::shard::mesh::ChannelMesh; @@ -483,7 +483,8 @@ pub(crate) async fn handle_connection_sharded_monoio< &ctx.shard_databases, ctx.shard_id, conn.selected_db, - &ctx.aof_tx, + &ctx.aof_pool, + &ctx.repl_state, ctx.cached_clock.ms(), ctx.num_shards, can_inline_writes, @@ -1066,7 +1067,7 @@ pub(crate) async fn handle_connection_sharded_monoio< } // Pre-classify write commands for AOF + tracking - let is_write = if ctx.aof_tx.is_some() || conn.tracking_state.enabled { + let is_write = if ctx.aof_pool.is_some() || conn.tracking_state.enabled { metadata::is_write(cmd) } else { false @@ -1120,10 +1121,27 @@ pub(crate) async fn handle_connection_sharded_monoio< } }; // AOF only on actual success (:1). Matches handler_single. + // H1 fix: durable path under `appendfsync=always` + // awaits the writer's fsync ack before responding to + // the client. if matches!(response, Frame::Integer(1)) { - if let Some(ref tx) = ctx.aof_tx { + if let Some(ref pool) = ctx.aof_pool { let serialized = aof::serialize_command(&frame); - let _ = tx.try_send(AofMessage::Append(serialized)); + let lsn = aof::AofWriterPool::issue_append_lsn( + &ctx.repl_state, + ctx.shard_id, + serialized.len(), + ); + if pool + .try_send_append_durable(ctx.shard_id, lsn, serialized) + .await + .is_err() + { + responses.push(Frame::Error(bytes::Bytes::from_static( + aof::AOF_FSYNC_ERR, + ))); + continue; + } } } responses.push(response); @@ -1185,10 +1203,25 @@ pub(crate) async fn handle_connection_sharded_monoio< }; // AOF only on actual success (:1). Matches handler_single // — `:0` (key absent / dst exists w/o REPLACE) is a no-op. + // H1: durable path awaits fsync under appendfsync=always. if matches!(response, Frame::Integer(1)) { - if let Some(ref tx) = ctx.aof_tx { + if let Some(ref pool) = ctx.aof_pool { let serialized = aof::serialize_command(&frame); - let _ = tx.try_send(AofMessage::Append(serialized)); + let lsn = aof::AofWriterPool::issue_append_lsn( + &ctx.repl_state, + ctx.shard_id, + serialized.len(), + ); + if pool + .try_send_append_durable(ctx.shard_id, lsn, serialized) + .await + .is_err() + { + responses.push(Frame::Error(bytes::Bytes::from_static( + aof::AOF_FSYNC_ERR, + ))); + continue; + } } } responses.push(response); @@ -1499,7 +1532,7 @@ pub(crate) async fn handle_connection_sharded_monoio< let sample_latency = (conn.cmd_counter & 0xF) == 0; let dispatch_start = sample_latency.then(std::time::Instant::now); - let response = match result { + let mut response = match result { DispatchResult::Response(f) => f, DispatchResult::Quit(f) => { should_quit = true; @@ -1533,13 +1566,38 @@ pub(crate) async fn handle_connection_sharded_monoio< } } - // AOF logging for successful local writes + // AOF logging for successful local writes. + // H1: durable path awaits fsync under appendfsync=always. + // On AOF failure we override `response` to an error + // frame and skip downstream side-effects (tracking + // invalidation, etc.) below — the client must see + // the failure, not a silent inconsistency. + let mut aof_failed = false; if !matches!(response, Frame::Error(_)) && is_write { - if let Some(ref tx) = ctx.aof_tx { + if let Some(ref pool) = ctx.aof_pool { let serialized = aof::serialize_command(&frame); - let _ = tx.try_send(AofMessage::Append(serialized)); + let lsn = aof::AofWriterPool::issue_append_lsn( + &ctx.repl_state, + ctx.shard_id, + serialized.len(), + ); + if pool + .try_send_append_durable(ctx.shard_id, lsn, serialized) + .await + .is_err() + { + response = + Frame::Error(bytes::Bytes::from_static(aof::AOF_FSYNC_ERR)); + aof_failed = true; + } } } + // Suppress downstream effects on AOF failure — the + // client sees the error frame, no tracking churn. + if aof_failed { + responses.push(response); + continue; + } // Phase 166 (Plan 02): record VectorIntents from HSET auto-index // into active cross-store TXN so TXN.ABORT can tombstone them. @@ -1768,7 +1826,7 @@ pub(crate) async fn handle_connection_sharded_monoio< let resp_idx = responses.len(); responses.push(Frame::Null); // placeholder, filled after batch dispatch // Pre-compute AOF bytes before moving frame into Arc - let aof_bytes = if ctx.aof_tx.is_some() && metadata::is_write(cmd) { + let aof_bytes = if ctx.aof_pool.is_some() && metadata::is_write(cmd) { Some(aof::serialize_command(&dispatch_frame)) } else { None @@ -1837,7 +1895,11 @@ pub(crate) async fn handle_connection_sharded_monoio< if !remote_groups.is_empty() { reply_futures.clear(); + // Capture `target` per batch so the cross-shard AOF write at the bottom + // of the loop can route to the owning shard's pool (not ctx.shard_id — + // mirrors the load-bearing fix at handler_sharded/mod.rs:1651). let mut oneshot_futures: Vec<( + usize, // target shard — owner for AOF append Vec<(usize, Option, Bytes)>, channel::OneshotReceiver>, )> = Vec::new(); @@ -1881,7 +1943,7 @@ pub(crate) async fn handle_connection_sharded_monoio< } } } - oneshot_futures.push((meta, reply_rx)); + oneshot_futures.push((target, meta, reply_rx)); } // Poll all shard responses via pending_wakers relay (monoio cross-thread waker fix). @@ -1889,7 +1951,7 @@ pub(crate) async fn handle_connection_sharded_monoio< // Instead, the connection task registers its waker in pending_wakers; the event // loop drains and wakes them after every SPSC cycle (~1ms). On wake, try_recv() // checks if the response arrived; if not, re-register and yield again. - for (meta, reply_rx) in oneshot_futures.drain(..) { + for (target, meta, reply_rx) in oneshot_futures.drain(..) { tracing::trace!( "Shard {}: awaiting cross-shard response via pending_wakers", ctx.shard_id @@ -1931,11 +1993,38 @@ pub(crate) async fn handle_connection_sharded_monoio< }; for ((resp_idx, aof_bytes, cmd_name), resp) in meta.into_iter().zip(shard_responses) { - // AOF logging for successful remote writes + // AOF logging for successful remote writes. + // Owner shard is `target` (NOT ctx.shard_id) — under PerShard + // layout the write must land in the target shard's AOF file + // since that shard owns the mutated data. Mirrors the + // load-bearing fix at handler_sharded/mod.rs:1651. if let Some(bytes) = aof_bytes { if !matches!(resp, Frame::Error(_)) { - if let Some(ref tx) = ctx.aof_tx { - let _ = tx.try_send(AofMessage::Append(bytes)); + if let Some(ref pool) = ctx.aof_pool { + // Cross-shard write: LSN must be sourced + // using `target`'s shard_id so the + // per-shard offset increment lands on the + // shard that owns the mutated data. + let lsn = aof::AofWriterPool::issue_append_lsn( + &ctx.repl_state, + target, + bytes.len(), + ); + // H1: durable path under appendfsync=always. + if pool + .try_send_append_durable(target, lsn, bytes) + .await + .is_err() + { + let err = Frame::Error(Bytes::from_static(aof::AOF_FSYNC_ERR)); + let err = apply_resp3_conversion( + &cmd_name, + err, + conn.protocol_version, + ); + responses[resp_idx] = err; + continue; + } } } } diff --git a/src/server/conn/handler_sharded/dispatch.rs b/src/server/conn/handler_sharded/dispatch.rs index e73b77f2..bcd4e98a 100644 --- a/src/server/conn/handler_sharded/dispatch.rs +++ b/src/server/conn/handler_sharded/dispatch.rs @@ -353,9 +353,9 @@ pub(super) fn try_handle_persistence( return true; } if cmd.eq_ignore_ascii_case(b"BGREWRITEAOF") { - if let Some(ref tx) = ctx.aof_tx { + if let Some(ref pool) = ctx.aof_pool { responses.push(crate::command::persistence::bgrewriteaof_start_sharded( - tx, + pool, ctx.shard_databases.clone(), )); } else { diff --git a/src/server/conn/handler_sharded/mod.rs b/src/server/conn/handler_sharded/mod.rs index 83256ca8..8e75b208 100644 --- a/src/server/conn/handler_sharded/mod.rs +++ b/src/server/conn/handler_sharded/mod.rs @@ -15,7 +15,7 @@ use std::collections::HashMap; use crate::command::connection as conn_cmd; use crate::command::metadata; use crate::command::{DispatchResult, dispatch, dispatch_read}; -use crate::persistence::aof::{self, AofMessage}; +use crate::persistence::aof; use crate::protocol::Frame; use crate::shard::dispatch::{ShardMessage, key_to_shard}; use crate::shard::mesh::ChannelMesh; @@ -1119,8 +1119,8 @@ pub(crate) async fn handle_connection_sharded_inner< } } - let is_write = if ctx.aof_tx.is_some() || conn.tracking_state.enabled { metadata::is_write(cmd) } else { false }; - let aof_bytes = if is_write && ctx.aof_tx.is_some() { Some(aof::serialize_command(&frame)) } else { None }; + let is_write = if ctx.aof_pool.is_some() || conn.tracking_state.enabled { metadata::is_write(cmd) } else { false }; + let aof_bytes = if is_write && ctx.aof_pool.is_some() { Some(aof::serialize_command(&frame)) } else { None }; if is_local { // LOCAL PATH: split into read/write to avoid exclusive lock on reads. @@ -1170,9 +1170,22 @@ pub(crate) async fn handle_connection_sharded_inner< }; // AOF only on actual success (:1). Matches handler_single // — `:0` (key absent) is a no-op and must not log. + // H1: durable path awaits fsync under appendfsync=always. if matches!(response, Frame::Integer(1)) { if let Some(ref bytes) = aof_bytes { - if let Some(ref tx) = ctx.aof_tx { let _ = tx.try_send(AofMessage::Append(bytes.clone())); } + if let Some(ref pool) = ctx.aof_pool { + let lsn = aof::AofWriterPool::issue_append_lsn(&ctx.repl_state, ctx.shard_id, bytes.len()); + if pool + .try_send_append_durable(ctx.shard_id, lsn, bytes.clone()) + .await + .is_err() + { + responses.push(Frame::Error(Bytes::from_static( + aof::AOF_FSYNC_ERR, + ))); + continue; + } + } } } responses.push(response); @@ -1214,9 +1227,22 @@ pub(crate) async fn handle_connection_sharded_inner< }; // AOF only on actual success (:1). Matches handler_single // — `:0` (key absent / dst exists w/o REPLACE) is a no-op. + // H1: durable path awaits fsync under appendfsync=always. if matches!(response, Frame::Integer(1)) { if let Some(ref bytes) = aof_bytes { - if let Some(ref tx) = ctx.aof_tx { let _ = tx.try_send(AofMessage::Append(bytes.clone())); } + if let Some(ref pool) = ctx.aof_pool { + let lsn = aof::AofWriterPool::issue_append_lsn(&ctx.repl_state, ctx.shard_id, bytes.len()); + if pool + .try_send_append_durable(ctx.shard_id, lsn, bytes.clone()) + .await + .is_err() + { + responses.push(Frame::Error(Bytes::from_static( + aof::AOF_FSYNC_ERR, + ))); + continue; + } + } } } responses.push(response); @@ -1315,7 +1341,7 @@ pub(crate) async fn handle_connection_sharded_inner< r }; - let (response, _sample_latency, dispatch_start): (Frame, bool, Option) = + let (mut response, _sample_latency, dispatch_start): (Frame, bool, Option) = match write_outcome { Ok(t) => t, Err(oom_frame) => { @@ -1425,11 +1451,29 @@ pub(crate) async fn handle_connection_sharded_inner< } } } + // H1: durable path under appendfsync=always. + let mut aof_failed = false; if let Some(bytes) = aof_bytes { if !matches!(response, Frame::Error(_)) { - if let Some(ref tx) = ctx.aof_tx { let _ = tx.try_send(AofMessage::Append(bytes)); } + if let Some(ref pool) = ctx.aof_pool { + let lsn = aof::AofWriterPool::issue_append_lsn(&ctx.repl_state, ctx.shard_id, bytes.len()); + if pool + .try_send_append_durable(ctx.shard_id, lsn, bytes) + .await + .is_err() + { + response = Frame::Error(Bytes::from_static( + aof::AOF_FSYNC_ERR, + )); + aof_failed = true; + } + } } } + if aof_failed { + responses.push(response); + continue; + } if conn.tracking_state.enabled && !matches!(response, Frame::Error(_)) { if let Some(key) = cmd_args.first().and_then(|f| extract_bytes(f)) { let senders = ctx.tracking_table.borrow_mut().invalidate_key(&key, client_id); @@ -1646,12 +1690,33 @@ pub(crate) async fn handle_connection_sharded_inner< for (meta, target) in reply_futures { let shard_responses = response_pool.future_for(target).await; for ((resp_idx, aof_bytes, cmd_name), resp) in meta.into_iter().zip(shard_responses) { + // AOF logging for successful remote writes. + // Owner shard is `target` (NOT ctx.shard_id) — under PerShard + // layout the write must land in the target shard's AOF file + // since that shard owns the mutated data. This was the + // pre-existing routing bug that motivated the per-shard AOF + // RFC (Option B): under TopLevel a single writer absorbed + // every cross-shard append, masking the wrong-owner write. + let mut resp_final = resp; if let Some(bytes) = aof_bytes { - if !matches!(resp, Frame::Error(_)) { - if let Some(ref tx) = ctx.aof_tx { let _ = tx.try_send(AofMessage::Append(bytes)); } + if !matches!(resp_final, Frame::Error(_)) { + if let Some(ref pool) = ctx.aof_pool { + // Cross-shard: LSN sourced for `target`. + let lsn = aof::AofWriterPool::issue_append_lsn(&ctx.repl_state, target, bytes.len()); + // H1: durable path under appendfsync=always. + if pool + .try_send_append_durable(target, lsn, bytes) + .await + .is_err() + { + resp_final = Frame::Error(Bytes::from_static( + aof::AOF_FSYNC_ERR, + )); + } + } } } - responses[resp_idx] = apply_resp3_conversion(&cmd_name, resp, proto_ver); + responses[resp_idx] = apply_resp3_conversion(&cmd_name, resp_final, proto_ver); } } } diff --git a/src/server/conn/handler_single.rs b/src/server/conn/handler_single.rs index 4a626cc7..d7a938fa 100644 --- a/src/server/conn/handler_single.rs +++ b/src/server/conn/handler_single.rs @@ -19,7 +19,6 @@ use crate::command::connection as conn_cmd; use crate::command::metadata; use crate::command::{DispatchResult, dispatch, dispatch_read}; use crate::config::{RuntimeConfig, ServerConfig}; -use crate::persistence::aof::AofMessage; use crate::protocol::Frame; use crate::pubsub::subscriber::Subscriber; use crate::pubsub::{self, PubSubRegistry}; @@ -34,6 +33,63 @@ use super::{ use crate::framevec; use crate::server::codec::RespCodec; +/// Flush AOF entries and responses under the `appendfsync=always` ordering contract (H1). +/// +/// **Invariant (H1):** awaits ALL fsync acks BEFORE sending ANY response to the +/// client. The client must never receive `+OK` before the entry is durable on +/// disk when `appendfsync=always` is configured. +/// +/// Returns `true` when the connection loop should break (sink send failed). +/// +/// Making the sink generic (`S: futures::Sink + Unpin`) lets unit tests +/// supply a lightweight recording sink instead of a real TcpStream, while the +/// production call site passes `&mut framed` (`Framed`) +/// unchanged — both satisfy the bound. +/// +/// # Arguments +/// +/// - `sink` — frame sink; production passes `Framed`, +/// tests pass any `Sink` mock. +/// - `responses` — per-command response slots; fsync failures patch the +/// corresponding slot to `Frame::Error`. +/// - `aof_entries` — `(resp_idx, bytes)`: bytes to fsync, slot to patch on failure. +/// - `pool` — AOF writer pool (caller must ensure Always policy). +/// - `repl_state` — replication state for LSN issuance (`&None` in tests). +/// - `change_counter` — auto-save dirty counter (`&None` if not configured). +pub(crate) async fn flush_with_aof_ack( + sink: &mut S, + mut responses: Vec, + aof_entries: Vec<(usize, Bytes)>, + pool: &crate::persistence::aof::AofWriterPool, + repl_state: &Option>>, + change_counter: &Option>, +) -> bool +where + S: futures::Sink + Unpin, +{ + // Phase 1 — await every fsync ack; patch failed slots. + for (resp_idx, bytes) in aof_entries { + let lsn = + crate::persistence::aof::AofWriterPool::issue_append_lsn(repl_state, 0, bytes.len()); + if pool.try_send_append_durable(0, lsn, bytes).await.is_err() && resp_idx < responses.len() + { + responses[resp_idx] = Frame::Error(Bytes::from_static(b"WRITEFAIL aof fsync failed")); + } + if let Some(counter) = change_counter { + counter.fetch_add(1, Ordering::Relaxed); + } + } + // Phase 2 — all acks received; flush responses to client. + let mut break_outer = false; + for response in responses { + if SinkExt::send(sink, response).await.is_err() { + break_outer = true; + break; + } + } + break_outer +} + /// Handle a single client connection. /// /// Reads frames from the TCP stream, dispatches commands, and writes responses. @@ -42,7 +98,10 @@ use crate::server::codec::RespCodec; /// When `requirepass` is set, clients must authenticate via AUTH before any other /// commands are accepted (except QUIT). /// -/// When `aof_tx` is provided, write commands are logged to the AOF file. +/// When `aof_pool` is provided, write commands are logged via the per-shard +/// AOF writer pool. handler_single is single-shard mode by definition +/// (num_shards = 1, shard_id = 0), so the pool is always a TopLevel layout +/// wrapping a single writer sender — see `AofWriterPool::top_level`. /// When `change_counter` is provided, write commands increment the counter for auto-save. /// /// Supports Pub/Sub subscriber mode: when a client subscribes to channels/patterns, @@ -61,7 +120,7 @@ pub async fn handle_connection( shutdown: CancellationToken, requirepass: Option, config: Arc, - aof_tx: Option>, + aof_pool: Option>, change_counter: Option>, pubsub_registry: Arc>, runtime_config: Arc>, @@ -332,7 +391,10 @@ pub async fn handle_connection( // Phase 1: Handle connection-level intercepts, collect dispatchable frames // Phase 2: Acquire ONE write lock, execute ALL dispatchable frames let mut responses: Vec = Vec::with_capacity(batch.len()); - let mut aof_entries: Vec = Vec::new(); + // Each entry carries (resp_idx, bytes) so the Always-policy flush path + // can patch responses[resp_idx] with WRITEFAIL when fsync fails, + // before any response is sent to the client (H1 fix — FIX-W1-1). + let mut aof_entries: Vec<(usize, Bytes)> = Vec::new(); let mut should_quit = false; let mut break_outer = false; @@ -608,8 +670,8 @@ pub async fn handle_connection( } // BGREWRITEAOF if cmd.eq_ignore_ascii_case(b"BGREWRITEAOF") { - let response = if let Some(ref tx) = aof_tx { - crate::command::persistence::bgrewriteaof_start(tx, db.clone()) + let response = if let Some(ref pool) = aof_pool { + crate::command::persistence::bgrewriteaof_start(pool, db.clone()) } else { Frame::Error(Bytes::from_static(b"ERR AOF is not enabled")) }; @@ -653,9 +715,14 @@ pub async fn handle_connection( )) } else { // WAL must be durable BEFORE the swap (no rollback - // path for SWAPDB). Try-send first; on failure return - // an error and leave both DBs untouched. - let wal_ok = if let Some(ref tx) = aof_tx { + // path for SWAPDB). Use try_send_append_durable so + // that the fsync policy is honoured: + // - appendfsync=always → await AppendSync ack + // (rendezvous guarantees data is on disk before +OK) + // - appendfsync=everysec/no → fire-and-forget (fast) + // On any Err the caller aborts and leaves both DBs + // untouched, preserving atomicity from the WAL's perspective. + let wal_ok = if let Some(ref pool) = aof_pool { let mut a_buf = itoa::Buffer::new(); let mut b_buf = itoa::Buffer::new(); let wal_frame = Frame::Array(crate::framevec![ @@ -671,12 +738,11 @@ pub async fn handle_connection( crate::persistence::aof::serialize_command( &wal_frame, ); - tx.try_send( - crate::persistence::aof::AofMessage::Append( - serialized, - ), - ) - .is_ok() + // Single-shard mode — shard_id = 0. + let lsn = crate::persistence::aof::AofWriterPool::issue_append_lsn(&repl_state, 0, serialized.len()); + pool.try_send_append_durable(0, lsn, serialized) + .await + .is_ok() } else { true // persistence disabled — no durability requirement }; @@ -849,7 +915,9 @@ pub async fn handle_connection( }; if let Some(bytes) = aof_bytes { if !matches!(&response, Frame::Error(_)) { - aof_entries.push(bytes); + // Carry resp_idx so the Always-policy flush can + // patch responses[resp_idx] on fsync failure. + aof_entries.push((resp_idx, bytes)); } } // Apply RESP3 response conversion if needed @@ -866,7 +934,52 @@ pub async fn handle_connection( break; } - // Flush accumulated responses first + // FIX-W1-1 + FIX-W2-4: Await AOF fsync ack for prior write + // commands BEFORE flushing their +OK responses. Ordering: + // (a) Under appendfsync=always: WRITEFAIL replaces +OK if + // fsync fails — no +OK is ever sent for a non-durable + // write. + // (b) The WRITEFAIL frame lands before the SUBSCRIBE + // response slot, not inside it (prior code flushed + // +OK first, then checked AOF, causing WRITEFAIL to + // be mistaken for the SUBSCRIBE ack by the client). + // + // For everysec/no policies, try_send_append_durable is + // fire-and-forget (returns Ok immediately) so no latency + // penalty. + // + // Note: aof_entries carries (resp_idx, bytes) from FIX-W1-1 + // — resp_idx is unused here because the all-or-nothing + // failure mode discards the entire response buffer; future + // per-slot patching could use it. + let mut aof_write_failed = false; + for (_resp_idx, bytes) in aof_entries.drain(..) { + if let Some(ref pool) = aof_pool { + let lsn = crate::persistence::aof::AofWriterPool::issue_append_lsn(&repl_state, 0, bytes.len()); + if let Err(_aof_err) = pool.try_send_append_durable(0, lsn, bytes).await { + aof_write_failed = true; + } + } + if let Some(ref counter) = change_counter { + counter.fetch_add(1, Ordering::Relaxed); + } + } + if aof_write_failed { + // Discard buffered +OK responses — the writes are not + // durable. Log at warn level so operators can correlate + // with disk I/O errors. + responses.clear(); + tracing::warn!( + "AOF fsync failed for prior write batch; returning error \ + to client and closing connection" + ); + let _ = framed.send(Frame::Error(Bytes::from_static( + crate::persistence::aof::AOF_FSYNC_ERR, + ))).await; + break; + } + // Flush accumulated +OK responses now that AOF durability + // has been confirmed (or is fire-and-forget). for resp in responses.drain(..) { if framed.send(resp).await.is_err() { break_outer = true; @@ -876,15 +989,6 @@ pub async fn handle_connection( if break_outer { break; } - // Send AOF entries accumulated so far - for bytes in aof_entries.drain(..) { - if let Some(ref tx) = aof_tx { - let _ = tx.send_async(AofMessage::Append(bytes)).await; - } - if let Some(ref counter) = change_counter { - counter.fetch_add(1, Ordering::Relaxed); - } - } // Handle subscribe if cmd_args.is_empty() { let cmd_lower = if cmd.eq_ignore_ascii_case(b"SUBSCRIBE") { "subscribe" } else { "psubscribe" }; @@ -1041,8 +1145,15 @@ pub async fn handle_connection( } conn.command_queue.clear(); conn.watched_keys.clear(); + // The EXEC response occupies responses[exec_resp_idx]. + // All txn AOF entries map to this same slot so that + // the Always-policy flush can patch the EXEC frame if + // any command's fsync fails. + let exec_resp_idx = responses.len(); responses.push(result); - aof_entries.extend(txn_aof_entries); + aof_entries.extend( + txn_aof_entries.into_iter().map(|b| (exec_resp_idx, b)), + ); } continue; } @@ -1509,15 +1620,31 @@ pub async fn handle_connection( let records = store.drain_wal(); (resp, records) }; + let mut graph_aof_failed = false; for record in wal_records { - if let Some(ref tx) = aof_tx { - let _ = tx.send_async(AofMessage::Append(bytes::Bytes::from(record))).await; + if let Some(ref pool) = aof_pool { + // Single-shard mode (shard_id = 0). + // `try_send_append_durable` awaits writer + // ack under `appendfsync=always` (H1 + // closure) and is fire-and-forget for + // everysec/no. + let bytes = bytes::Bytes::from(record); + let lsn = crate::persistence::aof::AofWriterPool::issue_append_lsn(&repl_state, 0, bytes.len()); + if let Err(_aof_err) = pool.try_send_append_durable(0, lsn, bytes).await { + graph_aof_failed = true; + } } if let Some(ref counter) = change_counter { counter.fetch_add(1, std::sync::atomic::Ordering::Relaxed); } } - responses.push(response); + if graph_aof_failed { + responses.push(Frame::Error(bytes::Bytes::from_static( + crate::persistence::aof::AOF_FSYNC_ERR, + ))); + } else { + responses.push(response); + } continue; } else { responses.push(Frame::Error(bytes::Bytes::from_static(b"ERR graph engine not initialized"))); @@ -1528,7 +1655,7 @@ pub async fn handle_connection( let is_write = metadata::is_write(cmd); // Serialize for AOF before dispatch - let aof_bytes = if is_write && aof_tx.is_some() { + let aof_bytes = if is_write && aof_pool.is_some() { let mut buf = BytesMut::new(); crate::protocol::serialize::serialize(&frame, &mut buf); Some(buf.freeze()) @@ -2109,7 +2236,7 @@ pub async fn handle_connection( }; if matches!(response, Frame::Integer(1)) { if let Some(bytes) = &aof_bytes { - aof_entries.push(bytes.clone()); + aof_entries.push((resp_idx, bytes.clone())); } } responses[resp_idx] = response; @@ -2138,7 +2265,7 @@ pub async fn handle_connection( }; if matches!(response, Frame::Integer(1)) { if let Some(bytes) = &aof_bytes { - aof_entries.push(bytes.clone()); + aof_entries.push((resp_idx, bytes.clone())); } } responses[resp_idx] = response; @@ -2202,7 +2329,7 @@ pub async fn handle_connection( } } if let Some(bytes) = aof_bytes { - aof_entries.push(bytes.clone()); + aof_entries.push((resp_idx, bytes.clone())); } } // Apply RESP3 response conversion if needed @@ -2222,21 +2349,48 @@ pub async fn handle_connection( } } // all locks dropped here -- BEFORE any await - // --- Write all responses OUTSIDE the lock --- - for response in responses { - if framed.send(response).await.is_err() { - break_outer = true; - break; - } - } + // FIX-W1-1: appendfsync=always ordering — H1 close for the single-shard + // tokio path. Under Always policy: await all AOF fsync acks FIRST, patch + // any failed response slots with WRITEFAIL, THEN flush responses to the + // client (delegated to `flush_with_aof_ack` so tests can call the real + // production path rather than reproducing it inline). + // Under EverySec/No: keep existing fire-and-forget ordering (flush + // responses first, then enqueue AOF in the background — no latency impact). + let use_always_ordering = aof_pool + .as_ref() + .map(|p| p.fsync_policy() == crate::persistence::aof::FsyncPolicy::Always) + .unwrap_or(false); - // --- Send AOF entries OUTSIDE the lock --- - for bytes in aof_entries { - if let Some(ref tx) = aof_tx { - let _ = tx.send_async(AofMessage::Append(bytes)).await; + if use_always_ordering { + // `use_always_ordering` is only true when aof_pool is Some + Always. + if let Some(ref pool) = aof_pool { + break_outer = flush_with_aof_ack( + &mut framed, + responses, + aof_entries, + pool, + &repl_state, + &change_counter, + ) + .await; } - if let Some(ref counter) = change_counter { - counter.fetch_add(1, Ordering::Relaxed); + } else { + // EverySec / No policy: flush responses first (zero added latency), + // then fire-and-forget AOF enqueue. + for response in responses { + if framed.send(response).await.is_err() { + break_outer = true; + break; + } + } + for (_, bytes) in aof_entries { + if let Some(ref pool) = aof_pool { + let lsn = crate::persistence::aof::AofWriterPool::issue_append_lsn(&repl_state, 0, bytes.len()); + let _ = pool.try_send_append_durable(0, lsn, bytes).await; + } + if let Some(ref counter) = change_counter { + counter.fetch_add(1, Ordering::Relaxed); + } } } @@ -2282,3 +2436,124 @@ pub async fn handle_connection( } crate::admin::metrics_setup::record_connection_closed(); } + +#[cfg(test)] +mod tests { + use super::*; + use crate::persistence::aof::{AofAck, AofMessage, AofWriterPool, FsyncPolicy}; + use crate::runtime::channel; + use std::pin::Pin; + use std::task::{Context, Poll}; + use std::time::{Duration, Instant}; + + // ── Minimal recording sink ────────────────────────────────────────────── + // + // Implements `futures::Sink` so `flush_with_aof_ack` can be called + // directly in unit tests without a real TcpStream. Each successful + // `start_send` appends `(frame, Instant::now())` to the internal log. + struct RecordingSink { + log: Vec<(Frame, Instant)>, + } + + impl RecordingSink { + fn new() -> Self { + Self { log: Vec::new() } + } + fn first_send_instant(&self) -> Option { + self.log.first().map(|(_, t)| *t) + } + } + + impl futures::Sink for RecordingSink { + type Error = (); + + fn poll_ready(self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn start_send(mut self: Pin<&mut Self>, item: Frame) -> Result<(), ()> { + self.log.push((item, Instant::now())); + Ok(()) + } + + fn poll_flush(self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn poll_close(self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + } + + /// FIX-W1-1 r3: discriminating ordering test for `flush_with_aof_ack`. + /// + /// This test calls the **real** `flush_with_aof_ack` function that the + /// production handler uses — not an inline copy. The H1 contract is: + /// + /// All AOF fsync acks MUST be awaited BEFORE any response is sent. + /// + /// Red state (broken ordering — send-before-ack): + /// If the fn were to flush responses BEFORE awaiting acks, the mock + /// 60ms fsync delay would NOT gate the first response, so + /// `elapsed_ms < 55` → test fails. + /// + /// Green state (ack-before-send — current production code): + /// `flush_with_aof_ack` awaits the 60ms ack BEFORE any `start_send`, + /// so `elapsed_ms >= 55` → test passes. + /// + /// Red verification was performed by temporarily inverting the fn body + /// (Phase 2 before Phase 1) and confirming `elapsed_ms ≈ 0ms` → FAIL. + /// The fn was then restored and the test passes consistently. + #[tokio::test] + async fn flush_with_aof_ack_ack_precedes_response() { + // Build an Always-policy pool backed by a real bounded channel. + let (tx, rx) = channel::mpsc_bounded::(4); + let pool = AofWriterPool::top_level_with_policy(tx, FsyncPolicy::Always); + + // Mock writer: receives one AppendSync, sleeps 60ms to simulate fsync, + // then sends Synced. Runs on a blocking thread because flume's + // `Receiver::recv()` is synchronous. + let mock_writer = tokio::task::spawn_blocking(move || { + let msg = rx.recv().expect("mock writer: message received"); + if let AofMessage::AppendSync { ack, .. } = msg { + std::thread::sleep(Duration::from_millis(60)); + let _ = ack.send(AofAck::Synced); + } else { + panic!("Always policy MUST send AppendSync; got non-AppendSync variant"); + } + }); + + let start = Instant::now(); + + let responses = vec![Frame::SimpleString(bytes::Bytes::from_static(b"OK"))]; + let aof_entries = vec![(0usize, bytes::Bytes::from_static(b"SET k v\r\n"))]; + let mut sink = RecordingSink::new(); + + let broke = flush_with_aof_ack( + &mut sink, + responses, + aof_entries, + &pool, + &None, // no replication state + &None, // no change counter + ) + .await; + + mock_writer.await.expect("mock writer completed cleanly"); + + assert!(!broke, "sink send must not have failed"); + assert_eq!(sink.log.len(), 1, "exactly one response must be sent"); + + let first_send = sink + .first_send_instant() + .expect("RecordingSink recorded at least one send"); + let elapsed_ms = first_send.duration_since(start).as_millis(); + + assert!( + elapsed_ms >= 55, + "H1 violation: first response sent {elapsed_ms}ms after start — \ + expected >= 55ms (mock fsync delay is 60ms). \ + This means +OK was flushed before the AOF ack." + ); + } +} diff --git a/src/server/conn/tests.rs b/src/server/conn/tests.rs index 2f6c74e1..9bcd6460 100644 --- a/src/server/conn/tests.rs +++ b/src/server/conn/tests.rs @@ -31,7 +31,7 @@ fn test_inline_get_hit() { } let mut read_buf = BytesMut::from(&b"*2\r\n$3\r\nGET\r\n$3\r\nfoo\r\n"[..]); let mut write_buf = BytesMut::new(); - let aof_tx: Option> = None; + let aof_pool: Option> = None; let rt_config = make_rt_config(); let result = try_inline_dispatch( @@ -40,7 +40,8 @@ fn test_inline_get_hit() { &dbs, 0, 0, - &aof_tx, + &aof_pool, + &None, 0, 1, false, @@ -56,7 +57,7 @@ fn test_inline_get_miss() { let dbs = make_dbs(); let mut read_buf = BytesMut::from(&b"*2\r\n$3\r\nGET\r\n$3\r\nfoo\r\n"[..]); let mut write_buf = BytesMut::new(); - let aof_tx: Option> = None; + let aof_pool: Option> = None; let rt_config = make_rt_config(); let result = try_inline_dispatch( @@ -65,7 +66,8 @@ fn test_inline_get_miss() { &dbs, 0, 0, - &aof_tx, + &aof_pool, + &None, 0, 1, false, @@ -84,7 +86,7 @@ fn test_inline_set_falls_through_when_writes_disabled() { let mut read_buf = BytesMut::from(&cmd[..]); let original_len = read_buf.len(); let mut write_buf = BytesMut::new(); - let aof_tx: Option> = None; + let aof_pool: Option> = None; let rt_config = make_rt_config(); let result = try_inline_dispatch( @@ -93,7 +95,8 @@ fn test_inline_set_falls_through_when_writes_disabled() { &dbs, 0, 0, - &aof_tx, + &aof_pool, + &None, 0, 1, false, @@ -111,7 +114,7 @@ fn test_inline_set_executes_when_writes_enabled() { let cmd = b"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$3\r\nbar\r\n"; let mut read_buf = BytesMut::from(&cmd[..]); let mut write_buf = BytesMut::new(); - let aof_tx: Option> = None; + let aof_pool: Option> = None; let rt_config = make_rt_config(); let result = try_inline_dispatch( @@ -120,7 +123,8 @@ fn test_inline_set_executes_when_writes_enabled() { &dbs, 0, 0, - &aof_tx, + &aof_pool, + &None, 0, 1, true, @@ -150,7 +154,7 @@ fn test_inline_set_with_options_falls_through() { let mut read_buf = BytesMut::from(&cmd[..]); let original_len = read_buf.len(); let mut write_buf = BytesMut::new(); - let aof_tx: Option> = None; + let aof_pool: Option> = None; let rt_config = make_rt_config(); let result = try_inline_dispatch( @@ -159,7 +163,8 @@ fn test_inline_set_with_options_falls_through() { &dbs, 0, 0, - &aof_tx, + &aof_pool, + &None, 0, 1, true, @@ -176,7 +181,7 @@ fn test_inline_fallthrough() { let mut read_buf = BytesMut::from(&ping_cmd[..]); let original_len = read_buf.len(); let mut write_buf = BytesMut::new(); - let aof_tx: Option> = None; + let aof_pool: Option> = None; let rt_config = make_rt_config(); let result = try_inline_dispatch( @@ -185,7 +190,8 @@ fn test_inline_fallthrough() { &dbs, 0, 0, - &aof_tx, + &aof_pool, + &None, 0, 1, false, @@ -211,7 +217,7 @@ fn test_inline_mixed_batch() { read_buf.extend_from_slice(b"*2\r\n$3\r\nGET\r\n$3\r\nfoo\r\n"); read_buf.extend_from_slice(b"*1\r\n$4\r\nPING\r\n"); let mut write_buf = BytesMut::new(); - let aof_tx: Option> = None; + let aof_pool: Option> = None; let rt_config = make_rt_config(); // Inline loop should process GET but leave PING @@ -221,7 +227,8 @@ fn test_inline_mixed_batch() { &dbs, 0, 0, - &aof_tx, + &aof_pool, + &None, 0, 1, false, @@ -244,7 +251,7 @@ fn test_inline_case_insensitive() { } let mut read_buf = BytesMut::from(&b"*2\r\n$3\r\nget\r\n$3\r\nfoo\r\n"[..]); let mut write_buf = BytesMut::new(); - let aof_tx: Option> = None; + let aof_pool: Option> = None; let rt_config = make_rt_config(); let result = try_inline_dispatch( @@ -253,7 +260,8 @@ fn test_inline_case_insensitive() { &dbs, 0, 0, - &aof_tx, + &aof_pool, + &None, 0, 1, false, @@ -271,7 +279,7 @@ fn test_inline_partial() { let mut read_buf = BytesMut::from(&b"*2\r\n$3\r\nGET\r\n$3\r\n"[..]); let original_len = read_buf.len(); let mut write_buf = BytesMut::new(); - let aof_tx: Option> = None; + let aof_pool: Option> = None; let rt_config = make_rt_config(); let result = try_inline_dispatch( @@ -280,7 +288,8 @@ fn test_inline_partial() { &dbs, 0, 0, - &aof_tx, + &aof_pool, + &None, 0, 1, false, @@ -296,7 +305,9 @@ fn test_inline_set_with_aof_falls_through_when_writes_disabled() { // SET falls through when can_inline_writes=false even with AOF. let dbs = make_dbs(); let (aof_sender, _aof_receiver) = channel::mpsc_bounded::(16); - let aof_tx: Option> = Some(aof_sender); + let aof_pool: Option> = Some( + crate::persistence::aof::AofWriterPool::top_level(aof_sender), + ); let cmd = b"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$3\r\nbar\r\n"; let mut read_buf = BytesMut::from(&cmd[..]); let original_len = read_buf.len(); @@ -310,7 +321,8 @@ fn test_inline_set_with_aof_falls_through_when_writes_disabled() { &dbs, 0, 0, - &aof_tx, + &aof_pool, + &None, 0, 1, false, @@ -342,7 +354,7 @@ fn test_inline_multiple_gets() { read_buf.extend_from_slice(b"*2\r\n$3\r\nGET\r\n$1\r\na\r\n"); read_buf.extend_from_slice(b"*2\r\n$3\r\nGET\r\n$1\r\nb\r\n"); let mut write_buf = BytesMut::new(); - let aof_tx: Option> = None; + let aof_pool: Option> = None; let rt_config = make_rt_config(); let total = try_inline_dispatch_loop( @@ -351,7 +363,8 @@ fn test_inline_multiple_gets() { &dbs, 0, 0, - &aof_tx, + &aof_pool, + &None, 0, 1, false, diff --git a/src/server/conn_state.rs b/src/server/conn_state.rs index 674857db..d2946f36 100644 --- a/src/server/conn_state.rs +++ b/src/server/conn_state.rs @@ -8,7 +8,7 @@ use ringbuf::HeapProd; use crate::blocking::BlockingRegistry; use crate::cluster::ClusterState; use crate::config::{RuntimeConfig, ServerConfig}; -use crate::persistence::aof::AofMessage; +use crate::persistence::aof::AofWriterPool; use crate::protocol::Frame; use crate::pubsub::PubSubRegistry; use crate::replication::state::ReplicationState; @@ -38,7 +38,12 @@ pub struct ConnectionContext { pub blocking_registry: Rc>, pub shutdown: CancellationToken, pub requirepass: Option, - pub aof_tx: Option>, + /// Per-shard AOF writer pool. **Sole AOF interface** after the + /// 2d/2e migration sequence. Built by spawn sites in `shard/conn_accept.rs` + /// from the manifest layout — TopLevel wraps a single sender, + /// PerShard owns one sender per shard. Step 2f flips spawn sites + /// to construct PerShard pools when the on-disk manifest demands it. + pub aof_pool: Option>, pub tracking_table: Rc>, pub repl_state: Option>>, pub cluster_state: Option>>, diff --git a/src/server/embedded.rs b/src/server/embedded.rs index 55a2b3fd..3d7e3cad 100644 --- a/src/server/embedded.rs +++ b/src/server/embedded.rs @@ -48,7 +48,7 @@ use parking_lot::RwLock; use tracing::info; use crate::config::ServerConfig; -use crate::persistence::aof::{self, AofMessage, FsyncPolicy}; +use crate::persistence::aof::{self, AofMessage, AofWriterPool, FsyncPolicy}; use crate::runtime::cancel::CancellationToken; use crate::runtime::channel; use crate::runtime::{RuntimeFactoryImpl, traits::RuntimeFactory}; @@ -113,9 +113,12 @@ pub async fn run_embedded( // AOF writer: dedicated std::thread (matches main.rs lifetime model). // We retain the JoinHandle so shutdown can wait for the writer to finish // flushing — dropping it would race the process exit and risk losing the - // final fsync (CodeRabbit #1). - let (aof_tx, aof_join): ( - Option>, + // final fsync (CodeRabbit #1). Step 2f-α: wrap the sender in a TopLevel + // `AofWriterPool` so every shard receives an Arc clone with a uniform + // API. Channel close still drives writer termination — dropping the last + // Arc drops the pool, which drops the underlying senders. + let (aof_pool, aof_join): ( + Option>, Option>, ) = if config.appendonly == "yes" { let (tx, rx) = channel::mpsc_bounded::(10_000); @@ -132,7 +135,10 @@ pub async fn run_embedded( }) .context("embedded moon: failed to spawn AOF writer thread")?; info!("embedded moon: AOF enabled (fsync: {:?})", fsync); - (Some(tx), Some(handle)) + ( + Some(AofWriterPool::top_level_with_policy(tx, fsync)), + Some(handle), + ) } else { (None, None) }; @@ -263,7 +269,7 @@ pub async fn run_embedded( let consumers = mesh.take_consumers(id); let conn_rx = mesh.take_conn_rx(id); let shard_cancel = cancel.clone(); - let shard_aof_tx = aof_tx.clone(); + let shard_aof_pool = aof_pool.clone(); let shard_bind_addr = bind_addr.clone(); let shard_persistence_dir = persistence_dir.clone(); let shard_snap_rx = snap_rx.clone(); @@ -293,7 +299,7 @@ pub async fn run_embedded( consumers, producers, shard_cancel, - shard_aof_tx, + shard_aof_pool, Some(shard_bind_addr), shard_persistence_dir, shard_snap_rx, @@ -354,10 +360,12 @@ pub async fn run_embedded( // Listener exited (cancel fired or fatal error). Shutdown ordering: // 1. cancel.cancel() — stops shard accept loops + producers. - // 2. Join shard threads — drops every shard-held `aof_tx` clone. - // 3. Drop our outer `aof_tx` — last sender goes away, the AOF writer's - // `recv_async()` returns `Err(_)` and the task flushes + fsyncs - // before exiting (see `aof::aof_writer_task` Err arm). + // 2. Join shard threads — drops every shard-held `Arc` + // clone (each `ConnectionContext` and each `shard_aof_pool` capture). + // 3. Drop our outer `aof_pool` Arc — the last reference goes away, the + // pool's `Drop` runs, dropping the underlying `Vec`. The + // AOF writer's `recv_async()` returns `Err(_)` and the task flushes + // + fsyncs before exiting (see `aof::aof_writer_task` Err arm). // 4. Join the AOF thread. // // This sequencing fixes Qodo bug #5: sending `AofMessage::Shutdown` before @@ -381,8 +389,9 @@ pub async fn run_embedded( } } - // Drop the last AOF sender so the writer's recv loop sees channel close. - drop(aof_tx); + // Drop the last `Arc` so the underlying senders close, + // triggering the writer's recv loop to drain + fsync + exit. + drop(aof_pool); let aof_panic = if let Some(handle) = aof_join { match handle.join() { diff --git a/src/server/listener.rs b/src/server/listener.rs index 607aef54..a93f26b7 100644 --- a/src/server/listener.rs +++ b/src/server/listener.rs @@ -13,7 +13,7 @@ use tracing::{debug, error, info}; use crate::command::connection as conn_cmd; use crate::config::ServerConfig; #[cfg(feature = "runtime-tokio")] -use crate::persistence::aof::{self, AofMessage, FsyncPolicy}; +use crate::persistence::aof::{self, AofMessage, AofWriterPool, FsyncPolicy}; #[cfg(feature = "runtime-tokio")] use crate::persistence::rdb; #[cfg(feature = "runtime-tokio")] @@ -115,15 +115,17 @@ pub async fn run_with_shutdown( let config = Arc::new(config); - // Set up AOF writer task if appendonly is enabled - let aof_tx: Option> = if config.appendonly == "yes" { + // Set up AOF writer task + wrap the sender as a TopLevel `AofWriterPool` + // (step 2f-α). Single-shard tokio path: `handler_single` receives the pool + // directly; per-shard layout is unreachable from this listener. + let aof_pool: Option> = if config.appendonly == "yes" { let (tx, rx) = channel::mpsc_bounded::(10_000); let aof_token = token.child_token(); let fsync = FsyncPolicy::from_str(&config.appendfsync); let aof_file_path = PathBuf::from(&config.dir).join(&config.appendfilename); tokio::spawn(aof::aof_writer_task(rx, aof_file_path, fsync, aof_token)); info!("AOF enabled with fsync policy: {:?}", fsync); - Some(tx) + Some(AofWriterPool::top_level_with_policy(tx, fsync)) } else { None }; @@ -235,7 +237,7 @@ pub async fn run_with_shutdown( let conn_token = token.child_token(); let requirepass = config.requirepass.clone(); let config = config.clone(); - let aof_tx = aof_tx.clone(); + let aof_pool_conn = aof_pool.clone(); let change_counter = Some(change_counter.clone()); let pubsub = pubsub_registry.clone(); let rt_config = runtime_config.clone(); @@ -249,7 +251,7 @@ pub async fn run_with_shutdown( let gs = graph_store.clone(); tokio::spawn(connection::handle_connection( stream, db, conn_token, requirepass, config, - aof_tx, change_counter, pubsub, rt_config, + aof_pool_conn, change_counter, pubsub, rt_config, tracking, cid, Some(rs), acl, Some(vs), Some(ts), #[cfg(feature = "graph")] @@ -263,9 +265,12 @@ pub async fn run_with_shutdown( } _ = token.cancelled() => { info!("Server shutting down"); - // Send shutdown to AOF writer - if let Some(ref tx) = aof_tx { - let _ = tx.send_async(AofMessage::Shutdown).await; + // Fan out shutdown to every AOF writer (single writer under TopLevel, + // one-per-shard under PerShard — step 2f-β). `broadcast_shutdown` + // is `try_send`-based so the writer must be draining; under tokio + // listener it always is. + if let Some(ref pool) = aof_pool { + pool.broadcast_shutdown(); } break; } diff --git a/src/shard/conn_accept.rs b/src/shard/conn_accept.rs index 432d65aa..de77e183 100644 --- a/src/shard/conn_accept.rs +++ b/src/shard/conn_accept.rs @@ -104,7 +104,7 @@ pub(crate) fn spawn_tokio_connection( pubsub_arc: &Arc>, blocking_rc: &Rc>, shutdown: &CancellationToken, - aof_tx: &Option>, + aof_pool: &Option>, tracking_rc: &Rc>, lua_rc: &Rc>>>, script_cache_rc: &Rc>, @@ -134,7 +134,6 @@ pub(crate) fn spawn_tokio_connection( let psr = pubsub_arc.clone(); let blk = blocking_rc.clone(); let sd = shutdown.clone(); - let aof = aof_tx.clone(); let trk = tracking_rc.clone(); let cid = conn_cmd::next_client_id(); let rs = repl_state.clone(); @@ -170,7 +169,10 @@ pub(crate) fn spawn_tokio_connection( set_tcp_keepalive(tcp_stream.as_raw_fd(), tcp_keepalive_secs); } - // Construct ConnectionContext from cloned shared state + // Construct ConnectionContext from cloned shared state. The pool is + // already built by the spawn site (main.rs / listener.rs / embedded.rs) + // and threaded through `Shard::run` — we just clone the Arc once here. + let pool_for_ctx = aof_pool.as_ref().map(Arc::clone); let conn_ctx = crate::server::conn::ConnectionContext::new( sdbs, shard_id, @@ -178,7 +180,7 @@ pub(crate) fn spawn_tokio_connection( psr, blk, reqpass, - aof, + pool_for_ctx, trk, rs, cs, @@ -272,7 +274,7 @@ pub(crate) fn spawn_migrated_tokio_connection( pubsub_arc: &Arc>, blocking_rc: &Rc>, shutdown: &CancellationToken, - aof_tx: &Option>, + aof_pool: &Option>, tracking_rc: &Rc>, lua_rc: &Rc>>>, script_cache_rc: &Rc>, @@ -326,7 +328,6 @@ pub(crate) fn spawn_migrated_tokio_connection( let psr = pubsub_arc.clone(); let blk = blocking_rc.clone(); let sd = shutdown.clone(); - let aof = aof_tx.clone(); let trk = tracking_rc.clone(); let cid = state.client_id; let rs = repl_state.clone(); @@ -354,6 +355,8 @@ pub(crate) fn spawn_migrated_tokio_connection( let migration_buf = take_migration_read_buf(&mut state); + // Pool is built by the spawn site and threaded through here. + let pool_for_ctx = aof_pool.as_ref().map(Arc::clone); let conn_ctx = crate::server::conn::ConnectionContext::new( sdbs, shard_id, @@ -361,7 +364,7 @@ pub(crate) fn spawn_migrated_tokio_connection( psr, blk, None, // requirepass: None = pre-authenticated - aof, + pool_for_ctx, trk, rs, cs, @@ -421,7 +424,7 @@ pub(crate) fn spawn_monoio_connection( pubsub_arc: &Arc>, blocking_rc: &Rc>, shutdown: &CancellationToken, - aof_tx: &Option>, + aof_pool: &Option>, tracking_rc: &Rc>, lua_rc: &Rc>>>, script_cache_rc: &Rc>, @@ -464,7 +467,6 @@ pub(crate) fn spawn_monoio_connection( let psr = pubsub_arc.clone(); let blk = blocking_rc.clone(); let sd = shutdown.clone(); - let aof = aof_tx.clone(); let trk = tracking_rc.clone(); let cid = conn_cmd::next_client_id(); let rs = repl_state.clone(); @@ -500,12 +502,38 @@ pub(crate) fn spawn_monoio_connection( .map(|a| a.to_string()) .unwrap_or_else(|_| "unknown".to_string()); - // Construct ConnectionContext from cloned shared state + // Construct ConnectionContext from cloned shared state. Pool is + // built by the spawn site and threaded through here. let reqpass = rtcfg.read().requirepass.clone(); + let pool_for_ctx = aof_pool.as_ref().map(Arc::clone); let conn_ctx = crate::server::conn::ConnectionContext::new( - sdbs, shard_id, num_shards, psr, blk, reqpass, aof, trk, rs, cs, lua, sc, cp, acl, - rtcfg, scfg, dtx, notifiers, snap_tx, clk, rsm, all_regs, all_rsm, aff, spill_tx, - spill_fid, do_dir, + sdbs, + shard_id, + num_shards, + psr, + blk, + reqpass, + pool_for_ctx, + trk, + rs, + cs, + lua, + sc, + cp, + acl, + rtcfg, + scfg, + dtx, + notifiers, + snap_tx, + clk, + rsm, + all_regs, + all_rsm, + aff, + spill_tx, + spill_fid, + do_dir, ); let maxclients = conn_ctx.runtime_config.read().maxclients; @@ -713,7 +741,7 @@ pub(crate) fn spawn_migrated_monoio_connection( pubsub_arc: &Arc>, blocking_rc: &Rc>, shutdown: &CancellationToken, - aof_tx: &Option>, + aof_pool: &Option>, tracking_rc: &Rc>, lua_rc: &Rc>>>, script_cache_rc: &Rc>, @@ -766,7 +794,6 @@ pub(crate) fn spawn_migrated_monoio_connection( let psr = pubsub_arc.clone(); let blk = blocking_rc.clone(); let sd = shutdown.clone(); - let aof = aof_tx.clone(); let trk = tracking_rc.clone(); let cid = state.client_id; let rs = repl_state.clone(); @@ -802,11 +829,36 @@ pub(crate) fn spawn_migrated_monoio_connection( let migration_buf = take_migration_read_buf(&mut state); + // Pool is built by the spawn site and threaded through here. + let pool_for_ctx = aof_pool.as_ref().map(Arc::clone); let conn_ctx = crate::server::conn::ConnectionContext::new( - sdbs, shard_id, num_shards, psr, blk, + sdbs, + shard_id, + num_shards, + psr, + blk, None, // requirepass: None = pre-authenticated - aof, trk, rs, cs, lua, sc, cp, acl, rtcfg, scfg, dtx, notifiers, snap_tx, clk, rsm, - all_regs, all_rsm, aff, spill_tx, spill_fid, do_dir, + pool_for_ctx, + trk, + rs, + cs, + lua, + sc, + cp, + acl, + rtcfg, + scfg, + dtx, + notifiers, + snap_tx, + clk, + rsm, + all_regs, + all_rsm, + aff, + spill_tx, + spill_fid, + do_dir, ); monoio::spawn(async move { diff --git a/src/shard/event_loop.rs b/src/shard/event_loop.rs index 2b9de27d..e2227897 100644 --- a/src/shard/event_loop.rs +++ b/src/shard/event_loop.rs @@ -56,7 +56,7 @@ impl super::Shard { mut consumers: Vec>, producers: Vec>, shutdown: CancellationToken, - aof_tx: Option>, + aof_pool: Option>, bind_addr: Option, persistence_dir: Option, snapshot_trigger_rx: channel::WatchReceiver, @@ -1047,7 +1047,7 @@ impl super::Shard { conn_accept::spawn_tokio_connection( tcp_stream, false, &tls_config, &shard_databases, &dispatch_tx, &pubsub_arc, &blocking_rc, - &shutdown, &aof_tx, &tracking_rc, &lua_rc, &script_cache_rc, + &shutdown, &aof_pool, &tracking_rc, &lua_rc, &script_cache_rc, &acl_table, &runtime_config, &server_config, &all_notifiers, &snapshot_trigger_tx, &repl_state, &cluster_state, &cached_clock, &remote_sub_map_arc, &all_pubsub_registries, @@ -1097,7 +1097,7 @@ impl super::Shard { conn_accept::spawn_tokio_connection( tcp_stream, is_tls, &tls_config, &shard_databases, &dispatch_tx, &pubsub_arc, &blocking_rc, - &shutdown, &aof_tx, &tracking_rc, &lua_rc, &script_cache_rc, + &shutdown, &aof_pool, &tracking_rc, &lua_rc, &script_cache_rc, &acl_table, &runtime_config, &server_config, &all_notifiers, &snapshot_trigger_tx, &repl_state, &cluster_state, &cached_clock, &remote_sub_map_arc, &all_pubsub_registries, @@ -1130,6 +1130,7 @@ impl super::Shard { server_config.graph_merge_max_segments, server_config.graph_dead_edge_trigger, &mut autovacuum_daemon, + aof_pool.as_ref(), // FIX-W1-2 ); }); } else { @@ -1145,6 +1146,7 @@ impl super::Shard { server_config.graph_merge_max_segments, server_config.graph_dead_edge_trigger, &mut autovacuum_daemon, + aof_pool.as_ref(), // FIX-W1-2 ); } // MA5: persist maintenance schedule when modified by RECLAMATION SCHEDULE. @@ -1180,7 +1182,7 @@ impl super::Shard { conn_accept::spawn_migrated_tokio_connection( fd, state, &shard_databases, &dispatch_tx, &pubsub_arc, &blocking_rc, - &shutdown, &aof_tx, &tracking_rc, &lua_rc, &script_cache_rc, + &shutdown, &aof_pool, &tracking_rc, &lua_rc, &script_cache_rc, &acl_table, &runtime_config, &server_config, &all_notifiers, &snapshot_trigger_tx, &repl_state, &cluster_state, &cached_clock, &remote_sub_map_arc, &all_pubsub_registries, @@ -1193,7 +1195,7 @@ impl super::Shard { conn_accept::spawn_migrated_monoio_connection( fd, state, &shard_databases, &dispatch_tx, &pubsub_arc, &blocking_rc, - &shutdown, &aof_tx, &tracking_rc, &lua_rc, &script_cache_rc, + &shutdown, &aof_pool, &tracking_rc, &lua_rc, &script_cache_rc, &acl_table, &runtime_config, &server_config, &all_notifiers, &snapshot_trigger_tx, &repl_state, &cluster_state, &cached_clock, &remote_sub_map_arc, &all_pubsub_registries, @@ -1227,6 +1229,7 @@ impl super::Shard { server_config.graph_merge_max_segments, server_config.graph_dead_edge_trigger, &mut autovacuum_daemon, + aof_pool.as_ref(), // FIX-W1-2 ); }); } else { @@ -1242,6 +1245,7 @@ impl super::Shard { server_config.graph_merge_max_segments, server_config.graph_dead_edge_trigger, &mut autovacuum_daemon, + aof_pool.as_ref(), // FIX-W1-2 ); } // MA5: persist maintenance schedule when modified by RECLAMATION SCHEDULE. @@ -1277,7 +1281,7 @@ impl super::Shard { conn_accept::spawn_migrated_tokio_connection( fd, state, &shard_databases, &dispatch_tx, &pubsub_arc, &blocking_rc, - &shutdown, &aof_tx, &tracking_rc, &lua_rc, &script_cache_rc, + &shutdown, &aof_pool, &tracking_rc, &lua_rc, &script_cache_rc, &acl_table, &runtime_config, &server_config, &all_notifiers, &snapshot_trigger_tx, &repl_state, &cluster_state, &cached_clock, &remote_sub_map_arc, &all_pubsub_registries, @@ -1290,7 +1294,7 @@ impl super::Shard { conn_accept::spawn_migrated_monoio_connection( fd, state, &shard_databases, &dispatch_tx, &pubsub_arc, &blocking_rc, - &shutdown, &aof_tx, &tracking_rc, &lua_rc, &script_cache_rc, + &shutdown, &aof_pool, &tracking_rc, &lua_rc, &script_cache_rc, &acl_table, &runtime_config, &server_config, &all_notifiers, &snapshot_trigger_tx, &repl_state, &cluster_state, &cached_clock, &remote_sub_map_arc, &all_pubsub_registries, @@ -1697,7 +1701,7 @@ impl super::Shard { &pubsub_arc, &blocking_rc, &shutdown, - &aof_tx, + &aof_pool, &tracking_rc, &lua_rc, &script_cache_rc, @@ -1739,7 +1743,7 @@ impl super::Shard { &pubsub_arc, &blocking_rc, &shutdown, - &aof_tx, + &aof_pool, &tracking_rc, &lua_rc, &script_cache_rc, @@ -1858,6 +1862,7 @@ impl super::Shard { server_config.graph_merge_max_segments, server_config.graph_dead_edge_trigger, &mut autovacuum_daemon, + aof_pool.as_ref(), // FIX-W1-2 ); }); } else { @@ -1884,6 +1889,7 @@ impl super::Shard { server_config.graph_merge_max_segments, server_config.graph_dead_edge_trigger, &mut autovacuum_daemon, + aof_pool.as_ref(), // FIX-W1-2 ); } if !pending_cdc_subscribes.is_empty() { @@ -1921,7 +1927,7 @@ impl super::Shard { &pubsub_arc, &blocking_rc, &shutdown, - &aof_tx, + &aof_pool, &tracking_rc, &lua_rc, &script_cache_rc, diff --git a/src/shard/mod.rs b/src/shard/mod.rs index f5c509f1..bfc14d17 100644 --- a/src/shard/mod.rs +++ b/src/shard/mod.rs @@ -411,6 +411,7 @@ mod tests { &mut crate::shard::autovacuum::AutovacuumDaemon::new( crate::shard::autovacuum::AutovacuumConfig::default(), ), + None, // aof_pool — None in tests ); // Subscriber now receives pre-serialized RESP bytes @@ -474,6 +475,7 @@ mod tests { &mut crate::shard::autovacuum::AutovacuumDaemon::new( crate::shard::autovacuum::AutovacuumConfig::default(), ), + None, // aof_pool — None in tests ); } diff --git a/src/shard/spsc_handler.rs b/src/shard/spsc_handler.rs index 1097a9b9..57e2514c 100644 --- a/src/shard/spsc_handler.rs +++ b/src/shard/spsc_handler.rs @@ -71,6 +71,9 @@ pub(crate) fn drain_spsc_shared( #[cfg_attr(not(feature = "graph"), allow(unused_variables))] graph_dead_edge_trigger: f64, // MA5: autovacuum daemon reference for RECLAMATION SCHEDULE commands. autovacuum_daemon: &mut crate::shard::autovacuum::AutovacuumDaemon, + // FIX-W1-2: per-shard AOF writer pool. Passed through to handle_shard_message_shared + // so cross-shard writes (MSET/MultiExecute) also land in the per-shard AOF files. + aof_pool: Option<&std::sync::Arc>, ) { const MAX_DRAIN_PER_CYCLE: usize = 256; let mut drained = 0; @@ -175,6 +178,7 @@ pub(crate) fn drain_spsc_shared( graph_merge_max_segments, graph_dead_edge_trigger, autovacuum_daemon, + aof_pool, // FIX-W1-2: thread AOF pool through SPSC drain ); } } @@ -206,6 +210,7 @@ pub(crate) fn drain_spsc_shared( graph_merge_max_segments, graph_dead_edge_trigger, autovacuum_daemon, + aof_pool, // FIX-W1-2: thread AOF pool through SPSC drain ); } } @@ -243,6 +248,9 @@ pub(crate) fn handle_shard_message_shared( #[cfg_attr(not(feature = "graph"), allow(unused_variables))] graph_dead_edge_trigger: f64, // MA5: autovacuum daemon reference for RECLAMATION SCHEDULE commands. autovacuum_daemon: &mut crate::shard::autovacuum::AutovacuumDaemon, + // FIX-W1-2: per-shard AOF writer pool. When Some, each successful write command + // is also routed to the owning shard's AOF file via fire-and-forget try_send_append. + aof_pool: Option<&std::sync::Arc>, ) { match msg { ShardMessage::Execute { @@ -514,6 +522,7 @@ pub(crate) fn handle_shard_message_shared( replica_txs, repl_state, shard_id, + aof_pool, // FIX-W1-2 ); } let _ = reply_tx.send(response); @@ -578,6 +587,7 @@ pub(crate) fn handle_shard_message_shared( replica_txs, repl_state, shard_id, + aof_pool, // FIX-W1-2 ); } let _ = reply_tx.send(response); @@ -618,6 +628,7 @@ pub(crate) fn handle_shard_message_shared( replica_txs, repl_state, shard_id, + aof_pool, // FIX-W1-2 ); } @@ -710,6 +721,7 @@ pub(crate) fn handle_shard_message_shared( replica_txs, repl_state, shard_id, + aof_pool, // FIX-W1-2 ); } @@ -845,6 +857,7 @@ pub(crate) fn handle_shard_message_shared( replica_txs, repl_state, shard_id, + aof_pool, // FIX-W1-2 ); let needs_wake = cmd.eq_ignore_ascii_case(b"LPUSH") @@ -924,6 +937,7 @@ pub(crate) fn handle_shard_message_shared( replica_txs, repl_state, shard_id, + aof_pool, // FIX-W1-2 ); // Wake blocked waiters for producer commands (same as Execute path) @@ -1016,6 +1030,13 @@ pub(crate) fn handle_shard_message_shared( replica_txs, repl_state, shard_id, + // FIX-W1-2 r2: PipelineBatch AOF is written by the + // connection handler coordinator AFTER collecting the + // shard response (handler_monoio/mod.rs:2004, + // handler_sharded/mod.rs:1703). Passing aof_pool here + // would cause a second write to the same shard's AOF + // file, doubling every cross-shard pipeline entry. + None, ); } @@ -1118,6 +1139,12 @@ pub(crate) fn handle_shard_message_shared( replica_txs, repl_state, shard_id, + // FIX-W1-2 r2: PipelineBatch AOF is handled by the + // connection-handler coordinator after collecting the + // shard response (handler_monoio/mod.rs:2004). Passing + // aof_pool here would produce a duplicate AOF entry for + // every cross-shard pipeline command. + None, ); } @@ -1230,6 +1257,7 @@ pub(crate) fn handle_shard_message_shared( replica_txs, repl_state, shard_id, + aof_pool, // FIX-W1-2 ); } @@ -1297,6 +1325,7 @@ pub(crate) fn handle_shard_message_shared( replica_txs, repl_state, shard_id, + aof_pool, // FIX-W1-2 ); } @@ -1389,6 +1418,7 @@ pub(crate) fn handle_shard_message_shared( replica_txs, repl_state, shard_id, + aof_pool, // FIX-W1-2 ); let needs_wake = cmd.eq_ignore_ascii_case(b"LPUSH") @@ -1465,6 +1495,7 @@ pub(crate) fn handle_shard_message_shared( replica_txs, repl_state, shard_id, + aof_pool, // FIX-W1-2 ); let needs_wake = cmd.eq_ignore_ascii_case(b"LPUSH") @@ -1557,6 +1588,12 @@ pub(crate) fn handle_shard_message_shared( replica_txs, repl_state, shard_id, + // FIX-W1-2 r2: PipelineBatchSlotted AOF is written by the + // connection-handler coordinator after collecting the shard + // response (handler_sharded/mod.rs:1703). Passing aof_pool + // here produces a duplicate AOF entry for every cross-shard + // pipeline command (double-write P0 bug). + None, ); } @@ -1655,6 +1692,10 @@ pub(crate) fn handle_shard_message_shared( replica_txs, repl_state, shard_id, + // FIX-W1-2 r2: PipelineBatchSlotted AOF (else branch — pre- + // ShardSlice path) is handled by handler_sharded/mod.rs:1703. + // Passing aof_pool here duplicates the AOF entry. + None, ); } @@ -2338,6 +2379,7 @@ pub(crate) fn handle_shard_message_shared( replica_txs, repl_state, shard_id, + aof_pool, // FIX-W1-2 ); // Perform the in-place swap under ascending-index write locks. @@ -2966,10 +3008,16 @@ pub(crate) fn cow_intercept( } /// Append WAL bytes, update the replication backlog, advance the monotonic shard offset, -/// and fan-out to all connected replica sender channels (non-blocking try_send). +/// fan-out to all connected replica sender channels (non-blocking try_send), and route +/// the entry to the per-shard AOF writer pool when AOF is enabled. /// /// CRITICAL: shard_offset in ReplicationState is SEPARATE from WalWriter::bytes_written. /// WalWriter::bytes_written resets on snapshot truncation; shard_offset NEVER resets. +/// +/// FIX-W1-2: `aof_pool` was added to route MSET/coordinator cross-shard writes +/// through the per-shard AOF pool. The SPSC drain is synchronous so we use +/// `try_send_append` (fire-and-forget). The `appendfsync=always` rendezvous is +/// handled by the connection handler (async context), not here. pub(crate) fn wal_append_and_fanout( data: &[u8], wal_writer: &mut Option, @@ -2978,6 +3026,7 @@ pub(crate) fn wal_append_and_fanout( replica_txs: &[(u64, channel::MpscSender)], repl_state: &Option>>, shard_id: usize, + aof_pool: Option<&std::sync::Arc>, ) { // S3.5b (2026-04-27): hot-path bypass when nothing actually has work. // ARM perf annotate showed `repl_backlog.lock()` (caslb/casab) and @@ -2986,7 +3035,14 @@ pub(crate) fn wal_append_and_fanout( // is fully derivable from the inputs — no flags or shared state needed. // Skipping leaves shard_offset un-advanced; that is fine since with no // WAL and no replicas the offsets are dead bytes (no consumer exists). - if wal_writer.is_none() && wal_v3_writer.is_none() && replica_txs.is_empty() { + // + // FIX-W1-2: also require `aof_pool.is_none()` so that per-shard AOF + // entries are not skipped when WAL/replication are off but AOF is on. + if wal_writer.is_none() + && wal_v3_writer.is_none() + && replica_txs.is_empty() + && aof_pool.is_none() + { return; } // WAL v3 supersedes v2 — skip v2 append when v3 is active to avoid @@ -3025,6 +3081,15 @@ pub(crate) fn wal_append_and_fanout( let _ = tx.try_send(bytes.clone()); } } + // 5. Per-shard AOF pool (FIX-W1-2): route to the owning shard's writer. + // Uses fire-and-forget (`try_send_append`) because this function is sync + // and cannot await the fsync rendezvous. The `appendfsync=always` ack is + // handled by the async connection handler (handler_sharded / handler_single). + // LSN=0 is safe here: per-shard order is preserved by write order; the LSN + // is only meaningful for cross-shard TXN merge (RFC step 5, not yet wired). + if let Some(pool) = aof_pool { + pool.try_send_append(shard_id, 0, bytes::Bytes::copy_from_slice(data)); + } } /// Extract command name and args from a Frame (static helper for SPSC dispatch). @@ -3068,6 +3133,7 @@ mod wal_append_tests { &[], // no replicas &None, // no repl_state 0, + None, // no aof_pool ); let final_end = backlog.lock().as_ref().unwrap().end_offset(); @@ -3095,6 +3161,7 @@ mod wal_append_tests { &replica_txs, &None, 0, + None, // no aof_pool ); let end = backlog.lock().as_ref().unwrap().end_offset(); @@ -3103,4 +3170,120 @@ mod wal_append_tests { "backlog must receive 5 bytes when at least one replica is connected" ); } + + /// FIX-W1-2: When an AofWriterPool is provided, wal_append_and_fanout must + /// route bytes to the pool even when there is no WAL writer and no replicas + /// (S3.5b bypass must NOT trigger when aof_pool is Some). + #[test] + fn test_wal_append_routes_to_aof_pool_when_provided() { + use crate::persistence::aof::{AofMessage, AofWriterPool, FsyncPolicy}; + use crate::runtime::channel::mpsc_bounded; + + let backlog: SharedBacklog = + std::sync::Arc::new(parking_lot::Mutex::new(Some(ReplicationBacklog::new(1024)))); + + // Build a pool backed by a real channel so we can observe what arrives. + let (tx, rx) = mpsc_bounded::(16); + let pool = AofWriterPool::top_level_with_policy(tx, FsyncPolicy::EverySec); + + wal_append_and_fanout( + b"world", + &mut None, // no v2 writer + &mut None, // no v3 writer + &backlog, + &[], // no replicas — S3.5b bypass triggered without pool guard + &None, // no repl_state + 0, // shard_id + Some(&pool), // aof_pool provided — bypass must NOT fire + ); + + // The pool should have received exactly one message. + let msg = rx + .try_recv() + .expect("pool must have received an AOF append"); + match msg { + AofMessage::Append { bytes, .. } => { + assert_eq!( + bytes.as_ref(), + b"world", + "pool must receive the correct bytes" + ); + } + AofMessage::AppendSync { .. } => panic!("expected Append, got AppendSync"), + AofMessage::Rewrite(_) => panic!("expected Append, got Rewrite"), + AofMessage::RewriteSharded(_) => panic!("expected Append, got RewriteSharded"), + AofMessage::Shutdown => panic!("expected Append, got Shutdown"), + } + } + + /// FIX-W1-2 r2: PipelineBatch/PipelineBatchSlotted arms MUST NOT forward + /// writes to the AofWriterPool. The connection-handler coordinator already + /// appends AOF for these arms after collecting the shard response + /// (handler_monoio/mod.rs:2004, handler_sharded/mod.rs:1703). + /// + /// Verify the invariant directly: `wal_append_and_fanout` called with + /// `None` (the PipelineBatch fix) must produce zero messages in the pool + /// channel, while the same call with `Some(&pool)` (the MultiExecute path) + /// must produce exactly one message. + /// + /// Red state (pre-fix): the PipelineBatch arms passed `aof_pool` instead + /// of `None`, so calling this test function using the arm's actual argument + /// would have produced 1 message instead of 0 — the double-write. + #[test] + fn pipeline_batch_arm_passes_none_to_prevent_double_write() { + use crate::persistence::aof::{AofMessage, AofWriterPool, FsyncPolicy}; + use crate::runtime::channel::mpsc_bounded; + + let backlog: SharedBacklog = + std::sync::Arc::new(parking_lot::Mutex::new(Some(ReplicationBacklog::new(1024)))); + + // Build a 2-shard pool so per_shard_with_policy's debug_assert passes. + let (tx0, rx0) = mpsc_bounded::(16); + let (tx1, rx1) = mpsc_bounded::(16); + let pool = AofWriterPool::per_shard_with_policy(vec![tx0, tx1], FsyncPolicy::EverySec); + + // ── PipelineBatch path: caller passes None ── + // Pre-fix this was `aof_pool` (Some), which caused the double-write. + wal_append_and_fanout( + b"*3\r\n$3\r\nSET\r\n$1\r\na\r\n$1\r\n1\r\n", + &mut None, // no v2 writer + &mut None, // no v3 writer + &backlog, + &[], // no replicas + &None, // no repl_state + 0, // shard_id + None, // PipelineBatch fix: None prevents double-write + ); + assert!( + rx0.try_recv().is_err(), + "PipelineBatch must NOT forward to aof_pool (coordinator handles it); \ + a message here means the double-write P0 bug is still present" + ); + assert!( + rx1.try_recv().is_err(), + "shard-1 pool must also be empty for PipelineBatch arm" + ); + + // ── MultiExecute path: caller passes Some(&pool) ── + // This arm has no coordinator-side AOF write, so the pool MUST receive + // the entry (otherwise the per-shard AOF would be silently empty for + // cross-shard MSET/DEL/EXISTS commands). + wal_append_and_fanout( + b"*3\r\n$4\r\nMSET\r\n$1\r\nb\r\n$1\r\n2\r\n", + &mut None, + &mut None, + &backlog, + &[], + &None, + 0, + Some(&pool), // MultiExecute: pool must receive this entry + ); + let msg = rx0 + .try_recv() + .expect("MultiExecute MUST forward to aof_pool; pool is empty — AOF silent drop"); + assert!( + matches!(msg, AofMessage::Append { .. }), + "expected AofMessage::Append from MultiExecute arm, got unexpected variant", + ); + } } diff --git a/tests/adversarial_v0110_fix01_set_delete_rollback.rs b/tests/adversarial_v0110_fix01_set_delete_rollback.rs index 38578700..8a2b8ffb 100644 --- a/tests/adversarial_v0110_fix01_set_delete_rollback.rs +++ b/tests/adversarial_v0110_fix01_set_delete_rollback.rs @@ -118,6 +118,9 @@ async fn start_txn_server(num_shards: usize) -> (u16, CancellationToken) { autovacuum_starvation_cap_secs: 300, vec_warm_mmap_budget: "2gb".to_string(), cold_orphan_sweep_interval_secs: 300, + migrate_aof_from: None, + migrate_aof_to: None, + migrate_aof_shards: 0, }; let cancel = token.clone(); diff --git a/tests/adversarial_v0110_fix02_err_path_intent.rs b/tests/adversarial_v0110_fix02_err_path_intent.rs index f2583efd..50d80e8c 100644 --- a/tests/adversarial_v0110_fix02_err_path_intent.rs +++ b/tests/adversarial_v0110_fix02_err_path_intent.rs @@ -121,6 +121,9 @@ async fn start_txn_server(num_shards: usize) -> (u16, CancellationToken) { autovacuum_starvation_cap_secs: 300, vec_warm_mmap_budget: "2gb".to_string(), cold_orphan_sweep_interval_secs: 300, + migrate_aof_from: None, + migrate_aof_to: None, + migrate_aof_shards: 0, }; let cancel = token.clone(); diff --git a/tests/adversarial_v0110_fix03_simplestring_graph.rs b/tests/adversarial_v0110_fix03_simplestring_graph.rs index 9670bbc5..e9ca1489 100644 --- a/tests/adversarial_v0110_fix03_simplestring_graph.rs +++ b/tests/adversarial_v0110_fix03_simplestring_graph.rs @@ -123,6 +123,9 @@ async fn start_txn_server(num_shards: usize) -> (u16, CancellationToken) { autovacuum_starvation_cap_secs: 300, vec_warm_mmap_budget: "2gb".to_string(), cold_orphan_sweep_interval_secs: 300, + migrate_aof_from: None, + migrate_aof_to: None, + migrate_aof_shards: 0, }; let cancel = token.clone(); diff --git a/tests/adversarial_v0110_fix04_shortest_path_call_parity.rs b/tests/adversarial_v0110_fix04_shortest_path_call_parity.rs index 4b69ba24..27f6da03 100644 --- a/tests/adversarial_v0110_fix04_shortest_path_call_parity.rs +++ b/tests/adversarial_v0110_fix04_shortest_path_call_parity.rs @@ -107,6 +107,9 @@ async fn start_server(num_shards: usize) -> (u16, CancellationToken) { autovacuum_starvation_cap_secs: 300, vec_warm_mmap_budget: "2gb".to_string(), cold_orphan_sweep_interval_secs: 300, + migrate_aof_from: None, + migrate_aof_to: None, + migrate_aof_shards: 0, }; let cancel = token.clone(); diff --git a/tests/adversarial_v0110_fix06_shortest_path_min_hops.rs b/tests/adversarial_v0110_fix06_shortest_path_min_hops.rs index 620755f2..490cec2e 100644 --- a/tests/adversarial_v0110_fix06_shortest_path_min_hops.rs +++ b/tests/adversarial_v0110_fix06_shortest_path_min_hops.rs @@ -112,6 +112,9 @@ async fn start_server(num_shards: usize) -> (u16, CancellationToken) { autovacuum_starvation_cap_secs: 300, vec_warm_mmap_budget: "2gb".to_string(), cold_orphan_sweep_interval_secs: 300, + migrate_aof_from: None, + migrate_aof_to: None, + migrate_aof_shards: 0, }; let cancel = token.clone(); diff --git a/tests/adversarial_v0110_fix07_multihop_edge_var_reject.rs b/tests/adversarial_v0110_fix07_multihop_edge_var_reject.rs index ee3ac6c8..5116a29f 100644 --- a/tests/adversarial_v0110_fix07_multihop_edge_var_reject.rs +++ b/tests/adversarial_v0110_fix07_multihop_edge_var_reject.rs @@ -114,6 +114,9 @@ async fn start_server(num_shards: usize) -> (u16, CancellationToken) { autovacuum_starvation_cap_secs: 300, vec_warm_mmap_budget: "2gb".to_string(), cold_orphan_sweep_interval_secs: 300, + migrate_aof_from: None, + migrate_aof_to: None, + migrate_aof_shards: 0, }; let cancel = token.clone(); diff --git a/tests/aof_fsync_err_subscribe_ordering.rs b/tests/aof_fsync_err_subscribe_ordering.rs new file mode 100644 index 00000000..f0d0ad30 --- /dev/null +++ b/tests/aof_fsync_err_subscribe_ordering.rs @@ -0,0 +1,173 @@ +//! FIX-W2-4 r3 — behavioral test: AOF_FSYNC_ERR canonical constant propagates +//! to the wire under appendfsync=always when a write precedes SUBSCRIBE. +//! +//! This test exercises the SUBSCRIBE ordering fix from handler_single.rs:875-918 +//! (and the parallel paths in handler_monoio / handler_sharded): +//! +//! 1. Connect to a server with `--appendonly yes --appendfsync always`. +//! 2. Issue `SET k v` (a write that must be fsync-ack'd before +OK). +//! 3. Issue `SUBSCRIBE chan` on the same connection immediately after. +//! 4. The writer returns FsyncFailed (injected via MOON_TEST_AOF_FSYNC_FAIL=1). +//! +//! Expected outcome: the first response received by the client is exactly the +//! canonical `ERR AOF fsync failed; write not durable` string — NOT +OK followed +//! by the subscribe confirmation. This proves: +//! (a) The AOF_FSYNC_ERR constant is used (not a hard-coded divergent string). +//! (b) The WRITEFAIL response lands BEFORE the SUBSCRIBE slot (ordering fix). +//! +//! Run: +//! cargo build --release +//! cargo test --release --test aof_fsync_err_subscribe_ordering -- --ignored +//! +//! Requires the release binary at ./target/release/moon. +//! The MOON_TEST_AOF_FSYNC_FAIL=1 env var is passed to the child process. + +use std::io::{BufRead, BufReader, Write}; +use std::net::TcpStream; +use std::path::PathBuf; +use std::process::{Child, Command}; +use std::time::Duration; +use std::{fs, thread}; + +/// Start moon with fault-injected AOF fsync failure. Returns (child, dir). +fn spawn_moon_with_fsync_fail(port: u16, shards: u16) -> (Child, PathBuf) { + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or(0); + let dir = std::env::temp_dir().join(format!( + "moon-w24-subscribe-{}-{}-{}", + std::process::id(), + port, + nanos + )); + fs::create_dir_all(&dir).expect("create temp dir"); + + let stderr_log = dir.join("moon.stderr.log"); + let stdout_log = dir.join("moon.stdout.log"); + + let child = Command::new("./target/release/moon") + .args([ + "--port", + &port.to_string(), + "--shards", + &shards.to_string(), + "--appendonly", + "yes", + "--appendfsync", + "always", + "--dir", + ]) + .arg(&dir) + .env("MOON_TEST_AOF_FSYNC_FAIL", "1") + .env("RUST_LOG", "warn") + .stdout(fs::File::create(&stdout_log).expect("create stdout log")) + .stderr(fs::File::create(&stderr_log).expect("create stderr log")) + .spawn() + .expect("spawn moon — run `cargo build --release` first"); + + (child, dir) +} + +/// Wait until the port is accepting connections, or panic after timeout. +fn wait_for_port(port: u16, timeout: Duration) { + let deadline = std::time::Instant::now() + timeout; + loop { + if TcpStream::connect(("127.0.0.1", port)).is_ok() { + return; + } + if std::time::Instant::now() >= deadline { + panic!("moon did not accept connections on port {port} within {timeout:?}"); + } + thread::sleep(Duration::from_millis(50)); + } +} + +/// Send a raw RESP command over a TCP stream without waiting for a response. +fn send_resp(stream: &mut TcpStream, args: &[&str]) { + let mut buf = format!("*{}\r\n", args.len()); + for arg in args { + buf.push_str(&format!("${}\r\n{}\r\n", arg.len(), arg)); + } + stream + .write_all(buf.as_bytes()) + .expect("write RESP command"); +} + +/// Read one complete RESP response (simple, error, or bulk) from the stream. +/// Returns the raw first line (e.g. "+OK", "-ERR ...", ":3", etc.). +fn read_resp_first_line(reader: &mut BufReader) -> String { + let mut line = String::new(); + reader + .read_line(&mut line) + .expect("read RESP response line"); + line.trim_end_matches("\r\n").to_string() +} + +/// Core assertion: SET + SUBSCRIBE with AOF fault injection must yield +/// the canonical AOF_FSYNC_ERR as the first response, not +OK. +/// +/// Tests both single-shard (handler_single path) and the sharded paths. +fn assert_aof_fsync_err_before_subscribe_ok(port: u16, shards: u16) { + let (mut child, dir) = spawn_moon_with_fsync_fail(port, shards); + + // Give moon up to 5 s to bind the port. + wait_for_port(port, Duration::from_secs(5)); + + let result = std::panic::catch_unwind(|| { + let stream = TcpStream::connect(("127.0.0.1", port)).expect("connect"); + stream.set_read_timeout(Some(Duration::from_secs(3))).ok(); + let stream_clone = stream.try_clone().expect("clone stream"); + let mut reader = BufReader::new(stream_clone); + let mut writer = stream; + + // Pipeline SET then SUBSCRIBE on the same connection. + // Under appendfsync=always the handler must await the fsync ack + // for SET before flushing +OK, and before the SUBSCRIBE response. + send_resp(&mut writer, &["SET", "testkey", "testval"]); + send_resp(&mut writer, &["SUBSCRIBE", "testchan"]); + + // Read the first response — must be the AOF error, NOT +OK. + let first = read_resp_first_line(&mut reader); + + // The canonical constant value: "ERR AOF fsync failed; write not durable" + // A `-` prefix means it's a RESP error frame. + assert!( + first.starts_with('-'), + "first response must be a RESP error frame (starts with '-'); got: {first:?}" + ); + assert!( + first.contains("AOF fsync failed"), + "error frame must contain 'AOF fsync failed' (canonical AOF_FSYNC_ERR); got: {first:?}" + ); + assert!( + first.contains("write not durable"), + "error frame must contain 'write not durable'; got: {first:?}" + ); + }); + + child.kill().ok(); + child.wait().ok(); + fs::remove_dir_all(&dir).ok(); + + if let Err(e) = result { + std::panic::resume_unwind(e); + } +} + +/// Single-shard path (handler_single): SET write followed by SUBSCRIBE. +/// The AOF_FSYNC_ERR must be the first response, proving the WRITEFAIL +/// frame lands before the SUBSCRIBE confirmation slot. +#[test] +#[ignore] +fn aof_fsync_err_propagates_before_subscribe_single_shard() { + assert_aof_fsync_err_before_subscribe_ok(17501, 1); +} + +/// Multi-shard path (handler_sharded): same ordering guarantee under PerShard +/// AOF layout. Uses --shards 2 so the PerShard writer path is exercised. +#[test] +#[ignore] +fn aof_fsync_err_propagates_before_subscribe_multi_shard() { + assert_aof_fsync_err_before_subscribe_ok(17502, 2); +} diff --git a/tests/aof_toplevel_multishard_refusal.rs b/tests/aof_toplevel_multishard_refusal.rs new file mode 100644 index 00000000..daaf9204 --- /dev/null +++ b/tests/aof_toplevel_multishard_refusal.rs @@ -0,0 +1,204 @@ +//! Integration test: Moon refuses to start with a TopLevel AOF manifest and +//! --shards >= 2 (FIX-W2-6 r2). +//! +//! Regression guard for: +//! - Hard refusal (exit code 2) when a v1 TopLevel manifest is detected with +//! --shards >= 2 (the data-loss safety gate). +//! - Correct inclusive range notation in the error message (1..=N-1, not 1..N-1). +//! - Error message refers to the runbook, not a nonexistent CLI flag. +//! +//! Run: +//! cargo build --release +//! cargo test --release --test aof_toplevel_multishard_refusal -- --ignored +//! +//! Requires the release binary at ./target/release/moon. + +use std::fs; +use std::io::Read as _; +use std::path::PathBuf; +use std::process::Command; +use std::time::Duration; + +/// Create a temp dir with a v1 TopLevel AOF manifest (layout: TopLevel). +/// This simulates a pre-PR#129 deployment that still has the legacy single-file +/// layout but is being restarted with --shards 2. +fn setup_toplevel_dir(suffix: &str) -> PathBuf { + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or(0); + let dir = std::env::temp_dir().join(format!( + "moon-w26-refusal-{}-{}-{}", + std::process::id(), + suffix, + nanos + )); + fs::create_dir_all(&dir).expect("create temp dir"); + + // Create appendonlydir/ with a minimal v1 (TopLevel) manifest. + let aof_dir = dir.join("appendonlydir"); + fs::create_dir_all(&aof_dir).expect("create appendonlydir"); + + // Minimal v1 manifest content (no `version` line = TopLevel layout). + // parse_v1 expects lines with prefixes `seq N`, `base `, `incr `. + // The `file` prefix is NOT a valid v1 keyword — parse_v1 would reject it + // with "no valid sequence number". The correct format is: + // seq 1 + // base moon.aof.1.base.rdb + // incr moon.aof.1.incr.aof + let manifest_content = "seq 1\nbase moon.aof.1.base.rdb\nincr moon.aof.1.incr.aof\n"; + fs::write(aof_dir.join("moon.aof.manifest"), manifest_content).expect("write manifest"); + + // Create stub base and incr files so the manifest path check passes. + fs::write(aof_dir.join("moon.aof.1.base.rdb"), b"").expect("write stub base"); + fs::write(aof_dir.join("moon.aof.1.incr.aof"), b"").expect("write stub incr"); + + dir +} + +/// Assert that starting moon with a TopLevel manifest and --shards 2 exits +/// with code 2 and prints a REFUSING TO START message to stderr. +/// +/// Red criterion (pre-fix): the error message contained `migrate-aof --dir` +/// (a nonexistent flag) and the range was printed as `1..1` (exclusive, looks +/// empty for --shards 2). The fix uses the inclusive form `1..=1` and removes +/// the nonexistent flag reference. +#[test] +#[ignore] +fn toplevel_manifest_with_multishard_exits_2_and_prints_refusing_to_start() { + let dir = setup_toplevel_dir("basic"); + let stderr_log = dir.join("moon.stderr.log"); + let stdout_log = dir.join("moon.stdout.log"); + + let mut child = Command::new("./target/release/moon") + .args([ + "--port", + "17399", // high port unlikely to clash + "--shards", + "2", + "--appendonly", + "yes", + "--dir", + ]) + .arg(&dir) + .stdout(fs::File::create(&stdout_log).expect("create stdout log")) + .stderr(fs::File::create(&stderr_log).expect("create stderr log")) + .spawn() + .expect("spawn moon (run `cargo build --release` first)"); + + // Moon should exit quickly (< 5 s) with code 2 — it does not even bind + // the port before the manifest check runs. + let deadline = std::time::Instant::now() + Duration::from_secs(5); + loop { + match child.try_wait().expect("try_wait") { + Some(status) => { + let code = status.code().expect("process terminated by signal"); + assert_eq!( + code, 2, + "expected exit code 2 (REFUSING TO START), got {}", + code + ); + break; + } + None => { + if std::time::Instant::now() >= deadline { + child.kill().ok(); + panic!( + "moon did not exit within 5 s — it should have refused immediately. \ + Check {}", + stderr_log.display() + ); + } + std::thread::sleep(Duration::from_millis(100)); + } + } + } + + // Read stderr and verify key phrases. + let mut stderr_content = String::new(); + fs::File::open(&stderr_log) + .expect("open stderr log") + .read_to_string(&mut stderr_content) + .expect("read stderr log"); + + assert!( + stderr_content.contains("REFUSING TO START"), + "stderr must contain 'REFUSING TO START'; got:\n{}", + stderr_content + ); + + // Post-fix: error message must reference the runbook, not a nonexistent + // CLI flag like `migrate-aof --dir`. + assert!( + stderr_content.contains("multi-shard-aof-rewrite.md"), + "stderr must reference the runbook (multi-shard-aof-rewrite.md); got:\n{}", + stderr_content + ); + + // Post-fix: range must use inclusive notation (1..=N-1 for --shards 2 → 1..=1). + // Pre-fix this was `1..1` (exclusive), which looks like an empty range to operators. + assert!( + stderr_content.contains("1..=1"), + "stderr must use inclusive range '1..=1' for --shards 2; got:\n{}", + stderr_content + ); + + // Cleanup + fs::remove_dir_all(&dir).ok(); +} + +/// Sanity check: --shards 1 with a TopLevel manifest must boot normally (no refusal). +/// This exercises the single-shard compatibility path that must remain unaffected. +#[test] +#[ignore] +fn toplevel_manifest_with_single_shard_is_allowed() { + let dir = setup_toplevel_dir("single"); + let stderr_log = dir.join("moon.stderr.log"); + let stdout_log = dir.join("moon.stdout.log"); + + let mut child = Command::new("./target/release/moon") + .args([ + "--port", + "17400", + "--shards", + "1", + "--appendonly", + "yes", + "--dir", + ]) + .arg(&dir) + .stdout(fs::File::create(&stdout_log).expect("create stdout log")) + .stderr(fs::File::create(&stderr_log).expect("create stderr log")) + .spawn() + .expect("spawn moon"); + + // Give it 3 s to either start or exit. + let deadline = std::time::Instant::now() + Duration::from_secs(3); + loop { + match child.try_wait().expect("try_wait") { + Some(status) => { + let code = status.code().unwrap_or(-1); + // If it exited with code 2 it incorrectly refused a single-shard TopLevel boot. + assert_ne!( + code, + 2, + "Moon must NOT refuse single-shard + TopLevel manifest; got exit 2. \ + stderr: {}", + fs::read_to_string(&stderr_log).unwrap_or_default() + ); + // Any other exit (e.g., port conflict) is fine for this test's purposes. + break; + } + None => { + if std::time::Instant::now() >= deadline { + // Still running — which means it did NOT refuse. That's the correct outcome. + child.kill().ok(); + break; + } + std::thread::sleep(Duration::from_millis(100)); + } + } + } + + fs::remove_dir_all(&dir).ok(); +} diff --git a/tests/crash_matrix_per_shard_aof.rs b/tests/crash_matrix_per_shard_aof.rs new file mode 100644 index 00000000..10d88afc --- /dev/null +++ b/tests/crash_matrix_per_shard_aof.rs @@ -0,0 +1,423 @@ +//! CRASH-01-LITE: per-shard AOF crash-recovery matrix (RFC step 8). +//! +//! Boots a multi-shard moon with `--appendonly yes`, drives a write load, +//! kills the process with SIGKILL, restarts, and asserts the recovered +//! state matches what was on disk pre-crash. Validates the full +//! step 2-5 pipeline end-to-end: +//! +//! handler write → AOF channel → per-shard writer → fsync +//! → SIGKILL → restart → PerShard manifest load → replay_per_shard +//! → ordered merge (empty today) → server accepts client traffic +//! +//! Test matrix (subset of RFC § 7 — "LITE" defers cross-shard TXN and +//! BGREWRITEAOF interleaving to step 9 + future steps): +//! - `--shards 2 --appendonly yes --appendfsync everysec` + SIGKILL +//! → ≥99% recover (everysec fsync window allows ≤1s loss). +//! - `--shards 2 --appendonly yes --appendfsync always` + SIGKILL +//! → 100% recover (every +OK must observe an fsync; H1 closure). +//! +//! Run with (monoio default — matches CI): +//! cargo build --release +//! cargo test --release --test crash_matrix_per_shard_aof -- --ignored +//! +//! Requires: built release binary (default features = runtime-monoio), `redis-cli` on PATH. +//! Crash-recovery is validated on runtime-monoio only. The PerShard AOF manifest +//! initialisation path (initialize_multi) is monoio-gated in main.rs:609; the +//! runtime-tokio binary does not initialise the PerShard manifest on fresh boot +//! and cannot pass crash-recovery validation. + +#![cfg(any(feature = "runtime-monoio", feature = "runtime-tokio"))] + +use std::process::{Child, Command, Stdio}; +use std::time::Duration; + +const KEY_COUNT: usize = 200; + +fn unique_port() -> u16 { + // Ask the OS to assign an available ephemeral port by binding to 0. + // The socket is immediately dropped after reading the port — there is a + // brief TOCTOU window, but it is far safer than the previous pid-modulo + // scheme which collides when multiple cargo test processes run in parallel + // (e.g., CI --test-threads > 1 across feature flag matrix jobs). + use std::net::TcpListener; + let listener = TcpListener::bind("127.0.0.1:0").expect("bind to port 0"); + let port = listener.local_addr().expect("local addr").port(); + drop(listener); + port +} + +fn unique_dir(suffix: &str) -> std::path::PathBuf { + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or(0); + std::env::temp_dir().join(format!( + "moon-crash-matrix-{}-{}-{}", + std::process::id(), + suffix, + nanos + )) +} + +fn start_moon(port: u16, dir: &std::path::Path) -> Child { + start_moon_with_fsync(port, dir, "everysec") +} + +fn start_moon_with_fsync(port: u16, dir: &std::path::Path, fsync: &str) -> Child { + Command::new("./target/release/moon") + .args([ + "--port", + &port.to_string(), + "--shards", + "2", + "--appendonly", + "yes", + "--appendfsync", + fsync, + // No `--unsafe-multishard-aof` — step 9 lifted the gate; this + // test now validates that the default `--shards 2 --appendonly + // yes` launch is crash-safe out of the box. + "--dir", + ]) + .arg(dir) + // Captured to a log file so a CI flake produces a real diagnostic + // rather than the silent "connection refused" symptom the project + // already paid for once (see feedback_silenced_child_stdio_flake). + .stdout(std::fs::File::create(dir.join("moon.stdout.log")).expect("create moon stdout log")) + .stderr(std::fs::File::create(dir.join("moon.stderr.log")).expect("create moon stderr log")) + .spawn() + .expect("spawn moon (run `cargo build --release` with default features first)") +} + +fn wait_for_port(port: u16) { + for _ in 0..80 { + if std::net::TcpStream::connect(format!("127.0.0.1:{}", port)).is_ok() { + std::thread::sleep(Duration::from_millis(200)); + return; + } + std::thread::sleep(Duration::from_millis(100)); + } + panic!("moon did not start within 8s on port {}", port); +} + +fn redis_set(port: u16, key: &str, value: &str) { + let out = Command::new("redis-cli") + .args(["-p", &port.to_string(), "SET", key, value]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .output() + .expect("redis-cli SET"); + assert!( + out.status.success(), + "redis-cli SET {} {} failed: {}", + key, + value, + String::from_utf8_lossy(&out.stderr) + ); +} + +fn redis_get(port: u16, key: &str) -> Option { + let out = Command::new("redis-cli") + .args(["-p", &port.to_string(), "GET", key]) + .output() + .expect("redis-cli GET"); + if !out.status.success() { + return None; + } + let s = String::from_utf8_lossy(&out.stdout).trim().to_string(); + if s.is_empty() || s == "(nil)" { + None + } else { + Some(s) + } +} + +/// Send N RPUSH commands to `key` in a single pipelined TCP write and drain +/// all N integer responses. Useful for double-write detection because +/// RPUSH is non-idempotent: N pushes → LLEN N; double-replay → LLEN 2*N. +fn pipeline_rpush(port: u16, key: &str, n: usize) { + use std::io::{BufRead, BufReader, Write}; + + let mut stream = + std::net::TcpStream::connect(format!("127.0.0.1:{}", port)).expect("connect for pipeline"); + stream.set_read_timeout(Some(Duration::from_secs(10))).ok(); + + // Build one TCP segment with N RPUSH commands (pipeline). + let mut buf: Vec = Vec::with_capacity(n * 64); + for i in 0..n { + let val = format!("v{}", i); + let cmd = format!( + "*3\r\n$5\r\nRPUSH\r\n${}\r\n{}\r\n${}\r\n{}\r\n", + key.len(), + key, + val.len(), + val + ); + buf.extend_from_slice(cmd.as_bytes()); + } + stream.write_all(&buf).expect("pipeline write"); + stream.flush().ok(); + + // Drain all N responses — each is `:N\r\n` (integer reply). + let mut reader = BufReader::new(&stream); + let mut line = String::new(); + for _ in 0..n { + line.clear(); + let _ = reader.read_line(&mut line); + } +} + +/// Query LLEN for `key`; returns -1 on parse failure. +fn redis_llen(port: u16, key: &str) -> i64 { + let out = Command::new("redis-cli") + .args(["-p", &port.to_string(), "LLEN", key]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .output() + .expect("redis-cli LLEN"); + let s = String::from_utf8_lossy(&out.stdout).trim().to_string(); + // redis-cli returns the integer as a bare decimal string (no `:` prefix). + s.parse().unwrap_or(-1) +} + +/// SIGKILL via `kill -9` (Child::kill on Unix already sends SIGKILL but +/// being explicit here documents intent and survives stdlib changes). +#[cfg(unix)] +fn sigkill(child: &mut Child) { + let pid = child.id() as i32; + unsafe { + libc::kill(pid, libc::SIGKILL); + } + // Wait for the kernel to reap the process so its file handles are + // released and the next spawn can lock the AOF files. + let _ = child.wait(); +} + +#[cfg(not(unix))] +fn sigkill(child: &mut Child) { + let _ = child.kill(); + let _ = child.wait(); +} + +#[test] +#[ignore] // Requires built release binary + redis-cli; run explicitly. +fn crash_01_lite_per_shard_aof_recovers_after_sigkill() { + let port = unique_port(); + let dir = unique_dir("crash01"); + std::fs::create_dir_all(&dir).expect("create test dir"); + + // -- Round 1 -------------------------------------------------------- + let mut child = start_moon(port, &dir); + wait_for_port(port); + + // Write KEY_COUNT keys. Use hash tags to deterministically spread + // across both shards (half on each) — confirms per-shard files are + // populated. + let mut expected: std::collections::HashMap = + std::collections::HashMap::with_capacity(KEY_COUNT); + for i in 0..KEY_COUNT { + // Alternate hash tag → shard partition. + let tag = if i % 2 == 0 { "a" } else { "b" }; + let key = format!("crash:{{{}}}:{}", tag, i); + let value = format!("v-{}", i); + redis_set(port, &key, &value); + expected.insert(key, value); + } + + // Wait > 1s so the everysec fsync window definitely flushed every + // entry to durable storage. + std::thread::sleep(Duration::from_millis(1500)); + + // SIGKILL — no graceful shutdown, no chance for in-flight buffers + // to drain. + sigkill(&mut child); + + // -- Round 2 (recovery) --------------------------------------------- + let mut child2 = start_moon(port, &dir); + wait_for_port(port); + + // Verify every key recovered. The everysec contract permits up to 1s + // of loss, but we slept 1.5s before the kill so we should see 100%. + let mut missing: Vec = Vec::new(); + let mut mismatched: Vec = Vec::new(); + for (key, want) in &expected { + match redis_get(port, key) { + None => missing.push(key.clone()), + Some(got) if got != *want => { + mismatched.push(format!("{}: want={} got={}", key, want, got)) + } + Some(_) => {} + } + } + + // Cleanup before any failure assertion so the temp dir isn't leaked + // when the assertion fires. + sigkill(&mut child2); + + assert!( + missing.is_empty() && mismatched.is_empty(), + "CRASH-01-LITE: {} missing, {} mismatched. Sample missing: {:?}, sample mismatched: {:?}", + missing.len(), + mismatched.len(), + missing.iter().take(5).collect::>(), + mismatched.iter().take(5).collect::>(), + ); + + // Successful run — clean up the temp dir. + let _ = std::fs::remove_dir_all(&dir); +} + +/// CRASH-01-LITE-ALWAYS: H1 (in-flight loss) closure. +/// +/// Under `appendfsync=always` the writer must fsync before the +OK is +/// observable by the client. The integration plumbs `AppendSync` through +/// the handlers (handler_monoio / handler_sharded) via +/// `AofWriterPool::try_send_append_durable`, which awaits the writer's +/// ack and converts AOF failure into `Frame::Error` so the client never +/// sees +OK for an entry that was not durable. +/// +/// This test SIGKILLs the server **without** any quiescing sleep — every +/// +OK observed by the client must therefore be backed by an fsync on +/// disk. Any data loss here proves the handler→writer ack handshake is +/// broken. +#[test] +#[ignore] // Requires built release binary + redis-cli; run explicitly. +fn crash_01_lite_always_per_shard_aof_recovers_after_sigkill() { + // Offset port so this test never collides with the everysec test + // when both run on the same dev host. + let port = unique_port().saturating_add(1); + let dir = unique_dir("crash01-always"); + std::fs::create_dir_all(&dir).expect("create test dir"); + + // -- Round 1 -------------------------------------------------------- + let mut child = start_moon_with_fsync(port, &dir, "always"); + wait_for_port(port); + + let mut expected: std::collections::HashMap = + std::collections::HashMap::with_capacity(KEY_COUNT); + for i in 0..KEY_COUNT { + let tag = if i % 2 == 0 { "a" } else { "b" }; + let key = format!("crash:{{{}}}:{}", tag, i); + let value = format!("v-{}", i); + // SET only returns +OK after the writer fsyncs under Always. + redis_set(port, &key, &value); + expected.insert(key, value); + } + + // NO quiescing sleep — H1 contract is that each +OK already saw fsync. + sigkill(&mut child); + + // -- Round 2 (recovery) --------------------------------------------- + let mut child2 = start_moon_with_fsync(port, &dir, "always"); + wait_for_port(port); + + let mut missing: Vec = Vec::new(); + let mut mismatched: Vec = Vec::new(); + for (key, want) in &expected { + match redis_get(port, key) { + None => missing.push(key.clone()), + Some(got) if got != *want => { + mismatched.push(format!("{}: want={} got={}", key, want, got)) + } + Some(_) => {} + } + } + + sigkill(&mut child2); + + assert!( + missing.is_empty() && mismatched.is_empty(), + "CRASH-01-LITE-ALWAYS: {} missing, {} mismatched. Sample missing: {:?}, sample mismatched: {:?}", + missing.len(), + mismatched.len(), + missing.iter().take(5).collect::>(), + mismatched.iter().take(5).collect::>(), + ); + + let _ = std::fs::remove_dir_all(&dir); +} + +/// PIPELINE-DOUBLE-WRITE: FIX-W1-2 discriminating regression test. +/// +/// Before the fix, `wal_append_and_fanout` was called with `aof_pool` inside +/// the `PipelineBatch` / `PipelineBatchSlotted` SPSC arms. The connection- +/// handler coordinator **also** writes the AOF entry after collecting each +/// shard response (handler_sharded/mod.rs:1703 / handler_monoio/mod.rs:2004). +/// The net effect was every cross-shard pipelined command written TWICE to +/// the target shard's AOF file. On recovery, the replay doubled the logical +/// effect of every write. +/// +/// `RPUSH` is non-idempotent: N pushes → LLEN N. +/// Double-replay → LLEN 2*N. This makes over-count directly observable +/// after crash+recovery — no AOF-binary inspection needed. +/// +/// Shard routing (CRC16 mod 2): +/// `{a}` → shard 1 | `{b}` → shard 0 +/// +/// A connection assigned to shard 0 routes `{a}` commands cross-shard +/// (PipelineBatchSlotted). A connection on shard 1 routes `{b}` commands +/// cross-shard. By pipelining BOTH sets in one call this test exercises the +/// PipelineBatchSlotted path regardless of which shard the OS assigns to the +/// connection (SO_REUSEPORT is non-deterministic at test time). +/// +/// **Red state (commit before 124cae2):** LLEN list{a} and/or list{b} == 2*N +/// — the SPSC-side duplicate write survives the fsync window and is replayed. +/// **Green state (post-fix):** both LLENs == N exactly. +#[test] +#[ignore] // Requires built release binary + redis-cli; run explicitly. +fn pipeline_batch_no_double_write_after_crash_recovery() { + const N: usize = 20; + + let port = unique_port().saturating_add(2); + let dir = unique_dir("pipeline-dbl"); + std::fs::create_dir_all(&dir).expect("create test dir"); + + // -- Round 1 -------------------------------------------------------- + let mut child = start_moon(port, &dir); + wait_for_port(port); + + // Pipeline N RPUSHes to list{a} (→ shard 1) and N to list{b} (→ shard 0) + // in two separate pipelined bursts. Each list should end up with exactly + // N elements after a crash+recovery cycle. + pipeline_rpush(port, "list{a}", N); + pipeline_rpush(port, "list{b}", N); + + // Wait > 1s so the everysec fsync window flushed all entries. + std::thread::sleep(Duration::from_millis(1500)); + + sigkill(&mut child); + + // -- Round 2 (recovery) --------------------------------------------- + let mut child2 = start_moon(port, &dir); + wait_for_port(port); + + let llen_a = redis_llen(port, "list{a}"); + let llen_b = redis_llen(port, "list{b}"); + + sigkill(&mut child2); + + // Failure message identifies which list was doubled and the expected count + // so the root cause is unambiguous in CI output. + assert_eq!( + llen_a, + N as i64, + "PIPELINE-DOUBLE-WRITE: list{{a}} LLEN after crash+recovery = {} (expected {}). \ + A value of {} indicates the PipelineBatchSlotted SPSC arm is still passing \ + aof_pool to wal_append_and_fanout, causing duplicate AOF entries (FIX-W1-2).", + llen_a, + N, + 2 * N as i64, + ); + assert_eq!( + llen_b, + N as i64, + "PIPELINE-DOUBLE-WRITE: list{{b}} LLEN after crash+recovery = {} (expected {}). \ + A value of {} indicates the PipelineBatchSlotted SPSC arm is still passing \ + aof_pool to wal_append_and_fanout, causing duplicate AOF entries (FIX-W1-2).", + llen_b, + N, + 2 * N as i64, + ); + + let _ = std::fs::remove_dir_all(&dir); +} diff --git a/tests/ft_search_as_of_boundary.rs b/tests/ft_search_as_of_boundary.rs index 80f70c1a..abcd60b1 100644 --- a/tests/ft_search_as_of_boundary.rs +++ b/tests/ft_search_as_of_boundary.rs @@ -106,6 +106,9 @@ fn build_config(port: u16, num_shards: usize) -> ServerConfig { autovacuum_starvation_cap_secs: 300, vec_warm_mmap_budget: "2gb".to_string(), cold_orphan_sweep_interval_secs: 300, + migrate_aof_from: None, + migrate_aof_to: None, + migrate_aof_shards: 0, } } diff --git a/tests/ft_search_as_of_filter.rs b/tests/ft_search_as_of_filter.rs index 1f1de273..01126e3e 100644 --- a/tests/ft_search_as_of_filter.rs +++ b/tests/ft_search_as_of_filter.rs @@ -114,6 +114,9 @@ fn build_config(port: u16, num_shards: usize) -> ServerConfig { autovacuum_starvation_cap_secs: 300, vec_warm_mmap_budget: "2gb".to_string(), cold_orphan_sweep_interval_secs: 300, + migrate_aof_from: None, + migrate_aof_to: None, + migrate_aof_shards: 0, } } diff --git a/tests/ft_search_concurrent_readers.rs b/tests/ft_search_concurrent_readers.rs index 1563064f..d1848d32 100644 --- a/tests/ft_search_concurrent_readers.rs +++ b/tests/ft_search_concurrent_readers.rs @@ -103,6 +103,9 @@ fn build_config(port: u16, num_shards: usize) -> ServerConfig { autovacuum_starvation_cap_secs: 300, vec_warm_mmap_budget: "2gb".to_string(), cold_orphan_sweep_interval_secs: 300, + migrate_aof_from: None, + migrate_aof_to: None, + migrate_aof_shards: 0, } } diff --git a/tests/ft_search_multi_shard_as_of.rs b/tests/ft_search_multi_shard_as_of.rs index c8f8fef1..f3fc89d1 100644 --- a/tests/ft_search_multi_shard_as_of.rs +++ b/tests/ft_search_multi_shard_as_of.rs @@ -46,6 +46,7 @@ fn build_config(port: u16, num_shards: usize) -> ServerConfig { databases: 16, requirepass: None, appendonly: "no".to_string(), + unsafe_multishard_aof: false, appendfsync: "everysec".to_string(), save: None, dir: ".".to_string(), @@ -116,6 +117,9 @@ fn build_config(port: u16, num_shards: usize) -> ServerConfig { autovacuum_starvation_cap_secs: 300, vec_warm_mmap_budget: "2gb".to_string(), cold_orphan_sweep_interval_secs: 300, + migrate_aof_from: None, + migrate_aof_to: None, + migrate_aof_shards: 0, } } diff --git a/tests/ft_search_temporal_parity.rs b/tests/ft_search_temporal_parity.rs index 065de5f9..7bd34aea 100644 --- a/tests/ft_search_temporal_parity.rs +++ b/tests/ft_search_temporal_parity.rs @@ -58,6 +58,7 @@ fn build_config(port: u16, num_shards: usize) -> ServerConfig { databases: 16, requirepass: None, appendonly: "no".to_string(), + unsafe_multishard_aof: false, appendfsync: "everysec".to_string(), save: None, dir: ".".to_string(), @@ -128,6 +129,9 @@ fn build_config(port: u16, num_shards: usize) -> ServerConfig { autovacuum_starvation_cap_secs: 300, vec_warm_mmap_budget: "2gb".to_string(), cold_orphan_sweep_interval_secs: 300, + migrate_aof_from: None, + migrate_aof_to: None, + migrate_aof_shards: 0, } } diff --git a/tests/graph_bench_compare.rs b/tests/graph_bench_compare.rs index 72835b46..789ed608 100644 --- a/tests/graph_bench_compare.rs +++ b/tests/graph_bench_compare.rs @@ -103,6 +103,9 @@ async fn start_moon() -> (u16, CancellationToken) { autovacuum_starvation_cap_secs: 300, vec_warm_mmap_budget: "2gb".to_string(), cold_orphan_sweep_interval_secs: 300, + migrate_aof_from: None, + migrate_aof_to: None, + migrate_aof_shards: 0, }; tokio::spawn(async move { diff --git a/tests/graph_bench_e2e.rs b/tests/graph_bench_e2e.rs index 1f13338c..136137eb 100644 --- a/tests/graph_bench_e2e.rs +++ b/tests/graph_bench_e2e.rs @@ -89,6 +89,9 @@ async fn start_server() -> (u16, CancellationToken) { autovacuum_starvation_cap_secs: 300, vec_warm_mmap_budget: "2gb".to_string(), cold_orphan_sweep_interval_secs: 300, + migrate_aof_from: None, + migrate_aof_to: None, + migrate_aof_shards: 0, }; tokio::spawn(async move { diff --git a/tests/graph_integration.rs b/tests/graph_integration.rs index 1834e92b..0afd3df3 100644 --- a/tests/graph_integration.rs +++ b/tests/graph_integration.rs @@ -89,6 +89,9 @@ async fn start_graph_server() -> (u16, CancellationToken) { autovacuum_starvation_cap_secs: 300, vec_warm_mmap_budget: "2gb".to_string(), cold_orphan_sweep_interval_secs: 300, + migrate_aof_from: None, + migrate_aof_to: None, + migrate_aof_shards: 0, }; tokio::spawn(async move { diff --git a/tests/graph_stress_deep.rs b/tests/graph_stress_deep.rs index 40da215a..9e30032c 100644 --- a/tests/graph_stress_deep.rs +++ b/tests/graph_stress_deep.rs @@ -92,6 +92,9 @@ async fn start_server() -> (u16, CancellationToken) { autovacuum_starvation_cap_secs: 300, vec_warm_mmap_budget: "2gb".to_string(), cold_orphan_sweep_interval_secs: 300, + migrate_aof_from: None, + migrate_aof_to: None, + migrate_aof_shards: 0, }; tokio::spawn(async move { diff --git a/tests/integration.rs b/tests/integration.rs index 1e6b51c8..9b10d892 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -32,6 +32,7 @@ async fn start_server() -> (u16, CancellationToken) { databases: 16, requirepass: None, appendonly: "no".to_string(), + unsafe_multishard_aof: false, appendfsync: "everysec".to_string(), save: None, dir: ".".to_string(), @@ -102,6 +103,9 @@ async fn start_server() -> (u16, CancellationToken) { autovacuum_starvation_cap_secs: 300, vec_warm_mmap_budget: "2gb".to_string(), cold_orphan_sweep_interval_secs: 300, + migrate_aof_from: None, + migrate_aof_to: None, + migrate_aof_shards: 0, }; tokio::spawn(async move { @@ -131,6 +135,7 @@ async fn start_server_with_pass(password: &str) -> (u16, CancellationToken) { databases: 16, requirepass: Some(password.to_string()), appendonly: "no".to_string(), + unsafe_multishard_aof: false, appendfsync: "everysec".to_string(), save: None, dir: ".".to_string(), @@ -201,6 +206,9 @@ async fn start_server_with_pass(password: &str) -> (u16, CancellationToken) { autovacuum_starvation_cap_secs: 300, vec_warm_mmap_budget: "2gb".to_string(), cold_orphan_sweep_interval_secs: 300, + migrate_aof_from: None, + migrate_aof_to: None, + migrate_aof_shards: 0, }; tokio::spawn(async move { @@ -1302,6 +1310,7 @@ async fn start_server_with_persistence( databases: 16, requirepass: None, appendonly: appendonly.to_string(), + unsafe_multishard_aof: false, appendfsync: appendfsync.to_string(), save: None, dir: dir.to_string_lossy().to_string(), @@ -1372,6 +1381,9 @@ async fn start_server_with_persistence( autovacuum_starvation_cap_secs: 300, vec_warm_mmap_budget: "2gb".to_string(), cold_orphan_sweep_interval_secs: 300, + migrate_aof_from: None, + migrate_aof_to: None, + migrate_aof_shards: 0, }; tokio::spawn(async move { @@ -2185,6 +2197,7 @@ async fn start_server_with_maxmemory(maxmemory: usize, policy: &str) -> (u16, Ca databases: 16, requirepass: None, appendonly: "no".to_string(), + unsafe_multishard_aof: false, appendfsync: "everysec".to_string(), save: None, dir: ".".to_string(), @@ -2255,6 +2268,9 @@ async fn start_server_with_maxmemory(maxmemory: usize, policy: &str) -> (u16, Ca autovacuum_starvation_cap_secs: 300, vec_warm_mmap_budget: "2gb".to_string(), cold_orphan_sweep_interval_secs: 300, + migrate_aof_from: None, + migrate_aof_to: None, + migrate_aof_shards: 0, }; tokio::spawn(async move { @@ -2595,6 +2611,7 @@ async fn start_sharded_server(num_shards: usize) -> (u16, CancellationToken) { databases: 16, requirepass: None, appendonly: "no".to_string(), + unsafe_multishard_aof: false, appendfsync: "everysec".to_string(), save: None, dir: ".".to_string(), @@ -2665,6 +2682,9 @@ async fn start_sharded_server(num_shards: usize) -> (u16, CancellationToken) { autovacuum_starvation_cap_secs: 300, vec_warm_mmap_budget: "2gb".to_string(), cold_orphan_sweep_interval_secs: 300, + migrate_aof_from: None, + migrate_aof_to: None, + migrate_aof_shards: 0, }; let cancel = token.clone(); @@ -3785,6 +3805,7 @@ async fn start_cluster_server() -> (u16, CancellationToken) { databases: 16, requirepass: None, appendonly: "no".to_string(), + unsafe_multishard_aof: false, appendfsync: "everysec".to_string(), save: None, dir: ".".to_string(), @@ -3855,6 +3876,9 @@ async fn start_cluster_server() -> (u16, CancellationToken) { autovacuum_starvation_cap_secs: 300, vec_warm_mmap_budget: "2gb".to_string(), cold_orphan_sweep_interval_secs: 300, + migrate_aof_from: None, + migrate_aof_to: None, + migrate_aof_shards: 0, }; std::thread::spawn(move || { @@ -4446,6 +4470,7 @@ async fn start_server_with_aclfile(acl_path: &str) -> (u16, CancellationToken) { databases: 16, requirepass: None, appendonly: "no".to_string(), + unsafe_multishard_aof: false, appendfsync: "everysec".to_string(), save: None, dir: ".".to_string(), @@ -4516,6 +4541,9 @@ async fn start_server_with_aclfile(acl_path: &str) -> (u16, CancellationToken) { autovacuum_starvation_cap_secs: 300, vec_warm_mmap_budget: "2gb".to_string(), cold_orphan_sweep_interval_secs: 300, + migrate_aof_from: None, + migrate_aof_to: None, + migrate_aof_shards: 0, }; tokio::spawn(async move { diff --git a/tests/kill_snapshot.rs b/tests/kill_snapshot.rs index 0a96c81f..51f870f5 100644 --- a/tests/kill_snapshot.rs +++ b/tests/kill_snapshot.rs @@ -28,6 +28,7 @@ fn base_config(port: u16, num_shards: usize) -> ServerConfig { databases: 1, requirepass: None, appendonly: "no".to_string(), + unsafe_multishard_aof: false, appendfsync: "no".to_string(), save: None, dir: ".".to_string(), @@ -98,6 +99,9 @@ fn base_config(port: u16, num_shards: usize) -> ServerConfig { graph_dead_edge_trigger: 0.20, autovacuum_starvation_cap_secs: 300, cold_orphan_sweep_interval_secs: 300, + migrate_aof_from: None, + migrate_aof_to: None, + migrate_aof_shards: 0, vec_warm_mmap_budget: "2gb".to_string(), } } diff --git a/tests/lunaris_cypher_shortest_path.rs b/tests/lunaris_cypher_shortest_path.rs index 5cc92e6b..86ba146f 100644 --- a/tests/lunaris_cypher_shortest_path.rs +++ b/tests/lunaris_cypher_shortest_path.rs @@ -155,6 +155,9 @@ fn build_config(port: u16, num_shards: usize) -> ServerConfig { autovacuum_starvation_cap_secs: 300, vec_warm_mmap_budget: "2gb".to_string(), cold_orphan_sweep_interval_secs: 300, + migrate_aof_from: None, + migrate_aof_to: None, + migrate_aof_shards: 0, } } diff --git a/tests/lunaris_cypher_temporal.rs b/tests/lunaris_cypher_temporal.rs index 9c34ab9e..345da2cb 100644 --- a/tests/lunaris_cypher_temporal.rs +++ b/tests/lunaris_cypher_temporal.rs @@ -164,6 +164,9 @@ fn build_config(port: u16, num_shards: usize) -> ServerConfig { autovacuum_starvation_cap_secs: 300, vec_warm_mmap_budget: "2gb".to_string(), cold_orphan_sweep_interval_secs: 300, + migrate_aof_from: None, + migrate_aof_to: None, + migrate_aof_shards: 0, } } diff --git a/tests/lunaris_hybrid_ft_search.rs b/tests/lunaris_hybrid_ft_search.rs index 28c1d164..dff7e88b 100644 --- a/tests/lunaris_hybrid_ft_search.rs +++ b/tests/lunaris_hybrid_ft_search.rs @@ -142,6 +142,9 @@ fn build_config(port: u16, num_shards: usize) -> ServerConfig { autovacuum_starvation_cap_secs: 300, vec_warm_mmap_budget: "2gb".to_string(), cold_orphan_sweep_interval_secs: 300, + migrate_aof_from: None, + migrate_aof_to: None, + migrate_aof_shards: 0, } } diff --git a/tests/mq_integration.rs b/tests/mq_integration.rs index 6423a8e8..49b9da57 100644 --- a/tests/mq_integration.rs +++ b/tests/mq_integration.rs @@ -44,6 +44,7 @@ async fn start_mq_server(num_shards: usize) -> (u16, CancellationToken) { databases: 16, requirepass: None, appendonly: "no".to_string(), + unsafe_multishard_aof: false, appendfsync: "everysec".to_string(), save: None, dir: ".".to_string(), @@ -114,6 +115,9 @@ async fn start_mq_server(num_shards: usize) -> (u16, CancellationToken) { autovacuum_starvation_cap_secs: 300, vec_warm_mmap_budget: "2gb".to_string(), cold_orphan_sweep_interval_secs: 300, + migrate_aof_from: None, + migrate_aof_to: None, + migrate_aof_shards: 0, }; let cancel = token.clone(); diff --git a/tests/pipeline_auto_index.rs b/tests/pipeline_auto_index.rs index c3d089d6..8d20fa71 100644 --- a/tests/pipeline_auto_index.rs +++ b/tests/pipeline_auto_index.rs @@ -112,6 +112,9 @@ fn build_config(port: u16, num_shards: usize) -> ServerConfig { autovacuum_starvation_cap_secs: 300, vec_warm_mmap_budget: "2gb".to_string(), cold_orphan_sweep_interval_secs: 300, + migrate_aof_from: None, + migrate_aof_to: None, + migrate_aof_shards: 0, } } diff --git a/tests/replication_test.rs b/tests/replication_test.rs index 3e36686c..55e1ebee 100644 --- a/tests/replication_test.rs +++ b/tests/replication_test.rs @@ -30,6 +30,7 @@ async fn start_server() -> (u16, CancellationToken) { databases: 16, requirepass: None, appendonly: "no".to_string(), + unsafe_multishard_aof: false, appendfsync: "everysec".to_string(), save: None, dir: dir_path, @@ -100,6 +101,9 @@ async fn start_server() -> (u16, CancellationToken) { autovacuum_starvation_cap_secs: 300, vec_warm_mmap_budget: "2gb".to_string(), cold_orphan_sweep_interval_secs: 300, + migrate_aof_from: None, + migrate_aof_to: None, + migrate_aof_shards: 0, }; tokio::spawn(async move { diff --git a/tests/txn_completeness_edge_cases.rs b/tests/txn_completeness_edge_cases.rs index a19ec75d..9199736e 100644 --- a/tests/txn_completeness_edge_cases.rs +++ b/tests/txn_completeness_edge_cases.rs @@ -115,6 +115,9 @@ fn build_config(port: u16, num_shards: usize) -> ServerConfig { autovacuum_starvation_cap_secs: 300, vec_warm_mmap_budget: "2gb".to_string(), cold_orphan_sweep_interval_secs: 300, + migrate_aof_from: None, + migrate_aof_to: None, + migrate_aof_shards: 0, } } diff --git a/tests/txn_cypher_write_rollback.rs b/tests/txn_cypher_write_rollback.rs index 1e26794f..fe108f86 100644 --- a/tests/txn_cypher_write_rollback.rs +++ b/tests/txn_cypher_write_rollback.rs @@ -127,6 +127,9 @@ async fn start_txn_server(num_shards: usize, persistence_dir: &str) -> (u16, Can autovacuum_starvation_cap_secs: 300, vec_warm_mmap_budget: "2gb".to_string(), cold_orphan_sweep_interval_secs: 300, + migrate_aof_from: None, + migrate_aof_to: None, + migrate_aof_shards: 0, }; let cancel = token.clone(); diff --git a/tests/txn_ft_search_snapshot.rs b/tests/txn_ft_search_snapshot.rs index f442afe1..f20b63a1 100644 --- a/tests/txn_ft_search_snapshot.rs +++ b/tests/txn_ft_search_snapshot.rs @@ -46,6 +46,7 @@ fn build_config(port: u16, num_shards: usize) -> ServerConfig { databases: 16, requirepass: None, appendonly: "no".to_string(), + unsafe_multishard_aof: false, appendfsync: "everysec".to_string(), save: None, dir: ".".to_string(), @@ -116,6 +117,9 @@ fn build_config(port: u16, num_shards: usize) -> ServerConfig { autovacuum_starvation_cap_secs: 300, vec_warm_mmap_budget: "2gb".to_string(), cold_orphan_sweep_interval_secs: 300, + migrate_aof_from: None, + migrate_aof_to: None, + migrate_aof_shards: 0, } } diff --git a/tests/txn_graph_wiring.rs b/tests/txn_graph_wiring.rs index f969f0e7..a2d80fec 100644 --- a/tests/txn_graph_wiring.rs +++ b/tests/txn_graph_wiring.rs @@ -137,6 +137,9 @@ async fn start_txn_server(num_shards: usize, persistence_dir: &str) -> (u16, Can autovacuum_starvation_cap_secs: 300, vec_warm_mmap_budget: "2gb".to_string(), cold_orphan_sweep_interval_secs: 300, + migrate_aof_from: None, + migrate_aof_to: None, + migrate_aof_shards: 0, }; let cancel = token.clone(); diff --git a/tests/txn_kv_wiring.rs b/tests/txn_kv_wiring.rs index 4a7ec97c..3a04ff14 100644 --- a/tests/txn_kv_wiring.rs +++ b/tests/txn_kv_wiring.rs @@ -49,6 +49,7 @@ async fn start_txn_server(num_shards: usize, persistence_dir: &str) -> (u16, Can databases: 16, requirepass: None, appendonly, + unsafe_multishard_aof: false, appendfsync: "everysec".to_string(), save: None, dir, @@ -119,6 +120,9 @@ async fn start_txn_server(num_shards: usize, persistence_dir: &str) -> (u16, Can autovacuum_starvation_cap_secs: 300, vec_warm_mmap_budget: "2gb".to_string(), cold_orphan_sweep_interval_secs: 300, + migrate_aof_from: None, + migrate_aof_to: None, + migrate_aof_shards: 0, }; let cancel = token.clone(); diff --git a/tests/vacuum_commands.rs b/tests/vacuum_commands.rs index 4b37fc59..86260b22 100644 --- a/tests/vacuum_commands.rs +++ b/tests/vacuum_commands.rs @@ -35,6 +35,7 @@ fn base_config(port: u16) -> ServerConfig { databases: 1, requirepass: None, appendonly: "no".to_string(), + unsafe_multishard_aof: false, appendfsync: "no".to_string(), save: None, dir: ".".to_string(), @@ -104,6 +105,9 @@ fn base_config(port: u16) -> ServerConfig { graph_dead_edge_trigger: 0.20, autovacuum_starvation_cap_secs: 300, cold_orphan_sweep_interval_secs: 300, + migrate_aof_from: None, + migrate_aof_to: None, + migrate_aof_shards: 0, vec_warm_mmap_budget: "2gb".to_string(), } } diff --git a/tests/workspace_integration.rs b/tests/workspace_integration.rs index fb833edd..a32fe0c6 100644 --- a/tests/workspace_integration.rs +++ b/tests/workspace_integration.rs @@ -37,6 +37,7 @@ async fn start_workspace_server(num_shards: usize) -> (u16, CancellationToken) { databases: 16, requirepass: None, appendonly: "no".to_string(), + unsafe_multishard_aof: false, appendfsync: "everysec".to_string(), save: None, dir: ".".to_string(), @@ -107,6 +108,9 @@ async fn start_workspace_server(num_shards: usize) -> (u16, CancellationToken) { autovacuum_starvation_cap_secs: 300, vec_warm_mmap_budget: "2gb".to_string(), cold_orphan_sweep_interval_secs: 300, + migrate_aof_from: None, + migrate_aof_to: None, + migrate_aof_shards: 0, }; let cancel = token.clone(); @@ -254,6 +258,7 @@ async fn start_workspace_server_with_auth( databases: 16, requirepass: Some(password), appendonly: "no".to_string(), + unsafe_multishard_aof: false, appendfsync: "everysec".to_string(), save: None, dir: ".".to_string(), @@ -324,6 +329,9 @@ async fn start_workspace_server_with_auth( autovacuum_starvation_cap_secs: 300, vec_warm_mmap_budget: "2gb".to_string(), cold_orphan_sweep_interval_secs: 300, + migrate_aof_from: None, + migrate_aof_to: None, + migrate_aof_shards: 0, }; let cancel = token.clone();