From ef577e47aec84763069e578dcf0d51a3130cdd4f Mon Sep 17 00:00:00 2001 From: Ray Walker Date: Sat, 25 Jul 2026 01:55:43 +1000 Subject: [PATCH 1/4] docs(protocol): specify File backend format (LAB-430) Co-authored-by: multica-agent --- .github/workflows/verify.yml | 3 +++ README.md | 1 + sdk-feature-matrix.md | 17 +++++++------- spec/file-backend-format.md | 38 +++++++++++++++++++++++++++++++ test-vectors/file-backend.json | 40 +++++++++++++++++++++++++++++++++ tools/file-backend-reference.py | 40 +++++++++++++++++++++++++++++++++ 6 files changed, 131 insertions(+), 8 deletions(-) create mode 100644 spec/file-backend-format.md create mode 100644 test-vectors/file-backend.json create mode 100644 tools/file-backend-reference.py diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index f19aafd..03f87b9 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -45,5 +45,8 @@ jobs: - name: Python-frame verify (stdlib only) run: python3 tools/python-frame-reference.py verify + - name: File-backend format verify (stdlib only) + run: python3 tools/file-backend-reference.py + - name: Python-frame JS cross-check (zero-dep independent reader, full round-trip) run: node tools/frame-crosscheck.mjs diff --git a/README.md b/README.md index ef68fc7..fe20fed 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,7 @@ layer's own store/retrieve flows are specified in | [spec/encryption.md](spec/encryption.md) | AES-256-GCM encryption, HKDF-SHA256 key derivation, AAD v0x03, counter-based nonces, key rotation | | [spec/saas-api.md](spec/saas-api.md) | REST API endpoints, binary wire protocol, error codes, metrics headers | | [spec/interop-mode.md](spec/interop-mode.md) | Cross-SDK cache sharing — language-neutral key format, canonical argument normalization *(specified, not yet implemented in any SDK)* | +| [spec/file-backend-format.md](spec/file-backend-format.md) | Shared local File backend filename, header, expiry, and fail-closed flag negotiation | | [sdk-feature-matrix.md](sdk-feature-matrix.md) | Feature parity tracking across Python, Rust, TypeScript, and PHP SDKs | | [decisions/key-rotation.md](decisions/key-rotation.md) | Decision records — master-key rotation via client-side keyring (rationale, rejected options, operator runbooks) | diff --git a/sdk-feature-matrix.md b/sdk-feature-matrix.md index 1b48eac..c1fc653 100644 --- a/sdk-feature-matrix.md +++ b/sdk-feature-matrix.md @@ -6,7 +6,7 @@ **Feature parity and compliance status across all CacheKit SDK implementations.** -*Last updated: 2026-07-24 — LAB-446: Python File backend gains full TTL inspection/refresh; Memcached gains `refresh_ttl` (touch) only (see [TTL management note](#reliability-features)). LAB-595 shipped: ts Cloudflare Workers flipped ❌ → ✅ via the `@cachekit-io/cachekit/workers` entrypoint on a wasm32 cachekit-core build (~55 KB gz measured); footnote ¹ records the phase-1 surface and semantics deltas. LAB-519: ts cold-miss single-flight (in-process, always on) + LockableBackend wired into `wrap()`'s miss path (opt-in); ts backpressure decision recorded; ts Redis lock/TTL capability cells refreshed for LAB-427.* +*Last updated: 2026-07-25 — LAB-430 shipped TypeScript Node-only Memcached and File backends; the protocol-owned File format and vectors now define fail-closed flag negotiation. LAB-446: Python File backend gains full TTL inspection/refresh; Memcached gains `refresh_ttl` (touch) only (see [TTL management note](#reliability-features)). LAB-595 shipped: ts Cloudflare Workers flipped ❌ → ✅ via the `@cachekit-io/cachekit/workers` entrypoint on a wasm32 cachekit-core build (~55 KB gz measured); footnote ¹ records the phase-1 surface and semantics deltas. LAB-519: ts cold-miss single-flight (in-process, always on) + LockableBackend wired into `wrap()`'s miss path (opt-in); ts backpressure decision recorded; ts Redis lock/TTL capability cells refreshed for LAB-427.* @@ -72,8 +72,8 @@ | Backend | Python | Rust | TypeScript | PHP | | :--- | :---: | :---: | :---: | :---: | | Redis (direct) | ✅ `backends/redis` (redis-py, required dep) | ✅ `redis` feature (fred) | ✅ `backends/redis.ts` (ioredis) | ❌ | -| Memcached | ✅ `backends/memcached` (`memcached` extra) | ✅ `memcached` feature (rust-memcache)³ | ❌ | ❌ | -| File (local) | ✅ `backends/file` (stdlib + mmap) | ✅ `file` feature (byte-compatible with py)³ | ❌ | ❌ | +| Memcached | ✅ `backends/memcached` (`memcached` extra) | ✅ `memcached` feature (rust-memcache)³ | ✅ `backends/memcached.ts` (optional `memjs` peer; Node-only)⁴ | ❌ | +| File (local) | ✅ `backends/file` (stdlib + mmap) | ✅ `file` feature (byte-compatible with py)³ | ✅ `backends/file.ts` (Node-only; [format](spec/file-backend-format.md))⁴ | ❌ | | CacheKit SaaS (HTTP) | ✅ `backends/cachekitio` (httpx) | ✅ `cachekitio` feature (reqwest, default) | ✅ `backends/cachekitio.ts` (fetch) | 🔜 Planned | | Cloudflare Workers | N/A | ✅ `workers` feature (`worker::Fetch`) | ✅ `@cachekit-io/cachekit/workers` (wasm32 core)¹ | N/A | | DynamoDB | ❌² | ❌ | ❌ | ❌ | @@ -85,6 +85,8 @@ > > ³ Added 2026-07-24 (LAB-429). Both are cargo features on cachekit-rs (`--features memcached` / `--features file`), native targets only (compile-error guarded against `workers`). The rs File backend shares py's on-disk format — Blake2b-128 hashed filenames, the 14-byte `CK` header, atomic write-then-rename, lazy expiry — verified by running the actual py `FileBackend` against an rs-written directory and vice versa. Not yet ported from py: LRU eviction/size caps and the mmap buffer read. rs Memcached is single-server (py's `HashClient` shards across servers); CI exercises it against a live memcached 1.6 container. +> ⁴ Added 2026-07-25 (LAB-430). TypeScript exposes these Node-only backends only through subpath exports, keeping `memjs` optional and `node:fs` out of browser and edge bundles. File implements full `TTLBackend`; Memcached deliberately offers refresh-only `touch`, not `TTLBackend`, because it cannot inspect remaining TTL. File names and headers follow [spec/file-backend-format.md](spec/file-backend-format.md); unknown reserved or flag values are misses preserved for a newer reader. + **Backend selection / env auto-detection:** Python is the only SDK that auto-detects *which* backend to use from the environment — a single unambiguous selector (`CACHEKIT_API_KEY` → SaaS, `CACHEKIT_REDIS_URL` → Redis, `CACHEKIT_MEMCACHED_SERVERS` → Memcached, `CACHEKIT_FILE_CACHE_DIR` → File; fallback `REDIS_URL` / localhost Redis; setting more than one selector raises `ConfigurationError`). Rust `CacheKit::from_env()` reads SaaS credentials only (`CACHEKIT_API_KEY` / `CACHEKIT_API_URL` / `CACHEKIT_MASTER_KEY` / `CACHEKIT_DEFAULT_TTL`) and always builds the CachekitIO backend — Redis/Memcached/File are wired explicitly via `.backend()`. TypeScript has no backend auto-detection: each preset fixes the backend type and env vars (`CACHEKIT_API_KEY`, `CACHEKIT_MASTER_KEY`) serve only as credential fallbacks. Whether rs/ts should gain selector-style auto-detection is an open product question, not a recorded decision (LAB-273 finding). --- @@ -107,7 +109,7 @@ The contract a storage backend must satisfy per SDK (bytes in / bytes out; seria | Capability | Python | Rust | TypeScript | | :--- | :--- | :--- | :--- | -| TTL inspect / refresh | `TTLInspectableBackend` — Redis ✅, SaaS ✅, File ✅, Memcached ⚠️ `refresh_ttl` only (LAB-446) | `TtlInspectable` — Redis ✅, SaaS ✅, File ✅, Memcached ⚠️ `refresh_ttl` only (LAB-429) | `TTLBackend` — Redis ✅ (LAB-427), SaaS ✅ (`TTLCachekitIO`) | +| TTL inspect / refresh | `TTLInspectableBackend` — Redis ✅, SaaS ✅, File ✅, Memcached ⚠️ `refresh_ttl` only (LAB-446) | `TtlInspectable` — Redis ✅, SaaS ✅, File ✅, Memcached ⚠️ `refresh_ttl` only (LAB-429) | `TTLBackend` — Redis ✅, SaaS ✅ (`TTLCachekitIO`), File ✅; Memcached ⚠️ `refreshTTL` only (LAB-430) | | Distributed locking | `LockableBackend` — Redis ✅ (`redis.lock.Lock`), SaaS ✅ | `LockableBackend` — SaaS ✅, Redis ✅ (`SET NX PX` + Lua compare-and-delete, `:lock` namespace shared with py; LAB-426), Workers ❌ | `LockableBackend` — Redis ✅ (LAB-427), SaaS ✅ (`LockableCachekitIO`) | | Per-operation timeout | `TimeoutConfigurableBackend` — Redis ✅ (SaaS ships a non-protocol `with_timeout` variant) | — no equivalent | — no equivalent | | Zero-copy buffer read | `BufferReadableBackend` / `BufferHandle` — File ✅ (mmap; #171) | — no equivalent | — no equivalent | @@ -128,7 +130,7 @@ The contract a storage backend must satisfy per SDK (bytes in / bytes out; seria | Distributed locking | ✅ Redis + SaaS backends | ✅ Redis + SaaS (`LockableBackend`; LAB-426. Workers lacks an impl) | ✅ Redis + SaaS backends; wired into `wrap()` cold miss (opt-in `stampede.distributedLock`, LAB-519) | ❌ | | L1/L2 dual-layer cache | ✅ | ✅ moka (native) / `l1` feature | ✅ | ❌ | | Cache stampede prevention | ✅ | ❌ | ✅ Cold-miss single-flight + SWR version tokens (LAB-519) | ❌ | -| TTL management | ✅ Redis + SaaS + File; Memcached refresh-only (see note) | ✅ Redis + SaaS + File (`TtlInspectable`); Memcached refresh-only (LAB-429) | ✅ Redis + SaaS (`TTLBackend`, LAB-427) | ❌ | +| TTL management | ✅ Redis + SaaS + File; Memcached refresh-only (see note) | ✅ Redis + SaaS + File (`TtlInspectable`); Memcached refresh-only (LAB-429) | ✅ Redis + SaaS + File (`TTLBackend`); Memcached refresh-only (LAB-430) | ❌ | | Stale-while-revalidate (server stale-grace) | 🚧 LAB-381 | ❌ | ❌ | ❌ | > **Lock id transport (CWE-532):** the unlock call carries the lock capability token in the `X-CacheKit-Lock-Id` request header, never the `?lock_id=` query string (which leaks via access/proxy logs and OTel `http.url` spans). **Migration complete in all three SDKs** (verified 2026-07-20, LAB-273): Python (#131, closed), Rust (#24, closed), TypeScript ships the header (ts#63 remains open only for an unrelated NAPI-rebuild item). SaaS dual-reads both during the rollout window. See [spec/saas-api.md](spec/saas-api.md#delete-v1cachekeylock). @@ -146,9 +148,7 @@ The contract a storage backend must satisfy per SDK (bytes in / bytes out; seria > `refresh_ttl_on_get` does not apply to it (it warns once, then serves the hit). > - **Rust (LAB-429):** File implements the full capability; Memcached implements the > refresh-only `touch` wrapper outside `TtlInspectable`, matching Python's split. -> - **TypeScript:** parity for its existing backends is tracked separately; when a -> Memcached/File backend lands there it must match this Python split (refresh-only Memcached, -> full File) — see LAB-430. +> - **TypeScript (LAB-430):** File implements the full `TTLBackend`; Memcached ships its `touch`-based `refreshTTL` method outside that capability, matching the Python and Rust split. --- @@ -247,6 +247,7 @@ its spec: - Monorepo: `@cachekit-io/cachekit` (SDK) + `@cachekit-io/cachekit-core-ts` (Rust NAPI bindings) - Redis backend via ioredis, CacheKit SaaS backend via fetch API +- Node-only Memcached (`memjs` optional peer) and File backends via subpath exports; File format is byte-verified against the shared protocol vectors - Encryption via Rust NAPI (AES-256-GCM, HKDF-SHA256, counter-based nonces) - AAD v0x03 compliant with Python cross-SDK test vectors - L1 LRU cache with background refresh, version tokens, namespace invalidation diff --git a/spec/file-backend-format.md b/spec/file-backend-format.md new file mode 100644 index 0000000..5fbc5a7 --- /dev/null +++ b/spec/file-backend-format.md @@ -0,0 +1,38 @@ +# File Backend Storage Format + +## Scope + +This optional transport format lets File backends in different CacheKit SDKs share one cache directory. It specifies only the file name and container header; the payload is the opaque byte sequence supplied by the SDK backend. It does not make an SDK auto-mode value representation cross-language compatible. + +## File name + +For a logical backend key encoded as UTF-8, calculate `BLAKE2b-128(key_utf8)` and write its lowercase hexadecimal encoding. The result is a 32-character file name in a flat cache directory. + +The File backend does not add a key prefix. Interop mode must therefore reject a configured hidden prefix before this mapping is used. + +## Entry layout + +Every entry is a 14-byte header followed by zero or more payload bytes. + +| Offset | Size | Field | Required value | +| ---: | ---: | --- | --- | +| 0 | 2 | Magic | ASCII `CK` (`0x43 0x4b`) | +| 2 | 1 | Version | `1` | +| 3 | 1 | Reserved | `0` | +| 4 | 2 | Flags | unsigned 16-bit big-endian; `0` | +| 6 | 8 | Expiry | unsigned 64-bit big-endian Unix seconds; `0` means no expiry | +| 14 | variable | Payload | opaque backend bytes | + +Writers for version 1 MUST set the reserved byte and flags to zero. The expiry is the integer UTC Unix-second deadline. An entry with a nonzero expiry is expired when the reader wall clock reaches that timestamp; readers MAY lazily remove expired entries. + +## Version and flag negotiation + +A truncated header or wrong magic/version is corrupt and may be removed as a cache miss. A nonzero reserved byte or flag value is different: it can indicate a future payload transform. A reader that does not implement every indicated transform MUST return a miss and MUST NOT delete, rewrite, or return the payload. This fails closed rather than exposing transformed bytes as plaintext. + +A future nonzero flag assignment requires a protocol update and canonical test vector. A writer MUST NOT set an unknown flag. A reader that implements a future flag must preserve the established fields and verify the transform before returning payload bytes. + +## Write and TTL behavior + +Entries are written through a temporary file in the same directory and atomically renamed into place. Refreshing TTL rewrites only bytes 6 through 13. Writers calculate the absolute deadline and store its whole-second Unix timestamp; a positive sub-second TTL can therefore expire within the current second. + +The canonical examples are in [`test-vectors/file-backend.json`](../test-vectors/file-backend.json). diff --git a/test-vectors/file-backend.json b/test-vectors/file-backend.json new file mode 100644 index 0000000..e3c3ed1 --- /dev/null +++ b/test-vectors/file-backend.json @@ -0,0 +1,40 @@ +{ + "version": "1.0.0", + "generator": "cachekit-py FileBackend format v1", + "format": "File name = BLAKE2b-128(UTF-8 key) lowercase hex; header = CK | version | reserved | flags u16 BE | expiry u64 BE | payload", + "vectors": [ + { + "name": "permanent_ascii", + "key_utf8": "test-key", + "filename": "0e2a03b49262c15a063c04d5a29c0158", + "header_hex": "434b010000000000000000000000", + "payload_hex": "68656c6c6f", + "file_hex": "434b01000000000000000000000068656c6c6f", + "expiry_unix_seconds": 0, + "flags": 0, + "reader_action": "return_payload" + }, + { + "name": "future_expiry", + "key_utf8": "unicode-ключ-🔑", + "filename": "f13b0df717f97cf1d2fe3da650525c79", + "header_hex": "434b0100000000000000f4865700", + "payload_hex": "68656c6c6f", + "file_hex": "434b0100000000000000f486570068656c6c6f", + "expiry_unix_seconds": 4102444800, + "flags": 0, + "reader_action": "return_payload" + }, + { + "name": "unknown_flag_preserved", + "key_utf8": "test-key", + "filename": "0e2a03b49262c15a063c04d5a29c0158", + "header_hex": "434b010000010000000000000000", + "payload_hex": "667574757265", + "file_hex": "434b010000010000000000000000667574757265", + "expiry_unix_seconds": 0, + "flags": 1, + "reader_action": "miss_preserve" + } + ] +} diff --git a/tools/file-backend-reference.py b/tools/file-backend-reference.py new file mode 100644 index 0000000..2ed64a5 --- /dev/null +++ b/tools/file-backend-reference.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +"""Validate the protocol-owned File backend vectors with Python stdlib.""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +import struct + +ROOT = Path(__file__).resolve().parents[1] +VECTORS = ROOT / "test-vectors" / "file-backend.json" + + +def main() -> None: + document = json.loads(VECTORS.read_text(encoding="utf-8")) + for vector in document["vectors"]: + key = vector["key_utf8"].encode("utf-8") + filename = hashlib.blake2b(key, digest_size=16).hexdigest() + assert filename == vector["filename"], vector["name"] + + image = bytes.fromhex(vector["file_hex"]) + assert len(image) >= 14, vector["name"] + assert image[:14].hex() == vector["header_hex"], vector["name"] + assert image[14:].hex() == vector["payload_hex"], vector["name"] + assert image[:2] == b"CK" and image[2] == 1, vector["name"] + assert image[3] == 0, vector["name"] + + flags = struct.unpack(">H", image[4:6])[0] + expiry = struct.unpack(">Q", image[6:14])[0] + assert flags == vector["flags"], vector["name"] + assert expiry == vector["expiry_unix_seconds"], vector["name"] + expected_action = "return_payload" if flags == 0 else "miss_preserve" + assert vector["reader_action"] == expected_action, vector["name"] + + print("validated", len(document["vectors"]), "File backend vectors") + + +if __name__ == "__main__": + main() From b1ac9e9669154216af6fe9a90d2df47be7156eee Mon Sep 17 00:00:00 2001 From: Ray Walker Date: Sun, 26 Jul 2026 00:10:09 +1000 Subject: [PATCH 2/4] fix(tools): fail closed under python -O and handle vector-load errors Replace data-validation asserts with explicit ValueError raises so the CI verifier cannot silently pass when asserts are stripped, and give missing/malformed vector files a clear one-line error and exit 1. Addresses kodus-27b review on #42 (LAB-805). Co-authored-by: multica-agent --- tools/file-backend-reference.py | 34 +++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/tools/file-backend-reference.py b/tools/file-backend-reference.py index 2ed64a5..846a644 100644 --- a/tools/file-backend-reference.py +++ b/tools/file-backend-reference.py @@ -7,31 +7,45 @@ import json from pathlib import Path import struct +import sys ROOT = Path(__file__).resolve().parents[1] VECTORS = ROOT / "test-vectors" / "file-backend.json" +def check(condition: bool, name: str, detail: str) -> None: + """Fail closed even under ``python -O`` (asserts would be stripped).""" + if not condition: + raise ValueError(f"{name}: {detail}") + + def main() -> None: - document = json.loads(VECTORS.read_text(encoding="utf-8")) + try: + document = json.loads(VECTORS.read_text(encoding="utf-8")) + except OSError as exc: + sys.exit(f"cannot read {VECTORS}: {exc}") + except json.JSONDecodeError as exc: + sys.exit(f"invalid JSON in {VECTORS}: {exc}") + for vector in document["vectors"]: + name = vector["name"] key = vector["key_utf8"].encode("utf-8") filename = hashlib.blake2b(key, digest_size=16).hexdigest() - assert filename == vector["filename"], vector["name"] + check(filename == vector["filename"], name, "filename mismatch") image = bytes.fromhex(vector["file_hex"]) - assert len(image) >= 14, vector["name"] - assert image[:14].hex() == vector["header_hex"], vector["name"] - assert image[14:].hex() == vector["payload_hex"], vector["name"] - assert image[:2] == b"CK" and image[2] == 1, vector["name"] - assert image[3] == 0, vector["name"] + check(len(image) >= 14, name, "file shorter than 14-byte header") + check(image[:14].hex() == vector["header_hex"], name, "header_hex mismatch") + check(image[14:].hex() == vector["payload_hex"], name, "payload_hex mismatch") + check(image[:2] == b"CK" and image[2] == 1, name, "bad magic or version") + check(image[3] == 0, name, "reserved byte not zero") flags = struct.unpack(">H", image[4:6])[0] expiry = struct.unpack(">Q", image[6:14])[0] - assert flags == vector["flags"], vector["name"] - assert expiry == vector["expiry_unix_seconds"], vector["name"] + check(flags == vector["flags"], name, "flags mismatch") + check(expiry == vector["expiry_unix_seconds"], name, "expiry mismatch") expected_action = "return_payload" if flags == 0 else "miss_preserve" - assert vector["reader_action"] == expected_action, vector["name"] + check(vector["reader_action"] == expected_action, name, "reader_action mismatch") print("validated", len(document["vectors"]), "File backend vectors") From bee3c277fd80648188284bfe29b0c13492b648e9 Mon Sep 17 00:00:00 2001 From: Ray Walker Date: Sun, 26 Jul 2026 08:27:19 +1000 Subject: [PATCH 3/4] fix(spec): atomic TTL refresh, reserved/expiry negotiation vectors, robust verifier MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - spec: TTL refresh must not expose a torn expiry — atomic-rename path or a single positioned 8-byte write per POSIX.1-2017 2.9.7, with single-read header loads. - vectors (1.1.0): add reserved_nonzero_preserved (fail-closed reserved negotiation) and expired_entry (deterministic reader_now at the >= expiry boundary, reader_action miss_expired). - verifier: reserved byte is reader negotiation, not a writer check; reader_action derived from reserved+flags+expiry with fail-closed precedence; structural/hex errors exit 1 with one-line messages. Addresses coderabbitai review on #42 (LAB-805). Co-authored-by: multica-agent --- spec/file-backend-format.md | 2 +- test-vectors/file-backend.json | 28 ++++++++++++++++++-- tools/file-backend-reference.py | 45 +++++++++++++++++++++++++-------- 3 files changed, 61 insertions(+), 14 deletions(-) diff --git a/spec/file-backend-format.md b/spec/file-backend-format.md index 5fbc5a7..6ca6860 100644 --- a/spec/file-backend-format.md +++ b/spec/file-backend-format.md @@ -33,6 +33,6 @@ A future nonzero flag assignment requires a protocol update and canonical test v ## Write and TTL behavior -Entries are written through a temporary file in the same directory and atomically renamed into place. Refreshing TTL rewrites only bytes 6 through 13. Writers calculate the absolute deadline and store its whole-second Unix timestamp; a positive sub-second TTL can therefore expire within the current second. +Entries are written through a temporary file in the same directory and atomically renamed into place. Refreshing TTL rewrites only bytes 6 through 13 and MUST NOT expose a torn expiry to a concurrent reader: a refresh either rewrites the entry through the same temporary-file and atomic-rename path, or updates the expiry with a single positioned 8-byte write on platforms that guarantee read/write atomicity for regular files (POSIX.1-2017 §2.9.7) — in which case readers MUST load the 14-byte header in a single read. Writers calculate the absolute deadline and store its whole-second Unix timestamp; a positive sub-second TTL can therefore expire within the current second. The canonical examples are in [`test-vectors/file-backend.json`](../test-vectors/file-backend.json). diff --git a/test-vectors/file-backend.json b/test-vectors/file-backend.json index e3c3ed1..1ef3d30 100644 --- a/test-vectors/file-backend.json +++ b/test-vectors/file-backend.json @@ -1,7 +1,7 @@ { - "version": "1.0.0", + "version": "1.1.0", "generator": "cachekit-py FileBackend format v1", - "format": "File name = BLAKE2b-128(UTF-8 key) lowercase hex; header = CK | version | reserved | flags u16 BE | expiry u64 BE | payload", + "format": "File name = BLAKE2b-128(UTF-8 key) lowercase hex; header = CK | version | reserved | flags u16 BE | expiry u64 BE | payload. reader_action is evaluated at reader_now_unix_seconds (default 0): miss_preserve for nonzero reserved/flags, miss_expired when now reaches a nonzero expiry, else return_payload. reserved defaults to 0.", "vectors": [ { "name": "permanent_ascii", @@ -35,6 +35,30 @@ "expiry_unix_seconds": 0, "flags": 1, "reader_action": "miss_preserve" + }, + { + "name": "reserved_nonzero_preserved", + "key_utf8": "test-key", + "filename": "0e2a03b49262c15a063c04d5a29c0158", + "header_hex": "434b010200000000000000000000", + "payload_hex": "7265736572766564", + "file_hex": "434b0102000000000000000000007265736572766564", + "expiry_unix_seconds": 0, + "flags": 0, + "reserved": 2, + "reader_action": "miss_preserve" + }, + { + "name": "expired_entry", + "key_utf8": "test-key", + "filename": "0e2a03b49262c15a063c04d5a29c0158", + "header_hex": "434b01000000000000003b9aca00", + "payload_hex": "65787069726564", + "file_hex": "434b01000000000000003b9aca0065787069726564", + "expiry_unix_seconds": 1000000000, + "flags": 0, + "reader_now_unix_seconds": 1000000000, + "reader_action": "miss_expired" } ] } diff --git a/tools/file-backend-reference.py b/tools/file-backend-reference.py index 846a644..5d57fcb 100644 --- a/tools/file-backend-reference.py +++ b/tools/file-backend-reference.py @@ -19,14 +19,16 @@ def check(condition: bool, name: str, detail: str) -> None: raise ValueError(f"{name}: {detail}") -def main() -> None: - try: - document = json.loads(VECTORS.read_text(encoding="utf-8")) - except OSError as exc: - sys.exit(f"cannot read {VECTORS}: {exc}") - except json.JSONDecodeError as exc: - sys.exit(f"invalid JSON in {VECTORS}: {exc}") +def expected_reader_action(reserved: int, flags: int, expiry: int, now: int) -> str: + """spec/file-backend-format.md: unknown reserved/flags fail closed before expiry.""" + if reserved != 0 or flags != 0: + return "miss_preserve" + if expiry != 0 and now >= expiry: + return "miss_expired" + return "return_payload" + +def verify(document: dict) -> int: for vector in document["vectors"]: name = vector["name"] key = vector["key_utf8"].encode("utf-8") @@ -38,16 +40,37 @@ def main() -> None: check(image[:14].hex() == vector["header_hex"], name, "header_hex mismatch") check(image[14:].hex() == vector["payload_hex"], name, "payload_hex mismatch") check(image[:2] == b"CK" and image[2] == 1, name, "bad magic or version") - check(image[3] == 0, name, "reserved byte not zero") + reserved = image[3] flags = struct.unpack(">H", image[4:6])[0] expiry = struct.unpack(">Q", image[6:14])[0] + check(reserved == vector.get("reserved", 0), name, "reserved mismatch") check(flags == vector["flags"], name, "flags mismatch") check(expiry == vector["expiry_unix_seconds"], name, "expiry mismatch") - expected_action = "return_payload" if flags == 0 else "miss_preserve" - check(vector["reader_action"] == expected_action, name, "reader_action mismatch") - print("validated", len(document["vectors"]), "File backend vectors") + now = vector.get("reader_now_unix_seconds", 0) + expected = expected_reader_action(reserved, flags, expiry, now) + check(vector["reader_action"] == expected, name, "reader_action mismatch") + + return len(document["vectors"]) + + +def main() -> None: + try: + document = json.loads(VECTORS.read_text(encoding="utf-8")) + except OSError as exc: + sys.exit(f"cannot read {VECTORS}: {exc}") + except json.JSONDecodeError as exc: + sys.exit(f"invalid JSON in {VECTORS}: {exc}") + + try: + count = verify(document) + except ValueError as exc: + sys.exit(f"invalid vector file: {exc}") + except (KeyError, TypeError, AttributeError) as exc: + sys.exit(f"invalid vector file: malformed structure ({exc!r})") + + print("validated", count, "File backend vectors") if __name__ == "__main__": From 348f15c305a39059e0c2e087cd7feba17ac13586 Mon Sep 17 00:00:00 2001 From: Ray Walker Date: Sun, 26 Jul 2026 10:12:45 +1000 Subject: [PATCH 4/4] style(tools): emit verifier success line via logging per team rule Addresses kodus-27b review on #42 (LAB-805). Co-authored-by: multica-agent --- tools/file-backend-reference.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/file-backend-reference.py b/tools/file-backend-reference.py index 5d57fcb..79b4d86 100644 --- a/tools/file-backend-reference.py +++ b/tools/file-backend-reference.py @@ -5,6 +5,7 @@ import hashlib import json +import logging from pathlib import Path import struct import sys @@ -70,8 +71,9 @@ def main() -> None: except (KeyError, TypeError, AttributeError) as exc: sys.exit(f"invalid vector file: malformed structure ({exc!r})") - print("validated", count, "File backend vectors") + logging.info("validated %d File backend vectors", count) if __name__ == "__main__": + logging.basicConfig(level=logging.INFO, format="%(message)s") main()