diff --git a/README.md b/README.md index 3712322..01d36e7 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ cachekit-ts/ │ │ ├── src/ │ │ │ ├── cache.ts # Core cache implementation │ │ │ ├── intents.ts # Intent-based API (.minimal, .production, .secure, .io) -│ │ │ ├── backends/ # Redis backend +│ │ │ ├── backends/ # Redis, CachekitIO, Memcached, File backends │ │ │ ├── l1/ # In-memory LRU cache │ │ │ ├── reliability/ # Circuit breaker, retry │ │ │ ├── encryption/ # Encryption manager diff --git a/packages/cachekit/README.md b/packages/cachekit/README.md index 504ce8d..5c3f8df 100644 --- a/packages/cachekit/README.md +++ b/packages/cachekit/README.md @@ -6,7 +6,7 @@ Production-ready Redis caching for TypeScript/Node.js. Hybrid TypeScript-Rust de ## Features -- **Dual-layer caching**: L1 in-memory (~50ns) + L2 Redis (~2-50ms) +- **Dual-layer caching**: L1 in-memory (~50ns) + pluggable L2 (Redis, CacheKit SaaS, Memcached, local File) - **Stale-while-revalidate**: Serve stale data while refreshing in background - **Zero-knowledge encryption**: Optional AES-256-GCM client-side encryption - **Circuit breaker**: Automatic failure isolation with exponential backoff @@ -133,6 +133,50 @@ const cache = createCache({ }); ``` +## Backends + +Four backends implement the same `Backend` interface (raw bytes in/out) and plug into `createCache({ backend })` interchangeably: + +| Backend | Import | Runtime | Notes | +| ----------------- | ----------------------------------------------------------- | ---------- | ------------------------------------------------------------------------------------ | +| Redis | `redis` from `@cachekit-io/cachekit` | Node | ioredis; TTL inspection + distributed locking | +| CachekitIO (SaaS) | `cachekitio` from `@cachekit-io/cachekit` | Node, edge | fetch-based; TTL + locking variants | +| Memcached | `memcached` from `@cachekit-io/cachekit/backends/memcached` | Node | memjs (binary protocol, multi-server); requires the optional `memjs` peer dependency | +| File | `file` from `@cachekit-io/cachekit/backends/file` | Node | local disk, on-disk format shared with cachekit-py | + +The Memcached and File backends are **Node-runtime only** and live behind subpath exports, so browser/edge bundles that import the package root never pull in `memjs` or `node:fs`. + +### Memcached + +```typescript +// pnpm add memjs (optional peer dependency, loaded lazily on first use) +import { createCache } from '@cachekit-io/cachekit'; +import { memcached } from '@cachekit-io/cachekit/backends/memcached'; + +const backend = memcached({ + servers: ['mc1:11211', 'mc2:11211'], // default: ['127.0.0.1:11211'] + keyPrefix: 'myapp:', +}); +const cache = createCache({ backend }); +``` + +Semantics match cachekit-py's Memcached backend: TTLs are clamped to the 30-day protocol maximum (larger values would be read as unix timestamps), values over `maxItemSizeBytes` (default 1 MiB, the server's default item-size limit) are rejected client-side with a loud error, `exists()` is GET-based (memcached has no EXISTS command), and omitting `ttl` with no `defaultTtl` means never expire. `refreshTTL(key, ttl)` is available via the `touch` command, but there is no `getTTL` — the memcached protocol cannot read a key's remaining TTL, so this backend deliberately does not implement `TTLBackend`. + +### File + +```typescript +import { createCache } from '@cachekit-io/cachekit'; +import { file } from '@cachekit-io/cachekit/backends/file'; + +const backend = file({ + cacheDir: '/var/cache/myapp', // default: os.tmpdir() + '/cachekit' + defaultTtl: 3600, // default: 0 = never expire +}); +const cache = createCache({ backend }); +``` + +The on-disk format is shared with cachekit-py's File backend — filenames are `blake2b(key, digestSize=16)` hex and each file carries the same 14-byte header (magic, version, flags, big-endian expiry), so Python and TypeScript processes can point at the same cache directory. Writes are atomic (write-to-temp, fsync, rename), expired or corrupt entries are unlinked on read, and symlinks are rejected (`O_NOFOLLOW`). Implements `TTLBackend` (`getTTL`/`refreshTTL` read and rewrite the on-disk expiry header). Unlike cachekit-py there is no LRU size eviction yet — cap growth with TTLs. The shared format is specified in [cachekit-io/protocol](https://github.com/cachekit-io/protocol/blob/main/spec/file-backend-format.md): version-1 writers set reserved and flags to zero, and this backend fails closed on a future nonzero value (misses without deleting or exposing the payload). Positive fractional TTLs round up to one second so they never become the permanent-entry sentinel. + ## API Reference ### createCache(options) diff --git a/packages/cachekit/package.json b/packages/cachekit/package.json index 6c084a5..b0f7e3a 100644 --- a/packages/cachekit/package.json +++ b/packages/cachekit/package.json @@ -21,6 +21,26 @@ "default": "./dist/cjs/index.js" } }, + "./backends/file": { + "import": { + "types": "./dist/backends/file.d.ts", + "default": "./dist/backends/file.js" + }, + "require": { + "types": "./dist/cjs/backends/file.d.ts", + "default": "./dist/cjs/backends/file.js" + } + }, + "./backends/memcached": { + "import": { + "types": "./dist/backends/memcached.d.ts", + "default": "./dist/backends/memcached.js" + }, + "require": { + "types": "./dist/cjs/backends/memcached.d.ts", + "default": "./dist/cjs/backends/memcached.js" + } + }, "./workers": { "types": "./dist/workers/index.d.ts", "default": "./dist/workers/index.js" @@ -74,10 +94,12 @@ "@cloudflare/vitest-pool-workers": "0.18.7", "@testcontainers/redis": "^11.13.0", "@types/ioredis": "^5.0.0", + "@types/memjs": "^1.3.3", "@types/node": "^25.5.0", "@vitest/coverage-v8": "^4.1.2", "esbuild": "0.28.1", "eslint": "^10.1.0", + "memjs": "^1.3.2", "prom-client": "^15.1.3", "testcontainers": "^11.13.0", "typescript": "^6.0.2", @@ -87,9 +109,13 @@ "node": ">=22.0.0" }, "peerDependencies": { + "memjs": ">=1.3.0", "prom-client": ">=14.0.0" }, "peerDependenciesMeta": { + "memjs": { + "optional": true + }, "prom-client": { "optional": true } diff --git a/packages/cachekit/src/backends/file.test.ts b/packages/cachekit/src/backends/file.test.ts new file mode 100644 index 0000000..32f7182 --- /dev/null +++ b/packages/cachekit/src/backends/file.test.ts @@ -0,0 +1,340 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { promises as fs } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { file, FileBackend } from './file.js'; +import { BackendError } from '../errors.js'; + +/** + * FileBackend unit tests — run against the real filesystem in a per-test + * tmpdir. No mocks: the on-disk format is a cross-SDK contract with + * cachekit-py, so the tests pin real bytes (see the "python golden vectors" + * block, generated with cachekit-py's hashlib/struct calls). + */ + +const HEADER_SIZE = 14; + +/** blake2b('test-key', digestSize=16) — py-verified (see PY_FILENAMES below). */ +const TEST_KEY_HASH = '0e2a03b49262c15a063c04d5a29c0158'; // pragma: allowlist secret + +/** Build a py-format file image: CK + version + reserved + flags + u64 BE expiry. */ +function fileImage(expirySeconds: bigint, payload: Uint8Array): Buffer { + const header = Buffer.alloc(HEADER_SIZE); + header.write('CK', 0, 'ascii'); + header[2] = 1; + header.writeBigUInt64BE(expirySeconds, 6); + return Buffer.concat([header, payload]); +} + +describe('FileBackend', () => { + let dir: string; + let backend: FileBackend; + + beforeEach(async () => { + dir = await fs.mkdtemp(path.join(os.tmpdir(), 'cachekit-file-test-')); + backend = file({ cacheDir: dir }); + }); + + afterEach(async () => { + await backend.close(); + await fs.rm(dir, { recursive: true, force: true }); + }); + + describe('basic operations', () => { + it('set/get round-trip preserves bytes', async () => { + const value = new Uint8Array([0, 1, 2, 255, 254, 128]); + await backend.set('test-key', value, 60); + expect(await backend.get('test-key')).toEqual(value); + }); + + it('get returns null for missing key', async () => { + expect(await backend.get('nope')).toBeNull(); + }); + + it('empty payload round-trips', async () => { + await backend.set('empty', new Uint8Array(0), 60); + expect(await backend.get('empty')).toEqual(new Uint8Array(0)); + }); + + it('overwrite replaces the value', async () => { + await backend.set('k', new Uint8Array([1]), 60); + await backend.set('k', new Uint8Array([2, 3]), 60); + expect(await backend.get('k')).toEqual(new Uint8Array([2, 3])); + }); + + it('delete returns true for existing key, false for missing', async () => { + await backend.set('k', new Uint8Array([1]), 60); + expect(await backend.delete('k')).toBe(true); + expect(await backend.delete('k')).toBe(false); + expect(await backend.get('k')).toBeNull(); + }); + + it('exists reflects presence', async () => { + expect(await backend.exists('k')).toBe(false); + await backend.set('k', new Uint8Array([1]), 60); + expect(await backend.exists('k')).toBe(true); + }); + + it('operations after close throw BackendError', async () => { + await backend.close(); + await expect(backend.get('k')).rejects.toThrow(BackendError); + await expect(backend.set('k', new Uint8Array([1]))).rejects.toThrow(BackendError); + }); + }); + + describe('python golden vectors (cross-SDK on-disk contract)', () => { + // hashlib.blake2b(key.encode('utf-8'), digest_size=16).hexdigest() + const PY_FILENAMES: Array<[string, string]> = [ + ['test-key', TEST_KEY_HASH], + [ + 'ns:myapp:func:mod.fn:args:0011223344556677889900112233445566778899001122334455667788990011:v1', + '48b25847fb31fc0b4f19b5484a11d6bb', // pragma: allowlist secret + ], + ['unicode-ключ-🔑', 'f13b0df717f97cf1d2fe3da650525c79'], // pragma: allowlist secret + ]; + + it.each(PY_FILENAMES)('key %s maps to the same filename as cachekit-py', async (key, hex) => { + await backend.set(key, new Uint8Array([1]), 60); + const entries = await fs.readdir(dir); + expect(entries).toContain(hex); + }); + + it('reads a file written by cachekit-py (permanent entry)', async () => { + // struct-packed by python: b'CK' + \x01 + \x00 + >H(0) + >Q(0) + b'hello' + const pyBytes = Buffer.from('434b01000000000000000000000068656c6c6f', 'hex'); + await fs.writeFile(path.join(dir, TEST_KEY_HASH), pyBytes); + + expect(await backend.get('test-key')).toEqual(new TextEncoder().encode('hello')); + expect(await backend.getTTL('test-key')).toBeNull(); // permanent → null + }); + + it('reads a file written by cachekit-py (expiry in 2100)', async () => { + // >Q(4102444800) — 2100-01-01T00:00:00Z + const pyBytes = Buffer.from('434b0100000000000000f486570068656c6c6f', 'hex'); + await fs.writeFile(path.join(dir, TEST_KEY_HASH), pyBytes); + + expect(await backend.get('test-key')).toEqual(new TextEncoder().encode('hello')); + const ttl = await backend.getTTL('test-key'); + expect(ttl).toBeGreaterThan(0); + expect(ttl).toBeLessThanOrEqual(4102444800 - Math.floor(Date.now() / 1000)); + }); + + it('writes headers python can parse (expiry field position and endianness)', async () => { + await backend.set('test-key', new TextEncoder().encode('hi'), 3600); + const raw = await fs.readFile(path.join(dir, TEST_KEY_HASH)); + + expect(raw.subarray(0, 2).toString('ascii')).toBe('CK'); + expect(raw[2]).toBe(1); // version + expect(raw[3]).toBe(0); // reserved + expect(raw.readUInt16BE(4)).toBe(0); // flags + const expiry = Number(raw.readBigUInt64BE(6)); + const now = Math.floor(Date.now() / 1000); + expect(expiry).toBeGreaterThanOrEqual(now + 3595); + expect(expiry).toBeLessThanOrEqual(now + 3605); + expect(raw.subarray(HEADER_SIZE).toString('utf-8')).toBe('hi'); + }); + + it('fails closed on nonzero future flags without deleting the entry', async () => { + const filePath = path.join(dir, TEST_KEY_HASH); + const image = fileImage(0n, new TextEncoder().encode('future-payload')); + image.writeUInt16BE(1, 4); + await fs.writeFile(filePath, image); + + expect(await backend.get('test-key')).toBeNull(); + expect(await backend.exists('test-key')).toBe(false); + expect(await backend.getTTL('test-key')).toBeNull(); + expect(await backend.refreshTTL('test-key', 60)).toBe(false); + await expect(fs.access(filePath)).resolves.toBeUndefined(); + }); + }); + + describe('TTL and expiry', () => { + it('ttl omitted and defaultTtl unset → permanent (py parity, not Redis 1h)', async () => { + await backend.set('k', new Uint8Array([1])); + const raw = await fs.readFile(path.join(dir, (await fs.readdir(dir))[0])); + expect(raw.readBigUInt64BE(6)).toBe(0n); + expect(await backend.getTTL('k')).toBeNull(); + }); + + it('defaultTtl config applies when ttl omitted', async () => { + const b = file({ cacheDir: dir, defaultTtl: 120 }); + await b.set('k', new Uint8Array([1])); + const ttl = await b.getTTL('k'); + expect(ttl).toBeGreaterThan(110); + expect(ttl).toBeLessThanOrEqual(120); + await b.close(); + }); + + it('expired entry: get returns null and unlinks the file', async () => { + const image = fileImage(BigInt(Math.floor(Date.now() / 1000) - 10), new Uint8Array([1])); + const filePath = path.join(dir, TEST_KEY_HASH); + await fs.writeFile(filePath, image); + + expect(await backend.get('test-key')).toBeNull(); + await expect(fs.access(filePath)).rejects.toThrow(); // unlinked + }); + + it('expired entry: exists returns false and unlinks the file', async () => { + const image = fileImage(BigInt(Math.floor(Date.now() / 1000) - 10), new Uint8Array([1])); + const filePath = path.join(dir, TEST_KEY_HASH); + await fs.writeFile(filePath, image); + + expect(await backend.exists('test-key')).toBe(false); + await expect(fs.access(filePath)).rejects.toThrow(); + }); + + it('getTTL: missing → null, live → remaining seconds, expired → null + unlink', async () => { + expect(await backend.getTTL('missing')).toBeNull(); + + await backend.set('live', new Uint8Array([1]), 300); + const ttl = await backend.getTTL('live'); + expect(ttl).toBeGreaterThan(290); + expect(ttl).toBeLessThanOrEqual(300); + + const image = fileImage(BigInt(Math.floor(Date.now() / 1000) - 10), new Uint8Array([1])); + const filePath = path.join(dir, TEST_KEY_HASH); + await fs.writeFile(filePath, image); + expect(await backend.getTTL('test-key')).toBeNull(); + await expect(fs.access(filePath)).rejects.toThrow(); + }); + + it('set rejects negative and >10y TTLs', async () => { + await expect(backend.set('k', new Uint8Array([1]), -1)).rejects.toThrow(BackendError); + await expect(backend.set('k', new Uint8Array([1]), 11 * 365 * 24 * 60 * 60)).rejects.toThrow( + BackendError + ); + }); + }); + + it('uses the fractional wall-clock expiry boundary from cachekit-py', async () => { + vi.useFakeTimers(); + try { + const filePath = path.join(dir, TEST_KEY_HASH); + const image = fileImage(1001n, new Uint8Array([1])); + await fs.writeFile(filePath, image); + + vi.setSystemTime(1_000_999); + expect(await backend.getTTL('test-key')).toBe(0); + + vi.setSystemTime(1_001_000); + expect(await backend.get('test-key')).toEqual(new Uint8Array([1])); + + vi.setSystemTime(1_001_001); + expect(await backend.get('test-key')).toBeNull(); + } finally { + vi.useRealTimers(); + } + }); + + describe('refreshTTL', () => { + it('refreshes a live key and reports the new TTL', async () => { + await backend.set('k', new Uint8Array([7, 8]), 10); + expect(await backend.refreshTTL('k', 500)).toBe(true); + const ttl = await backend.getTTL('k'); + expect(ttl).toBeGreaterThan(490); + expect(ttl).toBeLessThanOrEqual(500); + // payload untouched by the in-place header rewrite + expect(await backend.get('k')).toEqual(new Uint8Array([7, 8])); + }); + + it('refreshes a permanent key onto a TTL', async () => { + await backend.set('k', new Uint8Array([1])); // permanent + expect(await backend.refreshTTL('k', 60)).toBe(true); + expect(await backend.getTTL('k')).toBeGreaterThan(50); + }); + + it('returns false for missing key', async () => { + expect(await backend.refreshTTL('missing', 60)).toBe(false); + }); + + it('returns false for expired key and unlinks it', async () => { + const image = fileImage(BigInt(Math.floor(Date.now() / 1000) - 10), new Uint8Array([1])); + const filePath = path.join(dir, TEST_KEY_HASH); + await fs.writeFile(filePath, image); + + expect(await backend.refreshTTL('test-key', 60)).toBe(false); + await expect(fs.access(filePath)).rejects.toThrow(); + }); + + it('throws on ttl <= 0 rather than deleting or making permanent', async () => { + await backend.set('k', new Uint8Array([1]), 60); + await expect(backend.refreshTTL('k', 0)).rejects.toThrow(BackendError); + await expect(backend.refreshTTL('k', -5)).rejects.toThrow(BackendError); + expect(await backend.exists('k')).toBe(true); // still there, TTL untouched + }); + }); + + describe('corruption and safety', () => { + it('truncated file (< header size) → null + unlink', async () => { + const filePath = path.join(dir, TEST_KEY_HASH); + await fs.writeFile(filePath, Buffer.from('CK')); + expect(await backend.get('test-key')).toBeNull(); + await expect(fs.access(filePath)).rejects.toThrow(); + }); + + it('bad magic → null + unlink', async () => { + const image = fileImage(0n, new Uint8Array([1])); + image[0] = 0x58; // 'X' + const filePath = path.join(dir, TEST_KEY_HASH); + await fs.writeFile(filePath, image); + expect(await backend.get('test-key')).toBeNull(); + await expect(fs.access(filePath)).rejects.toThrow(); + }); + + it('unknown format version → null + unlink', async () => { + const image = fileImage(0n, new Uint8Array([1])); + image[2] = 99; + const filePath = path.join(dir, TEST_KEY_HASH); + await fs.writeFile(filePath, image); + expect(await backend.exists('test-key')).toBe(false); + await expect(fs.access(filePath)).rejects.toThrow(); + }); + + it('rejects values over maxValueBytes before touching disk', async () => { + const b = file({ cacheDir: dir, maxValueBytes: 4 }); + await expect(b.set('k', new Uint8Array(5))).rejects.toThrow(/maxValueBytes/); + expect(await fs.readdir(dir)).toEqual([]); + await b.close(); + }); + + it.skipIf(process.platform === 'win32')( + 'symlinked cache entry is treated as missing (O_NOFOLLOW)', + async () => { + const target = path.join(dir, 'target'); + await fs.writeFile(target, fileImage(0n, new TextEncoder().encode('attack'))); + await fs.symlink(target, path.join(dir, TEST_KEY_HASH)); + expect(await backend.get('test-key')).toBeNull(); + } + ); + + it('sweeps orphaned temp files older than 60s on first use', async () => { + const stale = path.join(dir, 'deadbeef.tmp.123.456'); + const fresh = path.join(dir, 'cafebabe.tmp.789.012'); + await fs.writeFile(stale, 'x'); + await fs.writeFile(fresh, 'x'); + const old = new Date(Date.now() - 120_000); + await fs.utimes(stale, old, old); + + const b = file({ cacheDir: dir }); + await b.get('anything'); // triggers lazy init + sweep + const entries = await fs.readdir(dir); + expect(entries).not.toContain('deadbeef.tmp.123.456'); + expect(entries).toContain('cafebabe.tmp.789.012'); // too young to sweep + await b.close(); + }); + + it('creates the cache directory with restrictive permissions', async () => { + const nested = path.join(dir, 'a', 'b'); + const b = file({ cacheDir: nested }); + await b.set('k', new Uint8Array([1]), 60); + if (process.platform !== 'win32') { + const stat = await fs.stat(nested); + expect(stat.mode & 0o777).toBe(0o700); + const [entry] = await fs.readdir(nested); + const fileStat = await fs.stat(path.join(nested, entry)); + expect(fileStat.mode & 0o777).toBe(0o600); + } + await b.close(); + }); + }); +}); diff --git a/packages/cachekit/src/backends/file.ts b/packages/cachekit/src/backends/file.ts new file mode 100644 index 0000000..d566b8e --- /dev/null +++ b/packages/cachekit/src/backends/file.ts @@ -0,0 +1,418 @@ +import { constants as fsConstants } from 'node:fs'; +import fs, { type FileHandle } from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { blake2b } from '@noble/hashes/blake2.js'; +import { bytesToHex } from '@noble/hashes/utils.js'; +import { Backend, FileBackendConfig, TTLBackend } from './types.js'; +import { BackendError } from '../errors.js'; + +// On-disk format shared with cachekit-py's FileBackend (14-byte header). +// Any change here breaks cross-SDK File readers — py and ts can point at the +// same cache directory, so the header layout and the blake2b-16 filename +// derivation are a frozen cross-SDK contract, not an implementation detail. +const MAGIC_0 = 0x43; // 'C' +const MAGIC_1 = 0x4b; // 'K' +const FORMAT_VERSION = 1; +const HEADER_SIZE = 14; // [0:2] magic, [2] version, [3] reserved, [4:6] flags u16 BE, [6:14] expiry u64 BE +const FLAGS_OFFSET = 4; +const EXPIRY_OFFSET = 6; + +/** TTL ceiling shared with py (10 years) — prevents u64 overflow games. */ +const MAX_TTL_SECONDS = 10 * 365 * 24 * 60 * 60; + +/** Orphaned temp files older than this are deleted on startup (matches py). */ +const TEMP_FILE_MAX_AGE_MS = 60_000; + +const DEFAULT_MAX_VALUE_BYTES = 100 * 1024 * 1024; // py: max_value_mb = 100 + +/** + * File-based backend for local disk caching (Node-runtime only). + * + * On-disk compatible with cachekit-py's FileBackend: filenames are + * `blake2b(key, digestSize=16)` hex (32 chars, flat directory), each file is a + * 14-byte header (magic "CK", version, flags, uint64 BE expiry — 0 = never + * expire) followed by the raw payload. Writes are atomic via + * write-to-temp → fsync → rename; expired or corrupt entries are unlinked on + * read. Symlinks are rejected with O_NOFOLLOW on every open. + * + * Deliberate deltas from py (documented, not accidental): + * - No LRU size eviction / max_entry_count: LAB-430 scopes parity to + * directory layout + TTL semantics. Size caps beyond maxValueBytes are a + * follow-up; until then the directory grows unbounded like any disk cache + * without a sweeper. + * - No flock/msvcrt file locks: Node is single-threaded per process and + * cross-process safety comes from atomic rename (readers see old or new + * bytes, never a torn file). The one in-place write (refreshTTL's 8-byte + * expiry field) has the same torn-write ceiling py accepted: worst case a + * wrong expiry, never a corrupt payload. + * + * @example + * ```typescript + * import { file } from '@cachekit-io/cachekit/backends/file'; + * + * const backend = file({ cacheDir: '/var/cache/myapp' }); + * await backend.set('key', new TextEncoder().encode('value'), 3600); + * const value = await backend.get('key'); + * await backend.close(); + * ``` + */ +export class FileBackend implements Backend, TTLBackend { + private readonly config: Required; + private closed = false; + /** Memoized directory init: mkdir + orphaned-temp-file sweep, once. */ + private ready: Promise | null = null; + + constructor(config: FileBackendConfig = {}) { + const defaultTtl = config.defaultTtl ?? 0; + if (defaultTtl < 0 || defaultTtl > MAX_TTL_SECONDS) { + throw new BackendError( + `defaultTtl ${defaultTtl} out of range [0, ${MAX_TTL_SECONDS}] (max 10 years)` + ); + } + this.config = { + cacheDir: config.cacheDir ?? path.join(os.tmpdir(), 'cachekit'), + defaultTtl, + maxValueBytes: config.maxValueBytes ?? DEFAULT_MAX_VALUE_BYTES, + fileMode: config.fileMode ?? 0o600, + dirMode: config.dirMode ?? 0o700, + }; + } + + async get(key: string): Promise { + this.ensureNotClosed(); + await this.ensureDir(); + const filePath = this.keyToPath(key); + + let handle: FileHandle; + try { + handle = await fs.open(filePath, fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW); + } catch (error) { + if (isMissingOrSymlink(error)) return null; + throw this.wrapError('get', error); + } + + try { + const data = await handle.readFile(); + const expiry = this.parseHeader(data); + if (expiry === 'unsupported') return null; + if (expiry === 'corrupt' || isExpired(expiry)) { + await handle.close(); + await this.safeUnlink(filePath); + return null; + } + return new Uint8Array(data.buffer, data.byteOffset + HEADER_SIZE, data.length - HEADER_SIZE); + } catch (error) { + throw this.wrapError('get', error); + } finally { + await handle.close().catch(() => undefined); + } + } + + async set(key: string, value: Uint8Array, ttl?: number): Promise { + this.ensureNotClosed(); + + if (this.config.maxValueBytes > 0 && value.length > this.config.maxValueBytes) { + throw new BackendError( + `Value size ${value.length} exceeds maxValueBytes (${this.config.maxValueBytes})` + ); + } + + const effectiveTtl = ttl ?? this.config.defaultTtl; + let expiry = 0n; + if (effectiveTtl !== 0) { + if (effectiveTtl < 0 || effectiveTtl > MAX_TTL_SECONDS) { + throw new BackendError( + `TTL ${effectiveTtl} out of range [0, ${MAX_TTL_SECONDS}] (max 10 years)` + ); + } + expiry = nowSeconds() + BigInt(Math.floor(effectiveTtl)); + } + + await this.ensureDir(); + const filePath = this.keyToPath(key); + // Unique temp name in the same directory so rename() is atomic (same fs). + const tempPath = `${filePath}.tmp.${process.pid}.${process.hrtime.bigint()}`; + + const header = Buffer.alloc(HEADER_SIZE); // zero-filled: reserved + flags stay 0 + header[0] = MAGIC_0; + header[1] = MAGIC_1; + header[2] = FORMAT_VERSION; + header.writeBigUInt64BE(expiry, EXPIRY_OFFSET); + + try { + const handle = await fs.open( + tempPath, + fsConstants.O_WRONLY | fsConstants.O_CREAT | fsConstants.O_EXCL | fsConstants.O_NOFOLLOW, + this.config.fileMode + ); + try { + await handle.writeFile(Buffer.concat([header, Buffer.from(value)])); + await handle.sync(); + } finally { + await handle.close(); + } + await fs.rename(tempPath, filePath); + } catch (error) { + await this.safeUnlink(tempPath); + throw this.wrapError('set', error); + } + } + + async delete(key: string): Promise { + this.ensureNotClosed(); + await this.ensureDir(); + + try { + await fs.unlink(this.keyToPath(key)); + return true; + } catch (error) { + if (isErrnoException(error) && error.code === 'ENOENT') return false; + throw this.wrapError('delete', error); + } + } + + async exists(key: string): Promise { + this.ensureNotClosed(); + const expiry = await this.readExpiry('exists', key); + return expiry !== null; + } + + /** + * See {@link TTLBackend.getTTL}: remaining seconds from the on-disk expiry + * header, or null when the key is missing, permanent (expiry field 0), or + * already expired (expired entries are unlinked, mirroring get/exists). + */ + async getTTL(key: string): Promise { + this.ensureNotClosed(); + const expiry = await this.readExpiry('getTTL', key); + if (expiry === null || expiry === 0n) return null; + const remaining = Number(expiry) - Date.now() / 1000; + if (remaining <= 0) { + await this.delete(key); + return null; + } + return Math.floor(remaining); + } + + /** + * See {@link TTLBackend.refreshTTL}: rewrites the 8-byte expiry field in + * place (payload and all other header fields untouched, so cross-SDK File + * readers stay compatible). Returns false when the key is missing or + * already expired (expired entries are unlinked, mirroring get). Throws on + * ttl <= 0 per the ts-wide TTLBackend contract — unlike cachekit-py's File + * refresh_ttl(0) = permanent, every ts refreshTTL rejects non-positive TTLs + * so swapping backends never changes zero-semantics. + */ + async refreshTTL(key: string, ttl: number): Promise { + this.ensureNotClosed(); + + const seconds = Math.floor(ttl); + if (seconds <= 0) { + throw new BackendError(`File refreshTTL requires ttl >= 1 second, got ${ttl}`); + } + if (seconds > MAX_TTL_SECONDS) { + throw new BackendError(`TTL ${ttl} out of range [1, ${MAX_TTL_SECONDS}] (max 10 years)`); + } + + await this.ensureDir(); + const filePath = this.keyToPath(key); + + let handle: FileHandle; + try { + handle = await fs.open(filePath, fsConstants.O_RDWR | fsConstants.O_NOFOLLOW); + } catch (error) { + if (isMissingOrSymlink(error)) return false; + throw this.wrapError('refreshTTL', error); + } + + try { + const header = Buffer.alloc(HEADER_SIZE); + const { bytesRead } = await handle.read(header, 0, HEADER_SIZE, 0); + const expiry = this.parseHeader(header.subarray(0, bytesRead)); + if (expiry === 'unsupported') return false; + if (expiry === 'corrupt' || isExpired(expiry)) { + await handle.close(); + await this.safeUnlink(filePath); + return false; + } + + const newExpiry = Buffer.alloc(8); + newExpiry.writeBigUInt64BE(nowSeconds() + BigInt(seconds)); + await handle.write(newExpiry, 0, 8, EXPIRY_OFFSET); + await handle.sync(); + return true; + } catch (error) { + throw this.wrapError('refreshTTL', error); + } finally { + await handle.close().catch(() => undefined); + } + } + + async close(): Promise { + this.closed = true; + } + + // ==================== private ==================== + + /** + * `blake2b(key, digestSize=16)` hex — 32-char filename, flat layout. + * Byte-identical to py's `_key_to_path`, so the same logical key maps to + * the same file from either SDK (this is also why Backend.keyPrefix stays + * unset: key identity is preserved, nothing is prefixed on the wire). + */ + private keyToPath(key: string): string { + const hash = bytesToHex(blake2b(new TextEncoder().encode(key), { dkLen: 16 })); + return path.join(this.config.cacheDir, hash); + } + + /** + * Expiry from a header buffer. Unknown reserved/flag bits are deliberately + * reported separately: they may denote a future transform, so this reader + * must fail closed without unlinking an entry a newer SDK understands. + */ + private parseHeader(data: Uint8Array): bigint | 'corrupt' | 'unsupported' { + if (data.length < HEADER_SIZE) return 'corrupt'; + if (data[0] !== MAGIC_0 || data[1] !== MAGIC_1 || data[2] !== FORMAT_VERSION) return 'corrupt'; + if ( + data[3] !== 0 || + Buffer.from(data.buffer, data.byteOffset, HEADER_SIZE).readUInt16BE(FLAGS_OFFSET) !== 0 + ) { + return 'unsupported'; + } + return Buffer.from(data.buffer, data.byteOffset, HEADER_SIZE).readBigUInt64BE(EXPIRY_OFFSET); + } + + /** + * Shared header-read path for exists/getTTL: expiry field of a live entry + * (0n = permanent), or null for missing/corrupt/expired — corrupt and + * expired entries are unlinked, mirroring py. + */ + private async readExpiry(operation: string, key: string): Promise { + await this.ensureDir(); + const filePath = this.keyToPath(key); + + let handle: FileHandle; + try { + handle = await fs.open(filePath, fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW); + } catch (error) { + if (isMissingOrSymlink(error)) return null; + throw this.wrapError(operation, error); + } + + try { + const header = Buffer.alloc(HEADER_SIZE); + const { bytesRead } = await handle.read(header, 0, HEADER_SIZE, 0); + const expiry = this.parseHeader(header.subarray(0, bytesRead)); + if (expiry === 'unsupported') return null; + if (expiry === 'corrupt' || isExpired(expiry)) { + await handle.close(); + await this.safeUnlink(filePath); + return null; + } + return expiry; + } catch (error) { + throw this.wrapError(operation, error); + } finally { + await handle.close().catch(() => undefined); + } + } + + private ensureDir(): Promise { + this.ready ??= (async () => { + try { + await fs.mkdir(this.config.cacheDir, { recursive: true, mode: this.config.dirMode }); + } catch (error) { + this.ready = null; // let a later call retry instead of caching the failure + throw this.wrapError('init', error); + } + await this.cleanupTempFiles(); + })(); + return this.ready; + } + + /** Best-effort sweep of orphaned `*.tmp.*` files older than 60s (matches py). */ + private async cleanupTempFiles(): Promise { + try { + const entries = await fs.readdir(this.config.cacheDir); + const cutoff = Date.now() - TEMP_FILE_MAX_AGE_MS; + for (const name of entries) { + if (!name.includes('.tmp.')) continue; + const tempPath = path.join(this.config.cacheDir, name); + try { + const stat = await fs.lstat(tempPath); // lstat: never follow symlinks + if (stat.isFile() && stat.mtimeMs < cutoff) { + await fs.unlink(tempPath); + } + } catch { + // raced with another process — fine + } + } + } catch { + // best-effort: never fail an operation over temp cleanup + } + } + + private async safeUnlink(filePath: string): Promise { + await fs.unlink(filePath).catch(() => undefined); + } + + private ensureNotClosed(): void { + if (this.closed) { + throw new BackendError('File backend is closed'); + } + } + + private wrapError(operation: string, error: unknown): Error { + if (error instanceof BackendError) return error; + if (isErrnoException(error)) { + // Disk-full and I/O errors may clear; inaccessible or read-only paths + // are permanent for this backend instance. + const permanent = error.code === 'EACCES' || error.code === 'EROFS' || error.code === 'EPERM'; + return new BackendError( + `File ${operation} failed: ${error.message}`, + permanent ? 'permanent' : 'transient', + { cause: error } + ); + } + if (error instanceof Error) { + return new BackendError(`File ${operation} failed: ${error.message}`, 'transient', { + cause: error, + }); + } + return new BackendError(`File ${operation} failed: Unknown error`); + } +} + +/** Missing file, or symlink rejected by O_NOFOLLOW — both mean "not found". */ +function isMissingOrSymlink(error: unknown): boolean { + return isErrnoException(error) && (error.code === 'ENOENT' || error.code === 'ELOOP'); +} + +function isErrnoException(error: unknown): error is NodeJS.ErrnoException { + return error instanceof Error && 'code' in error; +} + +/** Whole-second wall clock as bigint, the unit of the on-disk expiry field. */ +function nowSeconds(): bigint { + return BigInt(Math.floor(Date.now() / 1000)); +} + +/** Python compares the integer wire timestamp to its fractional wall clock. */ +function isExpired(expiry: bigint): boolean { + return expiry > 0n && BigInt(Date.now()) > expiry * 1000n; +} + +/** + * Factory function to create a File backend. + * + * @example + * ```typescript + * import { file } from '@cachekit-io/cachekit/backends/file'; + * + * const backend = file({ cacheDir: '/var/cache/myapp', defaultTtl: 3600 }); + * ``` + */ +export function file(config: FileBackendConfig = {}): FileBackend { + return new FileBackend(config); +} diff --git a/packages/cachekit/src/backends/memcached.test.ts b/packages/cachekit/src/backends/memcached.test.ts new file mode 100644 index 0000000..fdddf9e --- /dev/null +++ b/packages/cachekit/src/backends/memcached.test.ts @@ -0,0 +1,224 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { memcached, MemcachedBackend, MAX_MEMCACHED_TTL } from './memcached.js'; +import { BackendError, ConfigurationError, TimeoutError } from '../errors.js'; + +/** + * MemcachedBackend unit tests against a mocked memjs client. + * + * This mock IS the documented CI strategy for LAB-430: unit tests exercise + * every code path against a protocol-faithful fake (get returns + * `{value, flags}`, set/delete/touch return booleans — the real memjs + * promise API), while test/integration/memcached-backend.integration.test.ts + * runs the same operations against a real memcached container when Docker is + * available. CI's default `pnpm test` needs no memcached server. + */ + +const mockClient = { + get: vi.fn(), + set: vi.fn(), + delete: vi.fn(), + touch: vi.fn(), + quit: vi.fn(), +}; + +const clientCreate = vi.fn(() => mockClient); + +vi.mock('memjs', () => ({ + Client: { + create: (...args: unknown[]) => clientCreate(...args), + }, +})); + +describe('MemcachedBackend', () => { + let backend: MemcachedBackend; + + beforeEach(() => { + vi.clearAllMocks(); + mockClient.get.mockResolvedValue({ value: null, flags: null }); + mockClient.set.mockResolvedValue(true); + mockClient.delete.mockResolvedValue(true); + mockClient.touch.mockResolvedValue(true); + backend = memcached(); + }); + + describe('configuration', () => { + it('rejects an empty server list', () => { + expect(() => memcached({ servers: [] })).toThrow(ConfigurationError); + }); + + it.each(['no-port', 'host:notaport', 'host:0', 'host:70000'])( + 'rejects malformed server address %s', + (server) => { + expect(() => memcached({ servers: [server] })).toThrow(ConfigurationError); + } + ); + + it('passes servers and second-based timeouts to memjs', async () => { + const b = memcached({ + servers: ['mc1:11211', 'mc2:11212'], + timeout: 500, + connectTimeout: 3000, + retries: 5, + }); + await b.get('k'); // first op creates the client lazily + expect(clientCreate).toHaveBeenCalledWith('mc1:11211,mc2:11212', { + expires: 0, + timeout: 0.5, // ms → s conversion for memjs + conntimeout: 3, + retries: 5, + }); + }); + + it('creates the client once and reuses it', async () => { + await backend.get('a'); + await backend.get('b'); + expect(clientCreate).toHaveBeenCalledTimes(1); + }); + }); + + describe('get', () => { + it('returns stored bytes as Uint8Array', async () => { + mockClient.get.mockResolvedValue({ value: Buffer.from([1, 2, 3]), flags: null }); + expect(await backend.get('k')).toEqual(new Uint8Array([1, 2, 3])); + expect(mockClient.get).toHaveBeenCalledWith('k'); + }); + + it('returns null on miss', async () => { + expect(await backend.get('k')).toBeNull(); + }); + + it('wraps client errors in BackendError', async () => { + mockClient.get.mockRejectedValue(new Error('boom')); + await expect(backend.get('k')).rejects.toThrow(BackendError); + }); + + it('classifies timeouts as TimeoutError', async () => { + mockClient.get.mockRejectedValue(new Error('operation timed out')); + await expect(backend.get('k')).rejects.toThrow(TimeoutError); + }); + }); + + describe('set', () => { + it('stores bytes with the given TTL', async () => { + await backend.set('k', new Uint8Array([9]), 60); + expect(mockClient.set).toHaveBeenCalledWith('k', Buffer.from([9]), { expires: 60 }); + }); + + it('ttl omitted and defaultTtl unset → expires 0 / never (py parity, not Redis 1h)', async () => { + await backend.set('k', new Uint8Array([9])); + expect(mockClient.set).toHaveBeenCalledWith('k', Buffer.from([9]), { expires: 0 }); + }); + + it('applies defaultTtl when ttl omitted', async () => { + const b = memcached({ defaultTtl: 300 }); + await b.set('k', new Uint8Array([9])); + expect(mockClient.set).toHaveBeenCalledWith('k', Buffer.from([9]), { expires: 300 }); + }); + + it('clamps TTLs beyond 30 days (protocol treats them as unix timestamps)', async () => { + await backend.set('k', new Uint8Array([9]), MAX_MEMCACHED_TTL + 999); + expect(mockClient.set).toHaveBeenCalledWith('k', Buffer.from([9]), { + expires: MAX_MEMCACHED_TTL, + }); + }); + + it('rounds a positive fractional TTL up instead of making it permanent', async () => { + await backend.set('k', new Uint8Array([9]), 0.1); + expect(mockClient.set).toHaveBeenCalledWith('k', Buffer.from([9]), { expires: 1 }); + }); + + it('rejects oversized values client-side without calling the server', async () => { + const b = memcached({ maxItemSizeBytes: 8 }); + await expect(b.set('k', new Uint8Array(9))).rejects.toThrow(/max\s+item size/); + expect(mockClient.set).not.toHaveBeenCalled(); + }); + + it('maxItemSizeBytes: 0 disables the size guard', async () => { + const b = memcached({ maxItemSizeBytes: 0 }); + await b.set('k', new Uint8Array(2 * 1024 * 1024)); + expect(mockClient.set).toHaveBeenCalled(); + }); + }); + + describe('delete / exists', () => { + it('delete passes through the server result', async () => { + expect(await backend.delete('k')).toBe(true); + mockClient.delete.mockResolvedValue(false); + expect(await backend.delete('gone')).toBe(false); + }); + + it('exists is GET-based (memcached has no EXISTS command)', async () => { + mockClient.get.mockResolvedValue({ value: Buffer.from([1]), flags: null }); + expect(await backend.exists('k')).toBe(true); + mockClient.get.mockResolvedValue({ value: null, flags: null }); + expect(await backend.exists('k')).toBe(false); + expect(mockClient.get).toHaveBeenCalledTimes(2); + }); + }); + + describe('refreshTTL', () => { + it('touches the key with the clamped TTL', async () => { + expect(await backend.refreshTTL('k', 60)).toBe(true); + expect(mockClient.touch).toHaveBeenCalledWith('k', 60); + + await backend.refreshTTL('k', MAX_MEMCACHED_TTL + 1); + expect(mockClient.touch).toHaveBeenLastCalledWith('k', MAX_MEMCACHED_TTL); + }); + + it('returns false when the key does not exist', async () => { + mockClient.touch.mockResolvedValue(false); + expect(await backend.refreshTTL('missing', 60)).toBe(false); + }); + + it('throws on ttl <= 0 (ts-wide refreshTTL contract)', async () => { + await expect(backend.refreshTTL('k', 0)).rejects.toThrow(BackendError); + await expect(backend.refreshTTL('k', -1)).rejects.toThrow(BackendError); + expect(mockClient.touch).not.toHaveBeenCalled(); + }); + + it('does not implement TTLBackend (no getTTL — protocol cannot read TTLs)', () => { + expect('getTTL' in backend).toBe(false); + }); + }); + + describe('keyPrefix', () => { + it('is applied to every operation and exposed for the interop guard', async () => { + const b = memcached({ keyPrefix: 'app:' }); + expect(b.keyPrefix).toBe('app:'); + + await b.get('k'); + expect(mockClient.get).toHaveBeenCalledWith('app:k'); + await b.set('k', new Uint8Array([1]), 60); + expect(mockClient.set).toHaveBeenCalledWith('app:k', Buffer.from([1]), { expires: 60 }); + await b.delete('k'); + expect(mockClient.delete).toHaveBeenCalledWith('app:k'); + await b.refreshTTL('k', 60); + expect(mockClient.touch).toHaveBeenCalledWith('app:k', 60); + }); + + it('defaults to empty (keys stored verbatim, interop-safe)', () => { + expect(backend.keyPrefix).toBe(''); + }); + }); + + describe('close', () => { + it('quits the client and rejects further operations', async () => { + await backend.get('k'); // materialize the client + await backend.close(); + expect(mockClient.quit).toHaveBeenCalledTimes(1); + await expect(backend.get('k')).rejects.toThrow('closed'); + }); + + it('close before first use never creates a client', async () => { + await backend.close(); + expect(clientCreate).not.toHaveBeenCalled(); + }); + + it('close is idempotent', async () => { + await backend.get('k'); + await backend.close(); + await backend.close(); + expect(mockClient.quit).toHaveBeenCalledTimes(1); + }); + }); +}); diff --git a/packages/cachekit/src/backends/memcached.ts b/packages/cachekit/src/backends/memcached.ts new file mode 100644 index 0000000..35830c1 --- /dev/null +++ b/packages/cachekit/src/backends/memcached.ts @@ -0,0 +1,243 @@ +import type { Client as MemjsClient } from 'memjs'; +import { Backend, MemcachedBackendConfig } from './types.js'; +import { BackendError, ConfigurationError, TimeoutError } from '../errors.js'; + +/** + * Memcached maximum relative TTL: 30 days in seconds. The protocol treats any + * larger `expires` as an absolute UNIX timestamp, so cachekit clamps here + * (matching cachekit-py) instead of letting a "31 days" TTL expire instantly. + */ +export const MAX_MEMCACHED_TTL = 30 * 24 * 60 * 60; + +/** Server default item-size limit (-I flag): 1 MiB. */ +const DEFAULT_MAX_ITEM_SIZE_BYTES = 1024 * 1024; + +/** + * Memcached backend using memjs (binary protocol, multi-server support). + * + * Node-runtime only, behind the `@cachekit-io/cachekit/backends/memcached` + * subpath export. `memjs` is an optional peer dependency loaded lazily on + * first use — install it alongside cachekit (`pnpm add memjs`); browser/edge + * bundles that never import this subpath never see it. + * + * Capability surface matches cachekit-py's Memcached backend: base Backend + * only, plus a directly-callable {@link refreshTTL}. It is deliberately NOT a + * TTLBackend — the memcached protocol has no command to *read* a key's + * remaining TTL (memjs exposes no meta protocol), so `getTTL` cannot exist. + * `refreshTTL` ships anyway because the `touch` command makes it trivially + * free, exactly mirroring py's `refresh_ttl`. + * + * @example + * ```typescript + * import { memcached } from '@cachekit-io/cachekit/backends/memcached'; + * + * const backend = memcached({ servers: ['mc1:11211', 'mc2:11211'] }); + * await backend.set('key', new TextEncoder().encode('value'), 3600); + * await backend.close(); + * ``` + */ +export class MemcachedBackend implements Backend { + private readonly config: Required; + private closed = false; + /** Memoized lazy client — memjs is an optional peer dep, imported on first use. */ + private clientPromise: Promise | null = null; + + /** Applied client-side to every key (like py) — exposed so interop mode + * can fail closed; see Backend.keyPrefix for the contract. */ + get keyPrefix(): string { + return this.config.keyPrefix; + } + + constructor(config: MemcachedBackendConfig = {}) { + const servers = config.servers ?? ['127.0.0.1:11211']; + if (servers.length === 0) { + throw new ConfigurationError('At least one Memcached server must be specified'); + } + for (const server of servers) { + const port = Number(server.slice(server.lastIndexOf(':') + 1)); + if (!server.includes(':') || !Number.isInteger(port) || port < 1 || port > 65535) { + throw new ConfigurationError( + `Memcached server address must be 'host:port' with port 1-65535, got: ${server}` + ); + } + } + + this.config = { + servers, + defaultTtl: config.defaultTtl ?? 0, + timeout: config.timeout ?? 1000, + connectTimeout: config.connectTimeout ?? 2000, + retries: config.retries ?? 2, + keyPrefix: config.keyPrefix ?? '', + maxItemSizeBytes: config.maxItemSizeBytes ?? DEFAULT_MAX_ITEM_SIZE_BYTES, + }; + } + + async get(key: string): Promise { + this.ensureNotClosed(); + const client = await this.getClient(); + + try { + const { value } = await client.get(this.prefixedKey(key)); + return value ? new Uint8Array(value) : null; + } catch (error) { + throw this.wrapError('get', error); + } + } + + async set(key: string, value: Uint8Array, ttl?: number): Promise { + this.ensureNotClosed(); + + // Fail loudly BEFORE sending: the server rejects items over its -I limit + // (default 1 MiB), and that rejection is easy to lose — guard client-side + // so the caller can compress, shard, or switch backends (matches py). + const maxSize = this.config.maxItemSizeBytes; + if (maxSize > 0 && value.length > maxSize) { + throw new BackendError( + `Value for key '${key}' is ${value.length} bytes, which exceeds the Memcached max ` + + `item size of ${maxSize} bytes. Enable compression, use a larger-payload backend ` + + `(Redis/SaaS/File), or raise both the server's -I limit and maxItemSizeBytes.` + ); + } + + const effectiveTtl = ttl ?? this.config.defaultTtl; + const expires = + effectiveTtl > 0 ? Math.min(Math.max(1, Math.floor(effectiveTtl)), MAX_MEMCACHED_TTL) : 0; + + const client = await this.getClient(); + try { + await client.set(this.prefixedKey(key), Buffer.from(value), { expires }); + } catch (error) { + throw this.wrapError('set', error); + } + } + + async delete(key: string): Promise { + this.ensureNotClosed(); + const client = await this.getClient(); + + try { + return await client.delete(this.prefixedKey(key)); + } catch (error) { + throw this.wrapError('delete', error); + } + } + + /** Memcached has no native EXISTS command; GET and check for null (matches py). */ + async exists(key: string): Promise { + this.ensureNotClosed(); + const client = await this.getClient(); + + try { + const { value } = await client.get(this.prefixedKey(key)); + return value !== null; + } catch (error) { + throw this.wrapError('exists', error); + } + } + + /** + * Refresh a key's TTL via the memcached `touch` command. Returns false when + * the key doesn't exist. TTLs are clamped to the 30-day maximum like `set`. + * + * This is the ONLY half of TTLBackend memcached can ship (see class docs) — + * it is a plain method, not a TTLBackend implementation, so capability + * checks (`'getTTL' in backend`) correctly exclude this backend. Throws on + * ttl <= 0 per the ts-wide refreshTTL contract (py's refresh_ttl(0) means + * "make permanent"; ts rejects non-positive TTLs on every backend so + * swapping backends never changes zero-semantics). + */ + async refreshTTL(key: string, ttl: number): Promise { + this.ensureNotClosed(); + + const seconds = Math.floor(ttl); + if (seconds <= 0) { + throw new BackendError(`Memcached refreshTTL requires ttl >= 1 second, got ${ttl}`); + } + + const client = await this.getClient(); + try { + return await client.touch(this.prefixedKey(key), Math.min(seconds, MAX_MEMCACHED_TTL)); + } catch (error) { + throw this.wrapError('refreshTTL', error); + } + } + + async close(): Promise { + if (this.closed) return; + this.closed = true; + + if (this.clientPromise) { + // quit() flushes outstanding requests before closing (vs close()'s abort). + const client = await this.clientPromise.catch(() => null); + client?.quit(); + } + } + + // ==================== private ==================== + + private prefixedKey(key: string): string { + return this.config.keyPrefix ? `${this.config.keyPrefix}${key}` : key; + } + + private getClient(): Promise { + this.clientPromise ??= (async () => { + let memjs: typeof import('memjs'); + try { + memjs = await import('memjs'); + } catch (error) { + this.clientPromise = null; // don't cache the failure + throw new ConfigurationError( + "The Memcached backend requires the optional peer dependency 'memjs'. " + + 'Install it alongside @cachekit-io/cachekit: pnpm add memjs (or npm install memjs).', + { cause: error } + ); + } + // memjs timeouts are in (fractional) seconds; cachekit config is ms. + return memjs.Client.create(this.config.servers.join(','), { + expires: 0, // per-op expires is always passed explicitly in set() + timeout: this.config.timeout / 1000, + conntimeout: this.config.connectTimeout / 1000, + retries: this.config.retries, + }); + })(); + return this.clientPromise; + } + + private ensureNotClosed(): void { + if (this.closed) { + throw new BackendError('Memcached backend is closed'); + } + } + + private wrapError(operation: string, error: unknown): Error { + if (error instanceof Error) { + if (error.message.includes('timed out') || error.message.includes('timeout')) { + return new TimeoutError(`Memcached ${operation} timed out: ${error.message}`, { + cause: error, + }); + } + return new BackendError(`Memcached ${operation} failed: ${error.message}`, 'transient', { + cause: error, + }); + } + return new BackendError(`Memcached ${operation} failed: Unknown error`); + } +} + +/** + * Factory function to create a Memcached backend. + * + * @example + * ```typescript + * import { memcached } from '@cachekit-io/cachekit/backends/memcached'; + * + * const backend = memcached({ + * servers: ['127.0.0.1:11211'], + * keyPrefix: 'myapp:', + * }); + * ``` + */ +export function memcached(config: MemcachedBackendConfig = {}): MemcachedBackend { + return new MemcachedBackend(config); +} diff --git a/packages/cachekit/src/backends/types.ts b/packages/cachekit/src/backends/types.ts index 58f4b3c..81ed7aa 100644 --- a/packages/cachekit/src/backends/types.ts +++ b/packages/cachekit/src/backends/types.ts @@ -104,6 +104,68 @@ export interface RedisBackendConfig { keyPrefix?: string; } +/** + * Configuration for the Memcached backend (Node-runtime only). + * + * The backend lives at the `@cachekit-io/cachekit/backends/memcached` subpath + * export and requires the optional `memjs` peer dependency — neither enters + * the root bundle, so browser/edge consumers are unaffected. + */ +export interface MemcachedBackendConfig { + /** + * Memcached server addresses in "host:port" format + * (default: ["127.0.0.1:11211"], matching cachekit-py). + */ + servers?: string[]; + /** + * Default TTL in seconds for set operations without explicit TTL. + * Default 0 = never expire, matching cachekit-py's Memcached backend + * (NOT the Redis backend's 1-hour default). + */ + defaultTtl?: number; + /** Operation timeout in milliseconds (default: 1000, matching py's 1.0s) */ + timeout?: number; + /** Connection timeout in milliseconds (default: 2000, matching py's 2.0s) */ + connectTimeout?: number; + /** Retries on transient failures (default: 2, matching py) */ + retries?: number; + /** Key prefix for namespacing (exposed via Backend.keyPrefix for the interop guard) */ + keyPrefix?: string; + /** + * Reject values larger than this BEFORE sending to Memcached + * (default: 1 MiB, matching the server's default item-size limit; 0 disables). + * Memcached rejects oversized items server-side; guarding client-side keeps + * the failure loud and actionable (compress, shard, or switch backends). + */ + maxItemSizeBytes?: number; +} + +/** + * Configuration for the File backend (Node-runtime only). + * + * The backend lives at the `@cachekit-io/cachekit/backends/file` subpath + * export so its `node:fs` imports never enter the root bundle. + */ +export interface FileBackendConfig { + /** + * Directory for cache files + * (default: `${os.tmpdir()}/cachekit`, matching cachekit-py). + */ + cacheDir?: string; + /** + * Default TTL in seconds for set operations without explicit TTL. + * Default 0 = never expire, matching cachekit-py's File backend + * (NOT the Redis backend's 1-hour default). + */ + defaultTtl?: number; + /** Maximum single value size in bytes (default: 100 MiB, matching py's max_value_mb=100; 0 disables) */ + maxValueBytes?: number; + /** Cache file permissions (default: 0o600 — owner-only, matching py) */ + fileMode?: number; + /** Cache directory permissions (default: 0o700 — owner-only, matching py) */ + dirMode?: number; +} + /** * Configuration for CachekitIO (SaaS HTTP) backend. * diff --git a/packages/cachekit/src/index.ts b/packages/cachekit/src/index.ts index 1bd3b90..8b0b5b9 100644 --- a/packages/cachekit/src/index.ts +++ b/packages/cachekit/src/index.ts @@ -6,7 +6,16 @@ export { redis } from './backends/redis.js'; export * from './exports-common.js'; // ============ Node-only exports ============ -export type { RedisBackendConfig } from './backends/types.js'; +export type { + RedisBackendConfig, + MemcachedBackendConfig, + FileBackendConfig, +} from './backends/types.js'; +// NOTE: the Memcached and File backends themselves are deliberately NOT +// re-exported here. They are Node-runtime only (memjs / node:fs) and live +// behind subpath exports so browser/edge bundles never pull them in: +// import { file } from '@cachekit-io/cachekit/backends/file'; +// import { memcached } from '@cachekit-io/cachekit/backends/memcached'; export type { InvalidationConfig } from './types/cache.js'; export { EncryptionManager } from './encryption/manager.js'; export { ByteStorage } from '@cachekit-io/cachekit-core-ts'; diff --git a/packages/cachekit/test/integration/memcached-backend.integration.test.ts b/packages/cachekit/test/integration/memcached-backend.integration.test.ts new file mode 100644 index 0000000..95a06bb --- /dev/null +++ b/packages/cachekit/test/integration/memcached-backend.integration.test.ts @@ -0,0 +1,94 @@ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { GenericContainer, type StartedTestContainer } from 'testcontainers'; +import { execSync } from 'node:child_process'; +import { memcached } from '../../src/backends/memcached.js'; +import type { MemcachedBackend } from '../../src/backends/memcached.js'; + +// Skip if Docker is not available or on Windows (Testcontainers volume mount issues) +let dockerAvailable = false; +try { + if (process.platform !== 'win32') { + execSync('docker info', { stdio: 'ignore', timeout: 5000 }); + dockerAvailable = true; + } +} catch { + dockerAvailable = false; +} + +/** + * Memcached Integration Tests using Testcontainers + * + * Spins up a real memcached instance in Docker (same pattern as the Redis + * integration tests). Requires Docker; unit coverage for CI environments + * without Docker lives in src/backends/memcached.test.ts (documented mock). + */ +describe.skipIf(!dockerAvailable)('MemcachedBackend Integration (Testcontainers)', () => { + let container: StartedTestContainer; + let backend: MemcachedBackend; + + beforeAll(async () => { + container = await new GenericContainer('memcached:1.6-alpine').withExposedPorts(11211).start(); + backend = memcached({ + servers: [`${container.getHost()}:${container.getMappedPort(11211)}`], + keyPrefix: `test:${Date.now()}:`, + }); + }, 60000); // 60s timeout for container startup + + afterAll(async () => { + await backend?.close(); + await container?.stop(); + }); + + it('set and get round-trip', async () => { + const value = new Uint8Array([1, 2, 3, 4, 5]); + await backend.set('round-trip', value, 60); + expect(await backend.get('round-trip')).toEqual(value); + }); + + it('binary payloads survive intact', async () => { + const value = new Uint8Array(256).map((_, i) => i); + await backend.set('binary', value, 60); + expect(await backend.get('binary')).toEqual(value); + }); + + it('get returns null for missing key', async () => { + expect(await backend.get('does-not-exist')).toBeNull(); + }); + + it('delete removes the key and reports prior existence', async () => { + await backend.set('to-delete', new Uint8Array([1]), 60); + expect(await backend.delete('to-delete')).toBe(true); + expect(await backend.delete('to-delete')).toBe(false); + expect(await backend.get('to-delete')).toBeNull(); + }); + + it('exists reflects presence', async () => { + expect(await backend.exists('exists-check')).toBe(false); + await backend.set('exists-check', new Uint8Array([1]), 60); + expect(await backend.exists('exists-check')).toBe(true); + }); + + it('TTL expires entries', async () => { + await backend.set('short-lived', new Uint8Array([1]), 1); + expect(await backend.exists('short-lived')).toBe(true); + await new Promise((resolve) => setTimeout(resolve, 2100)); + expect(await backend.get('short-lived')).toBeNull(); + }, 10000); + + it('refreshTTL extends a dying key', async () => { + await backend.set('refresh-me', new Uint8Array([1]), 1); + expect(await backend.refreshTTL('refresh-me', 60)).toBe(true); + await new Promise((resolve) => setTimeout(resolve, 2100)); + expect(await backend.get('refresh-me')).toEqual(new Uint8Array([1])); // survived original TTL + }, 10000); + + it('refreshTTL returns false for a missing key', async () => { + expect(await backend.refreshTTL('never-existed', 60)).toBe(false); + }); + + it('oversized values are rejected client-side', async () => { + // memcached default -I is 1 MiB; cachekit guards before sending + const big = new Uint8Array(1024 * 1024 + 1); + await expect(backend.set('too-big', big)).rejects.toThrow(/max\s+item size/); + }); +}); diff --git a/packages/cachekit/test/protocol/fixtures/interop-mode.json b/packages/cachekit/test/protocol/fixtures/interop-mode.json index 10eaed3..d566c20 100644 --- a/packages/cachekit/test/protocol/fixtures/interop-mode.json +++ b/packages/cachekit/test/protocol/fixtures/interop-mode.json @@ -34,9 +34,7 @@ "description": "Issue #1 example: users:get_user with one integer argument", "namespace": "users", "operation": "get_user", - "args": [ - 42 - ], + "args": [42], "canonical_args_hex": "912a", "args_hash": "61598716255080080f6456eb065c2e51badfaa4320b0efe97469c29cffee8875", "expected_key": "users:get_user:61598716255080080f6456eb065c2e51badfaa4320b0efe97469c29cffee8875" @@ -74,16 +72,7 @@ "description": "Every unsigned width boundary: 127/128 (fixint/uint8), 255/256 (uint8/uint16), 65535/65536 (uint16/uint32), 4294967295/4294967296 (uint32/uint64)", "namespace": "t", "operation": "op", - "args": [ - 127, - 128, - 255, - 256, - 65535, - 65536, - 4294967295, - 4294967296 - ], + "args": [127, 128, 255, 256, 65535, 65536, 4294967295, 4294967296], "canonical_args_hex": "987fcc80ccffcd0100cdffffce00010000ceffffffffcf0000000100000000", "args_hash": "ccf107ae1fc53bde440609a16dc30794916bd692f06309832621b9c49c89f9b7", "expected_key": "t:op:ccf107ae1fc53bde440609a16dc30794916bd692f06309832621b9c49c89f9b7" @@ -93,16 +82,7 @@ "description": "Every signed width boundary: -32/-33 (fixint/int8), -128/-129 (int8/int16), -32768/-32769 (int16/int32), -2147483648/-2147483649 (int32/int64)", "namespace": "t", "operation": "op", - "args": [ - -32, - -33, - -128, - -129, - -32768, - -32769, - -2147483648, - -2147483649 - ], + "args": [-32, -33, -128, -129, -32768, -32769, -2147483648, -2147483649], "canonical_args_hex": "98e0d0dfd080d1ff7fd18000d2ffff7fffd280000000d3ffffffff7fffffff", "args_hash": "92fa9f5c0f336f282290472144b6a4cf86b2840a1cbd8366e42fdfa1d4faa98f", "expected_key": "t:op:92fa9f5c0f336f282290472144b6a4cf86b2840a1cbd8366e42fdfa1d4faa98f" @@ -112,9 +92,7 @@ "description": "UTF-8 string, no Unicode normalization applied", "namespace": "t", "operation": "op", - "args": [ - "h\u00e9llo w\u00f6rld \u2713" - ], + "args": ["h\u00e9llo w\u00f6rld \u2713"], "canonical_args_hex": "91b168c3a96c6c6f2077c3b6726c6420e29c93", "args_hash": "00fd1c3045fe23cfe4cac788a9e15773e1f9722ca84e28917f64be99dd7a5a3c", "expected_key": "t:op:00fd1c3045fe23cfe4cac788a9e15773e1f9722ca84e28917f64be99dd7a5a3c" @@ -157,41 +135,8 @@ "namespace": "t", "operation": "op", "args": [ - [ - 0, - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 11, - 12, - 13, - 14 - ], - [ - 0, - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 11, - 12, - 13, - 14, - 15 - ] + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] ], "canonical_args_hex": "929f000102030405060708090a0b0c0d0edc0010000102030405060708090a0b0c0d0e0f", "args_hash": "d121a3c3f0427c5df363068940c15d966120be41ea11d4befe62e3408a676130", @@ -248,24 +193,7 @@ "description": "16 arguments: the ROOT canonical argument array itself uses array16", "namespace": "t", "operation": "op", - "args": [ - 0, - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 11, - 12, - 13, - 14, - 15 - ], + "args": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], "canonical_args_hex": "dc0010000102030405060708090a0b0c0d0e0f", "args_hash": "c55dd1f50e57b0dfbcd8827965516e2e33e7de2788f18d7d88d82575e34f2116", "expected_key": "t:op:c55dd1f50e57b0dfbcd8827965516e2e33e7de2788f18d7d88d82575e34f2116" @@ -275,11 +203,7 @@ "description": "Booleans and null", "namespace": "t", "operation": "op", - "args": [ - true, - false, - null - ], + "args": [true, false, null], "canonical_args_hex": "93c3c2c0", "args_hash": "1e364f6582ca4dc93b8bed0ff4413162cddafd52407517bff6248286cb863ac1", "expected_key": "t:op:1e364f6582ca4dc93b8bed0ff4413162cddafd52407517bff6248286cb863ac1" @@ -317,9 +241,7 @@ "description": "Int 2 \u2014 must produce the same key as float_integral_collapse", "namespace": "t", "operation": "op", - "args": [ - 2 - ], + "args": [2], "canonical_args_hex": "9102", "args_hash": "6df5e966da1ba347f99784f56e50e2f570e775e54c9c9349dd8944cb7fe01ea7", "expected_key": "t:op:6df5e966da1ba347f99784f56e50e2f570e775e54c9c9349dd8944cb7fe01ea7" @@ -394,10 +316,7 @@ "b": 1, "a": 2 }, - "a": [ - 1, - 2 - ] + "a": [1, 2] } ], "canonical_args_hex": "9182a161920102a17a82a16102a16201", @@ -411,11 +330,7 @@ "operation": "op", "args": [ { - "$set": [ - 3, - 1, - 2 - ] + "$set": [3, 1, 2] } ], "canonical_args_hex": "9193010203", @@ -429,11 +344,7 @@ "operation": "op", "args": [ { - "$set": [ - "b", - 10, - "a" - ] + "$set": ["b", 10, "a"] } ], "canonical_args_hex": "91930aa161a162", diff --git a/packages/cachekit/test/protocol/interop-mode.protocol.test.ts b/packages/cachekit/test/protocol/interop-mode.protocol.test.ts index 8dd8ce0..7645c5c 100644 --- a/packages/cachekit/test/protocol/interop-mode.protocol.test.ts +++ b/packages/cachekit/test/protocol/interop-mode.protocol.test.ts @@ -214,9 +214,12 @@ describe('interop/v1 value vectors', () => { // Python float 2.0 to int. A bare JS number cannot carry that distinction // (`2.0 === 2` — the spec's "a JS-written 2 may come back to Python as // int" caveat); the harness expresses it with the InteropFloat wrapper. - it.each(vectors.value_vectors)('$name encodes canonically', ({ value, canonical_msgpack_hex }) => { - expect(bytesToHex(encodeInteropValue(fromTagged(value)))).toBe(canonical_msgpack_hex); - }); + it.each(vectors.value_vectors)( + '$name encodes canonically', + ({ value, canonical_msgpack_hex }) => { + expect(bytesToHex(encodeInteropValue(fromTagged(value)))).toBe(canonical_msgpack_hex); + } + ); it('accepts every published value payload on read (canonical or not)', () => { for (const v of vectors.value_vectors) { @@ -304,8 +307,6 @@ describe('interop/v1 error vectors (MUST reject)', () => { }); it('rejects a lone surrogate (self-test — inexpressible in portable JSON)', () => { - expect(() => encodeInteropArgs([String.fromCharCode(0xd800)])).toThrow( - /well-formed Unicode/ - ); + expect(() => encodeInteropArgs([String.fromCharCode(0xd800)])).toThrow(/well-formed Unicode/); }); }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2432393..633f63a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -66,6 +66,9 @@ importers: '@types/ioredis': specifier: ^5.0.0 version: 5.0.0 + '@types/memjs': + specifier: ^1.3.3 + version: 1.3.3 '@types/node': specifier: ^25.5.0 version: 25.9.4 @@ -78,6 +81,9 @@ importers: eslint: specifier: ^10.1.0 version: 10.5.0 + memjs: + specifier: ^1.3.2 + version: 1.3.2 prom-client: specifier: ^15.1.3 version: 15.1.3 @@ -1362,6 +1368,9 @@ packages: '@types/json-schema@7.0.15': resolution: {integrity: sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==} + '@types/memjs@1.3.3': + resolution: {integrity: sha512-OqZuZ2T0ErrygA/WUqQjmsB1GpxAt84OCgKPKbsSWwYKA/oe/lTiqvqMgFOv/Ngl8p+nytgdhrdxJtikOgimCg==} + '@types/node@18.19.130': resolution: {integrity: sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg==} @@ -2129,6 +2138,10 @@ packages: resolution: {integrity: sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==} engines: {node: '>=10'} + memjs@1.3.2: + resolution: {integrity: sha512-qUEg2g8vxPe+zPn09KidjIStHPtoBO8Cttm8bgJFWWabbsjQ9Av9Ky+6UcvKx6ue0LLb/LEhtcyQpRyKfzeXcg==} + engines: {node: '>=0.10.0'} + miniflare@4.20260721.0: resolution: {integrity: sha512-fBLaCxZ2i/nPH8iyLzvza0C8/sSF4sjD1ma1Skf+pkZVK0TlaW5ujHJlUHwcwR66v2JZt+Q28d4DCX/oaLG0cA==} engines: {node: '>=22.0.0'} @@ -3635,6 +3648,10 @@ snapshots: '@types/json-schema@7.0.15': {} + '@types/memjs@1.3.3': + dependencies: + '@types/node': 25.9.4 + '@types/node@18.19.130': dependencies: undici-types: 5.26.5 @@ -4422,6 +4439,8 @@ snapshots: dependencies: semver: 7.8.5 + memjs@1.3.2: {} + miniflare@4.20260721.0: dependencies: '@cspotcode/source-map-support': 0.8.1