diff --git a/Cargo.lock b/Cargo.lock index 15eaec4..c7675b7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2385,7 +2385,7 @@ dependencies = [ [[package]] name = "recalld" -version = "0.1.10" +version = "1.0.0" dependencies = [ "anyhow", "assert_matches", diff --git a/Cargo.toml b/Cargo.toml index bba2066..c912530 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "recalld" -version = "0.1.10" +version = "1.0.0" edition = "2024" rust-version = "1.94" description = "AI memory system" diff --git a/README.md b/README.md index 771f356..db57c8f 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,11 @@ claude mcp add --scope user --transport http recalld http://localhost:7680/mcp Then allow the MCP tools so Claude can use them without prompting each time. Add to your `~/.claude/settings.local.json` (global) or project `.claude/settings.local.json`: +`delete_namespace` is deliberately left out of the allowlist below. It +destroys every memory in a namespace and removes its vector file from +disk, with no undo and no backup — the one tool worth a confirmation +prompt each time. Add it only if you want that prompt gone. + ```json { "permissions": { @@ -78,10 +83,13 @@ Then allow the MCP tools so Claude can use them without prompting each time. Add "mcp__recalld__get_memory", "mcp__recalld__reinforce_memory", "mcp__recalld__forget_memory", + "mcp__recalld__forget_memories", "mcp__recalld__find_similar_memories", "mcp__recalld__create_namespace", "mcp__recalld__namespace_stats", - "mcp__recalld__list_memories" + "mcp__recalld__list_memories", + "mcp__recalld__list_namespaces", + "mcp__recalld__list_tags" ] } } @@ -173,7 +181,7 @@ See [docs/benchmark.md](docs/benchmark.md) for full methodology, per-category br ## Usage modes -**MCP server (stdio)** -- Runs as a Model Context Protocol server for AI tools like Claude Code. Exposes 10 tools: `store_memory`, `store_memories`, `recall_memories`, `get_memory`, `reinforce_memory`, `forget_memory`, `find_similar_memories`, `create_namespace`, `namespace_stats`, `list_memories`. +**MCP server (stdio)** -- Runs as a Model Context Protocol server for AI tools like Claude Code. Exposes 14 tools: `store_memory`, `store_memories`, `recall_memories`, `get_memory`, `reinforce_memory`, `forget_memory`, `forget_memories`, `find_similar_memories`, `create_namespace`, `delete_namespace`, `namespace_stats`, `list_memories`, `list_namespaces`, `list_tags`. ```sh recalld mcp diff --git a/docs/guide.md b/docs/guide.md index 467bfc9..8a74645 100644 --- a/docs/guide.md +++ b/docs/guide.md @@ -747,7 +747,9 @@ Shows decay forecast, at-risk memories, and storage breakdown. ## 6. MCP Tools Reference -recalld exposes 10 MCP tools. These are available to any MCP client (Claude Code, Cursor, etc.) when recalld is configured as an MCP server. +recalld exposes 14 MCP tools. These are available to any MCP client (Claude Code, Cursor, etc.) when recalld is configured as an MCP server. + +Deletion is worth reading before you use it. `forget_memory` and `forget_memories` do not free anything: a forgotten memory keeps its record and its graph edges for the life of the database, because the decay sweep never reclaims a tombstone. `delete_namespace` is the only operation that does — and it destroys everything else in the namespace along with them, with no undo and no backup. ### `store_memory` @@ -764,6 +766,7 @@ Store a new memory. | `namespace` | string | No | Target namespace (default: `"default"`). | | `parentId` | string | No | UUID of an existing memory in the same namespace, linked as this memory's parent. The store fails if the target does not exist or is in a different namespace. | | `supersedes` | string | No | UUID of an existing memory in the same namespace that this one replaces. Recall REMOVES the old memory from results and returns this one in its place. The store fails if the target does not exist or is in a different namespace. | +| `checkDuplicates` | boolean | No | Report existing memories that closely resemble this one (default: `true`). Never blocks the store; it only adds a `nearDuplicates` object to the result. | #### supersedes semantics @@ -795,13 +798,54 @@ always says what actually happened to the link. - **Counted.** The new memory's `edgeCount` includes the supersedes edge. Memories stored before this behaviour existed are not backfilled. +#### Near-duplicate warnings + +Every store compares the new memory against what is already there. When +something is at least 0.85 cosine similar, the result carries a +`nearDuplicates` object naming up to three existing memories, closest first, +each with its id, score, and summary (truncated to 240 bytes). + +- **Advisory, never a block.** The memory is stored either way. The field is + absent — not `null` — when nothing crossed the threshold, when + `checkDuplicates: false` was passed, or when the namespace's vectors are + not L2-normalized. +- **0.85 is the same number `find_similar_memories` scan mode uses**, so the + two surfaces cannot disagree about whether a pair of memories is a + duplicate. It is not configurable: the right value is a property of the + embedding model, not of the deployment. +- **Scope.** Full- and summary-phase memories in the same namespace only — + the same set auto-linking considers. The phase is re-checked against the + stored record when the report is built, so a memory that has decayed or been + deleted since it was indexed is dropped rather than reported with an empty + summary. +- **The recovery.** The warning arrives after the write, so acting on it + means undoing: `forget_memory` on the id you just created, then either + `reinforce_memory` on the existing memory (nothing new to say) or + `store_memory` again with `supersedes` set to the existing id (the fact + changed). Leave both only when they are genuinely distinct facts that + happen to read alike. +- **Cost.** The scan is a linear pass over every vector in the index — all + namespaces, since the index is one flat array with a per-entry namespace + filter — so it scales with the total number of memories, not with the size + of the namespace being written to. Pass `checkDuplicates: false` during + bulk ingestion. +- **Batches catch themselves.** `store_memories` indexes each item before + checking the next, so a batch containing the same fact twice reports the + second against the first. +- **Normalization caveat.** Scores are dot products, which are cosine + similarities only for L2-normalized vectors. In a `passthrough` namespace + the caller supplies the vectors and nothing validates them, so the check + verifies the query vector first and skips rather than reporting a number + that is not a similarity. The skip is logged at `debug` every time and at + `warn` once per process. + ### `store_memories` Store multiple memories in a single call. Each item has the same schema as `store_memory`. Returns an array of results, one per input memory. | Parameter | Type | Required | Description | |---|---|---|---| -| `memories` | array | Yes | Array of memory objects (max 100 per call). Each object has the same fields as `store_memory`, including `supersedes` and its preconditions. | +| `memories` | array | Yes | Array of memory objects (max 100 per call). Each object has the same fields as `store_memory`, including `supersedes` and its preconditions, and `checkDuplicates` — both are set per item, and `nearDuplicates` is reported per result entry. | ### `recall_memories` @@ -841,11 +885,19 @@ Strengthen a memory so it decays more slowly. ### `forget_memory` -Permanently delete a memory. +Delete a memory's content. The memory moves to the Tombstone phase: its summary, full text and tags are erased and it stops appearing in recall, but its record and its graph edges are kept so relationship chains stay intact, and its UUID is never reused. Tombstoned memories are never reclaimed by the decay sweep, so they persist for the life of the database. + +| Parameter | Type | Required | Description | +|---|---|---|---| +| `id` | string | Yes | Memory UUID to forget. The memory moves to the Tombstone phase; its graph edges are kept and its UUID is never reused. | + +### `forget_memories` + +Delete several memories in one call. Each moves to the Tombstone phase exactly as `forget_memory` describes. Returns one result per input id, in the order given; `deleted: false` means the id was unknown or the memory had already been forgotten. | Parameter | Type | Required | Description | |---|---|---|---| -| `id` | string | Yes | Memory UUID to delete. | +| `ids` | string[] | Yes | Memory UUIDs to forget (1-100). Each moves to the Tombstone phase; graph edges are kept and UUIDs are never reused. | ### `find_similar_memories` @@ -873,9 +925,24 @@ Create a new memory namespace. | `desiredRetention` | number | No | Target retention rate 0.0-1.0 (default: 0.9). | | `decayRateMultiplier` | number | No | Per-namespace decay rate multiplier. 1.0 = normal, 2.0 = 2x slower, 0.0 = disabled. Omit to inherit global setting. | +### `delete_namespace` + +**Destructive and irreversible. There is no undo and no backup.** Destroys a namespace and everything in it: every memory, every graph edge touching one, its search index entries, and its vector file on disk. + +A namespace that still holds live memories is refused unless you pass `force: true`; the refusal names the count and touches nothing. A namespace holding only tombstones counts as empty and deletes without `force` — the response reports those separately as `tombstonesPurged`. + +This is the only operation that reclaims tombstones. The `default` namespace cannot be deleted, with or without `force`, and whether or not it is empty: it is recreated on every startup under the same directory name but with a new namespace id, so deleting it would leave a stale `vectors.dat` that the recreated namespace could silently reuse. + +| Parameter | Type | Required | Description | +|---|---|---|---| +| `name` | string | Yes | Namespace to delete (1-64 chars). The `default` namespace cannot be deleted. | +| `force` | boolean | No | Delete the namespace even if it still holds memories (default: false). Forced deletion permanently destroys every memory in it and removes its vector file from disk. | + ### `namespace_stats` -Get statistics for a memory namespace including total memory count, phase breakdown (full/summary/ghost), permastore count, average strength, edge count, and vector storage size. +Get statistics for a memory namespace including the live memory count, tombstone count, phase breakdown (full/summary/ghost), permastore count, average strength, edge count, and vector storage size. + +`memoryCount` counts live memories only and always equals `full + summary + ghost`. Deleted memories appear separately as `tombstoneCount`. | Parameter | Type | Required | Description | |---|---|---|---| @@ -895,6 +962,24 @@ List memories in a namespace with pagination and optional filters. Unlike `recal | `timeRangeStart` | integer or string | No | Lower bound: epoch ms (integer) or ISO 8601 string. Only memories created at or after this time are returned. | | `timeRangeEnd` | integer or string | No | Upper bound: epoch ms (integer) or ISO 8601 string. Only memories created at or before this time are returned. | +A `limit` above the maximum is rejected, not clamped: silently returning 200 rows to a caller who asked for 1000 makes a paginating agent believe it has seen the whole namespace. + +### `list_namespaces` + +List every namespace with its ID, embedding dimensions, live memory count, and creation date. Call this first when you do not know which namespaces exist. Counts exclude deleted memories. + +Takes no parameters. + +### `list_tags` + +List the tag vocabulary with per-label memory counts, most common first. Use it to reuse an existing tag instead of inventing a near-duplicate. Results come in four buckets — plain `tags`, plus `entities`, `topics`, and `emotions` — with the derived buckets' prefixes stripped so their names can be passed straight back to `store_memory` or `list_memories`. Counts cover live memories only. + +| Parameter | Type | Required | Description | +|---|---|---|---| +| `namespace` | string | No | Namespace to count within (default: `"default"`). | +| `allNamespaces` | boolean | No | Count across every namespace (default: `false`). Cannot be combined with an explicit `namespace`. | +| `limit` | integer | No | Maximum labels per bucket (default: 50, max: 500). Each bucket also reports its untruncated total. | + --- ## 7. Namespaces diff --git a/docs/mcp.md b/docs/mcp.md index f4187ec..40ec1c0 100644 --- a/docs/mcp.md +++ b/docs/mcp.md @@ -44,10 +44,13 @@ By default, Claude Code will prompt you for approval each time an MCP tool is ca "mcp__recalld__get_memory", "mcp__recalld__reinforce_memory", "mcp__recalld__forget_memory", + "mcp__recalld__forget_memories", "mcp__recalld__find_similar_memories", "mcp__recalld__create_namespace", "mcp__recalld__namespace_stats", - "mcp__recalld__list_memories" + "mcp__recalld__list_memories", + "mcp__recalld__list_namespaces", + "mcp__recalld__list_tags" ] } } @@ -123,6 +126,10 @@ Configure per-project namespace defaults in a `.recalld.toml` file (see above), ## Available tools +### A note on deletion + +`forget_memory` and `forget_memories` move a memory to the Tombstone phase: content erased, but the record and its graph edges kept so relationship chains stay intact, and the UUID never reused. **Tombstones are never reclaimed by the decay sweep** — they persist for the life of the database. `delete_namespace` is the only operation that reclaims them. + ### A note on limits Every length limit below is measured in **UTF-8 bytes, not characters**. For plain ASCII the two are the same, so a 2000-byte `summary` holds 2000 characters. Non-ASCII characters cost more: accented letters and curly quotes are 2-3 bytes, em dashes (`—`) are 3 bytes, and emoji are 4 bytes. A summary of 1900 em dashes is 5700 bytes and will be rejected. Rejection messages report both numbers, e.g. `summary is 5700 bytes (1900 characters), which exceeds the 2000-byte limit.` @@ -135,7 +142,9 @@ Before this limit existed there was no per-element bound at all: an over-long en ### Argument validation -`store_memory` and `store_memories` validate their arguments strictly. A wrong-typed or unparseable argument is an error naming the field; it is never coerced, defaulted, or dropped. Earlier versions guessed silently — a `tags` array sent as a comma-joined string stored zero tags, a wrong-typed `namespace` fell back to the default partition, and an unparseable `supersedes` stored the memory with no link — so a client with a broken serializer could write a long run of quietly wrong memories without a single error reaching the agent. +`store_memory`, `store_memories`, `list_memories`, `list_tags`, `namespace_stats`, `recall_memories` and `find_similar_memories` validate their arguments strictly. A wrong-typed or unparseable argument is an error naming the field; it is never coerced, defaulted, or dropped. Earlier versions guessed silently — a `tags` array sent as a comma-joined string stored zero tags, a wrong-typed `namespace` fell back to the default partition, an unparseable `supersedes` stored the memory with no link, and a wrong-typed `timeRangeStart` was dropped, silently widening a range query to the whole namespace — so a client with a broken serializer could write a long run of quietly wrong memories, or read a long run of quietly wrong results, without a single error reaching the agent. + +**An out-of-range `limit` is rejected, not clamped.** `list_memories` used to clamp a `limit` above 200 down to 200, which told a paginating caller it had seen the whole namespace when it had seen the first page. The schema already declares the maximum, so only a schema-violating client is affected, and it now gets one clean error instead of a quietly short page. **Explicit `null` still means "absent."** `{"fullText": null}` is treated exactly like omitting `fullText`, for every optional field. SDKs that serialize optional fields as `null` need no changes. @@ -148,6 +157,13 @@ Example messages: | `"tags": "a,b"` | `Parameter 'tags' must be an array of strings (got string)` | | `"tags": ["a", 3]` | `Parameter 'tags' must be an array of strings (element at index 2 is a number)` | | `"supersedes": 123` | `Parameter 'supersedes' must be a string containing a UUID (got number)` | +| `"limit": "50"` | `Parameter 'limit' must be an integer (got string)` | +| `"limit": 1000` | `Parameter 'limit' must be between 1 and 200 (got 1000)` | +| `"limit": -1` | `Parameter 'limit' must be a non-negative whole number (got -1)` | +| `"namespace": ["work"]` | `Parameter 'namespace' must be a string (got array)` | +| `"allNamespaces": "true"` | `Parameter 'allNamespaces' must be a boolean (got string)` | +| `"allNamespaces": true` with `"namespace"` | `Parameter 'allNamespaces' cannot be combined with an explicit 'namespace'; pass one or the other` | +| `"timeRangeStart": ["x"]` | `Parameter 'timeRangeStart' must be an integer (epoch millis) or an ISO 8601 string (got array)` | | `"supersedes": "mem-1234"` | `Invalid UUID in parameter 'supersedes': "mem-1234"` | | a `memories` entry that is not an object | `Item at index 3 must be an object (got string)` | @@ -188,6 +204,7 @@ Store a new observation, fact, or piece of context. The system automatically gen | `namespace` | string | no | `"default"` | Memory partition | | `parentId` | string | no | -- | UUID of parent memory for hierarchical linking | | `supersedes` | string | no | -- | UUID of an existing memory in the same namespace that this one replaces. Recall returns this memory **in place of** the old one. The store fails if the target does not exist or is in a different namespace. See [supersedes semantics](#supersedes-semantics). | +| `checkDuplicates` | boolean | no | `true` | Report existing memories that closely resemble this one. Never blocks the store; it only adds a `nearDuplicates` object to the result. See [near-duplicate warnings](#near-duplicate-warnings). | #### Example @@ -214,6 +231,29 @@ Response: } ``` +Response when an existing memory closely resembles the new one: + +```json +{ + "id": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + "namespace": "default", + "phase": "Full", + "strength": 1.0, + "stability": 3.7145, + "createdAt": "2025-06-24 00:00:00 UTC", + "nearDuplicates": { + "threshold": 0.85, + "matches": [ + { + "id": "9f8e7d6c-5b4a-7392-8180-1a2b3c4d5e6f", + "score": 0.94, + "summary": "User writes Rust in snake_case and TypeScript in camelCase" + } + ] + } +} +``` + --- #### supersedes semantics @@ -250,6 +290,72 @@ replacement passes the query's own filters** -- namespace above all. If it does not, the original is returned instead, so a result is never dropped with nothing in its place. +--- + +#### Near-duplicate warnings + +Every store compares the new memory against what is already there and, when +something crosses the line, attaches a `nearDuplicates` object to the result: + +```json +"nearDuplicates": { + "threshold": 0.85, + "matches": [ + { "id": "…", "score": 0.94, "summary": "the existing memory's summary, truncated to 240 bytes" } + ] +} +``` + +**Advisory, never a block.** The memory is stored either way. The field is +absent -- not `null`, not an empty object -- when nothing crossed the threshold, +when the caller passed `checkDuplicates: false`, or when the namespace's vectors +are not L2-normalized (see below). At most three matches, closest first. + +**0.85 is the same number `find_similar_memories` scan mode uses.** One +definition of "duplicate" for the whole system, so scan mode and a store cannot +disagree about the same pair of memories. It is not configurable: the right +value is a property of the embedding model, not of the deployment, and a +per-call knob would hand the caller a lever to silence the warning. + +**Scope.** Full- and summary-phase memories in the *same namespace* only -- +the same set auto-linking considers. Ghost-phase memories have no summary left +to show and tombstoned ones are deleted. The phase is re-checked against the +stored record when the report is built, so a memory that has decayed or been +deleted since it was indexed is dropped rather than reported with an empty +summary. + +**The recovery.** The warning arrives after the write, so acting on it means +undoing: + +1. `forget_memory` on the id you just created. +2. Then either `reinforce_memory` on the existing memory (you had nothing new to + say) or `store_memory` again with `supersedes` set to the existing id (the + fact changed). + +Leave both only when they are genuinely distinct facts that happen to read +alike. + +**Cost, and when to turn it off.** The scan is a linear pass over every vector +in the index -- all namespaces, not just the target one, because the index is +one flat array with a namespace filter applied per entry. It is proportional to +the *total* number of memories and their dimensionality, not to the size of the +namespace being written to. At ten thousand memories that is well under the +fsync a store already pays; at a hundred thousand it is comparable. Pass +`checkDuplicates: false` during bulk ingestion, where you are importing known- +distinct records and paying one scan per memory buys nothing. + +**Batches catch themselves.** `store_memories` stores its items in order and +indexes each one before checking the next, so a batch containing the same fact +twice reports the second against the first. + +**Normalization caveat.** Scores are dot products, which are cosine +similarities only for L2-normalized vectors. In a namespace whose embeddings are +supplied by the caller (a `passthrough` provider) nothing validates that, so the +check verifies the query vector first and *skips* rather than reporting a number +that is not a similarity. A skip is logged at `debug` every time and at `warn` +once per process, so an operator gets one visible signal that the feature is +inert rather than silence. + ### store_memories Store multiple memories in a single call. Each item has the same schema as `store_memory`. Returns an array of results, one per input memory. @@ -273,6 +379,7 @@ Each object in the `memories` array accepts: | `namespace` | string | no | `"default"` | Memory partition | | `parentId` | string | no | -- | UUID of parent memory | | `supersedes` | string | no | -- | UUID of an older memory this one replaces | +| `checkDuplicates` | boolean | no | `true` | Report existing memories that closely resemble this item. Per item, not per batch. | #### Example @@ -328,6 +435,8 @@ Response: If individual memories fail validation (e.g., missing summary), their entry in the `results` array contains an `error` field instead of an `id`. Other memories in the batch are still stored. +A result entry carries `nearDuplicates` on the same terms as `store_memory` -- `checkDuplicates` is set per item, and the report appears per result entry. Because items are stored in order and each is indexed before the next is checked, a duplicate *within* the batch is reported too. + --- ### recall_memories @@ -504,13 +613,17 @@ Response: ### forget_memory -Permanently delete a memory. Use for incorrect or outdated information that should be removed immediately rather than allowed to decay. The memory transitions to Tombstone phase (graph edges are preserved). +Delete a memory's content. The memory moves to the Tombstone phase: its summary, full text and tags are erased and it stops appearing in recall, but its record and its graph edges are kept so relationship chains stay intact, and its UUID is never reused. + +Tombstoned memories are never reclaimed by the decay sweep, so they persist for the life of the database. + +Use for incorrect, outdated or harmful information that must stop being recalled immediately. To correct a memory, prefer storing the correction with `supersedes`. #### Parameters | Parameter | Type | Required | Default | Description | |-----------|------|----------|---------|-------------| -| `id` | string | yes | -- | Memory UUID to delete | +| `id` | string | yes | -- | Memory UUID to forget. The memory moves to the Tombstone phase; its graph edges are kept and its UUID is never reused. | #### Example @@ -533,6 +646,108 @@ Response: --- +### forget_memories + +Delete several memories in a single call. Each moves to the Tombstone phase exactly as `forget_memory` describes: content erased, record and graph edges kept, UUID never reused, never reclaimed by the decay sweep. + +Returns one result per input id, in the order given. A `deleted: false` result means the id was unknown or the memory had already been forgotten. A malformed UUID becomes a per-item error and does not fail the rest of the batch. + +#### Parameters + +| Parameter | Type | Required | Default | Description | +|-----------|------|----------|---------|-------------| +| `ids` | string[] | yes | -- | Memory UUIDs to forget. 1-100 items. Each moves to the Tombstone phase; its graph edges are kept and its UUID is never reused. | + +#### Example + +Request: + +```json +{ + "ids": [ + "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + "not-a-uuid" + ] +} +``` + +Response: + +```json +{ + "results": [ + { "index": 0, "id": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", "deleted": true }, + { "index": 1, "error": "Invalid UUID: not-a-uuid", "field": "ids[1]" } + ], + "total": 2, + "deleted": 1, + "errors": 1 +} +``` + +--- + +### delete_namespace + +**Destructive and irreversible. There is no undo and no backup.** + +Destroys a namespace and everything in it: every memory, every graph edge touching one (including edges that point into the namespace from outside), its full-text and vector index entries, and its vector file on disk. + +A namespace that still holds live memories is refused unless you pass `force: true`. The refusal names the count and touches nothing. + +A namespace holding only tombstones counts as empty and deletes without `force`. Those records hold no content — they have already been forgotten — and they report as `memoryCount: 0` everywhere else. The purge still reclaims them, and the response reports how many as `tombstonesPurged`. + +This is the **only** operation that reclaims tombstones. A memory deleted with `forget_memory` keeps its record and its graph edges for the life of the database; the decay sweep never removes them. + +The `default` namespace cannot be deleted, with or without `force`, and whether or not it is empty. It is recreated on every startup under the same directory name but with a new namespace id, so deleting it would leave a stale `vectors.dat` that the recreated namespace could silently reuse. + +#### Parameters + +| Parameter | Type | Required | Default | Description | +|-----------|------|----------|---------|-------------| +| `name` | string | yes | -- | Namespace to delete (1-64 chars). The `default` namespace cannot be deleted. | +| `force` | boolean | no | `false` | Delete the namespace even if it still holds memories. Without this, a non-empty namespace is refused. Forced deletion permanently destroys every memory in the namespace and removes its vector file from disk. | + +#### Example + +Request: + +```json +{ + "name": "scratch", + "force": true +} +``` + +Response: + +```json +{ + "name": "scratch", + "id": 4, + "deleted": true, + "memoriesDeleted": 128, + "tombstonesPurged": 12, + "edgesRemoved": 341, + "ftsRowsRemoved": 128, + "vectorFileRemoved": true +} +``` + +Refusal (a non-empty namespace without `force`): + +```json +{ + "isError": true, + "content": [{ + "type": "text", + "text": "Namespace 'scratch' still holds 12 memories. Deleting it permanently destroys them and removes its vector file from disk; there is no undo and no backup. Re-run with force: true to proceed." + }] +} +``` + +--- + ### find_similar_memories Find memories semantically similar to a given memory, or scan a namespace for duplicate clusters. Two modes: @@ -667,7 +882,9 @@ Response: ### namespace_stats -Get statistics for a memory namespace including total memory count, phase breakdown (full/summary/ghost), permastore count, average strength, edge count, and vector storage size. Use this to check how many memories exist or monitor namespace health. +Get statistics for a memory namespace including the live memory count, tombstone count, phase breakdown (full/summary/ghost), permastore count, average strength, edge count, and vector storage size. Use this to check how many memories exist or monitor namespace health. + +`memoryCount` counts **live** memories only and always equals `full + summary + ghost`, so it agrees with `list_memories.total` and `list_namespaces[].memoryCount`. Deleted memories appear separately as `tombstoneCount`: a namespace showing one memory where you stored eleven has ten tombstones, and nothing was lost. #### Parameters @@ -689,20 +906,23 @@ Response: ```json { - "namespace": "default", - "totalMemories": 42, - "phases": { + "name": "default", + "memoryCount": 42, + "tombstoneCount": 7, + "phaseCounts": { "full": 30, "summary": 8, "ghost": 4 }, "permastoreCount": 3, - "averageStrength": 0.72, + "avgStrength": 0.72, "edgeCount": 156, - "vectorStorageBytes": 1048576 + "vectorBytes": 258048 } ``` +`vectorBytes` is the logical size of the live vector data (`memoryCount` x embedding dimensions x 4). It is not the size of `vectors.dat`, which retains freed slots until compaction. + --- ### list_memories @@ -714,7 +934,7 @@ List memories in a namespace with pagination and optional filters. Unlike `recal | Parameter | Type | Required | Default | Description | |-----------|------|----------|---------|-------------| | `namespace` | string | no | `"default"` | Which namespace to list from | -| `limit` | integer | no | `50` | Maximum results per page (1-200) | +| `limit` | integer | no | `50` | Maximum results per page (1-200). A larger value is **rejected**, not clamped. | | `offset` | integer | no | `0` | Number of results to skip for pagination | | `tags` | string[] | no | `[]` | Only return memories with ALL of these tags | | `entities` | string[] | no | `[]` | Only return memories mentioning ALL of these entities | @@ -752,6 +972,101 @@ Response: } ``` +--- + +### list_namespaces + +List every memory namespace with its ID, embedding dimensions, live memory count, and creation date. Call this first when you do not know which namespaces exist, before targeting one with `list_memories`, `list_tags`, or `namespace_stats`. + +Counts exclude deleted (tombstoned) memories, so they agree with `list_memories.total` and `namespace_stats.memoryCount` for the same namespace. + +#### Parameters + +None. The tool takes no arguments. + +#### Example + +Request: + +```json +{} +``` + +Response: + +```json +[ + { + "id": 1, + "name": "default", + "embeddingDim": 1536, + "memoryCount": 128, + "createdAt": "2026-08-11 12:00:00 UTC" + } +] +``` + +The response is a bare array, deliberately: it is the same payload the `recalld://namespaces` resource returns, and the total is the array length. + +--- + +### list_tags + +List the tag vocabulary with per-label memory counts, most common first. Use this to discover what labels already exist before filtering `list_memories` or `recall_memories`, and before storing — so you reuse an existing tag instead of inventing a near-duplicate. + +Results come in four buckets: plain `tags`, plus the `entities`, `topics`, and `emotions` labels derived from those fields. **The derived buckets have their prefix stripped**, so a name from `entities` can be passed straight back to `store_memory` or `list_memories`; the prefixed form (`entity/recalld`) cannot, because those tools add the prefix themselves. + +A tag that merely contains a slash, such as `project/recalld`, stays in `tags`. Counts cover live memories only. + +#### Parameters + +| Parameter | Type | Required | Default | Description | +|-----------|------|----------|---------|-------------| +| `namespace` | string | no | `"default"` | Which namespace to count within. Reads that namespace's memory records, costing about one `list_memories` call. | +| `allNamespaces` | boolean | no | `false` | Count across every namespace instead of one. Cannot be combined with an explicit `namespace`. | +| `limit` | integer | no | `50` | Maximum labels returned **per bucket** (1-500). Each bucket also reports its untruncated total. | + +Passing `allNamespaces: true` together with an explicit `namespace` is an error, not a precedence rule — neither scope silently wins. + +#### Example + +Request: + +```json +{ + "namespace": "default", + "limit": 20 +} +``` + +Response: + +```json +{ + "namespace": "default", + "memoriesCounted": 128, + "tags": { + "total": 41, + "truncated": true, + "items": [ + { "name": "type/decision", "count": 12 }, + { "name": "project/recalld", "count": 9 } + ] + }, + "entities": { + "total": 120, + "truncated": true, + "items": [ + { "name": "recalld", "count": 30 } + ] + }, + "topics": { "total": 18, "truncated": false, "items": [] }, + "emotions": { "total": 3, "truncated": false, "items": [] } +} +``` + +In global scope (`allNamespaces: true`) both `namespace` and `memoriesCounted` are explicitly `null` rather than omitted, so the response always states its own scope. `memoriesCounted` is `null` there because the global path reads the tag index directly and touches no memory records. + ## Best practices ### What to store diff --git a/src/api/adapters.rs b/src/api/adapters.rs index 02e2b01..3d0548a 100644 --- a/src/api/adapters.rs +++ b/src/api/adapters.rs @@ -104,15 +104,25 @@ impl state::SearchPipeline for SearchPipelineAdapter { state::QueryInput::Vector(_) => None, }; - // Build proper filter from query tags/entities (Issue 2) + // A label that cannot become a tag is an error, not a dropped + // filter: dropping one from `includeTags` or `entities` WIDENS + // the query, and a caller cannot tell "every memory in the + // namespace" from a genuine match. let mut require_tags: Vec = - crate::model::parse_tags_lossy("search.includeTags", &query.include_tags); - // Convert entities to entity/ tags (Issue 4) - for e in &query.entities { - if let Ok(tag) = Tag::new(&format!("entity/{}", e.to_lowercase())) { - require_tags.push(tag); - } - } + crate::model::parse_filter_tags("includeTags", "", &query.include_tags) + .map_err(|e| crate::search::SearchError::InvalidFilter(e.to_string()))?; + require_tags.extend( + crate::model::parse_filter_tags( + "entities", + crate::model::constants::ENTITY_TAG_PREFIX, + &query.entities, + ) + .map_err(|e| crate::search::SearchError::InvalidFilter(e.to_string()))?, + ); + // `excludeTags` is the safe direction — dropping one can only + // ever return MORE than asked for in the same sense a missing + // exclusion does, and it never masquerades as a match — so it + // keeps the lossy parse. let exclude_tags: Vec = crate::model::parse_tags_lossy("search.excludeTags", &query.exclude_tags); let phases: Vec = query @@ -181,6 +191,17 @@ impl state::SearchPipeline for SearchPipelineAdapter { .collect()) } + async fn near_duplicates( + &self, + embedding: &[f32], + namespace_id: NamespaceId, + threshold: f32, + limit: usize, + ) -> Vec<(MemoryId, f32)> { + let index = self.vector_index.read().await; + crate::search::find_near_duplicates(&index, embedding, namespace_id, threshold, limit) + } + async fn index_memory(&self, id: MemoryId, embedding: &[f32], namespace_id: NamespaceId) { use crate::search::{VectorIndex, VectorMetadata}; let mut index = self.vector_index.write().await; @@ -290,15 +311,37 @@ impl state::SearchPipeline for SearchPipelineAdapter { pub struct StorageEngineAdapter { storage: Arc>, cache: Arc, + /// Held so `delete_memory` can clean the indexes storage cannot + /// reach. Without them this adapter could tombstone a record and + /// leave it indexed everywhere else. + vector_index: Arc>, + /// See [`vector_index`](Self::vector_index). + fts_index: Arc>, + /// See [`vector_index`](Self::vector_index). + entity_index: Arc>, + /// See [`vector_index`](Self::vector_index). + graph: crate::graph::SharedGraph, } impl StorageEngineAdapter { - /// Creates a new adapter wrapping the storage engine and cache. + /// Creates a new adapter wrapping the storage engine, the cache, and + /// the four indexes a delete has to clean. pub fn new( storage: Arc>, cache: Arc, + vector_index: Arc>, + fts_index: Arc>, + entity_index: Arc>, + graph: crate::graph::SharedGraph, ) -> Self { - Self { storage, cache } + Self { + storage, + cache, + vector_index, + fts_index, + entity_index, + graph, + } } } @@ -401,6 +444,10 @@ impl state::StorageEngine for StorageEngineAdapter { // as non-tombstoned, both tombstone it, and both free the same // vector slot -- corrupting the free list into a self-referential // cycle. + // Returns the PRE-tombstone record, not a bool: `tombstone` + // erases the tag list, and the tag list is where the entity + // names live. A bool here would leave the entity index with no + // way to know what to unlink. let deleted = tokio::task::spawn_blocking(move || { let mut storage_w = storage.write().map_err(|e| { crate::storage::StorageError::Io(std::io::Error::new( @@ -412,11 +459,15 @@ impl state::StorageEngine for StorageEngineAdapter { // Check if the record exists and is not already tombstoned. let existing_record = match storage_w.get_record(id)? { Some(r) => r, - None => return Ok::(false), + None => { + return Ok::, crate::storage::StorageError>( + None, + ); + } }; if existing_record.phase == DecayPhase::Tombstone { - return Ok(false); + return Ok(None); } // Tombstone the record. @@ -433,7 +484,7 @@ impl state::StorageEngine for StorageEngineAdapter { ); } - Ok(true) + Ok(Some(existing_record)) }) .await .map_err(|e| { @@ -443,10 +494,33 @@ impl state::StorageEngine for StorageEngineAdapter { )) })??; - if deleted { - self.cache.invalidate(id).await; - } - Ok(deleted) + let Some(record) = deleted else { + return Ok(false); + }; + + // This used to stop at `cache.invalidate`, so an HTTP-deleted + // memory kept its vector-index entry (burning a top-k slot in + // every search and reading as a live autolink target), kept its + // FTS row — which survives restart, because the FTS index is + // only rebuilt when it is empty — and kept a graph node + // reporting Full phase at strength 1.0, still propping up its + // neighbours' decay resistance. Same helper as the MCP path now, + // so the two cannot drift apart again. + crate::index_cleanup::purge_from_indexes( + &crate::index_cleanup::IndexHandles { + cache: &self.cache, + vector_index: &self.vector_index, + fts_index: &self.fts_index, + entity_index: &self.entity_index, + graph: &self.graph, + }, + std::slice::from_ref(&(id, record)), + crate::index_cleanup::FtsAction::PerRecord, + crate::index_cleanup::GraphAction::MarkTombstoned, + ) + .await; + + Ok(true) } async fn namespace_stats( @@ -466,6 +540,7 @@ impl state::StorageEngine for StorageEngineAdapter { let all_records = storage_r.scan_all()?; let mut memory_count: u64 = 0; + let mut tombstone_count: u64 = 0; let mut phase_1_count: u64 = 0; let mut phase_2_count: u64 = 0; let mut phase_3_count: u64 = 0; @@ -477,12 +552,23 @@ impl state::StorageEngine for StorageEngineAdapter { if NamespaceId::new(record.namespace_id) != namespace_id { continue; } + + // Mirrors the MCP path exactly: a tombstoned record is not + // a memory, and counting it dragged avg_strength toward + // zero while still contributing its (unzeroed) permastore + // flag and edge count. + if record.phase == DecayPhase::Tombstone { + tombstone_count += 1; + continue; + } memory_count += 1; match record.phase { DecayPhase::Full => phase_1_count += 1, DecayPhase::Summary => phase_2_count += 1, DecayPhase::Ghost => phase_3_count += 1, + // Unreachable: skipped above. An arm, not an + // `unreachable!()` -- this is a read path. DecayPhase::Tombstone => {} } @@ -502,6 +588,7 @@ impl state::StorageEngine for StorageEngineAdapter { Ok(state::NamespaceStats { memory_count, + tombstone_count, phase_1_count, phase_2_count, phase_3_count, @@ -1237,9 +1324,15 @@ impl state::NamespaceRegistry for NamespaceRegistryAdapter { Err(_) => return Vec::new(), }; + // Live memories only. Counting tombstones here made every + // namespace listing disagree with what a caller could actually + // fetch from it. let mut counts = std::collections::HashMap::::new(); if let Ok(all_records) = storage_r.scan_all() { for (_id, record) in &all_records { + if record.phase == DecayPhase::Tombstone { + continue; + } *counts.entry(record.namespace_id).or_default() += 1; } } @@ -1334,3 +1427,136 @@ impl state::MetricsCollector for NoopMetricsCollector { String::new() } } + +// ═══════════════════════════════════════════════════════════════════════ +// Tests +// ═══════════════════════════════════════════════════════════════════════ + +#[cfg(test)] +mod tests { + use super::*; + use crate::api::state::StorageEngine as _; + use crate::search::{VectorIndex as _, VectorMetadata}; + use crate::test_support::{DIM, Fixture, MemorySpec}; + + /// A vector of the fixture's width, so it matches the index. + fn unit_vector() -> Vec { + let mut v = vec![0.0; DIM]; + v[0] = 1.0; + v + } + + fn adapter(fx: &Fixture) -> StorageEngineAdapter { + StorageEngineAdapter::new( + fx.storage.clone(), + fx.cache.clone(), + fx.vector_index.clone(), + fx.fts_index.clone(), + fx.entity_index.clone(), + fx.graph.clone(), + ) + } + + /// Put a memory into every index, which is the state the running + /// system leaves after a successful store. + async fn indexed_memory(fx: &Fixture) -> MemoryId { + let ns = NamespaceId::new(1); + let id = fx + .insert_memory_with(MemorySpec { + tags: &["entity/sarah"], + ..MemorySpec::new(ns) + }) + .await; + + fx.fts_index + .lock() + .await + .add(ns, id, "a fact about Sarah", None, &[]) + .expect("fts add"); + fx.vector_index + .write() + .await + .add( + id, + &unit_vector(), + VectorMetadata { + namespace_id: ns, + decay_phase: 1, + tags: Vec::new(), + }, + ) + .expect("vector add"); + fx.entity_index + .write() + .await + .add(id, &["Sarah".to_string()]); + + id + } + + /// THE HTTP-delete regression. This path used to stop after + /// `cache.invalidate`, leaving the memory in all four indexes: a + /// vector entry burning a top-k slot in every search and reading as + /// a live autolink target, an FTS row that survives restart forever, + /// an entity entry, and a graph node still reporting Full phase at + /// strength 1.0 and so still propping up its neighbours' decay + /// resistance. + #[tokio::test] + async fn w35_http_delete_clears_every_index_and_marks_the_graph_node() { + let fx = Fixture::new(); + let id = indexed_memory(&fx).await; + + assert!(adapter(&fx).delete_memory(id).await.expect("delete")); + + assert!( + fx.fts_index.lock().await.is_empty().expect("fts"), + "FTS row survived an HTTP delete" + ); + assert_eq!( + fx.vector_index.read().await.len(), + 0, + "vector index entry survived an HTTP delete" + ); + assert_eq!( + fx.entity_index.read().await.len(), + 0, + "entity index entry survived an HTTP delete" + ); + assert_eq!( + fx.graph + .read() + .await + .get_node(&id) + .expect("the node is kept, only re-phased") + .decay_phase, + DecayPhase::Tombstone + ); + } + + /// A second delete is a no-op, not a second round of index churn. + #[tokio::test] + async fn w36_http_delete_of_an_already_tombstoned_memory_reports_false() { + let fx = Fixture::new(); + let id = indexed_memory(&fx).await; + let adapter = adapter(&fx); + + assert!(adapter.delete_memory(id).await.expect("first delete")); + + // Re-index it, so a second pass through the cleanup would be + // visible rather than idempotent by accident. + fx.fts_index + .lock() + .await + .add(NamespaceId::new(1), id, "re-indexed", None, &[]) + .expect("fts add"); + + assert!( + !adapter.delete_memory(id).await.expect("second delete"), + "an already-tombstoned memory must report false" + ); + assert!( + !fx.fts_index.lock().await.is_empty().expect("fts"), + "the second delete ran the index cleanup anyway" + ); + } +} diff --git a/src/api/errors.rs b/src/api/errors.rs index 5999bf8..434fe00 100644 --- a/src/api/errors.rs +++ b/src/api/errors.rs @@ -223,6 +223,12 @@ impl From for AppError { id: format!("{id}"), }, SearchError::StageTimeout { .. } => AppError::Timeout, + // The caller's filter, not our fault: 400, and the message + // already names the field and the offending label. + SearchError::InvalidFilter(message) => AppError::BadRequest { + message: message.clone(), + field: None, + }, _ => AppError::Internal { source: Box::new(err), }, diff --git a/src/api/handlers.rs b/src/api/handlers.rs index b2e9eb9..7f2da58 100644 --- a/src/api/handlers.rs +++ b/src/api/handlers.rs @@ -47,6 +47,78 @@ fn health_report_cache() CACHE.get_or_init(|| RwLock::new(std::collections::HashMap::new())) } +/// Build the near-duplicate advisory for a memory about to be created. +/// +/// Shared by the single and batch create endpoints so the two HTTP doors +/// cannot drift, and mirroring `McpStorageAdapter::store_memory` so the +/// HTTP and MCP doors cannot either. `MemoryResponse.supersedes` exists +/// for exactly that reason. +/// +/// **Call this BEFORE the memory is indexed.** `state.search.index_memory` +/// puts the new vector in the index; run after that and the memory matches +/// itself at score 1.0. +/// +/// Never fails a create: every error path inside +/// [`find_near_duplicates`](crate::search::find_near_duplicates) yields no +/// matches. +/// +/// The Full/Summary guarantee is enforced HERE, not by the +/// `decay_phases` filter `find_near_duplicates` hands the vector index: +/// `FlatVectorIndex` writes each entry's `decay_phase` once at add time +/// (always Full) and `VectorIndex::update_metadata` has no callers, so +/// that filter is inert in production. A candidate whose record is +/// missing or past Summary is dropped here instead. +async fn near_duplicate_report( + state: &AppState, + embedding: &[f32], + namespace_id: NamespaceId, + check_duplicates: Option, +) -> Option { + use crate::model::constants::{ + DUPLICATE_SIMILARITY_THRESHOLD, MAX_NEAR_DUPLICATE_MATCHES, + NEAR_DUPLICATE_SUMMARY_MAX_BYTES, + }; + use crate::model::{DecayPhase, NearDuplicateMatch, NearDuplicateReport, truncate_summary}; + + if !check_duplicates.unwrap_or(true) { + return None; + } + + let candidates = state + .search + .near_duplicates( + embedding, + namespace_id, + DUPLICATE_SIMILARITY_THRESHOLD as f32, + MAX_NEAR_DUPLICATE_MATCHES, + ) + .await; + + let mut matches = Vec::with_capacity(candidates.len()); + for (id, score) in candidates { + // `get_or_load` reads through the cache without recording an + // access — the endpoints that count a read call `record_access` + // separately afterwards. Loading a summary to warn about a + // duplicate is not a retrieval of that memory. + let Some(record) = state.cache.get_or_load(id, state.storage.as_ref()).await else { + // Hard-deleted, and the vector index has not caught up. + continue; + }; + // Ghost has no summary left to show and Tombstone is deleted; + // neither is something to be told it duplicates. + if !matches!(record.phase, DecayPhase::Full | DecayPhase::Summary) { + continue; + } + matches.push(NearDuplicateMatch { + id, + score, + summary: truncate_summary(&record.summary, NEAR_DUPLICATE_SUMMARY_MAX_BYTES), + }); + } + + NearDuplicateReport::from_matches(DUPLICATE_SIMILARITY_THRESHOLD, matches) +} + // ═══════════════════════════════════════════════════════════════════════ // Memory CRUD // ═══════════════════════════════════════════════════════════════════════ @@ -160,6 +232,12 @@ pub async fn create_memory( } }; + // --- Near-duplicate advisory --- + // Before the write, so the new memory is not yet in the vector index + // and cannot match itself. See `near_duplicate_report`. + let near_duplicates = + near_duplicate_report(&state, &embedding, ns.id, req.check_duplicates).await; + // --- Persist --- // Resolve initial stability: use caller-provided value, or fall back // to the namespace's configured initial_stability (matching the MCP @@ -250,6 +328,7 @@ pub async fn create_memory( let mut response = MemoryResponse::from_cached(&memory, ns.name.clone()); response.full_text = req.full_text; response.supersedes = supersedes_outcome; + response.near_duplicates = near_duplicates; let took = start.elapsed().as_micros() as u64; Ok(( @@ -1007,6 +1086,7 @@ pub async fn namespace_stats( name: ns.name, id: ns.id.get(), memory_count: stats.memory_count, + tombstone_count: stats.tombstone_count, phase_counts: PhaseCounts { full: stats.phase_1_count, summary: stats.phase_2_count, @@ -1371,6 +1451,13 @@ pub async fn batch_store( } }; + // Near-duplicate advisory, before this item is indexed. Items are + // created in order and each is indexed before the next is checked, + // so an intra-batch duplicate is reported too — same property the + // MCP batch tool has. + let near_duplicates = + near_duplicate_report(&state, &embedding, ns.id, mem_req.check_duplicates).await; + // Persist — resolve initial stability from namespace config (matching MCP) let resolved_stability = mem_req.initial_stability.unwrap_or(ns.initial_stability); let memory = match state @@ -1467,6 +1554,7 @@ pub async fn batch_store( id: memory.id, namespace: ns.name.clone(), supersedes: supersedes_outcome, + near_duplicates, }); } diff --git a/src/api/models.rs b/src/api/models.rs index cd1fe60..2a6eac3 100644 --- a/src/api/models.rs +++ b/src/api/models.rs @@ -75,6 +75,15 @@ pub struct CreateMemoryApiRequest { /// the memory is backdated to this time instead of using the current time. #[serde(skip_serializing_if = "Option::is_none", default)] pub created_at: Option, + + /// Whether to check the new memory against existing ones and report + /// near-duplicates on the response. Absent means enabled. + /// + /// `Option` rather than `bool` with a `default_true`: a bare + /// `#[serde(default)]` on a `bool` is `false`, which would silently + /// disable the check for every client that predates the field. + #[serde(default)] + pub check_duplicates: Option, } /// Default namespace name for request types. @@ -174,25 +183,35 @@ pub struct NamespaceStatsResponse { /// Namespace integer ID. pub id: u32, - /// Total memory count in this namespace. + /// Number of live memories in this namespace. Excludes tombstoned + /// records, and always equals the sum of `phaseCounts`. pub memory_count: u64, + /// Number of tombstoned records still occupying a metadata row. + /// + /// Reported rather than folded into `memoryCount`: a namespace showing + /// one live memory where eleven were stored has ten tombstones, and an + /// operator auditing it needs to see that rather than guess. + pub tombstone_count: u64, + /// Breakdown by decay phase. pub phase_counts: PhaseCounts, /// Number of permastore memories. pub permastore_count: u64, - /// Average decay strength across all memories. + /// Average decay strength across the live memories. pub avg_strength: f32, - /// Total edge count for memories in this namespace. + /// Total edge count for the live memories in this namespace. pub edge_count: u64, /// Embedding dimensionality. pub embedding_dim: u32, - /// Disk space used by this namespace's vectors in bytes. + /// Logical bytes of live vector data + /// (`memoryCount` x `embeddingDim` x 4). NOT the size of + /// `vectors.dat`, which retains freed slots until compaction. pub vector_bytes: u64, } @@ -573,6 +592,10 @@ pub struct BatchStoreResult { /// What became of the requested `supersedes` link, if one was asked for. #[serde(skip_serializing_if = "Option::is_none")] pub supersedes: Option, + /// Existing memories that closely resemble this item. Absent when + /// nothing crossed the threshold or the item disabled the check. + #[serde(skip_serializing_if = "Option::is_none")] + pub near_duplicates: Option, } /// A batch item that was rejected, and why. @@ -610,7 +633,7 @@ pub struct ScanDuplicatesRequest { } fn default_duplicate_threshold() -> f32 { - 0.85 + crate::model::constants::DUPLICATE_SIMILARITY_THRESHOLD as f32 } fn default_max_scan() -> usize { diff --git a/src/api/state.rs b/src/api/state.rs index 9b680f9..806d1b4 100644 --- a/src/api/state.rs +++ b/src/api/state.rs @@ -43,6 +43,24 @@ pub trait SearchPipeline: Send + Sync { /// Execute a search query, returning scored results. async fn search(&self, query: SearchQuery) -> Result, SearchError>; + /// Find existing memories at or above `threshold` similarity to a + /// pre-computed embedding, closest first, at most `limit` of them. + /// + /// Distinct from [`search`](Self::search) in the one way that matters + /// at store time: it records no accesses on what it returns, so + /// checking a new memory for duplicates does not age every + /// neighbour's decay schedule. + /// + /// Never errors — the caller is an advisory field that must not fail a + /// create. + async fn near_duplicates( + &self, + embedding: &[f32], + namespace_id: NamespaceId, + threshold: f32, + limit: usize, + ) -> Vec<(MemoryId, f32)>; + /// Index a memory's embedding in the vector index. async fn index_memory(&self, id: MemoryId, embedding: &[f32], namespace_id: NamespaceId); @@ -391,8 +409,10 @@ pub struct ReinforceResult { /// Statistics for a single namespace, returned by storage. #[derive(Debug, Clone)] pub struct NamespaceStats { - /// Total memory count. + /// Number of live memories. Excludes tombstoned records. pub memory_count: u64, + /// Number of tombstoned records still occupying a metadata row. + pub tombstone_count: u64, /// Memories in Full phase (phase 1). pub phase_1_count: u64, /// Memories in Summary phase (phase 2). diff --git a/src/daemon/bridge_adapters.rs b/src/daemon/bridge_adapters.rs index c5355d3..ebe314b 100644 --- a/src/daemon/bridge_adapters.rs +++ b/src/daemon/bridge_adapters.rs @@ -5,9 +5,9 @@ use async_trait::async_trait; use super::client::DaemonClient; use super::protocol; use crate::mcp::bridge::{ - self, BridgeError, CreateNamespaceInput, DuplicateCluster, HealthStatus, MemoryRecord, - NamespaceInfo, NamespaceStats, ReinforceResult, SearchHit, SearchInput, SearchPipeline, - SearchResponse, StoreInput, StoredMemory, SubsystemHealth, + self, BridgeError, CreateNamespaceInput, DuplicateCluster, HealthStatus, ListTagsInput, + ListTagsResponse, MemoryRecord, NamespaceInfo, NamespaceStats, ReinforceResult, SearchHit, + SearchInput, SearchPipeline, SearchResponse, StoreInput, StoredMemory, SubsystemHealth, }; use crate::model::MemoryId; @@ -116,6 +116,16 @@ impl bridge::StorageEngine for RemoteStorageAdapter { .map_err(|e| BridgeError::Internal(format!("response decode: {e}"))) } + async fn delete_memories(&self, ids: &[MemoryId]) -> Result, BridgeError> { + let params = serde_json::to_value(protocol::DeleteMemoriesParams { + ids: ids.iter().map(|id| id.to_string()).collect(), + }) + .map_err(|e| BridgeError::Internal(e.to_string()))?; + let result = self.client.call("delete_memories", params).await?; + serde_json::from_value(result) + .map_err(|e| BridgeError::Internal(format!("response decode: {e}"))) + } + async fn reinforce_memory( &self, id: MemoryId, @@ -141,6 +151,14 @@ impl bridge::StorageEngine for RemoteStorageAdapter { serde_json::from_value(result) .map_err(|e| BridgeError::Internal(format!("response decode: {e}"))) } + + async fn list_tags(&self, input: ListTagsInput) -> Result { + let params = + serde_json::to_value(&input).map_err(|e| BridgeError::Internal(e.to_string()))?; + let result = self.client.call("list_tags", params).await?; + serde_json::from_value(result) + .map_err(|e| BridgeError::Internal(format!("response decode: {e}"))) + } } // ═══════════════════════════════════════════════════════════════════════ @@ -181,6 +199,17 @@ impl bridge::NamespaceRegistry for RemoteNamespaceAdapter { .map_err(|e| BridgeError::Internal(format!("response decode: {e}"))) } + async fn delete_namespace( + &self, + input: bridge::DeleteNamespaceInput, + ) -> Result { + let params = + serde_json::to_value(&input).map_err(|e| BridgeError::Internal(e.to_string()))?; + let result = self.client.call("delete_namespace", params).await?; + serde_json::from_value(result) + .map_err(|e| BridgeError::Internal(format!("response decode: {e}"))) + } + async fn namespace_stats(&self, name: &str) -> Result { let params = serde_json::to_value(protocol::NamespaceStatsParams { name: name.to_string(), diff --git a/src/daemon/client.rs b/src/daemon/client.rs index 22de665..1325c1c 100644 --- a/src/daemon/client.rs +++ b/src/daemon/client.rs @@ -47,6 +47,18 @@ const MAX_STALE_SKIPS: usize = 8; /// state on each call (`reinforce_memory` moves the FSRS schedule), or have /// a return value that is not idempotent even where the effect is /// (`delete_memory` returns whether the memory existed). +/// +/// The destructive methods are excluded for a sharper reason than +/// "mutating". `delete_memories` returns a per-id flag vector whose +/// values all flip to false on a replay, so a caller that got no answer +/// the first time cannot distinguish "nothing was deleted" from +/// "everything was deleted, twice". `delete_namespace` is worse: it +/// removes a directory from disk, and a replay after an unknown outcome +/// could destroy a namespace of the same name that the user recreated in +/// between. Neither belongs on this list, and neither belongs on +/// `records_accesses_only` — the honest report for a destructive call +/// with an unknown outcome is the "may or may not have been applied" +/// warning. fn is_retry_safe(method: &str) -> bool { matches!( method, @@ -57,6 +69,7 @@ fn is_retry_safe(method: &str) -> bool { | "list_namespaces" | "namespace_stats" | "list_memories" + | "list_tags" ) } @@ -987,6 +1000,7 @@ mod tests { "create_namespace", "reinforce_memory", "delete_memory", + "delete_memories", "shutdown", // Not mutations, but not effect-free either: both record an // access per result, which moves the decay schedule. @@ -1003,11 +1017,28 @@ mod tests { "list_namespaces", "namespace_stats", "list_memories", + "list_tags", ] { assert!(is_retry_safe(method), "{method} should be replayable"); } } + /// The destructive methods must be on neither list. + /// `records_accesses_only` downgrades the caller's warning from "may + /// or may not have been applied" to "just ask again", which is the + /// wrong thing to tell someone whose namespace may have been + /// destroyed. + #[test] + fn w42_the_destructive_methods_are_neither_retry_safe_nor_access_only() { + for method in ["delete_memories", "delete_namespace"] { + assert!(!is_retry_safe(method), "{method} must not be replayed"); + assert!( + !records_accesses_only(method), + "{method} is destructive, not merely access-recording" + ); + } + } + #[test] fn negotiated_limit_ignores_unknown_and_absurd_announcements() { assert_eq!(negotiated_limit(&json!({})), LEGACY_MAX_MESSAGE_SIZE); diff --git a/src/daemon/protocol.rs b/src/daemon/protocol.rs index ac674a5..d113885 100644 --- a/src/daemon/protocol.rs +++ b/src/daemon/protocol.rs @@ -74,6 +74,14 @@ pub struct DeleteMemoryParams { pub id: String, } +/// Parameters for the `delete_memories` RPC method. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DeleteMemoriesParams { + /// Memory IDs to delete, in the order the caller supplied them. + /// The reply carries one flag per id in the same order. + pub ids: Vec, +} + /// Parameters for the `reinforce_memory` RPC method. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ReinforceParams { @@ -116,6 +124,8 @@ pub const ERR_SEARCH: i32 = -32004; pub const ERR_INTERNAL: i32 = -32603; /// RPC error code: the request or response did not fit in a protocol frame. pub const ERR_TOO_LARGE: i32 = -32005; +/// RPC error code: the operation failed after destroying something. +pub const ERR_PARTIALLY_APPLIED: i32 = -32006; // ── BridgeError conversion ─────────────────────────────────────────── @@ -146,6 +156,10 @@ impl From<&BridgeError> for DaemonRpcError { code: ERR_TOO_LARGE, message: msg.clone(), }, + BridgeError::PartiallyApplied(msg) => DaemonRpcError { + code: ERR_PARTIALLY_APPLIED, + message: msg.clone(), + }, } } } @@ -159,6 +173,7 @@ impl DaemonRpcError { ERR_STORAGE => BridgeError::Storage(self.message), ERR_SEARCH => BridgeError::Search(self.message), ERR_TOO_LARGE => BridgeError::TooLarge(self.message), + ERR_PARTIALLY_APPLIED => BridgeError::PartiallyApplied(self.message), _ => BridgeError::Internal(self.message), } } diff --git a/src/daemon/server.rs b/src/daemon/server.rs index 08455cf..8001e98 100644 --- a/src/daemon/server.rs +++ b/src/daemon/server.rs @@ -12,8 +12,9 @@ use super::lifecycle; use super::protocol::*; use crate::Recalld; use crate::mcp::bridge::{ - BridgeError, CreateNamespaceInput, HealthChecker, ListMemoriesInput, NamespaceRegistry, - SearchInput, SearchPipeline, StorageEngine as BridgeStorageEngine, StoreInput, + BridgeError, CreateNamespaceInput, DeleteNamespaceInput, HealthChecker, ListMemoriesInput, + ListTagsInput, NamespaceRegistry, SearchInput, SearchPipeline, + StorageEngine as BridgeStorageEngine, StoreInput, }; use crate::mcp::bridge_adapters::*; use crate::model::MemoryId; @@ -53,8 +54,15 @@ impl DaemonServer { std::sync::Arc::new(system.config().clone()), tz, )); - let namespaces: Arc = - Arc::new(McpNamespaceAdapter::new(system.storage().clone(), tz)); + let namespaces: Arc = Arc::new(McpNamespaceAdapter::new( + system.storage().clone(), + system.cache().clone(), + system.vector_index().clone(), + system.fts_index().clone(), + system.entity_index().clone(), + system.graph().clone(), + tz, + )); let health: Arc = Arc::new(McpHealthAdapter::new(system.storage().clone())); @@ -447,6 +455,27 @@ async fn dispatch( serde_json::to_value(result).map_err(|e| BridgeError::Internal(e.to_string())) } + "delete_memories" => { + let p: DeleteMemoriesParams = serde_json::from_value(params) + .map_err(|e| BridgeError::InvalidInput(e.to_string()))?; + // The socket is its own entry point: the tool layer's cap is + // not on this path, and an uncapped batch is one write + // transaction over an unbounded id list. + if p.ids.len() > crate::model::constants::MAX_BATCH_MEMORIES { + return Err(BridgeError::InvalidInput(format!( + "Too many ids ({}); the maximum is {} per call", + p.ids.len(), + crate::model::constants::MAX_BATCH_MEMORIES + ))); + } + let mut ids = Vec::with_capacity(p.ids.len()); + for id in &p.ids { + ids.push(parse_memory_id(id)?); + } + let result = storage.delete_memories(&ids).await?; + serde_json::to_value(result).map_err(|e| BridgeError::Internal(e.to_string())) + } + "reinforce_memory" => { let p: ReinforceParams = serde_json::from_value(params) .map_err(|e| BridgeError::InvalidInput(e.to_string()))?; @@ -462,6 +491,13 @@ async fn dispatch( serde_json::to_value(result).map_err(|e| BridgeError::Internal(e.to_string())) } + "list_tags" => { + let input: ListTagsInput = serde_json::from_value(params) + .map_err(|e| BridgeError::InvalidInput(e.to_string()))?; + let result = storage.list_tags(input).await?; + serde_json::to_value(result).map_err(|e| BridgeError::Internal(e.to_string())) + } + "list_namespaces" => { let result = namespaces.list_namespaces().await?; serde_json::to_value(result).map_err(|e| BridgeError::Internal(e.to_string())) @@ -474,6 +510,13 @@ async fn dispatch( serde_json::to_value(result).map_err(|e| BridgeError::Internal(e.to_string())) } + "delete_namespace" => { + let input: DeleteNamespaceInput = serde_json::from_value(params) + .map_err(|e| BridgeError::InvalidInput(e.to_string()))?; + let result = namespaces.delete_namespace(input).await?; + serde_json::to_value(result).map_err(|e| BridgeError::Internal(e.to_string())) + } + "namespace_stats" => { let p: NamespaceStatsParams = serde_json::from_value(params) .map_err(|e| BridgeError::InvalidInput(e.to_string()))?; @@ -496,8 +539,9 @@ async fn dispatch( mod tests { use super::*; use crate::mcp::bridge::{ - DuplicateCluster, HealthStatus, ListMemoriesResponse, MemoryRecord, NamespaceInfo, - NamespaceStats, ReinforceResult, SearchHit, SearchResponse, StoredMemory, + DeleteNamespaceResult, DuplicateCluster, HealthStatus, ListMemoriesResponse, + ListTagsResponse, MemoryRecord, NamespaceInfo, NamespaceStats, ReinforceResult, SearchHit, + SearchResponse, StoredMemory, }; /// Stub subsystems for the dispatch tests. Only the unknown-method path @@ -539,6 +583,9 @@ mod tests { async fn delete_memory(&self, _: MemoryId) -> Result { unreachable!() } + async fn delete_memories(&self, _: &[MemoryId]) -> Result, BridgeError> { + unreachable!() + } async fn reinforce_memory( &self, _: MemoryId, @@ -552,6 +599,9 @@ mod tests { ) -> Result { unreachable!() } + async fn list_tags(&self, _: ListTagsInput) -> Result { + unreachable!() + } } #[async_trait::async_trait] @@ -568,6 +618,12 @@ mod tests { async fn namespace_stats(&self, _: &str) -> Result { unreachable!() } + async fn delete_namespace( + &self, + _: DeleteNamespaceInput, + ) -> Result { + unreachable!() + } } #[async_trait::async_trait] @@ -766,6 +822,28 @@ mod tests { } // 24 + /// The tool layer caps a `forget_memories` batch at + /// `MAX_BATCH_MEMORIES`, but the socket is its own entry point and + /// never runs the tool handler. Without this the frame goes straight + /// to one write transaction over an unbounded id list — and the stub + /// `delete_memories` is `unreachable!()`, so reaching it fails the + /// test rather than passing it quietly. + #[tokio::test] + async fn w70_delete_memories_caps_the_batch_at_the_shared_constant() { + let over_cap: Vec = (0..crate::model::constants::MAX_BATCH_MEMORIES + 1) + .map(|_| crate::model::MemoryId::new().to_string()) + .collect(); + + let err = dispatch_stub("delete_memories", serde_json::json!({ "ids": over_cap })) + .await + .expect_err("an over-cap batch must be refused"); + + assert!( + matches!(err, BridgeError::InvalidInput(ref m) if m.contains("Too many ids")), + "{err}" + ); + } + #[tokio::test] async fn dispatch_unknown_method_returns_invalid_input() { let err = dispatch_stub("no_such_method", serde_json::json!({})) @@ -775,6 +853,17 @@ mod tests { assert!(err.to_string().contains("no_such_method"), "{err}"); } + /// `list_tags` params have no serde default for `limit`, so an empty + /// object is rejected at decode time -- which is also why the `Unused` + /// stub's `list_tags` is never reached. + #[tokio::test] + async fn dispatch_list_tags_rejects_params_it_cannot_decode() { + let err = dispatch_stub("list_tags", serde_json::json!({})) + .await + .unwrap_err(); + assert!(matches!(err, BridgeError::InvalidInput(_)), "{err}"); + } + #[tokio::test] async fn dispatch_ping_announces_the_protocol_version() { let result = dispatch_stub("ping", serde_json::json!({})).await.unwrap(); diff --git a/src/decay/sweep.rs b/src/decay/sweep.rs index 6f33844..6eb16f0 100644 --- a/src/decay/sweep.rs +++ b/src/decay/sweep.rs @@ -1299,6 +1299,36 @@ impl DecaySweepRunner { } } // Delete metadata record (requires write lock). + // + // KNOWN LEAK — documented, not yet fixed. + // + // This is the system's only hard delete, and it cleans + // only what storage owns: the meta.db row, the vector + // slot and the edge rows. The four indexes that live + // outside storage are untouched, because + // `DecaySweepRunner` holds handles to none of them: + // + // - the FTS5 row survives, and survives *restart* — + // system startup only rebuilds the FTS index when it + // is empty, so the row is permanent; + // - the `FlatVectorIndex` entry survives until the + // process exits; + // - the entity-index entry survives until restart. + // + // The damage is degraded recall, not wrong answers: + // search hydrates its candidates from meta.db, and an id + // with no record hydrates to nothing and is dropped. So + // an orphan row spends a top-k slot and returns no + // result. + // + // Fixing it means threading vector_index, fts_index and + // entity_index through `DecaySweepRunner::new`, the + // `start()` clone block, `execute_sweep` and into this + // path — a change to code that runs unattended on a + // background timer, which wants its own review rather + // than a ride along with a delete-path change. + // `FtsIndex::remove_namespace` is the manual remedy in + // the meantime. { let mut storage_w = storage_clone .write() diff --git a/src/graph/explicit_links.rs b/src/graph/explicit_links.rs index 5986d03..ee853ca 100644 --- a/src/graph/explicit_links.rs +++ b/src/graph/explicit_links.rs @@ -478,106 +478,7 @@ mod tests { // ── Live-storage fixtures ──────────────────────────────────────── - /// Embedding width used by the fixture namespace. Small on purpose: - /// these tests are about edge bookkeeping, not vectors. - const DIM: usize = 4; - - struct Fixture { - graph: SharedGraph, - storage: Arc>, - cache: Arc, - _dir: tempfile::TempDir, - } - - impl Fixture { - fn new() -> Self { - let dir = tempfile::tempdir().expect("tempdir"); - let mut engine = RedbStorageEngine::open(dir.path()).expect("open storage"); - let mut config = crate::model::NamespaceConfig::default_namespace(0); - config.embedding_dim = DIM as u32; - if engine - .get_namespace_by_name(&config.name) - .expect("namespace lookup") - .is_none() - { - engine.create_namespace(&config).expect("create namespace"); - } - - Self { - graph: Arc::new(tokio::sync::RwLock::new( - crate::graph::RelationshipGraph::new(), - )), - storage: Arc::new(std::sync::RwLock::new(engine)), - cache: Arc::new(crate::cache::CacheManager::new( - crate::cache::CacheConfig { - embedding_dim: DIM, - ..Default::default() - }, - None, - )), - _dir: dir, - } - } - - /// Writes a memory to storage and gives it a graph node, which is - /// the state an explicit edge is applied against. - async fn insert_memory(&self) -> MemoryId { - let id = MemoryId::new(); - let mut record = crate::model::DiskRecord { - version: crate::model::DiskRecord::CURRENT_VERSION, - id: *id.as_bytes(), - namespace_id: ns().get(), - created_at: 0, - last_accessed_at: 0, - phase: crate::model::DecayPhase::Full, - strength: 1.0, - decay_strength: 1.0, - stability: 1.0, - difficulty: 5.0, - is_permastore: 0, - vector_slot: 0, - edge_count: 0, - summary: "fixture".into(), - tags: Vec::new(), - access_history: Vec::new(), - text_offset: 0, - text_length: 0, - }; - - { - let mut storage_w = self.storage.write().expect("storage lock"); - storage_w - .insert_memory(id, ns(), &mut record, &[0.0; DIM], None) - .expect("insert memory"); - } - - let slot = record.vector_slot; - self.graph - .write() - .await - .add_node(id, ns(), crate::model::DecayPhase::Full, 1.0, slot) - .expect("add node"); - id - } - - fn edge_count(&self, id: MemoryId) -> u16 { - self.storage - .read() - .expect("storage lock") - .get_record(id) - .expect("get record") - .expect("record exists") - .edge_count - } - - fn persisted_edges(&self) -> Vec { - self.storage - .read() - .expect("storage lock") - .load_all_edges() - .expect("load edges") - } - } + use crate::test_support::Fixture; /// Regression: a `parentId` edge must move `edge_count`, whichever door /// created it. diff --git a/src/graph/structure.rs b/src/graph/structure.rs index 25e5e3c..1ea6401 100644 --- a/src/graph/structure.rs +++ b/src/graph/structure.rs @@ -635,6 +635,13 @@ impl RelationshipGraph { }; }; let Some(node) = self.nodes.get(node_key) else { + // Same stale-index case as `remove_memory`: the entry has to + // go, or `contains` keeps claiming the memory is here. + tracing::warn!( + memory_id = %memory_id, + "Graph id_index named a node that does not exist; dropping the stale entry" + ); + self.id_index.remove(&memory_id); return RemovalResult { removed_edges: Vec::new(), bridge_created: None, @@ -694,9 +701,50 @@ impl RelationshipGraph { } } - // Remove all edges connected to this node - // Re-fetch the node since we may have mutated other nodes above - let node = self.nodes.get(node_key).expect("node still exists"); + // Remove the node and its edges. Re-resolves the node key, + // since the bridge above may have mutated other nodes. + let removed_edges = self.remove_memory(memory_id); + + RemovalResult { + removed_edges, + bridge_created, + } + } + + /// Remove a memory's node and every edge incident to it, creating + /// no bridge. Returns the removed edges so the caller can delete + /// them from edges.db. + /// + /// Returns an empty vec if the memory has no node. + /// + /// # When to use this instead of bridging + /// + /// Use it when the memory is going away because its whole + /// **namespace** is going away. Bridging exists to preserve a chain + /// through a memory that decayed out from under it — the + /// relationship was real and only the waypoint is gone. In a + /// namespace deletion the relationship is exactly what the user + /// asked to erase, so manufacturing a new edge between two survivors + /// on the strength of it would resurrect, in summary form, the thing + /// that was just destroyed. + pub fn remove_memory(&mut self, memory_id: MemoryId) -> Vec { + let Some(&node_key) = self.id_index.get(&memory_id) else { + return Vec::new(); + }; + let Some(node) = self.nodes.get(node_key) else { + // An id_index key whose node is gone. Bailing out left the + // stale entry in place forever, so `contains` went on saying + // the memory was in the graph and every later removal took + // this same branch and did nothing. Removing it is the whole + // job that is left to do. + tracing::warn!( + memory_id = %memory_id, + "Graph id_index named a node that does not exist; dropping the stale entry" + ); + self.id_index.remove(&memory_id); + return Vec::new(); + }; + let all_edge_keys: Vec = node .outgoing .iter() @@ -726,10 +774,7 @@ impl RelationshipGraph { self.nodes.remove(node_key); self.id_index.remove(&memory_id); - RemovalResult { - removed_edges, - bridge_created, - } + removed_edges } } @@ -833,6 +878,38 @@ mod tests { (graph, ids) } + /// A stale `id_index` entry — a key whose `nodes` slot is gone — + /// used to make both removal paths bail out WITHOUT dropping it. So + /// `contains` went on reporting the memory as present, every later + /// removal took the same branch and did nothing, and the entry + /// outlived the process. + #[test] + fn w72_removal_drops_an_id_index_entry_whose_node_is_gone() { + for with_bridging in [false, true] { + let (mut graph, ids) = graph_with_nodes(1); + let id = ids[0]; + + // Remove the node behind the index's back. + let key = *graph.id_index.get(&id).expect("indexed"); + graph.nodes.remove(key); + assert!( + graph.contains(&id), + "the setup did not create a stale entry" + ); + + if with_bridging { + graph.remove_memory_with_bridging(id); + } else { + graph.remove_memory(id); + } + + assert!( + !graph.contains(&id), + "with_bridging={with_bridging}: the stale id_index entry survived" + ); + } + } + /// Add a Supersedes edge meaning "`new` replaces `old`". fn supersede( graph: &mut RelationshipGraph, @@ -953,4 +1030,53 @@ mod tests { assert_eq!(graph.supersedes_terminal(&b), None); assert_eq!(graph.supersedes_terminal(&c), None); } + + /// The same shape `remove_memory_with_bridging` bridges across — one + /// in, one out, matching type — must produce no bridge here. A + /// namespace deletion erases the relationship, so inventing a new + /// edge between two survivors on the strength of it would resurrect + /// in summary form the thing that was just destroyed. + #[test] + fn w23_remove_memory_removes_the_node_and_its_edges_without_bridging() { + /// A pass-through shape: predecessor -> middle -> successor, one + /// in, one out, same edge type. + fn pass_through() -> (RelationshipGraph, MemoryId, MemoryId, MemoryId) { + let (mut graph, ids) = graph_with_nodes(3); + graph + .add_edge(ids[0], ids[1], EdgeType::Associative, 1.0, false) + .expect("in edge"); + graph + .add_edge(ids[1], ids[2], EdgeType::Associative, 1.0, false) + .expect("out edge"); + (graph, ids[0], ids[1], ids[2]) + } + + // Establish that this shape really is one bridging would bridge, + // so the assertion below is about the new path and not about an + // input that would never have bridged anyway. + let (mut bridging, _, middle, _) = pass_through(); + assert!( + bridging + .remove_memory_with_bridging(middle) + .bridge_created + .is_some(), + "the shape under test is not one that bridging would bridge" + ); + + let (mut graph, predecessor, middle, successor) = pass_through(); + let removed = graph.remove_memory(middle); + + assert_eq!(removed.len(), 2); + assert!(!graph.contains(&middle)); + assert_eq!(graph.edge_count(), 0, "a bridge edge was manufactured"); + assert_eq!(graph.degree(&predecessor), 0); + assert_eq!(graph.degree(&successor), 0); + assert!(graph.contains(&predecessor) && graph.contains(&successor)); + } + + #[test] + fn w23b_remove_memory_of_an_unknown_id_is_empty() { + let (mut graph, _) = graph_with_nodes(1); + assert!(graph.remove_memory(MemoryId::new()).is_empty()); + } } diff --git a/src/health/report.rs b/src/health/report.rs index bce6b3f..29c3cef 100644 --- a/src/health/report.rs +++ b/src/health/report.rs @@ -322,10 +322,17 @@ pub async fn compute_metadata_stats( storage: &dyn StorageEngine, _namespace_filter: Option, ) -> Result> { - // list_tags returns all tags sorted by count descending - let all_tags = storage.list_tags().await?; + let mut all_tags = storage.list_tags().await?; let unique_tags = all_tags.len() as u64; + + // `list_tags` reads a redb B-tree keyed by tag, so it arrives in + // LEXICOGRAPHIC order -- the comment here used to claim it came back + // sorted by count, and the report published the first ten tags + // alphabetically under the heading "top tags". The "top" has to be + // selected here. + crate::model::sort_tag_counts(&mut all_tags); + let top_tags: Vec = all_tags .into_iter() .take(10) diff --git a/src/index_cleanup.rs b/src/index_cleanup.rs new file mode 100644 index 0000000..2a35034 --- /dev/null +++ b/src/index_cleanup.rs @@ -0,0 +1,186 @@ +//! Removing memories from the indexes that storage cannot reach. +//! +//! Deleting a memory is two jobs. Storage owns one of them — the +//! meta.db record, the vector slot, the edge rows — and returns. +//! The other job is the four indexes that live outside storage: the +//! FTS5 database, the in-memory `FlatVectorIndex`, the entity index and +//! the relationship graph. Nothing in `storage/` holds a handle to any +//! of them, so every delete path has to do that half itself. +//! +//! There are three such paths — the MCP tool, the HTTP endpoint, and +//! now namespace deletion — and until this module existed they were +//! three hand-written copies of the same six steps. Two of them agreed; +//! the HTTP one had quietly stopped after the cache invalidation, so an +//! HTTP-deleted memory kept a live vector-index entry (burning a top-k +//! slot in every search and looking like a valid autolink target), kept +//! its FTS row (which survives restart forever), and kept a graph node +//! reporting full strength, propping up its neighbours' decay +//! resistance. +//! +//! One implementation is the fix for that; a comment telling the next +//! author to keep three copies in step is not. +//! +//! # Best-effort by design +//! +//! Every step here runs after the storage-side removal has already +//! committed. There is nothing to roll back to, so a failure is warned +//! about and the remaining steps still run: a wedged FTS connection must +//! not be the reason the vector index keeps serving a deleted memory. + +use crate::cache::CacheManager; +use crate::graph::SharedGraph; +use crate::model::{DecayPhase, DiskRecord, MemoryId, NamespaceId}; +use crate::search::{EntityIndex, FlatVectorIndex, FtsIndex}; + +/// Borrowed handles to the four out-of-storage indexes, plus the cache. +pub(crate) struct IndexHandles<'a> { + /// Record cache — entries for deleted memories must not be served. + pub cache: &'a CacheManager, + /// In-memory vector index used by semantic search. + pub vector_index: &'a tokio::sync::RwLock, + /// SQLite FTS5 index used by keyword search. + pub fts_index: &'a tokio::sync::Mutex, + /// Entity -> memory inverted index. + pub entity_index: &'a tokio::sync::RwLock, + /// Relationship graph. + pub graph: &'a SharedGraph, +} + +/// What to do with each memory's graph node. +pub(crate) enum GraphAction { + /// Keep the node and its edges, set its phase to Tombstone. For + /// `forget_memory`: the record survives, so relationship chains + /// stay traversable. + MarkTombstoned, + /// Remove the node and every edge incident to it, with no bridging. + /// For namespace deletion: the record is gone, so a node pointing at + /// it would be a dangling reference. + RemoveNode, +} + +/// How to reach the FTS rows. +pub(crate) enum FtsAction { + /// One `remove` per supplied record. + PerRecord, + /// One `DELETE ... WHERE namespace_id = ?`. Reaches rows whose + /// meta.db record is already gone, which per-record removal cannot. + WholeNamespace(NamespaceId), +} + +/// What was actually removed, for reporting back to the caller. +#[derive(Debug, Default)] +pub(crate) struct IndexPurgeCounts { + /// FTS documents deleted. + pub fts_removed: usize, + /// Vector-index entries deleted. + pub vector_entries_removed: usize, + /// Graph nodes removed. Always 0 for [`GraphAction::MarkTombstoned`]. + pub graph_nodes_removed: usize, +} + +/// Remove a batch of memories from the cache and the four indexes. +/// +/// Takes **one** lock per subsystem for the whole slice, not one per +/// record. +/// +/// `records` must be the memories as they were **before** any tombstone +/// strip. `tombstone` erases the tag list, and the tag list is where the +/// entity names live — pass the stripped record and the entity index +/// keeps pointing at the deleted memory forever. +pub(crate) async fn purge_from_indexes( + handles: &IndexHandles<'_>, + records: &[(MemoryId, DiskRecord)], + fts: FtsAction, + graph: GraphAction, +) -> IndexPurgeCounts { + let mut counts = IndexPurgeCounts::default(); + + if records.is_empty() && matches!(fts, FtsAction::PerRecord) { + return counts; + } + + let ids: Vec = records.iter().map(|(id, _)| *id).collect(); + + // 1. Cache. + handles.cache.batch_invalidate(&ids).await; + + // 2. FTS5. + { + let fts_index = handles.fts_index.lock().await; + match fts { + FtsAction::PerRecord => { + for id in &ids { + match fts_index.remove(*id) { + Ok(true) => counts.fts_removed += 1, + Ok(false) => {} + Err(e) => tracing::warn!( + memory_id = %id, + %e, + "FTS5 removal failed (non-fatal)" + ), + } + } + } + FtsAction::WholeNamespace(namespace_id) => { + match fts_index.remove_namespace(namespace_id) { + Ok(removed) => counts.fts_removed = removed, + Err(e) => tracing::warn!( + namespace_id = namespace_id.get(), + %e, + "FTS5 namespace removal failed (non-fatal)" + ), + } + } + } + } + + // 3. Vector index. + { + use crate::search::VectorIndex; + let mut vector_index = handles.vector_index.write().await; + for id in &ids { + match vector_index.remove(*id) { + Ok(true) => counts.vector_entries_removed += 1, + Ok(false) => {} + Err(e) => tracing::warn!( + memory_id = %id, + %e, + "vector index removal failed (non-fatal)" + ), + } + } + } + + // 4. Entity index. Driven off the pre-strip tags. + { + let mut entity_index = handles.entity_index.write().await; + for (id, record) in records { + let metadata = crate::model::parse_structured_tags(&record.tags); + if !metadata.entities.is_empty() { + entity_index.remove(*id, &metadata.entities); + } + } + } + + // 5. Graph. + { + let mut graph_w = handles.graph.write().await; + for id in &ids { + match graph { + GraphAction::MarkTombstoned => { + // Err means the node is not in the graph, which is + // not a problem: there is nothing left to mark. + let _ = graph_w.update_node_state(*id, DecayPhase::Tombstone, 0.0); + } + GraphAction::RemoveNode => { + if graph_w.contains(id) { + graph_w.remove_memory(*id); + counts.graph_nodes_removed += 1; + } + } + } + } + } + + counts +} diff --git a/src/lib.rs b/src/lib.rs index f5e40cd..93f5e02 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -34,6 +34,7 @@ pub mod embedding; pub mod error; pub mod graph; pub mod health; +pub(crate) mod index_cleanup; pub mod mcp; pub mod model; pub mod rif; @@ -41,6 +42,8 @@ pub mod search; pub mod serialization; pub mod storage; pub mod system; +#[cfg(test)] +pub(crate) mod test_support; pub mod time; // ── Top-level re-exports ───────────────────────────────────────────── diff --git a/src/main.rs b/src/main.rs index 8fcdac9..6ab1063 100644 --- a/src/main.rs +++ b/src/main.rs @@ -435,6 +435,10 @@ async fn run_serve( let storage: Arc = Arc::new(StorageEngineAdapter::new( system.storage().clone(), system.cache().clone(), + system.vector_index().clone(), + system.fts_index().clone(), + system.entity_index().clone(), + system.graph().clone(), )); let cache: Arc = Arc::new(RecordCacheAdapter::new(system.cache().clone())); @@ -563,7 +567,15 @@ fn create_direct_mcp_bridge( tz, )); let namespaces: Arc = - Arc::new(McpNamespaceAdapter::new(system.storage().clone(), tz)); + Arc::new(McpNamespaceAdapter::new( + system.storage().clone(), + system.cache().clone(), + system.vector_index().clone(), + system.fts_index().clone(), + system.entity_index().clone(), + system.graph().clone(), + tz, + )); let health: Arc = Arc::new(McpHealthAdapter::new(system.storage().clone())); diff --git a/src/mcp/args.rs b/src/mcp/args.rs index 284cfc3..a3290a4 100644 --- a/src/mcp/args.rs +++ b/src/mcp/args.rs @@ -1,4 +1,4 @@ -//! Strict argument parsing for the MCP store tools. +//! Strict argument parsing for the MCP tools. //! //! The store handlers used to pull their fields out of the raw //! `serde_json::Value` with `.get(k).and_then(|v| v.as_str())` and @@ -15,7 +15,9 @@ //! three things and delegates the fourth: //! //! 1. **Extraction** with explicit types ([`require_str`], [`opt_str`], -//! [`opt_string_array`], [`opt_uuid`], [`require_uuid`]). +//! [`opt_string_array`], [`opt_uuid`], [`require_uuid`], +//! [`opt_u64_in_range`], [`opt_usize_in_range`], [`opt_bool`], +//! [`namespace_or`]). //! 2. **Type strictness** — a wrong-typed argument is an error naming //! the field and the type that arrived, never a default. //! 3. **Markup detection** ([`scan_client_markup`]) — text fields that @@ -24,8 +26,14 @@ //! [`crate::model::validation::validate_memory_input`], which the //! HTTP API shares. //! +//! A *numeric* bound is treated the same way. An out-of-range `limit` is +//! rejected rather than clamped: the tool schema already declares the +//! maximum, so only a schema-violating client can hit it, and quietly +//! returning 200 rows to a caller who asked for 1000 makes a paginating +//! agent believe it has enumerated the namespace. +//! //! Nothing here needs a bridge, a runtime, or storage: the composite -//! parser takes the default namespace as a `&str`, which is what makes +//! parsers take the default namespace as a `&str`, which is what makes //! the whole boundary unit-testable with no mocks. //! //! # Explicit JSON `null` means "absent" @@ -38,8 +46,11 @@ use serde_json::Value; -use crate::mcp::bridge::StoreInput; +use crate::mcp::bridge::{ListMemoriesInput, ListTagsInput, SearchInput, StoreInput}; use crate::model::MemoryId; +use crate::model::constants::{ + DEFAULT_LIST_LIMIT, DEFAULT_TAG_LIMIT, MAX_LIST_LIMIT, MAX_TAG_LIMIT, +}; use crate::model::validation::{MemoryInputRef, truncate_on_char_boundary, validate_memory_input}; // ═══════════════════════════════════════════════════════════════════════ @@ -242,6 +253,128 @@ pub fn require_uuid(obj: &Value, key: &str) -> Result { } } +/// Extract an optional integer parameter and check an inclusive range. +/// +/// A numeric *string* is an error, not a coercion: `{"limit": "50"}` used +/// to become 50 silently, the same class of bug as a comma-joined `tags` +/// string storing zero tags. +/// +/// Out-of-range is also an error rather than a clamp. The schema already +/// declares the bound, so only a schema-violating client can hit it, and +/// silently returning 200 rows to a caller who asked for 1000 makes a +/// paginating agent believe it has seen the whole namespace. +pub fn opt_u64_in_range( + obj: &Value, + key: &str, + min: u64, + max: u64, +) -> Result, ArgError> { + let Some(v) = field_opt(obj, key) else { + return Ok(None); + }; + let Value::Number(n) = v else { + return Err(ArgError::new( + key, + format!( + "Parameter '{key}' must be an integer (got {})", + json_type_name(v) + ), + )); + }; + let Some(value) = n.as_u64() else { + // Negative and fractional both land here. The literal is echoed + // through `elide` because JSON permits arbitrarily long numbers. + return Err(ArgError::new( + key, + format!( + "Parameter '{key}' must be a non-negative whole number (got {})", + elide(&n.to_string(), ECHO_MAX_BYTES) + ), + )); + }; + if value < min || value > max { + return Err(ArgError::new( + key, + format!("Parameter '{key}' must be between {min} and {max} (got {value})"), + )); + } + Ok(Some(value)) +} + +/// [`opt_u64_in_range`] returning `usize`. +/// +/// The cast lives here, done once against an already-checked bound, +/// rather than at four call sites where a truncating `as usize` could +/// hide. `max` is a `usize`, so on any target the range check runs before +/// the narrowing and a value that would wrap is rejected, not wrapped. +pub fn opt_usize_in_range( + obj: &Value, + key: &str, + min: usize, + max: usize, +) -> Result, ArgError> { + Ok(opt_u64_in_range(obj, key, min as u64, max as u64)?.map(|v| v as usize)) +} + +/// Extract an optional boolean parameter. +/// +/// `"true"` and `1` are errors. A client sending either has a broken +/// serializer, and guessing is how a `false` becomes a `true` — which for +/// a scope switch such as `allNamespaces` silently widens what the caller +/// reads. +pub fn opt_bool(obj: &Value, key: &str) -> Result, ArgError> { + match field_opt(obj, key) { + None => Ok(None), + Some(v) => match v.as_bool() { + Some(b) => Ok(Some(b)), + None => Err(ArgError::new( + key, + format!( + "Parameter '{key}' must be a boolean (got {})", + json_type_name(v) + ), + )), + }, + } +} + +/// Extract the shared `namespace` parameter, falling back to `default` +/// only when it is absent or `null`. +/// +/// A *wrong-typed* namespace is an error, never the default partition: +/// reading or writing the wrong partition because an array arrived where +/// a string was expected is precisely the silent-default failure this +/// module exists to remove. One definition of the rule, so the handlers +/// cannot drift. +pub fn namespace_or(obj: &Value, default: &str) -> Result { + Ok(opt_str(obj, "namespace")?.unwrap_or_else(|| default.to_string())) +} + +/// Extract an optional time bound, accepting epoch millis or ISO 8601. +/// +/// [`crate::time::parse_time_value`] maps every non-number, non-string to +/// `None`, which a handler cannot distinguish from "absent" — so +/// `{"timeRangeStart": ["x"]}` was silently ignored and the query widened +/// to the whole namespace. Here the wrong type is an error. +fn opt_time_bound(obj: &Value, key: &str) -> Result, ArgError> { + let Some(v) = field_opt(obj, key) else { + return Ok(None); + }; + match crate::time::parse_time_value(v) { + Some(Ok(ms)) => Ok(Some(ms)), + // Preserves the wording the handler used before this moved here. + Some(Err(e)) => Err(ArgError::new(key, format!("Invalid {key}: {e}"))), + None => Err(ArgError::new( + key, + format!( + "Parameter '{key}' must be an integer (epoch millis) or an ISO 8601 string \ + (got {})", + json_type_name(v) + ), + )), + } +} + // ═══════════════════════════════════════════════════════════════════════ // Client tool-call markup detection // ═══════════════════════════════════════════════════════════════════════ @@ -509,8 +642,8 @@ pub fn reject_client_markup(field: &str, text: &str) -> Result<(), ArgError> { /// Checks run in a fixed order so that an argument object broken in /// several ways always reports the same failure: object-ness, then /// `summary` (presence, type, markup), `fullText` (type, markup), the -/// four array fields, `namespace`, `parentId`, `supersedes`, and -/// finally the shared content limits from +/// four array fields, `namespace`, `parentId`, `supersedes`, +/// `checkDuplicates`, and finally the shared content limits from /// [`validate_memory_input`](crate::model::validation::validate_memory_input). pub fn parse_store_input(item: &Value, default_namespace: &str) -> Result { if !item.is_object() { @@ -533,10 +666,11 @@ pub fn parse_store_input(item: &Value, default_namespace: &str) -> Result Result Result, ArgError> { + let Some(v) = field_opt(obj, key) else { + return Ok(None); + }; + let Some(n) = v.as_f64() else { + return Err(ArgError::new( + key, + format!( + "Parameter '{key}' must be a number (got {})", + json_type_name(v) + ), + )); + }; + if n.is_nan() || n < min || n > max { + return Err(ArgError::new( + key, + format!("Parameter '{key}' must be between {min} and {max} (got {n})"), + )); + } + Ok(Some(n as f32)) +} + +/// Extract an optional string parameter constrained to a fixed set. +/// +/// A wrong-typed or unknown value is an error naming the alternatives. +/// `mode` used to be read with `.and_then(|v| v.as_str())`, so +/// `{"mode": 2}` silently ran "single" — a different operation from the +/// one asked for. +pub fn opt_enum(obj: &Value, key: &str, allowed: &[&str]) -> Result, ArgError> { + let Some(value) = opt_str(obj, key)? else { + return Ok(None); + }; + if !allowed.contains(&value.as_str()) { + return Err(ArgError::new( + key, + format!( + "Parameter '{key}' must be one of {} (got {:?})", + allowed + .iter() + .map(|a| format!("\"{a}\"")) + .collect::>() + .join(", "), + elide(&value, ECHO_MAX_BYTES) + ), + )); + } + Ok(Some(value)) +} + +/// A parsed `recall_memories` call. +pub struct RecallArgs { + /// What the search pipeline is given. + pub input: SearchInput, + /// Whether to render the token-efficient response shape. + pub compact: bool, +} + +/// Parse the arguments of a `recall_memories` call. +/// +/// Only `namespace` was ever checked here. Everything else went through +/// the `.get(k).and_then(..)` chain this module exists to replace, so +/// `{"tags": "topic/rust"}` became `[]` and ran UNFILTERED over the +/// whole namespace, `limit: 1000` was clamped to 100 (telling a +/// paginating caller it had seen everything), `compact: "false"` became +/// `true`, and `depth: 5` sailed past a schema that says 3. +pub fn parse_recall_input(obj: &Value, default_namespace: &str) -> Result { + let query = require_str(obj, "query")?; + let namespace = namespace_or(obj, default_namespace)?; + let limit = + opt_usize_in_range(obj, "limit", 1, MAX_SEARCH_LIMIT)?.unwrap_or(DEFAULT_SEARCH_LIMIT); + let tags = opt_string_array(obj, "tags")?; + let entities = opt_string_array(obj, "entities")?; + let topics = opt_string_array(obj, "topics")?; + let emotions = opt_string_array(obj, "emotions")?; + let min_strength = opt_f32_in_range(obj, "minStrength", 0.0, 1.0)?; + let depth = opt_u64_in_range(obj, "depth", 0, MAX_RECALL_DEPTH)?.unwrap_or(0) as u32; + let time_range_start = opt_time_bound(obj, "timeRangeStart")?; + let time_range_end = opt_time_bound(obj, "timeRangeEnd")?; + let compact = opt_bool(obj, "compact")?.unwrap_or(true); + + Ok(RecallArgs { + input: SearchInput { + query, + namespace, + limit, + tags, + entities, + topics, + emotions, + min_strength, + depth, + time_range_start, + time_range_end, + }, + compact, + }) +} + +/// A parsed `find_similar_memories` call in `single` mode. +pub struct FindSimilarArgs { + /// The memory to find neighbours of. + pub id: MemoryId, + /// Maximum neighbours to return. + pub limit: usize, + /// Minimum similarity, when the caller set one. + pub min_score: Option, + /// Whether to stay inside the source memory's namespace. + pub same_namespace: bool, +} + +/// Parse the `single`-mode arguments of a `find_similar_memories` call. +pub fn parse_find_similar_args(obj: &Value) -> Result { + let id = require_uuid(obj, "id")?; + let limit = + opt_usize_in_range(obj, "limit", 1, MAX_SEARCH_LIMIT)?.unwrap_or(DEFAULT_SEARCH_LIMIT); + let min_score = opt_f32_in_range(obj, "minScore", 0.0, 1.0)?; + let same_namespace = opt_bool(obj, "sameNamespace")?.unwrap_or(true); + + Ok(FindSimilarArgs { + id, + limit, + min_score, + same_namespace, + }) +} + +/// A parsed `find_similar_memories` call in `scan` mode. +pub struct DuplicateScanArgs { + /// The namespace to scan. + pub namespace: String, + /// Similarity at or above which two memories are a duplicate. + pub threshold: f32, +} + +/// Parse the `scan`-mode arguments of a `find_similar_memories` call. +pub fn parse_duplicate_scan_args( + obj: &Value, + default_namespace: &str, + default_threshold: f64, +) -> Result { + let namespace = namespace_or(obj, default_namespace)?; + let threshold = + opt_f32_in_range(obj, "threshold", 0.0, 1.0)?.unwrap_or(default_threshold as f32); + + Ok(DuplicateScanArgs { + namespace, + threshold, + }) +} + +// ═══════════════════════════════════════════════════════════════════════ +// Listing-tool parsers +// ═══════════════════════════════════════════════════════════════════════ + +/// Parse the arguments of a `list_memories` call. +/// +/// Checks run in a fixed order — `namespace`, `limit`, `offset`, `tags`, +/// `entities`, `timeRangeStart`, `timeRangeEnd` — so an argument object +/// broken in several ways always reports the same failure. +/// +/// Three things that used to pass silently now fail: a stringly-typed +/// `limit`, a `limit` above the schema's maximum (previously clamped to +/// [`MAX_LIST_LIMIT`], which told a paginating caller it had seen +/// everything), and a wrong-typed time bound (previously dropped, which +/// widened the query to the whole namespace). +pub fn parse_list_memories_input( + obj: &Value, + default_namespace: &str, +) -> Result { + let namespace = namespace_or(obj, default_namespace)?; + let limit = opt_usize_in_range(obj, "limit", 1, MAX_LIST_LIMIT)?.unwrap_or(DEFAULT_LIST_LIMIT); + let offset = opt_usize_in_range(obj, "offset", 0, usize::MAX)?.unwrap_or(0); + let tags = opt_string_array(obj, "tags")?; + let entities = opt_string_array(obj, "entities")?; + let time_range_start = opt_time_bound(obj, "timeRangeStart")?; + let time_range_end = opt_time_bound(obj, "timeRangeEnd")?; + + Ok(ListMemoriesInput { + namespace, + limit, + offset, + tags, + entities, + time_range_start, + time_range_end, + }) +} + +/// Parse the arguments of a `list_tags` call. +/// +/// `allNamespaces: true` and an explicit `namespace` are mutually +/// exclusive. Letting one silently win would hand back a histogram over a +/// scope the caller did not ask for, and the caller would have no way to +/// tell which rule applied — so the combination is rejected, attributed +/// to `allNamespaces`. +/// +/// `allNamespaces: false` alongside an explicit `namespace` is legal: a +/// client that always serializes the flag is not making a contradictory +/// request. +pub fn parse_list_tags_input( + obj: &Value, + default_namespace: &str, +) -> Result { + let explicit_namespace = opt_str(obj, "namespace")?; + let all_namespaces = opt_bool(obj, "allNamespaces")?.unwrap_or(false); + + if all_namespaces && explicit_namespace.is_some() { + return Err(ArgError::new( + "allNamespaces", + "Parameter 'allNamespaces' cannot be combined with an explicit 'namespace'; \ + pass one or the other", + )); + } + + let namespace = if all_namespaces { + None + } else { + Some(explicit_namespace.unwrap_or_else(|| default_namespace.to_string())) + }; + + let limit = opt_usize_in_range(obj, "limit", 1, MAX_TAG_LIMIT)?.unwrap_or(DEFAULT_TAG_LIMIT); + + Ok(ListTagsInput { namespace, limit }) +} + // ═══════════════════════════════════════════════════════════════════════ // Tests // ═══════════════════════════════════════════════════════════════════════ @@ -1288,6 +1672,51 @@ mod tests { assert_eq!(from_nulls.supersedes, from_absent.supersedes); } + /// The opt-out reaches `StoreInput`. One construction site, so this + /// covers the batch path too — `parse_store_item` delegates here. + #[test] + fn c19_parse_store_input_carries_check_duplicates() { + let item = json!({ "summary": "s", "checkDuplicates": false }); + assert_eq!( + parse_store_input(&item, NS).unwrap().check_duplicates, + Some(false) + ); + + let item = json!({ "summary": "s", "checkDuplicates": true }); + assert_eq!( + parse_store_input(&item, NS).unwrap().check_duplicates, + Some(true) + ); + } + + /// Absent means unset, not `false`. The adapter reads unset as + /// enabled; a `Some(false)` here would silently disable the check for + /// every caller that never heard of the parameter. + #[test] + fn c20_check_duplicates_is_none_when_absent_or_null() { + assert!( + parse_store_input(&json!({ "summary": "s" }), NS) + .unwrap() + .check_duplicates + .is_none() + ); + assert!( + parse_store_input(&json!({ "summary": "s", "checkDuplicates": null }), NS) + .unwrap() + .check_duplicates + .is_none() + ); + } + + /// `"false"` is a client bug, and JavaScript-style truthiness would + /// read it as `true` — turning an opt-out into an opt-in. + #[test] + fn c18_check_duplicates_rejects_a_stringly_typed_boolean() { + let err = parse_store_input(&json!({ "summary": "s", "checkDuplicates": "false" }), NS) + .unwrap_err(); + assert_eq!(err.field, "checkDuplicates"); + } + #[test] fn d3_wrong_typed_namespace_fails_instead_of_falling_back() { // Reproduces "namespace silently wrong": the old code took the @@ -1527,4 +1956,325 @@ mod tests { let err = parse_store_input(&json!({}), NS).unwrap_err(); assert_eq!(err.to_string(), err.message); } + + // ═══════════════════════════════════════════════════════════════ + // E. Numeric, boolean and namespace extractors + // ═══════════════════════════════════════════════════════════════ + + #[test] + fn e1_opt_u64_treats_absent_and_null_as_none() { + assert_eq!(opt_u64_in_range(&json!({}), "limit", 1, 200).unwrap(), None); + assert_eq!( + opt_u64_in_range(&json!({ "limit": null }), "limit", 1, 200).unwrap(), + None + ); + } + + #[test] + fn e2_opt_u64_accepts_both_inclusive_bounds() { + assert_eq!( + opt_u64_in_range(&json!({ "limit": 1 }), "limit", 1, 200).unwrap(), + Some(1) + ); + assert_eq!( + opt_u64_in_range(&json!({ "limit": 200 }), "limit", 1, 200).unwrap(), + Some(200) + ); + assert_eq!( + opt_u64_in_range(&json!({ "offset": 0 }), "offset", 0, 10).unwrap(), + Some(0) + ); + } + + /// The named regression: `{"limit": "50"}` used to coerce to 50. + #[test] + fn e3_opt_u64_rejects_a_numeric_string() { + let err = opt_u64_in_range(&json!({ "limit": "50" }), "limit", 1, 200).unwrap_err(); + assert_eq!(err.field, "limit"); + assert_eq!( + err.message, + "Parameter 'limit' must be an integer (got string)" + ); + } + + #[test] + fn e4_opt_u64_rejects_bool_array_and_object() { + for (value, name) in [ + (json!(true), "boolean"), + (json!([]), "array"), + (json!({}), "object"), + ] { + let err = opt_u64_in_range(&json!({ "limit": value }), "limit", 1, 200).unwrap_err(); + assert_eq!( + err.message, + format!("Parameter 'limit' must be an integer (got {name})") + ); + } + } + + #[test] + fn e5_opt_u64_rejects_negative_and_fractional() { + let err = opt_u64_in_range(&json!({ "limit": -1 }), "limit", 1, 200).unwrap_err(); + assert_eq!(err.field, "limit"); + assert_eq!( + err.message, + "Parameter 'limit' must be a non-negative whole number (got -1)" + ); + + let err = opt_u64_in_range(&json!({ "limit": 1.5 }), "limit", 1, 200).unwrap_err(); + assert_eq!(err.field, "limit"); + assert_eq!( + err.message, + "Parameter 'limit' must be a non-negative whole number (got 1.5)" + ); + } + + #[test] + fn e6_opt_u64_rejects_out_of_range_at_both_ends() { + let err = opt_u64_in_range(&json!({ "limit": 0 }), "limit", 1, 200).unwrap_err(); + assert_eq!( + err.message, + "Parameter 'limit' must be between 1 and 200 (got 0)" + ); + + let err = opt_u64_in_range(&json!({ "limit": 500 }), "limit", 1, 200).unwrap_err(); + assert_eq!( + err.message, + "Parameter 'limit' must be between 1 and 200 (got 500)" + ); + } + + /// Mirrors `c_opt_uuid_elides_a_long_bad_value`: a hostile literal + /// must not become a hostile error message. + #[test] + fn e7_opt_u64_keeps_the_message_bounded_for_a_huge_literal() { + let huge: Value = serde_json::from_str(&format!(r#"{{"limit": {}}}"#, "9".repeat(300))) + .expect("parses as a JSON number"); + let err = opt_u64_in_range(&huge, "limit", 1, 200).unwrap_err(); + assert_eq!(err.field, "limit"); + assert!(err.message.len() < 140, "{err}"); + } + + /// `u64::MAX` must be *rejected* by the range check, never narrowed + /// into a small `usize` by a truncating cast. + #[test] + fn e8_opt_usize_never_truncates() { + let err = opt_usize_in_range(&json!({ "limit": u64::MAX }), "limit", 1, 200).unwrap_err(); + assert_eq!( + err.message, + format!( + "Parameter 'limit' must be between 1 and 200 (got {})", + u64::MAX + ) + ); + assert_eq!( + opt_usize_in_range(&json!({ "limit": 7 }), "limit", 1, 200).unwrap(), + Some(7) + ); + } + + #[test] + fn e9_opt_bool_treats_absent_and_null_as_none() { + assert_eq!(opt_bool(&json!({}), "allNamespaces").unwrap(), None); + assert_eq!( + opt_bool(&json!({ "allNamespaces": null }), "allNamespaces").unwrap(), + None + ); + } + + #[test] + fn e10_opt_bool_accepts_true_and_false() { + assert_eq!( + opt_bool(&json!({ "allNamespaces": true }), "allNamespaces").unwrap(), + Some(true) + ); + assert_eq!( + opt_bool(&json!({ "allNamespaces": false }), "allNamespaces").unwrap(), + Some(false) + ); + } + + #[test] + fn e11_opt_bool_rejects_stringly_and_numeric_booleans() { + let err = opt_bool(&json!({ "allNamespaces": "true" }), "allNamespaces").unwrap_err(); + assert_eq!(err.field, "allNamespaces"); + assert_eq!( + err.message, + "Parameter 'allNamespaces' must be a boolean (got string)" + ); + + let err = opt_bool(&json!({ "allNamespaces": 1 }), "allNamespaces").unwrap_err(); + assert_eq!( + err.message, + "Parameter 'allNamespaces' must be a boolean (got number)" + ); + } + + #[test] + fn e12_namespace_or_falls_back_only_when_absent_or_null() { + assert_eq!(namespace_or(&json!({}), NS).unwrap(), NS); + assert_eq!(namespace_or(&json!({ "namespace": null }), NS).unwrap(), NS); + assert_eq!( + namespace_or(&json!({ "namespace": "personal" }), NS).unwrap(), + "personal" + ); + } + + /// The silent-default failure this module exists to remove: a + /// wrong-typed namespace must not read or write the default + /// partition. + #[test] + fn e13_namespace_or_rejects_a_wrong_typed_namespace() { + let err = namespace_or(&json!({ "namespace": ["work"] }), NS).unwrap_err(); + assert_eq!(err.field, "namespace"); + assert_eq!( + err.message, + "Parameter 'namespace' must be a string (got array)" + ); + } + + /// Documents the choice: an empty namespace is passed through rather + /// than silently defaulted, so it fails downstream as "not found" + /// naming what the caller actually sent. + #[test] + fn e14_namespace_or_passes_an_empty_string_through() { + assert_eq!(namespace_or(&json!({ "namespace": "" }), NS).unwrap(), ""); + } + + // ── parse_list_memories_input ──────────────────────────────────── + + #[test] + fn e15_list_memories_defaults_when_everything_is_absent() { + let input = parse_list_memories_input(&json!({}), NS).unwrap(); + assert_eq!(input.namespace, NS); + assert_eq!(input.limit, DEFAULT_LIST_LIMIT); + assert_eq!(input.offset, 0); + assert!(input.tags.is_empty()); + assert!(input.entities.is_empty()); + assert_eq!(input.time_range_start, None); + assert_eq!(input.time_range_end, None); + } + + #[test] + fn e16_list_memories_rejects_a_stringly_typed_limit() { + let err = parse_list_memories_input(&json!({ "limit": "50" }), NS).unwrap_err(); + assert_eq!(err.field, "limit"); + assert_eq!( + err.message, + "Parameter 'limit' must be an integer (got string)" + ); + } + + /// Pins the intentional behaviour change: an over-maximum `limit` is + /// an error, NOT a silent clamp to `MAX_LIST_LIMIT`. + #[test] + fn e17_list_memories_rejects_a_limit_above_the_maximum_rather_than_clamping() { + let err = parse_list_memories_input(&json!({ "limit": 1000 }), NS).unwrap_err(); + assert_eq!(err.field, "limit"); + assert_eq!( + err.message, + format!("Parameter 'limit' must be between 1 and {MAX_LIST_LIMIT} (got 1000)") + ); + + // The boundary itself is still accepted. + assert_eq!( + parse_list_memories_input(&json!({ "limit": MAX_LIST_LIMIT }), NS) + .unwrap() + .limit, + MAX_LIST_LIMIT + ); + } + + #[test] + fn e18_list_memories_rejects_bare_string_tags() { + let err = parse_list_memories_input(&json!({ "tags": "a,b" }), NS).unwrap_err(); + assert_eq!(err.field, "tags"); + assert!(err.message.contains("array of strings"), "{err}"); + } + + #[test] + fn e19_list_memories_accepts_epoch_millis_and_iso_8601_bounds() { + let input = + parse_list_memories_input(&json!({ "timeRangeStart": 1_700_000_000_000i64 }), NS) + .unwrap(); + assert_eq!(input.time_range_start, Some(1_700_000_000_000)); + + let input = + parse_list_memories_input(&json!({ "timeRangeEnd": "2026-01-01T00:00:00Z" }), NS) + .unwrap(); + assert!(input.time_range_end.is_some()); + } + + /// At HEAD this was silently ignored, widening the query to the whole + /// namespace. + #[test] + fn e20_list_memories_rejects_a_wrong_typed_time_bound() { + let err = parse_list_memories_input(&json!({ "timeRangeStart": ["x"] }), NS).unwrap_err(); + assert_eq!(err.field, "timeRangeStart"); + assert_eq!( + err.message, + "Parameter 'timeRangeStart' must be an integer (epoch millis) or an ISO 8601 \ + string (got array)" + ); + + let err = parse_list_memories_input(&json!({ "timeRangeEnd": true }), NS).unwrap_err(); + assert_eq!(err.field, "timeRangeEnd"); + assert!(err.message.ends_with("(got boolean)"), "{err}"); + } + + #[test] + fn e21_list_memories_keeps_the_invalid_time_prefix_for_an_unparseable_string() { + let err = + parse_list_memories_input(&json!({ "timeRangeStart": "not a date" }), NS).unwrap_err(); + assert_eq!(err.field, "timeRangeStart"); + assert!(err.message.starts_with("Invalid timeRangeStart: "), "{err}"); + } + + // ── parse_list_tags_input ──────────────────────────────────────── + + #[test] + fn e22_list_tags_defaults_when_everything_is_absent() { + let input = parse_list_tags_input(&json!({}), NS).unwrap(); + assert_eq!(input.namespace.as_deref(), Some(NS)); + assert_eq!(input.limit, DEFAULT_TAG_LIMIT); + } + + #[test] + fn e23_list_tags_all_namespaces_clears_the_namespace() { + let input = parse_list_tags_input(&json!({ "allNamespaces": true }), NS).unwrap(); + assert_eq!(input.namespace, None); + } + + /// Neither scope silently wins: the contradiction is reported, and it + /// is attributed to the flag that caused it. + #[test] + fn e24_list_tags_rejects_all_namespaces_with_an_explicit_namespace() { + let err = parse_list_tags_input(&json!({ "allNamespaces": true, "namespace": "work" }), NS) + .unwrap_err(); + assert_eq!(err.field, "allNamespaces"); + assert!(err.message.contains("cannot be combined"), "{err}"); + } + + #[test] + fn e25_list_tags_rejects_a_limit_above_the_maximum() { + let err = parse_list_tags_input(&json!({ "limit": MAX_TAG_LIMIT + 1 }), NS).unwrap_err(); + assert_eq!(err.field, "limit"); + assert_eq!( + err.message, + format!( + "Parameter 'limit' must be between 1 and {MAX_TAG_LIMIT} (got {})", + MAX_TAG_LIMIT + 1 + ) + ); + } + + /// A client that always serializes the flag is not contradicting + /// itself, so `false` plus an explicit namespace is legal. + #[test] + fn e26_list_tags_allows_all_namespaces_false_beside_an_explicit_namespace() { + let input = + parse_list_tags_input(&json!({ "allNamespaces": false, "namespace": "work" }), NS) + .unwrap(); + assert_eq!(input.namespace.as_deref(), Some("work")); + } } diff --git a/src/mcp/bridge.rs b/src/mcp/bridge.rs index 051745d..7df3c8d 100644 --- a/src/mcp/bridge.rs +++ b/src/mcp/bridge.rs @@ -61,6 +61,15 @@ pub trait StorageEngine: Send + Sync { /// Delete a memory by ID. Returns `true` if the memory existed and was deleted. async fn delete_memory(&self, id: MemoryId) -> Result; + /// Delete many memories in one pass. + /// + /// Returns one flag per input id, in input order. `false` means the + /// memory was missing **or** already tombstoned — the same + /// conflation [`delete_memory`](Self::delete_memory) already makes, + /// kept rather than replaced with a third state a caller would have + /// to learn for the batch alone. + async fn delete_memories(&self, ids: &[MemoryId]) -> Result, BridgeError>; + /// Reinforce a memory with the given FSRS quality rating (1-4). async fn reinforce_memory( &self, @@ -73,6 +82,13 @@ pub trait StorageEngine: Send + Sync { &self, input: ListMemoriesInput, ) -> Result; + + /// Tag vocabulary with per-label memory counts. + /// + /// On `StorageEngine` rather than `NamespaceRegistry` because + /// `list_tags` already lives on the storage traits and the tag index + /// is storage. + async fn list_tags(&self, input: ListTagsInput) -> Result; } /// Namespace registry interface. @@ -89,6 +105,17 @@ pub trait NamespaceRegistry: Send + Sync { /// Get detailed statistics for a namespace by name. async fn namespace_stats(&self, name: &str) -> Result; + + /// Destroy a namespace and every memory in it. + /// + /// **Destructive and irreversible.** There is no undo and no backup. + /// Implementations must refuse the default namespace outright and + /// must refuse a namespace still holding live memories unless + /// `force` is set. + async fn delete_namespace( + &self, + input: DeleteNamespaceInput, + ) -> Result; } /// Health check interface. @@ -265,6 +292,26 @@ pub struct StoreInput { pub parent_id: Option, /// ID of an older memory this one replaces. pub supersedes: Option, + /// Whether to check the new memory against existing ones and report + /// near-duplicates. Absent means enabled. + /// + /// `Option` rather than `bool` with a `default_true` function + /// is the load-bearing choice. This struct is deserialized straight + /// off the daemon socket, so an older client sends a payload with no + /// `checkDuplicates` key; serde already decodes a missing + /// `Option` as `None` — as the un-attributed `embedding` and + /// `initial_stability` beside it demonstrate — so "unset" arrives + /// intact and means "on". A `bool` would need + /// `#[serde(default = "…")]` and a bare `#[serde(default)]` on one + /// would yield `false`, silently disabling the feature for every + /// pre-existing client. + /// + /// The `#[serde(default)]` below is therefore redundant, and kept + /// only so the attribute set matches [`DeleteNamespaceInput::force`], + /// where it IS required. Do not read it as evidence that a + /// non-`Option` field is safe without one. + #[serde(default)] + pub check_duplicates: Option, } /// Result of storing a memory. @@ -286,12 +333,21 @@ pub struct StoredMemory { /// What became of the requested `supersedes` link, when one was asked /// for. Absent when the store named no target. /// - /// `#[serde(default)]` is load-bearing, not decoration: this struct - /// round-trips through the daemon protocol, so a new client talking to - /// an older daemon receives a payload without this key and must still - /// decode it. + /// `skip_serializing_if` is the load-bearing half: it keeps the key + /// OUT of the response when there is nothing to report, so existing + /// clients see the shape they saw before this field existed. The + /// `default` is redundant — serde decodes a missing `Option` as + /// `None` on its own — and is kept only for symmetry with the pair + /// below it. #[serde(default, skip_serializing_if = "Option::is_none")] pub supersedes: Option, + /// Existing memories that closely resemble this one. + /// + /// Absent when nothing crossed the threshold, when the caller passed + /// `checkDuplicates: false`, or when the namespace's vectors are not + /// L2-normalized. Advisory only: the memory was stored regardless. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub near_duplicates: Option, } /// A full memory record returned by get. @@ -360,6 +416,68 @@ pub struct ListMemoriesInput { pub time_range_end: Option, } +/// Input for a tag-vocabulary listing. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ListTagsInput { + /// Namespace to count within, or `None` for every namespace. + pub namespace: Option, + /// Maximum labels returned *per bucket*. Each bucket also reports the + /// untruncated total, so a truncated answer still states its own size. + pub limit: usize, +} + +/// One label and the number of live memories carrying it. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TagCount { + /// The label. In the `entities`, `topics` and `emotions` buckets the + /// derived prefix is **stripped**, so the value can be passed + /// straight back to `store_memory` or `list_memories` — which is not + /// true of the prefixed form, since those tools add the prefix + /// themselves. + pub name: String, + /// Live memories carrying this label. + pub count: u64, +} + +/// One bucket of a tag listing. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TagBucket { + /// Distinct labels in this bucket *before* truncation, so a truncated + /// answer still states its own size. + pub total: u64, + /// Whether `items` is shorter than `total`. + pub truncated: bool, + /// Count descending, ties broken on name ascending. + pub items: Vec, +} + +/// Tag vocabulary split into plain and derived buckets. +/// +/// Split rather than flat because in a real corpus every memory carries +/// several `entity/` tags, so a single top-50 list is nearly all entities +/// and useless for discovering the plain tags a caller would filter on. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ListTagsResponse { + /// The namespace counted, or `null` when every namespace was counted. + pub namespace: Option, + /// Live memories inspected. `null` in global scope, where the tag + /// index is read directly and no memory records are touched. + pub memories_counted: Option, + /// Labels that are not derived from `entities`, `topics` or + /// `emotions`. + pub tags: TagBucket, + /// Labels derived from `entities`, prefix stripped. + pub entities: TagBucket, + /// Labels derived from `topics`, prefix stripped. + pub topics: TagBucket, + /// Labels derived from `emotions`, prefix stripped. + pub emotions: TagBucket, +} + /// A single memory entry in a list response (lightweight, no full_text). #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -410,6 +528,53 @@ pub struct CreateNamespaceInput { pub decay_rate_multiplier: Option, } +/// Input for destroying a namespace. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DeleteNamespaceInput { + /// Name of the namespace to destroy. + pub name: String, + /// Destroy the namespace even if it still holds live memories. + /// + /// `#[serde(default)]` is required: this struct is deserialized + /// straight off the daemon socket, an older client sends a payload + /// with no `force` key, and `bool` has no "missing" representation — + /// so without the attribute such a call is a hard parse failure. + /// (`Option` fields elsewhere in this module get that for free from + /// serde and carry the attribute only for symmetry; this one does + /// not.) Defaulting to `false` is also the safe direction — a + /// missing key can only ever make the operation more cautious. + #[serde(default)] + pub force: bool, +} + +/// What a namespace deletion destroyed. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DeleteNamespaceResult { + /// Name of the namespace that was destroyed. + pub name: String, + /// Its numeric id. Never reused. + pub id: u32, + /// Always true on success; present so the response shape matches + /// `forget_memory`. + pub deleted: bool, + /// Live memories destroyed. + pub memories_deleted: u64, + /// Tombstoned records reclaimed. These held no content — they had + /// already been forgotten — but nothing else in the system ever + /// removes them, so they are reported separately rather than folded + /// into `memoriesDeleted`. + pub tombstones_purged: u64, + /// Graph edges removed, including edges that pointed into the + /// namespace from outside it. + pub edges_removed: u64, + /// Full-text search rows removed. + pub fts_rows_removed: u64, + /// Whether the namespace's vector file was removed from disk. + pub vector_file_removed: bool, +} + /// Namespace information. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -432,17 +597,34 @@ pub struct NamespaceInfo { pub struct NamespaceStats { /// Namespace name. pub name: String, - /// Total number of memories. + /// Number of live memories. Excludes tombstoned records, and always + /// equals `phase_counts.full + .summary + .ghost`. pub memory_count: u64, + /// Number of tombstoned records still occupying a metadata row. + /// + /// Reported rather than silently dropped: a namespace showing one + /// live memory where eleven were stored has ten tombstones, and an + /// agent auditing it needs to see that rather than guess whether + /// something was lost to a bug. + /// + /// `#[serde(default)]` is required here, unlike the `Option` fields + /// elsewhere in this module: `u64` has no "missing" representation, + /// so without the attribute a new client decoding an old daemon's + /// response — which has no `tombstoneCount` — fails outright, which + /// is strictly worse than the zero it reports instead. + #[serde(default)] + pub tombstone_count: u64, /// Memory counts broken down by decay phase. pub phase_counts: PhaseCounts, - /// Number of memories in permastore. + /// Number of live memories in permastore. pub permastore_count: u64, - /// Average memory strength across all memories. + /// Average memory strength across the live memories. pub avg_strength: f32, - /// Total number of graph edges. + /// Total number of graph edges on the live memories. pub edge_count: u64, - /// Total bytes used by vector storage. + /// Logical bytes of live vector data + /// (`memory_count` x `embeddingDim` x 4). NOT the size of + /// `vectors.dat`, which retains freed slots until compaction. pub vector_bytes: u64, } @@ -508,6 +690,18 @@ pub enum BridgeError { /// The request or response exceeded the daemon's frame size limit. #[error("Too large: {0}")] TooLarge(String), + + /// The operation failed after it had already changed something, and + /// the change is not undoable. + /// + /// Distinct from [`Storage`](Self::Storage) because the two demand + /// opposite reactions from the agent reading the message: a storage + /// failure means "nothing happened, try again or give up", and this + /// means "something was destroyed, re-run to finish the job". A + /// caller that renders both as a bare failure tells its user the + /// destroyed records are still there. + #[error("Partially applied: {0}")] + PartiallyApplied(String), } impl BridgeError { @@ -641,8 +835,10 @@ mod tests { /// Back-compat guard for the daemon protocol. `StoredMemory` is /// round-tripped through serde between client and daemon, so a new - /// client must still decode a response from a daemon that predates the - /// `supersedes` field. This is what `#[serde(default)]` buys. + /// client must still decode a response from a daemon that predates + /// the `supersedes` field. (Serde gives an `Option` field that for + /// free; the test is here because the guarantee matters, not because + /// an attribute is doing the work.) #[test] fn stored_memory_decodes_a_payload_without_the_supersedes_field() { let decoded: StoredMemory = @@ -703,4 +899,285 @@ mod tests { assert_eq!(outcome.status, SupersedesStatus::AppliedNotDurable); assert_eq!(outcome.detail.as_deref(), Some("disk write failed")); } + + // ── near-duplicate wire format ─────────────────────────────────── + + fn store_input_json() -> serde_json::Value { + json!({ + "summary": "a fact", + "fullText": null, + "tags": [], + "namespace": "default", + "embedding": null, + "initialStability": null, + "parentId": null, + "supersedes": null, + }) + } + + /// Response-side back-compat, matching the `supersedes` guard above: a + /// new client must decode a response from a daemon that predates the + /// field. + #[test] + fn b11_stored_memory_decodes_a_payload_without_near_duplicates() { + let decoded: StoredMemory = + serde_json::from_value(stored_memory_json()).expect("old daemon payload decodes"); + + assert!(decoded.near_duplicates.is_none()); + } + + /// A store with nothing to report serialises exactly as before, so an + /// existing client sees an unchanged response shape rather than a new + /// `"nearDuplicates": null`. + #[test] + fn b12_an_absent_report_is_omitted_from_the_output() { + let stored: StoredMemory = + serde_json::from_value(stored_memory_json()).expect("payload decodes"); + + let encoded = serde_json::to_value(&stored).expect("encodes"); + assert!(encoded.get("nearDuplicates").is_none()); + } + + /// The key names are the wire contract; camelCase on every level. + #[test] + fn b13_a_report_round_trips_with_camel_case_keys() { + let target = MemoryId::new(); + let mut stored: StoredMemory = + serde_json::from_value(stored_memory_json()).expect("payload decodes"); + stored.near_duplicates = crate::model::NearDuplicateReport::from_matches( + 0.85, + vec![crate::model::NearDuplicateMatch { + id: target, + score: 0.93, + summary: "an older phrasing of the same fact".to_string(), + }], + ); + + let encoded = serde_json::to_value(&stored).expect("encodes"); + let report = &encoded["nearDuplicates"]; + // `threshold` is `f64`, so the round number stays round. `score` + // is a genuine `f32` similarity and widens on the way out, which + // is why only this one is compared approximately. + assert_eq!(report["threshold"], json!(0.85)); + assert_eq!(report["matches"][0]["id"], json!(target.to_string())); + let score = report["matches"][0]["score"] + .as_f64() + .expect("score is a number"); + assert!((score - 0.93).abs() < 1e-6, "{score}"); + assert_eq!( + report["matches"][0]["summary"], + json!("an older phrasing of the same fact") + ); + + let decoded: StoredMemory = serde_json::from_value(encoded).expect("decodes"); + let report = decoded.near_duplicates.expect("report preserved"); + assert_eq!(report.matches.len(), 1); + assert_eq!(report.matches[0].id, target); + } + + /// The REQUEST-side guard. `StoreInput` is deserialized straight off + /// the daemon socket, so an older client sends a payload without + /// `checkDuplicates`, and it must decode as "unset" — which means + /// enabled. The `Option` is what carries that: a `bool` with a bare + /// `#[serde(default)]` would decode as `false` and silently disable + /// the feature for every pre-existing client. + #[test] + fn b14_store_input_decodes_a_payload_without_check_duplicates() { + let decoded: StoreInput = + serde_json::from_value(store_input_json()).expect("old client payload decodes"); + + assert!(decoded.check_duplicates.is_none()); + assert_eq!(decoded.summary, "a fact"); + } + + /// The executable form of the note on `check_duplicates`: serde + /// decodes a MISSING `Option` as `None` with no attribute at all. + /// `embedding`, `initialStability`, `parentId` and `supersedes` carry + /// no `#[serde(default)]` and are absent here. + /// + /// Written down because the comments used to claim the attribute was + /// what made these decode, which would tell the next author that a + /// non-`Option` field is equally safe without one. It is not — see + /// `DeleteNamespaceInput::force` and `NamespaceStats::tombstone_count`, + /// where the attribute is the only thing standing between an old + /// payload and a hard parse failure. + #[test] + fn w68_a_missing_option_field_needs_no_serde_default() { + let decoded: StoreInput = serde_json::from_value(json!({ + "summary": "a fact", + "tags": [], + "namespace": "default", + })) + .expect("a payload with every optional key absent decodes"); + + assert!(decoded.embedding.is_none()); + assert!(decoded.initial_stability.is_none()); + assert!(decoded.parent_id.is_none()); + assert!(decoded.supersedes.is_none()); + assert!(decoded.full_text.is_none()); + assert!(decoded.check_duplicates.is_none()); + } + + /// The case where `#[serde(default)]` is genuinely load-bearing: + /// `force` is a `bool`, not an `Option`, so a missing key has + /// no representation and the decode fails outright without it. An + /// older client sends no `force` key, and with the attribute that + /// missing key can only ever make the operation more cautious. + #[test] + fn w44_delete_namespace_input_decodes_a_payload_without_force() { + let decoded: DeleteNamespaceInput = serde_json::from_value(json!({ "name": "scratch" })) + .expect("old client payload decodes"); + + assert_eq!(decoded.name, "scratch"); + assert!( + !decoded.force, + "a missing force key must mean the safe path" + ); + + let decoded: DeleteNamespaceInput = + serde_json::from_value(json!({ "name": "scratch", "force": true })).expect("decodes"); + assert!(decoded.force); + } + + #[test] + fn b15_check_duplicates_round_trips_on_a_store_input() { + let mut payload = store_input_json(); + payload["checkDuplicates"] = json!(false); + + let decoded: StoreInput = serde_json::from_value(payload).expect("decodes"); + assert_eq!(decoded.check_duplicates, Some(false)); + + let encoded = serde_json::to_value(&decoded).expect("encodes"); + assert_eq!(encoded["checkDuplicates"], json!(false)); + } + + // ── namespace_stats wire format ────────────────────────────────── + + /// Back-compat guard for the daemon protocol. `NamespaceStats` is + /// round-tripped through serde in BOTH directions, so a new client + /// must still decode a response from a daemon that predates + /// `tombstoneCount`. `#[serde(default)]` really is doing the work + /// here — `tombstone_count` is a `u64`, which has no "missing" + /// representation — so without it the call fails outright. + #[test] + fn namespace_stats_decodes_a_payload_without_the_tombstone_count() { + let decoded: NamespaceStats = serde_json::from_value(json!({ + "name": "default", + "memoryCount": 3, + "phaseCounts": { "full": 3, "summary": 0, "ghost": 0 }, + "permastoreCount": 0, + "avgStrength": 1.0, + "edgeCount": 0, + "vectorBytes": 128, + })) + .expect("old daemon payload decodes"); + + assert_eq!(decoded.tombstone_count, 0); + assert_eq!(decoded.memory_count, 3); + } + + #[test] + fn namespace_stats_round_trips_the_tombstone_count() { + let stats = NamespaceStats { + name: "default".into(), + memory_count: 1, + tombstone_count: 10, + phase_counts: PhaseCounts { + full: 1, + summary: 0, + ghost: 0, + }, + permastore_count: 0, + avg_strength: 0.8, + edge_count: 0, + vector_bytes: 16, + }; + + let encoded = serde_json::to_value(&stats).expect("encodes"); + assert_eq!(encoded["tombstoneCount"], json!(10)); + + let decoded: NamespaceStats = serde_json::from_value(encoded).expect("decodes"); + assert_eq!(decoded.tombstone_count, 10); + assert_eq!(decoded.memory_count, 1); + } + + // ── list_tags wire format ──────────────────────────────────────── + + fn bucket(items: &[(&str, u64)], total: u64, truncated: bool) -> TagBucket { + TagBucket { + total, + truncated, + items: items + .iter() + .map(|(n, c)| TagCount { + name: (*n).to_string(), + count: *c, + }) + .collect(), + } + } + + fn empty_bucket() -> TagBucket { + bucket(&[], 0, false) + } + + #[test] + fn list_tags_response_round_trips_in_camel_case() { + let response = ListTagsResponse { + namespace: Some("work".into()), + memories_counted: Some(128), + tags: bucket(&[("type/decision", 12)], 41, false), + entities: bucket(&[("recalld", 30)], 120, true), + topics: empty_bucket(), + emotions: empty_bucket(), + }; + + let encoded = serde_json::to_value(&response).expect("encodes"); + assert_eq!(encoded["namespace"], json!("work")); + assert_eq!(encoded["memoriesCounted"], json!(128)); + assert_eq!(encoded["entities"]["truncated"], json!(true)); + assert_eq!(encoded["entities"]["total"], json!(120)); + assert_eq!(encoded["entities"]["items"][0]["name"], json!("recalld")); + assert_eq!(encoded["entities"]["items"][0]["count"], json!(30)); + + let decoded: ListTagsResponse = serde_json::from_value(encoded).expect("decodes"); + assert_eq!(decoded.tags.items[0].name, "type/decision"); + assert_eq!(decoded.memories_counted, Some(128)); + } + + /// Global scope emits EXPLICIT nulls rather than omitting the keys, + /// so the response always states its own scope instead of leaving the + /// caller to infer it from an absence. + #[test] + fn list_tags_response_states_global_scope_with_explicit_nulls() { + let response = ListTagsResponse { + namespace: None, + memories_counted: None, + tags: empty_bucket(), + entities: empty_bucket(), + topics: empty_bucket(), + emotions: empty_bucket(), + }; + + let encoded = serde_json::to_value(&response).expect("encodes"); + assert_eq!(encoded["namespace"], json!(null)); + assert_eq!(encoded["memoriesCounted"], json!(null)); + let obj = encoded.as_object().expect("object"); + assert!(obj.contains_key("namespace")); + assert!(obj.contains_key("memoriesCounted")); + } + + #[test] + fn list_tags_input_round_trips_a_global_scope() { + let input = ListTagsInput { + namespace: None, + limit: 50, + }; + let encoded = serde_json::to_value(&input).expect("encodes"); + assert_eq!(encoded["namespace"], json!(null)); + + let decoded: ListTagsInput = serde_json::from_value(encoded).expect("decodes"); + assert_eq!(decoded.namespace, None); + assert_eq!(decoded.limit, 50); + } } diff --git a/src/mcp/bridge_adapters.rs b/src/mcp/bridge_adapters.rs index bc973d6..45761ab 100644 --- a/src/mcp/bridge_adapters.rs +++ b/src/mcp/bridge_adapters.rs @@ -99,23 +99,33 @@ impl bridge::SearchPipeline for McpSearchAdapter { })??; }; - let mut require_tags: Vec = - crate::model::parse_tags_lossy("recall_memories.tags", &query.tags); - for e in &query.entities { - if let Ok(tag) = crate::model::Tag::new(&format!("entity/{}", e.to_lowercase())) { - require_tags.push(tag); - } - } - for t in &query.topics { - if let Ok(tag) = crate::model::Tag::new(&format!("topic/{}", t.to_lowercase())) { - require_tags.push(tag); - } - } - for em in &query.emotions { - if let Ok(tag) = crate::model::Tag::new(&format!("emotion/{}", em.to_lowercase())) { - require_tags.push(tag); - } - } + // A label that cannot become a tag is an ERROR, not a dropped + // filter: dropping one widens the query, and "every memory in + // the namespace" is indistinguishable from a real result set. + let require_tags = match filter_tags( + &[ + ("tags", "", &query.tags), + ( + "entities", + crate::model::constants::ENTITY_TAG_PREFIX, + &query.entities, + ), + ( + "topics", + crate::model::constants::TOPIC_TAG_PREFIX, + &query.topics, + ), + ( + "emotions", + crate::model::constants::EMOTION_TAG_PREFIX, + &query.emotions, + ), + ], + "recall_memories", + ) { + Ok(tags) => tags, + Err(e) => return Err(bridge::BridgeError::InvalidInput(e)), + }; let pipeline_query = crate::search::SearchQuery { text: Some(query.query), @@ -585,6 +595,109 @@ impl McpStorageAdapter { timezone, } } + + /// Borrow this adapter's out-of-storage indexes for + /// [`crate::index_cleanup::purge_from_indexes`]. + fn index_handles(&self) -> crate::index_cleanup::IndexHandles<'_> { + crate::index_cleanup::IndexHandles { + cache: &self.cache, + vector_index: &self.vector_index, + fts_index: &self.fts_index, + entity_index: &self.entity_index, + graph: &self.graph, + } + } + + /// Turn raw `(id, score)` candidates into a caller-facing report, + /// loading each summary from the cache and falling back to storage. + /// + /// Only runs when candidates were found, so the record loads are + /// conditional on the warning actually firing rather than a cost every + /// store pays. The common case is a cache hit — the memories most + /// likely to duplicate a new one are the ones just written. + /// + /// A summary that cannot be loaded yields an empty string rather than + /// dropping the match: "there is something very like this, id X" is + /// still the useful half of the advisory. + async fn hydrate_near_duplicates( + &self, + candidates: Vec<(MemoryId, f32)>, + threshold: f64, + ) -> Option { + use crate::model::constants::NEAR_DUPLICATE_SUMMARY_MAX_BYTES; + use crate::model::{NearDuplicateMatch, NearDuplicateReport, truncate_summary}; + + if candidates.is_empty() { + return None; + } + + // Summary AND phase. The phase is the load-bearing half: the + // `decay_phases` filter `find_near_duplicates` passes to the + // vector index is INERT, because `FlatVectorIndex` writes each + // entry's `decay_phase` once at add time (always Full) and + // `VectorIndex::update_metadata` has no callers. Without a check + // here, a Ghost — which has no summary left — and a + // hard-deleted memory both get reported as near-duplicates with + // an empty summary, and the tool description then tells the + // agent to reinforce them. + let mut hydrated: std::collections::HashMap = + std::collections::HashMap::new(); + let mut misses: Vec = Vec::new(); + for (id, _) in &candidates { + match self.cache.get(*id).await { + Some(cached) => { + hydrated.insert(*id, (cached.summary.clone(), cached.phase)); + } + None => misses.push(*id), + } + } + + // One blocking hop for every miss, not one per miss. A record + // that is not there at all stays out of `hydrated` and is + // dropped below: it was hard-deleted and the vector index has + // not caught up. + if !misses.is_empty() { + let storage = self.storage.clone(); + let loaded: Vec<(MemoryId, (String, DecayPhase))> = + tokio::task::spawn_blocking(move || { + let storage_r = match storage.read() { + Ok(s) => s, + Err(_) => return Vec::new(), + }; + misses + .iter() + .filter_map(|&mid| { + let record = storage_r.get_record(mid).ok().flatten()?; + Some((mid, (record.summary, record.phase))) + }) + .collect() + }) + .await + .unwrap_or_default(); + hydrated.extend(loaded); + } + + let matches: Vec = candidates + .into_iter() + .filter_map(|(id, score)| { + // Full and Summary only, matching what the (inert) + // index-level filter claims and what the docs promise. + // Ghost has no summary left to show and Tombstone is + // deleted; neither is something to be told it duplicates. + let (summary, phase) = hydrated.get(&id)?; + if !matches!(phase, DecayPhase::Full | DecayPhase::Summary) { + return None; + } + Some(NearDuplicateMatch { + id, + score, + summary: truncate_summary(summary, NEAR_DUPLICATE_SUMMARY_MAX_BYTES), + }) + }) + .collect(); + + NearDuplicateReport::from_matches(threshold, matches) + } } #[async_trait] @@ -594,6 +707,7 @@ impl bridge::StorageEngine for McpStorageAdapter { input: bridge::StoreInput, ) -> Result { use crate::model::Tag; + use crate::model::constants::{DUPLICATE_SIMILARITY_THRESHOLD, MAX_NEAR_DUPLICATE_MATCHES}; use crate::model::record::DiskRecord; // Resolve namespace. @@ -685,7 +799,11 @@ impl bridge::StorageEngine for McpStorageAdapter { Some(emb) => emb, None => { let mut embed_text = match &input.full_text { - Some(ft) => format!("{}\n\n{}", input.summary, ft), + Some(ft) => format!( + "{} +\n{}", + input.summary, ft + ), None => input.summary.clone(), }; if !merged_tags.is_empty() { @@ -698,6 +816,35 @@ impl bridge::StorageEngine for McpStorageAdapter { } }; + // Near-duplicate advisory. + // + // Runs BEFORE the write, for two reasons. The embedding is already + // in hand, so there is no second embed round-trip. And the new + // memory is not in the vector index yet — `index.add` is ~70 lines + // below — so it cannot match itself and no self-filter is needed. + // Moving this below the write silently turns every store into its + // own duplicate at score 1.0. + // + // Nothing here can fail the store: `find_near_duplicates` swallows + // every error path by contract, and `hydrate_near_duplicates` + // degrades to an empty summary rather than propagating. + let near_duplicates = if input.check_duplicates.unwrap_or(true) { + let candidates = { + let index = self.vector_index.read().await; + crate::search::find_near_duplicates( + &index, + &embedding, + ns_config.id, + DUPLICATE_SIMILARITY_THRESHOLD as f32, + MAX_NEAR_DUPLICATE_MATCHES, + ) + }; // index read lock released before the awaits below + self.hydrate_near_duplicates(candidates, DUPLICATE_SIMILARITY_THRESHOLD) + .await + } else { + None + }; + let now = chrono::Utc::now().timestamp_millis(); // Over-long tags cannot reach this point: the shared request // validator bounds every tag and reserves the `entity/` / @@ -935,6 +1082,7 @@ impl bridge::StorageEngine for McpStorageAdapter { stability: record.stability, created_at: format_timestamp(record.created_at, self.timezone), supersedes: supersedes_outcome, + near_duplicates, }) } @@ -1077,50 +1225,93 @@ impl bridge::StorageEngine for McpStorageAdapter { return Ok(false); }; - // 2. Invalidate cache entry. - self.cache.invalidate(id).await; + // 2. Cache and the four indexes storage cannot reach. Shared + // with the HTTP delete path and with namespace deletion, so + // the three cannot drift apart again. `existing_record` is + // the PRE-strip record, which is what keeps the entity names + // reachable. + crate::index_cleanup::purge_from_indexes( + &self.index_handles(), + std::slice::from_ref(&(id, existing_record)), + crate::index_cleanup::FtsAction::PerRecord, + crate::index_cleanup::GraphAction::MarkTombstoned, + ) + .await; - // 3. Remove from FTS5 index. - { - let fts = self.fts_index.lock().await; - if let Err(e) = fts.remove(id) { - tracing::warn!( - memory_id = %id, - %e, - "FTS5 removal failed (non-fatal)" - ); - } - } + Ok(true) + } - // 4. Remove from vector index. - { - use crate::search::VectorIndex; - let mut vi = self.vector_index.write().await; - if let Err(e) = vi.remove(id) { - tracing::warn!( - memory_id = %id, - %e, - "vector index removal failed (non-fatal)" - ); - } + async fn delete_memories(&self, ids: &[MemoryId]) -> Result, bridge::BridgeError> { + if ids.is_empty() { + return Ok(Vec::new()); } - // 5. Remove from entity index. - { - let metadata = crate::model::parse_structured_tags(&existing_record.tags); - if !metadata.entities.is_empty() { - let mut ei = self.entity_index.write().await; - ei.remove(id, &metadata.entities); - } - } + // ONE spawn_blocking holding ONE storage.write() for the whole + // batch. 100 memories go from 100 redb commits (100 fsyncs) to + // one, and from ~700 lock acquisitions to a handful. + // + // `batch_tombstone` is also the single source of truth for + // liveness here: only a `Tombstoned` outcome frees a vector + // slot, and a duplicate id reports `AlreadyTombstoned`, so the + // same slot cannot be freed twice — which would cycle the + // on-disk free list and underflow `live_count`. + let (flags, purged) = { + let storage = self.storage.clone(); + let ids = ids.to_vec(); + tokio::task::spawn_blocking(move || { + let mut storage_w = storage.write().map_err(|e| { + bridge::BridgeError::Internal(format!("storage lock poisoned: {e}")) + })?; - // 6. Update graph node phase to Tombstone (keep node and edges). - { - let mut graph_w = self.graph.write().await; - let _ = graph_w.update_node_state(id, DecayPhase::Tombstone, 0.0); - } + let outcomes = storage_w + .meta_store() + .batch_tombstone(&ids) + .map_err(|e| bridge::BridgeError::Storage(e.to_string()))?; - Ok(true) + let mut flags = Vec::with_capacity(outcomes.len()); + let mut purged: Vec<(MemoryId, crate::model::DiskRecord)> = Vec::new(); + + for (id, outcome) in ids.iter().zip(outcomes) { + match outcome { + crate::storage::metadata::TombstoneOutcome::Tombstoned(record) => { + let ns_id = NamespaceId::new(record.namespace_id); + if let Err(e) = storage_w.free_vector_slot(ns_id, record.vector_slot) { + tracing::warn!( + memory_id = %id, + vector_slot = record.vector_slot, + %e, + "vector slot free failed (non-fatal)" + ); + } + flags.push(true); + purged.push((*id, record)); + } + crate::storage::metadata::TombstoneOutcome::NotFound + | crate::storage::metadata::TombstoneOutcome::AlreadyTombstoned => { + flags.push(false) + } + } + } + + Ok::<_, bridge::BridgeError>((flags, purged)) + }) + .await + .map_err(|e| { + bridge::BridgeError::Internal(format!("blocking task join error: {e}")) + })?? + }; + + // One pass over the indexes for the whole batch, with the + // PRE-strip records so the entity names are still there. + crate::index_cleanup::purge_from_indexes( + &self.index_handles(), + &purged, + crate::index_cleanup::FtsAction::PerRecord, + crate::index_cleanup::GraphAction::MarkTombstoned, + ) + .await; + + Ok(flags) } async fn reinforce_memory( @@ -1245,14 +1436,23 @@ impl bridge::StorageEngine for McpStorageAdapter { &self, input: bridge::ListMemoriesInput, ) -> Result { - // Build tag filters: merge explicit tags + entity/topic tags for AND semantics. - let mut require_tags: Vec = - crate::model::parse_tags_lossy("list_memories.tags", &input.tags); - for entity in &input.entities { - if let Ok(tag) = crate::model::Tag::new(&format!("entity/{}", entity.to_lowercase())) { - require_tags.push(tag); - } - } + // Build tag filters: merge explicit tags + entity tags for AND + // semantics. An unfilterable label is an error — see + // `filter_tags`. + let require_tags = match filter_tags( + &[ + ("tags", "", &input.tags), + ( + "entities", + crate::model::constants::ENTITY_TAG_PREFIX, + &input.entities, + ), + ], + "list_memories", + ) { + Ok(tags) => tags, + Err(e) => return Err(bridge::BridgeError::InvalidInput(e)), + }; // Resolve namespace and query in a single blocking task. let storage = self.storage.clone(); @@ -1310,6 +1510,122 @@ impl bridge::StorageEngine for McpStorageAdapter { limit: input.limit, }) } + + async fn list_tags( + &self, + input: bridge::ListTagsInput, + ) -> Result { + let storage = self.storage.clone(); + let namespace = input.namespace.clone(); + let limit = input.limit; + + tokio::task::spawn_blocking(move || { + let storage_r = storage.read().map_err(|e| { + bridge::BridgeError::Internal(format!("storage lock poisoned: {e}")) + })?; + + let (counts, memories_counted) = match namespace.as_deref() { + Some(name) => { + let ns = storage_r + .get_namespace_by_name(name) + .map_err(|e| bridge::BridgeError::Storage(e.to_string()))? + .ok_or_else(|| { + bridge::BridgeError::NotFound(format!("namespace '{name}' not found")) + })?; + let (counts, memories) = storage_r + .meta_store() + .tag_counts_in_namespace(ns.id) + .map_err(|e| bridge::BridgeError::Storage(e.to_string()))?; + (counts.into_iter().collect::>(), Some(memories)) + } + // Global scope reads TAG_INDEX directly, which is keyed by + // tag alone: no memory records are touched, so there is no + // honest number to report for `memoriesCounted`. + None => ( + storage_r + .list_tags() + .map_err(|e| bridge::BridgeError::Storage(e.to_string()))?, + None, + ), + }; + + Ok(build_list_tags_response( + namespace, + memories_counted, + counts, + limit, + )) + }) + .await + .map_err(|e| bridge::BridgeError::Internal(format!("blocking task join error: {e}")))? + } +} + +/// Split a flat histogram into plain/entity/topic/emotion buckets, order +/// each most-common-first, and truncate to `limit`. +/// +/// A label whose prefix is one of the derived ones moves to that bucket +/// with the prefix stripped; everything else — including a tag that +/// merely *contains* a slash, such as `project/recalld` — stays in +/// `tags`. A bare prefix with an empty label also stays in `tags`, rather +/// than becoming an entity with no name. +/// +/// Pure, so the bucketing rules are testable without storage. +fn build_list_tags_response( + namespace: Option, + memories_counted: Option, + counts: Vec<(String, u64)>, + limit: usize, +) -> bridge::ListTagsResponse { + use crate::model::constants::{EMOTION_TAG_PREFIX, ENTITY_TAG_PREFIX, TOPIC_TAG_PREFIX}; + + let mut plain = Vec::new(); + let mut entities = Vec::new(); + let mut topics = Vec::new(); + let mut emotions = Vec::new(); + + for (name, count) in counts { + let bucketed = [ + (ENTITY_TAG_PREFIX, &mut entities), + (TOPIC_TAG_PREFIX, &mut topics), + (EMOTION_TAG_PREFIX, &mut emotions), + ] + .into_iter() + .find_map(|(prefix, bucket)| { + let label = name.strip_prefix(prefix)?; + // A bare `entity/` names nothing; leave it where it is + // rather than inventing an empty-named entity. + (!label.is_empty()).then(|| bucket.push((label.to_string(), count))) + }); + + if bucketed.is_none() { + plain.push((name, count)); + } + } + + let finish = |mut bucket: Vec<(String, u64)>| { + crate::model::sort_tag_counts(&mut bucket); + let total = bucket.len() as u64; + let truncated = bucket.len() > limit; + bucket.truncate(limit); + bridge::TagBucket { + total, + truncated, + items: bucket + .into_iter() + .map(|(name, count)| bridge::TagCount { name, count }) + .collect(), + } + }; + + bridge::ListTagsResponse { + namespace, + memories_counted, + tags: finish(plain), + entities: finish(entities), + topics: finish(topics), + emotions: finish(emotions), + } } // ═══════════════════════════════════════════════════════════════════════ @@ -1317,18 +1633,114 @@ impl bridge::StorageEngine for McpStorageAdapter { // ═══════════════════════════════════════════════════════════════════════ /// Adapts `RedbStorageEngine` to the MCP `NamespaceRegistry` trait. +/// +/// Holds the four out-of-storage indexes as well as storage, because +/// namespace deletion has to clean them: storage can destroy the +/// records, but a namespace whose FTS rows and vector-index entries +/// survive is a namespace that goes on burning top-k slots in every +/// search. Namespace lifecycle already lives on this trait — +/// create, list and stats are all here — so widening it is preferable +/// to splitting deletion onto `StorageEngine` because that is where the +/// handles happen to be. pub struct McpNamespaceAdapter { storage: Arc>, + cache: Arc, + vector_index: Arc>, + fts_index: Arc>, + entity_index: Arc>, + graph: SharedGraph, timezone: chrono_tz::Tz, } impl McpNamespaceAdapter { - /// Create a new namespace adapter wrapping the storage engine. + /// Create a new namespace adapter wrapping the storage engine and + /// the indexes a namespace deletion has to clean. pub fn new( storage: Arc>, + cache: Arc, + vector_index: Arc>, + fts_index: Arc>, + entity_index: Arc>, + graph: SharedGraph, timezone: chrono_tz::Tz, ) -> Self { - Self { storage, timezone } + Self { + storage, + cache, + vector_index, + fts_index, + entity_index, + graph, + timezone, + } + } + + /// Borrow this adapter's out-of-storage indexes for + /// [`crate::index_cleanup::purge_from_indexes`]. + fn index_handles(&self) -> crate::index_cleanup::IndexHandles<'_> { + crate::index_cleanup::IndexHandles { + cache: &self.cache, + vector_index: &self.vector_index, + fts_index: &self.fts_index, + entity_index: &self.entity_index, + graph: &self.graph, + } + } +} + +/// Build a query's `require_tags` from several labelled groups, failing +/// with a caller-facing message on the first label that is not a tag. +/// +/// `groups` is `(request field, tag prefix, labels)`. Order is fixed so +/// a request broken in several ways always reports the same failure. +/// +/// Deliberately NOT `parse_tags_lossy`: that drops what it cannot parse, +/// which is the right trade-off for a store (keep the memory, lose a +/// label) and exactly the wrong one for a filter. A dropped filter +/// WIDENS the query — `list_memories {"entities": ["José"]}` returned +/// every memory in the namespace — and the caller has no way to tell. +fn filter_tags( + groups: &[(&str, &str, &Vec)], + tool: &str, +) -> Result, String> { + let mut out = Vec::new(); + for (field, prefix, labels) in groups { + match crate::model::parse_filter_tags(field, prefix, labels) { + Ok(tags) => out.extend(tags), + Err(e) => { + tracing::warn!( + tool = %tool, + field = %e.field, + value = %e.value, + error = %e.source, + "Rejected an unfilterable label rather than widening the query" + ); + return Err(e.to_string()); + } + } + } + Ok(out) +} + +/// A namespace deletion that failed, and what it had already destroyed. +/// +/// Exists because `spawn_blocking` can only hand back one value: the +/// error the caller sees and the records the caller still owes index +/// cleanup for have to travel together, or the second is lost. +struct DeleteFailure { + /// Records destroyed before the failure. Empty when nothing was. + destroyed: Vec<(MemoryId, crate::model::DiskRecord)>, + /// What the caller is told. + error: bridge::BridgeError, +} + +impl DeleteFailure { + /// A failure that destroyed nothing. + fn nothing_destroyed(error: bridge::BridgeError) -> Self { + Self { + destroyed: Vec::new(), + error, + } } } @@ -1344,16 +1756,26 @@ impl bridge::NamespaceRegistry for McpNamespaceAdapter { let namespaces = storage_r .list_namespaces() .map_err(|e| bridge::BridgeError::Storage(e.to_string()))?; - Ok(namespaces - .into_iter() - .map(|ns| bridge::NamespaceInfo { + // `memory_count` was hardcoded to 0 here, so both this tool + // and the `recalld://namespaces` resource reported every + // namespace as empty. Counting live members costs one point + // lookup per member and agrees with `list_memories.total` by + // construction. + let mut out = Vec::with_capacity(namespaces.len()); + for ns in namespaces { + let memory_count = storage_r + .meta_store() + .count_memories_in_namespace(ns.id) + .map_err(|e| bridge::BridgeError::Storage(e.to_string()))?; + out.push(bridge::NamespaceInfo { id: ns.id.get(), name: ns.name.clone(), embedding_dim: ns.embedding_dim as u16, - memory_count: 0, + memory_count, created_at: format_timestamp(ns.created_at, tz), - }) - .collect()) + }); + } + Ok(out) }) .await .map_err(|e| bridge::BridgeError::Internal(format!("blocking task join error: {e}")))? @@ -1365,6 +1787,15 @@ impl bridge::NamespaceRegistry for McpNamespaceAdapter { ) -> Result { use crate::model::NamespaceConfig; + // Validated here as well as at the tool layer, and this is the + // load-bearing copy for the same reason the `default` guard in + // `delete_namespace` is: the daemon socket dispatches + // `create_namespace` straight into this adapter and never runs + // the tool handler. A namespace name becomes a directory name, + // so an unvalidated one becomes a path. + crate::model::validation::validate_namespace_name(&input.name) + .map_err(|e| bridge::BridgeError::InvalidInput(e.to_string()))?; + let now = chrono::Utc::now().timestamp_millis(); let storage = self.storage.clone(); let tz = self.timezone; @@ -1379,7 +1810,11 @@ impl bridge::NamespaceRegistry for McpNamespaceAdapter { storage .read() .ok() - .and_then(|s| s.get_namespace_by_name("default").ok().flatten()) + .and_then(|s| { + s.get_namespace_by_name(crate::model::constants::DEFAULT_NAMESPACE_NAME) + .ok() + .flatten() + }) .map(|ns| ns.embedding_dim) .unwrap_or(1536) }); @@ -1408,6 +1843,8 @@ impl bridge::NamespaceRegistry for McpNamespaceAdapter { id: assigned_id.get(), name: input_name, embedding_dim: dim as u16, + // Not a placeholder: a namespace created one line ago is + // empty. memory_count: 0, created_at: format_timestamp(now, tz), }) @@ -1439,6 +1876,7 @@ impl bridge::NamespaceRegistry for McpNamespaceAdapter { .map_err(|e| bridge::BridgeError::Storage(e.to_string()))?; let mut memory_count: u64 = 0; + let mut tombstone_count: u64 = 0; let mut full_count: u64 = 0; let mut summary_count: u64 = 0; let mut ghost_count: u64 = 0; @@ -1450,12 +1888,27 @@ impl bridge::NamespaceRegistry for McpNamespaceAdapter { if NamespaceId::new(record.namespace_id) != ns.id { continue; } + + // Tombstoned records were counted into `memory_count`, + // which made it disagree with `list_memories.total`, and + // then dragged every derived figure with them: `tombstone` + // zeroes strength and decay_strength but LEAVES + // is_permastore and edge_count intact, so a deleted memory + // still counted as permastore, still contributed its edges, + // and pulled avg_strength toward zero. + if record.phase == DecayPhase::Tombstone { + tombstone_count += 1; + continue; + } memory_count += 1; match record.phase { DecayPhase::Full => full_count += 1, DecayPhase::Summary => summary_count += 1, DecayPhase::Ghost => ghost_count += 1, + // Unreachable: skipped above. Left as an arm rather + // than an `unreachable!()` because this is a read path + // and a corrupt record must not panic a stats call. DecayPhase::Tombstone => {} } @@ -1478,6 +1931,7 @@ impl bridge::NamespaceRegistry for McpNamespaceAdapter { Ok(bridge::NamespaceStats { name: ns_name, memory_count, + tombstone_count, phase_counts: bridge::PhaseCounts { full: full_count, summary: summary_count, @@ -1492,6 +1946,190 @@ impl bridge::NamespaceRegistry for McpNamespaceAdapter { .await .map_err(|e| bridge::BridgeError::Internal(format!("blocking task join error: {e}")))? } + + async fn delete_namespace( + &self, + input: bridge::DeleteNamespaceInput, + ) -> Result { + use crate::model::constants::DEFAULT_NAMESPACE_NAME; + + // Validated before anything else, and for a blunter reason than + // tidiness: the name is joined onto the data directory and + // handed to `remove_dir_all`. `""` used to name a live, + // healthy namespace whose directory WAS the data directory. + crate::model::validation::validate_namespace_name(&input.name) + .map_err(|e| bridge::BridgeError::InvalidInput(e.to_string()))?; + + // The default namespace is refused here as well as at the tool + // layer, and this is the load-bearing copy: the daemon socket is + // its own entry point and never runs the tool handler. + // + // Refused even with `force: true`, and even when empty. Startup + // recreates a missing default under the SAME directory name with + // a NEW namespace id, so a deleted default leaves a stale + // vectors.dat that the recreated namespace opens and reuses — + // handing unrelated memories somebody else's embeddings. The + // boot sweep would remove that directory first, which makes the + // failure unlikely rather than impossible, and "unlikely" is not + // the right guarantee for silently wrong vectors. + // + // Compared case-insensitively, because the guard protects a + // DIRECTORY and macOS APFS resolves `Default/` to `default/`. + // With an exact compare, `create_namespace("Default")` + + // `delete_namespace("Default", force: true)` walked straight + // past this and unlinked the real default's vectors.dat; on the + // next boot every record's vector_slot pointed past EOF. + if input.name.eq_ignore_ascii_case(DEFAULT_NAMESPACE_NAME) { + return Err(bridge::BridgeError::InvalidInput(format!( + "The '{DEFAULT_NAMESPACE_NAME}' namespace cannot be deleted. It is \ + recreated on every startup under the same directory name, so deleting \ + it would leave a stale vectors.dat that a new namespace could silently \ + reuse." + ))); + } + + // Lookup, refuse-check and purge all happen inside ONE + // storage.write() in ONE blocking closure. Splitting them across + // two blocking hops would open a window in which a memory stored + // between the check and the purge is destroyed by a check that + // said "empty". + let outcome = { + let storage = self.storage.clone(); + let name = input.name.clone(); + let force = input.force; + tokio::task::spawn_blocking(move || { + let mut storage_w = storage.write().map_err(|e| { + DeleteFailure::nothing_destroyed(bridge::BridgeError::Internal(format!( + "storage lock poisoned: {e}" + ))) + })?; + + let config = storage_w + .get_namespace_by_name(&name) + .map_err(|e| { + DeleteFailure::nothing_destroyed(bridge::BridgeError::Storage( + e.to_string(), + )) + })? + .ok_or_else(|| { + DeleteFailure::nothing_destroyed(bridge::BridgeError::NotFound(format!( + "namespace '{name}' not found" + ))) + })?; + + if !force { + // Live memories only. A namespace holding nothing + // but tombstones reports memoryCount 0 everywhere + // else, so refusing it here would be unexplainable — + // and those tombstones hold no content the user has + // not already asked to forget. The purge still reaps + // them, and the result reports how many. + // + // `count_live_records_in_namespace`, NOT + // `count_memories_in_namespace`: the latter walks + // NAMESPACE_INDEX while the destruction scans + // META_TABLE, so a record whose index entry is + // missing was invisible to the check and destroyed + // by the effect. The guard applies the purge's own + // predicate. + let live = storage_w + .meta_store() + .count_live_records_in_namespace(config.id) + .map_err(|e| { + DeleteFailure::nothing_destroyed(bridge::BridgeError::Storage( + e.to_string(), + )) + })?; + if live > 0 { + // Nothing has been touched at this point. + return Err(DeleteFailure::nothing_destroyed( + bridge::BridgeError::InvalidInput(format!( + "Namespace '{name}' still holds {live} memories. Deleting it \ + permanently destroys them and removes its vector file from \ + disk; there is no undo and no backup. Re-run with \ + force: true to proceed." + )), + )); + } + } + + storage_w.purge_namespace(config.id).map_err(|e| { + let error = if e.is_partial() { + bridge::BridgeError::PartiallyApplied(format!( + "Namespace '{name}' was PARTIALLY destroyed: {} memories are \ + permanently gone, and the namespace still exists holding the \ + rest. Re-run delete_namespace to finish it. Cause: {}", + e.destroyed.len(), + e.source + )) + } else { + bridge::BridgeError::Storage(e.source.to_string()) + }; + DeleteFailure { + destroyed: e.destroyed, + error, + } + }) + }) + .await + .map_err(|e| bridge::BridgeError::Internal(format!("blocking task join error: {e}")))? + }; + + let purged = match outcome { + Ok(purged) => purged, + Err(failure) => { + // The records in `failure.destroyed` are already gone + // from meta.db and nothing else will ever reach their + // index entries: a retry looks in META_TABLE and finds + // them missing. Per-record FTS removal, not + // whole-namespace — the namespace still holds the + // records the purge did not get to, and their rows must + // stay. + if !failure.destroyed.is_empty() { + crate::index_cleanup::purge_from_indexes( + &self.index_handles(), + &failure.destroyed, + crate::index_cleanup::FtsAction::PerRecord, + crate::index_cleanup::GraphAction::RemoveNode, + ) + .await; + } + return Err(failure.error); + } + }; + + // Surviving memories whose outgoing edge_count storage just + // corrected. Their cached copy still holds the old number, and + // nothing else will ever notice: invalidate so the next read + // reloads the corrected record. + if !purged.peer_edge_counts.is_empty() { + let peers: Vec = purged.peer_edge_counts.iter().map(|(id, _)| *id).collect(); + self.cache.batch_invalidate(&peers).await; + } + + // Whole-namespace FTS removal, not per record: it also reaches + // rows whose meta.db record was already gone. RemoveNode, not + // MarkTombstoned: the records are destroyed, so a surviving node + // would dangle. + let index_counts = crate::index_cleanup::purge_from_indexes( + &self.index_handles(), + &purged.records, + crate::index_cleanup::FtsAction::WholeNamespace(purged.config.id), + crate::index_cleanup::GraphAction::RemoveNode, + ) + .await; + + Ok(bridge::DeleteNamespaceResult { + name: purged.config.name.clone(), + id: purged.config.id.get(), + deleted: true, + memories_deleted: purged.live_removed as u64, + tombstones_purged: purged.tombstones_removed as u64, + edges_removed: purged.edges_removed as u64, + fts_rows_removed: index_counts.fts_removed as u64, + vector_file_removed: purged.directory_removed, + }) + } } // ═══════════════════════════════════════════════════════════════════════ @@ -1534,3 +2172,1325 @@ impl bridge::HealthChecker for McpHealthAdapter { } } } + +// ═══════════════════════════════════════════════════════════════════════ +// Tests +// ═══════════════════════════════════════════════════════════════════════ + +#[cfg(test)] +mod tests { + use super::*; + use crate::mcp::bridge::{NamespaceRegistry, StorageEngine as BridgeStorageEngine}; + use crate::model::constants::DUPLICATE_SIMILARITY_THRESHOLD; + use crate::test_support::{Fixture, MemorySpec}; + + fn default_ns() -> NamespaceId { + NamespaceId::new(1) + } + + /// Find one namespace in a `list_namespaces` result by name. + fn find<'a>(namespaces: &'a [bridge::NamespaceInfo], name: &str) -> &'a bridge::NamespaceInfo { + namespaces + .iter() + .find(|n| n.name == name) + .unwrap_or_else(|| panic!("namespace '{name}' missing from list_namespaces")) + } + + // ── list_namespaces ────────────────────────────────────────────── + + /// The regression: `memory_count` was hardcoded to 0, so every + /// namespace reported as empty no matter what it held. + #[tokio::test] + async fn a41_list_namespaces_reports_the_live_memory_count() { + let fx = Fixture::new(); + for _ in 0..3 { + fx.insert_memory().await; + } + + let namespaces = fx.namespace_adapter().list_namespaces().await.unwrap(); + assert_eq!(find(&namespaces, "default").memory_count, 3); + } + + #[tokio::test] + async fn a42_list_namespaces_excludes_tombstoned_memories() { + let fx = Fixture::new(); + let doomed = fx.insert_memory().await; + fx.insert_memory().await; + fx.tombstone(doomed); + + let namespaces = fx.namespace_adapter().list_namespaces().await.unwrap(); + assert_eq!(find(&namespaces, "default").memory_count, 1); + } + + #[tokio::test] + async fn a43_list_namespaces_reports_zero_for_an_empty_namespace() { + let fx = Fixture::new(); + fx.create_namespace("empty"); + // A memory in `default` must not be counted against `empty`. + fx.insert_memory().await; + + let namespaces = fx.namespace_adapter().list_namespaces().await.unwrap(); + assert_eq!(find(&namespaces, "empty").memory_count, 0); + assert_eq!(find(&namespaces, "default").memory_count, 1); + } + + /// Each namespace is counted on its own members, not on the whole + /// database. + #[tokio::test] + async fn a43b_counts_are_per_namespace() { + let fx = Fixture::new(); + let other = fx.create_namespace("other"); + fx.insert_memory_with(MemorySpec::new(other)).await; + fx.insert_memory_with(MemorySpec::new(default_ns())).await; + + let namespaces = fx.namespace_adapter().list_namespaces().await.unwrap(); + assert_eq!(find(&namespaces, "other").memory_count, 1); + assert_eq!(find(&namespaces, "default").memory_count, 1); + } + + // ── namespace_stats ────────────────────────────────────────────── + + /// The regression: tombstoned records were counted as memories, so + /// `memoryCount` disagreed with what `list_memories` would return and + /// there was no way to tell a deleted memory from a lost one. + #[tokio::test] + async fn a44_namespace_stats_excludes_tombstones_and_reports_them_separately() { + let fx = Fixture::new(); + let a = fx.insert_memory().await; + let b = fx.insert_memory().await; + fx.insert_memory().await; + fx.tombstone(a); + fx.tombstone(b); + + let stats = fx + .namespace_adapter() + .namespace_stats("default") + .await + .unwrap(); + assert_eq!(stats.memory_count, 1); + assert_eq!(stats.tombstone_count, 2); + } + + /// The tell: `memoryCount` is assertable against the phase breakdown, + /// which it was not while tombstones were folded into it. + #[tokio::test] + async fn a45_memory_count_equals_the_sum_of_the_phase_counts() { + let fx = Fixture::new(); + for phase in [DecayPhase::Full, DecayPhase::Summary, DecayPhase::Ghost] { + fx.insert_memory_with(MemorySpec { + phase, + ..MemorySpec::new(default_ns()) + }) + .await; + } + let doomed = fx.insert_memory().await; + fx.tombstone(doomed); + + let stats = fx + .namespace_adapter() + .namespace_stats("default") + .await + .unwrap(); + let p = &stats.phase_counts; + assert_eq!(stats.memory_count, p.full + p.summary + p.ghost); + assert_eq!(stats.memory_count, 3); + } + + /// `tombstone` zeroes `decay_strength`, so counting tombstones dragged + /// the average toward zero: one live memory at 0.8 beside two + /// tombstones reported ~0.27. + #[tokio::test] + async fn a46_avg_strength_ignores_tombstoned_zero_strength_rows() { + let fx = Fixture::new(); + fx.insert_memory_with(MemorySpec { + decay_strength: 0.8, + ..MemorySpec::new(default_ns()) + }) + .await; + for _ in 0..2 { + let doomed = fx.insert_memory().await; + fx.tombstone(doomed); + } + + let stats = fx + .namespace_adapter() + .namespace_stats("default") + .await + .unwrap(); + assert!( + (stats.avg_strength - 0.8).abs() < 1e-5, + "expected ~0.8, got {}", + stats.avg_strength + ); + } + + #[tokio::test] + async fn a47_vector_bytes_counts_only_live_memories() { + let fx = Fixture::new(); + fx.insert_memory().await; + let doomed = fx.insert_memory().await; + fx.tombstone(doomed); + + let stats = fx + .namespace_adapter() + .namespace_stats("default") + .await + .unwrap(); + // One live memory x DIM dimensions x 4 bytes per f32. A + // tombstone's slot is freed, so it occupies no live vector bytes. + assert_eq!(stats.vector_bytes, crate::test_support::DIM as u64 * 4); + } + + /// `MetadataStore::tombstone` zeroes summary, tags and strength but + /// LEAVES `is_permastore` and `edge_count` intact, so these two only + /// come out right because the record is skipped entirely. + #[tokio::test] + async fn a48_edge_and_permastore_counts_exclude_tombstones() { + let fx = Fixture::new(); + let doomed = fx + .insert_memory_with(MemorySpec { + is_permastore: true, + edge_count: 2, + ..MemorySpec::new(default_ns()) + }) + .await; + fx.insert_memory_with(MemorySpec { + is_permastore: true, + edge_count: 3, + ..MemorySpec::new(default_ns()) + }) + .await; + fx.tombstone(doomed); + + let stats = fx + .namespace_adapter() + .namespace_stats("default") + .await + .unwrap(); + assert_eq!(stats.permastore_count, 1); + assert_eq!(stats.edge_count, 3); + } + + /// The three numbers a caller can compare have to agree after a + /// delete, or one of them is lying. + #[tokio::test] + async fn a49_stats_agree_with_list_memories_and_list_namespaces() { + let fx = Fixture::new(); + for _ in 0..3 { + fx.insert_memory().await; + } + let doomed = fx.insert_memory().await; + fx.tombstone(doomed); + + let stats = fx + .namespace_adapter() + .namespace_stats("default") + .await + .unwrap(); + let listed = fx + .storage_adapter() + .list_memories(bridge::ListMemoriesInput { + namespace: "default".into(), + limit: 50, + offset: 0, + tags: Vec::new(), + entities: Vec::new(), + time_range_start: None, + time_range_end: None, + }) + .await + .unwrap(); + let namespaces = fx.namespace_adapter().list_namespaces().await.unwrap(); + + assert_eq!(stats.memory_count, 3); + assert_eq!(listed.total, 3); + assert_eq!(find(&namespaces, "default").memory_count, 3); + } + + #[tokio::test] + async fn a50_namespace_stats_are_scoped_to_one_namespace() { + let fx = Fixture::new(); + let other = fx.create_namespace("other"); + fx.insert_memory_with(MemorySpec::new(default_ns())).await; + for _ in 0..2 { + fx.insert_memory_with(MemorySpec::new(other)).await; + } + + let default_stats = fx + .namespace_adapter() + .namespace_stats("default") + .await + .unwrap(); + let other_stats = fx + .namespace_adapter() + .namespace_stats("other") + .await + .unwrap(); + assert_eq!(default_stats.memory_count, 1); + assert_eq!(other_stats.memory_count, 2); + } + + // ── list_tags (live storage) ───────────────────────────────────── + + /// Tags as they appear on disk, prefixes included. + fn tagged<'a>(namespace: NamespaceId, tags: &'a [&'a str]) -> MemorySpec<'a> { + MemorySpec { + tags, + ..MemorySpec::new(namespace) + } + } + + fn names(bucket: &bridge::TagBucket) -> Vec<(&str, u64)> { + bucket + .items + .iter() + .map(|t| (t.name.as_str(), t.count)) + .collect() + } + + fn scoped(namespace: &str) -> bridge::ListTagsInput { + bridge::ListTagsInput { + namespace: Some(namespace.to_string()), + limit: 50, + } + } + + #[tokio::test] + async fn a51_list_tags_counts_only_the_requested_namespace() { + let fx = Fixture::new(); + let other = fx.create_namespace("other"); + fx.insert_memory_with(tagged(default_ns(), &["type/decision"])) + .await; + fx.insert_memory_with(tagged(default_ns(), &["type/decision"])) + .await; + fx.insert_memory_with(tagged(other, &["type/decision"])) + .await; + + let response = fx + .storage_adapter() + .list_tags(scoped("default")) + .await + .unwrap(); + assert_eq!(response.namespace.as_deref(), Some("default")); + assert_eq!(response.memories_counted, Some(2)); + assert_eq!(names(&response.tags), [("type/decision", 2)]); + } + + #[tokio::test] + async fn a52_list_tags_excludes_tombstoned_memories() { + let fx = Fixture::new(); + let doomed = fx + .insert_memory_with(tagged(default_ns(), &["type/decision"])) + .await; + fx.insert_memory_with(tagged(default_ns(), &["type/decision"])) + .await; + fx.tombstone(doomed); + + let response = fx + .storage_adapter() + .list_tags(scoped("default")) + .await + .unwrap(); + assert_eq!(response.memories_counted, Some(1)); + assert_eq!(names(&response.tags), [("type/decision", 1)]); + } + + #[tokio::test] + async fn a53_list_tags_aggregates_across_namespaces_in_global_scope() { + let fx = Fixture::new(); + let other = fx.create_namespace("other"); + fx.insert_memory_with(tagged(default_ns(), &["type/decision"])) + .await; + fx.insert_memory_with(tagged(other, &["type/decision"])) + .await; + + let response = fx + .storage_adapter() + .list_tags(bridge::ListTagsInput { + namespace: None, + limit: 50, + }) + .await + .unwrap(); + assert_eq!(names(&response.tags), [("type/decision", 2)]); + } + + #[tokio::test] + async fn a54_list_tags_rejects_an_unknown_namespace() { + let fx = Fixture::new(); + let err = fx + .storage_adapter() + .list_tags(scoped("nope")) + .await + .unwrap_err(); + assert!( + matches!(err, bridge::BridgeError::NotFound(_)), + "expected NotFound, got {err}" + ); + assert!(err.to_string().contains("nope"), "{err}"); + } + + /// A phase transition strips text, not tags, so a Summary or Ghost + /// memory is still part of the namespace's vocabulary. + #[tokio::test] + async fn a55_list_tags_counts_summary_and_ghost_phases() { + let fx = Fixture::new(); + for phase in [DecayPhase::Full, DecayPhase::Summary, DecayPhase::Ghost] { + fx.insert_memory_with(MemorySpec { + phase, + tags: &["type/decision"], + ..MemorySpec::new(default_ns()) + }) + .await; + } + + let response = fx + .storage_adapter() + .list_tags(scoped("default")) + .await + .unwrap(); + assert_eq!(response.memories_counted, Some(3)); + assert_eq!(names(&response.tags), [("type/decision", 3)]); + } + + /// Global scope reads the tag index directly and touches no memory + /// records, so there is no honest number to report. + #[tokio::test] + async fn a56_list_tags_reports_no_memory_count_in_global_scope() { + let fx = Fixture::new(); + fx.insert_memory_with(tagged(default_ns(), &["type/decision"])) + .await; + + let response = fx + .storage_adapter() + .list_tags(bridge::ListTagsInput { + namespace: None, + limit: 50, + }) + .await + .unwrap(); + assert_eq!(response.namespace, None); + assert_eq!(response.memories_counted, None); + } + + // ── build_list_tags_response (pure) ────────────────────────────── + + fn histogram(pairs: &[(&str, u64)]) -> Vec<(String, u64)> { + pairs.iter().map(|(n, c)| ((*n).to_string(), *c)).collect() + } + + #[test] + fn a57_build_orders_count_descending_then_name_ascending() { + let response = build_list_tags_response( + None, + None, + histogram(&[("zebra", 1), ("aardvark", 9), ("moose", 1)]), + 50, + ); + assert_eq!( + names(&response.tags), + [("aardvark", 9), ("moose", 1), ("zebra", 1)] + ); + } + + #[test] + fn a58_build_splits_derived_prefixes_and_strips_them() { + let response = build_list_tags_response( + None, + None, + histogram(&[ + ("entity/recalld", 3), + ("topic/rust", 2), + ("emotion/curious", 1), + ("type/decision", 4), + ]), + 50, + ); + assert_eq!(names(&response.entities), [("recalld", 3)]); + assert_eq!(names(&response.topics), [("rust", 2)]); + assert_eq!(names(&response.emotions), [("curious", 1)]); + assert_eq!(names(&response.tags), [("type/decision", 4)]); + } + + #[test] + fn a59_build_reports_the_untruncated_total_and_flags_truncation() { + let response = + build_list_tags_response(None, None, histogram(&[("a", 3), ("b", 2), ("c", 1)]), 2); + assert_eq!( + response.tags.total, 3, + "total is the size before truncation" + ); + assert!(response.tags.truncated); + assert_eq!(names(&response.tags), [("a", 3), ("b", 2)]); + + let untruncated = build_list_tags_response(None, None, histogram(&[("a", 1)]), 2); + assert_eq!(untruncated.tags.total, 1); + assert!(!untruncated.tags.truncated); + } + + /// Containing a slash does not make a tag derived. `project/recalld` + /// is a plain tag a caller filters on, and moving it to `entities` + /// would make it unusable there. + #[test] + fn a60_build_keeps_a_slashed_plain_tag_in_the_tags_bucket() { + let response = + build_list_tags_response(None, None, histogram(&[("project/recalld", 5)]), 50); + assert_eq!(names(&response.tags), [("project/recalld", 5)]); + assert!(response.entities.items.is_empty()); + assert_eq!(response.entities.total, 0); + } + + #[test] + fn a61_build_treats_a_bare_prefix_as_a_plain_tag() { + let response = build_list_tags_response( + None, + None, + histogram(&[("entity/", 2), ("topic/", 1), ("emotion/", 1)]), + 50, + ); + assert_eq!( + names(&response.tags), + [("entity/", 2), ("emotion/", 1), ("topic/", 1)], + "a bare prefix names nothing; it must never become an empty-named entity" + ); + assert!(response.entities.items.is_empty()); + assert!(response.topics.items.is_empty()); + assert!(response.emotions.items.is_empty()); + } + + // ── near-duplicate advisory ────────────────────────────────────── + // + // These route through `PassthroughProvider` and `StoreInput.embedding` + // — the exact un-validated caller-vector path the normalization guard + // exists for. The vectors are unit vectors, so they pass that guard, + // which also demonstrates the guard is not quietly disabling the + // feature for everyone. + + /// A store with a caller-supplied embedding, defaults elsewhere. + fn store_input(summary: &str, namespace: &str, embedding: Vec) -> bridge::StoreInput { + bridge::StoreInput { + summary: summary.to_string(), + full_text: None, + tags: Vec::new(), + entities: Vec::new(), + topics: Vec::new(), + emotions: Vec::new(), + namespace: namespace.to_string(), + embedding: Some(embedding), + initial_stability: None, + parent_id: None, + supersedes: None, + check_duplicates: None, + } + } + + fn unit_x() -> Vec { + vec![1.0, 0.0, 0.0, 0.0] + } + + /// The ordering guard, and the only test that catches it. The check + /// runs before `index.add`; move it below and the new memory finds + /// itself at score 1.0 and every store reports a duplicate. + #[tokio::test] + async fn f26_a_store_does_not_report_itself_as_its_own_duplicate() { + let fx = Fixture::new(); + let stored = fx + .storage_adapter() + .store_memory(store_input("the first fact", "default", unit_x())) + .await + .expect("store succeeds"); + + assert!( + stored.near_duplicates.is_none(), + "a store into an empty namespace reported itself: {:?}", + stored.near_duplicates + ); + } + + /// End to end, hydration included: the summary comes back, so the + /// caller can judge the duplicate without a second round trip. + #[tokio::test] + async fn f27_a_second_store_of_the_same_embedding_reports_the_first() { + let fx = Fixture::new(); + let adapter = fx.storage_adapter(); + let first = adapter + .store_memory(store_input("the first fact", "default", unit_x())) + .await + .expect("first store succeeds"); + + let second = adapter + .store_memory(store_input("the same fact again", "default", unit_x())) + .await + .expect("second store succeeds"); + + let report = second.near_duplicates.expect("duplicate reported"); + assert_eq!(report.threshold, DUPLICATE_SIMILARITY_THRESHOLD); + assert_eq!(report.matches.len(), 1); + assert_eq!(report.matches[0].id.to_string(), first.id); + assert!(report.matches[0].score >= 0.85, "{:?}", report.matches[0]); + assert_eq!(report.matches[0].summary, "the first fact"); + } + + /// The guarantee the docs make and the index could not keep. + /// `find_near_duplicates` asks the vector index for Full and Summary + /// only, but `FlatVectorIndex` writes each entry's `decay_phase` + /// once at add time — always Full — and `VectorIndex::update_metadata` + /// has no callers, so that filter is inert in production. A ghosted + /// memory was reported as a near-duplicate with an EMPTY summary, + /// and `store_memory`'s description then told the agent to reinforce + /// it. + #[tokio::test] + async fn w62_a_ghosted_memory_is_not_reported_as_a_near_duplicate() { + let fx = Fixture::new(); + let adapter = fx.storage_adapter(); + let first = adapter + .store_memory(store_input("the first fact", "default", unit_x())) + .await + .expect("first store succeeds"); + let first_id = MemoryId::from_uuid(uuid::Uuid::parse_str(&first.id).expect("uuid")); + + // Decay it to Ghost. The vector-index entry keeps saying Full, + // which is the whole point. + fx.storage + .read() + .expect("storage lock") + .update_decay_state(first_id, DecayPhase::Ghost, 0.1, 0.1, 1.0, false) + .expect("ghost the memory"); + fx.cache.invalidate(first_id).await; + + let second = adapter + .store_memory(store_input("the same fact again", "default", unit_x())) + .await + .expect("second store succeeds"); + + assert!( + second.near_duplicates.is_none(), + "a ghosted memory was reported as a near-duplicate: {:?}", + second.near_duplicates + ); + } + + /// The same guarantee for a memory whose record is gone entirely. + /// A hard delete that the in-memory vector index has not caught up + /// with used to surface as a match with an empty summary. + #[tokio::test] + async fn w63_a_hard_deleted_memory_is_not_reported_as_a_near_duplicate() { + let fx = Fixture::new(); + let adapter = fx.storage_adapter(); + let first = adapter + .store_memory(store_input("the first fact", "default", unit_x())) + .await + .expect("first store succeeds"); + let first_id = MemoryId::from_uuid(uuid::Uuid::parse_str(&first.id).expect("uuid")); + + fx.storage + .write() + .expect("storage lock") + .delete_memory(first_id) + .expect("hard delete"); + fx.cache.invalidate(first_id).await; + + let second = adapter + .store_memory(store_input("the same fact again", "default", unit_x())) + .await + .expect("second store succeeds"); + + assert!( + second.near_duplicates.is_none(), + "a hard-deleted memory was reported as a near-duplicate: {:?}", + second.near_duplicates + ); + } + + #[tokio::test] + async fn f28_check_duplicates_false_suppresses_the_report() { + let fx = Fixture::new(); + let adapter = fx.storage_adapter(); + adapter + .store_memory(store_input("the first fact", "default", unit_x())) + .await + .expect("first store succeeds"); + + let mut input = store_input("the same fact again", "default", unit_x()); + input.check_duplicates = Some(false); + let second = adapter.store_memory(input).await.expect("store succeeds"); + + assert!(second.near_duplicates.is_none()); + } + + #[tokio::test] + async fn f29_a_duplicate_in_another_namespace_is_not_reported() { + let fx = Fixture::new(); + fx.create_namespace("other"); + let adapter = fx.storage_adapter(); + adapter + .store_memory(store_input("the first fact", "default", unit_x())) + .await + .expect("first store succeeds"); + + let second = adapter + .store_memory(store_input("the same fact again", "other", unit_x())) + .await + .expect("second store succeeds"); + + assert!(second.near_duplicates.is_none()); + } + + // ── delete_memory: the shared index cleanup ────────────────────── + + /// Pins the refactor onto `index_cleanup::purge_from_indexes`. These + /// four removals used to be hand-written here; they are now shared + /// with the HTTP path and with namespace deletion, and the whole + /// point of sharing them is that all three keep doing all four. + #[tokio::test] + async fn w25_delete_memory_still_clears_every_index_and_marks_the_graph_node() { + let fx = Fixture::new(); + let adapter = fx.storage_adapter(); + + let mut input = store_input("a fact about Sarah", "default", unit_x()); + input.entities = vec!["Sarah".to_string()]; + let stored = adapter.store_memory(input).await.expect("store"); + let id = + MemoryId::from_uuid(uuid::Uuid::parse_str(&stored.id).expect("stored id is a uuid")); + + // Everything is indexed before the delete. + assert!( + fx.graph.read().await.contains(&id), + "precondition: the store created a graph node" + ); + assert!( + !fx.fts_index.lock().await.is_empty().expect("fts"), + "precondition: the memory is in the FTS index" + ); + assert_eq!(fx.entity_index.read().await.len(), 1); + + assert!(adapter.delete_memory(id).await.expect("delete")); + + assert!( + fx.fts_index.lock().await.is_empty().expect("fts"), + "FTS row survived the delete" + ); + assert_eq!( + { + use crate::search::VectorIndex; + fx.vector_index.read().await.len() + }, + 0, + "vector index entry survived the delete" + ); + assert_eq!( + fx.entity_index.read().await.len(), + 0, + "entity index entry survived the delete" + ); + + let graph_r = fx.graph.read().await; + let node = graph_r + .get_node(&id) + .expect("graph node is kept, not removed"); + assert_eq!(node.decay_phase, DecayPhase::Tombstone); + } + + // ── delete_memories (batch) ────────────────────────────────────── + + #[tokio::test] + async fn w26_delete_memories_returns_one_flag_per_id_in_input_order() { + let fx = Fixture::new(); + let adapter = fx.storage_adapter(); + + let live = fx.insert_memory().await; + let already = fx.insert_memory().await; + fx.tombstone(already); + let missing = MemoryId::new(); + + let flags = adapter + .delete_memories(&[missing, live, already]) + .await + .expect("batch delete"); + + assert_eq!(flags, vec![false, true, false]); + assert_eq!( + fx.record(live).expect("record survives").phase, + DecayPhase::Tombstone + ); + } + + #[tokio::test] + async fn w27_delete_memories_clears_every_index_for_every_deleted_id() { + let fx = Fixture::new(); + let adapter = fx.storage_adapter(); + + let mut ids = Vec::new(); + for name in ["Sarah", "Miguel"] { + let mut input = store_input(&format!("a fact about {name}"), "default", unit_x()); + input.entities = vec![name.to_string()]; + let stored = adapter.store_memory(input).await.expect("store"); + ids.push(MemoryId::from_uuid( + uuid::Uuid::parse_str(&stored.id).expect("stored id is a uuid"), + )); + } + + assert_eq!(fx.entity_index.read().await.len(), 2); + + let flags = adapter.delete_memories(&ids).await.expect("batch delete"); + assert_eq!(flags, vec![true, true]); + + assert!(fx.fts_index.lock().await.is_empty().expect("fts")); + assert_eq!( + { + use crate::search::VectorIndex; + fx.vector_index.read().await.len() + }, + 0 + ); + assert_eq!(fx.entity_index.read().await.len(), 0); + let graph_r = fx.graph.read().await; + for id in &ids { + assert_eq!( + graph_r.get_node(id).expect("node kept").decay_phase, + DecayPhase::Tombstone + ); + } + } + + /// Freeing one vector slot twice writes the slot's own index into + /// its next-pointer, turning the on-disk free list into a + /// self-cycle: the allocator then hands the same slot to two + /// different memories, and `live_count` underflows. The batch is the + /// only place a caller can present the same id twice, so this is + /// where it has to be impossible. + #[tokio::test] + async fn w28_a_duplicate_id_in_one_batch_frees_its_vector_slot_once() { + let fx = Fixture::new(); + let adapter = fx.storage_adapter(); + let id = fx.insert_memory().await; + + let flags = adapter + .delete_memories(&[id, id]) + .await + .expect("batch delete"); + assert_eq!(flags, vec![true, false], "the second pass deleted it again"); + + // A corrupted free list shows up as the next two allocations + // landing on the same slot. + let first = adapter + .store_memory(store_input("after the batch", "default", unit_x())) + .await + .expect("store"); + let second = adapter + .store_memory(store_input("and again", "default", unit_x())) + .await + .expect("store"); + + let slot_of = |stored: &bridge::StoredMemory| { + fx.record(MemoryId::from_uuid( + uuid::Uuid::parse_str(&stored.id).expect("uuid"), + )) + .expect("record") + .vector_slot + }; + assert_ne!( + slot_of(&first), + slot_of(&second), + "two memories were handed the same vector slot: the free list is a cycle" + ); + } + + // ── delete_namespace ───────────────────────────────────────────── + + /// Set up a namespace holding one fully indexed memory. + async fn indexed_namespace(fx: &Fixture, name: &str) -> (NamespaceId, MemoryId) { + let ns = fx.create_namespace(name); + let mut input = store_input("a fact worth keeping", name, unit_x()); + input.entities = vec!["Sarah".to_string()]; + let stored = fx + .storage_adapter() + .store_memory(input) + .await + .expect("store"); + let id = MemoryId::from_uuid(uuid::Uuid::parse_str(&stored.id).expect("uuid")); + (ns, id) + } + + /// THE most important test in this wave. A refusal that has already + /// destroyed something is worse than no refusal at all: the caller + /// reads an error, believes nothing happened, and the memories are + /// gone. The check runs before `purge_namespace` inside the same + /// write lock, and this asserts the whole state is untouched — not + /// just that an error came back. + #[tokio::test] + async fn w29_a_refused_non_force_delete_destroys_nothing() { + let fx = Fixture::new(); + let (_ns, id) = indexed_namespace(&fx, "scratch").await; + + let err = fx + .namespace_adapter() + .delete_namespace(bridge::DeleteNamespaceInput { + name: "scratch".into(), + force: false, + }) + .await + .expect_err("a non-empty namespace must be refused"); + + assert!( + matches!(err, bridge::BridgeError::InvalidInput(ref m) if m.contains("still holds 1")), + "the refusal must name the count: {err}" + ); + + assert!(fx.record(id).is_some(), "the record was destroyed"); + assert!( + fx.namespace_dir("scratch").join("vectors.dat").exists(), + "the vector file was destroyed" + ); + assert!( + !fx.fts_index.lock().await.is_empty().expect("fts"), + "the FTS rows were destroyed" + ); + assert!( + fx.storage + .read() + .expect("storage lock") + .get_namespace_by_name("scratch") + .expect("lookup") + .is_some(), + "the namespace config row was destroyed" + ); + } + + /// A namespace holding only tombstones reports memoryCount 0 in + /// `list_namespaces` and `namespace_stats`, so refusing it without + /// force would be a refusal the caller cannot explain. The purge + /// still reaps the rows — which is the whole point, since nothing + /// else in the system ever does — and the result says how many. + #[tokio::test] + async fn w30_a_tombstone_only_namespace_deletes_without_force() { + let fx = Fixture::new(); + let ns = fx.create_namespace("scratch"); + let a = fx.insert_memory_in(ns, &[]).await; + let b = fx.insert_memory_in(ns, &[]).await; + fx.tombstone(a); + fx.tombstone(b); + + let result = fx + .namespace_adapter() + .delete_namespace(bridge::DeleteNamespaceInput { + name: "scratch".into(), + force: false, + }) + .await + .expect("a namespace holding only tombstones is empty"); + + assert!(result.deleted); + assert_eq!(result.memories_deleted, 0); + assert_eq!( + result.tombstones_purged, 2, + "the tombstones were not reaped" + ); + assert!(fx.record(a).is_none()); + assert!(fx.record(b).is_none()); + } + + #[tokio::test] + async fn w31_a_forced_delete_removes_the_records_indexes_and_directory() { + let fx = Fixture::new(); + let (_ns, id) = indexed_namespace(&fx, "scratch").await; + let other = fx.insert_memory_in(NamespaceId::new(1), &[]).await; + + let result = fx + .namespace_adapter() + .delete_namespace(bridge::DeleteNamespaceInput { + name: "scratch".into(), + force: true, + }) + .await + .expect("forced delete"); + + assert!(result.deleted); + assert_eq!(result.memories_deleted, 1); + assert_eq!(result.fts_rows_removed, 1); + assert!(result.vector_file_removed); + + assert!(fx.record(id).is_none(), "the record survived"); + assert!( + !fx.namespace_dir("scratch").exists(), + "the directory survived" + ); + assert!(fx.fts_index.lock().await.is_empty().expect("fts")); + assert_eq!( + { + use crate::search::VectorIndex; + fx.vector_index.read().await.len() + }, + 0 + ); + assert_eq!(fx.entity_index.read().await.len(), 0); + assert!( + !fx.graph.read().await.contains(&id), + "the graph node survived" + ); + assert!( + fx.storage + .read() + .expect("storage lock") + .get_namespace_by_name("scratch") + .expect("lookup") + .is_none() + ); + assert!( + fx.record(other).is_some(), + "a memory in another namespace was destroyed" + ); + } + + /// Startup recreates a missing default under the SAME directory name + /// with a NEW namespace id, so a deleted default leaves a stale + /// vectors.dat that the recreated namespace would open and reuse — + /// handing unrelated memories somebody else's embeddings. Refused + /// even with force, and even when the namespace is empty, because + /// neither of those makes the recreation any less automatic. + #[tokio::test] + async fn w32_the_default_namespace_is_refused_even_forced_and_even_empty() { + let fx = Fixture::new(); + let adapter = fx.namespace_adapter(); + + for force in [false, true] { + let err = adapter + .delete_namespace(bridge::DeleteNamespaceInput { + name: "default".into(), + force, + }) + .await + .expect_err("the default namespace must never be deletable"); + assert!( + matches!(err, bridge::BridgeError::InvalidInput(_)), + "force={force}: {err}" + ); + } + + assert!( + fx.namespace_dir("default").join("vectors.dat").exists(), + "the default namespace's vector file was removed" + ); + } + + #[tokio::test] + async fn w33_an_unknown_namespace_is_not_found_and_touches_nothing() { + let fx = Fixture::new(); + let id = fx.insert_memory().await; + + let err = fx + .namespace_adapter() + .delete_namespace(bridge::DeleteNamespaceInput { + name: "no-such-namespace".into(), + force: true, + }) + .await + .expect_err("an unknown namespace is not found"); + + assert!(matches!(err, bridge::BridgeError::NotFound(_)), "{err}"); + assert!(fx.record(id).is_some()); + assert!(fx.namespace_dir("default").exists()); + } + + /// The stale-vectors.dat regression, from the other direction: a + /// namespace recreated under a name that was just deleted must get a + /// fresh, empty vector file. If the directory survived the delete, + /// the new namespace would open the old file and hand its slots out + /// again. + #[tokio::test] + async fn w34_recreating_a_deleted_namespace_gets_a_fresh_vector_file() { + let fx = Fixture::new(); + let (old_ns, old_id) = indexed_namespace(&fx, "scratch").await; + let namespace_adapter = fx.namespace_adapter(); + + namespace_adapter + .delete_namespace(bridge::DeleteNamespaceInput { + name: "scratch".into(), + force: true, + }) + .await + .expect("forced delete"); + + let recreated = namespace_adapter + .create_namespace(bridge::CreateNamespaceInput { + name: "scratch".into(), + embedding_dim: Some(crate::test_support::DIM as u16), + initial_stability: None, + desired_retention: None, + decay_rate_multiplier: None, + }) + .await + .expect("recreate"); + + assert_ne!( + recreated.id, + old_ns.get(), + "namespace ids must never be recycled" + ); + assert_eq!(recreated.memory_count, 0); + + let stored = fx + .storage_adapter() + .store_memory(store_input("a brand new fact", "scratch", unit_x())) + .await + .expect("store into the recreated namespace"); + let new_id = MemoryId::from_uuid(uuid::Uuid::parse_str(&stored.id).expect("uuid")); + + assert_eq!( + fx.record(new_id).expect("record").vector_slot, + 0, + "the recreated namespace inherited the deleted one's vector file" + ); + assert!(fx.record(old_id).is_none()); + } + + // ── unfilterable labels ────────────────────────────────────────── + + /// `entity/josé` fails `Tag`'s alphabet. The `if let Ok` around it + /// swallowed that, the filter was dropped, and `list_memories` + /// returned EVERY memory in the namespace — presented as the result + /// of filtering by entity. An agent acting on "these are the + /// memories about José" got the whole namespace. + #[tokio::test] + async fn w66_an_unfilterable_entity_is_an_error_not_a_wider_query() { + let fx = Fixture::new(); + let visible = fx.insert_memory().await; + + let err = fx + .storage_adapter() + .list_memories(bridge::ListMemoriesInput { + namespace: "default".into(), + limit: 10, + offset: 0, + tags: Vec::new(), + entities: vec!["José".to_string()], + time_range_start: None, + time_range_end: None, + }) + .await + .expect_err("an unfilterable entity must not widen the query"); + + assert!( + matches!(err, bridge::BridgeError::InvalidInput(ref m) if m.contains("José")), + "the error must name the label: {err:?}" + ); + assert!( + fx.record(visible).is_some(), + "the fixture memory should still exist" + ); + } + + /// The recall path builds the same filter from four groups; any of + /// them can carry the bad label, and each must name ITS field so the + /// caller knows which argument to fix. + #[test] + fn w67_filter_tags_names_the_group_that_carried_the_bad_label() { + let ok = vec!["fine".to_string()]; + let bad = vec!["José".to_string()]; + + for field in ["tags", "entities", "topics", "emotions"] { + let groups: Vec<(&str, &str, &Vec)> = + ["tags", "entities", "topics", "emotions"] + .iter() + .map(|g| { + let prefix = match *g { + "entities" => crate::model::constants::ENTITY_TAG_PREFIX, + "topics" => crate::model::constants::TOPIC_TAG_PREFIX, + "emotions" => crate::model::constants::EMOTION_TAG_PREFIX, + _ => "", + }; + (*g, prefix, if *g == field { &bad } else { &ok }) + }) + .collect(); + + let err = filter_tags(&groups, "recall_memories") + .expect_err("an unfilterable label must not be dropped"); + assert!(err.contains(field), "{field}: {err}"); + assert!(err.contains("José"), "{field}: {err}"); + } + + // All four groups valid: every label becomes a prefixed tag. + let groups: Vec<(&str, &str, &Vec)> = vec![ + ("tags", "", &ok), + ("entities", crate::model::constants::ENTITY_TAG_PREFIX, &ok), + ("topics", crate::model::constants::TOPIC_TAG_PREFIX, &ok), + ("emotions", crate::model::constants::EMOTION_TAG_PREFIX, &ok), + ]; + let tags: Vec = filter_tags(&groups, "recall_memories") + .expect("all four groups are valid") + .iter() + .map(|t| t.as_str().to_string()) + .collect(); + assert_eq!( + tags, + vec!["fine", "entity/fine", "topic/fine", "emotion/fine"] + ); + } + + // ── namespace-name validation on the socket's own entry point ──── + + /// `validate_namespace_name` ran at the tool layer and the HTTP + /// layer and nowhere else, but the daemon dispatches + /// `create_namespace` straight into this adapter. A namespace name + /// becomes a directory name, so an unvalidated one becomes a path. + #[tokio::test] + async fn w45_create_namespace_validates_the_name_in_the_adapter() { + let fx = Fixture::new(); + let adapter = fx.namespace_adapter(); + + for bad in ["", "..", "../evil", "has space", "ünïcode"] { + let err = adapter + .create_namespace(bridge::CreateNamespaceInput { + name: bad.to_string(), + embedding_dim: Some(crate::test_support::DIM as u16), + initial_stability: None, + desired_retention: None, + decay_rate_multiplier: None, + }) + .await + .expect_err("an unusable namespace name must be refused"); + assert!( + matches!(err, bridge::BridgeError::InvalidInput(_)), + "{bad:?}: {err:?}" + ); + } + + assert_eq!( + fx.storage + .read() + .expect("storage lock") + .list_namespaces() + .expect("list") + .len(), + 1, + "a refused create left a namespace row behind" + ); + } + + /// `delete_namespace` checked `!= "default"` and nothing else. `""` + /// passed that check, named a namespace that really was live, and + /// resolved to the data directory itself. + #[tokio::test] + async fn w46_delete_namespace_refuses_an_unusable_name_as_invalid_input() { + let fx = Fixture::new(); + let adapter = fx.namespace_adapter(); + + for bad in ["", "..", "../default", "has space"] { + let err = adapter + .delete_namespace(bridge::DeleteNamespaceInput { + name: bad.to_string(), + force: true, + }) + .await + .expect_err("an unusable namespace name must be refused"); + assert!( + matches!(err, bridge::BridgeError::InvalidInput(_)), + "{bad:?}: {err:?}" + ); + } + } + + /// The migration case, and the reason the refusal is by name rather + /// than by lookup: a database created before the validator reached + /// this layer can already hold a row named `""`, whose directory + /// **is** the data directory. Planting the row directly is the only + /// way to reach that state now. + #[tokio::test] + async fn w47_a_legacy_empty_named_namespace_cannot_destroy_the_database() { + let fx = Fixture::new(); + + { + let mut config = crate::model::NamespaceConfig::default_namespace(0); + config.name = String::new(); + config.embedding_dim = crate::test_support::DIM as u32; + fx.storage + .write() + .expect("storage lock") + .meta_store() + .create_namespace(&config) + .expect("plant a legacy row"); + } + + let err = fx + .namespace_adapter() + .delete_namespace(bridge::DeleteNamespaceInput { + name: String::new(), + force: true, + }) + .await + .expect_err("an empty namespace name must be refused"); + assert!( + matches!(err, bridge::BridgeError::InvalidInput(_)), + "{err:?}" + ); + + assert!( + fx.path().join("meta.db").exists() + && fx.path().join("edges.db").exists() + && fx.path().join("fulltext.dat").exists(), + "the database was destroyed" + ); + assert!( + fx.namespace_dir("default").join("vectors.dat").exists(), + "the default namespace's vectors were destroyed" + ); + } + + // ── case-insensitive namespace identity ────────────────────────── + + /// Chain B. `create_namespace("Default")` passed the exact-match + /// uniqueness check, committed its row, and only then failed on the + /// default namespace's exclusive flock — so the caller saw an error + /// and the row survived. `delete_namespace("Default", force: true)` + /// then walked past `"Default" != "default"` and pointed + /// `remove_dir_all` at the real `default/`, taking its vectors.dat + /// with it. On the next boot every record's vector_slot is past EOF. + #[tokio::test] + async fn w48_the_default_namespace_is_refused_under_any_casing() { + let fx = Fixture::new(); + let adapter = fx.namespace_adapter(); + + for spelling in ["Default", "DEFAULT", "dEfAuLt"] { + let create = adapter + .create_namespace(bridge::CreateNamespaceInput { + name: spelling.to_string(), + embedding_dim: Some(crate::test_support::DIM as u16), + initial_stability: None, + desired_retention: None, + decay_rate_multiplier: None, + }) + .await; + assert!( + create.is_err(), + "{spelling} was created alongside the default namespace" + ); + + let err = adapter + .delete_namespace(bridge::DeleteNamespaceInput { + name: spelling.to_string(), + force: true, + }) + .await + .expect_err("the default namespace must never be deletable"); + assert!( + matches!(err, bridge::BridgeError::InvalidInput(_)), + "{spelling}: {err:?}" + ); + } + + assert!( + fx.namespace_dir("default").join("vectors.dat").exists(), + "the default namespace's vector file was destroyed" + ); + assert_eq!( + fx.storage + .read() + .expect("storage lock") + .list_namespaces() + .expect("list") + .len(), + 1, + "a case-variant of 'default' left a namespace row behind" + ); + } +} diff --git a/src/mcp/tools.rs b/src/mcp/tools.rs index 88dc678..537abf9 100644 --- a/src/mcp/tools.rs +++ b/src/mcp/tools.rs @@ -1,17 +1,20 @@ //! MCP tool definitions and handlers for Recalld memory operations. //! -//! Defines 10 tools: store_memory, store_memories, recall_memories, get_memory, -//! reinforce_memory, forget_memory, find_similar_memories, create_namespace, -//! namespace_stats, list_memories. +//! Defines 14 tools: store_memory, store_memories, recall_memories, get_memory, +//! reinforce_memory, forget_memory, forget_memories, find_similar_memories, +//! create_namespace, delete_namespace, namespace_stats, list_memories, +//! list_namespaces, list_tags. use serde_json::json; use crate::mcp::args; -use crate::mcp::bridge::McpBridge; +use crate::mcp::bridge::{McpBridge, StoredMemory}; use crate::mcp::protocol::{ToolAnnotations, ToolCallResult, ToolInfo}; use crate::model::constants::{ - EMOTION_MAX_BYTES, ENTITY_MAX_BYTES, FULL_TEXT_MAX_BYTES, MAX_BATCH_MEMORIES, MAX_EMOTIONS, - MAX_ENTITIES, MAX_TAGS, MAX_TOPICS, SUMMARY_MAX_BYTES, TAG_MAX_BYTES, TOPIC_MAX_BYTES, + DEFAULT_LIST_LIMIT, DEFAULT_TAG_LIMIT, DUPLICATE_SIMILARITY_THRESHOLD, EMOTION_MAX_BYTES, + ENTITY_MAX_BYTES, FULL_TEXT_MAX_BYTES, MAX_BATCH_MEMORIES, MAX_EMOTIONS, MAX_ENTITIES, + MAX_LIST_LIMIT, MAX_TAG_LIMIT, MAX_TAGS, MAX_TOPICS, SUMMARY_MAX_BYTES, TAG_MAX_BYTES, + TOPIC_MAX_BYTES, }; use crate::model::validation::validate_namespace_name; @@ -41,13 +44,40 @@ pub fn tool_definitions() -> Vec { get_memory_def(), reinforce_memory_def(), forget_memory_def(), + forget_memories_def(), find_similar_memories_def(), create_namespace_def(), + delete_namespace_def(), namespace_stats_def(), list_memories_def(), + list_namespaces_def(), + list_tags_def(), ] } +/// Every tool name [`dispatch_tool`] has an arm for. +/// +/// Kept next to the match so a definition added without a dispatch arm — +/// which advertises a tool that then answers "Unknown tool" — fails a +/// test rather than a caller. +#[cfg(test)] +const DISPATCHED: [&str; 14] = [ + "store_memory", + "store_memories", + "recall_memories", + "get_memory", + "reinforce_memory", + "forget_memory", + "forget_memories", + "find_similar_memories", + "create_namespace", + "delete_namespace", + "namespace_stats", + "list_memories", + "list_namespaces", + "list_tags", +]; + /// Dispatch a tool call by name to the appropriate handler. pub async fn dispatch_tool( bridge: &McpBridge, @@ -61,10 +91,14 @@ pub async fn dispatch_tool( "get_memory" => handle_get_memory(bridge, arguments).await, "reinforce_memory" => handle_reinforce_memory(bridge, arguments).await, "forget_memory" => handle_forget_memory(bridge, arguments).await, + "forget_memories" => handle_forget_memories(bridge, arguments).await, "find_similar_memories" => handle_find_similar_memories(bridge, arguments).await, "create_namespace" => handle_create_namespace(bridge, arguments).await, + "delete_namespace" => handle_delete_namespace(bridge, arguments).await, "namespace_stats" => handle_namespace_stats(bridge, arguments).await, "list_memories" => handle_list_memories(bridge, arguments).await, + "list_namespaces" => handle_list_namespaces(bridge, arguments).await, + "list_tags" => handle_list_tags(bridge, arguments).await, _ => ToolCallResult::error(format!("Unknown tool: {name}")), } } @@ -77,10 +111,24 @@ fn store_memory_def() -> ToolInfo { ToolInfo { name: "store_memory".to_string(), title: Some("Store Memory".to_string()), + // The near-duplicate half of this description teaches the + // RECOVERY, not just the warning. The check is post-hoc by + // construction — the memory is already stored when the report + // arrives — so an agent told only "these look similar" has no + // action available and will ignore it. description: "Store a new observation, fact, or piece of context as a memory. \ The system automatically generates an embedding for semantic search. \ Use tags to categorize (e.g., \"topic/rust\", \"project/recalld\"). \ - Memories decay naturally over time unless reinforced." + Memories decay naturally over time unless reinforced. \ + The memory is always stored, but the result may carry a `nearDuplicates` \ + object listing existing memories at cosine similarity 0.85 or higher. \ + Act on it rather than ignoring it: if the new memory restates one of them, \ + call `forget_memory` on the id you just created, then either \ + `reinforce_memory` on the existing memory (nothing new to say) or \ + `store_memory` again with `supersedes` set to the existing id (the fact \ + changed). Leave both only when they are genuinely distinct facts that \ + happen to read alike. Pass `checkDuplicates: false` to skip the check \ + during bulk ingestion." .to_string(), input_schema: json!({ "type": "object", @@ -138,6 +186,11 @@ fn store_memory_def() -> ToolInfo { "supersedes": { "type": "string", "description": "UUID of an existing memory in the same namespace that this one replaces. Recall returns this memory in place of the old one. The store fails if the target does not exist or is in a different namespace." + }, + "checkDuplicates": { + "type": "boolean", + "description": "Report existing memories that closely resemble this one (default: true). Never blocks the store; it only adds a `nearDuplicates` object to the result. Set false for bulk ingestion, where one similarity scan per memory is not worth paying.", + "default": true } }, "required": ["summary"] @@ -191,7 +244,13 @@ fn store_memories_def() -> ToolInfo { title: Some("Store Memories (Batch)".to_string()), description: "Store multiple memories in a single call. Each item has the \ same schema as store_memory. Returns an array of results, one per \ - input memory. Saves round trips for bulk ingestion." + input memory. Saves round trips for bulk ingestion. \ + Items are stored in order and each is indexed before the next is \ + checked, so a result entry may carry a `nearDuplicates` object naming \ + an earlier item of the same batch as well as pre-existing memories. \ + The recovery is the same as for store_memory: forget the new id and \ + either reinforce the existing memory or re-store it with `supersedes`. \ + Set `checkDuplicates: false` on an item to skip its check." .to_string(), input_schema: json!({ "type": "object", @@ -249,6 +308,11 @@ fn store_memories_def() -> ToolInfo { "supersedes": { "type": "string", "description": "UUID of an existing memory in the same namespace that this one replaces. The store fails if the target does not exist or is in a different namespace." + }, + "checkDuplicates": { + "type": "boolean", + "description": "Report existing memories that closely resemble this one (default: true). Never blocks the store; it only adds a `nearDuplicates` object to this item's result. Set false for bulk ingestion, where one similarity scan per memory is not worth paying.", + "default": true } }, "required": ["summary"] @@ -266,6 +330,57 @@ fn store_memories_def() -> ToolInfo { } } +/// Build one `store_memories` result entry. +/// +/// Hand-built rather than `serde_json::to_value(stored)` because a batch +/// entry additionally carries its `index`, and existing consumers key off +/// that shape. +/// +/// Extracted into a pure function so the hand-built JSON can be tested +/// against `StoredMemory`'s own serialization. `json!` ignores +/// `skip_serializing_if`, so every optional field on `StoredMemory` has to +/// be inserted conditionally here or it is silently missing from batch +/// results while the single-memory tool emits it. That is a class of bug, +/// not one instance — `supersedes` was the first, `nearDuplicates` the +/// second, and the test is what stops the third. +fn store_result_entry(index: usize, stored: &StoredMemory) -> serde_json::Value { + let mut entry = json!({ + "index": index, + "id": stored.id, + "namespace": stored.namespace, + "phase": stored.phase, + "strength": stored.strength, + "stability": stored.stability, + "createdAt": stored.created_at, + }); + + let optional: [(&str, Option); 2] = [ + ( + "supersedes", + stored + .supersedes + .as_ref() + .and_then(|v| serde_json::to_value(v).ok()), + ), + ( + "nearDuplicates", + stored + .near_duplicates + .as_ref() + .and_then(|v| serde_json::to_value(v).ok()), + ), + ]; + if let Some(obj) = entry.as_object_mut() { + for (key, value) in optional { + if let Some(value) = value { + obj.insert(key.to_string(), value); + } + } + } + + entry +} + async fn handle_store_memories(bridge: &McpBridge, arguments: serde_json::Value) -> ToolCallResult { let memories_val = match arguments.get("memories") { Some(v) => v, @@ -312,29 +427,7 @@ async fn handle_store_memories(bridge: &McpBridge, arguments: serde_json::Value) }; match bridge.storage.store_memory(input).await { - Ok(stored) => { - let mut entry = json!({ - "index": index, - "id": stored.id, - "namespace": stored.namespace, - "phase": stored.phase, - "strength": stored.strength, - "stability": stored.stability, - "createdAt": stored.created_at, - }); - // `json!` ignores `skip_serializing_if`, so insert the outcome - // only when there is one. Otherwise every batch item would - // carry a `"supersedes": null` that the single-memory tool, - // which serializes `StoredMemory` directly, does not emit. - if let Some(outcome) = &stored.supersedes { - if let (Some(obj), Ok(value)) = - (entry.as_object_mut(), serde_json::to_value(outcome)) - { - obj.insert("supersedes".to_string(), value); - } - } - results.push(entry); - } + Ok(stored) => results.push(store_result_entry(index, &stored)), Err(e) => { results.push(json!({ "index": index, @@ -461,77 +554,23 @@ async fn handle_recall_memories( bridge: &McpBridge, arguments: serde_json::Value, ) -> ToolCallResult { - let query = match arguments.get("query").and_then(|v| v.as_str()) { - Some(s) => s.to_string(), - None => return ToolCallResult::error("Missing required parameter: query"), - }; - - let limit = arguments - .get("limit") - .and_then(|v| v.as_u64()) - .unwrap_or(10) as usize; - let namespace = arguments - .get("namespace") - .and_then(|v| v.as_str()) - .map(String::from) - .unwrap_or_else(|| bridge.default_namespace().to_string()); - let tags: Vec = arguments - .get("tags") - .and_then(|v| serde_json::from_value(v.clone()).ok()) - .unwrap_or_default(); - let entities: Vec = arguments - .get("entities") - .and_then(|v| serde_json::from_value(v.clone()).ok()) - .unwrap_or_default(); - let topics: Vec = arguments - .get("topics") - .and_then(|v| serde_json::from_value(v.clone()).ok()) - .unwrap_or_default(); - let emotions: Vec = arguments - .get("emotions") - .and_then(|v| serde_json::from_value(v.clone()).ok()) - .unwrap_or_default(); - let min_strength = arguments - .get("minStrength") - .and_then(|v| v.as_f64()) - .map(|f| f as f32); - let depth = arguments.get("depth").and_then(|v| v.as_u64()).unwrap_or(0) as u32; - let compact = arguments - .get("compact") - .and_then(|v| v.as_bool()) - .unwrap_or(true); - - // Parse time range values: accept either integer (millis) or ISO 8601 string. - let time_range_start = match arguments.get("timeRangeStart") { - Some(v) => match crate::time::parse_time_value(v) { - Some(Ok(ms)) => Some(ms), - Some(Err(e)) => return ToolCallResult::error(format!("Invalid timeRangeStart: {e}")), - None => None, - }, - None => None, - }; - let time_range_end = match arguments.get("timeRangeEnd") { - Some(v) => match crate::time::parse_time_value(v) { - Some(Ok(ms)) => Some(ms), - Some(Err(e)) => return ToolCallResult::error(format!("Invalid timeRangeEnd: {e}")), - None => None, - }, - None => None, - }; - - let input = crate::mcp::bridge::SearchInput { - query, - namespace, - limit: limit.min(100), - tags, - entities, - topics, - emotions, - min_strength, - depth, - time_range_start, - time_range_end, - }; + // Every wrong-typed or out-of-range argument is rejected in `args`. + // Only `namespace` used to be: `{"tags": "topic/rust"}` became `[]` + // and ran UNFILTERED over the whole namespace, `limit: 1000` was + // clamped to 100, and `compact: "false"` became `true`. + let args::RecallArgs { input, compact } = + match args::parse_recall_input(&arguments, bridge.default_namespace()) { + Ok(parsed) => parsed, + Err(e) => { + tracing::warn!( + tool = "recall_memories", + field = %e.field, + error = %e.message, + "Rejected malformed MCP tool arguments" + ); + return ToolCallResult::error(e.to_string()); + } + }; match bridge.search.search(input).await { Ok(search_response) => { @@ -721,16 +760,27 @@ fn forget_memory_def() -> ToolInfo { ToolInfo { name: "forget_memory".to_string(), title: Some("Forget Memory".to_string()), - description: "Permanently delete a memory. Use for incorrect, outdated, \ - or harmful information that should be immediately removed rather \ - than allowed to decay naturally." + // "Permanently delete" was wrong in both directions: the record + // survives (only its content is erased), and the tombstone it + // leaves behind is never reclaimed. An agent told "permanent" + // reasonably assumes the row is gone and the space is back. + description: "Delete a memory's content. The memory moves to the Tombstone \ + phase: its summary, full text and tags are erased and it stops appearing \ + in recall, but its record and its graph edges are kept so relationship \ + chains stay intact, and its UUID is never reused. Tombstoned memories are \ + never reclaimed by the decay sweep, so they persist for the life of the \ + database. Use for incorrect, outdated or harmful information that must \ + stop being recalled immediately. To correct a memory, prefer storing the \ + correction with `supersedes`." .to_string(), input_schema: json!({ "type": "object", "properties": { "id": { "type": "string", - "description": "Memory UUID to delete" + "description": "Memory UUID to forget. The memory moves to the \ + Tombstone phase; its graph edges are kept and its UUID is \ + never reused." } }, "required": ["id"] @@ -771,6 +821,166 @@ async fn handle_forget_memory(bridge: &McpBridge, arguments: serde_json::Value) } } +// ═══════════════════════════════════════════════════════════════════════ +// Tool 5b: forget_memories (batch) +// ═══════════════════════════════════════════════════════════════════════ + +fn forget_memories_def() -> ToolInfo { + ToolInfo { + name: "forget_memories".to_string(), + title: Some("Forget Memories (Batch)".to_string()), + description: "Delete several memories in a single call. Each moves to the \ + Tombstone phase exactly as forget_memory describes: content erased, \ + record and graph edges kept, UUID never reused, never reclaimed by the \ + decay sweep. Returns an array of results, one per input id, in the order \ + given. A `deleted: false` result means the id was unknown or the memory \ + had already been forgotten." + .to_string(), + input_schema: json!({ + "type": "object", + "properties": { + "ids": { + "type": "array", + "minItems": 1, + "maxItems": MAX_BATCH_MEMORIES, + "items": { "type": "string", "format": "uuid" }, + "description": "Memory UUIDs to forget. Each moves to the Tombstone \ + phase; its graph edges are kept and its UUID is never reused." + } + }, + "required": ["ids"] + }), + annotations: Some(ToolAnnotations { + read_only_hint: Some(false), + destructive_hint: Some(true), + idempotent_hint: Some(true), + open_world_hint: Some(false), + }), + } +} + +async fn handle_forget_memories( + bridge: &McpBridge, + arguments: serde_json::Value, +) -> ToolCallResult { + // Same guard order as store_memories: missing key, wrong type, + // empty, over cap. + let ids_val = match arguments.get("ids") { + Some(v) => v, + None => return ToolCallResult::error("Missing required parameter: ids"), + }; + + let ids_arr = match ids_val.as_array() { + Some(arr) => arr, + None => return ToolCallResult::error("Parameter 'ids' must be an array"), + }; + + if ids_arr.is_empty() { + return ToolCallResult::error("Parameter 'ids' must not be empty"); + } + + if ids_arr.len() > MAX_BATCH_MEMORIES { + return ToolCallResult::error(format!( + "Too many ids (maximum {MAX_BATCH_MEMORIES} per call)" + )); + } + + // UUIDs are parsed here, not at the bridge, so one malformed id + // becomes a per-item error instead of failing the whole batch. Each + // valid id keeps its original index so the results can be merged + // back in caller order. + let mut results: Vec = vec![serde_json::Value::Null; ids_arr.len()]; + let mut parsed: Vec<(usize, crate::model::MemoryId)> = Vec::with_capacity(ids_arr.len()); + + for (index, item) in ids_arr.iter().enumerate() { + let Some(id_str) = item.as_str() else { + results[index] = json!({ + "index": index, + "error": format!("Parameter 'ids[{index}]' must be a string"), + "field": format!("ids[{index}]"), + }); + continue; + }; + match uuid::Uuid::parse_str(id_str) { + Ok(uuid) => parsed.push((index, crate::model::MemoryId::from_uuid(uuid))), + Err(_) => { + results[index] = json!({ + "index": index, + "error": format!("Invalid UUID: {id_str}"), + "field": format!("ids[{index}]"), + }); + } + } + } + + if !parsed.is_empty() { + let ids: Vec = parsed.iter().map(|(_, id)| *id).collect(); + match bridge.storage.delete_memories(&ids).await { + // One flag per id, in input order, is the contract. The + // in-process adapter keeps it by construction, but this also + // decodes a `Vec` off the daemon socket — and `zip` + // stops at the shorter side, so a short response would leave + // `Value::Null` entries in `results`, under-count `deleted`, + // and report all of it as a success. + Ok(flags) if flags.len() != parsed.len() => { + let detail = format!( + "delete_memories returned {} results for {} ids; no per-id outcome \ + can be reported. Re-read the ids to find out which were deleted.", + flags.len(), + parsed.len() + ); + tracing::error!( + tool = "forget_memories", + returned = flags.len(), + requested = parsed.len(), + "delete_memories broke its one-flag-per-id contract" + ); + for (index, id) in &parsed { + results[*index] = json!({ + "index": index, + "id": id.to_string(), + "error": detail, + }); + } + } + Ok(flags) => { + for ((index, id), deleted) in parsed.iter().zip(flags) { + results[*index] = json!({ + "index": index, + "id": id.to_string(), + "deleted": deleted, + }); + } + } + Err(e) => { + for (index, id) in &parsed { + results[*index] = json!({ + "index": index, + "id": id.to_string(), + "error": format!("Failed to delete memory: {e}"), + }); + } + } + } + } + + let deleted_count = results + .iter() + .filter(|r| r.get("deleted").and_then(|v| v.as_bool()) == Some(true)) + .count(); + let error_count = results.iter().filter(|r| r.get("error").is_some()).count(); + let response = json!({ + "results": results, + "total": results.len(), + "deleted": deleted_count, + "errors": error_count, + }); + match ToolCallResult::json(&response) { + Ok(r) => r, + Err(e) => ToolCallResult::error(format!("Serialization error: {e}")), + } +} + // ═══════════════════════════════════════════════════════════════════════ // Tool 6: find_similar_memories // ═══════════════════════════════════════════════════════════════════════ @@ -816,7 +1026,9 @@ fn find_similar_memories_def() -> ToolInfo { "threshold": { "type": "number", "description": "Similarity threshold for \"scan\" mode duplicate detection (0.0-1.0, default: 0.85)", - "default": 0.85, + // `f64`, so this serializes as `0.85` and not as the + // widened `0.8500000238418579` an `f32` would produce. + "default": DUPLICATE_SIMILARITY_THRESHOLD, "minimum": 0.0, "maximum": 1.0 }, @@ -840,14 +1052,26 @@ async fn handle_find_similar_memories( bridge: &McpBridge, arguments: serde_json::Value, ) -> ToolCallResult { - let mode = arguments - .get("mode") - .and_then(|v| v.as_str()) - .unwrap_or("single"); + // A wrong-typed `mode` is an error, never a silent "single": those + // are two different operations, and running the wrong one is not a + // degradation the caller can detect. + let mode = match args::opt_enum(&arguments, "mode", &["single", "scan"]) { + Ok(m) => m.unwrap_or_else(|| "single".to_string()), + Err(e) => { + tracing::warn!( + tool = "find_similar_memories", + field = %e.field, + error = %e.message, + "Rejected malformed MCP tool arguments" + ); + return ToolCallResult::error(e.to_string()); + } + }; - match mode { + match mode.as_str() { "single" => handle_find_similar_single(bridge, &arguments).await, "scan" => handle_find_similar_scan(bridge, &arguments).await, + // Unreachable: `opt_enum` already rejected anything else. other => ToolCallResult::error(format!( "Invalid mode: \"{other}\". Must be \"single\" or \"scan\"." )), @@ -859,42 +1083,32 @@ async fn handle_find_similar_single( bridge: &McpBridge, arguments: &serde_json::Value, ) -> ToolCallResult { - let id_str = match arguments.get("id").and_then(|v| v.as_str()) { - Some(s) => s, - None => { - return ToolCallResult::error( - "Missing required parameter: id (required for \"single\" mode)", + let args::FindSimilarArgs { + id, + limit, + min_score, + same_namespace, + } = match args::parse_find_similar_args(arguments) { + Ok(parsed) => parsed, + Err(e) => { + tracing::warn!( + tool = "find_similar_memories", + field = %e.field, + error = %e.message, + "Rejected malformed MCP tool arguments" ); + return ToolCallResult::error(e.to_string()); } }; - let uuid = match uuid::Uuid::parse_str(id_str) { - Ok(u) => u, - Err(_) => return ToolCallResult::error(format!("Invalid UUID: {id_str}")), - }; - let id = crate::model::MemoryId::from_uuid(uuid); - - let limit = arguments - .get("limit") - .and_then(|v| v.as_u64()) - .unwrap_or(10) as usize; - let min_score = arguments - .get("minScore") - .and_then(|v| v.as_f64()) - .map(|f| f as f32); - let same_namespace = arguments - .get("sameNamespace") - .and_then(|v| v.as_bool()) - .unwrap_or(true); - match bridge .search - .find_similar(id, limit.min(100), min_score, same_namespace) + .find_similar(id, limit, min_score, same_namespace) .await { Ok(hits) => { let response = json!({ - "sourceId": id_str, + "sourceId": id.to_string(), "memories": hits, "count": hits.len(), }); @@ -912,20 +1126,25 @@ async fn handle_find_similar_scan( bridge: &McpBridge, arguments: &serde_json::Value, ) -> ToolCallResult { - let namespace = arguments - .get("namespace") - .and_then(|v| v.as_str()) - .map(String::from) - .unwrap_or_else(|| bridge.default_namespace().to_string()); - - let threshold = arguments - .get("threshold") - .and_then(|v| v.as_f64()) - .unwrap_or(0.85) as f32; - - if !(0.0..=1.0).contains(&threshold) { - return ToolCallResult::error("threshold must be between 0.0 and 1.0"); - } + let args::DuplicateScanArgs { + namespace, + threshold, + } = match args::parse_duplicate_scan_args( + arguments, + bridge.default_namespace(), + DUPLICATE_SIMILARITY_THRESHOLD, + ) { + Ok(parsed) => parsed, + Err(e) => { + tracing::warn!( + tool = "find_similar_memories", + field = %e.field, + error = %e.message, + "Rejected malformed MCP tool arguments" + ); + return ToolCallResult::error(e.to_string()); + } + }; // Max memories to sample from the namespace (bounded to 200). const MAX_SCAN_MEMORIES: usize = 200; @@ -1054,6 +1273,113 @@ async fn handle_create_namespace( } } +// ═══════════════════════════════════════════════════════════════════════ +// Tool 8b: delete_namespace +// ═══════════════════════════════════════════════════════════════════════ + +fn delete_namespace_def() -> ToolInfo { + ToolInfo { + name: "delete_namespace".to_string(), + title: Some("Delete Namespace".to_string()), + description: "Destroy a namespace and everything in it: every memory, every \ + graph edge touching one, its search index entries, and its vector file on \ + disk. There is no undo and no backup. A namespace that still holds \ + memories is refused unless you pass force: true. \ + This is also the only operation that reclaims tombstones — memories \ + deleted with forget_memory keep a record for the life of the database, and \ + nothing else ever removes it. \ + The 'default' namespace cannot be deleted." + .to_string(), + input_schema: json!({ + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 64, + "description": "Namespace to delete. The 'default' namespace cannot be deleted." + }, + "force": { + "type": "boolean", + "default": false, + "description": "Delete the namespace even if it still holds memories. \ + Without this, a non-empty namespace is refused. Forced deletion \ + permanently destroys every memory in the namespace and removes \ + its vector file from disk. There is no undo and no backup." + } + }, + "required": ["name"] + }), + annotations: Some(ToolAnnotations { + read_only_hint: Some(false), + destructive_hint: Some(true), + // Deliberately NOT idempotent, unlike forget_memory. This + // removes a directory from disk and the namespace id is + // never reused, so a blind retry after an unknown outcome + // could destroy a same-named namespace the user recreated in + // between. + idempotent_hint: Some(false), + open_world_hint: Some(false), + }), + } +} + +async fn handle_delete_namespace( + bridge: &McpBridge, + arguments: serde_json::Value, +) -> ToolCallResult { + use crate::model::constants::DEFAULT_NAMESPACE_NAME; + + let name = match arguments.get("name").and_then(|v| v.as_str()) { + Some(s) => s.to_string(), + None => return ToolCallResult::error("Missing required parameter: name"), + }; + + // The name becomes a directory name that `remove_dir_all` is + // pointed at, so it is checked before anything is done with it. + // Refused here as well as in the adapter; the adapter's copy is the + // load-bearing one, because the socket is its own entry point. + if let Err(e) = validate_namespace_name(&name) { + return ToolCallResult::error(e.to_string()); + } + + // Refused here as well as in the adapter. This copy saves a daemon + // round trip; the adapter's copy is the load-bearing one, because + // the socket is its own entry point. + // Case-insensitive: the guard protects a directory, and a + // case-insensitive filesystem resolves `Default/` to `default/`. + if name.eq_ignore_ascii_case(DEFAULT_NAMESPACE_NAME) { + return ToolCallResult::error(format!( + "The '{DEFAULT_NAMESPACE_NAME}' namespace cannot be deleted. It is recreated \ + on every startup under the same directory name, so deleting it would leave \ + a stale vectors.dat that a new namespace could silently reuse." + )); + } + + // A wrong-typed `force` is an error naming the field, never a silent + // false: quietly downgrading to the safe path would make a caller + // with a broken serializer see a refusal they cannot explain. + let force = match args::opt_bool(&arguments, "force") { + Ok(v) => v.unwrap_or(false), + Err(e) => return ToolCallResult::error(e.to_string()), + }; + + let input = crate::mcp::bridge::DeleteNamespaceInput { name, force }; + + match bridge.namespaces.delete_namespace(input).await { + Ok(result) => match ToolCallResult::json(&result) { + Ok(r) => r, + Err(e) => ToolCallResult::error(format!("Serialization error: {e}")), + }, + // A partial purge is not a failed delete. Prefixing it with + // "Failed to delete namespace" would tell the agent nothing + // happened, when in fact memories were destroyed and the job + // needs finishing — so the message passes through unwrapped. + Err(crate::mcp::bridge::BridgeError::PartiallyApplied(msg)) => ToolCallResult::error(msg), + Err(e) => ToolCallResult::error(format!("Failed to delete namespace: {e}")), + } +} + // ═══════════════════════════════════════════════════════════════════════ // Tool 9: namespace_stats // ═══════════════════════════════════════════════════════════════════════ @@ -1062,10 +1388,16 @@ fn namespace_stats_def() -> ToolInfo { ToolInfo { name: "namespace_stats".to_string(), title: Some("Namespace Stats".to_string()), - description: "Get statistics for a memory namespace including total memory count, \ + description: "Get statistics for a memory namespace: live memory count, \ phase breakdown (full/summary/ghost), permastore count, average strength, \ edge count, and vector storage size. Use this to check how many memories \ - exist or monitor namespace health." + exist or monitor namespace health. \ + `memoryCount` and every figure derived from it EXCLUDE deleted \ + (tombstoned) memories, and always equal full + summary + ghost, so it \ + agrees with list_memories and list_namespaces. Tombstoned records are \ + reported separately as `tombstoneCount`: they hold no content, but nothing \ + except delete_namespace ever reclaims them, so a namespace showing one \ + memory where eleven were stored has ten of them." .to_string(), input_schema: json!({ "type": "object", @@ -1090,11 +1422,18 @@ async fn handle_namespace_stats( bridge: &McpBridge, arguments: serde_json::Value, ) -> ToolCallResult { - let namespace = arguments - .get("namespace") - .and_then(|v| v.as_str()) - .map(String::from) - .unwrap_or_else(|| bridge.default_namespace().to_string()); + let namespace = match args::namespace_or(&arguments, bridge.default_namespace()) { + Ok(ns) => ns, + Err(e) => { + tracing::warn!( + tool = "namespace_stats", + field = %e.field, + error = %e.message, + "Rejected malformed MCP tool arguments" + ); + return ToolCallResult::error(e.to_string()); + } + }; match bridge.namespaces.namespace_stats(&namespace).await { Ok(stats) => match ToolCallResult::json(&stats) { @@ -1129,10 +1468,13 @@ fn list_memories_def() -> ToolInfo { }, "limit": { "type": "integer", - "description": "Maximum results per page (default: 50, max: 200)", - "default": 50, + "description": format!( + "Maximum results per page (default: {DEFAULT_LIST_LIMIT}, \ + max: {MAX_LIST_LIMIT}). A larger value is rejected, not clamped." + ), + "default": DEFAULT_LIST_LIMIT, "minimum": 1, - "maximum": 200 + "maximum": MAX_LIST_LIMIT }, "offset": { "type": "integer", @@ -1176,66 +1518,149 @@ fn list_memories_def() -> ToolInfo { } async fn handle_list_memories(bridge: &McpBridge, arguments: serde_json::Value) -> ToolCallResult { - let namespace = arguments - .get("namespace") - .and_then(|v| v.as_str()) - .map(String::from) - .unwrap_or_else(|| bridge.default_namespace().to_string()); - - let limit = arguments - .get("limit") - .and_then(|v| v.as_u64()) - .unwrap_or(50) as usize; - let limit = limit.min(200); + // Every wrong-typed or out-of-range argument is rejected in `args`, + // so this handler cannot silently list the default namespace or + // silently return a smaller page than was asked for. + let input = match args::parse_list_memories_input(&arguments, bridge.default_namespace()) { + Ok(input) => input, + Err(e) => { + tracing::warn!( + tool = "list_memories", + field = %e.field, + error = %e.message, + "Rejected malformed MCP tool arguments" + ); + return ToolCallResult::error(e.to_string()); + } + }; - let offset = arguments - .get("offset") - .and_then(|v| v.as_u64()) - .unwrap_or(0) as usize; - - let tags: Vec = arguments - .get("tags") - .and_then(|v| serde_json::from_value(v.clone()).ok()) - .unwrap_or_default(); - let entities: Vec = arguments - .get("entities") - .and_then(|v| serde_json::from_value(v.clone()).ok()) - .unwrap_or_default(); - - // Parse time range values: accept either integer (millis) or ISO 8601 string. - let time_range_start = match arguments.get("timeRangeStart") { - Some(v) => match crate::time::parse_time_value(v) { - Some(Ok(ms)) => Some(ms), - Some(Err(e)) => return ToolCallResult::error(format!("Invalid timeRangeStart: {e}")), - None => None, + match bridge.storage.list_memories(input).await { + Ok(response) => match ToolCallResult::json(&response) { + Ok(r) => r, + Err(e) => ToolCallResult::error(format!("Serialization error: {e}")), }, - None => None, - }; - let time_range_end = match arguments.get("timeRangeEnd") { - Some(v) => match crate::time::parse_time_value(v) { - Some(Ok(ms)) => Some(ms), - Some(Err(e)) => return ToolCallResult::error(format!("Invalid timeRangeEnd: {e}")), - None => None, + Err(e) => ToolCallResult::error(format!("List memories failed: {e}")), + } +} + +// ═══════════════════════════════════════════════════════════════════════ +// Tool 11: list_namespaces +// ═══════════════════════════════════════════════════════════════════════ + +fn list_namespaces_def() -> ToolInfo { + ToolInfo { + name: "list_namespaces".to_string(), + title: Some("List Namespaces".to_string()), + description: "List every memory namespace with its ID, embedding dimensions, live \ + memory count, and creation date. Call this first when you do not know which \ + namespaces exist, before targeting one with list_memories, list_tags, or \ + namespace_stats. Counts exclude deleted (tombstoned) memories." + .to_string(), + input_schema: json!({ + "type": "object", + "properties": {} + }), + annotations: Some(ToolAnnotations { + read_only_hint: Some(true), + destructive_hint: Some(false), + idempotent_hint: Some(true), + open_world_hint: Some(false), + }), + } +} + +/// The response is the bare serialization of `Vec`, not a +/// `{"namespaces": [...], "total": n}` envelope. Reusing the type +/// unchanged makes this tool and the `recalld://namespaces` resource +/// provably the same payload, and `total` is the array length. +async fn handle_list_namespaces( + bridge: &McpBridge, + _arguments: serde_json::Value, +) -> ToolCallResult { + match bridge.namespaces.list_namespaces().await { + Ok(namespaces) => match ToolCallResult::json(&namespaces) { + Ok(r) => r, + Err(e) => ToolCallResult::error(format!("Serialization error: {e}")), }, - None => None, - }; + Err(e) => ToolCallResult::error(format!("List namespaces failed: {e}")), + } +} - let input = crate::mcp::bridge::ListMemoriesInput { - namespace, - limit, - offset, - tags, - entities, - time_range_start, - time_range_end, +// ═══════════════════════════════════════════════════════════════════════ +// Tool 12: list_tags +// ═══════════════════════════════════════════════════════════════════════ + +fn list_tags_def() -> ToolInfo { + ToolInfo { + name: "list_tags".to_string(), + title: Some("List Tags".to_string()), + description: "List the tag vocabulary with per-label memory counts, most common \ + first. Use this to discover what labels already exist before filtering \ + list_memories or recall_memories, and before storing — so you reuse an existing \ + tag instead of inventing a near-duplicate. Results come in four buckets: plain \ + tags, plus the entity, topic, and emotion labels derived from those fields, \ + returned without their prefixes so they can be passed straight back to \ + store_memory or list_memories. Counts cover live memories only; deleted \ + memories are excluded." + .to_string(), + input_schema: json!({ + "type": "object", + "properties": { + "namespace": { + "type": "string", + "description": "Which namespace to count within (default: \"default\"). \ + Reads that namespace's memory records, costing about one \ + list_memories call.", + "default": "default" + }, + "allNamespaces": { + "type": "boolean", + "description": "Count across every namespace instead of one. \ + Cannot be combined with an explicit namespace.", + "default": false + }, + "limit": { + "type": "integer", + "description": format!( + "Maximum labels returned per bucket (default: {DEFAULT_TAG_LIMIT}, \ + max: {MAX_TAG_LIMIT}). Each bucket also reports its untruncated \ + total." + ), + "default": DEFAULT_TAG_LIMIT, + "minimum": 1, + "maximum": MAX_TAG_LIMIT + } + } + }), + annotations: Some(ToolAnnotations { + read_only_hint: Some(true), + destructive_hint: Some(false), + idempotent_hint: Some(true), + open_world_hint: Some(false), + }), + } +} + +async fn handle_list_tags(bridge: &McpBridge, arguments: serde_json::Value) -> ToolCallResult { + let input = match args::parse_list_tags_input(&arguments, bridge.default_namespace()) { + Ok(input) => input, + Err(e) => { + tracing::warn!( + tool = "list_tags", + field = %e.field, + error = %e.message, + "Rejected malformed MCP tool arguments" + ); + return ToolCallResult::error(e.to_string()); + } }; - match bridge.storage.list_memories(input).await { + match bridge.storage.list_tags(input).await { Ok(response) => match ToolCallResult::json(&response) { Ok(r) => r, Err(e) => ToolCallResult::error(format!("Serialization error: {e}")), }, - Err(e) => ToolCallResult::error(format!("List memories failed: {e}")), + Err(e) => ToolCallResult::error(format!("List tags failed: {e}")), } } @@ -1320,4 +1745,755 @@ mod tests { Some(MAX_BATCH_MEMORIES as u64) ); } + + // ── near-duplicate advisory ────────────────────────────────────── + + /// The `checkDuplicates` sub-schema of each store tool, keyed by tool + /// name — same shape as [`summary_schemas`], and for the same reason: + /// the two schemas are written out separately and drift silently. + fn check_duplicates_schemas() -> Vec<(&'static str, serde_json::Value)> { + let single = store_memory_def().input_schema["properties"]["checkDuplicates"].clone(); + let batch = store_memories_def().input_schema["properties"]["memories"]["items"] + ["properties"]["checkDuplicates"] + .clone(); + vec![("store_memory", single), ("store_memories", batch)] + } + + #[test] + fn d21_both_store_tools_declare_check_duplicates_defaulting_to_true() { + for (tool, schema) in check_duplicates_schemas() { + assert_eq!(schema["type"], json!("boolean"), "{tool}"); + assert_eq!( + schema["default"], + json!(true), + "{tool}: the opt-out must advertise itself as on by default" + ); + assert!( + schema["description"] + .as_str() + .is_some_and(|d| !d.is_empty()), + "{tool}: checkDuplicates has no description" + ); + } + } + + /// The warning is post-hoc, so the description has to name both the + /// field the caller will see and the recovery that acts on it. A + /// description that mentions neither leaves the agent with a report it + /// has no instruction to use. + #[test] + fn d22_store_memory_description_names_the_report_and_the_recovery() { + let description = store_memory_def().description; + for needle in [ + "nearDuplicates", + "supersedes", + "forget_memory", + "reinforce_memory", + "checkDuplicates", + ] { + assert!( + description.contains(needle), + "store_memory description does not mention {needle}: {description}" + ); + } + } + + /// `json!` widens an `f32` to `f64` on the way into a `Value`, so a + /// threshold declared `0.85f32` publishes as `0.8500000238418579` — + /// a schema change nobody made and nobody would notice. The constant + /// is `f64` for exactly this reason; this test is what keeps it that + /// way. + #[test] + fn d23_scan_mode_threshold_default_serializes_as_a_clean_decimal() { + let schema = find_similar_memories_def().input_schema; + assert_eq!( + schema["properties"]["threshold"]["default"], + json!(0.85), + "threshold default is not a clean decimal" + ); + } + + // ── The batch trap ─────────────────────────────────────────────── + + fn stored_memory( + supersedes: Option, + near_duplicates: Option, + ) -> StoredMemory { + StoredMemory { + id: "0195e2c0-0000-7000-8000-000000000001".to_string(), + namespace: "default".to_string(), + phase: "Full".to_string(), + strength: 1.0, + stability: 3.7145, + created_at: "2026-08-11T12:00:00Z".to_string(), + supersedes, + near_duplicates, + } + } + + /// `json!` ignores `skip_serializing_if`, so a batch entry is built by + /// hand and every optional field has to be re-remembered there. This + /// compares the hand-built entry against `StoredMemory`'s own + /// serialization: every key the single-memory tool emits must appear + /// in the batch entry with an equal value. + /// + /// It would have caught the original `supersedes` omission, and it + /// catches the next field somebody adds to `StoredMemory` without + /// touching `store_result_entry`. + #[test] + fn e24_batch_entry_carries_every_field_the_single_store_serializes() { + let stored = stored_memory( + Some(crate::model::SupersedesOutcome::new( + crate::model::MemoryId::new(), + crate::model::SupersedesStatus::Applied, + )), + crate::model::NearDuplicateReport::from_matches( + 0.85, + vec![crate::model::NearDuplicateMatch { + id: crate::model::MemoryId::new(), + score: 0.91, + summary: "an older phrasing".to_string(), + }], + ), + ); + + let direct = serde_json::to_value(&stored).expect("StoredMemory serializes"); + let batch = store_result_entry(7, &stored); + + for (key, value) in direct.as_object().expect("direct output is an object") { + // `createdAt` is the one renamed key: `StoredMemory` and the + // batch entry agree on it, so nothing to special-case. + assert_eq!( + batch.get(key), + Some(value), + "batch entry is missing or disagrees about '{key}'" + ); + } + assert_eq!(batch["index"], json!(7)); + } + + /// The other half: an absent optional field must be ABSENT, not + /// `null`, so the batch shape matches what `store_memory` emits for + /// the same store. + #[test] + fn e25_batch_entry_omits_absent_optional_fields() { + let batch = store_result_entry(0, &stored_memory(None, None)); + let obj = batch.as_object().expect("entry is an object"); + + assert!(!obj.contains_key("supersedes"), "{batch}"); + assert!(!obj.contains_key("nearDuplicates"), "{batch}"); + } + + // ── Batch forget ───────────────────────────────────────────────── + + /// The published bound and the enforced bound are the same constant, + /// so a client that trusts the schema cannot be told its request was + /// out of range. `minItems` matters too: an empty array is an error, + /// not a no-op, and the schema has to say so. + #[test] + fn w37_forget_memories_ids_bounds_match_the_batch_constant() { + let ids = forget_memories_def().input_schema["properties"]["ids"].clone(); + assert_eq!(ids["type"], json!("array")); + assert_eq!(ids["maxItems"].as_u64(), Some(MAX_BATCH_MEMORIES as u64)); + assert_eq!(ids["minItems"].as_u64(), Some(1)); + assert_eq!(ids["items"]["type"], json!("string")); + assert_eq!(ids["items"]["format"], json!("uuid")); + assert_eq!( + forget_memories_def().input_schema["required"], + json!(["ids"]) + ); + } + + // ── delete_namespace ───────────────────────────────────────────── + + /// `force` has to advertise its default. A caller that reads the + /// schema and omits the key must be able to predict which way the + /// call goes, because the two answers are "refused" and "everything + /// is destroyed". + #[test] + fn w38_delete_namespace_declares_a_required_name_and_force_defaulting_to_false() { + let schema = delete_namespace_def().input_schema; + assert_eq!(schema["required"], json!(["name"])); + + let force = &schema["properties"]["force"]; + assert_eq!(force["type"], json!("boolean")); + assert_eq!(force["default"], json!(false)); + assert!( + force["description"] + .as_str() + .is_some_and(|d| d.contains("no undo")), + "force must say there is no undo: {force}" + ); + + let name = &schema["properties"]["name"]; + assert_eq!(name["type"], json!("string")); + assert_eq!(name["minLength"].as_u64(), Some(1)); + assert_eq!(name["maxLength"].as_u64(), Some(64)); + } + + // ── Destructive-tool descriptions ──────────────────────────────── + + /// `forget_memory` used to promise a permanent delete. It does not + /// perform one: the record stays in meta.db, only its content is + /// stripped, and no code path ever hard-deletes a tombstone. An agent + /// reading "permanently delete" will believe the row and its disk + /// space are gone, and will not reach for the tool that actually + /// reclaims them. + #[test] + fn w39_forget_memory_description_does_not_promise_a_permanent_delete() { + let description = forget_memory_def().description; + assert!( + !description.to_lowercase().contains("permanent"), + "forget_memory still claims a permanent delete: {description}" + ); + assert!( + description.contains("Tombstone"), + "forget_memory does not name the phase the memory moves to: {description}" + ); + assert!( + description.contains("never reclaimed"), + "forget_memory does not say tombstones are never reclaimed: {description}" + ); + } + + /// `list_namespaces` and `list_tags` both got an "excludes deleted" + /// sentence when the counts changed; `namespace_stats` shipped the + /// same semantic change and kept saying "total memory count". The + /// tool description is what the agent actually reads — the docs are + /// not in its context — so a stale one is the version that governs. + #[test] + fn w69_namespace_stats_description_states_what_its_counts_exclude() { + let description = namespace_stats_def().description; + assert!( + !description.contains("total memory count"), + "namespace_stats still calls its count a total: {description}" + ); + assert!( + description.contains("tombstoneCount"), + "namespace_stats does not name the field it added: {description}" + ); + assert!( + description.to_lowercase().contains("exclude"), + "namespace_stats does not say its counts exclude deleted memories: \ + {description}" + ); + } + + /// The annotation an MCP client uses to decide whether to prompt. + /// Getting it wrong on these three means a destructive call goes + /// through unconfirmed. + #[test] + fn w41_every_destructive_tool_declares_the_destructive_hint() { + for def in [ + forget_memory_def(), + forget_memories_def(), + delete_namespace_def(), + ] { + let annotations = def + .annotations + .as_ref() + .unwrap_or_else(|| panic!("{} has no annotations", def.name)); + assert_eq!( + annotations.destructive_hint, + Some(true), + "{} does not declare itself destructive", + def.name + ); + assert_eq!(annotations.read_only_hint, Some(false), "{}", def.name); + } + + // Deliberately not idempotent, unlike the two forget tools: this + // removes a directory from disk and the namespace id is never + // reused, so a blind retry could destroy a same-named namespace + // recreated in between. + assert_eq!( + delete_namespace_def() + .annotations + .expect("annotations") + .idempotent_hint, + Some(false) + ); + } + + // ── Registry shape ─────────────────────────────────────────────── + + #[test] + fn t34_every_tool_definition_has_a_unique_name() { + let mut names: Vec = tool_definitions().into_iter().map(|t| t.name).collect(); + let before = names.len(); + names.sort(); + names.dedup(); + assert_eq!( + names.len(), + before, + "duplicate tool name in tool_definitions" + ); + } + + /// The count the module doc, the README and the guide all repeat. If + /// it moves, they have to move with it. + #[test] + fn t35_declares_fourteen_tools() { + assert_eq!(tool_definitions().len(), 14); + } + + /// A bridge whose every method fails. The point is to reach + /// [`dispatch_tool`]'s match arms without standing up a real system: + /// what a handler does after it finds its arm is somebody else's + /// test, but *finding* the arm is this one's. + struct Unroutable; + + macro_rules! refuse { + () => { + Err(crate::mcp::bridge::BridgeError::Internal( + "stub".to_string(), + )) + }; + } + + #[async_trait::async_trait] + impl crate::mcp::bridge::SearchPipeline for Unroutable { + async fn search( + &self, + _: crate::mcp::bridge::SearchInput, + ) -> Result { + refuse!() + } + async fn find_similar( + &self, + _: crate::model::MemoryId, + _: usize, + _: Option, + _: bool, + ) -> Result, crate::mcp::bridge::BridgeError> { + refuse!() + } + async fn scan_duplicates( + &self, + _: &str, + _: f32, + _: usize, + ) -> Result, crate::mcp::bridge::BridgeError> + { + refuse!() + } + } + + #[async_trait::async_trait] + impl crate::mcp::bridge::StorageEngine for Unroutable { + async fn store_memory( + &self, + _: crate::mcp::bridge::StoreInput, + ) -> Result { + refuse!() + } + async fn get_memory( + &self, + _: crate::model::MemoryId, + ) -> Result, crate::mcp::bridge::BridgeError> + { + refuse!() + } + async fn delete_memory( + &self, + _: crate::model::MemoryId, + ) -> Result { + refuse!() + } + async fn delete_memories( + &self, + _: &[crate::model::MemoryId], + ) -> Result, crate::mcp::bridge::BridgeError> { + refuse!() + } + async fn reinforce_memory( + &self, + _: crate::model::MemoryId, + _: u8, + ) -> Result { + refuse!() + } + async fn list_memories( + &self, + _: crate::mcp::bridge::ListMemoriesInput, + ) -> Result + { + refuse!() + } + async fn list_tags( + &self, + _: crate::mcp::bridge::ListTagsInput, + ) -> Result { + refuse!() + } + } + + #[async_trait::async_trait] + impl crate::mcp::bridge::NamespaceRegistry for Unroutable { + async fn list_namespaces( + &self, + ) -> Result, crate::mcp::bridge::BridgeError> + { + refuse!() + } + async fn create_namespace( + &self, + _: crate::mcp::bridge::CreateNamespaceInput, + ) -> Result { + refuse!() + } + async fn namespace_stats( + &self, + _: &str, + ) -> Result { + refuse!() + } + async fn delete_namespace( + &self, + _: crate::mcp::bridge::DeleteNamespaceInput, + ) -> Result + { + refuse!() + } + } + + #[async_trait::async_trait] + impl crate::mcp::bridge::HealthChecker for Unroutable { + async fn check_health(&self) -> crate::mcp::bridge::HealthStatus { + unreachable!("no tool reaches the health checker") + } + } + + fn unroutable_bridge() -> McpBridge { + let stub = std::sync::Arc::new(Unroutable); + McpBridge { + search: stub.clone(), + storage: stub.clone(), + namespaces: stub.clone(), + health: stub, + default_namespace: "default".to_string(), + timezone: chrono_tz::UTC, + } + } + + /// [`DISPATCHED`] is hand-maintained next to the match, so it can + /// drift from the match itself. This one goes through the real + /// [`dispatch_tool`]: a declared tool with no arm answers "Unknown + /// tool", and the point of declaring it was that it would not. + #[tokio::test] + async fn w40_dispatch_tool_routes_every_declared_tool() { + let bridge = unroutable_bridge(); + + for tool in tool_definitions() { + let result = dispatch_tool(&bridge, &tool.name, json!({})).await; + let rendered = serde_json::to_string(&result).expect("result serializes"); + assert!( + !rendered.contains("Unknown tool"), + "{} is declared but dispatch_tool does not route it", + tool.name + ); + } + + let unknown = dispatch_tool(&bridge, "no_such_tool", json!({})).await; + assert!( + serde_json::to_string(&unknown) + .expect("result serializes") + .contains("Unknown tool"), + "the unknown-tool path stopped reporting unknown tools" + ); + } + + /// A definition without a dispatch arm advertises a tool that answers + /// "Unknown tool". Cheap to catch here, expensive to catch in the + /// field. + #[test] + fn t36_every_declared_tool_has_a_dispatch_arm() { + for tool in tool_definitions() { + assert!( + DISPATCHED.contains(&tool.name.as_str()), + "{} is declared but has no dispatch arm", + tool.name + ); + } + assert_eq!(DISPATCHED.len(), tool_definitions().len()); + } + + /// The published bound and the enforced bound are the same constant, + /// so a client that trusts the schema cannot be told its request was + /// out of range. + #[test] + fn t37_listing_schema_limits_match_their_constants() { + let list = list_memories_def().input_schema; + assert_eq!( + list["properties"]["limit"]["maximum"].as_u64(), + Some(MAX_LIST_LIMIT as u64) + ); + assert_eq!( + list["properties"]["limit"]["default"].as_u64(), + Some(DEFAULT_LIST_LIMIT as u64) + ); + assert_eq!(list["properties"]["limit"]["minimum"].as_u64(), Some(1)); + + let tags = list_tags_def().input_schema; + assert_eq!( + tags["properties"]["limit"]["maximum"].as_u64(), + Some(MAX_TAG_LIMIT as u64) + ); + assert_eq!( + tags["properties"]["limit"]["default"].as_u64(), + Some(DEFAULT_TAG_LIMIT as u64) + ); + assert_eq!(tags["properties"]["limit"]["minimum"].as_u64(), Some(1)); + } + + #[test] + fn t38_read_only_tools_declare_the_read_only_hint() { + const READ_ONLY: [&str; 7] = [ + "recall_memories", + "get_memory", + "find_similar_memories", + "namespace_stats", + "list_memories", + "list_namespaces", + "list_tags", + ]; + for tool in tool_definitions() { + let Some(annotations) = tool.annotations else { + panic!("{} has no annotations", tool.name); + }; + let expected = READ_ONLY.contains(&tool.name.as_str()); + assert_eq!( + annotations.read_only_hint, + Some(expected), + "{} read_only_hint", + tool.name + ); + } + } + + #[test] + fn t39_list_namespaces_takes_no_parameters() { + let schema = list_namespaces_def().input_schema; + assert_eq!(schema["type"].as_str(), Some("object")); + assert_eq!( + schema["properties"].as_object().map(|o| o.len()), + Some(0), + "list_namespaces must not require the caller to know anything first" + ); + } + + /// Both scope switches must be discoverable from the schema, or a + /// caller cannot know that global scope exists -- nor that the two + /// cannot be combined. + #[test] + fn t40_list_tags_declares_both_scope_switches() { + let props = list_tags_def().input_schema["properties"].clone(); + assert_eq!(props["namespace"]["type"].as_str(), Some("string")); + assert_eq!(props["allNamespaces"]["type"].as_str(), Some("boolean")); + assert_eq!(props["allNamespaces"]["default"].as_bool(), Some(false)); + assert!( + props["allNamespaces"]["description"] + .as_str() + .unwrap_or_default() + .contains("Cannot be combined"), + "the mutual exclusion has to be in the schema, not just the error" + ); + } + + // ── partial namespace purge ────────────────────────────────────── + + /// A purge that destroyed 2,000 memories and then failed is not a + /// delete that failed. Rendering it as "Failed to delete namespace" + /// tells the agent nothing happened, so it moves on and the + /// half-purged namespace is never finished — and the memories are + /// gone either way. + #[tokio::test] + async fn w52_a_partial_purge_says_partially_destroyed_not_failed() { + let bridge = crate::test_support::StubBridge { + delete_namespace_error: Some(crate::mcp::bridge::BridgeError::PartiallyApplied( + "Namespace 'scratch' was PARTIALLY destroyed: 2000 memories are \ + permanently gone. Re-run delete_namespace to finish it." + .to_string(), + )), + ..Default::default() + } + .into_bridge(); + + let result = + handle_delete_namespace(&bridge, json!({ "name": "scratch", "force": true })).await; + + assert_eq!(result.is_error, Some(true)); + let text = tool_text(&result); + assert!( + text.contains("PARTIALLY destroyed") && text.contains("Re-run"), + "{text}" + ); + assert!( + !text.contains("Failed to delete namespace"), + "a partial purge must not be reported as a plain failure: {text}" + ); + } + + /// The other direction: an ordinary failure keeps the prefix that + /// says nothing was destroyed. + #[tokio::test] + async fn w53_an_ordinary_delete_failure_keeps_the_failed_prefix() { + let bridge = crate::test_support::StubBridge { + delete_namespace_error: Some(crate::mcp::bridge::BridgeError::Storage( + "disk on fire".to_string(), + )), + ..Default::default() + } + .into_bridge(); + + let result = + handle_delete_namespace(&bridge, json!({ "name": "scratch", "force": true })).await; + + assert_eq!(result.is_error, Some(true)); + assert!( + tool_text(&result).contains("Failed to delete namespace"), + "{:?}", + tool_text(&result) + ); + } + + // ── batch result assembly ──────────────────────────────────────── + + /// `zip` stops at the shorter side. A `delete_memories` that came + /// back one flag short — this decodes a bare `Vec` off the + /// daemon socket, so nothing structural prevents it — left a + /// `Value::Null` in `results`, under-counted `deleted`, and reported + /// the whole thing as a success. + #[tokio::test] + async fn w71_a_short_delete_memories_response_is_an_error_per_id() { + let ids: Vec = (0..3) + .map(|_| crate::model::MemoryId::new().to_string()) + .collect(); + + let bridge = crate::test_support::StubBridge { + // Two flags for three ids. + delete_memories_result: Some(vec![true, true]), + ..Default::default() + } + .into_bridge(); + + let result = handle_forget_memories(&bridge, json!({ "ids": ids })).await; + let payload: serde_json::Value = + serde_json::from_str(&tool_text(&result)).expect("the result is JSON"); + + assert_eq!(payload["total"], json!(3)); + assert_eq!( + payload["deleted"], + json!(0), + "a broken contract must not be reported as a partial success: {payload}" + ); + assert_eq!(payload["errors"], json!(3)); + for entry in payload["results"].as_array().expect("results is an array") { + assert!( + !entry.is_null(), + "a null entry survived into the response: {payload}" + ); + assert!(entry["error"].is_string(), "{entry}"); + } + } + + // ── strict argument checking on the search tools ───────────────── + + /// `docs/mcp.md` claims `recall_memories` validates strictly; only + /// its `namespace` was retrofitted. Everything here used to be a + /// silent default, and the flagship case is the first one: a + /// comma-joined `tags` string became `[]` and the recall ran + /// UNFILTERED over the whole namespace, which is a *wider* answer + /// than was asked for, returned as a success. + #[tokio::test] + async fn w64_recall_memories_rejects_every_malformed_argument() { + let bridge = crate::test_support::StubBridge::default().into_bridge(); + + let cases = [ + ("tags", json!({ "query": "q", "tags": "topic/rust" })), + ("entities", json!({ "query": "q", "entities": "Sarah" })), + ("topics", json!({ "query": "q", "topics": "rust" })), + ("emotions", json!({ "query": "q", "emotions": "happy" })), + ("limit", json!({ "query": "q", "limit": 1000 })), + ("limit", json!({ "query": "q", "limit": "10" })), + ("depth", json!({ "query": "q", "depth": 5 })), + ("compact", json!({ "query": "q", "compact": "false" })), + ("minStrength", json!({ "query": "q", "minStrength": 5 })), + ( + "timeRangeStart", + json!({ "query": "q", "timeRangeStart": ["x"] }), + ), + ("query", json!({})), + ]; + + for (field, arguments) in cases { + // The stub's `search` is `unreachable!()`, so reaching the + // bridge at all is the failure: it means the handler ran the + // query instead of rejecting the arguments. + let result = handle_recall_memories(&bridge, arguments.clone()).await; + assert_eq!( + result.is_error, + Some(true), + "{field} was accepted: {arguments}" + ); + assert!( + tool_text(&result).contains(field), + "the error must name the field: {}", + tool_text(&result) + ); + } + } + + #[tokio::test] + async fn w65_find_similar_memories_rejects_every_malformed_argument() { + let bridge = crate::test_support::StubBridge::default().into_bridge(); + let id = crate::model::MemoryId::new().to_string(); + + let cases = [ + ("mode", json!({ "mode": 2, "id": id })), + ("mode", json!({ "mode": "sngle", "id": id })), + ("id", json!({ "mode": "single" })), + ("id", json!({ "mode": "single", "id": "not-a-uuid" })), + ( + "limit", + json!({ "mode": "single", "id": id, "limit": 1000 }), + ), + ( + "minScore", + json!({ "mode": "single", "id": id, "minScore": 5 }), + ), + ( + "sameNamespace", + json!({ "mode": "single", "id": id, "sameNamespace": "true" }), + ), + ("threshold", json!({ "mode": "scan", "threshold": 5 })), + ("namespace", json!({ "mode": "scan", "namespace": ["a"] })), + ]; + + for (field, arguments) in cases { + let result = handle_find_similar_memories(&bridge, arguments.clone()).await; + assert_eq!( + result.is_error, + Some(true), + "{field} was accepted: {arguments}" + ); + assert!( + tool_text(&result).contains(field), + "the error must name the field: {}", + tool_text(&result) + ); + } + } + + /// The text of a tool result, joined across content blocks. + fn tool_text(result: &ToolCallResult) -> String { + result + .content + .iter() + .map(|c| match c { + crate::mcp::protocol::ContentBlock::Text { text } => text.as_str(), + }) + .collect::>() + .join("\n") + } } diff --git a/src/model/constants.rs b/src/model/constants.rs index 4d4f526..046202b 100644 --- a/src/model/constants.rs +++ b/src/model/constants.rs @@ -132,6 +132,60 @@ pub const MAX_EMOTIONS: usize = 32; /// Maximum number of memories accepted in a single batch store call. pub const MAX_BATCH_MEMORIES: usize = 100; +// ── Namespaces ─────────────────────────────────────────────────────── +/// Name of the namespace every store and recall falls back to. +/// +/// Startup recreates it whenever it is missing, and it is recreated +/// under the *same directory name* with a *new* namespace id — which is +/// why `delete_namespace` refuses it outright. One constant so the +/// coupling between the refusal and the recreation is a single grep. +pub const DEFAULT_NAMESPACE_NAME: &str = "default"; + +// ── Near-Duplicate Detection ───────────────────────────────────────── +/// Cosine similarity at or above which two memories count as +/// near-duplicates of each other. +/// +/// One number, three surfaces: the `find_similar_memories` scan mode, the +/// REST duplicate scan, and the advisory a store returns. They have to +/// agree, or "duplicate" means two different things inside one namespace. +/// +/// Declared `f64`, not `f32`, and that is load-bearing: this value is +/// embedded in published JSON schemas via `json!`, and `serde_json` widens +/// an `f32` to `f64` on the way in — `0.85f32` serializes as +/// `0.8500000238418579`, silently changing the schema clients read. `f64` +/// also matches [`crate::config::GraphConfig::auto_link_threshold`] and the +/// `.as_f64()` used to parse the override. Cast to `f32` at the use sites. +pub const DUPLICATE_SIMILARITY_THRESHOLD: f64 = 0.85; + +/// Maximum near-duplicate matches reported on a single store. +/// +/// Three, not one: one match hides "this is the fourth restatement of the +/// same fact". Three shows a cluster forming and is still cheap. +pub const MAX_NEAR_DUPLICATE_MATCHES: usize = 3; + +/// Byte budget for each reported near-duplicate's summary. +/// +/// Summaries are capped at [`SUMMARY_MAX_BYTES`], so three untruncated ones +/// would be a 6 KB tax on a store response. +pub const NEAR_DUPLICATE_SUMMARY_MAX_BYTES: usize = 240; + +// ── Listing Limits ─────────────────────────────────────────────────── +// +// Named because two places have to agree about each one: the `default` +// and `maximum` a tool publishes in its JSON schema, and the bound the +// argument parser enforces. When those drift the schema stops describing +// the server, and a client that trusts the schema starts getting errors +// it was told could not happen. A schema test asserts the pairing. + +/// Default page size of a `list_memories` call. +pub const DEFAULT_LIST_LIMIT: usize = 50; +/// Largest page size a `list_memories` call may ask for. +pub const MAX_LIST_LIMIT: usize = 200; +/// Default number of labels returned per bucket by `list_tags`. +pub const DEFAULT_TAG_LIMIT: usize = 50; +/// Largest number of labels per bucket a `list_tags` call may ask for. +pub const MAX_TAG_LIMIT: usize = 500; + // ── Access History ─────────────────────────────────────────────────── /// Maximum number of `AccessEvent` entries retained per memory. pub const ACCESS_HISTORY_MAX: usize = 32; diff --git a/src/model/duplicates.rs b/src/model/duplicates.rs new file mode 100644 index 0000000..b9fa361 --- /dev/null +++ b/src/model/duplicates.rs @@ -0,0 +1,123 @@ +//! The near-duplicate advisory a store returns. +//! +//! These are wire types, shared by every door into the system: the MCP +//! `store_memory` result and the HTTP create response both carry the same +//! report, for the same reason [`SupersedesOutcome`](crate::model::SupersedesOutcome) +//! lives here rather than in one transport's module — two copies of a wire +//! type drift, and the drift is only visible to whichever client happens to +//! use the door that fell behind. +//! +//! The report is *advisory*. It never blocks a store and never fails one; +//! it tells the caller that something very like what they just wrote is +//! already there, so they can supersede or reinforce instead of +//! accumulating restatements. + +use serde::{Deserialize, Serialize}; + +use crate::model::MemoryId; + +/// One existing memory that closely resembles a newly stored one. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct NearDuplicateMatch { + /// The existing memory. + pub id: MemoryId, + /// Cosine similarity to the new memory, in `[threshold, 1.0]`. + pub score: f32, + /// The existing memory's summary, truncated to + /// [`NEAR_DUPLICATE_SUMMARY_MAX_BYTES`](crate::model::constants::NEAR_DUPLICATE_SUMMARY_MAX_BYTES). + /// + /// Carried inline on purpose: without it the caller has to round-trip + /// `get_memory` before it can judge whether the new memory restates + /// this one, which defeats an advisory meant to change behaviour in + /// place. Empty when the record could not be loaded — a match with an + /// unknown summary is still worth reporting. + pub summary: String, +} + +/// The set of existing memories that closely resemble a newly stored one. +/// +/// `matches` is never empty: when nothing crosses the threshold the report +/// is *absent*, not empty. That is what [`from_matches`](Self::from_matches) +/// enforces, and it is why the field on each response type is an `Option` +/// with `skip_serializing_if`. +/// +/// There is deliberately no `topId` convenience field. `matches[0]` is the +/// closest match — a duplicate scalar could only ever disagree with it. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct NearDuplicateReport { + /// The similarity at or above which a memory was included. Carried so + /// the object is self-describing and a caller need not know which + /// constant the server was built with. + /// + /// `f64` for the same reason + /// [`DUPLICATE_SIMILARITY_THRESHOLD`](crate::model::constants::DUPLICATE_SIMILARITY_THRESHOLD) + /// is: this is that constant echoed onto the wire, and an `f32` would + /// publish it as `0.8500000238418579`. `score` below stays `f32` + /// because it is a genuine `f32` computation, not a round number + /// travelling through a narrower type than it was written in. + pub threshold: f64, + /// Closest matches first. + pub matches: Vec, +} + +impl NearDuplicateReport { + /// Build a report, or `None` when there is nothing to report. + pub fn from_matches(threshold: f64, matches: Vec) -> Option { + if matches.is_empty() { + None + } else { + Some(Self { threshold, matches }) + } + } +} + +/// The marker appended to a summary that had to be cut. +const ELLIPSIS: &str = "…"; + +/// Truncate `summary` to at most `max_bytes` UTF-8 bytes, appending an +/// ellipsis when anything was cut. +/// +/// The ellipsis is charged against the budget, so the result is never +/// longer than `max_bytes` (for any `max_bytes >= ELLIPSIS.len()`, which +/// every caller satisfies by a wide margin). The cut lands on a `char` +/// boundary, so the result is always valid UTF-8 — slicing a summary at a +/// fixed byte offset would panic the moment one contained an em dash. +pub fn truncate_summary(summary: &str, max_bytes: usize) -> String { + if summary.len() <= max_bytes { + return summary.to_string(); + } + let mut end = max_bytes.saturating_sub(ELLIPSIS.len()); + while end > 0 && !summary.is_char_boundary(end) { + end -= 1; + } + format!("{}{ELLIPSIS}", &summary[..end]) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn an_empty_match_list_produces_no_report() { + assert!(NearDuplicateReport::from_matches(0.85, Vec::new()).is_none()); + } + + #[test] + fn truncate_summary_leaves_a_short_summary_alone() { + assert_eq!(truncate_summary("short", 240), "short"); + } + + /// A fixed byte-offset slice would panic here; the boundary walk is + /// the whole point of the helper. + #[test] + fn truncate_summary_cuts_on_a_char_boundary_and_stays_within_budget() { + // Six 3-byte chars = 18 bytes. + let s = "日本語日本語"; + let cut = truncate_summary(s, 10); + assert!(cut.len() <= 10, "{} bytes", cut.len()); + assert!(cut.ends_with(ELLIPSIS), "{cut}"); + assert_eq!(cut, "日本…"); + } +} diff --git a/src/model/mod.rs b/src/model/mod.rs index 507a874..8beda69 100644 --- a/src/model/mod.rs +++ b/src/model/mod.rs @@ -18,6 +18,7 @@ pub mod constants; pub mod decay; +pub mod duplicates; pub mod edge; pub mod error; pub mod id; @@ -29,6 +30,7 @@ pub mod validation; // Re-export primary types at module level for convenience. pub use self::decay::DecayPhase; +pub use self::duplicates::{NearDuplicateMatch, NearDuplicateReport, truncate_summary}; pub use self::edge::{EdgeType, SupersedesOutcome, SupersedesStatus}; pub use self::error::{DecodeError, TagError, ValidationError}; pub use self::id::{MemoryId, NamespaceId}; @@ -36,6 +38,7 @@ pub use self::memory::{AccessEvent, AccessKind, Memory}; pub use self::namespace::{NamespaceConfig, PhaseThresholds}; pub use self::record::{CachedRecord, DiskRecord}; pub use self::tag::{ - StructuredMetadata, Tag, entity_overlap, parse_structured_tags, parse_tags_lossy, + StructuredMetadata, Tag, TagFilterError, entity_overlap, parse_filter_tags, + parse_structured_tags, parse_tags_lossy, sort_tag_counts, }; pub use self::validation::{MemoryInputRef, validate_memory_input}; diff --git a/src/model/tag.rs b/src/model/tag.rs index 602f02c..b4103ee 100644 --- a/src/model/tag.rs +++ b/src/model/tag.rs @@ -125,6 +125,53 @@ pub fn parse_tags_lossy(context: &str, raw: &[String]) -> Vec { .collect() } +/// A query filter that names a label no tag can carry. +#[derive(Debug, thiserror::Error)] +#[error( + "Filter '{field}' cannot be applied: {value:?} is not a valid tag ({source}). \ + Tags allow only a-z, 0-9 and - _ / . : — entity, topic and emotion filters are \ + matched as `entity/`, `topic/` and `emotion/` tags, so no \ + memory can carry this label." +)] +pub struct TagFilterError { + /// Which request field the bad label came from. + pub field: String, + /// The label, as sent. + pub value: String, + /// Why it is not a tag. + pub source: TagError, +} + +/// Parse `raw` into [`Tag`]s for a query FILTER, failing on the first +/// element that will not validate. `prefix` is prepended to each element +/// (`"entity/"`, `"topic/"`, `"emotion/"`, or `""` for plain tags). +/// +/// The mirror image of [`parse_tags_lossy`], and deliberately the +/// opposite trade-off. Dropping a tag from a store keeps the memory and +/// loses one label. Dropping a tag from a filter **widens the query**: +/// `list_memories {"entities": ["José"]}` derived `entity/josé`, failed +/// the alphabet, swallowed the error and returned EVERY memory in the +/// namespace as though it had matched a filter. A caller cannot tell a +/// widened answer from a genuine one, and an agent acting on "these are +/// the memories about José" gets the whole namespace instead. +/// +/// An error is recoverable. A wrong answer that looks right is not. +pub fn parse_filter_tags( + field: &str, + prefix: &str, + raw: &[String], +) -> Result, TagFilterError> { + raw.iter() + .map(|label| { + Tag::new(format!("{prefix}{}", label.to_lowercase())).map_err(|source| TagFilterError { + field: field.to_string(), + value: label.clone(), + source, + }) + }) + .collect() +} + /// Structured metadata extracted from hierarchical tags. #[derive(Debug, Clone, Default)] pub struct StructuredMetadata { @@ -171,6 +218,17 @@ pub fn parse_structured_tags(tags: &[Tag]) -> StructuredMetadata { result } +/// Order a tag histogram most-common-first, breaking ties on the tag +/// name so a report is stable between runs. +/// +/// `MetadataStore::list_tags` reads a redb B-tree keyed by tag, so it +/// returns **lexicographic** order. Every caller that wants "top tags" +/// has to apply this: `health::report` did not, and published the first +/// ten tags alphabetically under the heading "top tags". +pub fn sort_tag_counts(counts: &mut [(String, u64)]) { + counts.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0))); +} + /// Compute the Jaccard similarity between two sets of entities. /// Returns 0.0 if either set is empty. Comparison is case-insensitive. pub fn entity_overlap(query_entities: &[String], memory_entities: &[String]) -> f32 { @@ -196,3 +254,40 @@ pub fn entity_overlap(query_entities: &[String], memory_entities: &[String]) -> intersection_count as f32 / union_size as f32 } + +// ═══════════════════════════════════════════════════════════════════════ +// Tests +// ═══════════════════════════════════════════════════════════════════════ + +#[cfg(test)] +mod tests { + use super::*; + + fn counts(pairs: &[(&str, u64)]) -> Vec<(String, u64)> { + pairs.iter().map(|(n, c)| ((*n).to_string(), *c)).collect() + } + + /// The health-report bug in miniature: `list_tags` arrives in + /// lexicographic order, so without this the "top tags" heading names + /// the alphabetically first tags instead of the most common ones. + #[test] + fn t27_sort_orders_most_common_first() { + let mut c = counts(&[("aardvark", 1), ("zebra", 99), ("moose", 50)]); + sort_tag_counts(&mut c); + assert_eq!( + c.iter().map(|(n, _)| n.as_str()).collect::>(), + ["zebra", "moose", "aardvark"] + ); + } + + #[test] + fn t28_ties_break_on_name_ascending() { + let mut c = counts(&[("zebra", 7), ("aardvark", 7), ("moose", 7)]); + sort_tag_counts(&mut c); + assert_eq!( + c.iter().map(|(n, _)| n.as_str()).collect::>(), + ["aardvark", "moose", "zebra"], + "a stable order is what makes two runs of the same report comparable" + ); + } +} diff --git a/src/search/duplicates.rs b/src/search/duplicates.rs new file mode 100644 index 0000000..599daf8 --- /dev/null +++ b/src/search/duplicates.rs @@ -0,0 +1,255 @@ +//! Near-duplicate detection against the vector index. +//! +//! Detection only: this module answers "what is already in the index that +//! looks like this?" and nothing else. Turning ids into a caller-facing +//! report — loading summaries, truncating them — belongs to the transport +//! adapters, which is what keeps this function synchronous and therefore +//! unable to `await` while the index read lock is held. + +use std::sync::atomic::{AtomicBool, Ordering}; + +use crate::model::{DecayPhase, MemoryId, NamespaceId}; +use crate::search::{FlatVectorIndex, SearchFilter, VectorIndex, is_normalized}; + +/// Tolerance on `| ||v||^2 - 1 |` for calling a query vector normalized. +const NORM_EPSILON: f32 = 1e-3; + +/// Set the first time a non-normalized query vector is seen, so the +/// `warn!` fires once per process instead of once per store. +static UNNORMALIZED_WARNED: AtomicBool = AtomicBool::new(false); + +/// Find existing memories at least `threshold` similar to `embedding`, +/// closest first, at most `limit` of them. +/// +/// # Candidates, not a report +/// +/// The phase filter below is a cheap PREFILTER and nothing more. +/// [`FlatVectorIndex`] writes each entry's `decay_phase` once, at +/// [`add`](crate::search::VectorIndex::add) time — always +/// [`DecayPhase::Full`] — and +/// [`update_metadata`](crate::search::VectorIndex::update_metadata) has +/// no callers, so in production the entry's phase is a snapshot of the +/// moment it was indexed rather than the memory's phase now. Every +/// caller must therefore re-check the phase on the record it hydrates, +/// and drop candidates whose record is missing or past +/// [`DecayPhase::Summary`]. That is where the Full/Summary guarantee is +/// actually kept. +/// +/// Deliberately **not** [`QueryEngine::similar`](crate::search::QueryEngine): +/// that requires the memory to already exist, and it records an associative +/// access on every result it returns. At store time that would age every +/// neighbour's FSRS schedule on every single write — a decay-model change +/// smuggled in behind an advisory string. +/// +/// Never errors. A dimension mismatch, an empty index, or a query vector +/// that is not L2-normalized all yield an empty vector, because the only +/// consumer is an advisory field that must never fail a store. +/// +/// # Normalization +/// +/// The index stores dot products, not cosines — [`VectorIndex::add`] +/// requires normalized input and [`dot_product_simd`](crate::search::dot_product_simd) +/// divides by no norm. For a caller-supplied vector (a passthrough +/// namespace never validates one) an un-normalized query would produce +/// scores above 1.0 that look like similarities and are not. The check is +/// *skipped* rather than fixed up: normalizing here would either lie about +/// the stored vector or report a number the rest of the system disagrees +/// with, and an advisory's only value is that its number is trustworthy. +pub fn find_near_duplicates( + index: &FlatVectorIndex, + embedding: &[f32], + namespace_id: NamespaceId, + threshold: f32, + limit: usize, +) -> Vec<(MemoryId, f32)> { + if limit == 0 || index.len() == 0 { + return Vec::new(); + } + + if !is_normalized(embedding, NORM_EPSILON) { + tracing::debug!( + %namespace_id, + "skipping near-duplicate check: query vector is not L2-normalized" + ); + if !UNNORMALIZED_WARNED.swap(true, Ordering::Relaxed) { + tracing::warn!( + "near-duplicate detection is inert for un-normalized embeddings; \ + scores would not be cosine similarities. This is logged once per process." + ); + } + return Vec::new(); + } + + let filter = SearchFilter { + namespace_id: Some(namespace_id), + // The same phases autolink scans, so the two similarity views of a + // namespace stay comparable. Ghost has no summary left to show and + // Tombstone is deleted; neither is something to be told it + // duplicates. + // + // A PREFILTER only — see the note on this function. It catches + // nothing in production, because nothing ever updates an entry's + // stored phase; the hydration step is what enforces this. + decay_phases: Some(vec![DecayPhase::Full.as_u8(), DecayPhase::Summary.as_u8()]), + // Prunes the result heap, not the dot products — the scan is + // O(index) either way. + min_score: Some(threshold), + ..SearchFilter::default() + }; + + match index.search(embedding, limit, &filter) { + // Already descending by score. + Ok(results) => results.into_iter().map(|r| (r.id, r.score)).collect(), + Err(e) => { + tracing::warn!(%e, "near-duplicate vector search failed (non-fatal)"); + Vec::new() + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::model::constants::DUPLICATE_SIMILARITY_THRESHOLD; + use crate::search::VectorMetadata; + + const NS: u32 = 1; + const OTHER_NS: u32 = 2; + + fn threshold() -> f32 { + DUPLICATE_SIMILARITY_THRESHOLD as f32 + } + + fn ns(id: u32) -> NamespaceId { + NamespaceId::new(id) + } + + /// Add a vector to the index and hand back its id. + fn add( + index: &mut FlatVectorIndex, + vector: &[f32], + namespace: u32, + phase: DecayPhase, + ) -> MemoryId { + let id = MemoryId::new(); + index + .add( + id, + vector, + VectorMetadata { + namespace_id: ns(namespace), + decay_phase: phase.as_u8(), + tags: Vec::new(), + }, + ) + .expect("add vector"); + id + } + + /// An index holding one full-phase unit vector along the first axis. + fn one_vector_index() -> (FlatVectorIndex, MemoryId) { + let mut index = FlatVectorIndex::new(4); + let id = add(&mut index, &[1.0, 0.0, 0.0, 0.0], NS, DecayPhase::Full); + (index, id) + } + + #[test] + fn a1_an_empty_index_reports_nothing() { + let index = FlatVectorIndex::new(4); + let found = find_near_duplicates(&index, &[1.0, 0.0, 0.0, 0.0], ns(NS), threshold(), 3); + assert!(found.is_empty()); + } + + #[test] + fn a2_an_identical_vector_scores_one() { + let (index, id) = one_vector_index(); + let found = find_near_duplicates(&index, &[1.0, 0.0, 0.0, 0.0], ns(NS), threshold(), 3); + assert_eq!(found.len(), 1); + assert_eq!(found[0].0, id); + assert!((found[0].1 - 1.0).abs() < 1e-5, "{}", found[0].1); + } + + #[test] + fn a3_a_vector_below_the_threshold_is_ignored() { + let (index, _) = one_vector_index(); + // Orthogonal: cosine 0.0, far below 0.85. + let found = find_near_duplicates(&index, &[0.0, 1.0, 0.0, 0.0], ns(NS), threshold(), 3); + assert!(found.is_empty()); + } + + #[test] + fn a4_the_scan_is_scoped_to_the_query_namespace() { + let mut index = FlatVectorIndex::new(4); + add( + &mut index, + &[1.0, 0.0, 0.0, 0.0], + OTHER_NS, + DecayPhase::Full, + ); + let found = find_near_duplicates(&index, &[1.0, 0.0, 0.0, 0.0], ns(NS), threshold(), 3); + assert!(found.is_empty()); + } + + #[test] + fn a5_ghost_and_tombstone_memories_are_excluded() { + let mut index = FlatVectorIndex::new(4); + add(&mut index, &[1.0, 0.0, 0.0, 0.0], NS, DecayPhase::Ghost); + add(&mut index, &[1.0, 0.0, 0.0, 0.0], NS, DecayPhase::Tombstone); + let summary = add(&mut index, &[1.0, 0.0, 0.0, 0.0], NS, DecayPhase::Summary); + + let found = find_near_duplicates(&index, &[1.0, 0.0, 0.0, 0.0], ns(NS), threshold(), 5); + assert_eq!(found.len(), 1, "{found:?}"); + assert_eq!(found[0].0, summary); + } + + #[test] + fn a6_the_result_is_capped_at_the_limit() { + let mut index = FlatVectorIndex::new(4); + for _ in 0..5 { + add(&mut index, &[1.0, 0.0, 0.0, 0.0], NS, DecayPhase::Full); + } + let found = find_near_duplicates(&index, &[1.0, 0.0, 0.0, 0.0], ns(NS), threshold(), 3); + assert_eq!(found.len(), 3); + } + + #[test] + fn a7_matches_come_back_in_descending_score_order() { + let mut index = FlatVectorIndex::new(4); + // Three unit vectors at increasing angles from the query. + let far = add(&mut index, &[0.88, 0.475, 0.0, 0.0], NS, DecayPhase::Full); + let exact = add(&mut index, &[1.0, 0.0, 0.0, 0.0], NS, DecayPhase::Full); + let near = add(&mut index, &[0.96, 0.28, 0.0, 0.0], NS, DecayPhase::Full); + + let found = find_near_duplicates(&index, &[1.0, 0.0, 0.0, 0.0], ns(NS), threshold(), 5); + let ids: Vec = found.iter().map(|(id, _)| *id).collect(); + assert_eq!(ids, vec![exact, near, far], "{found:?}"); + assert!(found.windows(2).all(|w| w[0].1 >= w[1].1), "{found:?}"); + } + + /// The executable form of "guard the query, skip the check". `[2,0,0,0]` + /// against a stored `[1,0,0,0]` has a dot product of 2.0 — a number that + /// looks like a similarity, is above every threshold, and is not one. + /// Reporting nothing is the correct answer; reporting 2.0 is not. + #[test] + fn a8_skips_the_check_when_the_query_vector_is_not_l2_normalized() { + let (index, _) = one_vector_index(); + let found = find_near_duplicates(&index, &[2.0, 0.0, 0.0, 0.0], ns(NS), threshold(), 3); + assert!(found.is_empty(), "{found:?}"); + } + + #[test] + fn a9_a_dimension_mismatch_yields_no_matches_rather_than_an_error() { + let (index, _) = one_vector_index(); + // Normalized, but eight-dimensional against a four-dimensional index. + let query = [0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5]; + let found = find_near_duplicates(&index, &query, ns(NS), threshold(), 3); + assert!(found.is_empty()); + } + + #[test] + fn a10_a_zero_limit_reports_nothing() { + let (index, _) = one_vector_index(); + let found = find_near_duplicates(&index, &[1.0, 0.0, 0.0, 0.0], ns(NS), threshold(), 0); + assert!(found.is_empty()); + } +} diff --git a/src/search/error.rs b/src/search/error.rs index 1d019b6..2b134aa 100644 --- a/src/search/error.rs +++ b/src/search/error.rs @@ -38,6 +38,11 @@ pub enum SearchError { #[error("empty query: at least one of text, tags, or memory_id must be provided")] EmptyQuery, + /// A filter names a label that no memory can carry, so applying it + /// is impossible and DROPPING it would widen the query. + #[error("{0}")] + InvalidFilter(String), + /// An unexpected internal error occurred. #[error("internal error: {0}")] Internal(String), diff --git a/src/search/fts.rs b/src/search/fts.rs index f421aad..1a645c6 100644 --- a/src/search/fts.rs +++ b/src/search/fts.rs @@ -213,6 +213,78 @@ impl FtsIndex { Ok(out) } + /// Remove every document belonging to a namespace. Returns how many + /// were removed. + /// + /// # Not an optimization — the only way to reach some of these rows + /// + /// Calling [`remove`](Self::remove) per id would only be correct for + /// ids that meta.db still knows about. The decay sweep hard-deletes + /// a Ghost record without touching this index, so a namespace can + /// hold FTS rows whose meta.db record is already gone — ids no + /// caller can enumerate. Filtering on `namespace_id` is the only + /// handle on them. + /// + /// # `fts_content` rows have to go first + /// + /// `fts_content` is declared `contentless_delete=1`, so its rows are + /// reachable ONLY by rowid, and the rowid lives in `id_map`. A bare + /// `DELETE FROM id_map` therefore orphans every content row + /// permanently: nothing can ever name them again. + /// + /// What that costs is not a later collision — `id_map.rowid` is + /// `AUTOINCREMENT` and [`add`](Self::add) removes the old `id_map` + /// row first, so re-indexing the same memory takes a fresh rowid and + /// the `memory_id` UNIQUE constraint is never in play. It costs: + /// + /// - the space, for the life of the database, with no way to reclaim + /// it short of rebuilding the index; + /// - **corpus-wide ranking drift.** `bm25()` scores against the + /// whole `fts_content` table, so orphaned rows go on contributing + /// to the document count and to per-term document frequencies. The + /// rows can never be *returned* — [`search`](Self::search) inner + /// joins `id_map` — but every query in every namespace is scored + /// against an IDF that counts them. + /// + /// So: collect the rowids, delete the content rows, and only then + /// delete the id_map rows, all in one transaction. + /// + /// `id_map.namespace_id` is not indexed, so this is a table scan. + /// Acceptable for a rare administrative operation. + pub fn remove_namespace(&self, namespace_id: NamespaceId) -> Result { + let tx = self.conn.unchecked_transaction()?; + + let rowids: Vec = { + let mut stmt = self + .conn + .prepare("SELECT rowid FROM id_map WHERE namespace_id = ?1")?; + let rows = stmt.query_map(rusqlite::params![namespace_id.get()], |row| row.get(0))?; + rows.collect::, _>>()? + }; + + if rowids.is_empty() { + tx.commit()?; + return Ok(0); + } + + { + let mut stmt = self + .conn + .prepare("DELETE FROM fts_content WHERE rowid = ?1")?; + for rowid in &rowids { + stmt.execute(rusqlite::params![rowid])?; + } + } + + self.conn.execute( + "DELETE FROM id_map WHERE namespace_id = ?1", + rusqlite::params![namespace_id.get()], + )?; + + tx.commit()?; + Ok(rowids.len()) + } + /// Whether the index contains no documents. pub fn is_empty(&self) -> Result { let count: i64 = self @@ -403,6 +475,110 @@ mod tests { let _ = index.search(test_ns_id(), " ", 10).unwrap(); } + /// `fts_content` is contentless_delete, so its rows are reachable + /// only by rowid — which lives in `id_map`. Deleting the id_map rows + /// first orphans every content row permanently. + /// + /// This asserts on `fts_content` DIRECTLY, because nothing else can: + /// an orphaned content row is invisible to `search` (which inner + /// joins `id_map`), does not collide with anything on re-index + /// (`id_map.rowid` is AUTOINCREMENT), and does not show up in + /// `is_empty` (which counts `id_map`). Every observable this test + /// used to check passed with the guard removed. The row count is the + /// one that does not. + #[test] + fn w20_remove_namespace_deletes_id_map_and_fts_content_together() { + let dir = tempfile::TempDir::new().unwrap(); + let index = FtsIndex::new(dir.path()).unwrap(); + + let id = MemoryId::new(); + index + .add(test_ns_id(), id, "obsolete content here", None, &[]) + .unwrap(); + assert_eq!(fts_content_rows(&index), 1, "the setup did not index"); + + assert_eq!(index.remove_namespace(test_ns_id()).unwrap(), 1); + assert!(index.is_empty().unwrap()); + assert_eq!( + fts_content_rows(&index), + 0, + "an orphaned fts_content row survived: unreachable, unreclaimable, and \ + still counted by bm25's corpus statistics" + ); + + // The rest of the original guard: the memory re-indexes cleanly + // and the stale text does not come back. + index + .add(test_ns_id(), id, "replacement content", None, &[]) + .unwrap(); + let results = index.search(test_ns_id(), "replacement", 10).unwrap(); + assert_eq!(results.len(), 1); + assert_eq!(results[0].0, id); + assert_eq!(fts_content_rows(&index), 1); + } + + /// The same assertion for the per-record path, which has the same + /// ordering requirement and the same invisible failure mode. + #[test] + fn w61_remove_deletes_the_content_row_too() { + let dir = tempfile::TempDir::new().unwrap(); + let index = FtsIndex::new(dir.path()).unwrap(); + + let id = MemoryId::new(); + index + .add(test_ns_id(), id, "obsolete content here", None, &[]) + .unwrap(); + + assert!(index.remove(id).unwrap()); + assert_eq!( + fts_content_rows(&index), + 0, + "an orphaned fts_content row survived a per-record removal" + ); + } + + /// Rows actually present in the FTS5 virtual table, orphans included. + fn fts_content_rows(index: &FtsIndex) -> i64 { + index + .conn + .query_row("SELECT count(*) FROM fts_content", [], |row| row.get(0)) + .expect("count fts_content rows") + } + + #[test] + fn w21_remove_namespace_leaves_other_namespaces_searchable() { + let dir = tempfile::TempDir::new().unwrap(); + let index = FtsIndex::new(dir.path()).unwrap(); + + let doomed = MemoryId::new(); + let kept = MemoryId::new(); + index + .add(test_ns_id(), doomed, "shared keyword alpha", None, &[]) + .unwrap(); + index + .add(test_ns_id_2(), kept, "shared keyword alpha", None, &[]) + .unwrap(); + + index.remove_namespace(test_ns_id()).unwrap(); + + assert!(index.search(test_ns_id(), "alpha", 10).unwrap().is_empty()); + let survivors = index.search(test_ns_id_2(), "alpha", 10).unwrap(); + assert_eq!(survivors.len(), 1); + assert_eq!(survivors[0].0, kept); + } + + #[test] + fn w22_remove_namespace_of_an_empty_namespace_is_zero() { + let dir = tempfile::TempDir::new().unwrap(); + let index = FtsIndex::new(dir.path()).unwrap(); + index + .add(test_ns_id_2(), MemoryId::new(), "kept", None, &[]) + .unwrap(); + + assert_eq!(index.remove_namespace(test_ns_id()).unwrap(), 0); + assert!(!index.is_empty().unwrap()); + } + #[test] fn escape_query_sanitizes() { let result = escape_query("Sarah trip Japan"); diff --git a/src/search/mod.rs b/src/search/mod.rs index 98dea58..334a7ca 100644 --- a/src/search/mod.rs +++ b/src/search/mod.rs @@ -12,6 +12,9 @@ mod index; mod simd; +// Near-duplicate detection at store time. +mod duplicates; + // CS-26: FTS5 full-text search (replaces custom BM25) mod fts; @@ -30,6 +33,9 @@ mod response; pub use index::{FilterEntry, FlatVectorIndex, TagInterner}; pub use simd::{dot_product_simd, is_normalized, normalize_l2}; +// --- Near-duplicate detection --- +pub use duplicates::find_near_duplicates; + // --- CS-26 re-exports --- pub use fts::FtsIndex; diff --git a/src/serialization/json.rs b/src/serialization/json.rs index 1685f79..2c72bdd 100644 --- a/src/serialization/json.rs +++ b/src/serialization/json.rs @@ -137,6 +137,14 @@ pub struct MemoryResponse { /// named a target — so GET, search and list bodies are unchanged. #[serde(default, skip_serializing_if = "Option::is_none")] pub supersedes: Option, + + /// Existing memories that closely resemble this one. + /// + /// Like `supersedes`, only the create endpoint populates this, and + /// only when something crossed the threshold — so GET, search and list + /// bodies are unchanged. Advisory: the memory was created regardless. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub near_duplicates: Option, } impl MemoryResponse { @@ -166,6 +174,7 @@ impl MemoryResponse { embedding: None, access_history: None, supersedes: None, + near_duplicates: None, } } } @@ -334,3 +343,73 @@ pub struct NamespaceResponse { /// Total number of memories in this namespace. pub memory_count: u64, } + +// ═══════════════════════════════════════════════════════════════════════ +// Tests +// ═══════════════════════════════════════════════════════════════════════ + +#[cfg(test)] +mod tests { + use super::*; + use crate::model::{NearDuplicateMatch, NearDuplicateReport}; + + /// A `MemoryResponse` as every read endpoint produces it. + fn response() -> MemoryResponse { + let record = CachedRecord { + id: MemoryId::new(), + namespace_id: crate::model::NamespaceId::new(1), + created_at: 0, + last_accessed_at: 0, + phase: DecayPhase::Full, + strength: 1.0, + decay_strength: 1.0, + stability: 1.0, + difficulty: 5.0, + is_permastore: false, + edge_count: 0, + summary: "a fact".to_string(), + tags: Vec::new(), + vector_slot: 0, + entities: Vec::new(), + }; + MemoryResponse::from_cached(&record, "default".to_string()) + } + + /// Only the create endpoint populates the advisory, so GET, search and + /// list bodies have to be byte-identical to what they were. + #[test] + fn g30_a_read_response_omits_the_near_duplicate_report() { + let encoded = serde_json::to_value(response()).expect("encodes"); + assert!(encoded.get("nearDuplicates").is_none(), "{encoded}"); + } + + /// Same wire shape as the MCP door's report — the two create paths + /// must not drift. + #[test] + fn g30b_a_populated_report_round_trips_with_camel_case_keys() { + let target = MemoryId::new(); + let mut memory = response(); + memory.near_duplicates = NearDuplicateReport::from_matches( + 0.85, + vec![NearDuplicateMatch { + id: target, + score: 0.9, + summary: "an older phrasing".to_string(), + }], + ); + + let encoded = serde_json::to_value(&memory).expect("encodes"); + let report = &encoded["nearDuplicates"]; + assert_eq!(report["threshold"], serde_json::json!(0.85)); + assert_eq!( + report["matches"][0]["id"], + serde_json::json!(target.to_string()) + ); + assert_eq!(report["matches"][0]["summary"], "an older phrasing"); + + let decoded: MemoryResponse = serde_json::from_value(encoded).expect("decodes"); + let report = decoded.near_duplicates.expect("report preserved"); + assert_eq!(report.matches.len(), 1); + assert_eq!(report.matches[0].id, target); + } +} diff --git a/src/storage/engine.rs b/src/storage/engine.rs index ac4ba3c..c6a1ece 100644 --- a/src/storage/engine.rs +++ b/src/storage/engine.rs @@ -7,9 +7,11 @@ //! //! See Spec 04 Section 7 for the design rationale. -use std::collections::HashSet; -use std::path::{Path, PathBuf}; +use std::collections::{HashMap, HashSet}; +use std::ffi::OsStr; +use std::path::{Component, Path, PathBuf}; +use crate::model::validation::validate_namespace_name; use crate::model::{ AccessKind, DecayPhase, DiskRecord, EdgeType, MemoryId, NamespaceConfig, NamespaceId, }; @@ -342,6 +344,8 @@ impl RedbStorageEngine { /// 1. Rebuild phase index from meta.db. /// 2. Validate fulltext.dat header. /// 3. Clean up orphaned edges. + /// 4. Sweep namespace directories left behind by an interrupted + /// namespace deletion into quarantine. fn startup_validation(&mut self) -> Result<(), StorageError> { tracing::info!("Running storage startup validation..."); @@ -362,10 +366,235 @@ impl RedbStorageEngine { ); } + // 4. Sweep orphaned namespace directories. Deliberately not + // allowed to fail startup: this is disk hygiene, and a + // database that refuses to open because a stray directory + // could not be inspected is a far worse outcome than the + // wasted space. + match self.sweep_orphan_namespace_dirs() { + Ok(removed) if removed > 0 => { + tracing::warn!( + quarantined = removed, + path = %self.db_path.join(QUARANTINE_DIR_NAME).display(), + "Moved orphaned namespace directories left by an interrupted \ + namespace deletion into quarantine. Nothing was deleted; remove \ + the quarantine directory by hand once you are satisfied." + ); + } + Ok(_) => {} + Err(e) => { + tracing::warn!( + error = %e, + "Orphaned-namespace-directory sweep failed; continuing startup" + ); + } + } + tracing::info!("Storage startup validation complete"); Ok(()) } + /// Quarantine namespace directories that belong to no live + /// namespace. + /// + /// Cleans up after a `purge_namespace` that was interrupted between + /// deleting the `NAMESPACE_TABLE` row and removing the directory. + /// Returns the number of directories moved. + /// + /// Directories are MOVED to `/.orphaned/`, not deleted — + /// see [`quarantine_orphan_dir`](Self::quarantine_orphan_dir) for + /// why the delete had no lower bound. + /// + /// # Why this cannot delete a live namespace + /// + /// [`open`](Self::open) step 6 calls `VectorManager::open_or_create` + /// for **every** namespace in `NAMESPACE_TABLE`, and + /// `open_or_create` creates the directory. That runs before step 7, + /// which is this. So at sweep time every live namespace provably + /// has a directory on disk and a name in `live`; there is no window + /// in which a live-but-not-yet-opened namespace could be swept. + /// + /// # Guards, and why each one is here + /// + /// `db_path` is user-configurable and may hold directories that have + /// nothing to do with recalld, so "the name matches no live + /// namespace" is not on its own a licence to recurse and delete. + /// Every candidate must additionally: + /// + /// - be a real directory. Types come from `read_dir`, which does + /// **not** follow symlinks, so a symlink pointing outside + /// `db_path` is reported as a symlink and skipped rather than + /// followed into someone else's tree. + /// - have a UTF-8 name (a name we cannot compare against `live` is + /// a name we cannot prove is dead). The comparison itself is ASCII + /// case-folded: on a case-insensitive filesystem the live + /// namespace `Notes` and the directory `notes/` are the same + /// directory, and an exact-string compare would call it dead. + /// - not start with `.` — dotfile directories are conventionally + /// somebody else's state. + /// - **contain `vectors.dat` and nothing else.** This is the load- + /// bearing guard: it is the shape this code writes, so anything + /// else in there is evidence the directory was not created by us. + /// + /// Anything that fails a guard is logged and left strictly alone. + fn sweep_orphan_namespace_dirs(&self) -> Result { + // Case-folded, because the identity that matters here is the + // one the FILESYSTEM uses and macOS APFS is case-insensitive by + // default. An exact-string compare let a live namespace named + // `Notes` fail to claim the directory `notes/` that it is + // actually reading and writing — and the shape guard passes, + // because it is genuinely a namespace directory. The sweep then + // deleted a live namespace's vectors, unattended, at boot. + // + // Folding is the conservative direction: it can only ever make + // this refuse to delete something, never delete more. + let live: HashSet = self + .meta_store + .list_namespaces()? + .into_iter() + .map(|ns| ns.name.to_ascii_lowercase()) + .collect(); + + let mut removed = 0usize; + + for entry in std::fs::read_dir(&self.db_path)? { + let entry = match entry { + Ok(e) => e, + Err(e) => { + tracing::warn!(error = %e, "Could not read a data-directory entry; skipping"); + continue; + } + }; + + // read_dir file types do not follow symlinks. + match entry.file_type() { + Ok(ft) if ft.is_dir() => {} + Ok(_) => continue, + Err(e) => { + tracing::warn!( + path = %entry.path().display(), + error = %e, + "Could not stat a data-directory entry; skipping" + ); + continue; + } + } + + let file_name = entry.file_name(); + let Some(name) = file_name.to_str() else { + tracing::warn!( + path = %entry.path().display(), + "Data-directory entry has a non-UTF-8 name; leaving it alone" + ); + continue; + }; + + if name.starts_with('.') || live.contains(&name.to_ascii_lowercase()) { + continue; + } + + let dir = entry.path(); + if !Self::looks_like_a_namespace_dir(&dir) { + tracing::warn!( + path = %dir.display(), + "Directory in the data directory matches no namespace but does not \ + have the shape of one; leaving it alone" + ); + continue; + } + + match self.quarantine_orphan_dir(name, &dir) { + Ok(moved_to) => { + tracing::info!( + namespace = name, + from = %dir.display(), + to = %moved_to.display(), + "Quarantined orphaned namespace directory" + ); + removed += 1; + } + Err(e) => { + tracing::warn!( + path = %dir.display(), + error = %e, + "Failed to quarantine orphaned namespace directory" + ); + } + } + } + + Ok(removed) + } + + /// Move an orphaned namespace directory into + /// `/.orphaned/-/` instead of deleting it. + /// Returns where it went. + /// + /// # Why a rename and not `remove_dir_all` + /// + /// The sweep's whole argument for being safe is "every live + /// namespace has a row in `NAMESPACE_TABLE`, so a directory with no + /// row is dead". That argument has no lower bound: if `meta.db` is + /// absent or restored from an older backup, `live` is EMPTY and + /// every namespace directory in the data directory is a candidate. + /// The guards all pass — they are real namespace directories — and + /// a recursive delete of all of them is unrecoverable. + /// + /// A rename keeps the entire hygiene benefit (the directories stop + /// occupying the data directory and stop being reopened) with none + /// of the blast radius: whatever the sweep got wrong is still on + /// disk, named, timestamped, and can be moved back. + /// + /// Refusing to sweep when `live` is empty was the alternative. It is + /// strictly weaker — it does nothing for the case where `live` is + /// merely STALE rather than empty, which is the same restored-backup + /// scenario one namespace later — and it would break the legitimate + /// case of a fresh database with a leftover directory. + fn quarantine_orphan_dir(&self, name: &str, dir: &Path) -> Result { + let quarantine = self.db_path.join(QUARANTINE_DIR_NAME); + std::fs::create_dir_all(&quarantine)?; + + // Same filesystem by construction — both are under db_path — so + // this is a rename, not a copy. + let stamp = chrono::Utc::now().format("%Y%m%dT%H%M%SZ"); + let mut dest = quarantine.join(format!("{name}-{stamp}")); + // Two sweeps in the same second must not merge into one + // directory, and `rename` onto an existing directory is not an + // error on every platform. + let mut attempt = 1u32; + while dest.exists() { + dest = quarantine.join(format!("{name}-{stamp}-{attempt}")); + attempt += 1; + } + + std::fs::rename(dir, &dest)?; + Ok(dest) + } + + /// Whether `dir` has exactly the shape this code creates for a + /// namespace: one entry, named `vectors.dat`. + /// + /// Conservative on every uncertainty — an unreadable directory, an + /// unreadable entry, or a non-UTF-8 name all answer `false`, which + /// means "leave it alone". + fn looks_like_a_namespace_dir(dir: &Path) -> bool { + let Ok(entries) = std::fs::read_dir(dir) else { + return false; + }; + + let mut saw_vectors = false; + for entry in entries { + let Ok(entry) = entry else { + return false; + }; + if entry.file_name() != std::ffi::OsStr::new("vectors.dat") { + return false; + } + saw_vectors = true; + } + saw_vectors + } + /// Return a reference to the underlying MetadataStore. /// Useful for callers that need direct access (e.g., batch /// operations not covered by the trait). @@ -394,6 +623,459 @@ impl RedbStorageEngine { } } +// ═══════════════════════════════════════════════════════════════════════ +// Namespace purge +// ═══════════════════════════════════════════════════════════════════════ + +/// What [`RedbStorageEngine::purge_namespace`] destroyed. +#[derive(Debug)] +pub struct PurgedNamespace { + /// The configuration of the namespace that was removed. + pub config: NamespaceConfig, + /// Every record removed, as it was on disk. The caller needs these + /// to clean the indexes storage cannot reach. + pub records: Vec<(MemoryId, DiskRecord)>, + /// How many of `records` were live (not tombstoned). + pub live_removed: usize, + /// How many of `records` were already tombstones. + pub tombstones_removed: usize, + /// Edge rows deleted from edges.db, counting edges in which a + /// purged memory was the target as well as the source. + pub edges_removed: usize, + /// Surviving memories whose outgoing `edge_count` was corrected + /// because an edge of theirs pointed into the purged namespace, with + /// their new count. The caller owes these an invalidation: their + /// cached record still holds the old number. + pub peer_edge_counts: Vec<(MemoryId, u16)>, + /// Whether the namespace's vector directory was removed. `false` + /// means it was already gone, or a guard refused to touch it. + pub directory_removed: bool, +} + +/// A [`RedbStorageEngine::purge_namespace`] that failed. +/// +/// A purge is chunked and multi-step, so "it failed" says nothing about +/// how much survived. `destroyed` is the answer, and it is not +/// diagnostic decoration: those records are permanently gone from +/// `META_TABLE` and still occupy the FTS, vector, entity and graph +/// indexes, which only the caller can reach. Dropping the list strands +/// them there forever — a retry cannot find them, because it looks in +/// `META_TABLE`. +#[derive(Debug)] +pub struct NamespacePurgeError { + /// Records this purge destroyed before it stopped. Empty when it + /// failed before touching anything. + pub destroyed: Vec<(MemoryId, DiskRecord)>, + /// What stopped it. + pub source: StorageError, +} + +impl NamespacePurgeError { + /// A failure that happened before any record was committed. + fn nothing_destroyed(source: StorageError) -> Self { + Self { + destroyed: Vec::new(), + source, + } + } + + /// A failure that happened after `destroyed` was already gone. + fn partial(destroyed: Vec<(MemoryId, DiskRecord)>, source: StorageError) -> Self { + Self { destroyed, source } + } + + /// Whether anything was destroyed before the failure. A caller that + /// reports a bare failure for this case is lying to its user. + pub fn is_partial(&self) -> bool { + !self.destroyed.is_empty() + } +} + +impl std::fmt::Display for NamespacePurgeError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if self.is_partial() { + write!( + f, + "the namespace was PARTIALLY destroyed: {} memories are permanently \ + gone and the namespace still exists. Re-run the deletion to finish \ + it. Cause: {}", + self.destroyed.len(), + self.source + ) + } else { + write!(f, "{}", self.source) + } + } +} + +impl std::error::Error for NamespacePurgeError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + Some(&self.source) + } +} + +impl RedbStorageEngine { + /// Destroy a namespace: its records, its edges, its configuration + /// row and its vector file. + /// + /// # DESTRUCTIVE AND IRREVERSIBLE + /// + /// Every memory in the namespace is gone when this returns. There + /// is no undo and no backup. Callers are expected to have obtained + /// explicit confirmation before getting here. + /// + /// # The caller must finish the job + /// + /// This method can only reach what storage owns. The returned + /// records are the caller's input for cleaning the FTS index, the + /// in-memory vector index, the entity index and the relationship + /// graph — none of which this layer holds a handle to. Skip that and + /// the namespace is gone from disk but still burning top-k slots in + /// every search. + /// + /// # Ordering, and what a crash costs + /// + /// 1. Purge `META_TABLE` rows (chunked). + /// 2. Purge edges. + /// 3. Delete the `NAMESPACE_TABLE` row. **Point of no return.** + /// 4. Drop the `VectorStore`, then remove the directory. + /// + /// A crash before (3) leaves a live namespace holding fewer + /// memories: ugly, but consistent, and re-running the call finishes + /// it. A crash after (3) leaves an orphan directory and orphan FTS + /// rows — wasted disk, no wrong answers — and the directory is + /// reclaimed by the boot sweep in `startup_validation`. + /// + /// No crash marker is needed. Namespace ids are never recycled and + /// `NAMESPACE_TABLE` is authoritative, so "the row is gone" is + /// itself the durable record that the namespace is dead. + /// + /// # A failure is not the same as "nothing happened" + /// + /// Step 1 commits in chunks, so an ERROR — not merely a crash — part + /// of the way through has already destroyed everything up to that + /// point. The error type carries those records for exactly the + /// reason the success type does: without them the caller cannot + /// clean the indexes, and nothing ever can, because a retry finds + /// their `META_TABLE` rows already gone and reports an empty purge. + pub fn purge_namespace( + &mut self, + namespace_id: NamespaceId, + ) -> Result { + let config = match self.meta_store.get_namespace(namespace_id) { + Ok(Some(config)) => config, + Ok(None) => { + return Err(NamespacePurgeError::nothing_destroyed( + StorageError::NamespaceNotFound(namespace_id.get()), + )); + } + Err(e) => return Err(NamespacePurgeError::nothing_destroyed(e)), + }; + + tracing::info!( + namespace = %config.name, + namespace_id = namespace_id.get(), + "Purging namespace (destructive, irreversible)" + ); + + // 1. Records. This is the pass that reaches tombstones. + let records = match self + .meta_store + .purge_namespace_records(namespace_id, PURGE_CHUNK_SIZE) + { + Ok(records) => records, + Err(partial) => { + tracing::error!( + namespace = %config.name, + destroyed = partial.destroyed.len(), + error = %partial.source, + "Namespace purge failed part-way; the records it had already \ + committed are permanently gone" + ); + return Err(NamespacePurgeError { + destroyed: partial.destroyed, + source: partial.source, + }); + } + }; + + let live_removed = records + .iter() + .filter(|(_, r)| r.phase != DecayPhase::Tombstone) + .count(); + let tombstones_removed = records.len() - live_removed; + + // From here on every record in `records` is already destroyed, + // so any failure hands them back rather than dropping them. + // 2. Edges. `remove_all_edges` covers both directions, so an + // edge from a surviving memory into a purged one goes too — + // and the survivor's cached outgoing count has to follow it. + let ids: Vec = records.iter().map(|(id, _)| *id).collect(); + let (edges_removed, peer_edge_counts) = match self.remove_edges_and_repair_peers(&ids) { + Ok(pair) => pair, + Err(e) => return Err(NamespacePurgeError::partial(records, e)), + }; + + // 3. Configuration row. Point of no return. + if let Err(e) = self.meta_store.delete_namespace(namespace_id) { + return Err(NamespacePurgeError::partial(records, e)); + } + + // 4. Vector file. The store must be dropped before the unlink: + // it holds an fs2 exclusive lock and a live mmap. + let store = self.vector_manager.remove(namespace_id); + drop(store); + + let directory_removed = self.remove_namespace_dir(&config.name); + + tracing::info!( + namespace = %config.name, + namespace_id = namespace_id.get(), + live_removed, + tombstones_removed, + edges_removed, + directory_removed, + "Namespace purge complete" + ); + + Ok(PurgedNamespace { + config, + records, + live_removed, + tombstones_removed, + edges_removed, + peer_edge_counts, + directory_removed, + }) + } + + /// Remove every edge touching `ids`, and repair the outgoing + /// `edge_count` of every SURVIVING memory that had an edge into one + /// of them. Returns the edges removed and the `(peer, new_count)` + /// pairs that were rewritten. + /// + /// # Why anything needs repairing + /// + /// `DiskRecord.edge_count` counts OUTGOING edges and only the + /// source's count ever moves — see + /// `explicit_links::bump_edge_count`. Nothing in the system + /// decrements it, and until a memory is *destroyed* nothing needs + /// to: `tombstone` keeps the record and its edges, so the count + /// stays true. + /// + /// Destroying a memory breaks that. `survivor -> destroyed` + /// disappears from edges.db while the survivor's record goes on + /// counting it, permanently, and that number feeds degree + /// centrality and cache weight. Cross-namespace survivors are the + /// visible case — a namespace purge is the one operation that + /// removes edges from memories it is not deleting. + fn remove_edges_and_repair_peers( + &self, + ids: &[MemoryId], + ) -> Result<(usize, Vec<(MemoryId, u16)>), StorageError> { + let doomed: HashSet = ids.iter().copied().collect(); + let mut lost: HashMap = HashMap::new(); + let mut edges_removed = 0usize; + + for id in ids { + // Read before removing: these are the memories whose + // OUTGOING edge is about to vanish. A source that is itself + // doomed needs no repair — its record is going too. + for (source, _) in self.edge_store.get_incoming(*id)? { + if !doomed.contains(&source) { + let entry = lost.entry(source).or_insert(0); + *entry = entry.saturating_add(1); + } + } + edges_removed += self.edge_store.remove_all_edges(*id)?; + } + + let mut repaired = Vec::with_capacity(lost.len()); + for (peer, n) in lost { + let Some(record) = self.meta_store.get(peer)? else { + continue; + }; + let new_count = record.edge_count.saturating_sub(n); + self.meta_store.update_edge_count(peer, new_count)?; + repaired.push((peer, new_count)); + } + + Ok((edges_removed, repaired)) + } + + /// Resolve a namespace name to its directory under `db_path`, + /// refusing anything that is not exactly one ordinary path + /// component. + /// + /// `Path::join` does **not** normalize, so this is the whole + /// defence between a namespace name and an arbitrary directory: + /// `db_path.join("..")` is the parent of the data directory and + /// `db_path.join("")` is the data directory itself. Both are + /// `remove_dir_all` targets in [`Self::remove_namespace_dir`]. + /// + /// Modelled on the check + /// [`VectorManager::open_or_create`](crate::storage::vectors::VectorManager::open_or_create) + /// already performs — the non-destructive path had no business + /// being the better-guarded one. + fn namespace_dir(&self, name: &str) -> Result { + let reject = |reason: &str| { + Err(StorageError::InvalidNamespaceName { + name: name.to_string(), + reason: reason.to_string(), + }) + }; + + // Separators are refused explicitly rather than left to + // `components`: a backslash is a separator on Windows and an + // ordinary character on Unix, and a name that means two + // different paths on two platforms is not a name this may act + // on. + if name.is_empty() { + return reject("must not be empty"); + } + if name.contains('/') || name.contains('\\') { + return reject("must not contain '/' or '\\'"); + } + + let mut components = Path::new(name).components(); + let first_is_the_whole_name = + matches!(components.next(), Some(Component::Normal(c)) if c == OsStr::new(name)); + if !first_is_the_whole_name || components.next().is_some() { + return reject( + "must be exactly one ordinary path component (not '.', '..', or a path)", + ); + } + + Ok(self.db_path.join(name)) + } + + /// Remove one namespace's vector directory. Returns whether it was + /// actually removed. + /// + /// Unlike the boot sweep this joins a name straight onto `db_path` + /// rather than walking `read_dir`, so it does its own guarding: + /// + /// - [`Self::namespace_dir`] refuses anything that is not one + /// ordinary path component, which is what keeps `..` and `""` + /// away from `remove_dir_all`. + /// - a symlink is refused rather than followed: `remove_dir_all` + /// follows a symlinked *root*, and a namespace directory replaced + /// by a symlink would hand it an arbitrary tree to delete. + /// - the canonicalized target must be a strict child of the + /// canonicalized `db_path`. The component check already implies + /// this; canonicalizing is what *proves* it, and proof is what a + /// recursive delete is worth. + /// + /// Never returns an error. Step 3 of the purge has already + /// committed by the time this runs, so a failure here is wasted disk + /// to be logged and swept at next boot, not a reason to report the + /// deletion as failed. + fn remove_namespace_dir(&self, name: &str) -> bool { + let dir = match self.namespace_dir(name) { + Ok(dir) => dir, + Err(e) => { + tracing::error!( + namespace = name, + error = %e, + "Refusing to remove a namespace directory for an unusable name" + ); + return false; + } + }; + + let metadata = match std::fs::symlink_metadata(&dir) { + Ok(m) => m, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return false, + Err(e) => { + tracing::warn!( + path = %dir.display(), + error = %e, + "Could not stat namespace directory; leaving it on disk" + ); + return false; + } + }; + + if metadata.file_type().is_symlink() { + tracing::warn!( + path = %dir.display(), + "Namespace directory is a symlink; refusing to follow it. Remove the \ + link target by hand if it really is namespace data." + ); + return false; + } + + if !metadata.is_dir() { + tracing::warn!( + path = %dir.display(), + "Namespace path is not a directory; leaving it alone" + ); + return false; + } + + // The last word before the recursive delete: resolve both ends + // and require a strict parent/child relationship. + let canonical_base = match self.db_path.canonicalize() { + Ok(p) => p, + Err(e) => { + tracing::warn!( + path = %self.db_path.display(), + error = %e, + "Could not canonicalize the data directory; leaving the namespace \ + directory on disk" + ); + return false; + } + }; + let canonical_dir = match dir.canonicalize() { + Ok(p) => p, + Err(e) => { + tracing::warn!( + path = %dir.display(), + error = %e, + "Could not canonicalize the namespace directory; leaving it on disk" + ); + return false; + } + }; + if canonical_dir.parent() != Some(canonical_base.as_path()) { + tracing::error!( + namespace = name, + path = %canonical_dir.display(), + base = %canonical_base.display(), + "Namespace directory does not resolve to a direct child of the data \ + directory; refusing to remove it" + ); + return false; + } + + match std::fs::remove_dir_all(&dir) { + Ok(()) => true, + Err(e) => { + tracing::warn!( + path = %dir.display(), + error = %e, + "Failed to remove namespace directory; it will be swept at next startup" + ); + false + } + } + } +} + +/// Where the boot sweep moves orphaned namespace directories. +/// +/// Leading dot on purpose: the sweep skips dotfile directories, so its +/// own quarantine can never become a candidate for the next sweep. +const QUARANTINE_DIR_NAME: &str = ".orphaned"; + +/// How many records one `purge_namespace_records` transaction commits. +/// +/// Bounds how long a purge holds the redb write lock. Small enough that +/// a concurrent reader is not stalled for seconds; large enough that a +/// hundred-thousand-record namespace is not a hundred thousand fsyncs. +const PURGE_CHUNK_SIZE: usize = 1_000; + // ═══════════════════════════════════════════════════════════════════════ // StorageEngine Implementation // ═══════════════════════════════════════════════════════════════════════ @@ -510,8 +1192,9 @@ impl StorageEngine for RedbStorageEngine { let _ = vector_store.free_slot(record.vector_slot); } - // 3. Remove all edges involving this memory. - self.edge_store.remove_all_edges(id)?; + // 3. Remove all edges involving this memory, repairing the + // outgoing count of every survivor that pointed at it. + self.remove_edges_and_repair_peers(&[id])?; // 4. Text.log space is reclaimed lazily via compaction. @@ -623,12 +1306,45 @@ impl StorageEngine for RedbStorageEngine { // ── Namespaces ────────────────────────────────────────────────── + /// # The row must never outlive the directory + /// + /// This used to commit the `NAMESPACE_TABLE` row and only then ask + /// `open_or_create` whether the name could be a directory. A + /// rejected name therefore left a committed row for a namespace + /// whose directory can never be created — and [`Self::open`] step 6 + /// calls `open_or_create` for **every** row and `?`-propagates, so + /// the next boot failed outright. One rejected `create_namespace` + /// call wedged the database. + /// + /// So: validate first, and roll the row back if the vector store + /// still refuses. Validation cannot see a directory already flocked + /// by another store or a full disk, and both wedge the boot in + /// exactly the same way. fn create_namespace(&mut self, config: &NamespaceConfig) -> Result { + validate_namespace_name(&config.name).map_err(|e| StorageError::InvalidNamespaceName { + name: config.name.clone(), + reason: e.to_string(), + })?; + let ns_id = self.meta_store.create_namespace(config)?; // Open/create the vector file for this namespace. - self.vector_manager - .open_or_create(ns_id, &config.name, config.embedding_dim as usize)?; + if let Err(e) = + self.vector_manager + .open_or_create(ns_id, &config.name, config.embedding_dim as usize) + { + if let Err(rollback) = self.meta_store.delete_namespace(ns_id) { + tracing::error!( + namespace = %config.name, + namespace_id = ns_id.get(), + error = %rollback, + "Could not roll back the namespace row after its vector store failed \ + to open. The next startup will fail to open this namespace; remove \ + the row by hand." + ); + } + return Err(e.into()); + } Ok(ns_id) } @@ -784,3 +1500,698 @@ impl StorageEngine for RedbStorageEngine { self.meta_store.persist_phase_index() } } + +// ═══════════════════════════════════════════════════════════════════════ +// Tests +// ═══════════════════════════════════════════════════════════════════════ + +#[cfg(test)] +mod tests { + use super::*; + use crate::model::EdgeType; + + const DIM: usize = 4; + + fn open_engine(dir: &Path) -> RedbStorageEngine { + RedbStorageEngine::open(dir).expect("open engine") + } + + fn make_namespace(engine: &mut RedbStorageEngine, name: &str) -> NamespaceId { + let mut config = NamespaceConfig::default_namespace(0); + config.name = name.to_string(); + config.embedding_dim = DIM as u32; + engine.create_namespace(&config).expect("create namespace") + } + + fn add_memory(engine: &mut RedbStorageEngine, namespace_id: NamespaceId) -> MemoryId { + let id = MemoryId::new(); + let mut record = DiskRecord { + version: DiskRecord::CURRENT_VERSION, + id: *id.as_bytes(), + namespace_id: namespace_id.get(), + created_at: 0, + last_accessed_at: 0, + phase: DecayPhase::Full, + strength: 1.0, + decay_strength: 1.0, + stability: 1.0, + difficulty: 5.0, + is_permastore: 0, + vector_slot: 0, + edge_count: 0, + summary: "fixture".into(), + tags: Vec::new(), + access_history: Vec::new(), + text_offset: 0, + text_length: 0, + }; + engine + .insert_memory(id, namespace_id, &mut record, &[0.0; DIM], None) + .expect("insert memory"); + id + } + + /// The directory shape the boot sweep recognizes, planted by hand. + /// True when `root` lives on a case-insensitive filesystem, as APFS on + /// macOS is by default. + /// + /// Tests that plant a case variant of a live namespace's directory have + /// to know this: on such a filesystem the variant is not a second + /// directory, it is the same one, so planting into it overwrites the + /// live namespace's real `vectors.dat`. + fn filesystem_is_case_insensitive(root: &Path) -> bool { + let probe = root.join("CaseProbe"); + std::fs::create_dir_all(&probe).expect("create case probe"); + let insensitive = root.join("caseprobe").exists(); + std::fs::remove_dir_all(&probe).expect("remove case probe"); + insensitive + } + + fn plant_orphan_dir(root: &Path, name: &str) -> PathBuf { + let dir = root.join(name); + std::fs::create_dir_all(&dir).expect("create orphan dir"); + std::fs::write(dir.join("vectors.dat"), b"not really a vector file") + .expect("write vectors.dat"); + dir + } + + // ── purge_namespace ────────────────────────────────────────────── + + #[test] + fn w13_purge_removes_records_edges_the_config_row_and_the_directory() { + let dir = tempfile::tempdir().expect("tempdir"); + let mut engine = open_engine(dir.path()); + + let ns = make_namespace(&mut engine, "scratch"); + let a = add_memory(&mut engine, ns); + let b = add_memory(&mut engine, ns); + engine + .add_edge(a, b, EdgeType::Associative, 0.5, false, 0) + .expect("add edge"); + + let purged = engine.purge_namespace(ns).expect("purge"); + + assert_eq!(purged.config.name, "scratch"); + assert_eq!(purged.records.len(), 2); + assert_eq!(purged.live_removed, 2); + assert_eq!(purged.tombstones_removed, 0); + assert_eq!(purged.edges_removed, 1); + assert!(purged.directory_removed); + + assert!(engine.get_record(a).expect("get").is_none()); + assert!(engine.get_record(b).expect("get").is_none()); + assert!(engine.get_namespace(ns).expect("get namespace").is_none()); + assert!(engine.load_all_edges().expect("edges").is_empty()); + assert!(!dir.path().join("scratch").exists()); + } + + /// An edge from a surviving memory *into* a purged one is as dead as + /// one that starts there. Leaving it behind would point the graph at + /// a memory that no longer exists. + #[test] + fn w14_purge_removes_edges_that_point_into_the_namespace_from_outside() { + let dir = tempfile::tempdir().expect("tempdir"); + let mut engine = open_engine(dir.path()); + + let keep = make_namespace(&mut engine, "keep"); + let doomed = make_namespace(&mut engine, "doomed"); + let survivor = add_memory(&mut engine, keep); + let victim = add_memory(&mut engine, doomed); + engine + .add_edge(survivor, victim, EdgeType::Associative, 0.5, false, 0) + .expect("add edge"); + + let purged = engine.purge_namespace(doomed).expect("purge"); + + assert_eq!(purged.edges_removed, 1); + assert!(engine.load_all_edges().expect("edges").is_empty()); + assert!( + engine.get_record(survivor).expect("get").is_some(), + "the surviving memory was destroyed along with the edge" + ); + } + + /// A purge that resumes after a crash between the config-row delete + /// and the unlink finds no directory. That is a completed deletion, + /// not a failure. + #[test] + fn w15_purge_tolerates_a_directory_that_is_already_gone() { + let dir = tempfile::tempdir().expect("tempdir"); + let mut engine = open_engine(dir.path()); + + let ns = make_namespace(&mut engine, "scratch"); + add_memory(&mut engine, ns); + + // Drop the store first, exactly as the purge would. + drop(engine.vector_manager_mut().remove(ns)); + std::fs::remove_dir_all(dir.path().join("scratch")).expect("remove dir early"); + + let purged = engine.purge_namespace(ns).expect("purge"); + assert!(!purged.directory_removed); + assert_eq!(purged.records.len(), 1); + } + + #[test] + fn w16_boot_sweep_removes_a_namespace_shaped_directory_with_no_live_namespace() { + let dir = tempfile::tempdir().expect("tempdir"); + { + let _engine = open_engine(dir.path()); + } + let orphan = plant_orphan_dir(dir.path(), "ghost-namespace"); + + let _engine = open_engine(dir.path()); + assert!( + !orphan.exists(), + "orphan directory was left in the data directory" + ); + assert_eq!( + quarantined(dir.path()).len(), + 1, + "the orphan was destroyed rather than quarantined" + ); + } + + /// The sweep's safety argument — "no NAMESPACE_TABLE row means dead" + /// — has no lower bound: an absent or restored-from-backup meta.db + /// makes `live` empty and EVERY namespace directory a candidate. + /// Every guard passes, because they really are namespace + /// directories. Quarantine is what makes that survivable. + #[test] + fn w59_a_swept_directory_is_moved_not_destroyed() { + let dir = tempfile::tempdir().expect("tempdir"); + { + let mut engine = open_engine(dir.path()); + make_namespace(&mut engine, "irreplaceable"); + std::fs::write( + dir.path().join("irreplaceable").join("vectors.dat"), + b"the only copy", + ) + .expect("write vectors.dat"); + } + + // The restored-backup shape: the namespace rows are gone, the + // directories are not. + std::fs::remove_file(dir.path().join("meta.db")).expect("remove meta.db"); + + { + let _engine = open_engine(dir.path()); + } + + let survivors = quarantined(dir.path()); + assert_eq!(survivors.len(), 1, "{survivors:?}"); + assert!( + survivors[0] + .file_name() + .and_then(|n| n.to_str()) + .is_some_and(|n| n.starts_with("irreplaceable-")), + "the quarantined directory must be named after the namespace: {survivors:?}" + ); + assert_eq!( + std::fs::read(survivors[0].join("vectors.dat")).expect("read"), + b"the only copy", + "the quarantined directory does not hold the original contents" + ); + } + + /// The quarantine directory must never become the next sweep's + /// candidate, or the recovery copy is swept into a copy of itself + /// every boot. + #[test] + fn w60_the_quarantine_directory_is_not_itself_swept() { + let dir = tempfile::tempdir().expect("tempdir"); + { + let _engine = open_engine(dir.path()); + } + plant_orphan_dir(dir.path(), "ghost"); + + { + let _engine = open_engine(dir.path()); + } + let after_first = quarantined(dir.path()); + assert_eq!(after_first.len(), 1); + + { + let _engine = open_engine(dir.path()); + } + assert_eq!( + quarantined(dir.path()), + after_first, + "the second boot moved the quarantine" + ); + } + + /// Everything currently in `/.orphaned/`, sorted. + fn quarantined(db_path: &Path) -> Vec { + let quarantine = db_path.join(QUARANTINE_DIR_NAME); + let Ok(entries) = std::fs::read_dir(&quarantine) else { + return Vec::new(); + }; + let mut out: Vec = entries.filter_map(|e| e.ok()).map(|e| e.path()).collect(); + out.sort(); + out + } + + /// `db_path` is user-configurable and may hold directories that are + /// nothing to do with recalld. "Matches no live namespace" is not on + /// its own a licence to recurse and delete. + #[test] + fn w17_boot_sweep_leaves_a_directory_that_is_not_shaped_like_a_namespace() { + let dir = tempfile::tempdir().expect("tempdir"); + { + let _engine = open_engine(dir.path()); + } + + let with_extra = plant_orphan_dir(dir.path(), "not-ours"); + std::fs::write(with_extra.join("important.txt"), b"someone else's data") + .expect("write extra file"); + + let no_vectors = dir.path().join("empty-dir"); + std::fs::create_dir_all(&no_vectors).expect("create dir"); + + let dotted = plant_orphan_dir(dir.path(), ".hidden-state"); + + let _engine = open_engine(dir.path()); + + assert!(with_extra.join("important.txt").exists()); + assert!(no_vectors.exists()); + assert!(dotted.exists()); + } + + /// THE catastrophic-regression test. The boot sweep deletes + /// directories, it runs unattended on every open, and its blast + /// radius if it misjudges "live" is every memory in the database. + /// + /// What makes it safe is an ordering guarantee: `open` step 6 opens + /// (and thereby creates) a directory for every namespace in + /// `NAMESPACE_TABLE` before step 7 runs the sweep, so at sweep time + /// every live namespace has both a directory and a name in `live`. + #[test] + fn w18_boot_sweep_leaves_every_live_namespace_directory_on_a_plain_reopen() { + let dir = tempfile::tempdir().expect("tempdir"); + let mut ids = Vec::new(); + + { + let mut engine = open_engine(dir.path()); + for name in ["default", "work", "personal"] { + let ns = make_namespace(&mut engine, name); + ids.push(add_memory(&mut engine, ns)); + } + } + + let engine = open_engine(dir.path()); + + for name in ["default", "work", "personal"] { + let ns_dir = dir.path().join(name); + assert!( + ns_dir.join("vectors.dat").exists(), + "the boot sweep destroyed live namespace '{name}'" + ); + } + for id in ids { + assert!( + engine.get_record(id).expect("get").is_some(), + "a live record went missing across a plain reopen" + ); + } + } + + /// The crash-recovery shape from D5: everything about the namespace + /// is gone after a restart, including the edges that pointed at it. + #[test] + fn w19_a_purged_namespace_stays_gone_across_a_reopen() { + let dir = tempfile::tempdir().expect("tempdir"); + + { + let mut engine = open_engine(dir.path()); + let ns = make_namespace(&mut engine, "scratch"); + let a = add_memory(&mut engine, ns); + let b = add_memory(&mut engine, ns); + engine + .add_edge(a, b, EdgeType::Associative, 0.5, false, 0) + .expect("add edge"); + engine.purge_namespace(ns).expect("purge"); + } + + let engine = open_engine(dir.path()); + assert!( + engine + .get_namespace_by_name("scratch") + .expect("lookup") + .is_none() + ); + assert!(!dir.path().join("scratch").exists()); + assert!(engine.load_all_edges().expect("edges").is_empty()); + assert_eq!(engine.count().expect("count"), 0); + } + + // ── peer edge counts ───────────────────────────────────────────── + + /// `edge_count` counts OUTGOING edges. A purge removes the edge + /// `survivor -> purged` from edges.db but used to leave the + /// survivor's record still counting it — permanently, and in a + /// number that feeds degree centrality and cache weight. The + /// cross-namespace case is the one a user sees: the survivor is not + /// in the namespace being deleted and nothing else will ever touch + /// its record. + #[test] + fn w56_a_purge_repairs_the_outgoing_edge_count_of_a_surviving_peer() { + let dir = tempfile::tempdir().expect("tempdir"); + let mut engine = open_engine(dir.path()); + + let keep_ns = make_namespace(&mut engine, "keep"); + let doomed_ns = make_namespace(&mut engine, "doomed"); + + let survivor = add_memory(&mut engine, keep_ns); + let other_survivor = add_memory(&mut engine, keep_ns); + let doomed_a = add_memory(&mut engine, doomed_ns); + let doomed_b = add_memory(&mut engine, doomed_ns); + + // Two edges into the doomed namespace and one that stays. + for target in [doomed_a, doomed_b, other_survivor] { + engine + .add_edge(survivor, target, EdgeType::Associative, 0.5, false, 0) + .expect("add edge"); + } + engine.update_edge_count(survivor, 3).expect("seed count"); + + let purged = engine.purge_namespace(doomed_ns).expect("purge"); + + assert_eq!( + engine + .get_record(survivor) + .expect("get") + .expect("record") + .edge_count, + 1, + "the survivor still counts edges that no longer exist" + ); + assert_eq!(purged.peer_edge_counts, vec![(survivor, 1)]); + } + + /// A peer that is itself being purged needs no repair, and must not + /// be reported as one: its record is gone. + #[test] + fn w57_a_purge_does_not_repair_a_peer_it_is_also_destroying() { + let dir = tempfile::tempdir().expect("tempdir"); + let mut engine = open_engine(dir.path()); + + let ns = make_namespace(&mut engine, "doomed"); + let a = add_memory(&mut engine, ns); + let b = add_memory(&mut engine, ns); + engine + .add_edge(a, b, EdgeType::Associative, 0.5, false, 0) + .expect("add edge"); + engine.update_edge_count(a, 1).expect("seed count"); + + let purged = engine.purge_namespace(ns).expect("purge"); + + assert!(purged.peer_edge_counts.is_empty(), "{purged:?}"); + } + + /// The same repair on the single-memory hard delete, which removes + /// edges for exactly the same reason and had exactly the same gap. + #[test] + fn w58_a_hard_delete_repairs_the_outgoing_edge_count_of_a_surviving_peer() { + let dir = tempfile::tempdir().expect("tempdir"); + let mut engine = open_engine(dir.path()); + + let ns = make_namespace(&mut engine, "scratch"); + let survivor = add_memory(&mut engine, ns); + let doomed = add_memory(&mut engine, ns); + engine + .add_edge(survivor, doomed, EdgeType::Associative, 0.5, false, 0) + .expect("add edge"); + engine.update_edge_count(survivor, 1).expect("seed count"); + + engine.delete_memory(doomed).expect("delete"); + + assert_eq!( + engine + .get_record(survivor) + .expect("get") + .expect("record") + .edge_count, + 0, + "the survivor still counts an edge that no longer exists" + ); + } + + // ── namespace name → directory ─────────────────────────────────── + + /// `Path::join` does **not** normalize, so `db_path.join("..")` is + /// the parent of the data directory and `remove_dir_all` on it takes + /// everything recalld owns — and everything that happens to sit + /// beside it. The old guard checked only symlink-and-is-dir, both of + /// which `..` passes. + #[test] + fn w45_a_dot_dot_name_cannot_delete_the_parent_of_the_data_directory() { + let outer = tempfile::tempdir().expect("tempdir"); + let db_path = outer.path().join("data"); + std::fs::create_dir_all(&db_path).expect("create data dir"); + let bystander = outer.path().join("someone-elses-files"); + std::fs::create_dir_all(&bystander).expect("create bystander"); + + let engine = open_engine(&db_path); + + assert!( + !engine.remove_namespace_dir(".."), + "'..' was accepted as a namespace directory name" + ); + assert!( + bystander.exists(), + "the parent of the data directory was destroyed" + ); + assert!( + db_path.join("meta.db").exists(), + "the database was destroyed" + ); + } + + /// The one that needs no `..` and no daemon frame. `db_path.join("")` + /// **is** `db_path`, `VectorManager::open_or_create` accepted `""`, + /// so the namespace was live and healthy — and deleting it pointed + /// `remove_dir_all` at the whole database. + #[test] + fn w46_an_empty_name_cannot_delete_the_data_directory() { + let dir = tempfile::tempdir().expect("tempdir"); + let engine = open_engine(dir.path()); + + assert!( + !engine.remove_namespace_dir(""), + "'' was accepted as a namespace directory name" + ); + assert!(db_is_intact(dir.path()), "the database was destroyed"); + } + + #[test] + fn w47_only_a_single_ordinary_path_component_is_accepted() { + let dir = tempfile::tempdir().expect("tempdir"); + let engine = open_engine(dir.path()); + + for bad in [ + "", + ".", + "..", + "/", + "sub/dir", + "..\\up", + "scratch/", + "./scratch", + ] { + assert!( + engine.namespace_dir(bad).is_err(), + "{bad:?} was accepted as a namespace directory name" + ); + } + + assert_eq!( + engine.namespace_dir("scratch").expect("a plain name"), + dir.path().join("scratch") + ); + } + + /// `create_namespace` used to commit the `NAMESPACE_TABLE` row and + /// only THEN ask `open_or_create` whether the name could be a + /// directory. The caller saw an error and the row survived — and + /// [`RedbStorageEngine::open`] step 6 `?`-propagates that same + /// rejection for every row, so one bad `create_namespace` call meant + /// the database would not start again. + #[test] + fn w48_a_rejected_name_leaves_no_row_and_the_database_still_opens() { + let dir = tempfile::tempdir().expect("tempdir"); + + { + let mut engine = open_engine(dir.path()); + for bad in ["", "..", "a/b", "has space", "ünïcode", &"x".repeat(65)] { + let mut config = NamespaceConfig::default_namespace(0); + config.name = bad.to_string(); + config.embedding_dim = DIM as u32; + + engine + .create_namespace(&config) + .expect_err(&format!("{bad:?} must be rejected")); + + assert!( + engine + .list_namespaces() + .expect("list") + .iter() + .all(|ns| ns.name != bad), + "{bad:?} left a NAMESPACE_TABLE row behind" + ); + } + } + + RedbStorageEngine::open(dir.path()) + .expect("a rejected create_namespace must not wedge the next startup"); + } + + // ── case-insensitive namespace identity ────────────────────────── + + /// Chain A. On a case-insensitive filesystem `notes/` and `Notes/` + /// are one directory, so two rows pointing at it is two namespaces + /// sharing one `vectors.dat`. The uniqueness check compared names + /// exactly and let the second row in. + #[test] + fn w49_a_namespace_name_differing_only_in_case_is_a_duplicate() { + let dir = tempfile::tempdir().expect("tempdir"); + let mut engine = open_engine(dir.path()); + + make_namespace(&mut engine, "notes"); + + for collision in ["Notes", "NOTES", "nOtEs"] { + let mut config = NamespaceConfig::default_namespace(0); + config.name = collision.to_string(); + config.embedding_dim = DIM as u32; + let err = engine + .create_namespace(&config) + .expect_err("a case-variant name must collide"); + assert!( + matches!(err, StorageError::DuplicateName(_)), + "{collision}: {err}" + ); + } + + assert_eq!(engine.list_namespaces().expect("list").len(), 1); + } + + /// The lookup has to agree with the uniqueness rule, or + /// `create_namespace` reports "duplicate" and `namespace_stats` + /// reports "not found" for the same string. + #[test] + fn w50_a_namespace_resolves_by_name_regardless_of_case() { + let dir = tempfile::tempdir().expect("tempdir"); + let mut engine = open_engine(dir.path()); + let ns = make_namespace(&mut engine, "notes"); + + for spelling in ["notes", "Notes", "NOTES"] { + assert_eq!( + engine + .get_namespace_by_name(spelling) + .expect("lookup") + .map(|c| c.id), + Some(ns), + "{spelling} did not resolve" + ); + } + } + + /// The unattended half of chain A. A live namespace whose directory + /// differs from its row only in case used to be invisible to the + /// sweep's exact-string compare, and the shape guard cannot save it + /// — it really is a namespace directory. Boot then deleted a live + /// namespace's vectors. + #[test] + fn w51_the_boot_sweep_does_not_delete_a_case_variant_live_directory() { + let dir = tempfile::tempdir().expect("tempdir"); + + // Probe before the engine exists, so the throwaway directory cannot + // be seen by the boot sweep. + let case_insensitive = filesystem_is_case_insensitive(dir.path()); + + { + let mut engine = open_engine(dir.path()); + make_namespace(&mut engine, "Notes"); + // On a case-sensitive filesystem the lowercase variant has to be + // planted, so that the sweep is made to choose between deleting + // it and recognising it as the live namespace's directory. + // + // On a case-insensitive one there is nothing to plant: creating + // `Notes` already produced the directory `notes` resolves to. + // Planting would write a stub over the live namespace's real + // vectors.dat and the reopen below would fail to parse it -- + // which is exactly how this test used to fail on macOS while + // passing on Linux. + if !case_insensitive { + plant_orphan_dir(dir.path(), "notes"); + } + } + + let _engine = open_engine(dir.path()); + + assert!( + dir.path().join("notes").join("vectors.dat").exists(), + "the boot sweep destroyed a live namespace's vectors" + ); + } + + // ── partial purge ──────────────────────────────────────────────── + + /// A failure before anything was committed must say so, or the + /// caller cleans indexes for records that still exist. + #[test] + fn w52_a_purge_that_destroyed_nothing_reports_nothing_destroyed() { + let dir = tempfile::tempdir().expect("tempdir"); + let mut engine = open_engine(dir.path()); + + let err = engine + .purge_namespace(NamespaceId::new(999)) + .expect_err("an unknown namespace cannot be purged"); + + assert!(!err.is_partial(), "{err}"); + assert!(err.destroyed.is_empty()); + assert!( + !err.to_string().contains("PARTIALLY"), + "a failure that destroyed nothing must not claim otherwise: {err}" + ); + } + + /// The other half of the contract, and the one that matters: an + /// error that arrives after chunks have committed carries them, and + /// says what the caller has to do about it. Without this the records + /// are gone from meta.db, still in every index, and unreachable by a + /// retry — which looks in meta.db and finds nothing. + #[test] + fn w53_a_partial_purge_carries_its_destroyed_records_and_asks_for_a_retry() { + let dir = tempfile::tempdir().expect("tempdir"); + let mut engine = open_engine(dir.path()); + let ns = make_namespace(&mut engine, "scratch"); + let a = add_memory(&mut engine, ns); + let b = add_memory(&mut engine, ns); + + let destroyed: Vec<(MemoryId, DiskRecord)> = [a, b] + .into_iter() + .map(|id| (id, engine.get_record(id).expect("get").expect("record"))) + .collect(); + + let err = NamespacePurgeError::partial( + destroyed, + StorageError::CorruptIndex("simulated mid-purge fault".into()), + ); + + assert!(err.is_partial()); + assert_eq!(err.destroyed.len(), 2); + let message = err.to_string(); + assert!(message.contains("PARTIALLY destroyed"), "{message}"); + assert!(message.contains('2'), "the count must be in it: {message}"); + assert!(message.contains("Re-run"), "{message}"); + } + + /// Whether the database files a purge must never touch are all still + /// where they belong. + fn db_is_intact(db_path: &Path) -> bool { + db_path.join("meta.db").exists() + && db_path.join("edges.db").exists() + && db_path.join("fulltext.dat").exists() + } +} diff --git a/src/storage/error.rs b/src/storage/error.rs index ba8c766..baa95ad 100644 --- a/src/storage/error.rs +++ b/src/storage/error.rs @@ -46,6 +46,15 @@ pub enum StorageError { #[error("duplicate namespace name: {0}")] DuplicateName(String), + /// A namespace name is not usable as an on-disk directory name. + #[error("invalid namespace name '{name}': {reason}")] + InvalidNamespaceName { + /// The rejected name, echoed back for the caller's message. + name: String, + /// Why it was rejected. + reason: String, + }, + /// No namespace exists for the given ID. #[error("namespace not found: {0}")] NamespaceNotFound(u32), diff --git a/src/storage/indexes.rs b/src/storage/indexes.rs index a5e876a..b6e3499 100644 --- a/src/storage/indexes.rs +++ b/src/storage/indexes.rs @@ -132,6 +132,18 @@ impl PhaseIndex { } } + /// Drop every bitmap belonging to a namespace, in all four phases. + /// + /// For namespace deletion, which removes records without going + /// through [`remove`](Self::remove) per record. Without this the + /// bitmaps outlive the namespace, and + /// [`all_slots_in_phase`](Self::all_slots_in_phase) would keep + /// handing the decay sweep `(namespace_id, vector_slot)` pairs whose + /// records no longer exist. + pub fn remove_namespace(&mut self, namespace_id: u32) { + self.namespaces.remove(&namespace_id); + } + /// Return all `(namespace_id, vector_slot)` pairs in the given /// phase across all namespaces. pub fn all_slots_in_phase(&self, phase: DecayPhase) -> Vec<(u32, u32)> { @@ -313,3 +325,49 @@ impl PhaseIndex { index } } + +// ═══════════════════════════════════════════════════════════════════════ +// Tests +// ═══════════════════════════════════════════════════════════════════════ + +#[cfg(test)] +mod tests { + use super::*; + + /// Namespace deletion removes records without going through + /// `remove` per record, so without a wholesale drop the bitmaps + /// outlive the namespace and `all_slots_in_phase` keeps offering the + /// decay sweep slots whose records are gone. + #[test] + fn w10_remove_namespace_clears_every_phase_for_that_namespace_only() { + let mut index = PhaseIndex::new(); + + index.insert(1, 0); + index.insert(1, 1); + index.transition(1, 1, DecayPhase::Full, DecayPhase::Ghost); + index.insert(1, 2); + index.transition(1, 2, DecayPhase::Full, DecayPhase::Summary); + index.insert(2, 0); + + index.remove_namespace(1); + + for phase in [ + DecayPhase::Full, + DecayPhase::Summary, + DecayPhase::Ghost, + DecayPhase::Tombstone, + ] { + let survivors: Vec<(u32, u32)> = index + .all_slots_in_phase(phase) + .into_iter() + .filter(|(ns, _)| *ns == 1) + .collect(); + assert!( + survivors.is_empty(), + "{phase:?} still holds ns 1: {survivors:?}" + ); + } + + assert_eq!(index.all_slots_in_phase(DecayPhase::Full), vec![(2, 0)]); + } +} diff --git a/src/storage/metadata.rs b/src/storage/metadata.rs index cdb51a5..ea48d85 100644 --- a/src/storage/metadata.rs +++ b/src/storage/metadata.rs @@ -7,6 +7,7 @@ //! //! See CS-07 for the full specification. +use std::collections::HashMap; use std::path::Path; use redb::{ @@ -539,6 +540,49 @@ impl MetadataStore { Ok(tags) } + + /// Tag histogram over the live memories of one namespace. + /// + /// `TAG_INDEX` is keyed by tag alone with no namespace dimension, so + /// this walks `NAMESPACE_INDEX` and does one `META_TABLE` point + /// lookup per member — the same work as one unfiltered + /// [`list_memories_filtered`](Self::list_memories_filtered) call, not + /// a `scan_all`. + /// + /// Returns `(tag -> memory count, live memory count)`. Tombstoned + /// records are skipped; `Summary` and `Ghost` are included, because a + /// phase transition strips text, not tags. + pub fn tag_counts_in_namespace( + &self, + namespace_id: NamespaceId, + ) -> Result<(HashMap, u64), StorageError> { + let read_txn = self.db.begin_read()?; + let ns_table = read_txn.open_multimap_table(NAMESPACE_INDEX)?; + let meta_table = read_txn.open_table(META_TABLE)?; + + let mut counts: HashMap = HashMap::new(); + let mut memories = 0u64; + + for result in ns_table.get(namespace_id.get())? { + let value = result?; + let bytes: [u8; 16] = value.value().try_into().map_err(|_| { + StorageError::CorruptIndex("invalid UUID length in namespace index".into()) + })?; + let Some(record_value) = meta_table.get(bytes.as_slice())? else { + continue; + }; + let record = DiskRecord::from_bytes(record_value.value())?; + if record.phase == DecayPhase::Tombstone { + continue; + } + memories += 1; + for tag in &record.tags { + *counts.entry(tag.as_str().to_string()).or_insert(0) += 1; + } + } + + Ok((counts, memories)) + } } // ── Namespace Index (Section 6.3) ─────────────────────────────────── @@ -565,6 +609,49 @@ impl MetadataStore { Ok(ids) } + /// Number of live (non-tombstoned) memories in `namespace_id`. + /// + /// Applies the same predicate as [`list_memories_filtered`] with no + /// filters — walk `NAMESPACE_INDEX`, load each record, skip + /// `Tombstone` — so `list_namespaces[].memoryCount`, + /// `namespace_stats.memoryCount` and `list_memories.total` agree *by + /// construction* rather than by invariant. + /// + /// One point lookup per member of the namespace, not a scan of the + /// database. + /// + /// [`memories_in_namespace`](Self::memories_in_namespace)`.len()` + /// would be cheaper — index cardinality only, no `META_TABLE` reads — + /// and `tombstone` does remove from the index, so the two agree in + /// practice. But they diverge for an index entry whose `META_TABLE` + /// row is missing, which `list_memories_filtered` explicitly + /// tolerates, and the whole point here is that this count agrees with + /// what a caller can actually list. + pub fn count_memories_in_namespace( + &self, + namespace_id: NamespaceId, + ) -> Result { + let read_txn = self.db.begin_read()?; + let ns_table = read_txn.open_multimap_table(NAMESPACE_INDEX)?; + let meta_table = read_txn.open_table(META_TABLE)?; + + let mut count = 0u64; + for result in ns_table.get(namespace_id.get())? { + let value = result?; + let bytes: [u8; 16] = value.value().try_into().map_err(|_| { + StorageError::CorruptIndex("invalid UUID length in namespace index".into()) + })?; + let Some(record_value) = meta_table.get(bytes.as_slice())? else { + continue; + }; + if DiskRecord::from_bytes(record_value.value())?.phase != DecayPhase::Tombstone { + count += 1; + } + } + + Ok(count) + } + /// List memories in a namespace with optional tag/entity/time filters, /// sorted by creation date (newest first) with offset/limit pagination. /// @@ -807,12 +894,31 @@ impl MetadataStore { let new_id; { // Check name uniqueness by scanning existing namespaces. + // + // Case-INSENSITIVELY, because the name becomes a directory + // name and macOS APFS, Windows NTFS and a case-insensitive + // ext4 volume all give "notes" and "Notes" the SAME + // directory. Two rows sharing one directory is not a + // cosmetic collision: + // + // - delete "notes" (row gone, remove_dir_all fails, warn + // only), create "Notes", and the new namespace adopts the + // old one's `vectors.dat` — somebody else's embeddings. + // - `create_namespace("Default")` used to pass the + // uniqueness check, commit its row, no-op `create_dir_all` + // onto `default/` and only then fail on the exclusive + // flock. The row survived, and `delete_namespace("Default")` + // walked past the `!= "default"` guard and unlinked the + // real default's vectors. + // + // `validate_namespace_name` restricts names to ASCII, so + // ASCII case folding is total here. let ns_table = write_txn.open_table(NAMESPACE_TABLE)?; for result in ns_table.iter()? { let (_, value) = result?; let existing: NamespaceConfig = serde_json::from_slice(value.value()) .map_err(|e| StorageError::Deserialize(e.to_string()))?; - if existing.name == config.name { + if existing.name.eq_ignore_ascii_case(&config.name) { return Err(StorageError::DuplicateName(config.name.clone())); } } @@ -871,45 +977,42 @@ impl MetadataStore { /// Get a namespace by name (linear scan -- acceptable for <100 /// namespaces). + /// + /// Matching is case-insensitive, to agree with the uniqueness rule + /// in [`create_namespace`](Self::create_namespace) — a name that + /// cannot be created because it collides case-insensitively must + /// also resolve case-insensitively, or `create` says "duplicate" + /// while `stats` says "not found" for the same string. + /// + /// An exact match wins over a case-folded one. That only matters for + /// a database written before the uniqueness rule folded case, which + /// can already hold both `notes` and `Notes`; preferring the exact + /// match keeps each of those rows individually addressable rather + /// than resolving both to whichever the scan reaches first. pub fn get_namespace_by_name( &self, name: &str, ) -> Result, StorageError> { let all = self.list_namespaces()?; - Ok(all.into_iter().find(|c| c.name == name)) - } - - /// Update a namespace's mutable fields (name, thresholds, etc.). - /// Immutable fields (id, embedding_dim, created_at) are preserved. - pub fn update_namespace( - &self, - id: NamespaceId, - updated: &NamespaceConfig, - ) -> Result<(), StorageError> { - let write_txn = self.db.begin_write()?; - { - let mut table = write_txn.open_table(NAMESPACE_TABLE)?; - let existing: NamespaceConfig = { - let existing_bytes = table - .get(id.get())? - .ok_or(StorageError::NamespaceNotFound(id.get()))?; - serde_json::from_slice(existing_bytes.value()) - .map_err(|e| StorageError::Deserialize(e.to_string()))? - }; - - let mut to_store = updated.clone(); - to_store.id = existing.id; - to_store.embedding_dim = existing.embedding_dim; - to_store.created_at = existing.created_at; - - let bytes = serde_json::to_vec(&to_store) - .map_err(|e| StorageError::Serialize(e.to_string()))?; - table.insert(id.get(), bytes.as_slice())?; + if let Some(exact) = all.iter().find(|c| c.name == name) { + return Ok(Some(exact.clone())); } - write_txn.commit()?; - Ok(()) + Ok(all.into_iter().find(|c| c.name.eq_ignore_ascii_case(name))) } + // `update_namespace` used to live here: a read-modify-write of the + // NAMESPACE_TABLE row that preserved id, embedding_dim and + // created_at and let everything else through — including `name`. + // It had zero callers, and a rename is not a metadata edit. The name + // IS the directory: renaming the row leaves the memories reading + // `/vectors.dat` until the next boot, where step 6 creates a + // fresh empty `/` and + // `RedbStorageEngine::sweep_orphan_namespace_dirs` finds `/` + // matching no live name and removes it. A namespace rename would + // have been a delayed, unattended shredder for every embedding in + // it. Deleted rather than documented; if a rename is ever wanted it + // has to move the directory in the same operation. + /// Delete a namespace record from NAMESPACE_TABLE. /// /// Only removes the config entry. Memory cleanup is the caller's @@ -929,33 +1032,310 @@ impl MetadataStore { Ok(config) } - /// Drain all memories belonging to a namespace. + /// Remove every memory record belonging to a namespace, live and + /// tombstoned alike, together with its tag and namespace index + /// entries. + /// + /// # DESTRUCTIVE AND IRREVERSIBLE + /// + /// There is no undo and no backup. Returns the removed + /// `(MemoryId, DiskRecord)` pairs so the caller can free vector + /// slots and clean the FTS, vector and entity indexes, none of + /// which this store can reach. + /// + /// # Why a `META_TABLE` scan and not `NAMESPACE_INDEX` /// - /// Processes in batches of 1,000 to avoid holding a write lock - /// for too long. Returns the list of deleted DiskRecords. - pub fn drain_namespace_memories( + /// [`tombstone`](Self::tombstone) unlinks a record from + /// `NAMESPACE_INDEX` while leaving its `META_TABLE` row in place, so + /// an index-driven walk structurally cannot see tombstones — and + /// tombstones are exactly the rows nothing else ever reclaims. The + /// full scan is the only way to reach them. `record.namespace_id` is + /// the filter. + /// + /// # Chunking, and what the caller owes it + /// + /// Removal is committed in chunks of `batch_size` rather than one + /// transaction, so a large namespace does not hold the write lock + /// for the whole purge. That makes the operation *resumable but not + /// atomic*: an interrupted purge leaves a live namespace holding + /// fewer memories. The caller must therefore delete the + /// `NAMESPACE_TABLE` row **after** this returns, never before — + /// deleting it first turns a crash mid-purge into orphan records + /// belonging to a namespace that no longer exists. + pub fn purge_namespace_records( &self, namespace_id: NamespaceId, - ) -> Result, StorageError> { - let mut all_deleted = Vec::new(); - - loop { - let batch_ids = { - let ids = self.memories_in_namespace(namespace_id)?; - if ids.is_empty() { - break; + batch_size: usize, + ) -> Result, PartialPurge> { + // chunks(0) panics, and a zero batch would never terminate + // anyway. + let batch_size = batch_size.max(1); + let ns = namespace_id.get(); + + // Pass 1: read scan. Collects tombstones as well as live rows. + // Nothing is committed yet, so a failure here destroyed nothing. + let victims = match self.scan_namespace_records(ns) { + Ok(v) => v, + Err(source) => { + return Err(PartialPurge { + destroyed: Vec::new(), + source, + }); + } + }; + + if victims.is_empty() { + return Ok(Vec::new()); + } + + // Pass 2: chunked removal. Each chunk that commits is gone + // forever, so the count of committed chunks travels with the + // error rather than being thrown away with it. + let mut destroyed: Vec<(MemoryId, DiskRecord)> = Vec::with_capacity(victims.len()); + for chunk in victims.chunks(batch_size) { + if let Err(source) = self.purge_one_chunk(ns, chunk) { + // The bitmaps for what DID go are dropped slot by slot: + // `remove_namespace` would drop the survivors' entries + // too, and the survivors are still live records. + { + let mut pi = self.phase_index.write().unwrap_or_else(|e| e.into_inner()); + for (_, record) in &destroyed { + pi.remove(ns, record.vector_slot, record.phase); + } } - ids.into_iter().take(1000).collect::>() - }; + return Err(PartialPurge { destroyed, source }); + } + destroyed.extend_from_slice(chunk); + } + + // The whole namespace is gone, so drop its bitmaps wholesale + // rather than slot by slot. + { + let mut pi = self.phase_index.write().unwrap_or_else(|e| e.into_inner()); + pi.remove_namespace(ns); + } + + Ok(destroyed) + } + + /// Count the live records a [`purge_namespace_records`] call would + /// destroy. + /// + /// # Why not `count_memories_in_namespace` + /// + /// That one walks `NAMESPACE_INDEX` and confirms each hit against + /// `META_TABLE`; the purge scans `META_TABLE` and filters on + /// `record.namespace_id`. They read different sources of truth, so a + /// `META_TABLE` row whose index entry is missing is invisible to the + /// count and destroyed by the purge — which is precisely the row a + /// non-force "is this namespace empty?" guard exists to protect. A + /// check and an effect that disagree is not a guard. + /// + /// Tombstones are excluded, matching the guard's rule that a + /// namespace holding only tombstones is empty: they hold no content + /// the user has not already asked to forget, and they report as zero + /// everywhere else. + /// + /// This is a full scan. It runs only on the non-force deletion path, + /// immediately before a purge that scans the same table anyway. + pub fn count_live_records_in_namespace( + &self, + namespace_id: NamespaceId, + ) -> Result { + let ns = namespace_id.get(); + let read_txn = self.db.begin_read()?; + let table = read_txn.open_table(META_TABLE)?; + + let mut count = 0u64; + for result in table.iter()? { + let (_, value) = result?; + let record = DiskRecord::from_bytes(value.value())?; + if record.namespace_id == ns && record.phase != DecayPhase::Tombstone { + count += 1; + } + } + Ok(count) + } + + /// Every `META_TABLE` row belonging to `ns`, tombstones included. + fn scan_namespace_records(&self, ns: u32) -> Result, StorageError> { + let read_txn = self.db.begin_read()?; + let table = read_txn.open_table(META_TABLE)?; + let mut found = Vec::new(); + for result in table.iter()? { + let (key_bytes, value) = result?; + let record = DiskRecord::from_bytes(value.value())?; + if record.namespace_id != ns { + continue; + } + let uuid = uuid::Uuid::from_slice(key_bytes.value()) + .map_err(|_| StorageError::CorruptIndex("invalid UUID in meta table".into()))?; + found.push((MemoryId::from_uuid(uuid), record)); + } + Ok(found) + } + + /// Remove one chunk of records, with their tag and namespace index + /// entries, in a single transaction. + fn purge_one_chunk( + &self, + ns: u32, + chunk: &[(MemoryId, DiskRecord)], + ) -> Result<(), StorageError> { + let write_txn = self.db.begin_write()?; + { + let mut meta = write_txn.open_table(META_TABLE)?; + let mut tags = write_txn.open_multimap_table(TAG_INDEX)?; + let mut ns_idx = write_txn.open_multimap_table(NAMESPACE_INDEX)?; + + for (id, record) in chunk { + let key = id.as_bytes().as_slice(); + meta.remove(key)?; + for tag in &record.tags { + tags.remove(tag.as_str(), key)?; + } + // A tombstoned record was already unlinked from + // NAMESPACE_INDEX; removing again is a no-op. + ns_idx.remove(ns, key)?; + } + } + write_txn.commit()?; + Ok(()) + } +} + +/// A [`MetadataStore::purge_namespace_records`] that stopped part-way. +/// +/// Removal is chunked, so a failure in chunk N leaves chunks `0..N` +/// committed and permanently gone. Carrying that list out with the error +/// is the difference between a caller that can finish the job — free the +/// vector slots and clean the FTS, vector, entity and graph entries those +/// records still occupy — and one that reports a bare failure and strands +/// them in every index for the life of the database, where a retry cannot +/// find them because their `META_TABLE` rows are already gone. +#[derive(Debug)] +pub struct PartialPurge { + /// Records already removed. Permanently destroyed; the caller still + /// owes them index cleanup. + pub destroyed: Vec<(MemoryId, DiskRecord)>, + /// What stopped the purge. + pub source: StorageError, +} + +impl std::fmt::Display for PartialPurge { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{} records were destroyed before the purge failed: {}", + self.destroyed.len(), + self.source + ) + } +} + +impl std::error::Error for PartialPurge { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + Some(&self.source) + } +} + +// ── Batch Tombstoning ─────────────────────────────────────────────── + +/// What [`MetadataStore::batch_tombstone`] did to one requested id. +#[derive(Debug, Clone)] +pub enum TombstoneOutcome { + /// The record was live and has been tombstoned. Carries the record + /// **as it was before the strip**, so the caller still has the + /// vector slot to free and the tags to clean out of the entity + /// index — both of which the stripped record no longer holds. + Tombstoned(DiskRecord), + /// No record exists for this id. + NotFound, + /// The record exists but was already in the Tombstone phase. + AlreadyTombstoned, +} + +impl MetadataStore { + /// Tombstone many memories in a single write transaction. + /// + /// Returns one outcome per input id, in input order. A per-id miss + /// (`NotFound`, `AlreadyTombstoned`) is an outcome, not an error, + /// and does not abort the batch; only a genuine storage fault + /// aborts, in which case the transaction is dropped and *nothing* + /// is tombstoned. + /// + /// # Duplicate ids are safe by construction + /// + /// The second occurrence of an id reads back the first occurrence's + /// uncommitted write from within the same transaction, sees + /// `DecayPhase::Tombstone`, and reports `AlreadyTombstoned`. Since + /// only `Tombstoned` carries a record, a caller that frees a vector + /// slot per `Tombstoned` outcome cannot free the same slot twice — + /// which would build a cycle in the free list and underflow + /// `live_count`. + pub fn batch_tombstone(&self, ids: &[MemoryId]) -> Result, StorageError> { + if ids.is_empty() { + return Ok(Vec::new()); + } + + let mut outcomes = Vec::with_capacity(ids.len()); + + let write_txn = self.db.begin_write()?; + { + let mut meta = write_txn.open_table(META_TABLE)?; + let mut tags = write_txn.open_multimap_table(TAG_INDEX)?; + let mut ns_idx = write_txn.open_multimap_table(NAMESPACE_INDEX)?; + + for id in ids { + let key = id.as_bytes().as_slice(); + + let existing = match meta.get(key)? { + Some(value) => DiskRecord::from_bytes(value.value())?, + None => { + outcomes.push(TombstoneOutcome::NotFound); + continue; + } + }; + + if existing.phase == DecayPhase::Tombstone { + outcomes.push(TombstoneOutcome::AlreadyTombstoned); + continue; + } + + let mut stripped = existing.clone(); + stripped.summary = String::new(); + stripped.tags = Vec::new(); + stripped.text_offset = 0; + stripped.text_length = 0; + stripped.phase = DecayPhase::Tombstone; + stripped.strength = 0.0; + stripped.decay_strength = 0.0; + + meta.insert(key, stripped.to_bytes().as_slice())?; + + for tag in &existing.tags { + tags.remove(tag.as_str(), key)?; + } + ns_idx.remove(existing.namespace_id, key)?; + + outcomes.push(TombstoneOutcome::Tombstoned(existing)); + } + } + write_txn.commit()?; - for id in batch_ids { - if let Some(record) = self.delete(id)? { - all_deleted.push(record); + // Phase bitmaps only after the commit: a bitmap update rolled + // forward against a transaction that never landed would point + // the decay sweep at slots that are still live. + { + let mut pi = self.phase_index.write().unwrap_or_else(|e| e.into_inner()); + for outcome in &outcomes { + if let TombstoneOutcome::Tombstoned(record) = outcome { + pi.remove(record.namespace_id, record.vector_slot, record.phase); } } } - Ok(all_deleted) + Ok(outcomes) } } @@ -1010,3 +1390,349 @@ impl MetadataStore { Ok(table.get(id.as_bytes().as_slice())?.is_some()) } } + +// ═══════════════════════════════════════════════════════════════════════ +// Tests +// ═══════════════════════════════════════════════════════════════════════ + +#[cfg(test)] +mod tests { + use super::*; + use crate::model::Tag; + + fn open_store() -> (tempfile::TempDir, MetadataStore) { + let dir = tempfile::tempdir().expect("tempdir"); + let store = MetadataStore::open(&dir.path().join("meta.db")).expect("open meta.db"); + (dir, store) + } + + /// A live, Full-phase record in `namespace_id` occupying `slot`. + fn record_in(namespace_id: u32, slot: u32, tags: &[&str]) -> DiskRecord { + DiskRecord { + version: DiskRecord::CURRENT_VERSION, + id: [0u8; 16], + namespace_id, + created_at: 0, + last_accessed_at: 0, + phase: DecayPhase::Full, + strength: 1.0, + decay_strength: 1.0, + stability: 1.0, + difficulty: 5.0, + is_permastore: 0, + vector_slot: slot, + edge_count: 0, + summary: "fixture".into(), + tags: tags + .iter() + .map(|t| Tag::new(*t).expect("valid tag")) + .collect(), + access_history: Vec::new(), + text_offset: 0, + text_length: 0, + } + } + + fn insert(store: &MetadataStore, namespace_id: u32, slot: u32, tags: &[&str]) -> MemoryId { + let id = MemoryId::new(); + let mut record = record_in(namespace_id, slot, tags); + record.id = *id.as_bytes(); + store.insert(id, &record).expect("insert"); + id + } + + // ── the non-force emptiness guard ──────────────────────────────── + + /// The check and the effect have to read the same source of truth. + /// `count_memories_in_namespace` walks NAMESPACE_INDEX; the purge + /// scans META_TABLE. A record whose index entry went missing — the + /// corruption `list_memories_filtered` explicitly tolerates — was + /// therefore invisible to the "is this namespace empty?" guard and + /// destroyed anyway by the purge that guard was protecting it from. + #[test] + fn w54_the_emptiness_guard_sees_a_record_the_namespace_index_lost() { + let (_dir, store) = open_store(); + let orphan = insert(&store, 1, 0, &[]); + + // Drop the index entry, keeping the META_TABLE row. + { + let write_txn = store.db.begin_write().expect("begin write"); + { + let mut ns_idx = write_txn + .open_multimap_table(NAMESPACE_INDEX) + .expect("open namespace index"); + ns_idx + .remove(1, orphan.as_bytes().as_slice()) + .expect("remove index entry"); + } + write_txn.commit().expect("commit"); + } + + let ns = NamespaceId::new(1); + assert_eq!( + store + .count_memories_in_namespace(ns) + .expect("index-driven count"), + 0, + "the setup did not actually orphan the record" + ); + assert_eq!( + store + .count_live_records_in_namespace(ns) + .expect("scan-driven count"), + 1, + "the guard cannot see a record the purge will destroy" + ); + assert_eq!( + store.purge_namespace_records(ns, 100).expect("purge").len(), + 1, + "the purge and the guard disagree about what is in the namespace" + ); + } + + /// The rule the guard deliberately keeps: a namespace holding only + /// tombstones is empty, because they report as zero everywhere else + /// and hold no content the user has not already forgotten. + #[test] + fn w55_tombstones_do_not_make_a_namespace_non_empty() { + let (_dir, store) = open_store(); + let dead = insert(&store, 1, 0, &[]); + store.tombstone(dead).expect("tombstone"); + + assert_eq!( + store + .count_live_records_in_namespace(NamespaceId::new(1)) + .expect("count"), + 0 + ); + } + + // ── purge_namespace_records ────────────────────────────────────── + + /// The whole reason this scans META_TABLE instead of walking + /// NAMESPACE_INDEX: `tombstone` unlinks the record from that index, + /// so an index-driven purge cannot see a tombstone — and tombstones + /// are the rows nothing else in the system ever reclaims. + #[test] + fn w01_purge_removes_live_and_tombstoned_rows_of_the_target_namespace_only() { + let (_dir, store) = open_store(); + + let live = insert(&store, 1, 0, &[]); + let dead = insert(&store, 1, 1, &[]); + store.tombstone(dead).expect("tombstone"); + let survivor = insert(&store, 2, 0, &[]); + + let purged = store + .purge_namespace_records(NamespaceId::new(1), 100) + .expect("purge"); + + assert_eq!(purged.len(), 2, "both the live row and the tombstone"); + let purged_ids: Vec = purged.iter().map(|(id, _)| *id).collect(); + assert!(purged_ids.contains(&live)); + assert!(purged_ids.contains(&dead), "the tombstone was not reached"); + + assert!(store.get(live).expect("get").is_none()); + assert!(store.get(dead).expect("get").is_none()); + assert!( + store.get(survivor).expect("get").is_some(), + "a record in another namespace was destroyed" + ); + } + + #[test] + fn w02_purge_clears_the_tag_index() { + let (_dir, store) = open_store(); + + insert(&store, 1, 0, &["topic/doomed"]); + insert(&store, 2, 0, &["topic/kept"]); + + store + .purge_namespace_records(NamespaceId::new(1), 100) + .expect("purge"); + + assert!( + store + .memories_with_tag("topic/doomed") + .expect("tag") + .is_empty(), + "tag index still points at purged records" + ); + assert_eq!(store.memories_with_tag("topic/kept").expect("tag").len(), 1); + } + + #[test] + fn w03_purge_clears_the_namespace_index() { + let (_dir, store) = open_store(); + + insert(&store, 1, 0, &[]); + insert(&store, 1, 1, &[]); + insert(&store, 2, 0, &[]); + + store + .purge_namespace_records(NamespaceId::new(1), 100) + .expect("purge"); + + assert!( + store + .memories_in_namespace(NamespaceId::new(1)) + .expect("ns index") + .is_empty() + ); + assert_eq!( + store + .memories_in_namespace(NamespaceId::new(2)) + .expect("ns index") + .len(), + 1 + ); + } + + #[test] + fn w04_purge_of_an_unknown_or_empty_namespace_is_an_empty_ok() { + let (_dir, store) = open_store(); + insert(&store, 1, 0, &[]); + + assert!( + store + .purge_namespace_records(NamespaceId::new(9), 100) + .expect("purge") + .is_empty() + ); + assert_eq!(store.count().expect("count"), 1, "nothing else was touched"); + } + + /// Chunking is what keeps a large purge off the write lock. It must + /// not also be what makes a large purge incomplete. + #[test] + fn w05_purge_chunking_removes_every_record() { + let (_dir, store) = open_store(); + for slot in 0..5 { + insert(&store, 1, slot, &[]); + } + + let purged = store + .purge_namespace_records(NamespaceId::new(1), 2) + .expect("purge"); + + assert_eq!(purged.len(), 5); + assert_eq!(store.count().expect("count"), 0); + } + + // ── batch_tombstone ────────────────────────────────────────────── + + #[test] + fn w06_batch_tombstone_reports_one_outcome_per_id_in_input_order() { + let (_dir, store) = open_store(); + + let live = insert(&store, 1, 0, &[]); + let already = insert(&store, 1, 1, &[]); + store.tombstone(already).expect("tombstone"); + let missing = MemoryId::new(); + + let outcomes = store + .batch_tombstone(&[missing, live, already]) + .expect("batch tombstone"); + + assert!(matches!(outcomes[0], TombstoneOutcome::NotFound)); + assert!(matches!(outcomes[1], TombstoneOutcome::Tombstoned(_))); + assert!(matches!(outcomes[2], TombstoneOutcome::AlreadyTombstoned)); + } + + /// A double free of one vector slot builds a cycle in the on-disk + /// free list and underflows `live_count`, handing the same slot to + /// two memories. The batch is the only place a caller can hit it, so + /// the batch is where it has to be impossible: the second occurrence + /// reads the first's uncommitted write and reports + /// `AlreadyTombstoned`, which carries no record and so frees no slot. + #[test] + fn w07_a_duplicate_id_in_one_batch_yields_exactly_one_tombstoned_outcome() { + let (_dir, store) = open_store(); + let id = insert(&store, 1, 7, &[]); + + let outcomes = store.batch_tombstone(&[id, id]).expect("batch tombstone"); + + assert!(matches!(outcomes[0], TombstoneOutcome::Tombstoned(_))); + assert!(matches!(outcomes[1], TombstoneOutcome::AlreadyTombstoned)); + assert_eq!( + outcomes + .iter() + .filter(|o| matches!(o, TombstoneOutcome::Tombstoned(_))) + .count(), + 1, + "a caller freeing one slot per Tombstoned outcome would double-free" + ); + } + + /// The batch path is a second implementation of `tombstone`. It has + /// to leave the database in the state the single-record path would. + #[test] + fn w08_batch_tombstone_leaves_the_same_state_as_the_single_record_path() { + let (_dir, store) = open_store(); + + let single = insert(&store, 1, 0, &["topic/alpha"]); + let batched = insert(&store, 1, 1, &["topic/beta"]); + + store.tombstone(single).expect("tombstone"); + let outcomes = store.batch_tombstone(&[batched]).expect("batch tombstone"); + + let TombstoneOutcome::Tombstoned(pre_strip) = &outcomes[0] else { + panic!("expected Tombstoned, got {:?}", outcomes[0]); + }; + assert_eq!( + pre_strip.vector_slot, 1, + "the outcome must carry the PRE-strip record, or the caller has no slot to free" + ); + assert_eq!(pre_strip.tags.len(), 1, "pre-strip tags must survive"); + + for id in [single, batched] { + let record = store.get(id).expect("get").expect("record survives"); + assert_eq!(record.phase, DecayPhase::Tombstone); + assert!(record.summary.is_empty()); + assert!(record.tags.is_empty()); + } + + assert!( + store + .memories_with_tag("topic/alpha") + .expect("tag") + .is_empty() + ); + assert!( + store + .memories_with_tag("topic/beta") + .expect("tag") + .is_empty() + ); + assert!( + store + .memories_in_namespace(NamespaceId::new(1)) + .expect("ns index") + .is_empty() + ); + assert!( + store + .ids_in_phase(DecayPhase::Full) + .expect("phase") + .is_empty(), + "phase bitmaps still claim both slots are live" + ); + } + + #[test] + fn w09_batch_tombstone_of_an_empty_slice_is_an_empty_ok() { + let (_dir, store) = open_store(); + let id = insert(&store, 1, 0, &[]); + + assert!( + store + .batch_tombstone(&[]) + .expect("batch tombstone") + .is_empty() + ); + assert_eq!( + store.get(id).expect("get").expect("record").phase, + DecayPhase::Full, + "an empty batch must not open a transaction that changes anything" + ); + } +} diff --git a/src/storage/vectors.rs b/src/storage/vectors.rs index 0257ece..8fc66dc 100644 --- a/src/storage/vectors.rs +++ b/src/storage/vectors.rs @@ -802,4 +802,86 @@ impl VectorManager { pub fn iter(&self) -> impl Iterator { self.stores.iter().map(|(&id, store)| (id, store)) } + + /// Forget a namespace's store, handing it back to the caller. + /// + /// Returns `None` if the namespace was never opened. + /// + /// # The returned store is still holding the file + /// + /// A [`VectorStore`] owns an fs2 **exclusive advisory lock** on + /// `vectors.dat` and a live `Mmap` over it, and it has no close + /// method — both are released by its `Drop`. A caller that is about + /// to `remove_dir_all` the namespace directory **must drop the + /// returned store first**. On Windows the unlink fails outright + /// while the mapping is open; on Unix it succeeds but the lock and + /// the mapping survive against the now-unlinked inode, so a + /// namespace recreated under the same directory name would be + /// locked out by a file nobody can see. + /// + /// Returning the store rather than dropping it here is deliberate: + /// it puts the drop at the call site, next to the unlink it has to + /// precede. + #[must_use = "the returned store holds an exclusive lock and a live mmap; \ + drop it before removing the namespace directory"] + pub fn remove(&mut self, namespace_id: NamespaceId) -> Option { + self.stores.remove(&namespace_id) + } +} + +// ═══════════════════════════════════════════════════════════════════════ +// Tests +// ═══════════════════════════════════════════════════════════════════════ + +#[cfg(test)] +mod manager_tests { + use super::*; + + const DIM: usize = 4; + + /// `VectorStore` holds an fs2 exclusive advisory lock and a live + /// mmap with no close method, so a namespace deletion that unlinks + /// the directory while the store is still alive leaves the lock held + /// against an inode nobody can see — and the next namespace created + /// under the same name cannot open its own file. Handing the store + /// back and dropping it at the call site is what releases both. + #[test] + fn w11_a_removed_store_releases_its_lock_so_the_directory_can_be_replaced() { + let dir = tempfile::tempdir().expect("tempdir"); + let mut manager = VectorManager::new(dir.path().to_path_buf()); + let ns = NamespaceId::new(1); + + manager + .open_or_create(ns, "scratch", DIM) + .expect("open_or_create") + .insert_vector(&[1.0; DIM]) + .expect("insert vector"); + + let store = manager.remove(ns); + assert!(store.is_some(), "the store was open, so it must come back"); + drop(store); + + let ns_dir = dir.path().join("scratch"); + std::fs::remove_dir_all(&ns_dir).expect("remove namespace directory"); + assert!(!ns_dir.exists()); + + // The real proof: a fresh store opens at the same path. If the + // lock or the mapping had survived, this would fail or hand back + // the old file's contents. + let recreated = manager + .open_or_create(ns, "scratch", DIM) + .expect("recreate at the same path"); + assert_eq!( + recreated.slot_count(), + 0, + "the recreated namespace inherited the deleted one's vectors" + ); + } + + #[test] + fn w12_removing_a_namespace_that_was_never_opened_is_none() { + let dir = tempfile::tempdir().expect("tempdir"); + let mut manager = VectorManager::new(dir.path().to_path_buf()); + assert!(manager.remove(NamespaceId::new(42)).is_none()); + } } diff --git a/src/system.rs b/src/system.rs index 57995cc..d2caa4a 100644 --- a/src/system.rs +++ b/src/system.rs @@ -4,6 +4,7 @@ //! subsystem. Constructed by [`Recalld::new()`], which executes //! the ordered startup sequence. Torn down by [`Recalld::shutdown()`]. +use std::collections::BTreeSet; use std::net::SocketAddr; use std::sync::Arc; use std::sync::atomic::{AtomicU32, Ordering}; @@ -20,12 +21,161 @@ use crate::embedding::{self, EmbeddingProvider}; use crate::error::{RecalldError, Result}; use crate::graph::activation::ActivationConfig; use crate::graph::{self, RelationshipGraph, SharedGraph}; +use crate::model::{DecayPhase, DiskRecord, MemoryId, NamespaceId}; use crate::rif::{RifConfig, RifEngine}; -use crate::search::{EntityIndex, FlatVectorIndex, FtsIndex, QueryEngine}; +use crate::search::{ + EntityIndex, FlatVectorIndex, FtsIndex, QueryEngine, VectorError, VectorIndex as _, + VectorMetadata, +}; use crate::storage::engine::RedbStorageEngine; // Import the StorageEngine trait so its methods are in scope. use crate::storage::StorageEngine as _; +// ═══════════════════════════════════════════════════════════════════════ +// Startup vector-index load +// ═══════════════════════════════════════════════════════════════════════ + +/// What the startup vector load did. +/// +/// Counts rather than per-record logging: a store with a changed +/// embedding width would otherwise emit one warning per memory, and the +/// one number an operator needs — how many — would be the one thing not +/// on the line. +#[derive(Debug, Default, PartialEq, Eq)] +struct VectorLoadStats { + /// Vectors added to the index. + loaded: usize, + /// Records skipped for being tombstones. Tombstones have had their + /// content stripped and are deliberately absent from every search + /// index; loading one would resurrect it as a vector hit. + tombstoned: usize, + /// Records whose `vector_slot` points past the end of their + /// namespace's `vectors.dat`. + missing: usize, + /// Records whose namespace's vector file could not be read at all — + /// in practice, a namespace directory that is gone or was never + /// opened. + unreadable: usize, + /// Records whose stored vector is a different width than the index. + dim_mismatch: usize, + /// Namespaces that produced at least one [`dim_mismatch`], sorted so + /// the log line is stable across boots. + /// + /// [`dim_mismatch`]: VectorLoadStats::dim_mismatch + mismatched_namespaces: BTreeSet, + /// Namespaces that produced at least one [`unreadable`] record. + /// + /// [`unreadable`]: VectorLoadStats::unreadable + unreadable_namespaces: BTreeSet, +} + +impl VectorLoadStats { + /// Every record that did not make it into the index, for the + /// one-line summary. + fn skipped(&self) -> usize { + self.tombstoned + self.missing + self.unreadable + self.dim_mismatch + } +} + +/// Fill `index` from the vectors already on disk. +/// +/// [`FlatVectorIndex`] is pure RAM. It is created empty, and until this +/// existed nothing but the store path ever put anything in it — so every +/// restart began with an empty index and stayed empty until the process +/// stored its first memory. `find_similar_memories` failed outright +/// ("no embedding found for memory "), and `recall_memories` +/// silently degraded to FTS, entity and graph matching, which returns +/// accurate-looking results that ignore semantic similarity entirely. +/// This load is what makes a restart invisible. +/// +/// # Nothing here is fatal +/// +/// A missing vector, an unreadable namespace, a width that does not +/// match: all are counted and skipped. The consequence of skipping is +/// that one memory is not reachable by vector search — which is +/// precisely the state it was in before this function existed, so +/// failing startup over it would trade a degraded system for no system. +/// +/// `records` is the caller's existing `scan_all()`. This function does +/// not scan; it reads one vector per live record. +fn load_vector_index( + storage: &RedbStorageEngine, + records: &[(MemoryId, DiskRecord)], + index: &mut FlatVectorIndex, +) -> VectorLoadStats { + let mut stats = VectorLoadStats::default(); + + // Size the buffer from the live records, not all of them. A store + // that is mostly tombstones would otherwise reserve — and actually + // allocate — several hundred MB it never uses. + let live = records + .iter() + .filter(|(_, r)| r.phase != DecayPhase::Tombstone) + .count(); + index.reserve(live); + + for (memory_id, record) in records { + if record.phase == DecayPhase::Tombstone { + stats.tombstoned += 1; + continue; + } + + let namespace_id = NamespaceId::new(record.namespace_id); + let vector = match storage.get_vector(namespace_id, record.vector_slot) { + Ok(Some(v)) => v, + Ok(None) => { + stats.missing += 1; + continue; + } + Err(_) => { + stats.unreadable += 1; + stats.unreadable_namespaces.insert(record.namespace_id); + continue; + } + }; + + let metadata = VectorMetadata { + namespace_id, + // The phase the record ACTUALLY has. The store path writes a + // hardcoded `Full` and `VectorIndex::update_metadata` has no + // callers, so before this every index-level phase filter + // matched everything and was in effect inert. + // + // KNOWN GAP, stated plainly: this makes the filter correct + // only as of the last restart. A memory that transitions + // Full -> Summary -> Ghost while the process is running + // still carries its boot-time phase in the index, because + // the decay sweep holds no handle on the vector index. + // Giving it one is a separate change; do not paper over it + // here. + decay_phase: record.phase.as_u8(), + // From the record, not from the store request. `DiskRecord` + // tags are already normalized `Tag`s, which is the same form + // the query side derives its filter tags in — so a tag + // filter behaves the same before and after a restart. + tags: record.tags.iter().map(|t| t.to_string()).collect(), + }; + + match index.add(*memory_id, &vector, metadata) { + Ok(()) => stats.loaded += 1, + Err(VectorError::DimensionMismatch { .. }) => { + stats.dim_mismatch += 1; + stats.mismatched_namespaces.insert(record.namespace_id); + } + Err(e) => { + // `add` has no other failure mode today. If it grows + // one, count it where an operator will see it rather + // than dropping the record silently. + stats.unreadable += 1; + stats.unreadable_namespaces.insert(record.namespace_id); + tracing::warn!(memory_id = %memory_id, %e, "vector rejected by the index"); + } + } + } + + stats +} + /// System readiness state. Checked by the health endpoint. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum SystemState { @@ -265,12 +415,64 @@ impl Recalld { Arc::new(engine) }; - // -- Step 6: Create FlatVectorIndex --------------------------- + // -- Step 6: Create FlatVectorIndex and load persisted vectors - let vector_index = { - let index = FlatVectorIndex::new(config.embedding.dimensions); + let mut index = FlatVectorIndex::new(config.embedding.dimensions); + + // The index lives in RAM only; without this the process + // starts semantically blind and stays that way until it + // stores something. See `load_vector_index`. + let started = std::time::Instant::now(); + let stats = match storage.read() { + Ok(storage_r) => load_vector_index(&storage_r, &all_records, &mut index), + Err(e) => { + // Unreachable — step 2 already took this lock and + // would have failed first. Degrade rather than + // refuse to boot, on principle: an empty vector + // index is the old behaviour, not a new outage. + tracing::warn!(%e, "storage lock poisoned; vector index left empty"); + VectorLoadStats::default() + } + }; + let elapsed = started.elapsed(); + + if stats.dim_mismatch > 0 { + tracing::warn!( + skipped = stats.dim_mismatch, + index_dimensions = config.embedding.dimensions, + namespaces = ?stats.mismatched_namespaces, + "stored vectors are a different width than the vector index and were \ + skipped; those memories are unreachable by vector search. Either \ + embedding.dimensions changed since they were written, or the namespace \ + was created with its own embeddingDim" + ); + } + if stats.unreadable > 0 { + tracing::warn!( + skipped = stats.unreadable, + namespaces = ?stats.unreadable_namespaces, + "vectors could not be read and were skipped; those memories are \ + unreachable by vector search" + ); + } + if stats.missing > 0 { + tracing::warn!( + skipped = stats.missing, + "records point at vector slots that do not exist and were skipped; \ + those memories are unreachable by vector search" + ); + } + tracing::info!( dimensions = config.embedding.dimensions, - "flat vector index created" + loaded = stats.loaded, + skipped = stats.skipped(), + skipped_tombstoned = stats.tombstoned, + skipped_missing_slot = stats.missing, + skipped_unreadable = stats.unreadable, + skipped_dimension_mismatch = stats.dim_mismatch, + elapsed_ms = elapsed.as_millis() as u64, + "flat vector index created and loaded from storage" ); Arc::new(RwLock::new(index)) }; @@ -462,7 +664,7 @@ impl Recalld { source: None, })?; let has_default = storage_r - .get_namespace_by_name("default") + .get_namespace_by_name(crate::model::constants::DEFAULT_NAMESPACE_NAME) .map_err(|e| RecalldError::Init { step: "check_default_namespace", message: format!("failed to check default namespace: {}", e), @@ -474,7 +676,7 @@ impl Recalld { if !has_default { let ns_config = crate::model::NamespaceConfig { id: crate::model::NamespaceId::UNSET, - name: "default".to_string(), + name: crate::model::constants::DEFAULT_NAMESPACE_NAME.to_string(), embedding_dim: config.embedding.dimensions as u32, initial_stability: 3.7145, default_difficulty: 5.0, @@ -740,3 +942,288 @@ impl Recalld { Ok(()) } } + +// ═══════════════════════════════════════════════════════════════════════ +// Tests +// ═══════════════════════════════════════════════════════════════════════ + +#[cfg(test)] +mod tests { + use super::*; + use crate::search::SearchFilter; + use crate::test_support::{DIM, Fixture, MemorySpec}; + + fn ns() -> NamespaceId { + NamespaceId::new(1) + } + + /// Run the startup load against a fixture's storage, into a fresh + /// index of `dim` — which is what step 6 does, minus the logging. + fn load(fx: &Fixture, dim: usize) -> (FlatVectorIndex, VectorLoadStats) { + let storage_r = fx.storage.read().expect("storage lock"); + let records = storage_r.scan_all().expect("scan_all"); + let mut index = FlatVectorIndex::new(dim); + let stats = load_vector_index(&storage_r, &records, &mut index); + (index, stats) + } + + /// A config pointed at `dir` that will not talk to the network or + /// start a sweep. + fn offline_config(dir: &std::path::Path) -> RecalldConfig { + let mut config = RecalldConfig::default(); + config.storage.data_dir = dir.to_str().expect("utf-8 temp dir").to_string(); + config.embedding.provider = crate::config::EmbeddingProvider::Passthrough; + config.embedding.dimensions = DIM; + config.decay.disable_sweep = true; + config + } + + // ── the regression ─────────────────────────────────────────────── + + /// THE bug. `FlatVectorIndex` is RAM-only and was only ever filled + /// by the store path, so every restart came up with an empty index: + /// `find_similar_memories` failed with "no embedding found for + /// memory " and `recall_memories` quietly fell back to FTS and + /// graph matching until something was stored in that process. + /// + /// This drives the real `Recalld::new`, not the helper, because the + /// helper is not what was broken — the wiring was. Delete the load + /// from step 6 and this fails with `0 == 3`. + #[tokio::test] + async fn a_restart_reloads_the_vector_index_from_disk() { + let dir = { + let fx = Fixture::new(); + for _ in 0..3 { + fx.insert_memory().await; + } + fx.close() + }; + + let system = Recalld::new(offline_config(dir.path())) + .await + .expect("system starts over the existing store"); + + assert_eq!( + system.vector_index().read().await.len(), + 3, + "the three memories on disk should be searchable by vector \ + immediately, before this process stores anything" + ); + } + + /// The same property one layer down, where the counts are visible. + #[tokio::test] + async fn the_load_reads_vectors_that_outlived_the_engine_that_wrote_them() { + let dir = { + let fx = Fixture::new(); + fx.insert_memory().await; + fx.insert_memory().await; + fx.close() + }; + + let engine = RedbStorageEngine::open(dir.path()).expect("reopen engine"); + let records = engine.scan_all().expect("scan_all"); + let mut index = FlatVectorIndex::new(DIM); + let stats = load_vector_index(&engine, &records, &mut index); + + assert_eq!(stats.loaded, 2); + assert_eq!(stats.skipped(), 0); + assert_eq!(index.len(), 2); + } + + // ── what gets skipped ──────────────────────────────────────────── + + /// A tombstone has had its content stripped and is deliberately + /// absent from every search index. Loading its vector would + /// resurrect it as a similarity hit. + #[tokio::test] + async fn a_tombstoned_memory_is_not_loaded() { + let fx = Fixture::new(); + let live = fx.insert_memory().await; + let dead = fx.insert_memory().await; + fx.tombstone(dead); + + let (index, stats) = load(&fx, DIM); + + assert_eq!(stats.loaded, 1); + assert_eq!(stats.tombstoned, 1); + assert!(index.get_vector(live).is_some()); + assert!( + index.get_vector(dead).is_none(), + "a tombstone must not come back as a vector hit" + ); + } + + /// A record pointing past the end of its namespace's `vectors.dat` — + /// what a truncated or replaced vector file leaves behind. The + /// memory is simply not vector-searchable, which is what it was + /// before this load existed; startup carries on. + #[tokio::test] + async fn a_record_whose_vector_is_gone_is_skipped_and_startup_continues() { + let fx = Fixture::new(); + let good = fx.insert_memory().await; + fx.insert_memory().await; + + let storage_r = fx.storage.read().expect("storage lock"); + let mut records = storage_r.scan_all().expect("scan_all"); + // Point one record at a slot that was never allocated. + let orphan = records + .iter_mut() + .find(|(id, _)| *id != good) + .expect("two records"); + orphan.1.vector_slot = 9_999; + + let mut index = FlatVectorIndex::new(DIM); + let stats = load_vector_index(&storage_r, &records, &mut index); + + assert_eq!(stats.missing, 1); + assert_eq!(stats.loaded, 1); + assert!( + index.get_vector(good).is_some(), + "one bad record must not cost the others their vectors" + ); + } + + /// A record in a namespace that no longer has a vector store at all. + #[tokio::test] + async fn a_record_in_an_unreadable_namespace_is_skipped() { + let fx = Fixture::new(); + fx.insert_memory().await; + + let storage_r = fx.storage.read().expect("storage lock"); + let mut records = storage_r.scan_all().expect("scan_all"); + records[0].1.namespace_id = 4_242; + + let mut index = FlatVectorIndex::new(DIM); + let stats = load_vector_index(&storage_r, &records, &mut index); + + assert_eq!(stats.unreadable, 1); + assert_eq!(stats.loaded, 0); + assert!(stats.unreadable_namespaces.contains(&4_242)); + assert_eq!(index.len(), 0); + } + + // ── the real decay phase ───────────────────────────────────────── + + /// The store path hardcodes `Full`, so every index-level phase + /// filter was inert. Loading the record's true phase is what makes + /// the filter mean something. + #[tokio::test] + async fn the_loaded_entry_carries_the_records_real_phase() { + let fx = Fixture::new(); + let ghost = fx + .insert_memory_with(MemorySpec { + phase: DecayPhase::Ghost, + ..MemorySpec::new(ns()) + }) + .await; + let summary = fx + .insert_memory_with(MemorySpec { + phase: DecayPhase::Summary, + ..MemorySpec::new(ns()) + }) + .await; + let full = fx.insert_memory().await; + + let (index, stats) = load(&fx, DIM); + assert_eq!(stats.loaded, 3); + + // Asserted through the filter, because the filter is the point. + let only = |phase: DecayPhase| { + let filter = SearchFilter { + decay_phases: Some(vec![phase.as_u8()]), + ..SearchFilter::default() + }; + let hits = index + .search(&[1.0, 0.0, 0.0, 0.0], 10, &filter) + .expect("search"); + hits.into_iter().map(|h| h.id).collect::>() + }; + + assert_eq!(only(DecayPhase::Ghost), vec![ghost]); + assert_eq!(only(DecayPhase::Summary), vec![summary]); + assert_eq!(only(DecayPhase::Full), vec![full]); + } + + // ── per-namespace widths ───────────────────────────────────────── + + /// The index has one width; namespaces each have their own. A stored + /// vector of a different width cannot go in, and that is a warning, + /// not a failed boot. + #[tokio::test] + async fn vectors_of_the_wrong_width_are_counted_not_fatal() { + let fx = Fixture::new(); + let narrow = fx.insert_memory().await; + + let wide_ns = fx.with_namespace("wide", 8); + fx.insert_memory_with(MemorySpec { + embedding: Some(&[0.0; 8]), + ..MemorySpec::new(wide_ns) + }) + .await; + + let (index, stats) = load(&fx, DIM); + + assert_eq!(stats.loaded, 1, "the matching-width memory still loads"); + assert_eq!(stats.dim_mismatch, 1); + assert_eq!( + stats.mismatched_namespaces, + [wide_ns.get()].into_iter().collect(), + "the warning names the namespace an operator has to go look at" + ); + assert!(index.get_vector(narrow).is_some()); + } + + /// The other route to the same state, and the likelier one: the + /// operator changed `embedding.dimensions` between runs. Every + /// stored vector is now the wrong width. + #[tokio::test] + async fn a_changed_embedding_width_skips_everything_without_failing() { + let fx = Fixture::new(); + fx.insert_memory().await; + fx.insert_memory().await; + + let (index, stats) = load(&fx, DIM * 2); + + assert_eq!(stats.loaded, 0); + assert_eq!(stats.dim_mismatch, 2); + assert_eq!(index.len(), 0); + } + + // ── end to end ─────────────────────────────────────────────────── + + /// The property the bug report was actually about: after a restart, + /// asking for a memory's neighbours works. `find_similar` fails at + /// `vector_indexes.get_vector(id)` — the same lookup this asserts — + /// so an index that has the vector is the whole of the fix. + #[tokio::test] + async fn a_stored_memory_still_has_its_embedding_after_a_restart() { + let (dir, id) = { + let fx = Fixture::new(); + let id = fx + .insert_memory_with(MemorySpec { + embedding: Some(&[1.0, 0.0, 0.0, 0.0]), + ..MemorySpec::new(ns()) + }) + .await; + (fx.close(), id) + }; + + let system = Recalld::new(offline_config(dir.path())) + .await + .expect("system starts over the existing store"); + + let index = system.vector_index().read().await; + assert_eq!( + index.get_vector(id), + Some(vec![1.0, 0.0, 0.0, 0.0]), + "find_similar_memories resolves the source embedding through \ + exactly this lookup; None is the 'no embedding found for \ + memory ' error the user saw" + ); + let hits = index + .search(&[1.0, 0.0, 0.0, 0.0], 5, &SearchFilter::default()) + .expect("search"); + assert_eq!(hits.first().map(|h| h.id), Some(id)); + } +} diff --git a/src/test_support.rs b/src/test_support.rs new file mode 100644 index 0000000..a9401cb --- /dev/null +++ b/src/test_support.rs @@ -0,0 +1,497 @@ +//! Shared test fixtures for tests that need live storage. +//! +//! A unit test that only needs a struct builds one inline. This module is for +//! the other kind: tests whose subject is what the *system* does — an edge +//! counter that has to survive a redb write, a namespace statistic that has to +//! agree with what `list_memories` would return. Those need a real +//! [`RedbStorageEngine`] over a real temp directory, and standing one up is +//! ~50 lines that would otherwise be copied per module and drift. +//! +//! Compiled only under `cfg(test)`. + +// A fixture accessor is dead code from the point of view of any single test +// module -- the point of the module is that different callers use different +// subsets. Warning on that would just push each addition to grow a fake +// caller. +#![allow(dead_code)] + +use std::path::Path; +use std::sync::Arc; + +use crate::graph::SharedGraph; +use crate::model::{DecayPhase, MemoryId, NamespaceId}; +use crate::storage::engine::RedbStorageEngine; +use crate::storage::{PersistedEdge, StorageEngine as StorageEngineTrait}; + +/// Embedding width used by the fixture namespace. Small on purpose: +/// these tests are about bookkeeping, not vectors. +pub(crate) const DIM: usize = 4; + +/// The namespace the fixture's memories are stored in by default. +fn ns() -> NamespaceId { + NamespaceId::new(1) +} + +/// What to write for one fixture memory. +/// +/// Every field has a sensible default, so a test states only what it is +/// actually about. Use struct-update syntax: +/// +/// ```ignore +/// let spec = MemorySpec { tags: &["type/decision"], ..MemorySpec::new(ns) }; +/// ``` +pub(crate) struct MemorySpec<'a> { + pub(crate) namespace: NamespaceId, + pub(crate) summary: &'a str, + /// Written verbatim to `DiskRecord.tags`, prefixes included — this is + /// the on-disk form, not the API form. + pub(crate) tags: &'a [&'a str], + pub(crate) phase: DecayPhase, + pub(crate) decay_strength: f32, + pub(crate) is_permastore: bool, + pub(crate) edge_count: u16, + /// The embedding to write. `None` means [`DIM`] zeros, which is what + /// a test that is not about vectors wants. Set it when the namespace + /// has a width of its own — `insert_vector` rejects anything that is + /// not exactly the namespace's width. + pub(crate) embedding: Option<&'a [f32]>, +} + +impl MemorySpec<'_> { + /// A live, full-phase, full-strength memory with no tags. + pub(crate) fn new(namespace: NamespaceId) -> Self { + Self { + namespace, + summary: "fixture", + tags: &[], + phase: DecayPhase::Full, + decay_strength: 1.0, + is_permastore: false, + edge_count: 0, + embedding: None, + } + } +} + +pub(crate) struct Fixture { + pub(crate) graph: SharedGraph, + pub(crate) storage: Arc>, + pub(crate) cache: Arc, + /// Held on the fixture, not built per adapter call, so a test can + /// assert on the same index the adapter under test just wrote to. + pub(crate) vector_index: Arc>, + /// See [`vector_index`](Self::vector_index). + pub(crate) fts_index: Arc>, + /// See [`vector_index`](Self::vector_index). + pub(crate) entity_index: Arc>, + _dir: tempfile::TempDir, +} + +impl Fixture { + pub(crate) fn new() -> Self { + let dir = tempfile::tempdir().expect("tempdir"); + let mut engine = RedbStorageEngine::open(dir.path()).expect("open storage"); + let mut config = crate::model::NamespaceConfig::default_namespace(0); + config.embedding_dim = DIM as u32; + if engine + .get_namespace_by_name(&config.name) + .expect("namespace lookup") + .is_none() + { + engine.create_namespace(&config).expect("create namespace"); + } + + let fts_index = crate::search::FtsIndex::new(dir.path()).expect("open fts index"); + + Self { + graph: Arc::new(tokio::sync::RwLock::new( + crate::graph::RelationshipGraph::new(), + )), + storage: Arc::new(std::sync::RwLock::new(engine)), + cache: Arc::new(crate::cache::CacheManager::new( + crate::cache::CacheConfig { + embedding_dim: DIM, + ..Default::default() + }, + None, + )), + vector_index: Arc::new(tokio::sync::RwLock::new( + crate::search::FlatVectorIndex::new(DIM), + )), + fts_index: Arc::new(tokio::sync::Mutex::new(fts_index)), + entity_index: Arc::new(tokio::sync::RwLock::new(crate::search::EntityIndex::new())), + _dir: dir, + } + } + + /// The temp directory backing this fixture's storage files. Also the + /// storage engine's `db_path`, so namespace directories hang + /// directly off it. + pub(crate) fn path(&self) -> &Path { + self._dir.path() + } + + /// Where a namespace's `vectors.dat` directory lives. + pub(crate) fn namespace_dir(&self, name: &str) -> std::path::PathBuf { + self.path().join(name) + } + + /// Shut this fixture's storage down and hand back the directory, + /// still alive, for someone else to open. + /// + /// This is how a test spells "restart". A [`RedbStorageEngine`] + /// holds an exclusive `recalld.lock` and a live mmap per namespace + /// for its entire lifetime, so nothing — not a second engine, not a + /// whole `Recalld` — can open the same directory until this one is + /// dropped. Consuming the fixture is what guarantees the drop + /// happens, and returning the [`tempfile::TempDir`] is what keeps + /// the files from going with it. + pub(crate) fn close(self) -> tempfile::TempDir { + let Self { + graph, + storage, + cache, + vector_index, + fts_index, + entity_index, + _dir, + } = self; + drop((graph, cache, vector_index, fts_index, entity_index)); + + // Not a formality: if a test built an adapter and kept it, the + // engine would outlive this call and the next open would fail + // with `DatabaseLocked` somewhere far from the cause. + let engine = Arc::try_unwrap(storage).unwrap_or_else(|_| { + panic!( + "close(): something still holds the storage Arc, so the engine's file \ + locks would outlive this call and the next open would fail" + ) + }); + drop(engine); + + _dir + } + + /// Create an extra namespace, so a test can prove a per-namespace + /// count really is per-namespace. + pub(crate) fn create_namespace(&self, name: &str) -> NamespaceId { + self.with_namespace(name, DIM) + } + + /// [`create_namespace`](Self::create_namespace) with an explicit + /// embedding width. + pub(crate) fn with_namespace(&self, name: &str, dim: usize) -> NamespaceId { + let mut config = crate::model::NamespaceConfig::default_namespace(0); + config.name = name.to_string(); + config.embedding_dim = dim as u32; + self.storage + .write() + .expect("storage lock") + .create_namespace(&config) + .expect("create namespace") + } + + /// The on-disk record for `id`, or `None` if it was hard-deleted. + pub(crate) fn record(&self, id: MemoryId) -> Option { + self.storage + .read() + .expect("storage lock") + .get_record(id) + .expect("get record") + } + + /// Writes a memory to storage and gives it a graph node, which is + /// the state an explicit edge is applied against. + pub(crate) async fn insert_memory(&self) -> MemoryId { + self.insert_memory_with(MemorySpec::new(ns())).await + } + + /// A live memory in `namespace` carrying `tags`. + pub(crate) async fn insert_memory_in(&self, namespace: NamespaceId, tags: &[&str]) -> MemoryId { + self.insert_memory_with(MemorySpec { + tags, + ..MemorySpec::new(namespace) + }) + .await + } + + /// [`insert_memory`](Self::insert_memory) with every field a test might + /// need to vary. + pub(crate) async fn insert_memory_with(&self, spec: MemorySpec<'_>) -> MemoryId { + let id = MemoryId::new(); + let mut record = crate::model::DiskRecord { + version: crate::model::DiskRecord::CURRENT_VERSION, + id: *id.as_bytes(), + namespace_id: spec.namespace.get(), + created_at: 0, + last_accessed_at: 0, + phase: spec.phase, + strength: spec.decay_strength, + decay_strength: spec.decay_strength, + stability: 1.0, + difficulty: 5.0, + is_permastore: u8::from(spec.is_permastore), + vector_slot: 0, + edge_count: spec.edge_count, + summary: spec.summary.into(), + tags: spec + .tags + .iter() + .map(|t| crate::model::Tag::new(*t).expect("fixture tag is valid")) + .collect(), + access_history: Vec::new(), + text_offset: 0, + text_length: 0, + }; + + { + let zeros = [0.0; DIM]; + let embedding = spec.embedding.unwrap_or(&zeros); + let mut storage_w = self.storage.write().expect("storage lock"); + storage_w + .insert_memory(id, spec.namespace, &mut record, embedding, None) + .expect("insert memory"); + } + + let slot = record.vector_slot; + self.graph + .write() + .await + .add_node(id, spec.namespace, spec.phase, spec.decay_strength, slot) + .expect("add node"); + id + } + + /// Delete a memory the way the product does. + /// + /// Calls [`RedbStorageEngine::tombstone_memory`], which is what + /// `McpStorageAdapter::delete_memory` calls — going straight to + /// `MetadataStore::tombstone` would skip whatever the engine layer does + /// and leave the fixture in a state the running system never reaches. + pub(crate) fn tombstone(&self, id: MemoryId) { + self.storage + .read() + .expect("storage lock") + .tombstone_memory(id) + .expect("tombstone memory"); + } + + /// The MCP namespace adapter over this fixture's storage. + pub(crate) fn namespace_adapter(&self) -> crate::mcp::bridge_adapters::McpNamespaceAdapter { + crate::mcp::bridge_adapters::McpNamespaceAdapter::new( + self.storage.clone(), + self.cache.clone(), + self.vector_index.clone(), + self.fts_index.clone(), + self.entity_index.clone(), + self.graph.clone(), + chrono_tz::UTC, + ) + } + + /// The MCP storage adapter over this fixture's storage. + /// + /// `McpStorageAdapter::new` takes nine subsystems. They are built + /// here once — empty and real, not mocked — and the index handles + /// come off the fixture, so a test can assert against the very index + /// the adapter wrote to. + pub(crate) fn storage_adapter(&self) -> crate::mcp::bridge_adapters::McpStorageAdapter { + crate::mcp::bridge_adapters::McpStorageAdapter::new( + self.storage.clone(), + self.cache.clone(), + Arc::new(crate::embedding::PassthroughProvider::new(DIM)), + self.vector_index.clone(), + self.fts_index.clone(), + self.entity_index.clone(), + self.graph.clone(), + Arc::new(crate::config::RecalldConfig::default()), + chrono_tz::UTC, + ) + } + + pub(crate) fn edge_count(&self, id: MemoryId) -> u16 { + self.storage + .read() + .expect("storage lock") + .get_record(id) + .expect("get record") + .expect("record exists") + .edge_count + } + + pub(crate) fn persisted_edges(&self) -> Vec { + self.storage + .read() + .expect("storage lock") + .load_all_edges() + .expect("load edges") + } +} + +// ═══════════════════════════════════════════════════════════════════════ +// Stub MCP bridge +// ═══════════════════════════════════════════════════════════════════════ + +/// A bridge whose every operation panics, except the ones a test asks +/// for. +/// +/// The subject is the tool handler's own behaviour — the argument +/// checking and the error rendering it does BEFORE and AFTER the bridge +/// call. `unreachable!()` on everything else is the assertion: a handler +/// that was supposed to reject its arguments and instead ran the query +/// fails loudly rather than silently returning an empty result that +/// looks like a pass. +#[derive(Default)] +pub(crate) struct StubBridge { + /// What `delete_namespace` returns instead of doing anything. + pub(crate) delete_namespace_error: Option, + /// What `delete_memories` returns instead of doing anything. + /// Deliberately settable to a WRONG length, which is the contract + /// violation the handler has to notice. + pub(crate) delete_memories_result: Option>, +} + +impl StubBridge { + /// Wrap this stub in an [`McpBridge`](crate::mcp::bridge::McpBridge) + /// with `default` as the session namespace. + pub(crate) fn into_bridge(self) -> crate::mcp::bridge::McpBridge { + let shared = Arc::new(self); + crate::mcp::bridge::McpBridge { + search: shared.clone(), + storage: shared.clone(), + namespaces: shared.clone(), + health: shared, + default_namespace: crate::model::constants::DEFAULT_NAMESPACE_NAME.to_string(), + timezone: chrono_tz::UTC, + } + } +} + +#[async_trait::async_trait] +impl crate::mcp::bridge::SearchPipeline for StubBridge { + async fn search( + &self, + _: crate::mcp::bridge::SearchInput, + ) -> Result { + unreachable!("the handler was expected to reject its arguments before searching") + } + async fn find_similar( + &self, + _: MemoryId, + _: usize, + _: Option, + _: bool, + ) -> Result, crate::mcp::bridge::BridgeError> { + unreachable!("the handler was expected to reject its arguments before searching") + } + async fn scan_duplicates( + &self, + _: &str, + _: f32, + _: usize, + ) -> Result, crate::mcp::bridge::BridgeError> { + unreachable!("the handler was expected to reject its arguments before scanning") + } +} + +#[async_trait::async_trait] +impl crate::mcp::bridge::StorageEngine for StubBridge { + async fn store_memory( + &self, + _: crate::mcp::bridge::StoreInput, + ) -> Result { + unreachable!("the handler was expected to reject its arguments before storing") + } + async fn get_memory( + &self, + _: MemoryId, + ) -> Result, crate::mcp::bridge::BridgeError> { + unreachable!("the handler was expected to reject its arguments before reading") + } + async fn delete_memory(&self, _: MemoryId) -> Result { + unreachable!("the handler was expected to reject its arguments before deleting") + } + async fn delete_memories( + &self, + _: &[MemoryId], + ) -> Result, crate::mcp::bridge::BridgeError> { + match &self.delete_memories_result { + Some(flags) => Ok(flags.clone()), + None => { + unreachable!("the handler was expected to reject its arguments before deleting") + } + } + } + async fn reinforce_memory( + &self, + _: MemoryId, + _: u8, + ) -> Result { + unreachable!("the handler was expected to reject its arguments before reinforcing") + } + async fn list_memories( + &self, + _: crate::mcp::bridge::ListMemoriesInput, + ) -> Result { + unreachable!("the handler was expected to reject its arguments before listing") + } + async fn list_tags( + &self, + _: crate::mcp::bridge::ListTagsInput, + ) -> Result { + unreachable!("the handler was expected to reject its arguments before listing") + } +} + +#[async_trait::async_trait] +impl crate::mcp::bridge::NamespaceRegistry for StubBridge { + async fn list_namespaces( + &self, + ) -> Result, crate::mcp::bridge::BridgeError> { + unreachable!("the handler was expected to reject its arguments before listing") + } + async fn create_namespace( + &self, + _: crate::mcp::bridge::CreateNamespaceInput, + ) -> Result { + unreachable!("the handler was expected to reject its arguments before creating") + } + async fn namespace_stats( + &self, + _: &str, + ) -> Result { + unreachable!("the handler was expected to reject its arguments before reading stats") + } + async fn delete_namespace( + &self, + _: crate::mcp::bridge::DeleteNamespaceInput, + ) -> Result { + match &self.delete_namespace_error { + Some(e) => Err(clone_bridge_error(e)), + None => { + unreachable!("the handler was expected to reject its arguments before deleting") + } + } + } +} + +#[async_trait::async_trait] +impl crate::mcp::bridge::HealthChecker for StubBridge { + async fn check_health(&self) -> crate::mcp::bridge::HealthStatus { + unreachable!("no test in this module checks health") + } +} + +/// `BridgeError` is not `Clone`; this is enough of one for a stub. +fn clone_bridge_error(e: &crate::mcp::bridge::BridgeError) -> crate::mcp::bridge::BridgeError { + use crate::mcp::bridge::BridgeError as E; + match e { + E::NotFound(m) => E::NotFound(m.clone()), + E::InvalidInput(m) => E::InvalidInput(m.clone()), + E::Storage(m) => E::Storage(m.clone()), + E::Search(m) => E::Search(m.clone()), + E::Internal(m) => E::Internal(m.clone()), + E::TooLarge(m) => E::TooLarge(m.clone()), + E::PartiallyApplied(m) => E::PartiallyApplied(m.clone()), + } +}