diff --git a/.claude/request-composition.md b/.claude/request-composition.md new file mode 100644 index 00000000..98b62804 --- /dev/null +++ b/.claude/request-composition.md @@ -0,0 +1,225 @@ +# Request Composition and the Caching Contract + +Every request to the Anthropic API is assembled from three regions, in this order: `tools`, +`system`, `messages`. Prompt caching matches on an exact prefix, so a change to any region +invalidates that region and everything after it. Tools sit first, which is why a single toggled +tool costs the entire cached prompt. + +This document is the reference for what goes into each region, who builds it, and what can change +it. It exists so that a change to any of those things can be checked against the caching contract +before it ships, rather than discovered later as a cost increase. + +**Keeping it current.** Any change that adds a content block, adds a source of prompt text, moves a +cache breakpoint, or introduces a new way for one of the regions to vary belongs here in the same +change. The invariants near the end are the part that must not silently rot: if a change breaks one, +either the change is wrong or the invariant needs re-deciding on purpose. + +## The four cache breakpoints + +The API allows at most four `cache_control` breakpoints per request. All four are spent, and there +is no headroom. They are applied in `RequestBuilder.buildRequestParams` against a request-only clone +of the conversation (`Conversation.cloneForRequest`), so no marker is ever stored in history. + +| # | Position | Applied by | Behaviour | +|---|----------|-----------|-----------| +| 1 | Last tool in the `tools` array | `buildRequestParams` | Static. Covers every tool. | +| 2 | Last block of the `system` array | `buildRequestParams` | Static. Covers tools plus system. | +| 3 | `content[cachedReminders.length - 1]` of the first user message | `cacheClaudeMdPrefix` | Pinned. Covers the skill catalogue and CLAUDE.md. | +| 4 | Last non-thinking block of the last user message | `cacheLastUserMessage` | Moves forward each turn, so only new content is a write. | + +Breakpoint 3 exists only when there is cached reminder content. With CLAUDE.md present the request +spends all four, so any future feature needing a fifth breakpoint is rejected by the API and needs a +different design. + +Breakpoint 3 is positional, not content-addressed. It counts blocks using the live +`cachedReminders.length` and applies that index to frozen history. A sanity guard refuses to mark a +block that is not a ``, but the guard cannot tell CLAUDE.md apart from any other +reminder, so it catches an unexpected shape and nothing more. + +## Region 1: tools + +Built in `DurableConfigFactory.#build()`, converted to wire form in `RequestBuilder.toWireTool`, then +passed through `transformTool`. + +Order on the wire is server tools first, then client tools. + +**Server tools** come from `buildServerTools`: `web_search` and `web_fetch`, each carrying its +version and `allowed_callers`. When advanced tool use is enabled, `tool_search_tool_regex` or +`tool_search_tool_bm25` is appended. + +**Client tools** are the array `createAppTools` builds once at startup, filtered by +`config.disabledTools`. Each becomes `{ name, description, input_schema, input_examples }`, where the +schema is generated from the tool's zod type. `transformTool` is +`withPathNote(buildAtuTransform(...))`: the ATU transform adds `defer_loading` and `allowed_callers` +when advanced tool use is on and strips `input_examples` when it is off, and the path note appends a +normalisation sentence to every `isPath`-marked schema field. + +No tool description or schema is derived from live session state. Az and Azure DevOps account +configuration in particular never reaches a schema. + +### What changes this region + +Fixed for the life of the process: + +- `tools.exec`, `tools.execV2`, `tools.execV3` +- `skillDirs` +- `tsAvailable`, which is `tsserverPath != null`. When typescript cannot be resolved the four TS + tools are left out entirely, so a SEA launch and a dev launch can send different tool sets. + +Changeable at runtime through config reload, with no restart: + +- `disabledTools` +- `serverTools.webSearch` and `serverTools.webFetch`, including version and allowed callers +- `advancedTools.enabled`, `.searchTool`, `.codeExecutionTool`, `.allowProgrammaticExecution` + +The advanced tool use toggle is the widest of these because it rewrites every tool object, not just +the array membership. + +## Region 2: system + +Assembled in `RequestBuilder.buildRequestParams` as `[AGENT_SDK_PREFIX, ...options.systemPrompts]`, +where `systemPrompts` comes from `DurableConfigFactory`. + +1. `AGENT_SDK_PREFIX`, a constant. +2. The `` block, if an identity body is present. +3. The output of `composeSystemPrompts`: SYSTEM.md sections in source order (user, project, + projectClaude, local) each wrapped in ``, then `config.systemPrompt.text`, then the + `--system` flag text wrapped in ``. + +Item 3 is resolved once per session by `resolveSystemPromptsFor(sessionId)` and reused until the +session id changes. Item 2 is read from disk on every turn by `TurnCoordinator.runTurn` and pushed in +through `updateIdentityBody`. + +### What changes this region + +- The identity file changing on disk, picked up on the next turn. +- A new session id, which re-resolves SYSTEM.md, the config text and the flag. +- `--system` at launch, and `config.systemPrompt.*` at reload. + +The identity sits at position 2, ahead of everything else. Reordering it would not help on its own: +with one breakpoint at the end of the region, a change anywhere in the region costs the whole region. +Confining the loss would require a breakpoint between the stable and volatile parts, and there is +none spare. + +## Region 3: messages + +The array sent is `trimToLastCompaction(items)`, deep-cloned per request. A compaction therefore +replaces the message array wholesale rather than appending to it. + +### Block types on the wire + +User-role messages carry: + +- `text` for `` blocks, the clock stamp, and the operator's own prompt +- `tool_result`, whose content is a text block followed by any native `document` or `image` + attachments a tool returned +- `compaction` blocks, converted to `text` when compaction is disabled + +Assistant-role messages carry `text`, `thinking`, `redacted_thinking`, `tool_use`, +`server_tool_use`, `compaction`, and the server tool result types: `web_search_tool_result`, +`web_fetch_tool_result`, `code_execution_tool_result`, `bash_code_execution_tool_result`, +`text_editor_code_execution_tool_result`, `tool_search_tool_result`, `mcp_tool_result`. The mapping +lives in `TurnRunner.mapBlock`. + +### The reminder inventory + +Seven producers put `` blocks into messages. Position relative to breakpoints 3 and +4 is what decides whether a reminder is cached, so it is the column that matters. + +| Reminder | Source | Read when | Placement | In history | Relative to marker | +|----------|--------|-----------|-----------|------------|--------------------| +| Skill catalogue | `resolveSkillCatalogue()` | Startup, once | Leading, first user message | Yes | Inside breakpoint 3 | +| CLAUDE.md and `--claudeMd` | `ClaudeMdLoader.getContent()` | Every turn | Leading, first user message | Yes | Breakpoint 3 sits on it | +| Scratchpad | `DurableConfigFactory.#conversationReminders()` | Derived per config read | Leading, after the cached run | Yes | After breakpoint 3 | +| Skill catalogue delta | `SkillCatalogueTracker.scanForDelta()` | Per query | Leading, that query's message | Yes | After breakpoint 3 | +| Working directory | `CwdTracker.scanForDelta()` | Per query | Leading, that query's message | Yes | After breakpoint 3 | +| Git delta | `GitStateMonitor.getDelta()` | Per query | Trailing | No | After breakpoint 4, deliberately uncached | +| Clock stamp | `TurnRunner` | Every turn, on the tip | After the leading run, before real content | Yes | After breakpoint 3 | + +`QueryRunner` composes the leading run in breakpoint order: cached reminders first, then +conversation reminders, then the query's own persisted-leading reminders. That order is what keeps +breakpoint 3 on the last cached block. + +The clock stamp is written by `TurnRunner` directly into the conversation tip before the request +clone is taken. It skips any leading `tool_result` and any leading `` so it lands +immediately before the message's real content, and a stale stamp from a rolled-back attempt is +stripped before the new one goes in. A tip that is nothing but a leading run, which is what a +tool-loop continuation looks like, is left unstamped. + +### What rewrites message history + +These change messages that have already been sent, so they move content under the cache: + +- `TurnRunner` calls `healDanglingToolUse` before every request +- `QueryRunner.removeLast()` on the empty-tool-use retry, up to twice +- Role-alternation merge in `Conversation.push`: a cancelled query followed by a new prompt merges + two user messages into one that was already on the wire +- Compaction, which replaces the whole slice +- `ConversationSession` calling `setHistory` on resume or new session +- `Conversation.remove(id)` exists on the interface for tagged pruning but has no CLI caller yet + +## Outside the three regions + +These are not part of the message prefix but are part of the cache key or otherwise invalidate it. +They are worth listing because several are reachable by a single keypress. + +- `model`. Changed by the `--model` flag, by config reload, or at runtime through + `ModelOverrides.setModel`. A model change is a total miss. +- `thinking` and `output_config.effort`. Bound to the `t` and `e` keys in command mode via + `ModelOverrides.cycleThinking` and `cycleEffort`. Changing thinking parameters invalidates cached + message blocks. This is documented Anthropic behaviour rather than something proven in this + codebase, and should be confirmed before being relied on. +- `betas`. The advanced tool use flag changes both the beta header and the shape of every tool. +- `context_management` edits and the `compact` config. +- `cacheTtl`, currently hardcoded to one hour in `DurableConfigFactory`. + +## Invariants + +Break one of these and caching degrades quietly. Nothing fails loudly. + +1. **Ephemeral reminders must be trailing.** `RequestBuilder` honours `position: 'leading'` for an + ephemeral reminder by unshifting it onto the head of the last user message, which puts it inside + breakpoint 4's prefix. Because it is not persisted, the next turn sends that message without it, + the prefix no longer matches, and the whole conversation tail falls out of cache. Nothing produces + a leading ephemeral reminder today. Nothing should. +2. **Anything read per turn must not reach regions 1 or 2.** A file read on every turn is a file that + can change between turns, and a change in tools or system costs everything downstream. +3. **The leading reminder run stays in breakpoint order.** Cached reminders first, then anything that + varies. A new reminder added ahead of the cached run moves breakpoint 3 onto the wrong block. +4. **`cachedReminders.length` must agree with what is frozen in the first user message.** Breakpoint + 3 is an index, not a match. If the count changes while history does not, the marked prefix stops + corresponding to what was cached. +5. **Availability is not membership.** Telling the model a tool cannot be used belongs in a reminder + after the breakpoints. Removing it from the `tools` array invalidates from the very start of the + request. `ToolRegistry.resolve` already refuses a disabled tool with `unavailable` regardless of + what the wire list contains, so enforcement does not depend on membership. + +## Known gaps + +Recorded here so they are not rediscovered. None are fixed yet. + +- **Compaction can permanently drop CLAUDE.md.** `ensureClaudeMdReminders` decides the reminders are + already present by testing whether the first block of the first user message is a + ``. The clock stamp is also a ``, and `TurnRunner` writes it + before that check runs. On the first turn after a compaction, when the tip is also the first user + message of the surviving slice, the stamp lands at index 0, the guard trips, and CLAUDE.md plus the + skill catalogue are never re-injected for that turn or any turn after it. Breakpoint 3 then marks + the stamp block instead. +- **`disabledTools` filters the wire list.** Toggling a tool changes region 1, so it invalidates the + entire cached prompt. See invariant 5 for where that decision should live instead. +- **`ConfigDisabledToolsProvider` is dead on the request path.** It computes an identity-aware + disabled set covering `AzCli`, `EscalatedAzCli` and the Azure DevOps PR tools, and + `ToolRegistry.wireTools` consumes it, but the request path builds tools from + `DurableConfigFactory`, which filters on raw `config.disabledTools`. Those tools reach the wire + whether or not an identity is configured, despite the provider's comment claiming otherwise. +- **Per-turn file reads.** CLAUDE.md and the system identity body are both read on every turn. The + intended policy is to read them once and re-read only across a compaction, since that is the point + at which the message prefix is being rebuilt anyway. + +## Measured behaviour + +Within a single session the caching is already optimal. Across 40 consecutive turns of a 408-turn +session, every turn's `cache_read` equalled the previous turn's `cache_read + cache_creation` +exactly, meaning the prefix was never re-written. Across sessions, the first request reads roughly +29k tokens, which is tools plus system surviving the restart, and writes the reminder run, which +differs per project. diff --git a/CLAUDE.md b/CLAUDE.md index fec73702..efc5b272 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -230,11 +230,13 @@ File watcher on both config paths (home + local). 100ms debounce. Only reloads d `SystemPromptBuilder` collects `SystemPromptProvider` instances. Providers run in parallel via `Promise.all`. Two built-in: `GitProvider` (branch/sha/status) and `UsageProvider` (time/context/cost). -### Cache markers +### Request composition and cache markers -A request carries cache breakpoints so a stable prefix is served from cache instead of re-billed each turn. Anthropic allows at most 4 per request. Three are always set: the system prompt, the tools, and a moving marker on the last user message that advances each turn so only the new message is a cache write. The fourth is added only when CLAUDE.md is present: a stable-prefix marker pinned to the end of the assembled CLAUDE.md content, held at the same position every turn so that content is a cache read after the first turn. +A request is assembled from three regions in prefix order: `tools`, `system`, `messages`. Caching matches on an exact prefix, so a change to any region invalidates that region and everything after it. Tools sit first, which is why toggling a single tool costs the entire cached prompt. -With CLAUDE.md present the request spends all 4 breakpoints. There is no headroom left, so any future change that needs a fifth breakpoint will be rejected by the API. +Anthropic allows at most 4 cache breakpoints per request, and all 4 are spent: the last tool, the last system block, a pinned marker at the end of the cached CLAUDE.md and skill-catalogue run, and a moving marker on the last user message so only new content is a write. With CLAUDE.md present there is no headroom, so any future change needing a fifth breakpoint will be rejected by the API. + +Full detail: `.claude/request-composition.md`. It lists every block type on the wire, every source of prompt text, what can change each region and when, and the invariants that keep caching intact. **Any change that adds a content block, adds a source of prompt text, moves a breakpoint, or introduces a new way for a region to vary must update that document in the same change.** ## Test Infrastructure diff --git a/apps/claude-sdk-cli/CHANGELOG.md b/apps/claude-sdk-cli/CHANGELOG.md index d6f2c625..5d0a0f54 100644 --- a/apps/claude-sdk-cli/CHANGELOG.md +++ b/apps/claude-sdk-cli/CHANGELOG.md @@ -82,6 +82,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Support reading PDF and image files as native API content blocks - Survive a mid-turn network drop: keep the machine awake during a request, persist the conversation as each message is sent and answered, and resume an interrupted turn from an empty submit - Tell the model the working directory: state it up front, and report the from/to when it changes mid-session +- The status bar now warns before a cache invalidation is paid for. Changing the model, thinking or effort mid-conversation re-writes the whole cached prefix on the next request, which on a long conversation is dollars; the warning names what moved, from what to what, and what re-writing the prefix would cost. Nothing is spent until a request goes out, so cycling back clears it for free - Track session history per working directory for future session picker - Write BetaMessage per turn to ~/.claude/audit/.jsonl @@ -89,6 +90,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - --config startup display now shows only the keys the payload actually named, not the full merged config - A broken dependency wiring now fails the build or startup +- A resumed conversation now comes up on the model, thinking and effort it last ran under, read back from its own audit, instead of on whatever the config defaults happen to be. A conversation switched away from and back no longer pays to re-write its cached prefix for having been left. A --model flag or a runtime change still wins, and now shows as a warning rather than passing silently - Add a plain-ASCII fast path to the TUI cell-grid layout, skipping Intl.Segmenter and stringWidth for rows with no ANSI styling and no wide or combining characters, cutting per-frame layout cost for plain-text rows - Adopt core-di-lite property injection end to end: the container resolves the whole graph eagerly, SQLite databases are created through a registered factory, and CLI startup moves into main() so the entry module's only import-time effect is invoking it - Az account config: reader/holder identities are now configured with a type (cert or interactive) and optional subscriptionIds, replacing readerClientId/holderClientId @@ -126,6 +128,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Update runtime and build dependencies - Updated patch and minor dependencies - Updated patch dependencies +- When a model change is what moved, the cache warning now asks the model being switched to how many tokens the conversation actually is, instead of reusing the count the previous model gave. A token count is a property of the model, not of the content: the same conversation came back as 12,223 tokens on sonnet-4-6 and 15,948 on sonnet-5, so reusing it under-stated a re-write by a third. The count starts as soon as the model editor holds a name the catalogue recognises, so the figure is usually there the instant the change is made; while it is not, the size and cost read ?? rather than showing a number from the wrong model - Wrap injected content presented to the model (attachments, git delta, CLAUDE.md, SYSTEM.md, system identity) in XML-like tags instead of custom markers, so the model-facing format is consistent - Write session ID marker on save instead of on creation diff --git a/apps/claude-sdk-cli/changes.jsonl b/apps/claude-sdk-cli/changes.jsonl index 59f03769..37e80a23 100644 --- a/apps/claude-sdk-cli/changes.jsonl +++ b/apps/claude-sdk-cli/changes.jsonl @@ -173,3 +173,6 @@ {"description":"The scratchpad is unavailable on platforms with no user id to separate one user's files from another's","category":"changed"} {"description":"A tool call refused without a prompt now says what refused it: the permission setting that decided, the operation it judged, and the paths that selected that setting. It previously reported only that the tool 'is configured to be denied automatically', which was untrue of every case and left both Claude and the operator guessing at a decision the CLI had already made","category":"fixed"} {"description":"Deleting a symlink inside the scratchpad is now approved. Removing a link never touches what it points at, so judging the delete by its destination made any link Claude created in its own scratchpad permanently undeletable. Writes still follow a link to where they land, and a delete whose parent directory resolves outside the scratchpad is still refused","category":"fixed"} +{"description":"The status bar now warns before a cache invalidation is paid for. Changing the model, thinking or effort mid-conversation re-writes the whole cached prefix on the next request, which on a long conversation is dollars; the warning names what moved, from what to what, and what re-writing the prefix would cost. Nothing is spent until a request goes out, so cycling back clears it for free","category":"added"} +{"description":"A resumed conversation now comes up on the model, thinking and effort it last ran under, read back from its own audit, instead of on whatever the config defaults happen to be. A conversation switched away from and back no longer pays to re-write its cached prefix for having been left. A --model flag or a runtime change still wins, and now shows as a warning rather than passing silently","category":"changed"} +{"description":"When a model change is what moved, the cache warning now asks the model being switched to how many tokens the conversation actually is, instead of reusing the count the previous model gave. A token count is a property of the model, not of the content: the same conversation came back as 12,223 tokens on sonnet-4-6 and 15,948 on sonnet-5, so reusing it under-stated a re-write by a third. The count starts as soon as the model editor holds a name the catalogue recognises, so the figure is usually there the instant the change is made; while it is not, the size and cost read ?? rather than showing a number from the wrong model","category":"changed"} diff --git a/apps/claude-sdk-cli/src/AuditStats.ts b/apps/claude-sdk-cli/src/AuditStats.ts index 6881786d..1859a207 100644 --- a/apps/claude-sdk-cli/src/AuditStats.ts +++ b/apps/claude-sdk-cli/src/AuditStats.ts @@ -1,8 +1,9 @@ import type { BetaMessage } from '@anthropic-ai/sdk/resources/beta/messages/messages.js'; import { IFileSystem } from '@shellicar/claude-core/fs/interfaces'; -import { type CacheTtl, calculateCost, calculateCostSplit, getContextWindow, reconstructCacheSplit } from '@shellicar/claude-sdk'; +import { type CacheTtl, calculateCost, calculateCostSplit, getContextWindow, reconstructCacheSplit, type ThinkingEffort } from '@shellicar/claude-sdk'; import { dependsOn } from '@shellicar/core-di'; import { auditPathFor } from './conversations/auditPath.js'; +import type { CacheParameters } from './model/ModelSettings.js'; import type { StatusTotals } from './model/StatusState.js'; /** An audit line: the stored BetaMessage plus the fields the audit now adds — @@ -11,6 +12,21 @@ import type { StatusTotals } from './model/StatusState.js'; type AuditLine = BetaMessage & { costUsd?: number; cacheCreation?: { fiveMinute: number; oneHour: number }; + request?: { thinking: boolean; effort: ThinkingEffort | null }; +}; + +/** Everything one pass over a conversation's audit yields: the running totals the status bar + * shows, and what its last request was sent with. Both come off the same read because the cache + * warning needs the prefix size and the parameters together, and two reads could disagree. */ +export type AuditDerivation = { + totals: StatusTotals; + /** What the cached prefix was written under, or null when no line carries it: nothing has been + * sent yet, or the lines predate the parameters being recorded. */ + cached: CacheParameters | null; + /** The model of the last assistant line, or null when the conversation has sent nothing. The API + * reports the model on every line however old, so this is known even where `cached` is not, and + * a non-null value is also what says the conversation has a cached prefix at all. */ + lastModel: string | null; }; /** @@ -52,10 +68,10 @@ export class AuditStats { * empty. `cacheTtl` is the configured TTL, consulted only for the last-resort * legacy fallback in #lineCost. */ - public async derive(id: string, cacheTtl: CacheTtl): Promise { + public async derive(id: string, cacheTtl: CacheTtl): Promise { const path = auditPathFor(this.fs, id); if (!(await this.fs.exists(path))) { - return { ...EMPTY }; + return { totals: { ...EMPTY }, cached: null, lastModel: null }; } const raw = await this.fs.readFile(path); // The audit file is now an alternating user/assistant transcript; only the @@ -67,7 +83,7 @@ export class AuditStats { .map((l) => JSON.parse(l)) .filter(isUsableLine); if (lines.length === 0) { - return { ...EMPTY }; + return { totals: { ...EMPTY }, cached: null, lastModel: null }; } const totals: StatusTotals = { ...EMPTY }; for (const line of lines) { @@ -80,7 +96,10 @@ export class AuditStats { const last = lines[lines.length - 1]; totals.lastContextUsed = last.usage.input_tokens + (last.usage.cache_creation_input_tokens ?? 0) + (last.usage.cache_read_input_tokens ?? 0); totals.contextWindow = getContextWindow(last.model); - return totals; + // The model comes off the line's own `model` field, which the API reported, rather than from + // anything the CLI wrote about the request it thought it was making. + const cached = last.request != null ? { model: last.model, thinking: last.request.thinking, effort: last.request.effort } : null; + return { totals, cached, lastModel: last.model }; } /** diff --git a/apps/claude-sdk-cli/src/AuditWriter.ts b/apps/claude-sdk-cli/src/AuditWriter.ts index 44dbbc3a..604a27d9 100644 --- a/apps/claude-sdk-cli/src/AuditWriter.ts +++ b/apps/claude-sdk-cli/src/AuditWriter.ts @@ -6,6 +6,7 @@ import { calculateCostSplit, type MessageIdentity, reconstructCacheSplit } from import { dependsOn } from '@shellicar/core-di'; import { auditPathFor } from './conversations/auditPath.js'; import { logger } from './logger.js'; +import type { CacheParameters } from './model/ModelSettings.js'; import { toHistoryBlocks } from './persistence/historyBlocks.js'; /** @@ -19,7 +20,7 @@ export class AuditWriter { @dependsOn(IFileSystem) private readonly fs!: IFileSystem; @dependsOn(IHistoryWriter) private readonly index!: IHistoryWriter; - public write(conversationId: string, request: BetaMessageParam | undefined, msg: BetaMessage, identity?: MessageIdentity): void { + public write(conversationId: string, request: BetaMessageParam | undefined, msg: BetaMessage, identity?: MessageIdentity, sent?: CacheParameters | null): void { const path = auditPathFor(this.fs, conversationId); const timestamp = new Date().toISOString(); // Store the derived cost and the reconstructed per-duration breakdown so @@ -38,7 +39,12 @@ export class AuditWriter { // v2 stamps the turn's `turnId`/`queryId` onto both lines; the assistant keeps its API `id` (spread from msg). // A legacy round (no identity) writes the id-less v1 shape unchanged. const turnIds = identity != null ? { turnId: identity.turnId, queryId: identity.queryId } : {}; - const assistant = { timestamp, costUsd, cacheCreation: { fiveMinute, oneHour }, ...msg, ...turnIds }; + // The two request parameters the prompt cache keys on that the response does not echo back; + // `model` is already on `msg`, so recording it again would be our claim over the API's fact. + // Placed after `...msg` so they land in the line's tail: `scanAuditSummary` reads model and + // cost from a bounded head window, and anything inserted ahead of them eats into it. + const sentParams = sent != null ? { request: { thinking: sent.thinking, effort: sent.effort } } : {}; + const assistant = { timestamp, costUsd, cacheCreation: { fiveMinute, oneHour }, ...msg, ...sentParams, ...turnIds }; // The user delta and the assistant response are the alternating pair for this // API call; both take the commit timestamp (the turn is stamped as a unit). // One appendFile so the pair lands together. `request` is always present in a diff --git a/apps/claude-sdk-cli/src/controller/CommandIntentExecutor.ts b/apps/claude-sdk-cli/src/controller/CommandIntentExecutor.ts index 0a40b5ff..83e35e8f 100644 --- a/apps/claude-sdk-cli/src/controller/CommandIntentExecutor.ts +++ b/apps/claude-sdk-cli/src/controller/CommandIntentExecutor.ts @@ -13,6 +13,7 @@ import { ModelSettings } from '../model/ModelSettings.js'; import { IPrimaryViewState } from '../model/PrimaryViewState.js'; import { StatusState } from '../model/StatusState.js'; import { IWorkingDirectory } from '../model/WorkingDirectory.js'; +import { ICacheWarning } from '../setup/CacheWarning.js'; import { IConversationSwitcher } from '../setup/ConversationSwitcher.js'; export type CommandIntent = 'pasteText' | 'pasteFile' | 'pasteImage' | 'removeAttachment' | 'togglePreview' | 'newSession' | 'selectPrev' | 'selectNext' | 'enterModelSubMode' | 'cycleThinking' | 'cycleEffort' | 'openModelEditor' | 'submitModel' | 'enterCdSubMode' | 'openCdEditor' | 'submitCd'; @@ -47,6 +48,7 @@ export class CommandIntentExecutor { @dependsOn(IFileSystem) private readonly fs!: IFileSystem; @dependsOn(IWorkingDirectory) private readonly workingDirectory!: IWorkingDirectory; @dependsOn(IModelCatalog) private readonly modelCatalog!: IModelCatalog; + @dependsOn(ICacheWarning) private readonly cacheWarning!: ICacheWarning; public async execute(intent: CommandIntent): Promise { try { @@ -81,11 +83,16 @@ export class CommandIntentExecutor { case 'enterModelSubMode': this.commandModeState.enterModelSubMode(); return; + // Each of these three moves a parameter the prompt cache keys on, so each is followed by a + // fresh reading of what sending under the new value would cost. The change itself is free: + // nothing is spent until a request goes out, and cycling back clears the warning. case 'cycleThinking': this.modelSettings.cycleThinking(); + this.cacheWarning.refresh(); return; case 'cycleEffort': this.modelSettings.cycleEffort(); + this.cacheWarning.refresh(); return; case 'openModelEditor': this.commandModeState.openModelEditor(this.statusState.model); @@ -141,6 +148,7 @@ export class CommandIntentExecutor { } const text = editorText(editor).trim(); this.modelSettings.setModel(text.length > 0 ? text : null); + this.cacheWarning.refresh(); this.commandModeState.closeModelEditor(); } diff --git a/apps/claude-sdk-cli/src/controller/CommandKeyHandler.ts b/apps/claude-sdk-cli/src/controller/CommandKeyHandler.ts index 356eb4d3..4608b5e7 100644 --- a/apps/claude-sdk-cli/src/controller/CommandKeyHandler.ts +++ b/apps/claude-sdk-cli/src/controller/CommandKeyHandler.ts @@ -1,6 +1,7 @@ import type { KeyAction } from '@shellicar/claude-core/input'; import { dependsOn } from '@shellicar/core-di'; import { type CommandContext, ICommandModeState } from '../model/CommandModeState.js'; +import { ICacheWarning } from '../setup/CacheWarning.js'; import { type CommandIntent, CommandIntentExecutor } from './CommandIntentExecutor.js'; import type { InputHandler } from './InputHandler.js'; @@ -50,6 +51,7 @@ export const COMMAND_BINDINGS_BY_CONTEXT: ReadonlyMap(); public get totalInputTokens(): number { @@ -84,6 +106,9 @@ export class StatusState { public get cwdBasename(): string { return this.#cwdBasename; } + public get cacheDivergence(): CacheDivergence | null { + return this.#cacheDivergence; + } public constructor(cwdBasename: string) { this.#cwdBasename = cwdBasename; @@ -135,6 +160,13 @@ export class StatusState { this.#emitter.emit('change'); } + /** Set or clear the pending cache cost. Null means the live settings and the cached prefix + * agree, so there is nothing to warn about. */ + public setCacheDivergence(divergence: CacheDivergence | null): void { + this.#cacheDivergence = divergence; + this.#emitter.emit('change'); + } + /** * Replace the running totals wholesale from a derived snapshot. Called when * the figures are re-derived from the audit for the current conversation id diff --git a/apps/claude-sdk-cli/src/setup/CacheWarning.ts b/apps/claude-sdk-cli/src/setup/CacheWarning.ts new file mode 100644 index 00000000..4c220140 --- /dev/null +++ b/apps/claude-sdk-cli/src/setup/CacheWarning.ts @@ -0,0 +1,162 @@ +import { buildRequestParams, calculateCostSplit, IConversation, IDurableConfigProvider, ITokenCounter } from '@shellicar/claude-sdk'; +import { dependsOn } from '@shellicar/core-di'; +import type { AuditDerivation } from '../AuditStats.js'; +import { type CacheParameters, ModelSettings } from '../model/ModelSettings.js'; +import { type CacheParameterChange, StatusState } from '../model/StatusState.js'; + +/** The parameters a request would go out under right now. */ +export const liveCacheParameters = (config: IDurableConfigProvider): CacheParameters => ({ + model: config.getEffectiveModel(), + thinking: config.getEffectiveThinkingEnabled(), + effort: config.getEffectiveEffort() ?? null, +}); + +/** The cache warning's contract; register abstract→concrete and depend on the abstract (DI rule). */ +export abstract class ICacheWarning { + /** Recompute what the next request would cost in cache terms and land it in the status bar. + * Call wherever either side can move: an operator toggle, a config reload, a conversation + * move, or the send that resolves the divergence. */ + public abstract refresh(): void; + + /** What a conversation being adopted should have its divergence measured against, given what its + * audit yielded. */ + public abstract baselineFor(audit: AuditDerivation): CacheParameters | null; + + /** Start counting the conversation under a model the operator is looking at but has not chosen. + * Called while the model editor holds a name the catalogue recognises, so the size is usually + * known by the time the choice is made and the warning has a figure the instant it appears. */ + public abstract prefetch(model: string): void; +} + +/** Each parameter the cache keys on, and how it reads to an operator. */ +const PARAMETERS: readonly { name: string; of: (p: CacheParameters) => string }[] = [ + { name: 'model', of: (p) => p.model }, + { name: 'thinking', of: (p) => (p.thinking ? 'on' : 'off') }, + { name: 'effort', of: (p) => p.effort ?? 'default' }, +]; + +/** + * Warns before an invalidation is paid for rather than after. + * + * Changing the model, thinking or effort mid-conversation re-writes the entire cached prefix on the + * next request, and on a long conversation that is dollars, not cents. The change itself is free: + * nothing is spent until a request goes out, so between the keypress and the send there is a window + * where the operator can still cycle back and pay nothing. This is what fills that window. + * + * Deliberately not held by `ModelOverrides`, which would need the effective values it is itself an + * input to. `DurableConfigFactory` depends on the overrides, so the overrides cannot depend back on + * it; the comparison lives here, above both. + */ +export class CacheWarning extends ICacheWarning { + @dependsOn(IDurableConfigProvider) private readonly configFactory!: IDurableConfigProvider; + @dependsOn(ModelSettings) private readonly settings!: ModelSettings; + @dependsOn(StatusState) private readonly statusState!: StatusState; + @dependsOn(ITokenCounter) private readonly counter!: ITokenCounter; + @dependsOn(IConversation) private readonly conversation!: IConversation; + // Bumped by every refresh, so a count that lands after the operator has moved on is recognised as + // an answer to a question nobody is asking any more and dropped. + #generation = 0; + // A count taken ahead of the choice, spent the moment that model is chosen. Holding it for longer + // would mean holding a figure for a conversation that has since grown. + #prefetched: { model: string; tokens: number } | null = null; + #counting: string | null = null; + + /** + * A conversation whose audit records the parameters gives them outright. One that has sent nothing + * has no cached prefix, so there is nothing to measure. The case in between is every conversation + * that predates the parameters being recorded: it has a prefix, written under settings nobody + * wrote down. Assuming it was written under the current ones is what lets the warning work on a + * conversation that started before this existed, rather than staying silent until its next turn. + * + * The model is not assumed. The API reports it on every audit line however old, so that much is + * known and only thinking and effort are guesses. A wrong guess can miss a warning, or raise one + * for a cost already paid, but only on a conversation that predates the recording and only until + * its next turn writes down the truth. + */ + public baselineFor(audit: AuditDerivation): CacheParameters | null { + if (audit.cached != null) { + return audit.cached; + } + if (audit.lastModel == null) { + return null; + } + return { ...liveCacheParameters(this.configFactory), model: audit.lastModel }; + } + + public refresh(): void { + const generation = ++this.#generation; + const cached = this.settings.cached; + if (cached == null) { + this.statusState.setCacheDivergence(null); + return; + } + const live = liveCacheParameters(this.configFactory); + const changes = PARAMETERS.flatMap(({ name, of }): CacheParameterChange[] => { + const from = of(cached); + const to = of(live); + return from === to ? [] : [{ name, from, to }]; + }); + if (changes.length === 0) { + this.statusState.setCacheDivergence(null); + return; + } + // A token count belongs to the model that produced it, so the last turn's count is already the + // right number unless the model itself is what moved. + if (cached.model === live.model) { + this.#publish(changes, this.statusState.lastContextUsed, live.model); + return; + } + // The models may not tokenise alike: the same prefix came back as 12,223 tokens on sonnet-4-6 + // and 15,948 on sonnet-5. So the old count is not shown while the new model's is unknown; a + // count taken ahead of the choice usually means it never is. + const prefetched = this.#prefetched?.model === live.model ? this.#prefetched.tokens : null; + this.#prefetched = null; + this.#publish(changes, prefetched, live.model); + if (prefetched == null) { + void this.#countUnderLiveModel(generation, changes, live.model); + } + } + + public prefetch(model: string): void { + if (model === this.#prefetched?.model || model === this.#counting) { + return; + } + this.#counting = model; + void this.#countFor(model).then((tokens) => { + this.#counting = null; + if (tokens != null) { + this.#prefetched = { model, tokens }; + } + }); + } + + #publish(changes: readonly CacheParameterChange[], tokens: number | null, model: string): void { + // The whole of the last request's context is what gets re-written, priced at the model the next + // request would use and at the one-hour write rate, the TTL every breakpoint is written with. + const costUsd = tokens == null ? null : calculateCostSplit({ inputTokens: 0, cacheCreation5mTokens: 0, cacheCreation1hTokens: tokens, cacheReadTokens: 0, outputTokens: 0 }, model); + this.statusState.setCacheDivergence({ changes, tokens, costUsd }); + } + + async #countUnderLiveModel(generation: number, changes: readonly CacheParameterChange[], model: string): Promise { + const counted = await this.#countFor(model); + if (counted == null || generation !== this.#generation) { + return; + } + this.#publish(changes, counted, model); + } + + /** + * Counts the conversation as a request under one model, which may not be the one the config + * currently names: a prefetch runs for a model the operator is only looking at. + * + * `durable` is the same object TurnRunner reads its builder options from, so this is the request + * the send would actually make. `cloneForRequest` carries the CLAUDE.md and skill reminders, + * which are persisted into the first user message; the per-turn ephemeral ones are absent, and + * are a few dozen tokens against a prefix measured in tens of thousands. + */ + async #countFor(model: string): Promise { + const durable = this.configFactory.config; + const messages = this.conversation.cloneForRequest(durable.compact?.enabled ?? false); + return this.counter.count(buildRequestParams({ ...durable, model }, messages)); + } +} diff --git a/apps/claude-sdk-cli/src/setup/ConfigChangeCoordinator.ts b/apps/claude-sdk-cli/src/setup/ConfigChangeCoordinator.ts index 31e78bda..380602ff 100644 --- a/apps/claude-sdk-cli/src/setup/ConfigChangeCoordinator.ts +++ b/apps/claude-sdk-cli/src/setup/ConfigChangeCoordinator.ts @@ -6,6 +6,7 @@ import { IConversationState } from '../model/ConversationState.js'; import { DisabledToolsNoticeGate } from '../model/DisabledToolsNoticeGate.js'; import { PermissionsNoticeGate } from '../model/PermissionsNoticeGate.js'; import { StatusState } from '../model/StatusState.js'; +import { ICacheWarning } from './CacheWarning.js'; import { IRulesConfigNotifier } from './ConfigRulesConfigProvider.js'; import { ModelOverrides } from './ModelOverrides.js'; import { ITurnCoordinator } from './TurnCoordinator.js'; @@ -40,6 +41,7 @@ export class ConfigChangeCoordinator extends IConfigChangeCoordinator { @dependsOn(ModelOverrides) private readonly overrides!: ModelOverrides; @dependsOn(IDisabledToolsProvider) private readonly disabledToolsProvider!: IDisabledToolsProvider; @dependsOn(DisabledToolsNoticeGate) private readonly disabledToolsNoticeGate!: DisabledToolsNoticeGate; + @dependsOn(ICacheWarning) private readonly cacheWarning!: ICacheWarning; public wire(): void { this.rulesConfigNotifier.onNotice((notice) => { @@ -61,6 +63,9 @@ export class ConfigChangeCoordinator extends IConfigChangeCoordinator { if (!this.turnCoordinator.inProgress) { this.statusState.setModel(this.configFactory.getEffectiveModel(), this.overrides.model != null); this.statusState.setShowConversationId(config.statusBar.showConversationId); + // A reload can move the model, thinking or effort defaults out from under a conversation whose + // prefix was cached under the old ones, which costs the same as an operator toggling them. + this.cacheWarning.refresh(); } const disabledToolsNotice = this.disabledToolsNoticeGate.update(this.disabledToolsProvider.disabledTools); if (disabledToolsNotice != null) { diff --git a/apps/claude-sdk-cli/src/setup/ConversationBootSequence.ts b/apps/claude-sdk-cli/src/setup/ConversationBootSequence.ts index 53706fa3..cca2a173 100644 --- a/apps/claude-sdk-cli/src/setup/ConversationBootSequence.ts +++ b/apps/claude-sdk-cli/src/setup/ConversationBootSequence.ts @@ -13,6 +13,7 @@ import { StatusState } from '../model/StatusState.js'; import { HistorySweepScheduler } from '../persistence/HistorySweepScheduler.js'; import { replayHistory } from '../replayHistory.js'; import { IWorkspace, scratchpadUnavailableNotice } from '../workspace/Workspace.js'; +import { ICacheWarning } from './CacheWarning.js'; import { ModelOverrides } from './ModelOverrides.js'; import { ISdkEventBridge } from './SdkEventBridge.js'; import { IShutdownSequence } from './ShutdownSequence.js'; @@ -47,6 +48,7 @@ export class ConversationBootSequence extends IConversationBootSequence { @dependsOn(ModelOverrides) private readonly overrides!: ModelOverrides; @dependsOn(IConversationSession) private readonly session!: IConversationSession; @dependsOn(AuditStats) private readonly auditStats!: AuditStats; + @dependsOn(ICacheWarning) private readonly cacheWarning!: ICacheWarning; @dependsOn(ViewHost) private readonly host!: ViewHost; public async run(configOverride: Record | undefined): Promise { @@ -100,12 +102,20 @@ export class ConversationBootSequence extends IConversationBootSequence { if (configOverride !== undefined) { this.conversationState.addBlocks([{ type: 'meta', content: formatEffectiveConfig({ ...this.configLoader.config, model: this.configFactory.getEffectiveModel() }, configOverride) }]); } + // One read of the current id's audit yields both halves: the usage figures that replace the zero state, + // and what its last request was sent with. A resumed id reads both back; a fresh id has no audit file, so + // both read empty. The configured TTL is passed for the legacy fallback that prices any pre-existing + // flat-only lines of a resumed id. + const audit = await this.auditStats.derive(this.session.id, this.configFactory.config.cacheTtl ?? CacheTtl.OneHour); + // Adopted before the model reaches the status bar, because the overrides resolve through it: a resumed + // conversation comes up on the settings its cached prefix was written under, not the config default. + this.overrides.adopt(this.cacheWarning.baselineFor(audit)); this.statusState.setModel(this.configFactory.getEffectiveModel(), this.overrides.model != null); this.statusState.setShowConversationId(this.configLoader.config.statusBar.showConversationId); - // Re-derive the status figures from the current id's audit, replacing the zero state. A resumed id reads its - // usage back; a fresh id has no audit file, so it reads empty. The configured TTL is passed for the legacy - // fallback that prices any pre-existing flat-only lines of a resumed id. - this.statusState.resetTo(await this.auditStats.derive(this.session.id, this.configFactory.config.cacheTtl ?? CacheTtl.OneHour)); + this.statusState.resetTo(audit.totals); + // After the totals land, since the warning prices the prefix size they carry. A `--model` flag is the one + // thing that can already have diverged before a single key is pressed. + this.cacheWarning.refresh(); this.host.renderNow(); } } diff --git a/apps/claude-sdk-cli/src/setup/ConversationSwitcher.ts b/apps/claude-sdk-cli/src/setup/ConversationSwitcher.ts index 684a9fb4..522c1399 100644 --- a/apps/claude-sdk-cli/src/setup/ConversationSwitcher.ts +++ b/apps/claude-sdk-cli/src/setup/ConversationSwitcher.ts @@ -9,10 +9,12 @@ import { IConvServe } from '../conv/ConvServe.js'; import { IConversationSession } from '../model/ConversationSession.js'; import { IConversationState } from '../model/ConversationState.js'; import { ISystemIdentity } from '../model/ISystemIdentity.js'; +import { type CacheParameters, ModelSettings } from '../model/ModelSettings.js'; import { IPrimaryViewState } from '../model/PrimaryViewState.js'; import { StatusState } from '../model/StatusState.js'; import { replayHistory } from '../replayHistory.js'; import { IWorkspace, scratchpadUnavailableNotice } from '../workspace/Workspace.js'; +import { ICacheWarning } from './CacheWarning.js'; /** The switcher's contract; register abstract→concrete and depend on the abstract (DI rule). */ export abstract class IConversationSwitcher { @@ -37,6 +39,7 @@ export class ConversationSwitcher extends IConversationSwitcher { @dependsOn(IConversationSession) private readonly session!: IConversationSession; @dependsOn(IConversationState) private readonly conversationState!: IConversationState; @dependsOn(ISystemIdentity) private readonly systemIdentity!: ISystemIdentity; + @dependsOn(ModelSettings) private readonly modelSettings!: ModelSettings; @dependsOn(IAgentPresence) private readonly agentPresence!: IAgentPresence; @dependsOn(IConvServe) private readonly convServe!: IConvServe; @dependsOn(AuditStats) private readonly auditStats!: AuditStats; @@ -46,6 +49,7 @@ export class ConversationSwitcher extends IConversationSwitcher { @dependsOn(IPrimaryViewState) private readonly primaryViewState!: IPrimaryViewState; @dependsOn(ConfigLoader) private readonly configLoader!: ConfigLoader; @dependsOn(IWorkspace) private readonly workspace!: IWorkspace; + @dependsOn(ICacheWarning) private readonly cacheWarning!: ICacheWarning; public async createNew(): Promise { if (this.primaryViewState.conversationMoving) { return; @@ -82,10 +86,12 @@ export class ConversationSwitcher extends IConversationSwitcher { await this.session.createNew(); this.#rebind(previousId); this.systemIdentity.inherit(this.session.id); + this.modelSettings.carryOver(); this.conversationState.clear(); // After the transcript is cleared, or the notice would be cleared with it. await this.#resolveWorkspace(); await this.#resetStatus(); + this.cacheWarning.refresh(); } async #switchTo(id: string): Promise { @@ -101,7 +107,10 @@ export class ConversationSwitcher extends IConversationSwitcher { this.#replayHistory(); // After the transcript is rebuilt, or the notice would be cleared with it. await this.#resolveWorkspace(); - await this.#resetStatus(); + // The adopted conversation comes up on whatever it last sent, so it does not pay to re-write its + // cached prefix just for having been switched away from and back. + this.modelSettings.adopt(await this.#resetStatus()); + this.cacheWarning.refresh(); } /** @@ -138,11 +147,14 @@ export class ConversationSwitcher extends IConversationSwitcher { this.agentPresence.attach(this.session.id, this.fs.cwd()); } - /** Re-derive the status figures for the current id. A fresh id has no audit file, so this reads - * empty and the "clear on new" behaviour falls out of the single id-keyed rule. The TTL is - * inert except for a legacy flat-only audit line, so the default is passed rather than - * threading the config provider. */ - async #resetStatus(): Promise { - this.statusState.resetTo(await this.auditStats.derive(this.session.id, CacheTtl.OneHour)); + /** Re-derive the status figures for the current id, returning what its last request was sent + * with so the caller can settle the model settings off the same read. A fresh id has no audit + * file, so this reads empty and the "clear on new" behaviour falls out of the single id-keyed + * rule. The TTL is inert except for a legacy flat-only audit line, so the default is passed + * rather than threading the config provider. */ + async #resetStatus(): Promise { + const audit = await this.auditStats.derive(this.session.id, CacheTtl.OneHour); + this.statusState.resetTo(audit.totals); + return this.cacheWarning.baselineFor(audit); } } diff --git a/apps/claude-sdk-cli/src/setup/ModelOverrides.ts b/apps/claude-sdk-cli/src/setup/ModelOverrides.ts index 0ab27ffc..6e671b5f 100644 --- a/apps/claude-sdk-cli/src/setup/ModelOverrides.ts +++ b/apps/claude-sdk-cli/src/setup/ModelOverrides.ts @@ -1,7 +1,7 @@ import { ConfigLoader } from '@shellicar/claude-core/Config/ConfigLoader'; import type { ThinkingEffort } from '@shellicar/claude-sdk'; import { dependsOn } from '@shellicar/core-di'; -import { ModelSettings } from '../model/ModelSettings.js'; +import { type CacheParameters, ModelSettings } from '../model/ModelSettings.js'; import { StatusState } from '../model/StatusState.js'; import { IRuntimeOptions } from './IRuntimeOptions.js'; @@ -15,13 +15,33 @@ export class ModelOverrides extends ModelSettings { #thinking: 'on' | 'off' | null = null; #effort: ThinkingEffort | null = null; #model: string | null = null; - // Distinguishes "never set at runtime" (fall back to the --model flag) from - // "cleared at runtime" (fall back to the config model). One override slot, - // seeded by --model; command mode reads, sets, and clears the same slot. + // Each slot distinguishes "never set at runtime" (defer to the flag, then to what the + // conversation last sent) from "cleared at runtime" (a deliberate null the operator chose). + // Command mode reads, sets, and clears the same slots. #modelTouched = false; + #thinkingTouched = false; + #effortTouched = false; + #cached: CacheParameters | null = null; + /** + * Precedence, highest first: a runtime change the operator just made, the `--model` launch flag, + * then what this conversation last sent. A runtime change wins because it is the most recent + * expression of intent, which is also what lets `C-/ m` back to the cached value clear a + * divergence the flag introduced. + * + * The cached value only fills the slot when it actually differs from the config model. An + * override slot means "not what the config says", and the status bar's `*` marks exactly that, + * so a conversation cached on the config's own model must read as no override at all. + */ public get model(): string | null { - return this.#modelTouched ? this.#model : this.runtime.modelOverride; + if (this.#modelTouched) { + return this.#model; + } + if (this.runtime.modelOverride != null) { + return this.runtime.modelOverride; + } + const cached = this.#cached?.model ?? null; + return cached !== this.configLoader.config.model ? cached : null; } public setModel(id: string | null): void { @@ -33,22 +53,66 @@ export class ModelOverrides extends ModelSettings { } public get thinking(): 'on' | 'off' | null { - return this.#thinking; + if (this.#thinkingTouched) { + return this.#thinking; + } + const cached = this.#cached; + if (cached == null || cached.thinking === this.configLoader.config.thinking.enabled) { + return null; + } + return cached.thinking ? 'on' : 'off'; } public get effort(): ThinkingEffort | null { - return this.#effort; + if (this.#effortTouched) { + return this.#effort; + } + const cached = this.#cached?.effort ?? null; + return cached !== (this.configLoader.config.thinking.effort ?? null) ? cached : null; } + // Both cycles advance from the EFFECTIVE current value, not from the raw slot, so the first press + // after resuming a conversation steps on from what that conversation was actually using rather + // than restarting the cycle from its head. public cycleThinking(): void { - const idx = THINKING_CYCLE.indexOf(this.#thinking); - this.#thinking = THINKING_CYCLE[(idx + 1) % THINKING_CYCLE.length]; + const idx = THINKING_CYCLE.indexOf(this.thinking); + this.#thinking = THINKING_CYCLE[(idx + 1) % THINKING_CYCLE.length] ?? null; + this.#thinkingTouched = true; this.statusState.setThinkingOverride(this.#thinking); } public cycleEffort(): void { - const idx = EFFORT_CYCLE.indexOf(this.#effort); + const idx = EFFORT_CYCLE.indexOf(this.effort); this.#effort = EFFORT_CYCLE[(idx + 1) % EFFORT_CYCLE.length] ?? null; + this.#effortTouched = true; this.statusState.setEffortOverride(this.#effort); } + + public get cached(): CacheParameters | null { + return this.#cached; + } + + public markSent(params: CacheParameters): void { + this.#cached = params; + } + + public adopt(cached: CacheParameters | null): void { + this.#cached = cached; + this.#modelTouched = false; + this.#thinkingTouched = false; + this.#effortTouched = false; + this.#syncStatus(); + } + + public carryOver(): void { + this.#cached = null; + this.#syncStatus(); + } + + #syncStatus(): void { + const override = this.model; + this.statusState.setModel(override ?? this.configLoader.config.model, override != null); + this.statusState.setThinkingOverride(this.thinking); + this.statusState.setEffortOverride(this.effort); + } } diff --git a/apps/claude-sdk-cli/src/setup/SdkEventBridge.ts b/apps/claude-sdk-cli/src/setup/SdkEventBridge.ts index cbf2da00..e454d1c7 100644 --- a/apps/claude-sdk-cli/src/setup/SdkEventBridge.ts +++ b/apps/claude-sdk-cli/src/setup/SdkEventBridge.ts @@ -8,6 +8,7 @@ import { IConvTelemetryProjector } from '../conv/ConvTelemetryProjector.js'; import { telemetryLeaf } from '../conv/telemetryLeaf.js'; import { encode, stamp } from '../conv/wire.js'; import { IConversationSession } from '../model/ConversationSession.js'; +import { ModelSettings } from '../model/ModelSettings.js'; import { SdkChannel } from './SdkChannel.js'; /** A round's closing reason, recognised off its telemetry but not committal until the closing @@ -39,11 +40,14 @@ export class SdkEventBridge extends ISdkEventBridge { @dependsOn(IConvTelemetryProjector) private readonly convTelemetry!: IConvTelemetryProjector; @dependsOn(Clock) private readonly clock!: Clock; @dependsOn(IConversationSession) private readonly session!: IConversationSession; + @dependsOn(ModelSettings) private readonly modelSettings!: ModelSettings; #pendingQueryClose: PendingQueryClose | null = null; /** Wire both directions. Call once at startup, after every dependency above is live. */ public wire(): void { - this.processor.on('final_message', (msg, request, identity) => this.auditWriter.write(this.session.id, request, msg, identity)); + // The settings are the ones noted before the request went out, so the line records what this turn was + // actually sent with rather than whatever the operator has selected by the time the response lands. + this.processor.on('final_message', (msg, request, identity) => this.auditWriter.write(this.session.id, request, msg, identity, this.modelSettings.cached)); this.processor.on('message_start', () => this.sdkChannel.send({ type: 'message_start' })); this.processor.on('message_usage', (usage) => this.sdkChannel.send({ type: 'message_usage', ...usage })); this.processor.on('message_text', (text) => this.sdkChannel.send({ type: 'message_text', text })); diff --git a/apps/claude-sdk-cli/src/setup/TurnCoordinator.ts b/apps/claude-sdk-cli/src/setup/TurnCoordinator.ts index f4792a1d..b62c73b4 100644 --- a/apps/claude-sdk-cli/src/setup/TurnCoordinator.ts +++ b/apps/claude-sdk-cli/src/setup/TurnCoordinator.ts @@ -18,6 +18,7 @@ import { buildRunAgentInput, runAgent, type UserInput } from '../runAgent.js'; import { flushSealedToScroll } from '../view/flushSealedToScroll.js'; import { TerminalRenderer } from '../view/TerminalRenderer.js'; import { AppToolsService } from './AppToolsService.js'; +import { ICacheWarning, liveCacheParameters } from './CacheWarning.js'; import { CwdTracker } from './CwdTracker.js'; import { ModelOverrides } from './ModelOverrides.js'; import { ISdkEventBridge } from './SdkEventBridge.js'; @@ -73,6 +74,7 @@ export class TurnCoordinator extends ITurnCoordinator { @dependsOn(AppToolsService) private readonly appTools!: AppToolsService; @dependsOn(IConvChangePublisher) private readonly convChanges!: IConvChangePublisher; @dependsOn(ISdkEventBridge) private readonly sdkEventBridge!: ISdkEventBridge; + @dependsOn(ICacheWarning) private readonly cacheWarning!: ICacheWarning; #currentAbortController: AbortController | null = null; #turnInProgress = false; @@ -131,6 +133,12 @@ export class TurnCoordinator extends ITurnCoordinator { const skillDelta = await this.skillTracker.scanForDelta(); const cwdDelta = this.cwdTracker.scanForDelta(); const agentInput = buildRunAgentInput(userInput); + // Noted on the way out rather than on the way back. The API has processed the prefix and written + // the cache before a stream can be cut off, so an aborted turn has still moved what the cache is + // keyed on; recording after the response would leave the next turn measuring against a stale value + // and warning about a cost already paid. These are also the parameters this turn's audit line carries. + this.overrides.markSent(liveCacheParameters(this.configFactory)); + this.cacheWarning.refresh(); await runAgent( this.queryRunner, agentInput, @@ -145,7 +153,6 @@ export class TurnCoordinator extends ITurnCoordinator { { git: gitDelta, skill: skillDelta, cwd: cwdDelta }, ); await this.gitMonitor.takeSnapshot(); - this.statusState.setModel(this.configFactory.getEffectiveModel(), this.overrides.model != null); await this.session.saveConversation(); this.convChanges.flush(this.session.id); diff --git a/apps/claude-sdk-cli/src/setup/container.ts b/apps/claude-sdk-cli/src/setup/container.ts index 5672e9f8..fd29fd78 100644 --- a/apps/claude-sdk-cli/src/setup/container.ts +++ b/apps/claude-sdk-cli/src/setup/container.ts @@ -48,6 +48,7 @@ import { ISdkMessagePublisher, ISkillGateProvider, IStreamProcessor, + ITokenCounter, ITokenEndpoint, IToolBlockNotifier, IToolProvider, @@ -61,6 +62,7 @@ import { QueryRunner, StreamInterruptListener, StreamProcessor, + TokenCounter, ToolBlockNotifier, ToolRegistry, TurnRunner, @@ -162,6 +164,7 @@ import { IWorkspace, Workspace } from '../workspace/Workspace.js'; import { AgentBusActivator, IAgentBusActivator } from './AgentBusActivator.js'; import { Application, IApplication } from './Application.js'; import { AppToolsService } from './AppToolsService.js'; +import { CacheWarning, ICacheWarning } from './CacheWarning.js'; import { ConfigChangeCoordinator, IConfigChangeCoordinator } from './ConfigChangeCoordinator.js'; import { ConfigDisabledToolsProvider } from './ConfigDisabledToolsProvider.js'; import { ConfigRulesConfigProvider, IRulesConfigNotifier, readToolsRaw } from './ConfigRulesConfigProvider.js'; @@ -405,6 +408,10 @@ export function buildContainer(options: ContainerOptions): IServiceCollection { .register(ModelCatalog) .using([ICredentialProvider, ILogger], (credentials, log) => new ModelCatalog(credentials, log)) .as(IModelCatalog); + services + .register(TokenCounter) + .using([ICredentialProvider, ILogger], (credentials, log) => new TokenCounter(credentials, log)) + .as(ITokenCounter); services.register(ApprovalCoordinator).asSelf(); // AccountLimitNotice and AccountLimitListener share identity from this one register() call. services.register(AccountLimitNotice).asSelf().as(AccountLimitListener); @@ -450,6 +457,7 @@ export function buildContainer(options: ContainerOptions): IServiceCollection { services.register(NodeSipsBridge).asSelf().as(SipsBridge); // ModelOverrides and ModelSettings share identity from this one register() call. services.register(ModelOverrides).asSelf().as(ModelSettings); + services.register(CacheWarning).as(ICacheWarning); // --- state stores --- services diff --git a/apps/claude-sdk-cli/src/view/renderStatus.ts b/apps/claude-sdk-cli/src/view/renderStatus.ts index 4d75a6c6..b63bb582 100644 --- a/apps/claude-sdk-cli/src/view/renderStatus.ts +++ b/apps/claude-sdk-cli/src/view/renderStatus.ts @@ -1,6 +1,6 @@ import type { Duration } from '@js-joda/core'; import versionInfo from '@shellicar/build-version/version'; -import { BOLD_WHITE, CYAN, DIM, RESET, YELLOW } from '@shellicar/claude-core/ansi'; +import { BOLD_WHITE, CYAN, DIM, RED, RESET, YELLOW } from '@shellicar/claude-core/ansi'; import { StatusLineBuilder } from '@shellicar/claude-core/status-line'; import type { ClockRole, ClockSnapshot } from '../model/ITurnClock.js'; import type { StatusState } from '../model/StatusState.js'; @@ -30,13 +30,41 @@ export function renderModel(state: StatusState, _cols: number, conversationId: s const idSuffix = state.showConversationId && conversationId ? ` ${conversationId}` : ''; const identity = state.identityName != null ? ` ${CYAN}${state.identityName}${RESET}` : ''; const buildVersion = ` ${DIM}v${versionInfo.version}${RESET}`; + const cache = renderCacheWarning(state); if (!model) { - return ` ${label}${identity}${thinking}${effort}${idSuffix}${buildVersion}`; + return ` ${label}${identity}${thinking}${effort}${cache}${idSuffix}${buildVersion}`; } const { name, version } = parseModelName(model); const versionPart = version != null ? ` ${version}` : ''; const overridePart = state.isModelOverridden ? '*' : ''; - return ` ${YELLOW}⚡ ${name}${versionPart}${overridePart}${RESET} ${label}${identity}${thinking}${effort}${idSuffix}${buildVersion}`; + return ` ${YELLOW}⚡ ${name}${versionPart}${overridePart}${RESET} ${label}${identity}${thinking}${effort}${cache}${idSuffix}${buildVersion}`; +} + +/** + * The cost the next request would pay for a setting that has moved away from what the cached + * prefix was written under, or '' while the two agree. + * + * Sits beside the thinking and effort segments rather than at the end of the line: it is about the + * value the operator just cycled, and the segments after it (conversation id, build version) are + * the ones worth losing first when the terminal is narrow. + * + * Names the changes before the price, because the number means nothing until you recognise which + * keypress caused it. Nothing has been spent at this point: cycling back clears it for free. + * + * `??` stands where the size is not known yet, which happens only while the model being switched to + * is still counting the conversation. The model being left has a count, but it is the wrong one by + * up to a third across a tokeniser change, and a wrong number reads as an answer where `??` does + * not. + */ +function renderCacheWarning(state: StatusState): string { + const divergence = state.cacheDivergence; + if (divergence == null) { + return ''; + } + const changes = divergence.changes.map((change) => `${change.name} ${change.from}\u2192${change.to}`).join(', '); + const size = divergence.tokens == null ? '??' : formatTokens(divergence.tokens); + const cost = divergence.costUsd == null ? '$??' : `$${divergence.costUsd.toFixed(2)}`; + return ` ${RED}\u26a0 ${changes} rewrites ${size} (${cost})${RESET}`; } function formatTokens(n: number): string { diff --git a/apps/claude-sdk-cli/test/AuditStats.spec.ts b/apps/claude-sdk-cli/test/AuditStats.spec.ts index f7924483..04baf508 100644 --- a/apps/claude-sdk-cli/test/AuditStats.spec.ts +++ b/apps/claude-sdk-cli/test/AuditStats.spec.ts @@ -1,5 +1,5 @@ import { IFileSystem } from '@shellicar/claude-core/fs/interfaces'; -import { CacheTtl } from '@shellicar/claude-sdk'; +import { CacheTtl, type ThinkingEffort } from '@shellicar/claude-sdk'; import { createServiceCollection, Lifetime } from '@shellicar/core-di'; import { describe, expect, it } from 'vitest'; import { AuditStats } from '../src/AuditStats.js'; @@ -26,6 +26,7 @@ type LineFields = { costUsd?: number; // stored derived cost cacheSplit?: { fiveMinute: number; oneHour: number }; // stored normalized breakdown ephemeral?: { ephemeral_5m_input_tokens: number; ephemeral_1h_input_tokens: number }; // raw usage.cache_creation + request?: { thinking: boolean; effort: ThinkingEffort | null }; // the parameters the turn was sent with }; function auditLine(fields: LineFields): string { @@ -45,6 +46,9 @@ function auditLine(fields: LineFields): string { if (fields.cacheSplit !== undefined) { entry.cacheCreation = fields.cacheSplit; } + if (fields.request !== undefined) { + entry.request = fields.request; + } return JSON.stringify(entry); } @@ -56,35 +60,35 @@ describe('AuditStats — derive', () => { it('returns zero inputTokens when the id has no audit file', async () => { const stats = buildAuditStats(new MemoryFileSystem({}, '/home/user')); const expected = 0; - const actual = (await stats.derive('missing-id', CacheTtl.OneHour)).inputTokens; + const actual = (await stats.derive('missing-id', CacheTtl.OneHour)).totals.inputTokens; expect(actual).toBe(expected); }); it('sums inputTokens across lines', async () => { const stats = buildAuditStats(fsWithAudit('c1', [auditLine({ input: 100 }), auditLine({ input: 40 })])); const expected = 140; - const actual = (await stats.derive('c1', CacheTtl.OneHour)).inputTokens; + const actual = (await stats.derive('c1', CacheTtl.OneHour)).totals.inputTokens; expect(actual).toBe(expected); }); it('sums outputTokens across lines', async () => { const stats = buildAuditStats(fsWithAudit('c1', [auditLine({ output: 20 }), auditLine({ output: 5 })])); const expected = 25; - const actual = (await stats.derive('c1', CacheTtl.OneHour)).outputTokens; + const actual = (await stats.derive('c1', CacheTtl.OneHour)).totals.outputTokens; expect(actual).toBe(expected); }); it('takes lastContextUsed from the final line', async () => { const stats = buildAuditStats(fsWithAudit('c1', [auditLine({ input: 1000 }), auditLine({ input: 10, cacheCreation: 20, cacheRead: 30 })])); const expected = 60; // 10 + 20 + 30 from the last line only - const actual = (await stats.derive('c1', CacheTtl.OneHour)).lastContextUsed; + const actual = (await stats.derive('c1', CacheTtl.OneHour)).totals.lastContextUsed; expect(actual).toBe(expected); }); it('derives contextWindow from the final line model', async () => { const stats = buildAuditStats(fsWithAudit('c1', [auditLine({ model: 'claude-fable-5' })])); const expected = 1_000_000; - const actual = (await stats.derive('c1', CacheTtl.OneHour)).contextWindow; + const actual = (await stats.derive('c1', CacheTtl.OneHour)).totals.contextWindow; expect(actual).toBe(expected); }); @@ -92,7 +96,7 @@ describe('AuditStats — derive', () => { // A costUsd the pricing would never produce, so a match proves it was read, not computed. const stats = buildAuditStats(fsWithAudit('c1', [auditLine({ input: 1_000_000, costUsd: 999 })])); const expected = 999; - const actual = (await stats.derive('c1', CacheTtl.OneHour)).costUsd; + const actual = (await stats.derive('c1', CacheTtl.OneHour)).totals.costUsd; expect(actual).toBe(expected); }); @@ -100,7 +104,7 @@ describe('AuditStats — derive', () => { // fable-5: 5m at 12.5/M, 1h at 20/M → 1M each = 32.5 const stats = buildAuditStats(fsWithAudit('c1', [auditLine({ model: 'claude-fable-5', cacheSplit: { fiveMinute: 1_000_000, oneHour: 1_000_000 } })])); const expected = 32.5; - const actual = (await stats.derive('c1', CacheTtl.OneHour)).costUsd; + const actual = (await stats.derive('c1', CacheTtl.OneHour)).totals.costUsd; expect(actual).toBe(expected); }); @@ -108,7 +112,7 @@ describe('AuditStats — derive', () => { // flat 1M all 1h → priced at the 1h rate = 20 const stats = buildAuditStats(fsWithAudit('c1', [auditLine({ model: 'claude-fable-5', cacheCreation: 1_000_000, ephemeral: { ephemeral_5m_input_tokens: 0, ephemeral_1h_input_tokens: 1_000_000 } })])); const expected = 20; - const actual = (await stats.derive('c1', CacheTtl.OneHour)).costUsd; + const actual = (await stats.derive('c1', CacheTtl.OneHour)).totals.costUsd; expect(actual).toBe(expected); }); @@ -116,14 +120,14 @@ describe('AuditStats — derive', () => { // No costUsd, no stored breakdown, no ephemeral object: flat 1M against the configured TTL. const stats = buildAuditStats(fsWithAudit('c1', [auditLine({ model: 'claude-fable-5', cacheCreation: 1_000_000 })])); const expected = 20; - const actual = (await stats.derive('c1', CacheTtl.OneHour)).costUsd; + const actual = (await stats.derive('c1', CacheTtl.OneHour)).totals.costUsd; expect(actual).toBe(expected); }); it('prices a legacy flat-only line at the 5m rate when 5m is configured', async () => { const stats = buildAuditStats(fsWithAudit('c1', [auditLine({ model: 'claude-fable-5', cacheCreation: 1_000_000 })])); const expected = 12.5; - const actual = (await stats.derive('c1', CacheTtl.FiveMinutes)).costUsd; + const actual = (await stats.derive('c1', CacheTtl.FiveMinutes)).totals.costUsd; expect(actual).toBe(expected); }); @@ -131,7 +135,7 @@ describe('AuditStats — derive', () => { const userLine = JSON.stringify({ role: 'user', content: 'hello' }); const stats = buildAuditStats(fsWithAudit('c1', [auditLine({ input: 100 }), userLine, auditLine({ input: 40 })])); const expected = 140; - const actual = (await stats.derive('c1', CacheTtl.OneHour)).inputTokens; + const actual = (await stats.derive('c1', CacheTtl.OneHour)).totals.inputTokens; expect(actual).toBe(expected); }); @@ -139,7 +143,7 @@ describe('AuditStats — derive', () => { const malformed = JSON.stringify({ model: 'claude-fable-5' }); const stats = buildAuditStats(fsWithAudit('c1', [auditLine({ input: 100 }), malformed])); const expected = 100; - const actual = (await stats.derive('c1', CacheTtl.OneHour)).inputTokens; + const actual = (await stats.derive('c1', CacheTtl.OneHour)).totals.inputTokens; expect(actual).toBe(expected); }); @@ -147,7 +151,7 @@ describe('AuditStats — derive', () => { const systemLine = JSON.stringify({ role: 'system', content: 'hi' }); const stats = buildAuditStats(fsWithAudit('c1', [auditLine({ input: 100 }), systemLine])); const expected = 100; - const actual = (await stats.derive('c1', CacheTtl.OneHour)).inputTokens; + const actual = (await stats.derive('c1', CacheTtl.OneHour)).totals.inputTokens; expect(actual).toBe(expected); }); @@ -155,7 +159,52 @@ describe('AuditStats — derive', () => { const userLine = JSON.stringify({ role: 'user', content: 'hello' }); const stats = buildAuditStats(fsWithAudit('c1', [userLine, userLine])); const expected = 0; - const actual = (await stats.derive('c1', CacheTtl.OneHour)).inputTokens; + const actual = (await stats.derive('c1', CacheTtl.OneHour)).totals.inputTokens; + expect(actual).toBe(expected); + }); +}); + +describe('AuditStats — cached parameters', () => { + it('reads what the last request was sent with', async () => { + const stats = buildAuditStats(fsWithAudit('c1', [auditLine({ model: 'claude-fable-5', request: { thinking: true, effort: 'high' } })])); + const expected = { model: 'claude-fable-5', thinking: true, effort: 'high' }; + const actual = (await stats.derive('c1', CacheTtl.OneHour)).cached; + expect(actual).toEqual(expected); + }); + + it('takes the model from the line the API reported it on, not from the recorded request', async () => { + const stats = buildAuditStats(fsWithAudit('c1', [auditLine({ model: 'claude-opus-4-8', request: { thinking: false, effort: null } })])); + const expected = 'claude-opus-4-8'; + const actual = (await stats.derive('c1', CacheTtl.OneHour)).cached?.model; + expect(actual).toBe(expected); + }); + + it('takes the parameters from the final line when earlier lines differ', async () => { + const lines = [auditLine({ request: { thinking: true, effort: 'low' } }), auditLine({ request: { thinking: true, effort: 'max' } })]; + const stats = buildAuditStats(fsWithAudit('c1', lines)); + const expected = 'max'; + const actual = (await stats.derive('c1', CacheTtl.OneHour)).cached?.effort; + expect(actual).toBe(expected); + }); + + it('carries a null effort through as the model default rather than dropping it', async () => { + const stats = buildAuditStats(fsWithAudit('c1', [auditLine({ request: { thinking: true, effort: null } })])); + const expected = null; + const actual = (await stats.derive('c1', CacheTtl.OneHour)).cached?.effort; + expect(actual).toBe(expected); + }); + + it('reads nothing when the lines predate the parameters being recorded', async () => { + const stats = buildAuditStats(fsWithAudit('c1', [auditLine({ input: 100 })])); + const expected = null; + const actual = (await stats.derive('c1', CacheTtl.OneHour)).cached; + expect(actual).toBe(expected); + }); + + it('reads nothing when the id has no audit file', async () => { + const stats = buildAuditStats(new MemoryFileSystem({}, '/home/user')); + const expected = null; + const actual = (await stats.derive('missing-id', CacheTtl.OneHour)).cached; expect(actual).toBe(expected); }); }); diff --git a/apps/claude-sdk-cli/test/AuditWriter.spec.ts b/apps/claude-sdk-cli/test/AuditWriter.spec.ts index 748ea8c7..e6638de2 100644 --- a/apps/claude-sdk-cli/test/AuditWriter.spec.ts +++ b/apps/claude-sdk-cli/test/AuditWriter.spec.ts @@ -6,6 +6,7 @@ import type { MessageIdentity } from '@shellicar/claude-sdk'; import { createServiceCollection, Lifetime } from '@shellicar/core-di'; import { describe, expect, it } from 'vitest'; import { AuditWriter } from '../src/AuditWriter.js'; +import { scanAuditSummary } from '../src/conversations/scanAuditSummary.js'; import { MemoryFileSystem } from './MemoryFileSystem.js'; // AuditWriter now derives its dir as `${fs.homedir()}/.claude/audit`; with the @@ -399,3 +400,66 @@ describe('AuditWriter — best-effort index projection', () => { expect(act).not.toThrow(); }); }); + +// --------------------------------------------------------------------------- +// the parameters the prompt cache keys on +// --------------------------------------------------------------------------- + +describe('AuditWriter — the parameters a turn was sent with', () => { + const readAssistantLine = async (fs: MemoryFileSystem, id: string): Promise> => { + const content = await fs.readFile(`${AUDIT_DIR}/${id}.jsonl`); + const lines = content.trimEnd().split('\n'); + return JSON.parse(lines[lines.length - 1] ?? '{}'); + }; + + it('records the thinking and effort the request went out under', async () => { + const fs = new MemoryFileSystem({}, '/home/user'); + const writer = buildAuditWriter(fs); + writer.write('conv-p1', undefined, makeMessage(), makeIdentity(), { model: 'claude-fable-5', thinking: true, effort: 'high' }); + + await new Promise((r) => setTimeout(r, 10)); + + const expected = { thinking: true, effort: 'high' }; + const actual = (await readAssistantLine(fs, 'conv-p1')).request; + expect(actual).toEqual(expected); + }); + + it('leaves the model to the field the API reported it on rather than recording it twice', async () => { + const fs = new MemoryFileSystem({}, '/home/user'); + const writer = buildAuditWriter(fs); + writer.write('conv-p2', undefined, makeMessage(), makeIdentity(), { model: 'a-model-we-claim', thinking: true, effort: null }); + + await new Promise((r) => setTimeout(r, 10)); + + const expected = 'claude-sonnet-4-20250514'; + const actual = (await readAssistantLine(fs, 'conv-p2')).model; + expect(actual).toBe(expected); + }); + + it('writes no request field at all when the parameters are unknown', async () => { + const fs = new MemoryFileSystem({}, '/home/user'); + const writer = buildAuditWriter(fs); + writer.write('conv-p3', undefined, makeMessage(), makeIdentity(), null); + + await new Promise((r) => setTimeout(r, 10)); + + const expected = false; + const actual = 'request' in (await readAssistantLine(fs, 'conv-p3')); + expect(actual).toBe(expected); + }); + + it('leaves the model where the conversation list scanner still finds it', async () => { + // The scanner reads the model from a bounded head window, so anything written ahead of it eats + // into that budget. This is what holds the parameters to the line's tail. + const fs = new MemoryFileSystem({}, '/home/user'); + const writer = buildAuditWriter(fs); + writer.write('conv-p4', undefined, makeMessage(), makeIdentity(), { model: 'claude-fable-5', thinking: true, effort: 'high' }); + + await new Promise((r) => setTimeout(r, 10)); + + const content = await fs.readFile(`${AUDIT_DIR}/conv-p4.jsonl`); + const expected = 'claude-sonnet-4-20250514'; + const actual = scanAuditSummary(Buffer.from(content, 'utf8')).model; + expect(actual).toBe(expected); + }); +}); diff --git a/apps/claude-sdk-cli/test/CacheWarning.spec.ts b/apps/claude-sdk-cli/test/CacheWarning.spec.ts new file mode 100644 index 00000000..bc4b2ac0 --- /dev/null +++ b/apps/claude-sdk-cli/test/CacheWarning.spec.ts @@ -0,0 +1,404 @@ +import { IConversation, IDurableConfigProvider, ITokenCounter } from '@shellicar/claude-sdk'; +import { createServiceCollection, Lifetime } from '@shellicar/core-di'; +import { describe, expect, it } from 'vitest'; +import type { AuditDerivation } from '../src/AuditStats.js'; +import type { CacheParameters } from '../src/model/ModelSettings.js'; +import { ModelSettings } from '../src/model/ModelSettings.js'; +import { StatusState } from '../src/model/StatusState.js'; +import { CacheWarning, ICacheWarning } from '../src/setup/CacheWarning.js'; +import { FakeModelSettings } from './FakeModelSettings.js'; + +const MODEL = 'claude-fable-5'; + +// `effort` is read by presence, not by ??, so a deliberate null reaches the subject as the model +// default rather than being swallowed back into the fallback. +function parameters(fields: Partial = {}): CacheParameters { + return { model: fields.model ?? MODEL, thinking: fields.thinking ?? true, effort: 'effort' in fields ? (fields.effort ?? null) : 'low' }; +} + +/** + * A counter whose answers land when the test says so, so the window between asking the API and + * hearing back is a thing the test can stand inside rather than a race it has to hope about. + */ +class FakeTokenCounter extends ITokenCounter { + readonly #resolvers: Array<(value: number | null) => void> = []; + + public get asked(): number { + return this.#resolvers.length; + } + + public count(): Promise { + return new Promise((resolve) => { + this.#resolvers.push(resolve); + }); + } + + /** Answer the nth outstanding question and let the continuation run. */ + public async land(value: number | null, at = 0): Promise { + this.#resolvers[at]?.(value); + await new Promise((resolve) => setTimeout(resolve, 0)); + } +} + +/** The live settings a request would go out under right now, held by the config provider the + * warning reads through. */ +function build(live: CacheParameters, contextTokens = 0): { warning: ICacheWarning; settings: FakeModelSettings; status: StatusState; counter: FakeTokenCounter } { + const settings = new FakeModelSettings(); + const status = new StatusState('test'); + const counter = new FakeTokenCounter(); + status.resetTo({ inputTokens: 0, cacheCreationTokens: 0, cacheReadTokens: 0, outputTokens: 0, costUsd: 0, lastContextUsed: contextTokens, contextWindow: 1_000_000 }); + const services = createServiceCollection({ defaultLifetime: Lifetime.Singleton }); + services + .register(IDurableConfigProvider) + .using( + () => + ({ + getEffectiveModel: () => live.model, + getEffectiveThinkingEnabled: () => live.thinking, + getEffectiveEffort: () => live.effort ?? undefined, + config: { model: live.model, maxTokens: 16, tools: [] }, + }) as unknown as IDurableConfigProvider, + ) + .asSelf(); + services + .register(ITokenCounter) + .using(() => counter) + .asSelf(); + services + .register(IConversation) + .using(() => ({ cloneForRequest: () => [] }) as unknown as IConversation) + .asSelf(); + services + .register(ModelSettings) + .using(() => settings) + .asSelf(); + services + .register(StatusState) + .using(() => status) + .asSelf(); + services.register(CacheWarning).as(ICacheWarning); + return { warning: services.buildProvider().resolve(ICacheWarning), settings, status, counter }; +} + +describe('CacheWarning — when there is nothing to warn about', () => { + it('says nothing for a conversation that has sent nothing', () => { + const { warning, status } = build(parameters()); + + warning.refresh(); + + const actual = status.cacheDivergence; + + expect(actual).toBeNull(); + }); + + it('says nothing while the live settings still match what was sent', () => { + const { warning, settings, status } = build(parameters()); + settings.adopt(parameters()); + + warning.refresh(); + + const actual = status.cacheDivergence; + + expect(actual).toBeNull(); + }); + + it('stops warning once a request goes out under the new settings', () => { + const { warning, settings, status } = build(parameters({ effort: 'max' })); + settings.adopt(parameters({ effort: 'low' })); + warning.refresh(); + + settings.markSent(parameters({ effort: 'max' })); + warning.refresh(); + + const actual = status.cacheDivergence; + + expect(actual).toBeNull(); + }); +}); + +describe('CacheWarning — what has moved', () => { + it('names the effort that has moved away from the cached one', () => { + const { warning, settings, status } = build(parameters({ effort: 'max' })); + settings.adopt(parameters({ effort: 'low' })); + + warning.refresh(); + + const expected = [{ name: 'effort', from: 'low', to: 'max' }]; + const actual = status.cacheDivergence?.changes; + + expect(actual).toEqual(expected); + }); + + it('names the model that has moved away from the cached one', () => { + const { warning, settings, status } = build(parameters({ model: 'claude-opus-4-8' })); + settings.adopt(parameters({ model: MODEL })); + + warning.refresh(); + + const expected = [{ name: 'model', from: MODEL, to: 'claude-opus-4-8' }]; + const actual = status.cacheDivergence?.changes; + + expect(actual).toEqual(expected); + }); + + it('reads thinking as on and off rather than as a boolean', () => { + const { warning, settings, status } = build(parameters({ thinking: false })); + settings.adopt(parameters({ thinking: true })); + + warning.refresh(); + + const expected = [{ name: 'thinking', from: 'on', to: 'off' }]; + const actual = status.cacheDivergence?.changes; + + expect(actual).toEqual(expected); + }); + + it('reads an absent effort as the model default rather than as nothing', () => { + const { warning, settings, status } = build(parameters({ effort: null })); + settings.adopt(parameters({ effort: 'low' })); + + warning.refresh(); + + const expected = 'default'; + const actual = status.cacheDivergence?.changes[0]?.to; + + expect(actual).toBe(expected); + }); + + it('names every parameter that has moved, not just the first', () => { + const { warning, settings, status } = build(parameters({ model: 'claude-opus-4-8', thinking: false, effort: 'max' })); + settings.adopt(parameters({ model: MODEL, thinking: true, effort: 'low' })); + + warning.refresh(); + + const expected = ['model', 'thinking', 'effort']; + const actual = status.cacheDivergence?.changes.map((change) => change.name); + + expect(actual).toEqual(expected); + }); +}); + +describe('CacheWarning — what it would cost', () => { + it('counts the whole of the last request context as the prefix that would be re-written', () => { + const { warning, settings, status } = build(parameters({ effort: 'max' }), 250_000); + settings.adopt(parameters({ effort: 'low' })); + + warning.refresh(); + + const expected = 250_000; + const actual = status.cacheDivergence?.tokens; + + expect(actual).toBe(expected); + }); + + it('prices that prefix at the one-hour cache write rate of the model the next request would use', () => { + // claude-fable-5 writes a 1h cache at $20/M, so a million tokens is $20. + const { warning, settings, status } = build(parameters({ effort: 'max' }), 1_000_000); + settings.adopt(parameters({ effort: 'low' })); + + warning.refresh(); + + const expected = 20; + const actual = status.cacheDivergence?.costUsd; + + expect(actual).toBe(expected); + }); +}); + +describe('CacheWarning — what a conversation is measured against when it is adopted', () => { + const derivation = (fields: Partial): AuditDerivation => ({ totals: {}, cached: fields.cached ?? null, lastModel: fields.lastModel ?? null }) as AuditDerivation; + + it('uses what the audit recorded when the audit recorded it', () => { + const { warning } = build(parameters({ effort: 'max' })); + const audit = derivation({ cached: parameters({ effort: 'low' }), lastModel: MODEL }); + + const expected = 'low'; + const actual = warning.baselineFor(audit)?.effort; + + expect(actual).toBe(expected); + }); + + it('assumes the live effort for a conversation whose turns predate the recording', () => { + const { warning } = build(parameters({ effort: 'max' })); + const audit = derivation({ cached: null, lastModel: MODEL }); + + const expected = 'max'; + const actual = warning.baselineFor(audit)?.effort; + + expect(actual).toBe(expected); + }); + + it('takes the model from the audit rather than assuming it, since every line carries one', () => { + const { warning } = build(parameters({ model: 'claude-haiku-4-5' })); + const audit = derivation({ cached: null, lastModel: 'claude-opus-4-8' }); + + const expected = 'claude-opus-4-8'; + const actual = warning.baselineFor(audit)?.model; + + expect(actual).toBe(expected); + }); + + it('measures nothing for a conversation that has sent nothing, which has no prefix to lose', () => { + const { warning } = build(parameters()); + const audit = derivation({ cached: null, lastModel: null }); + + const actual = warning.baselineFor(audit); + + expect(actual).toBeNull(); + }); +}); + +describe('CacheWarning — a size only the new model can give', () => { + it('trusts the last count when the model has not moved', () => { + const { warning, settings, status } = build(parameters({ effort: 'max' }), 250_000); + settings.adopt(parameters({ effort: 'low' })); + + warning.refresh(); + + const expected = 250_000; + const actual = status.cacheDivergence?.tokens; + + expect(actual).toBe(expected); + }); + + it('asks nobody when the model has not moved, since the count already belongs to it', () => { + const { warning, settings, counter } = build(parameters({ effort: 'max' }), 250_000); + settings.adopt(parameters({ effort: 'low' })); + + warning.refresh(); + + const expected = 0; + const actual = counter.asked; + + expect(actual).toBe(expected); + }); + + it('reports no size at all while the new model is still counting', () => { + const { warning, settings, status } = build(parameters({ model: 'claude-sonnet-5' }), 12_226); + settings.adopt(parameters({ model: 'claude-sonnet-4-6' })); + + warning.refresh(); + + const actual = status.cacheDivergence?.tokens; + + expect(actual).toBeNull(); + }); + + it('reports no cost while there is no size to price', () => { + const { warning, settings, status } = build(parameters({ model: 'claude-sonnet-5' }), 12_226); + settings.adopt(parameters({ model: 'claude-sonnet-4-6' })); + + warning.refresh(); + + const actual = status.cacheDivergence?.costUsd; + + expect(actual).toBeNull(); + }); + + it('still names what moved while the size is unknown', () => { + const { warning, settings, status } = build(parameters({ model: 'claude-sonnet-5' }), 12_226); + settings.adopt(parameters({ model: 'claude-sonnet-4-6' })); + + warning.refresh(); + + const expected = ['model']; + const actual = status.cacheDivergence?.changes.map((change) => change.name); + + expect(actual).toEqual(expected); + }); + + it('takes the new model count when it lands', async () => { + const { warning, settings, status, counter } = build(parameters({ model: 'claude-sonnet-5' }), 12_226); + settings.adopt(parameters({ model: 'claude-sonnet-4-6' })); + warning.refresh(); + + await counter.land(15_910); + + const expected = 15_910; + const actual = status.cacheDivergence?.tokens; + + expect(actual).toBe(expected); + }); + + it('leaves the size unknown when the API cannot say', async () => { + const { warning, settings, status, counter } = build(parameters({ model: 'claude-sonnet-5' }), 12_226); + settings.adopt(parameters({ model: 'claude-sonnet-4-6' })); + warning.refresh(); + + await counter.land(null); + + const actual = status.cacheDivergence?.tokens; + + expect(actual).toBeNull(); + }); + + it('drops a count that answers a comparison the operator has already moved past', async () => { + const { warning, settings, status, counter } = build(parameters({ model: 'claude-sonnet-5' }), 12_226); + settings.adopt(parameters({ model: 'claude-sonnet-4-6' })); + warning.refresh(); + // A second keypress supersedes the first, and only then does the first answer arrive. + settings.adopt(parameters({ model: 'claude-sonnet-5' })); + warning.refresh(); + + await counter.land(15_910, 0); + + const actual = status.cacheDivergence; + + expect(actual).toBeNull(); + }); +}); + +describe('CacheWarning — counting ahead of the choice', () => { + it('has the size the instant the model is chosen', async () => { + const { warning, settings, status, counter } = build(parameters({ model: 'claude-sonnet-5' }), 12_226); + settings.adopt(parameters({ model: 'claude-sonnet-4-6' })); + warning.prefetch('claude-sonnet-5'); + await counter.land(15_910); + + warning.refresh(); + + const expected = 15_910; + const actual = status.cacheDivergence?.tokens; + + expect(actual).toBe(expected); + }); + + it('asks nobody again for a model it has already counted', async () => { + const { warning, settings, counter } = build(parameters({ model: 'claude-sonnet-5' }), 12_226); + settings.adopt(parameters({ model: 'claude-sonnet-4-6' })); + warning.prefetch('claude-sonnet-5'); + await counter.land(15_910); + + warning.refresh(); + + const expected = 1; + const actual = counter.asked; + + expect(actual).toBe(expected); + }); + + it('asks once while a count for the same model is still outstanding', () => { + const { warning, counter } = build(parameters({ model: 'claude-sonnet-5' }), 12_226); + + warning.prefetch('claude-sonnet-5'); + warning.prefetch('claude-sonnet-5'); + + const expected = 1; + const actual = counter.asked; + + expect(actual).toBe(expected); + }); + + it('ignores a count taken for a model the operator did not end up choosing', async () => { + const { warning, settings, status, counter } = build(parameters({ model: 'claude-sonnet-5' }), 12_226); + settings.adopt(parameters({ model: 'claude-sonnet-4-6' })); + warning.prefetch('claude-haiku-4-5'); + await counter.land(4_000); + + warning.refresh(); + + const actual = status.cacheDivergence?.tokens; + + expect(actual).toBeNull(); + }); +}); diff --git a/apps/claude-sdk-cli/test/CommandIntentExecutor.spec.ts b/apps/claude-sdk-cli/test/CommandIntentExecutor.spec.ts index 3f7db8e4..71f5410d 100644 --- a/apps/claude-sdk-cli/test/CommandIntentExecutor.spec.ts +++ b/apps/claude-sdk-cli/test/CommandIntentExecutor.spec.ts @@ -27,10 +27,13 @@ import { StatusState } from '../src/model/StatusState.js'; import { SystemIdentity } from '../src/model/SystemIdentity.js'; import { IWorkingDirectory, WorkingDirectory } from '../src/model/WorkingDirectory.js'; import { ISqliteSessionStore, SqliteSessionStore } from '../src/persistence/SqliteSessionStore.js'; +import { ICacheWarning } from '../src/setup/CacheWarning.js'; import { ConversationSwitcher, IConversationSwitcher } from '../src/setup/ConversationSwitcher.js'; import { IWorkspace } from '../src/workspace/Workspace.js'; import { buildCommandModeState } from './buildCommandModeState.js'; import { FakeAttachmentSource } from './FakeAttachmentSource.js'; +import { FakeCacheWarning } from './FakeCacheWarning.js'; +import { FakeModelSettings } from './FakeModelSettings.js'; import { FakeWorkspace } from './FakeWorkspace.js'; import { MemoryFileSystem } from './MemoryFileSystem.js'; import { MemoryObjectStore } from './MemoryObjectStore.js'; @@ -48,19 +51,8 @@ function makeExecutor(source: AttachmentSource) { const commandModeState = buildCommandModeState(); const fs = new MemoryFileSystem({}, '/home/user', '/test'); const conversation = new Conversation(); - const cycleCalls = { thinking: 0, effort: 0 }; - const modelCalls: { model: (string | null)[] } = { model: [] }; - const modelSettings: ModelSettings = { - cycleThinking: () => { - cycleCalls.thinking += 1; - }, - cycleEffort: () => { - cycleCalls.effort += 1; - }, - setModel: (id) => { - modelCalls.model.push(id); - }, - }; + const modelSettings = new FakeModelSettings(); + const { cycleCalls, modelCalls } = modelSettings; const catalogueModels: ModelInfo[] = [{ id: 'claude-opus-4-8', displayName: 'Claude Opus 4.8' }]; const modelCatalog: IModelCatalog = { list: () => Promise.resolve(catalogueModels) }; const services = createServiceCollection({ defaultLifetime: Lifetime.Singleton }); @@ -107,6 +99,10 @@ function makeExecutor(source: AttachmentSource) { .register(ModelSettings) .using(() => modelSettings) .asSelf(); + services + .register(ICacheWarning) + .using(() => new FakeCacheWarning()) + .asSelf(); services .register(IModelCatalog) .using(() => modelCatalog) diff --git a/apps/claude-sdk-cli/test/CommandKeyHandler.spec.ts b/apps/claude-sdk-cli/test/CommandKeyHandler.spec.ts index 49d771f5..affced75 100644 --- a/apps/claude-sdk-cli/test/CommandKeyHandler.spec.ts +++ b/apps/claude-sdk-cli/test/CommandKeyHandler.spec.ts @@ -28,10 +28,13 @@ import { StatusState } from '../src/model/StatusState.js'; import { SystemIdentity } from '../src/model/SystemIdentity.js'; import { IWorkingDirectory, WorkingDirectory } from '../src/model/WorkingDirectory.js'; import { ISqliteSessionStore, SqliteSessionStore } from '../src/persistence/SqliteSessionStore.js'; +import { ICacheWarning } from '../src/setup/CacheWarning.js'; import { ConversationSwitcher, IConversationSwitcher } from '../src/setup/ConversationSwitcher.js'; import { IWorkspace } from '../src/workspace/Workspace.js'; import { buildCommandModeState } from './buildCommandModeState.js'; import { FakeAttachmentSource } from './FakeAttachmentSource.js'; +import { FakeCacheWarning } from './FakeCacheWarning.js'; +import { FakeModelSettings } from './FakeModelSettings.js'; import { FakeWorkspace } from './FakeWorkspace.js'; import { MemoryFileSystem } from './MemoryFileSystem.js'; import { MemoryObjectStore } from './MemoryObjectStore.js'; @@ -52,16 +55,8 @@ function makeHandler(sourceText: string | null = null) { const fs = new MemoryFileSystem({}, '/home/user', '/test'); const conversation = new Conversation(); const source = new FakeAttachmentSource({ text: sourceText }); - const cycleCalls = { thinking: 0, effort: 0 }; - const modelSettings: ModelSettings = { - cycleThinking: () => { - cycleCalls.thinking += 1; - }, - cycleEffort: () => { - cycleCalls.effort += 1; - }, - setModel: () => {}, - }; + const modelSettings = new FakeModelSettings(); + const { cycleCalls } = modelSettings; const modelCatalog: IModelCatalog = { list: () => Promise.resolve([]) }; const services = createServiceCollection({ defaultLifetime: Lifetime.Singleton }); services.register(IntlGraphemeSegmenter).asSelf().as(IGraphemeSegmenter); @@ -107,6 +102,10 @@ function makeHandler(sourceText: string | null = null) { .register(ModelSettings) .using(() => modelSettings) .asSelf(); + services + .register(ICacheWarning) + .using(() => new FakeCacheWarning()) + .asSelf(); services .register(IModelCatalog) .using(() => modelCatalog) diff --git a/apps/claude-sdk-cli/test/ConversationBootSequence.spec.ts b/apps/claude-sdk-cli/test/ConversationBootSequence.spec.ts index f4fe6bec..a288eafa 100644 --- a/apps/claude-sdk-cli/test/ConversationBootSequence.spec.ts +++ b/apps/claude-sdk-cli/test/ConversationBootSequence.spec.ts @@ -3,6 +3,7 @@ import { ConfigLoader } from '@shellicar/claude-core/Config/ConfigLoader'; import { IConfigWatcher } from '@shellicar/claude-core/Config/interfaces'; import { IFileSystem } from '@shellicar/claude-core/fs/interfaces'; import { ILogger } from '@shellicar/claude-core/logging/ILogger'; +import { IObjectStore } from '@shellicar/claude-core/persistence/interfaces'; import { CacheTtl, Conversation, IConversation, IDurableConfigProvider } from '@shellicar/claude-sdk'; import { createServiceCollection, Lifetime } from '@shellicar/core-di'; import { describe, expect, it } from 'vitest'; @@ -15,14 +16,17 @@ import { ConversationState, IConversationState } from '../src/model/Conversation import { ISystemIdentity } from '../src/model/ISystemIdentity.js'; import { StatusState } from '../src/model/StatusState.js'; import { HistorySweepScheduler } from '../src/persistence/HistorySweepScheduler.js'; +import { ICacheWarning } from '../src/setup/CacheWarning.js'; import { ConversationBootSequence } from '../src/setup/ConversationBootSequence.js'; import { IRuntimeOptions } from '../src/setup/IRuntimeOptions.js'; import { ModelOverrides } from '../src/setup/ModelOverrides.js'; import { ISdkEventBridge } from '../src/setup/SdkEventBridge.js'; import { IShutdownSequence } from '../src/setup/ShutdownSequence.js'; import { IWorkspace, type Refusal } from '../src/workspace/Workspace.js'; +import { FakeCacheWarning } from './FakeCacheWarning.js'; import { FakeWorkspace } from './FakeWorkspace.js'; import { MemoryFileSystem } from './MemoryFileSystem.js'; +import { MemoryObjectStore } from './MemoryObjectStore.js'; const CONVERSATION_ID = 'boot-conversation'; const REFUSAL = { reason: '/tmp/claude-501 is owned by another user', remedy: 'Nothing on your side can change that; the scratchpad stays off.' }; @@ -63,6 +67,10 @@ function buildBootSequence(options: { refusal?: Refusal | null; history?: boolea .asSelf() .as(IConversation); services.register(ConversationState).asSelf().as(IConversationState); + services + .register(IObjectStore) + .using(() => new MemoryObjectStore()) + .asSelf(); services .register(StatusState) .using(() => new StatusState('test')) @@ -115,11 +123,15 @@ function buildBootSequence(options: { refusal?: Refusal | null; history?: boolea .asSelf(); services .register(ModelOverrides) - .using(() => ({ model: null }) as unknown as ModelOverrides) + .using(() => ({ model: null, adopt: () => {} }) as unknown as ModelOverrides) .asSelf(); services .register(AuditStats) - .using(() => ({ derive: async () => ({}) }) as unknown as AuditStats) + .using(() => ({ derive: async () => ({ totals: {}, cached: null, lastModel: null }) }) as unknown as AuditStats) + .asSelf(); + services + .register(ICacheWarning) + .using(() => new FakeCacheWarning()) .asSelf(); services .register(ViewHost) diff --git a/apps/claude-sdk-cli/test/ConversationSwitcher.spec.ts b/apps/claude-sdk-cli/test/ConversationSwitcher.spec.ts index c3bffbd9..342eff13 100644 --- a/apps/claude-sdk-cli/test/ConversationSwitcher.spec.ts +++ b/apps/claude-sdk-cli/test/ConversationSwitcher.spec.ts @@ -14,12 +14,16 @@ import { logger } from '../src/logger.js'; import { ConversationSession, IConversationSession } from '../src/model/ConversationSession.js'; import { ConversationState, IConversationState } from '../src/model/ConversationState.js'; import { ISystemIdentity } from '../src/model/ISystemIdentity.js'; +import { ModelSettings } from '../src/model/ModelSettings.js'; import { IPrimaryViewState, PrimaryViewState } from '../src/model/PrimaryViewState.js'; import { StatusState } from '../src/model/StatusState.js'; import { SystemIdentity } from '../src/model/SystemIdentity.js'; import { ISqliteSessionStore, SqliteSessionStore } from '../src/persistence/SqliteSessionStore.js'; +import { ICacheWarning } from '../src/setup/CacheWarning.js'; import { ConversationSwitcher, IConversationSwitcher } from '../src/setup/ConversationSwitcher.js'; import { IWorkspace } from '../src/workspace/Workspace.js'; +import { FakeCacheWarning } from './FakeCacheWarning.js'; +import { FakeModelSettings } from './FakeModelSettings.js'; import { FakeWorkspace } from './FakeWorkspace.js'; import { MemoryFileSystem } from './MemoryFileSystem.js'; import { MemoryObjectStore } from './MemoryObjectStore.js'; @@ -64,6 +68,14 @@ function makeSwitcher(workspace = new FakeWorkspace()) { .using(() => new MemoryObjectStore()) .asSelf(); services.register(SystemIdentity).as(ISystemIdentity); + services + .register(ModelSettings) + .using(() => new FakeModelSettings()) + .asSelf(); + services + .register(ICacheWarning) + .using(() => new FakeCacheWarning()) + .asSelf(); services.register(PrimaryViewState).asSelf().as(IPrimaryViewState); services .register(ConfigLoader) diff --git a/apps/claude-sdk-cli/test/DisabledToolsRequestWiring.spec.ts b/apps/claude-sdk-cli/test/DisabledToolsRequestWiring.spec.ts index 127e2f29..ce658d28 100644 --- a/apps/claude-sdk-cli/test/DisabledToolsRequestWiring.spec.ts +++ b/apps/claude-sdk-cli/test/DisabledToolsRequestWiring.spec.ts @@ -7,6 +7,7 @@ import { IConfigFileReader } from '@shellicar/claude-core/Config/interfaces'; import { readConfig } from '@shellicar/claude-core/Config/readConfig'; import { IFileSystem } from '@shellicar/claude-core/fs/interfaces'; import { ILogger } from '@shellicar/claude-core/logging/ILogger'; +import { IObjectStore } from '@shellicar/claude-core/persistence/interfaces'; import { IRandomProvider } from '@shellicar/claude-core/providers/IRandomProvider'; import { ISleepProvider } from '@shellicar/claude-core/providers/ISleepProvider'; import { @@ -138,6 +139,10 @@ function buildHarness(tools: AnyToolDefinition[], disabledTools: string[]) { .register(ConfigLoader) .using(() => makeLoader(disabledTools)) .asSelf(); + services + .register(IObjectStore) + .using(() => new MemoryObjectStore()) + .asSelf(); services.register(ModelOverrides).asSelf(); services .register(AppToolsService) diff --git a/apps/claude-sdk-cli/test/FakeCacheWarning.ts b/apps/claude-sdk-cli/test/FakeCacheWarning.ts new file mode 100644 index 00000000..737d0d75 --- /dev/null +++ b/apps/claude-sdk-cli/test/FakeCacheWarning.ts @@ -0,0 +1,23 @@ +import type { AuditDerivation } from '../src/AuditStats.js'; +import type { CacheParameters } from '../src/model/ModelSettings.js'; +import { ICacheWarning } from '../src/setup/CacheWarning.js'; + +/** A counting no-op `ICacheWarning`, for specs that exercise a seam the warning happens to sit on + * without asserting anything about it. What the warning says is proved in its own spec. */ +export class FakeCacheWarning extends ICacheWarning { + public refreshes = 0; + public readonly prefetched: string[] = []; + + public refresh(): void { + this.refreshes += 1; + } + + /** Takes the audit at its word and assumes nothing, so a spec sees only what the audit held. */ + public baselineFor(audit: AuditDerivation): CacheParameters | null { + return audit.cached; + } + + public prefetch(model: string): void { + this.prefetched.push(model); + } +} diff --git a/apps/claude-sdk-cli/test/FakeModelSettings.ts b/apps/claude-sdk-cli/test/FakeModelSettings.ts new file mode 100644 index 00000000..bad6835b --- /dev/null +++ b/apps/claude-sdk-cli/test/FakeModelSettings.ts @@ -0,0 +1,57 @@ +import type { ThinkingEffort } from '@shellicar/claude-sdk'; +import { type CacheParameters, ModelSettings } from '../src/model/ModelSettings.js'; + +/** + * In-memory `ModelSettings` for tests that only need the command-mode capability. Counts the cycle + * calls and keeps every model set, so a spec asserts on what the executor asked for rather than on + * how the real overrides resolve precedence. + */ +export class FakeModelSettings extends ModelSettings { + public readonly cycleCalls = { thinking: 0, effort: 0 }; + public readonly modelCalls: { model: (string | null)[] } = { model: [] }; + #cached: CacheParameters | null = null; + #model: string | null = null; + #thinking: 'on' | 'off' | null = null; + #effort: ThinkingEffort | null = null; + + public cycleThinking(): void { + this.cycleCalls.thinking += 1; + } + + public cycleEffort(): void { + this.cycleCalls.effort += 1; + } + + public setModel(id: string | null): void { + this.#model = id; + this.modelCalls.model.push(id); + } + + public get model(): string | null { + return this.#model; + } + + public get thinking(): 'on' | 'off' | null { + return this.#thinking; + } + + public get effort(): ThinkingEffort | null { + return this.#effort; + } + + public get cached(): CacheParameters | null { + return this.#cached; + } + + public markSent(params: CacheParameters): void { + this.#cached = params; + } + + public adopt(cached: CacheParameters | null): void { + this.#cached = cached; + } + + public carryOver(): void { + this.#cached = null; + } +} diff --git a/apps/claude-sdk-cli/test/ModelOverrides.spec.ts b/apps/claude-sdk-cli/test/ModelOverrides.spec.ts new file mode 100644 index 00000000..c7b46039 --- /dev/null +++ b/apps/claude-sdk-cli/test/ModelOverrides.spec.ts @@ -0,0 +1,198 @@ +import { ConfigLoader } from '@shellicar/claude-core/Config/ConfigLoader'; +import type { ThinkingEffort } from '@shellicar/claude-sdk'; +import { createServiceCollection, Lifetime } from '@shellicar/core-di'; +import { describe, expect, it } from 'vitest'; +import type { CacheParameters } from '../src/model/ModelSettings.js'; +import { StatusState } from '../src/model/StatusState.js'; +import { IRuntimeOptions } from '../src/setup/IRuntimeOptions.js'; +import { ModelOverrides } from '../src/setup/ModelOverrides.js'; + +const CONFIG_MODEL = 'claude-sonnet-4-5'; + +type ConfigDefaults = { model?: string; thinkingEnabled?: boolean; effort?: ThinkingEffort }; + +function build(modelFlag: string | null = null, defaults: ConfigDefaults = {}): ModelOverrides { + const config = { + model: defaults.model ?? CONFIG_MODEL, + thinking: { enabled: defaults.thinkingEnabled ?? false, effort: defaults.effort }, + }; + const services = createServiceCollection({ defaultLifetime: Lifetime.Singleton }); + services + .register(IRuntimeOptions) + .using(() => ({ modelOverride: modelFlag, systemFlagText: null, claudeMdFlagText: null, tsAvailable: false }) satisfies IRuntimeOptions) + .asSelf(); + services + .register(StatusState) + .using(() => new StatusState('test')) + .asSelf(); + services + .register(ConfigLoader) + .using(() => ({ config }) as unknown as ConfigLoader) + .asSelf(); + services.register(ModelOverrides).asSelf(); + return services.buildProvider().resolve(ModelOverrides); +} + +function sent(fields: Partial = {}): CacheParameters { + return { model: fields.model ?? CONFIG_MODEL, thinking: fields.thinking ?? false, effort: fields.effort ?? null }; +} + +describe('ModelOverrides — what a conversation resumes on', () => { + it('comes up on the model its cached prefix was written under', () => { + const overrides = build(); + + overrides.adopt(sent({ model: 'claude-opus-4-8' })); + + const expected = 'claude-opus-4-8'; + const actual = overrides.model; + + expect(actual).toBe(expected); + }); + + it('comes up on the effort its cached prefix was written under', () => { + const overrides = build(); + + overrides.adopt(sent({ effort: 'high' })); + + const expected = 'high'; + const actual = overrides.effort; + + expect(actual).toBe(expected); + }); + + it('comes up on the thinking its cached prefix was written under', () => { + const overrides = build(null, { thinkingEnabled: false }); + + overrides.adopt(sent({ thinking: true })); + + const expected = 'on'; + const actual = overrides.thinking; + + expect(actual).toBe(expected); + }); + + it('reports no model override when the cached model is the one the config already names', () => { + const overrides = build(); + + overrides.adopt(sent({ model: CONFIG_MODEL })); + + const expected = null; + const actual = overrides.model; + + expect(actual).toBe(expected); + }); + + it('reports no effort override when the cached effort is the one the config already names', () => { + const overrides = build(null, { effort: 'high' }); + + overrides.adopt(sent({ effort: 'high' })); + + const expected = null; + const actual = overrides.effort; + + expect(actual).toBe(expected); + }); + + it('holds no override for a conversation that has sent nothing', () => { + const overrides = build(); + + overrides.adopt(null); + + const expected = null; + const actual = overrides.model; + + expect(actual).toBe(expected); + }); + + it('drops a runtime change made against the conversation being left', () => { + const overrides = build(); + overrides.setModel('claude-haiku-4-5'); + + overrides.adopt(sent({ model: 'claude-opus-4-8' })); + + const expected = 'claude-opus-4-8'; + const actual = overrides.model; + + expect(actual).toBe(expected); + }); +}); + +describe('ModelOverrides — precedence', () => { + it('prefers the launch flag over what the conversation last sent', () => { + const overrides = build('claude-haiku-4-5'); + + overrides.adopt(sent({ model: 'claude-opus-4-8' })); + + const expected = 'claude-haiku-4-5'; + const actual = overrides.model; + + expect(actual).toBe(expected); + }); + + it('prefers a runtime change over the launch flag, so the operator can clear a divergence', () => { + const overrides = build('claude-haiku-4-5'); + + overrides.setModel('claude-opus-4-8'); + + const expected = 'claude-opus-4-8'; + const actual = overrides.model; + + expect(actual).toBe(expected); + }); + + it('advances effort from the value the conversation came up on, not from the head of the cycle', () => { + const overrides = build(); + overrides.adopt(sent({ effort: 'medium' })); + + overrides.cycleEffort(); + + const expected = 'high'; + const actual = overrides.effort; + + expect(actual).toBe(expected); + }); +}); + +describe('ModelOverrides — what the cached prefix was written under', () => { + it('holds nothing before the conversation has sent anything', () => { + const overrides = build(); + + const actual = overrides.cached; + + expect(actual).toBeNull(); + }); + + it('holds the parameters the last request was sent with', () => { + const overrides = build(); + + overrides.markSent(sent({ model: 'claude-opus-4-8', thinking: true, effort: 'max' })); + + const expected = { model: 'claude-opus-4-8', thinking: true, effort: 'max' }; + const actual = overrides.cached; + + expect(actual).toEqual(expected); + }); + + it('holds nothing for a new conversation, which has no cached prefix to lose', () => { + const overrides = build(); + overrides.markSent(sent({ model: 'claude-opus-4-8' })); + + overrides.carryOver(); + + const actual = overrides.cached; + + expect(actual).toBeNull(); + }); + + it('keeps the operator selection when a new conversation carries it over', () => { + const overrides = build(); + overrides.setModel('claude-opus-4-8'); + + overrides.carryOver(); + + const expected = 'claude-opus-4-8'; + const actual = overrides.model; + + expect(actual).toBe(expected); + }); +}); diff --git a/apps/claude-sdk-cli/test/ThinkingRequestWiring.spec.ts b/apps/claude-sdk-cli/test/ThinkingRequestWiring.spec.ts index f2b648ab..72bce25e 100644 --- a/apps/claude-sdk-cli/test/ThinkingRequestWiring.spec.ts +++ b/apps/claude-sdk-cli/test/ThinkingRequestWiring.spec.ts @@ -7,6 +7,7 @@ import { IConfigFileReader } from '@shellicar/claude-core/Config/interfaces'; import { readConfig } from '@shellicar/claude-core/Config/readConfig'; import { IFileSystem } from '@shellicar/claude-core/fs/interfaces'; import { ILogger } from '@shellicar/claude-core/logging/ILogger'; +import { IObjectStore } from '@shellicar/claude-core/persistence/interfaces'; import { IRandomProvider } from '@shellicar/claude-core/providers/IRandomProvider'; import { ISleepProvider } from '@shellicar/claude-core/providers/ISleepProvider'; import { AccountLimitListener, Conversation, type DurableConfig, IDurableConfigProvider, IMessageStreamer, IRequestClockListener, IStreamProcessor, IToolRegistry, IWakeLock, StreamInterruptListener, StreamProcessor, type ThinkingEffort, ToolRegistry, TurnRunner, type WakeLockHandle } from '@shellicar/claude-sdk'; @@ -176,6 +177,10 @@ function makeFactory(thinking: ThinkingConfig, override: Override): IDurableConf .register(ConfigLoader) .using(() => makeLoader(thinking)) .asSelf(); + services + .register(IObjectStore) + .using(() => new MemoryObjectStore()) + .asSelf(); services.register(ModelOverrides).asSelf(); services .register(AppToolsService) diff --git a/apps/claude-sdk-cli/test/ViewHost.spec.ts b/apps/claude-sdk-cli/test/ViewHost.spec.ts index 3dced99b..442f767a 100644 --- a/apps/claude-sdk-cli/test/ViewHost.spec.ts +++ b/apps/claude-sdk-cli/test/ViewHost.spec.ts @@ -41,6 +41,7 @@ import { IToolApprovalState, ToolApprovalState } from '../src/model/ToolApproval import { TurnClock } from '../src/model/TurnClock.js'; import { IWorkingDirectory, WorkingDirectory } from '../src/model/WorkingDirectory.js'; import { ISqliteSessionStore } from '../src/persistence/SqliteSessionStore.js'; +import { ICacheWarning } from '../src/setup/CacheWarning.js'; import { ConsumerChannel } from '../src/setup/ConsumerChannel.js'; import { ConversationSwitcher, IConversationSwitcher } from '../src/setup/ConversationSwitcher.js'; import { PrimaryView } from '../src/view/PrimaryView.js'; @@ -50,6 +51,8 @@ import { IWorkspace } from '../src/workspace/Workspace.js'; import { buildCommandModeState } from './buildCommandModeState.js'; import { buildEditorBuffer } from './buildEditorBuffer.js'; import { FakeAttachmentSource } from './FakeAttachmentSource.js'; +import { FakeCacheWarning } from './FakeCacheWarning.js'; +import { FakeModelSettings } from './FakeModelSettings.js'; import { FakeWorkspace } from './FakeWorkspace.js'; import { MemoryFileSystem } from './MemoryFileSystem.js'; import { MemoryObjectStore } from './MemoryObjectStore.js'; @@ -290,7 +293,11 @@ describe('ViewHost — escape routing through the primary chains', () => { .asSelf(); services .register(ModelSettings) - .using(() => ({ cycleThinking: () => {}, cycleEffort: () => {}, setModel: () => {} })) + .using(() => new FakeModelSettings()) + .asSelf(); + services + .register(ICacheWarning) + .using(() => new FakeCacheWarning()) .asSelf(); services .register(IModelCatalog) diff --git a/apps/claude-sdk-cli/test/renderStatus.spec.ts b/apps/claude-sdk-cli/test/renderStatus.spec.ts index d0555b91..ac843395 100644 --- a/apps/claude-sdk-cli/test/renderStatus.spec.ts +++ b/apps/claude-sdk-cli/test/renderStatus.spec.ts @@ -450,3 +450,118 @@ describe('renderModel — build version', () => { expect(actual).toBe(expected); }); }); + +// --------------------------------------------------------------------------- +// The pending cache cost +// --------------------------------------------------------------------------- + +describe('renderModel — the pending cache cost', () => { + it('shows nothing while the live settings and the cached prefix agree', () => { + const state = makeStatusState(); + state.setModel('claude-opus-4-8'); + + const expected = false; + const actual = renderModel(state, 200, '').includes('rewrites'); + + expect(actual).toBe(expected); + }); + + it('names the setting that moved and where it moved from', () => { + const state = makeStatusState(); + state.setModel('claude-opus-4-8'); + state.setCacheDivergence({ changes: [{ name: 'effort', from: 'low', to: 'high' }], tokens: 101_900, costUsd: 1.019 }); + + const expected = true; + const actual = renderModel(state, 200, '').includes('effort low\u2192high'); + + expect(actual).toBe(expected); + }); + + it('shows the size of the prefix that would be re-written', () => { + const state = makeStatusState(); + state.setModel('claude-opus-4-8'); + state.setCacheDivergence({ changes: [{ name: 'effort', from: 'low', to: 'high' }], tokens: 101_900, costUsd: 1.019 }); + + const expected = true; + const actual = renderModel(state, 200, '').includes('rewrites 101.9k'); + + expect(actual).toBe(expected); + }); + + it('shows what re-writing it would cost', () => { + const state = makeStatusState(); + state.setModel('claude-opus-4-8'); + state.setCacheDivergence({ changes: [{ name: 'effort', from: 'low', to: 'high' }], tokens: 101_900, costUsd: 1.019 }); + + const expected = true; + const actual = renderModel(state, 200, '').includes('($1.02)'); + + expect(actual).toBe(expected); + }); + + it('separates several moved settings rather than showing only the first', () => { + const state = makeStatusState(); + state.setModel('claude-opus-4-8'); + const changes = [ + { name: 'model', from: 'claude-fable-5', to: 'claude-opus-4-8' }, + { name: 'effort', from: 'low', to: 'high' }, + ]; + state.setCacheDivergence({ changes, tokens: 1000, costUsd: 0.01 }); + + const expected = true; + const actual = renderModel(state, 200, '').includes('claude-fable-5\u2192claude-opus-4-8, effort low\u2192high'); + + expect(actual).toBe(expected); + }); + + it('keeps the conversation id after the warning, so the warning survives a narrow terminal', () => { + const state = makeStatusState(); + state.setModel('claude-opus-4-8'); + state.setShowConversationId(true); + state.setCacheDivergence({ changes: [{ name: 'effort', from: 'low', to: 'high' }], tokens: 1000, costUsd: 0.01 }); + + const line = renderModel(state, 200, 'conv-id-here'); + + const expected = true; + const actual = line.indexOf('rewrites') < line.indexOf('conv-id-here'); + + expect(actual).toBe(expected); + }); +}); + +describe('renderModel — a size the new model has not given yet', () => { + const unknown = { changes: [{ name: 'model', from: 'claude-sonnet-4-6', to: 'claude-sonnet-5' }], tokens: null, costUsd: null }; + + it('shows the size as unknown rather than as the old model count', () => { + const state = makeStatusState(); + state.setModel('claude-sonnet-5'); + state.setCacheDivergence(unknown); + + const expected = true; + const actual = renderModel(state, 200, '').includes('rewrites ??'); + + expect(actual).toBe(expected); + }); + + it('shows the cost as unknown too, since it is that size times a rate', () => { + const state = makeStatusState(); + state.setModel('claude-sonnet-5'); + state.setCacheDivergence(unknown); + + const expected = true; + const actual = renderModel(state, 200, '').includes('($??)'); + + expect(actual).toBe(expected); + }); + + it('still names what moved while the size is unknown', () => { + const state = makeStatusState(); + state.setModel('claude-sonnet-5'); + state.setCacheDivergence(unknown); + + const expected = true; + const actual = renderModel(state, 200, '').includes('model claude-sonnet-4-6\u2192claude-sonnet-5'); + + expect(actual).toBe(expected); + }); +}); diff --git a/packages/claude-sdk/CHANGELOG.md b/packages/claude-sdk/CHANGELOG.md index a073f331..8e85ab9d 100644 --- a/packages/claude-sdk/CHANGELOG.md +++ b/packages/claude-sdk/CHANGELOG.md @@ -20,6 +20,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add support for Claude Opus 4.8 - Add the 'escalate' tool operation: a tool that crosses a privilege boundary always prompts for approval, independent of the read/write/delete cwd-zone matrix or any auto-approve config - Add updateIdentityBody to the durable config provider, folding a live system-identity body in as the first system prompt on the next config read +- buildRequestParams is now exported, along with the RequestBuilderOptions and RequestParams types, so a consumer can assemble the exact request the SDK would send and inspect it without sending it - Carry the request delta and its message, turn, and query ids through the final_message event, so the CLI can record each turn as a user/assistant pair - Classify a mid-stream connection drop and retry it on a bounded fixed schedule instead of surfacing it as a fatal error, with injection seams to hold a wake lock and signal a reconnect - Deliver tool attachments as native content blocks inside tool results @@ -31,6 +32,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Export `IMessageStreamer` from the public barrel - Inject a live per-turn date/time stamp into every request - isSystemReminderBlock is now exported, so a consumer can tell a block apart from a message's own words without reimplementing the test +- ITokenCounter asks the API how many input tokens a request would count as under its own model, over the same OAuth transport the message client uses. Advisory like the model catalogue: a failure returns null rather than throwing, so a caller shows what it already knew - Mark a tool-schema field as a filesystem path and normalise all marked paths once from that marker, so the display, the permission check, and handler execution read one produced path - Publish defineTool, ToolCancelledError, ToolRefusedError, and pathSchema as their own subpath exports, so a consumer can import just one without pulling in the whole SDK module graph - Stamp `messageId`, `turnId`, and `queryId` into each conversation record as nested fields, carried through the jsonl save and load round-trip diff --git a/packages/claude-sdk/changes.jsonl b/packages/claude-sdk/changes.jsonl index edbf95e9..11369baa 100644 --- a/packages/claude-sdk/changes.jsonl +++ b/packages/claude-sdk/changes.jsonl @@ -69,3 +69,5 @@ {"description":"Stored credentials and the browser login are now separate services a consumer resolves and can substitute (ICredentialProvider and ILoginFlow), replacing AnthropicAuth. A per-request caller holds the credential provider, which cannot open a browser","category":"changed"} {"description":"The OAuth callback's state is checked against the authorisation request it was built for, so a callback arriving from anywhere else is refused instead of exchanged","category":"security"} {"description":"DurableConfig gains conversationReminders, for standing facts about the current conversation. They are injected and re-injected exactly as cachedReminders are, but sit after them, so the prefix cache marker still falls on the last cached block and a per-conversation value cannot cost the shared prefix its reuse","category":"added"} +{"description":"buildRequestParams is now exported, along with the RequestBuilderOptions and RequestParams types, so a consumer can assemble the exact request the SDK would send and inspect it without sending it","category":"added"} +{"description":"ITokenCounter asks the API how many input tokens a request would count as under its own model, over the same OAuth transport the message client uses. Advisory like the model catalogue: a failure returns null rather than throwing, so a caller shows what it already knew","category":"added"} diff --git a/packages/claude-sdk/src/index.ts b/packages/claude-sdk/src/index.ts index 5f86238e..c29570e4 100644 --- a/packages/claude-sdk/src/index.ts +++ b/packages/claude-sdk/src/index.ts @@ -18,8 +18,9 @@ import { IMessageStreamer } from './private/MessageStreamer'; import { IModelCatalog, ModelCatalog } from './private/ModelCatalog'; import { calculateCost, calculateCostSplit, getContextWindow, reconstructCacheSplit } from './private/pricing'; import { QueryRunner } from './private/QueryRunner'; -import { isSystemReminderBlock, toWireTool } from './private/RequestBuilder'; +import { buildRequestParams, isSystemReminderBlock, toWireTool } from './private/RequestBuilder'; import { StreamProcessor } from './private/StreamProcessor'; +import { ITokenCounter, TokenCounter } from './private/TokenCounter'; import { ToolBlockNotifier } from './private/ToolBlockNotifier'; import { ToolRegistry } from './private/ToolRegistry'; import { TurnRunner } from './private/TurnRunner'; @@ -76,6 +77,7 @@ export type { BetaToolUnion } from '@anthropic-ai/sdk/resources/beta.mjs'; export type { ILogger } from '@shellicar/claude-core/logging/ILogger'; export type { HistoryItem, MessageIdentity, Sender } from './private/Conversation'; export type { ModelInfo } from './private/ModelCatalog'; +export type { RequestBuilderOptions, RequestParams } from './private/RequestBuilder'; export type { SchemaResolver } from './public/pathSchema'; export type { AnthropicBetaFlags, @@ -122,6 +124,7 @@ export { AnthropicClient, ApprovalCoordinator, annotatePathDescriptions, + buildRequestParams, CacheTtl, COMPACT_BETA, ControlChannel, @@ -154,6 +157,7 @@ export { ISdkMessagePublisher, ISkillGateProvider, IStreamProcessor, + ITokenCounter, ITokenEndpoint, IToolBlockNotifier, IToolProvider, @@ -174,6 +178,7 @@ export { StreamInterruptListener, StreamProcessor, TOOL_INPUT_KEYED_BY, + TokenCounter, ToolBlockNotifier, ToolCancelledError, ToolRefusedError, diff --git a/packages/claude-sdk/src/private/TokenCounter.ts b/packages/claude-sdk/src/private/TokenCounter.ts new file mode 100644 index 00000000..c7de0957 --- /dev/null +++ b/packages/claude-sdk/src/private/TokenCounter.ts @@ -0,0 +1,80 @@ +import versionJson from '@shellicar/build-version/version'; +import type { ILogger } from '@shellicar/claude-core/logging/ILogger'; +import type { ICredentialProvider } from './Client/Auth/interfaces'; +import { customFetch } from './http/customFetch'; +import type { RequestParams } from './RequestBuilder'; + +const COUNT_URL = 'https://api.anthropic.com/v1/messages/count_tokens?beta=true'; +const ANTHROPIC_VERSION = '2023-06-01'; + +/** + * How many input tokens a request would count as, asked of the model that would answer it. + * + * A token count is a property of the model, not of the content: the same bytes came back as 12,223 + * tokens on sonnet-4-6 and 15,948 on sonnet-5. So a count taken under one model says nothing about + * what another would charge for the same prompt, and there is no way to derive it locally. This is + * the only way to know. + */ +export abstract class ITokenCounter { + /** The count, or null when the API could not say. Advisory: a caller shows what it already knew + * rather than treating the absence as an error. */ + public abstract count(request: RequestParams): Promise; +} + +/** + * Counts over the same OAuth-bearer transport the message client uses, sending the request the + * caller assembled minus the fields that describe a generation rather than a prompt. + * + * Advisory like the model catalogue: a failure logs and returns null rather than throwing, because + * every caller is showing a figure it can fall back on and none of them should fail for want of a + * better one. Nothing is memoised, since each call is a different prompt. + */ +export class TokenCounter extends ITokenCounter { + readonly #credentials: ICredentialProvider; + readonly #logger: ILogger; + readonly #fetch: typeof fetch; + readonly #defaultHeaders: Record = { + 'user-agent': `@shellicar/claude-sdk/${versionJson.version}`, + }; + + public constructor(credentials: ICredentialProvider, logger: ILogger) { + super(); + this.#credentials = credentials; + this.#logger = logger; + this.#fetch = customFetch(logger) as typeof fetch; + } + + public async count(request: RequestParams): Promise { + try { + return await this.#count(request); + } catch (err) { + this.#logger.warn('token count failed', err); + return null; + } + } + + async #count(request: RequestParams): Promise { + const { claudeAiOauth } = await this.#credentials.get(); + // The endpoint takes a prompt, not a generation, so the fields that bound the response go. + const { max_tokens: _maxTokens, stream: _stream, ...prompt } = request.body as unknown as Record; + const response = await this.#fetch(COUNT_URL, { + method: 'POST', + headers: { + 'content-type': 'application/json', + 'anthropic-version': ANTHROPIC_VERSION, + authorization: `Bearer ${claudeAiOauth.accessToken}`, + ...this.#defaultHeaders, + ...request.headers, + }, + body: JSON.stringify(prompt), + }); + if (!response.ok) { + throw new Error(`token count request failed: ${response.status} ${response.statusText}`); + } + const json = (await response.json()) as { input_tokens?: unknown }; + if (typeof json.input_tokens !== 'number') { + throw new Error('token count response carried no input_tokens'); + } + return json.input_tokens; + } +} diff --git a/scripts/src/cache-probe.ts b/scripts/src/cache-probe.ts new file mode 100644 index 00000000..869ae15a --- /dev/null +++ b/scripts/src/cache-probe.ts @@ -0,0 +1,363 @@ +// Measures prompt-cache behaviour of the real request the CLI sends. +// +// The conversation is entirely synthetic: fixed user messages and canned assistant +// replies, so the request bytes are identical run to run. That is what makes a second +// run meaningful. A real assistant reply would differ every time and every turn after +// it would miss for reasons that have nothing to do with the thing being measured. +// +// Run from scripts/: +// pnpm tsx src/cache-probe.ts --dry hashes only, no API calls, free +// pnpm tsx src/cache-probe.ts live, 5 turns +// pnpm tsx src/cache-probe.ts --server-tools-at 3 region 1 changes at turn 3 +// pnpm tsx src/cache-probe.ts --model claude-opus-4-8 --effort low --effort-at 3 high +// thinking effort changes at turn 3 +// pnpm tsx src/cache-probe.ts --model claude-opus-4-8 --thinking on --thinking-at 3 off +// thinking is turned off at turn 3 +// pnpm tsx src/cache-probe.ts --model claude-opus-4-8 --model-at 3 claude-opus-5 +// the model changes at turn 3 +// pnpm tsx src/cache-probe.ts --model claude-sonnet-5 --count +// asks the API what it counts, sends no message +// +// Run a thinking switch WITHOUT --effort. Effort rides output_config, which the builder sends only +// while thinking is enabled, so a run carrying both would drop two request parameters at the same +// turn and could not say which one moved the cache. + +import { createHash } from 'node:crypto'; +import { readFile } from 'node:fs/promises'; +import { homedir } from 'node:os'; +import { join } from 'node:path'; +import { Clock } from '@js-joda/core'; +import type { IFileSystem } from '@shellicar/claude-core/fs/interfaces'; +import { IHistoryReader } from '@shellicar/claude-core/history/interfaces'; +import type { HistoryReadRequest, HistorySearchHit, HistorySearchQuery, HistoryWindow } from '@shellicar/claude-core/history/types'; +import { ILogger } from '@shellicar/claude-core/logging/ILogger'; +import { IMemoryStore } from '@shellicar/claude-core/memory/interfaces'; +import type { MemoryDraft, MemoryEntry, MemorySearchHit, MemoryTypeCount } from '@shellicar/claude-core/memory/types'; +import { IObjectStore } from '@shellicar/claude-core/persistence/interfaces'; +import { AnthropicBeta, AnthropicClient, type AuthCredentials, type BetaMessageParam, type BetaToolUnion, buildRequestParams, CacheTtl, ICredentialProvider, type RequestBuilderOptions, type ThinkingEffort } from '@shellicar/claude-sdk'; +import { createAppTools } from '@shellicar/claude-sdk-cli/src/createAppTools.js'; +import { ISecrets } from '@shellicar/claude-sdk-cli/src/secrets/Secrets.js'; +import { IEnvProvider, StaticRulesConfigProvider } from '@shellicar/claude-sdk-tools/ExecV3'; +import type { ITypeScriptService } from '@shellicar/claude-sdk-tools/TsService'; + +type TextBlock = { type: 'text'; text: string }; + +const TURNS = 5; +const CREDENTIALS_PATH = join(homedir(), '.claude', '.credentials.json'); + +const args = process.argv.slice(2); +function flag(name: string): string | null { + const i = args.indexOf(name); + return i === -1 ? null : (args[i + 1] ?? null); +} + +const dryRun = args.includes('--dry'); +const countOnly = args.includes('--count'); +const MODEL = flag('--model') ?? 'claude-haiku-4-5'; +const switchModelAt = flag('--model-at') == null ? null : Number(flag('--model-at')); +const switchModel = switchModelAt == null ? null : (args[args.indexOf('--model-at') + 2] ?? null); +const serverToolsAt = flag('--server-tools-at') == null ? null : Number(flag('--server-tools-at')); +const baseEffort = flag('--effort') as ThinkingEffort | null; +const switchEffortAt = flag('--effort-at') == null ? null : Number(flag('--effort-at')); +const switchEffort = (switchEffortAt == null ? null : (args[args.indexOf('--effort-at') + 2] ?? null)) as ThinkingEffort | null; +const thinkingFlag = flag('--thinking'); +const switchThinkingAt = flag('--thinking-at') == null ? null : Number(flag('--thinking-at')); +const switchThinking = switchThinkingAt == null ? null : (args[args.indexOf('--thinking-at') + 2] ?? null); +// Thinking follows `--effort` unless said otherwise, so every invocation that predates `--thinking` +// keeps behaving exactly as it did. +const baseThinking = thinkingFlag != null ? thinkingFlag === 'on' : baseEffort != null; + +/** The model in force for a given turn, so a model switch is a per-turn fact rather than a mode. */ +function modelFor(turn: number): string { + if (switchModelAt != null && switchModel != null && turn >= switchModelAt) { + return switchModel; + } + return MODEL; +} + +/** The effort in force for a given turn, so an effort switch is a per-turn fact rather than a mode. */ +function effortFor(turn: number): ThinkingEffort | undefined { + if (switchEffortAt != null && switchEffort != null && turn >= switchEffortAt) { + return switchEffort; + } + return baseEffort ?? undefined; +} + +/** Whether thinking is on for a given turn, so a thinking switch is a per-turn fact rather than a mode. */ +function thinkingFor(turn: number): boolean { + if (switchThinkingAt != null && switchThinking != null && turn >= switchThinkingAt) { + return switchThinking === 'on'; + } + return baseThinking; +} + +// Fixed for the whole run, including one where thinking flips. It was derived from the thinking +// state, which would have made a thinking switch change two request parameters at once and left the +// result unattributable. A run that thinks at any point pays the larger budget throughout. +const MAX_TOKENS = baseThinking || switchThinking === 'on' ? 4096 : 16; + +class StubObjectStore extends IObjectStore { + public set(): void {} + public get(): string | undefined { + return undefined; + } +} + +class StubMemoryStore extends IMemoryStore { + public async write(draft: MemoryDraft): Promise { + return { id: '', title: draft.title, body: draft.body, type: draft.type, keywords: draft.keywords, environment: {}, createdAt: '' }; + } + public async read(): Promise { + return undefined; + } + public async search(): Promise { + return []; + } + public async delete(): Promise {} + public async types(): Promise { + return []; + } +} + +class StubHistoryReader extends IHistoryReader { + public search(_query: HistorySearchQuery): HistorySearchHit[] { + return []; + } + public read(_request: HistoryReadRequest): HistoryWindow[] { + return []; + } +} + +class StubSecrets extends ISecrets { + public ghHolderToken(): string { + return ''; + } + public ghReaderToken(): string { + return ''; + } + public azCert(): string { + return ''; + } +} + +class StubEnvProvider extends IEnvProvider { + public buildEnv(cmdEnv?: NodeJS.ProcessEnv): NodeJS.ProcessEnv { + return { ...process.env, ...cmdEnv }; + } +} + +class SilentLogger extends ILogger { + public trace(): void {} + public debug(): void {} + public info(): void {} + public warn(): void {} + public error(): void {} +} + +/** Reads the credentials the CLI already stored. Refuses an expired token rather than + * refreshing it, so this probe never writes to the shared credential file. */ +class StoredCredentialProvider extends ICredentialProvider { + public async get(): Promise { + const raw = await readFile(CREDENTIALS_PATH, 'utf8'); + const parsed = JSON.parse(raw) as AuthCredentials; + if (parsed.claudeAiOauth.expiresAt <= Date.now()) { + throw new Error('Stored credentials have expired. Start the CLI once to refresh them, then re-run.'); + } + return parsed; + } +} + +const { tools } = createAppTools({ + fs: null as unknown as IFileSystem, + tsServer: null as unknown as ITypeScriptService, + toolsConfig: { exec: false, execV2: false, execV3: true }, + rulesProvider: new StaticRulesConfigProvider(), + objects: new StubObjectStore(), + memory: new StubMemoryStore(), + history: new StubHistoryReader(), + currentSessionId: () => '', + clock: Clock.systemUTC(), + tsAvailable: false, + logger: new SilentLogger(), + secrets: new StubSecrets(), + envProvider: new StubEnvProvider(), + getAzAccounts: () => ({}), +}); + +const SYSTEM_PROMPTS = ['You are a probe. Answer every message with the single word: ok.', 'Stay terse. One word only. Never explain.']; + +const CACHED_REMINDERS = ['The following skills are available for use with the Skill tool:\n\n- probe: a fixed catalogue entry, standing in for the real one.', 'Codebase and user instructions are shown below.\n\nThis is fixed CLAUDE.md content, standing in for the real file.']; + +const SERVER_TOOLS: BetaToolUnion[] = [{ name: 'web_search', type: 'web_search_20260209', allowed_callers: ['direct'] } as BetaToolUnion]; + +/** Fixed user text per turn. Short: the point is the prefix, not these. */ +const USER_TEXTS = Array.from({ length: TURNS }, (_, i) => `Probe message ${i + 1}. Reply with ok.`); + +/** A canned assistant reply, so the conversation replays byte-identically. */ +const ASSISTANT_REPLY: BetaMessageParam = { role: 'assistant', content: [{ type: 'text', text: 'ok' }] }; + +function reminderBlocks(texts: string[]): TextBlock[] { + return texts.map((text, i, arr) => ({ type: 'text' as const, text: `\n${text}\n\n${i === arr.length - 1 ? '\n' : ''}` })); +} + +/** The conversation as it stands before turn `turn` (1-based), ending on a user message. */ +function messagesFor(turn: number): BetaMessageParam[] { + const out: BetaMessageParam[] = []; + for (let i = 0; i < turn; i++) { + const text = USER_TEXTS[i] ?? ''; + const content = i === 0 ? [...reminderBlocks(CACHED_REMINDERS), { type: 'text' as const, text }] : [{ type: 'text' as const, text }]; + out.push({ role: 'user', content }); + if (i < turn - 1) { + out.push(ASSISTANT_REPLY); + } + } + return out; +} + +function hash(value: unknown): string { + return createHash('sha256').update(JSON.stringify(value)).digest('hex').slice(0, 12); +} + +/** Every cache_control marker in the body, in prefix order, so the marker layout is visible + * rather than assumed. */ +function markers(body: { tools?: unknown[]; system?: unknown; messages: BetaMessageParam[] }): string[] { + const found: string[] = []; + body.tools?.forEach((t, i) => { + if ((t as { cache_control?: unknown }).cache_control != null) { + found.push(`tools[${i}]`); + } + }); + if (Array.isArray(body.system)) { + body.system.forEach((b, i) => { + if ((b as { cache_control?: unknown }).cache_control != null) { + found.push(`system[${i}]`); + } + }); + } + body.messages.forEach((m, mi) => { + if (!Array.isArray(m.content)) { + return; + } + m.content.forEach((b: unknown, bi: number) => { + if ((b as { cache_control?: unknown }).cache_control != null) { + found.push(`messages[${mi}].content[${bi}]`); + } + }); + }); + return found; +} + +function optionsFor(turn: number): RequestBuilderOptions { + const withServerTools = serverToolsAt != null && turn >= serverToolsAt; + return { + model: modelFor(turn), + maxTokens: MAX_TOKENS, + thinking: thinkingFor(turn), + thinkingEffort: effortFor(turn), + tools, + serverTools: withServerTools ? SERVER_TOOLS : [], + transformTool: (tool) => { + const { input_examples: _drop, ...rest } = tool as BetaToolUnion & { input_examples?: unknown }; + return rest as BetaToolUnion; + }, + betas: { [AnthropicBeta.ClaudeCodeAuth]: true }, + systemPrompts: SYSTEM_PROMPTS, + cachedReminders: CACHED_REMINDERS, + cacheTtl: CacheTtl.OneHour, + }; +} + +const COUNT_URL = 'https://api.anthropic.com/v1/messages/count_tokens?beta=true'; +const ANTHROPIC_VERSION = '2023-06-01'; + +/** + * Asks the API what a request counts as under a given model, the only way to know the token total + * for a model that has not seen the conversation. Token counts are a property of the model, not of + * the content: the same bytes came back as 12,223 on sonnet-4-6 and 15,948 on sonnet-5. + * + * `max_tokens` and `stream` are dropped because this endpoint takes the prompt, not a generation. + */ +async function countTokens(body: Record, requestHeaders: Record, token: string): Promise { + const { max_tokens: _maxTokens, stream: _stream, ...prompt } = body; + const response = await fetch(COUNT_URL, { + method: 'POST', + headers: { + 'content-type': 'application/json', + 'anthropic-version': ANTHROPIC_VERSION, + authorization: `Bearer ${token}`, + ...requestHeaders, + }, + body: JSON.stringify(prompt), + }); + if (!response.ok) { + throw new Error(`count_tokens ${response.status}: ${await response.text()}`); + } + const json = (await response.json()) as { input_tokens: number }; + return json.input_tokens; +} + +type Usage = { input: number; write: number; read: number; output: number }; + +/** What the numbers mean, stated so a run reads without arithmetic. A cold prefix reads nothing + * and writes everything, which is the signature of an invalidation. */ +function verdict(u: Usage): string { + if (u.read === 0) { + return 'COLD: nothing cached, whole prefix written'; + } + if (u.write === 0) { + return 'FULL HIT: nothing written'; + } + return `PARTIAL: ${u.read} cached, ${u.write} written`; +} + +async function send(client: AnthropicClient, body: Parameters[0], headers: Record): Promise { + const usage: Usage = { input: 0, write: 0, read: 0, output: 0 }; + for await (const event of client.stream(body, { headers })) { + if (event.type === 'message_start') { + const u = event.message.usage; + usage.input = u.input_tokens; + usage.write = u.cache_creation_input_tokens ?? 0; + usage.read = u.cache_read_input_tokens ?? 0; + } + if (event.type === 'message_delta') { + usage.output = event.usage.output_tokens ?? 0; + } + } + return usage; +} + +async function main(): Promise { + const client = dryRun || countOnly ? null : new AnthropicClient(new StoredCredentialProvider(), new SilentLogger()); + const authToken = dryRun || !countOnly ? null : (await new StoredCredentialProvider().get()).claudeAiOauth.accessToken; + + console.log( + `model=${MODEL} turns=${TURNS} tools=${tools.length} thinking=${baseThinking} maxTokens=${MAX_TOKENS}${serverToolsAt == null ? '' : ` server-tools-from=${serverToolsAt}`}${switchEffortAt == null ? '' : ` effort-switch=${switchEffort}@${switchEffortAt}`}${switchThinkingAt == null ? '' : ` thinking-switch=${switchThinking}@${switchThinkingAt}`}${switchModelAt == null ? '' : ` model-switch=${switchModel}@${switchModelAt}`}${dryRun ? ' (dry run, no API calls)' : ''}${countOnly ? ' (count only, no messages sent)' : ''}`, + ); + console.log(''); + console.log('turn model think effort toolsHash systemHash claudeMdHash markers'); + + for (let turn = 1; turn <= TURNS; turn++) { + const { body, headers } = buildRequestParams(optionsFor(turn), messagesFor(turn)); + const firstUser = body.messages[0]; + const claudeMdPrefix = Array.isArray(firstUser?.content) ? firstUser.content.slice(0, CACHED_REMINDERS.length) : []; + + const toolsHash = hash(body.tools); + const systemHash = hash(body.system); + const claudeMdHash = hash(claudeMdPrefix); + console.log(`${String(turn).padEnd(6)}${modelFor(turn).padEnd(19)}${(thinkingFor(turn) ? 'on' : 'off').padEnd(7)}${(effortFor(turn) ?? '-').padEnd(8)}${toolsHash} ${systemHash} ${claudeMdHash} ${markers(body).join(' ')}`); + + if (authToken != null) { + const counted = await countTokens(body as unknown as Record, headers, authToken); + console.log(` count_tokens: input=${counted}`); + } + if (client != null) { + const u = await send(client, body, headers); + console.log(` usage: in=${u.input} write=${u.write} read=${u.read} out=${u.output} ${verdict(u)}`); + } + } +} + +main().catch((err: unknown) => { + console.error(err instanceof Error ? err.message : String(err)); + process.exitCode = 1; +});