diff --git a/.env.example b/.env.example index a3243d6c..963147b6 100644 --- a/.env.example +++ b/.env.example @@ -198,6 +198,11 @@ LS_PORT=42100 # regeneration, system-prompt drift, and response truncation; parallel dialogs on # the same key stay independent. In-memory only (no persistence across restarts). # DEVIN_CONNECT_SESSION_REUSE=0 +# --- Think-text reroute (Thinking-core item 1, loop break) --- +# When the model emits a LEADING think-tagged span on the CONTENT channel, reroute it +# to the thinking channel so clients do not store it as visible assistant text and +# resend it into a self-reinforcing loop. Anthropic Messages egress. Default 0 (off). +# DEVIN_CONNECT_THINKTEXT_REROUTE=0 # State TTL before a dormant dialog's session_id is forgotten. Default 1800000 (30 min). # DEVIN_CONNECT_SESSION_TTL_MS=1800000 # Max tracked dialogs (LRU-evicted). Default 500. diff --git a/src/devin-connect-openai.js b/src/devin-connect-openai.js index 3ea0e56c..655f3b4b 100644 --- a/src/devin-connect-openai.js +++ b/src/devin-connect-openai.js @@ -24,6 +24,15 @@ import { log } from './config.js'; import { systemFingerprint } from './system-fingerprint.js'; import { applyStop, StopSequenceGate } from './stop-sequences.js'; import { normalizeToolCallArgs, recordArgRepair } from './handlers/cline-compat.js'; +import { ThinkTextClassifier } from './response-classifier.js'; + +// Gate for the leading think-tag reroute (Item 1, loop break). The classifier +// lives HERE, in the connect layer, so the DEVIN_CONNECT_ prefix is accurate: +// only the connect backend's streams are reclassified. Default OFF — the +// reroute is a behavior change and must be opted into per deployment. +function thinkTextRerouteEnabled() { + return String(process.env.DEVIN_CONNECT_THINKTEXT_REROUTE || '') === '1'; +} // Apply the Cline compat tool-arg shim when active: normalize an arguments // string @ai-sdk/openai-compatible would reject (empty / whitespace / non-JSON) @@ -213,8 +222,38 @@ async function* streamChatWithEmptyRetry(params, { env = process.env } = {}) { let sawReasoning = false; let sawReasoningText = ''; let finishEv = null; + // Item 1 (loop break): leading think-tagged CONTENT is reclassified into the + // reasoning channel HERE, at the stream-event level, BEFORE the rescue + // decision below. A whole turn of ` thinking…` therefore stays a + // reasoning-only stream. + // Honest rescue interplay (corrected after review): such a turn does NOT + // fire the #238 rescue — the :284 nudge path requires tools, and + // isEmptyCompletion reads sawContent, which the thinking branch below sets + // at :238. The real outcome: reasoning delivered as thinking blocks, text + // empty, no retry. That is accepted: the reasoning is not lost (the client + // receives thinking blocks, not the #238 whole-turn vanish), and sawContent + // genuinely means "upstream said something", which is true here. + // Deliberately NOT teaching isEmptyCompletion to read sawText — that would + // redraw the #238/#241 rescue boundary for a cosmetic gain. + // (Known limitation: only the ` thinking`…` dialect is recognized; Kimi K2's + // family uses `◁think▷…◁/think▷` — extending the classifier to that dialect + // is left for a future PR.) + const thinkClassifier = thinkTextRerouteEnabled() ? new ThinkTextClassifier() : null; for await (const ev of streamChatImpl(attemptParams)) { - if (ev.type === 'content' || ev.type === 'reasoning') { + if (ev.type === 'content' && thinkClassifier) { + const routed = thinkClassifier.feed(ev.text); + if (routed.thinking) { + sawContent = true; + sawReasoning = true; + sawReasoningText += routed.thinking; + yield { type: 'reasoning', text: routed.thinking }; + } + if (routed.text) { + sawContent = true; + sawText = true; + yield { type: 'content', text: routed.text }; + } + } else if (ev.type === 'content' || ev.type === 'reasoning') { if (ev.text) { sawContent = true; if (ev.type === 'content') sawText = true; @@ -230,6 +269,17 @@ async function* streamChatWithEmptyRetry(params, { env = process.env } = {}) { yield ev; } } + // Flush anything the classifier still holds BEFORE the rescue decision: an + // unterminated/undecided tail is delivered as text (visible beats dropped), + // and it must count as real text for the rescue gate above. + if (thinkClassifier) { + const rest = thinkClassifier.flush(); + if (rest) { + sawContent = true; + sawText = true; + yield { type: 'content', text: rest }; + } + } // swe-1-7 (Kimi K2 fine-tune) intermittently spends the whole turn in reasoning // declaring tool intent without emitting the call; a corrective nudge measurably // (24/24 live probe) forces emission; empty assistant turns poison upstream into diff --git a/src/handlers/messages.js b/src/handlers/messages.js index 375664b0..5bc86f9e 100644 --- a/src/handlers/messages.js +++ b/src/handlers/messages.js @@ -35,6 +35,12 @@ function mapStopReason(finishReason) { return STOP_REASON_MAP[finishReason] || 'end_turn'; } +// Item 1 (loop break) classifier moved to the connect layer: leading think-tagged +// CONTENT is now reclassified into the reasoning channel at the stream-event level +// in devin-connect-openai.js (streamChatWithEmptyRetry), BEFORE the #238 rescue +// decision — see docs/GOAL-2026-08-06-PR-REWORK.md. The egress translators here stay +// passive: they render whatever channel the events arrive on. + // B5: Anthropic sets stop_reason:'stop_sequence' AND echoes the matched string // in stop_sequence when generation halts on a caller-supplied stop sequence. // The internal OpenAI path reports finish_reason:'stop' for both a natural @@ -837,6 +843,10 @@ export function openAIToAnthropic(result, model, msgId, cachePolicy = null, stop }); } } else { + // Invariant: the text block is ALWAYS present, even when empty. The think-tag + // reroute lives in the connect layer now (devin-connect-openai.js), so by the + // time a completion reaches this translator the content channel carries no + // think markers — a reasoning-only turn arrives as reasoning_content only. content.push({ type: 'text', text: choice?.message?.content || '' }); } // B5: resolve stop_reason (incl. content_filter→refusal) and back-fill diff --git a/src/response-classifier.js b/src/response-classifier.js new file mode 100644 index 00000000..b52f1c44 --- /dev/null +++ b/src/response-classifier.js @@ -0,0 +1,106 @@ +// Response-content classifier (Thinking-core item 2, applied in item 1). +// +// Purpose: decide which parts of a streamed answer are *reasoning* vs *actionable +// text*, so misrouted content can be corrected at the egress and self-reinforcing +// loops can be broken. +// +// Degradation addressed: the model sometimes emits its reasoning through the CONTENT +// channel wrapped in its native markers. The client stores that as visible assistant +// text and resends it next turn, re-priming more reasoning-as-text — the loop. +// +// This module detects the markers in the content stream and lets the egress reroute +// the marked spans to the thinking channel (which clients do not resend), breaking the +// loop. Scope: a LEADING reasoning block only (marker at the very start, optional +// whitespace before it). No synthetic signatures, no router reliance, no guesses about +// unmarked or mid-answer content. Handles markers split across stream deltas. + +const THINK_OPEN = '<' + 'think' + '>'; +const THINK_CLOSE = '<' + '/' + 'think' + '>'; + +const MAX_PENDING = 32000; // hold ceiling for an unterminated think span +const MAX_LEAD = 8192; // undecided-hold ceiling before committing to text + + +export class ThinkTextClassifier { + constructor() { + this.pending = ''; + this.mode = 'undecided'; // undecided | text | think + } + + // Feed a content delta; returns { text, thinking } — slices to emit on each channel + // right now ('' when nothing is due). + feed(delta) { + if (!delta) return { text: '', thinking: '' }; + if (this.mode === 'text') return { text: delta, thinking: '' }; + if (this.mode === 'think') return this._feedThink(delta); + return this._feedUndecided(delta); + } + + _feedUndecided(delta) { + this.pending += delta; + + // Enough held without a decision -> it is plain text; commit. + if (this.pending.length > MAX_LEAD) return this._commitText(); + + const oi = this.pending.indexOf(THINK_OPEN); + if (oi >= 0) { + if (this.pending.slice(0, oi).trim() === '') { + // Leading marker -> enter think mode; drop whitespace-only prefix. + this.mode = 'think'; + this.pending = this.pending.slice(oi + THINK_OPEN.length); + return this._feedThink(''); // process the remainder of this same delta + } + // Marker present but real text precedes it -> inline, not a leak. + return this._commitText(); + } + + // No full marker yet. Could the pending still grow into one? + const core = this.pending.replace(/^\s+/, ''); + if (core.length === 0) return { text: '', thinking: '' }; // only whitespace so far + if (THINK_OPEN.startsWith(core)) return { text: '', thinking: '' }; // partial marker + return this._commitText(); // definitively not a leading marker + } + + _commitText() { + const out = this.pending; + this.pending = ''; + this.mode = 'text'; + return { text: out, thinking: '' }; + } + + _feedThink(delta) { + this.pending += delta; + + const ci = this.pending.indexOf(THINK_CLOSE); + if (ci >= 0) { + const span = this.pending.slice(0, ci); + const rest = this.pending.slice(ci + THINK_CLOSE.length); + this.pending = ''; + this.mode = 'undecided'; // may catch a following block; normal text commits + const cont = this._feedUndecided(rest); + return { text: cont.text, thinking: span + cont.thinking }; + } + + if (this.pending.length > MAX_PENDING) { + // Pathological unterminated span -> deliver as text (visible beats dropped, + // and loop-break value is gone at this size anyway). + const dump = this.pending; + this.pending = ''; + this.mode = 'text'; + return { text: dump, thinking: '' }; + } + + return { text: '', thinking: '' }; // buffer until the close marker proves it + } + + // Stream end: flush whatever is held. Undecided content is text; an unterminated + // think span is NOT rerouted (it never proved it was reasoning) — delivered as text. + flush() { + const out = this.pending; + this.pending = ''; + this.mode = 'text'; + return out; + } +} + +export const THINK_MARKERS = { open: THINK_OPEN, close: THINK_CLOSE }; diff --git a/test/devin-connect-openai.test.js b/test/devin-connect-openai.test.js index d190de2d..29fbe73c 100644 --- a/test/devin-connect-openai.test.js +++ b/test/devin-connect-openai.test.js @@ -1,4 +1,4 @@ -import { afterEach, describe, it } from 'node:test'; +import { afterEach, beforeEach, describe, it } from 'node:test'; import assert from 'node:assert/strict'; import { toChatCompletion, @@ -1295,3 +1295,134 @@ describe('thinking-only rescue & promotion', () => { }); }); +// Pin tests — GOAL-2026-08-06-PR-REWORK item 3. The leading think-tag reroute +// moved from the messages.js egress translators into the connect layer, at the +// stream-event level (streamChatWithEmptyRetry), BEFORE the #238 rescue +// decision. These pin the combined behavior: reclassification + rescue + the +// egress invariant "text block always present". +describe('think-text reroute (connect layer, DEVIN_CONNECT_THINKTEXT_REROUTE)', () => { + const OPEN = '<' + 'think' + '>'; + const CLOSE = '<' + '/' + 'think' + '>'; + const SAMPLE_TOOLS = [{ type: 'function', function: { name: 'read_file' } }]; + let prev; + beforeEach(() => { prev = process.env.DEVIN_CONNECT_THINKTEXT_REROUTE; process.env.DEVIN_CONNECT_THINKTEXT_REROUTE = '1'; }); + afterEach(() => { if (prev === undefined) delete process.env.DEVIN_CONNECT_THINKTEXT_REROUTE; else process.env.DEVIN_CONNECT_THINKTEXT_REROUTE = prev; }); + + it('(a) whole turn = think block: the #238 rescue fires and the client gets an answer', async () => { + // Attempt 1 is an ENTIRE turn on the content channel, think-tagged. With the + // classifier at the event level this decodes to reasoning-only, so sawText + // stays false and the rescue fires naturally — it never did before the move: + // the egress reroute ran AFTER the rescue had already given up, leaving a + // reasoning-only finish (#238 form, APIEmptyResponseError on strict clients). + let callCount = 0; + let lastParams = null; + __setStreamChatForTest(async function* (params) { + callCount++; + lastParams = params; + if (callCount === 1) { + yield { type: 'content', text: OPEN + 'Let me think through the whole request. ' + CLOSE }; + yield { type: 'finish', reason: 'stop', usage: null }; + } else { + yield { type: 'content', text: 'Here is the answer.' }; + yield { type: 'finish', reason: 'stop', usage: null }; + } + }); + const frames = []; + const result = await streamChatCompletion( + { model: 'swe-1-7', messages: [{ role: 'user', content: 'hi' }], tools: SAMPLE_TOOLS }, + (f) => frames.push(f), + { emulateTools: true }, + ); + assert.equal(callCount, 2, 'the #238 rescue must fire for a whole-think turn'); + assert.ok( + lastParams.messages.at(-1).content.includes('Stop reasoning. Emit the tool call markup now.'), + 'the rescue nudge reaches the second attempt', + ); + // The rescued attempt's answer is what reaches the client — the abandoned + // think-only attempt is not concatenated onto it. + assert.equal(result.content, 'Here is the answer.'); + }); + + it('(a) whole-think turn without tools: promotion keeps a visible answer (no empty message)', async () => { + // No tools -> no rescue (plain chat legitimately ends in reasoning/text). The + // think-only content must still reach the client as the visible answer via + // the existing reasoning->content promotion, so the anthropic egress below + // (openAIToAnthropic) always has a text block to push. + __setStreamChatForTest(fakeStream([ + { type: 'content', text: OPEN + 'reasoning only, no tools. ' + CLOSE }, + { type: 'finish', reason: 'stop', usage: null }, + ])); + const { body } = await toChatCompletion({ model: 'swe-1-7', messages: [] }); + const msg = body.choices[0].message; + assert.equal(msg.content, 'reasoning only, no tools. ', 'promoted into the visible content'); + assert.equal(msg.reasoning_content, undefined, 'promotion moves, not copies'); + }); + + it('(b) think + answer still splits into thinking and text', async () => { + __setStreamChatForTest(fakeStream([ + { type: 'content', text: OPEN + 'inner reasoning. ' + CLOSE }, + { type: 'content', text: 'The answer.' }, + { type: 'finish', reason: 'stop', usage: null }, + ])); + const { body } = await toChatCompletion({ model: 'swe-1-7', messages: [] }); + const msg = body.choices[0].message; + assert.equal(msg.reasoning_content, 'inner reasoning. ', 'think span lands on the reasoning channel'); + assert.equal(msg.content, 'The answer.', 'the answer stays on the content channel'); + }); + + it('(c) stream variant: think + answer renders as reasoning_content deltas then content deltas', async () => { + __setStreamChatForTest(fakeStream([ + { type: 'content', text: OPEN + 'inner reasoning. ' + CLOSE }, + { type: 'content', text: 'The answer.' }, + { type: 'finish', reason: 'stop', usage: null }, + ])); + const frames = []; + const result = await streamChatCompletion( + { model: 'swe-1-7', messages: [] }, + (f) => frames.push(f), + {}, + ); + assert.equal(result.reasoning, 'inner reasoning. '); + assert.equal(result.content, 'The answer.'); + const reasoningDeltas = frames.filter((f) => f.choices?.[0]?.delta?.reasoning_content).map((f) => f.choices[0].delta.reasoning_content).join(''); + assert.equal(reasoningDeltas, 'inner reasoning. '); + const contentDeltas = frames.filter((f) => f.choices?.[0]?.delta?.content).map((f) => f.choices[0].delta.content).join(''); + assert.equal(contentDeltas, 'The answer.'); + }); + + it('respects the gate: with DEVIN_CONNECT_THINKTEXT_REROUTE off, think-tagged content stays text', async () => { + process.env.DEVIN_CONNECT_THINKTEXT_REROUTE = '0'; + __setStreamChatForTest(fakeStream([ + { type: 'content', text: OPEN + 'inner reasoning. ' + CLOSE + 'The answer.' }, + { type: 'finish', reason: 'stop', usage: null }, + ])); + const { body } = await toChatCompletion({ model: 'swe-1-7', messages: [] }); + const msg = body.choices[0].message; + assert.equal(msg.content, OPEN + 'inner reasoning. ' + CLOSE + 'The answer.'); + assert.equal(msg.reasoning_content, undefined); + }); + + it('history isolation: a leading think block inside an INBOUND history message is forwarded upstream untouched, even with the gate on', async () => { + // The classifier runs ONLY on live upstream output events. Caller-pasted + // content (a quoted transcript, a literal "what does the tag mean") rides + // the inbound message list and must never be reclassified — that would + // lose attribution, the mirror image of the leak this feature fixes. + process.env.DEVIN_CONNECT_THINKTEXT_REROUTE = '1'; + let captured = null; + __setStreamChatForTest(async function* (params) { + captured = params; + yield { type: 'content', text: 'ok' }; + yield { type: 'finish', reason: 'stop', usage: null }; + }); + const history = [ + { role: 'user', content: 'what does ' + OPEN + CLOSE + ' mean?' }, + { role: 'assistant', content: OPEN + 'quoted from a log' + CLOSE + ' and here is my real answer' }, + { role: 'user', content: 'go on' }, + ]; + const { status, body } = await toChatCompletion({ model: 'swe-1-7', messages: history }); + assert.equal(status, 200); + assert.deepEqual(captured.messages, history); + assert.equal(body.choices[0].message.content, 'ok'); + }); +}); + diff --git a/test/messages.test.js b/test/messages.test.js index c5c8a044..d1bb40ce 100644 --- a/test/messages.test.js +++ b/test/messages.test.js @@ -1,4 +1,4 @@ -import { afterEach, describe, it } from 'node:test'; +import { afterEach, beforeEach, describe, it } from 'node:test'; import assert from 'node:assert/strict'; import { annotateRiskyReadToolResult, extractCallerSubKey, handleMessages, handleCountTokens, toAnthropicError } from '../src/handlers/messages.js'; import { applyJsonResponseHint, extractRequestedJsonKeys, isExplicitJsonRequested, stabilizeJsonPayload } from '../src/handlers/chat.js'; @@ -1676,3 +1676,140 @@ describe('Anthropic count_tokens', () => { assert.ok(a.body.input_tokens >= 1); }); }); + +// Item 1 (loop break) reroute MOVED to the connect layer: leading think-tagged +// content is reclassified at the stream-event level in devin-connect-openai.js +// (streamChatWithEmptyRetry), BEFORE the #238 rescue decision. The egress +// translators here are passive — they render whatever channel the events arrive +// on — and the one invariant they must keep is: the text block is always present. +describe('think-text reroute (egress is passive; text block always present)', () => { + const OPEN = '<' + 'think' + '>'; + const CLOSE = '<' + '/' + 'think' + '>'; + + it('stream: think-tagged content arrives on the content channel and renders as text (reroute lives below)', async () => { + const result = await handleMessages({ + model: 'claude-sonnet-4.6', + stream: true, + messages: [{ role: 'user', content: 'hi' }], + }, { + async handleChatCompletions() { + return { + status: 200, + stream: true, + async handler(res) { + res.write(chatChunk({ choices: [{ index: 0, delta: { role: 'assistant', content: OPEN + 'inner reasoning. ' + CLOSE }, finish_reason: null }] })); + res.write(chatChunk({ choices: [{ index: 0, delta: { content: 'The answer.' }, finish_reason: null }] })); + res.write(chatChunk({ choices: [{ index: 0, delta: {}, finish_reason: 'stop' }] })); + res.write(chatChunk({ choices: [], usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 } })); + res.end('data: [DONE]\n\n'); + }, + }; + }, + }); + const res = fakeRes(); + await result.handler(res); + const events = parseAnthropicEvents(res.body); + const blocks = events.filter(e => e.event === 'content_block_start').map(e => e.data.content_block.type); + assert.deepEqual(blocks, ['text'], 'egress translator is passive: no thinking block at this layer'); + const textDeltas = events.filter(e => e.event === 'content_block_delta' && e.data.delta?.type === 'text_delta').map(e => e.data.delta.text).join(''); + assert.equal(textDeltas, OPEN + 'inner reasoning. ' + CLOSE + 'The answer.'); + }); + + it('non-stream: think-tagged content stays in the text block (invariant: text block always present)', async () => { + const result = await handleMessages({ + model: 'claude-sonnet-4.6', + stream: false, + messages: [{ role: 'user', content: 'hi' }], + }, { + async handleChatCompletions() { + return { + status: 200, + body: { + choices: [{ index: 0, message: { role: 'assistant', content: OPEN + 'inner reasoning. ' + CLOSE + 'The answer.' }, finish_reason: 'stop' }], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + }, + }; + }, + }); + const blocks = result.body.content; + assert.equal(blocks.length, 1); + assert.equal(blocks[0].type, 'text'); + assert.equal(blocks[0].text, OPEN + 'inner reasoning. ' + CLOSE + 'The answer.'); + }); + + it('invariant: a reasoning-only completion (reasoning_content, empty content) still yields a text block', async () => { + const result = await handleMessages({ + model: 'claude-sonnet-4.6', + stream: false, + messages: [{ role: 'user', content: 'hi' }], + }, { + async handleChatCompletions() { + return { + status: 200, + body: { + choices: [{ index: 0, message: { role: 'assistant', content: '', reasoning_content: 'deep thoughts' }, finish_reason: 'stop' }], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + }, + }; + }, + }); + const blocks = result.body.content; + assert.equal(blocks[0].type, 'thinking'); + assert.equal(blocks[0].thinking, 'deep thoughts'); + assert.equal(blocks[1].type, 'text', 'the text block is ALWAYS present, even when empty'); + assert.equal(blocks[1].text, ''); + }); + + it('non-stream: with reasoning_content present, content is left untouched (no second thinking block)', async () => { + const result = await handleMessages({ + model: 'claude-sonnet-4.6', + stream: false, + messages: [{ role: 'user', content: 'hi' }], + }, { + async handleChatCompletions() { + return { + status: 200, + body: { + choices: [{ index: 0, message: { role: 'assistant', content: OPEN + 'x' + CLOSE, reasoning_content: 'real reasoning' }, finish_reason: 'stop' }], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + }, + }; + }, + }); + const blocks = result.body.content; + // one thinking block (from reasoning_content), content untouched + const thinkingBlocks = blocks.filter(b => b.type === 'thinking'); + assert.equal(thinkingBlocks.length, 1); + assert.equal(thinkingBlocks[0].thinking, 'real reasoning'); + const textBlock = blocks.find(b => b.type === 'text'); + assert.equal(textBlock.text, OPEN + 'x' + CLOSE); + }); + + it('with plain content, everything still flows as text on the stream path', async () => { + const result = await handleMessages({ + model: 'claude-sonnet-4.6', + stream: true, + messages: [{ role: 'user', content: 'hi' }], + }, { + async handleChatCompletions() { + return { + status: 200, + stream: true, + async handler(res) { + res.write(chatChunk({ choices: [{ index: 0, delta: { role: 'assistant', content: 'Just a normal reply.' }, finish_reason: null }] })); + res.write(chatChunk({ choices: [{ index: 0, delta: {}, finish_reason: 'stop' }] })); + res.write(chatChunk({ choices: [], usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 } })); + res.end('data: [DONE]\n\n'); + }, + }; + }, + }); + const res = fakeRes(); + await result.handler(res); + const events = parseAnthropicEvents(res.body); + const blocks = events.filter(e => e.event === 'content_block_start').map(e => e.data.content_block.type); + assert.deepEqual(blocks, ['text']); + const textDeltas = events.filter(e => e.event === 'content_block_delta' && e.data.delta?.type === 'text_delta').map(e => e.data.delta.text).join(''); + assert.equal(textDeltas, 'Just a normal reply.'); + }); +}); diff --git a/test/mutations/retry-rescue-budget-split.json b/test/mutations/retry-rescue-budget-split.json index f5d8a64e..c62410d1 100644 --- a/test/mutations/retry-rescue-budget-split.json +++ b/test/mutations/retry-rescue-budget-split.json @@ -3,7 +3,7 @@ "test/retry-rescue-budget-split.test.js", "test/devin-connect-openai.test.js" ], - "expectBaselinePass": 81, + "expectBaselinePass": 87, "mutations": [ { "name": "the original defect: the empty arm reads the shared loop counter again", diff --git a/test/mutations/think-text-reroute.json b/test/mutations/think-text-reroute.json new file mode 100644 index 00000000..2c2de66e --- /dev/null +++ b/test/mutations/think-text-reroute.json @@ -0,0 +1,27 @@ +{ + "tests": [ + "test/response-classifier.test.js", + "test/messages.test.js" + ], + "expectBaselinePass": 90, + "mutations": [ + { + "name": "leading-marker reroute disabled entirely — think-tagged content is never rerouted", + "file": "src/response-classifier.js", + "anchor": " if (this.pending.slice(0, oi).trim() === '') {", + "replacement": " if (false) {" + }, + { + "name": "unterminated think span silently dropped at flush instead of delivered as text", + "file": "src/response-classifier.js", + "anchor": " const out = this.pending;\n this.pending = '';\n this.mode = 'text';\n return out;", + "replacement": " const out = this.pending;\n this.pending = '';\n this.mode = 'text';\n return '';" + }, + { + "name": "think-mode buffering removed — think content leaks to the text channel mid-span", + "file": "src/response-classifier.js", + "anchor": " return { text: '', thinking: '' }; // buffer until the close marker proves it", + "replacement": " return { text: delta, thinking: '' }; // buffer until the close marker proves it" + } + ] +} \ No newline at end of file diff --git a/test/response-classifier.test.js b/test/response-classifier.test.js new file mode 100644 index 00000000..c548bf43 --- /dev/null +++ b/test/response-classifier.test.js @@ -0,0 +1,82 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { ThinkTextClassifier } from '../src/response-classifier.js'; + +const OPEN = '<' + 'think' + '>'; +const CLOSE = '<' + '/' + 'think' + '>'; + +function runAll(parts) { + const c = new ThinkTextClassifier(); + let text = '', thinking = ''; + for (const p of parts) { + const r = c.feed(p); + text += r.text; + thinking += r.thinking; + } + text += c.flush(); + return { text, thinking }; +} + +describe('ThinkTextClassifier', () => { + it('passes plain text through unchanged', () => { + const { text, thinking } = runAll(['Hello, ', 'world!']); + assert.equal(text, 'Hello, world!'); + assert.equal(thinking, ''); + }); + + it('routes a full think block to thinking', () => { + const { text, thinking } = runAll([OPEN + 'Let me compute. ' + CLOSE, 'The answer is 42.']); + assert.equal(thinking, 'Let me compute. '); + assert.equal(text, 'The answer is 42.'); + }); + + it('routes think block split across many deltas', () => { + const { text, thinking } = runAll(['<' + 'thin', 'k' + '>step ', 'one; step two', '<' + '/thi', 'nk' + '>', 'Final: 7.']); + assert.equal(thinking, 'step one; step two'); + assert.equal(text, 'Final: 7.'); + }); + + it('routes a bare opening marker (loop artifact) to thinking', () => { + const { text, thinking } = runAll([OPEN]); + assert.equal(thinking, ''); + assert.equal(text, ''); + }); + + it('does not reroute text that merely mentions the word think', () => { + const { text, thinking } = runAll(['I think this is correct.']); + assert.equal(text, 'I think this is correct.'); + assert.equal(thinking, ''); + }); + + it('does not reroute when real text precedes the marker in the same delta', () => { + const { text, thinking } = runAll(['Here is a sample: ' + OPEN + 'inner' + CLOSE + ' done.']); + assert.equal(thinking, ''); + assert.equal(text, 'Here is a sample: ' + OPEN + 'inner' + CLOSE + ' done.'); + }); + + it('allows leading whitespace before the marker', () => { + const { text, thinking } = runAll([' \n' + OPEN + 'reasoning' + CLOSE + 'Answer.']); + assert.equal(thinking, 'reasoning'); + assert.equal(text, 'Answer.'); // leading whitespace-only prefix is dropped + }); + + it('does not reroute once real text was committed (mid-answer marker)', () => { + const { text, thinking } = runAll(['Real answer here. ', OPEN + 'late noise' + CLOSE]); + assert.equal(thinking, ''); + assert.equal(text, 'Real answer here. ' + OPEN + 'late noise' + CLOSE); + }); + + it('flush releases an unterminated span as text (visible beats dropped)', () => { + const c = new ThinkTextClassifier(); + const r = c.feed(OPEN + 'dangling reasoning without close'); + assert.equal(r.text, ''); + assert.equal(r.thinking, ''); + assert.equal(c.flush(), 'dangling reasoning without close'); + }); + + it('empty and null deltas are inert', () => { + const c = new ThinkTextClassifier(); + assert.deepEqual(c.feed(''), { text: '', thinking: '' }); + assert.equal(c.flush(), ''); + }); +});