diff --git a/.changeset/fresh-pandas-listen.md b/.changeset/fresh-pandas-listen.md new file mode 100644 index 000000000..51e0e558c --- /dev/null +++ b/.changeset/fresh-pandas-listen.md @@ -0,0 +1,5 @@ +--- +'@livekit/agents': patch +--- + +Support AssemblyAI inference STT agent context carryover. diff --git a/.changeset/stt-context-options.md b/.changeset/stt-context-options.md new file mode 100644 index 000000000..b0cfeb39c --- /dev/null +++ b/.changeset/stt-context-options.md @@ -0,0 +1,6 @@ +--- +'@livekit/agents': patch +'@livekit/agents-plugin-assemblyai': patch +--- + +Add STT context options for keyterms and chat context forwarding. diff --git a/agents/src/inference/stt.test.ts b/agents/src/inference/stt.test.ts index 65df44750..a41237b86 100644 --- a/agents/src/inference/stt.test.ts +++ b/agents/src/inference/stt.test.ts @@ -1,12 +1,14 @@ // SPDX-FileCopyrightText: 2025 LiveKit, Inc. // // SPDX-License-Identifier: Apache-2.0 -import { beforeAll, describe, expect, it } from 'vitest'; +import { afterEach, beforeAll, describe, expect, it, vi } from 'vitest'; import * as agents from '../index.js'; import { normalizeLanguage } from '../language.js'; +import { AgentHandoffItem, ChatMessage } from '../llm/index.js'; import { initializeLogger } from '../log.js'; import { type APIConnectOptions, DEFAULT_API_CONNECT_OPTIONS } from '../types.js'; import { VAD, type VADStream } from '../vad.js'; +import { createConversationItemAddedEvent } from '../voice/events.js'; import { STT, type STTFallbackModel, @@ -21,6 +23,10 @@ beforeAll(() => { initializeLogger({ level: 'silent', pretty: false }); }); +afterEach(() => { + vi.restoreAllMocks(); +}); + /** Helper to create STT with required credentials. */ function makeStt(overrides: Record = {}) { const defaults = { @@ -32,6 +38,16 @@ function makeStt(overrides: Record = {}) { return new STT({ ...defaults, ...overrides }); } +function makeAssemblyStt(overrides: Record = {}) { + return makeStt({ model: 'assemblyai/universal-3-5-pro', ...overrides }); +} + +function assistantItemEvent(text: string) { + return createConversationItemAddedEvent( + ChatMessage.create({ role: 'assistant', content: [text] }), + ); +} + describe('parseSTTModelString', () => { it('simple model without language', () => { const [model, language] = parseSTTModelString('deepgram'); @@ -329,6 +345,122 @@ describe('STT diarization capabilities', () => { }); }); +describe('STT agent_context forwarding', () => { + it('enables chat context for AssemblyAI U3 Pro family models', () => { + for (const model of ['assemblyai/u3-rt-pro', 'assemblyai/universal-3-5-pro'] as const) { + const stt = makeAssemblyStt({ model }); + expect(stt.capabilities.chatContext).toBe(true); + } + }); + + it('disables chat context for unsupported AssemblyAI models', () => { + const stt = makeAssemblyStt({ model: 'assemblyai/universal-streaming' }); + + expect(stt.capabilities.chatContext).toBe(false); + }); + + it('disables chat context for non-AssemblyAI models', () => { + const stt = makeAssemblyStt({ model: 'deepgram/nova-3' }); + expect(stt.capabilities.chatContext).toBe(false); + }); + + it('derives chatContext capability from model support only', () => { + const stt = makeAssemblyStt({ modelOptions: { previous_context_n_turns: 0 } }); + expect(stt.capabilities.chatContext).toBe(true); + }); + + it('emits active carryover shaping in the session.update wire payload', () => { + const stt = makeAssemblyStt(); + const stream = stt.stream(); + const sent: string[] = []; + Reflect.set(stream, 'activeWs', { + readyState: 1, + send: (payload: string) => sent.push(payload), + }); + + stt._pushConversationItem(assistantItemEvent('Your room is booked for Tuesday.')); + + expect(stt['opts'].modelOptions).toHaveProperty( + 'agent_context', + 'Your room is booked for Tuesday.', + ); + expect(sent).toHaveLength(1); + expect(JSON.parse(sent[0]!)).toMatchObject({ + type: 'session.update', + settings: { + extra: { + agent_context: 'Your room is booked for Tuesday.', + }, + }, + }); + stream.close(); + }); + + it('truncates oversize replies keeping the tail', () => { + const text = 'a'.repeat(2000) + 'b'.repeat(1750); + const stt = makeAssemblyStt(); + stt._pushConversationItem(assistantItemEvent(text)); + expect(stt['opts'].modelOptions).toHaveProperty('agent_context', 'b'.repeat(1750)); + }); + + it('ignores non-assistant items', () => { + const stt = makeAssemblyStt(); + + stt._pushConversationItem( + createConversationItemAddedEvent(ChatMessage.create({ role: 'user', content: ['hi there'] })), + ); + expect(stt['opts'].modelOptions).not.toHaveProperty('agent_context'); + + stt._pushConversationItem( + createConversationItemAddedEvent(ChatMessage.create({ role: 'assistant', content: [] })), + ); + expect(stt['opts'].modelOptions).not.toHaveProperty('agent_context'); + }); + + it('ignores agent handoff items', () => { + const stt = makeAssemblyStt(); + stt._pushConversationItem( + createConversationItemAddedEvent(AgentHandoffItem.create({ newAgentId: 'agent-2' })), + ); + expect(stt['opts'].modelOptions).not.toHaveProperty('agent_context'); + }); + + it('preserves explicit agent_context and later overwrites it with carryover', () => { + const stt = makeAssemblyStt({ + modelOptions: { agent_context: 'The agent asked for a booking date.' }, + }); + expect(stt['opts'].modelOptions).toHaveProperty( + 'agent_context', + 'The agent asked for a booking date.', + ); + + stt._pushConversationItem(assistantItemEvent('And your zip code?')); + expect(stt['opts'].modelOptions).toHaveProperty('agent_context', 'And your zip code?'); + }); + + it('starts forwarding after changing from an unsupported to a supported model', () => { + const stt = makeAssemblyStt({ model: 'assemblyai/universal-streaming' }); + + stt._pushConversationItem(assistantItemEvent('ignored before transition')); + expect(stt['opts'].modelOptions).not.toHaveProperty('agent_context'); + + stt.updateOptions({ model: 'assemblyai/universal-3-5-pro' }); + stt._pushConversationItem(assistantItemEvent('forwarded after transition')); + + expect(stt['opts'].modelOptions).toHaveProperty('agent_context', 'forwarded after transition'); + }); + + it('stops forwarding after changing from a supported to an unsupported model', () => { + const stt = makeAssemblyStt(); + stt._pushConversationItem(assistantItemEvent('last supported context')); + + stt.updateOptions({ model: 'assemblyai/universal-streaming' }); + stt._pushConversationItem(assistantItemEvent('ignored after transition')); + + expect(stt['opts'].modelOptions).toHaveProperty('agent_context', 'last supported context'); + }); +}); + describe('STT session keyterms', () => { it('updateOptions does not bake session keyterms into the user baseline', () => { const stt = makeStt({ model: 'deepgram/nova-3' }); diff --git a/agents/src/inference/stt.ts b/agents/src/inference/stt.ts index 3cfc8e852..2e8744375 100644 --- a/agents/src/inference/stt.ts +++ b/agents/src/inference/stt.ts @@ -7,6 +7,7 @@ import type { WebSocket } from 'ws'; import { APIError, APIStatusError } from '../_exceptions.js'; import { AudioByteStream } from '../audio.js'; import { type LanguageCode, areLanguagesEquivalent, normalizeLanguage } from '../language.js'; +import { ChatMessage } from '../llm/index.js'; import { log } from '../log.js'; import { createStreamChannel } from '../stream/stream_channel.js'; import { @@ -19,6 +20,7 @@ import { import { type APIConnectOptions, DEFAULT_API_CONNECT_OPTIONS } from '../types.js'; import { type AudioBuffer, Event, Task, cancelAndWait, shortuuid, waitForAbort } from '../utils.js'; import { type VAD, VADEventType, type VADStream } from '../vad.js'; +import type { ConversationItemAddedEvent } from '../voice/events.js'; import { type TimedString, createTimedString } from '../voice/io.js'; import { type SttServerEvent, @@ -120,8 +122,10 @@ export interface AssemblyAIOptions { keyterms_prompt?: string[]; /** Enable speaker diarization. Default: false. */ speaker_labels?: boolean; - /** Context to bias recognition. Only supported with u3-rt-pro. Max 1500 chars. */ + /** Context to bias recognition. Only supported with u3-rt-pro. Max 1750 chars. */ agent_context?: string; + /** Prior turns carried as context; 0 disables carryover. Only supported with u3-rt-pro. */ + previous_context_n_turns?: number; /** Isolate the primary voice. Only supported with u3-rt-pro. */ voice_focus?: 'near-field' | 'far-field'; /** Background suppression strength. Only supported with u3-rt-pro. */ @@ -270,6 +274,17 @@ function keytermsExtraForModel( return { [key]: [...new Set([...existing, ...sessionKeyterms])] }; } +const ASSEMBLYAI_CARRYOVER_MODELS = [ + 'assemblyai/u3-rt-pro', + 'assemblyai/universal-3-5-pro', +] as const; + +const ASSEMBLYAI_MAX_AGENT_CONTEXT_CHARS = 1750; + +function supportsChatContext(model: string | undefined): boolean { + return model === ASSEMBLYAI_CARRYOVER_MODELS[0] || model === ASSEMBLYAI_CARRYOVER_MODELS[1]; +} + type _STTModels = | DeepgramModels | DeepgramFluxModels @@ -412,19 +427,32 @@ export class STT extends BaseSTT { vad?: VAD; }) { const modelOptions = (opts?.modelOptions ?? {}) as STTOptions; + // Parse language from model string if provided: "provider/model:language". + let nextModel = opts?.model; + let nextLanguage = opts?.language; + let hasLanguageConflict = false; + if (typeof nextModel === 'string') { + const [parsedModel, parsedLanguage] = parseSTTModelString(nextModel); + if (parsedLanguage !== undefined) { + hasLanguageConflict = + !!nextLanguage && !areLanguagesEquivalent(nextLanguage, parsedLanguage); + if (!hasLanguageConflict) { + nextLanguage = parsedLanguage as STTLanguages; + } + nextModel = parsedModel as TModel; + } + } super({ streaming: true, interimResults: true, alignedTranscript: 'word', diarization: diarizationEnabled(modelOptions as Record), keyterms: - keytermsExtraForModel(typeof opts?.model === 'string' ? opts.model : undefined) !== - undefined, + keytermsExtraForModel(typeof nextModel === 'string' ? nextModel : undefined) !== undefined, + chatContext: supportsChatContext(typeof nextModel === 'string' ? nextModel : undefined), }); const { - model, - language, baseURL, encoding = DEFAULT_ENCODING, sampleRate = DEFAULT_SAMPLE_RATE, @@ -447,22 +475,11 @@ export class STT extends BaseSTT { throw new Error('apiSecret is required: pass apiSecret or set LIVEKIT_API_SECRET'); } - // Parse language from model string if provided: "provider/model:language" - let nextModel = model; - let nextLanguage = language; - if (typeof nextModel === 'string') { - const [parsedModel, parsedLanguage] = parseSTTModelString(nextModel); - if (parsedLanguage !== undefined) { - if (nextLanguage && !areLanguagesEquivalent(nextLanguage, parsedLanguage)) { - this.#logger.warn( - '`language` is provided via both argument and model, using the one from the argument', - { language: nextLanguage, model: nextModel }, - ); - } else { - nextLanguage = parsedLanguage as STTLanguages; - } - nextModel = parsedModel as TModel; - } + if (hasLanguageConflict) { + this.#logger.warn( + '`language` is provided via both argument and model, using the one from the argument', + { language: nextLanguage, model: opts?.model }, + ); } const normalizedFallback = fallback ? normalizeSTTFallback(fallback) : undefined; this.vad = resolveVADForModel(nextModel, vad); @@ -531,6 +548,7 @@ export class STT extends BaseSTT { this._vadPromise = undefined; this.updateCapabilities({ keyterms: keytermsExtraForModel(this.opts.model) !== undefined, + chatContext: supportsChatContext(this.opts.model), }); } @@ -587,6 +605,25 @@ export class STT extends BaseSTT { } } + override _pushConversationItem(ev: ConversationItemAddedEvent): void { + if (!this.capabilities.chatContext) { + return; + } + + const chatItem = ev.item; + if (chatItem instanceof ChatMessage && chatItem.role === 'assistant' && chatItem.textContent) { + let text = chatItem.textContent; + if (text.length > ASSEMBLYAI_MAX_AGENT_CONTEXT_CHARS) { + this.#logger.debug( + { fromChars: text.length, toChars: ASSEMBLYAI_MAX_AGENT_CONTEXT_CHARS }, + 'truncating agent_context carryover', + ); + text = text.slice(-ASSEMBLYAI_MAX_AGENT_CONTEXT_CHARS); + } + this.updateOptions({ modelOptions: { agent_context: text } as STTOptions }); + } + } + stream(options?: { language?: STTLanguages | string; connOptions?: APIConnectOptions; diff --git a/agents/src/stt/fallback_adapter.test.ts b/agents/src/stt/fallback_adapter.test.ts index db616664b..472b2a38a 100644 --- a/agents/src/stt/fallback_adapter.test.ts +++ b/agents/src/stt/fallback_adapter.test.ts @@ -4,9 +4,14 @@ import type { EventEmitter } from 'node:events'; import { beforeAll, describe, expect, it, vi } from 'vitest'; import { APIConnectionError, APIError } from '../_exceptions.js'; +import { ChatMessage } from '../llm/index.js'; import { initializeLogger } from '../log.js'; import type { APIConnectOptions } from '../types.js'; import { type AudioBuffer, delay } from '../utils.js'; +import { + type ConversationItemAddedEvent, + createConversationItemAddedEvent, +} from '../voice/events.js'; import { FallbackAdapter } from './fallback_adapter.js'; import { STT, type SpeechEvent, SpeechEventType, SpeechStream } from './stt.js'; import { FakeSTT, RecognizeSentinel, emptyAudioFrame } from './testing/fake_stt.js'; @@ -123,6 +128,55 @@ describe('FallbackAdapter', () => { expect(adapter.capabilities.streaming).toBe(true); }); + it('forwards conversation context only to children that support it', () => { + class ContextRecordingSTT extends FakeSTT { + readonly conversationItems: ConversationItemAddedEvent[] = []; + + constructor(label: string, supportsChatContext: boolean) { + super({ label }); + this.updateCapabilities({ chatContext: supportsChatContext }); + } + + override _pushConversationItem(ev: ConversationItemAddedEvent): void { + this.conversationItems.push(ev); + } + } + + const supported = new ContextRecordingSTT('supported', true); + const unsupported = new ContextRecordingSTT('unsupported', false); + const adapter = new FallbackAdapter({ sttInstances: [supported, unsupported] }); + const event = createConversationItemAddedEvent( + ChatMessage.create({ role: 'assistant', content: ['hello'] }), + ); + + adapter._pushConversationItem(event); + + expect(supported.conversationItems).toEqual([event]); + expect(unsupported.conversationItems).toEqual([]); + }); + + it('tracks dynamic child conversation-context capabilities and removes listeners on close', async () => { + class DynamicContextSTT extends FakeSTT { + setChatContext(supported: boolean): void { + this.updateCapabilities({ chatContext: supported }); + } + } + + const child = new DynamicContextSTT(); + const adapter = new FallbackAdapter({ sttInstances: [child] }); + expect(adapter.capabilities.chatContext).toBe(false); + expect(child.listenerCount('capabilities_changed')).toBe(1); + + child.setChatContext(true); + expect(adapter.capabilities.chatContext).toBe(true); + + child.setChatContext(false); + expect(adapter.capabilities.chatContext).toBe(false); + + await adapter.close(); + expect(child.listenerCount('capabilities_changed')).toBe(0); + }); + it('_recognize falls through to the next instance on error', async () => { const primary = new FakeSTT({ label: 'primary', diff --git a/agents/src/stt/fallback_adapter.ts b/agents/src/stt/fallback_adapter.ts index f733545bb..0ad05815e 100644 --- a/agents/src/stt/fallback_adapter.ts +++ b/agents/src/stt/fallback_adapter.ts @@ -98,6 +98,7 @@ export class FallbackAdapter extends STT { private _status: STTStatus[] = []; private _logger = log(); private _metricsForwarders = new Map void>(); + private _capabilitiesForwarders = new Map void>(); // Last child that produced output or returned a recognize result. Surfaced // via the dynamic label/model/provider getters so OTel attributes like // `gen_ai.request.model` on `user_turn` (refreshed on every STT event by @@ -197,9 +198,10 @@ export class FallbackAdapter extends STT { } override _pushConversationItem(ev: ConversationItemAddedEvent): void { - // forward to every underlying STT; unsupported ones warn-and-skip internally for (const sttInstance of this.sttInstances) { - sttInstance._pushConversationItem(ev); + if (sttInstance.capabilities.chatContext) { + sttInstance._pushConversationItem(ev); + } } } @@ -213,8 +215,15 @@ export class FallbackAdapter extends STT { // SpeechStream.mainTask emits that on this STT instance naturally. for (const s of this.sttInstances) { const metricsForwarder = (metrics: STTMetrics) => this.emit('metrics_collected', metrics); + const capabilitiesForwarder = () => { + this.updateCapabilities({ + chatContext: this.sttInstances.some((stt) => !!stt.capabilities.chatContext), + }); + }; this._metricsForwarders.set(s, metricsForwarder); + this._capabilitiesForwarders.set(s, capabilitiesForwarder); s.on('metrics_collected', metricsForwarder); + s.on('capabilities_changed', capabilitiesForwarder); } } @@ -325,8 +334,11 @@ export class FallbackAdapter extends STT { for (const s of this.sttInstances) { const m = this._metricsForwarders.get(s); if (m) s.off('metrics_collected' as keyof STTCallbacks, m); + const c = this._capabilitiesForwarders.get(s); + if (c) s.off('capabilities_changed', c); } this._metricsForwarders.clear(); + this._capabilitiesForwarders.clear(); } } diff --git a/agents/src/stt/stt.ts b/agents/src/stt/stt.ts index b6f157519..f0f19061b 100644 --- a/agents/src/stt/stt.ts +++ b/agents/src/stt/stt.ts @@ -154,6 +154,7 @@ export interface STTError { export type STTCallbacks = { ['metrics_collected']: (metrics: STTMetrics) => void; ['error']: (error: STTError) => void; + ['capabilities_changed']: (capabilities: STTCapabilities) => void; }; /** @@ -180,7 +181,20 @@ export abstract class STT extends (EventEmitter as new () => TypedEmitter): void { - this.#capabilities = { ...this.#capabilities, ...caps }; + const next = { ...this.#capabilities, ...caps }; + const changed = + next.streaming !== this.#capabilities.streaming || + next.interimResults !== this.#capabilities.interimResults || + next.alignedTranscript !== this.#capabilities.alignedTranscript || + next.diarization !== this.#capabilities.diarization || + next.keyterms !== this.#capabilities.keyterms || + next.chatContext !== this.#capabilities.chatContext; + if (!changed) { + return; + } + + this.#capabilities = next; + this.emit('capabilities_changed', next); } /** diff --git a/agents/src/voice/agent_activity.ts b/agents/src/voice/agent_activity.ts index 5e7eb372c..81af9c465 100644 --- a/agents/src/voice/agent_activity.ts +++ b/agents/src/voice/agent_activity.ts @@ -618,18 +618,13 @@ export class AgentActivity implements RecognitionHooks { if (this.stt instanceof STT) { // bind the session's keyterm detector to this activity's STT (detection uses its - // own LLM, configured via keytermsOptions, not the agent's) + // own LLM, configured via sttContextOptions, not the agent's) this.agentSession._keytermDetector.start(this.agentSession, this.stt); - // forward conversation turns to STTs that consume context natively; gated by the - // STT's own capability (toggled via the STT's args). stateless and activity-scoped, - // so it lives here rather than in the detector. - if (this.stt.capabilities.chatContext) { - this.agentSession.on( - AgentSessionEventTypes.ConversationItemAdded, - this.pushConversationItemToStt, - ); - } + // Forward conversation turns to STTs that consume context natively. Keep the + // listener synchronized with dynamic capability changes and the session opt-out. + this.stt.on('capabilities_changed', this.syncConversationItemForwarding); + this.syncConversationItemForwarding(); } // Bundled-default VAD is treated as absent when the RealtimeModel does @@ -1219,6 +1214,23 @@ export class AgentActivity implements RecognitionHooks { } }; + private syncConversationItemForwarding = (): void => { + this.agentSession.off( + AgentSessionEventTypes.ConversationItemAdded, + this.pushConversationItemToStt, + ); + if ( + this.stt instanceof STT && + this.stt.capabilities.chatContext && + this.agentSession.sessionOptions.sttContextOptions.forwardChatContext + ) { + this.agentSession.on( + AgentSessionEventTypes.ConversationItemAdded, + this.pushConversationItemToStt, + ); + } + }; + private onError(ev: RealtimeModelError | STTError | TTSError | LLMError): void { if (ev.type === 'realtime_model_error') { const errorEvent = createErrorEvent(ev, this.llm); @@ -4617,6 +4629,7 @@ export class AgentActivity implements RecognitionHooks { if (this.stt instanceof STT) { this.stt.off('metrics_collected', this.onMetricsCollected); this.stt.off('error', this.onModelError); + this.stt.off('capabilities_changed', this.syncConversationItemForwarding); this.agentSession.off( AgentSessionEventTypes.ConversationItemAdded, this.pushConversationItemToStt, diff --git a/agents/src/voice/agent_activity_stt_context.test.ts b/agents/src/voice/agent_activity_stt_context.test.ts new file mode 100644 index 000000000..5b874fb61 --- /dev/null +++ b/agents/src/voice/agent_activity_stt_context.test.ts @@ -0,0 +1,106 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { describe, expect, it, vi } from 'vitest'; +import { STT as InferenceSTT } from '../inference/stt.js'; +import { ChatMessage } from '../llm/index.js'; +import { initializeLogger } from '../log.js'; +import { Agent } from './agent.js'; +import { AgentSession } from './agent_session.js'; +import { AgentSessionEventTypes, createConversationItemAddedEvent } from './events.js'; + +describe('AgentActivity STT conversation-context lifecycle', () => { + initializeLogger({ pretty: false, level: 'silent' }); + + it('forwards assistant context by default and removes listeners on close', async () => { + const stt = new InferenceSTT({ + model: 'assemblyai/universal-3-5-pro', + apiKey: 'test-key', + apiSecret: 'test-secret', + baseURL: 'https://example.livekit.cloud', + }); + const pushSpy = vi.spyOn(stt, '_pushConversationItem'); + const session = new AgentSession({ stt }); + const event = createConversationItemAddedEvent( + ChatMessage.create({ role: 'assistant', content: ['hello'] }), + ); + + await session.start({ agent: new Agent({ instructions: 'test' }) }); + expect(session.listenerCount(AgentSessionEventTypes.ConversationItemAdded)).toBe(1); + + session.emit(AgentSessionEventTypes.ConversationItemAdded, event); + expect(pushSpy).toHaveBeenCalledTimes(1); + + await session.close(); + expect(session.listenerCount(AgentSessionEventTypes.ConversationItemAdded)).toBe(0); + expect(stt.listenerCount('capabilities_changed')).toBe(0); + }); + + it('does not forward context when the session opts out', async () => { + const stt = new InferenceSTT({ + model: 'assemblyai/universal-3-5-pro', + apiKey: 'test-key', + apiSecret: 'test-secret', + baseURL: 'https://example.livekit.cloud', + }); + const pushSpy = vi.spyOn(stt, '_pushConversationItem'); + const session = new AgentSession({ + stt, + sttContextOptions: { forwardChatContext: false }, + }); + + await session.start({ agent: new Agent({ instructions: 'test' }) }); + session.emit( + AgentSessionEventTypes.ConversationItemAdded, + createConversationItemAddedEvent( + ChatMessage.create({ role: 'assistant', content: ['hello'] }), + ), + ); + + expect(pushSpy).not.toHaveBeenCalled(); + expect(session.listenerCount(AgentSessionEventTypes.ConversationItemAdded)).toBe(0); + await session.close(); + }); + + it('tracks model-only capability through dynamic model transitions', async () => { + const stt = new InferenceSTT({ + model: 'assemblyai/universal-streaming', + apiKey: 'test-key', + apiSecret: 'test-secret', + baseURL: 'https://example.livekit.cloud', + }); + const pushSpy = vi.spyOn(stt, '_pushConversationItem'); + const session = new AgentSession({ stt }); + const event = createConversationItemAddedEvent( + ChatMessage.create({ role: 'assistant', content: ['hello'] }), + ); + + await session.start({ agent: new Agent({ instructions: 'test' }) }); + expect(stt.listenerCount('capabilities_changed')).toBe(1); + + session.emit(AgentSessionEventTypes.ConversationItemAdded, event); + expect(pushSpy).not.toHaveBeenCalled(); + + stt.updateOptions({ model: 'assemblyai/universal-3-5-pro' }); + session.emit(AgentSessionEventTypes.ConversationItemAdded, event); + expect(pushSpy).toHaveBeenCalledTimes(1); + + stt.updateOptions({ modelOptions: { previous_context_n_turns: 0 } }); + expect(stt.capabilities.chatContext).toBe(true); + session.emit(AgentSessionEventTypes.ConversationItemAdded, event); + expect(pushSpy).toHaveBeenCalledTimes(2); + + stt.updateOptions({ model: 'assemblyai/universal-streaming' }); + expect(stt.capabilities.chatContext).toBe(false); + session.emit(AgentSessionEventTypes.ConversationItemAdded, event); + expect(pushSpy).toHaveBeenCalledTimes(2); + + stt.updateOptions({ model: 'assemblyai/universal-3-5-pro' }); + expect(stt.capabilities.chatContext).toBe(true); + session.emit(AgentSessionEventTypes.ConversationItemAdded, event); + expect(pushSpy).toHaveBeenCalledTimes(3); + + await session.close(); + expect(stt.listenerCount('capabilities_changed')).toBe(0); + }); +}); diff --git a/agents/src/voice/agent_session.test.ts b/agents/src/voice/agent_session.test.ts index 0c7fbebed..7c6c37a06 100644 --- a/agents/src/voice/agent_session.test.ts +++ b/agents/src/voice/agent_session.test.ts @@ -2,6 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 import { describe, expect, it, vi } from 'vitest'; +import { log } from '../log.js'; import { AgentSession, resolveRecordingOptions } from './agent_session.js'; import { AgentSessionEventTypes, createUserInputTranscribedEvent } from './events.js'; import { SpeechHandle } from './speech_handle.js'; @@ -91,6 +92,47 @@ describe('AgentSession recording state', () => { }); }); +describe('AgentSession STT context options', () => { + it('defaults forwardChatContext on', () => { + const session = new AgentSession({ vad: null }); + + expect(session.sessionOptions.sttContextOptions.forwardChatContext).toBe(true); + }); + + it('passes through sttContextOptions', () => { + const session = new AgentSession({ + vad: null, + sttContextOptions: { keyterms: ['LiveKit'], forwardChatContext: false }, + }); + + expect(session.sessionOptions.sttContextOptions.keyterms).toEqual(['LiveKit']); + expect(session.sessionOptions.sttContextOptions.forwardChatContext).toBe(false); + }); + + it('maps deprecated keytermsOptions to sttContextOptions and warns', () => { + const warn = vi.spyOn(log(), 'warn'); + + const session = new AgentSession({ vad: null, keytermsOptions: { keyterms: ['Acme'] } }); + + expect(session.sessionOptions.sttContextOptions.keyterms).toEqual(['Acme']); + expect(session.sessionOptions.sttContextOptions.forwardChatContext).toBe(true); + expect(warn).toHaveBeenCalledWith( + 'keytermsOptions is deprecated, use sttContextOptions instead', + ); + warn.mockRestore(); + }); + + it('sttContextOptions wins over keytermsOptions', () => { + const session = new AgentSession({ + vad: null, + sttContextOptions: { keyterms: ['new'] }, + keytermsOptions: { keyterms: ['old'] }, + }); + + expect(session.sessionOptions.sttContextOptions.keyterms).toEqual(['new']); + }); +}); + describe('AgentSession user input transcription', () => { it('resets the away timer on final transcripts when not speaking', () => { const session = new AgentSession({ userAwayTimeout: 15 }); diff --git a/agents/src/voice/agent_session.ts b/agents/src/voice/agent_session.ts index cc0f61576..2cabe86dd 100644 --- a/agents/src/voice/agent_session.ts +++ b/agents/src/voice/agent_session.ts @@ -95,7 +95,8 @@ import { AgentInput, AgentOutput } from './io.js'; import { KeytermDetector, type KeytermsOptions, - resolveKeytermsOptions, + type ResolvedSTTContextOptions, + type STTContextOptions, } from './keyterm_detection.js'; import { RecorderIO } from './recorder_io/index.js'; import { RoomSessionTransport, SessionHost } from './remote_session.js'; @@ -181,6 +182,7 @@ export function resolveRecordingOptions( export interface InternalSessionOptions extends AgentSessionOptions { turnHandling: InternalTurnHandlingOptions; + sttContextOptions: ResolvedSTTContextOptions; useTtsAlignedTranscript: boolean; maxToolSteps: number; userAwayTimeout: number | null; @@ -310,9 +312,16 @@ export type AgentSessionOptions = { turnHandling?: Partial; /** - * Keyterm biasing for the STT. Holds static `keyterms` plus `keytermDetection` - * (LLM extraction). Applies to STTs that accept a term list; on others it warns - * and is ignored. + * Conversation-aware context for the STT: static `keyterms` plus `keytermDetection` for STTs + * that accept a term list, and `forwardChatContext` (on by default) that forwards conversation + * turns to STTs that consume context directly. Applied where the STT supports it, ignored + * otherwise. + */ + sttContextOptions?: STTContextOptions; + + /** + * @deprecated Use `sttContextOptions` instead. Its `keyterms`/`keytermDetection` keys map onto + * the new option. */ keytermsOptions?: KeytermsOptions; @@ -573,10 +582,9 @@ export class AgentSession< this._chatCtx = ChatContext.empty(); this.sessionOptions = resolvedSessionOptions; - const keytermsOptions = resolveKeytermsOptions(this.sessionOptions.keytermsOptions); this._keytermDetector = new KeytermDetector({ - staticKeyterms: keytermsOptions.keyterms, - options: keytermsOptions.keytermDetection, + staticKeyterms: this.sessionOptions.sttContextOptions.keyterms, + options: this.sessionOptions.sttContextOptions.keytermDetection, }); this.options = legacyVoiceOptions; diff --git a/agents/src/voice/index.ts b/agents/src/voice/index.ts index 3e2ea3ccd..f3b788751 100644 --- a/agents/src/voice/index.ts +++ b/agents/src/voice/index.ts @@ -40,6 +40,7 @@ export { KeytermDetector, type KeytermDetectionOptions, type KeytermsOptions, + type STTContextOptions, } from './keyterm_detection.js'; export { AudioInput, diff --git a/agents/src/voice/keyterm_detection.test.ts b/agents/src/voice/keyterm_detection.test.ts index 8802c7232..64dd4205c 100644 --- a/agents/src/voice/keyterm_detection.test.ts +++ b/agents/src/voice/keyterm_detection.test.ts @@ -21,6 +21,8 @@ import { formatInput, parseToolCall, resolveDetection, + resolveSTTContextOptions, + sttContextFromKeytermsOptions, } from './keyterm_detection.js'; type DetectionResult = [string[], string[], string[]]; @@ -676,4 +678,39 @@ describe('module helpers', () => { expect(resolved.turnInterval).toBe(1); expect(resolved.maxKeyterms).toBeUndefined(); }); + + it('resolve STT context options defaults', () => { + const resolved = resolveSTTContextOptions(undefined); + expect(resolved.keyterms).toEqual([]); + expect(resolved.keytermDetection.enabled).toBe(false); + expect(resolved.forwardChatContext).toBe(true); + }); + + it('resolve STT context options passthrough', () => { + const resolved = resolveSTTContextOptions({ + keyterms: ['LiveKit'], + keytermDetection: { enabled: true, turnInterval: 2 }, + forwardChatContext: false, + }); + + expect(resolved.keyterms).toEqual(['LiveKit']); + expect(resolved.keytermDetection.enabled).toBe(true); + expect(resolved.keytermDetection.turnInterval).toBe(2); + expect(resolved.forwardChatContext).toBe(false); + }); + + it('STT context from keyterms options maps keys', () => { + const mapped = sttContextFromKeytermsOptions({ + keyterms: ['Acme'], + keytermDetection: { enabled: true }, + }); + expect(mapped).toEqual({ keyterms: ['Acme'], keytermDetection: { enabled: true } }); + + const resolved = resolveSTTContextOptions(mapped); + expect(resolved.keyterms).toEqual(['Acme']); + expect(resolved.keytermDetection.enabled).toBe(true); + expect(resolved.forwardChatContext).toBe(true); + + expect(sttContextFromKeytermsOptions(undefined)).toEqual({}); + }); }); diff --git a/agents/src/voice/keyterm_detection.ts b/agents/src/voice/keyterm_detection.ts index 317a3ef66..a38f7447b 100644 --- a/agents/src/voice/keyterm_detection.ts +++ b/agents/src/voice/keyterm_detection.ts @@ -17,16 +17,8 @@ import { AgentSessionEventTypes, type ConversationItemAddedEvent } from './event /** * Keyterm biasing for STTs that accept a term list. * - * Can be passed as a plain object: - * - * ```ts - * new AgentSession({ - * keytermsOptions: { - * keyterms: ['LiveKit', 'Acme Corp'], - * keytermDetection: { enabled: true, turnInterval: 1 }, - * }, - * }); - * ``` + * @deprecated Use {@link STTContextOptions} (`new AgentSession({ sttContextOptions: ... })`) + * instead; its `keyterms` and `keytermDetection` keys are identical. */ export interface KeytermsOptions { /** Static keyterms applied wherever the STT accepts a term list; never touched by detection. */ @@ -63,6 +55,33 @@ export interface KeytermDetectionOptions { timeout?: number; } +/** + * Conversation-aware context for the STT. + * + * Can be passed as a plain object: + * + * ```ts + * new AgentSession({ + * sttContextOptions: { + * keyterms: ['LiveKit', 'Acme Corp'], + * keytermDetection: { enabled: true, turnInterval: 1 }, + * forwardChatContext: true, + * }, + * }); + * ``` + */ +export interface STTContextOptions { + /** Static keyterms applied wherever the STT accepts a term list; never touched by detection. */ + keyterms?: string[]; + /** LLM-based keyterm extraction, for STTs that accept a term list. */ + keytermDetection?: KeytermDetectionOptions; + /** + * Forward conversation turns to STTs that consume context directly (e.g. AssemblyAI U3 Pro). + * Defaults to `true`; only STTs that advertise the `chatContext` capability act on it. + */ + forwardChatContext?: boolean; +} + // bound a single pass so a stuck LLM call can't hold the single-flight guard forever and // stall detection for the rest of the call; a timed-out pass simply makes no change const DETECTION_TIMEOUT = 10_000; @@ -86,10 +105,11 @@ export interface ResolvedKeytermDetectionOptions { timeout: number; } -/** A fully-defaulted keyterms config. @internal */ -export interface ResolvedKeytermsOptions { +/** A fully-defaulted STT-context config. @internal */ +export interface ResolvedSTTContextOptions { keyterms: string[]; keytermDetection: ResolvedKeytermDetectionOptions; + forwardChatContext: boolean; } /** A pending term not confirmed within this many passes is dropped. @internal */ @@ -117,15 +137,28 @@ export function resolveDetection( }; } -/** Return a fully-defaulted keyterms config. @internal */ -export function resolveKeytermsOptions(config?: KeytermsOptions): ResolvedKeytermsOptions { +/** Return a fully-defaulted STT-context config. @internal */ +export function resolveSTTContextOptions(config?: STTContextOptions): ResolvedSTTContextOptions { const cfg = config ?? {}; return { keyterms: [...(cfg.keyterms ?? [])], keytermDetection: resolveDetection(cfg.keytermDetection), + forwardChatContext: cfg.forwardChatContext ?? true, }; } +/** Map deprecated `keytermsOptions` onto the new {@link STTContextOptions} shape. @internal */ +export function sttContextFromKeytermsOptions(config?: KeytermsOptions): STTContextOptions { + const out: STTContextOptions = {}; + if (config?.keyterms !== undefined) { + out.keyterms = config.keyterms; + } + if (config?.keytermDetection !== undefined) { + out.keytermDetection = config.keytermDetection; + } + return out; +} + /** * Resolve the configured detection `llm`: an `LLM` instance is used directly; a * model string (or the default model when unset) is created via the inference gateway. diff --git a/agents/src/voice/turn_config/utils.ts b/agents/src/voice/turn_config/utils.ts index 100cda367..86d8191e1 100644 --- a/agents/src/voice/turn_config/utils.ts +++ b/agents/src/voice/turn_config/utils.ts @@ -9,6 +9,7 @@ import { type TurnDetectionMode, type VoiceOptions, } from '../agent_session.js'; +import { resolveSTTContextOptions, sttContextFromKeytermsOptions } from '../keyterm_detection.js'; import { type EndpointingOptions, defaultEndpointingOptions, @@ -50,6 +51,8 @@ export function migrateLegacyOptions(legacyOptions: AgentSessionOption tts, userData, connOptions, + sttContextOptions, + keytermsOptions, ...sessionOptions } = legacyOptions; @@ -59,6 +62,15 @@ export function migrateLegacyOptions(legacyOptions: AgentSessionOption ); } + if (keytermsOptions !== undefined) { + logger.warn('keytermsOptions is deprecated, use sttContextOptions instead'); + } + + const resolvedSTTContextOptions = + sttContextOptions !== undefined + ? resolveSTTContextOptions(sttContextOptions) + : resolveSTTContextOptions(sttContextFromKeytermsOptions(keytermsOptions)); + const turnHandling: TurnHandlingOptions = { interruption: { discardAudioIfUninterruptible: voiceOptions?.discardAudioIfUninterruptible, @@ -130,6 +142,7 @@ export function migrateLegacyOptions(legacyOptions: AgentSessionOption ...defaultSessionOptions, ...migratedVoiceOptions, ...sessionOptions, + sttContextOptions: resolvedSTTContextOptions, turnHandling: mergeWithDefaults(turnHandling), // repopulate the deprecated voice options with migrated options for backwards compatibility voiceOptions: legacyVoiceOptions, diff --git a/examples/src/basic_agent.ts b/examples/src/basic_agent.ts index fc9b371de..9d03143e9 100644 --- a/examples/src/basic_agent.ts +++ b/examples/src/basic_agent.ts @@ -89,7 +89,7 @@ export default defineAgent({ }, }, // automatically detect keyterms and apply them to the STT per user turn - keytermsOptions: { + sttContextOptions: { keyterms: ['LiveKit'], keytermDetection: { enabled: true, diff --git a/plugins/assemblyai/src/stt.test.ts b/plugins/assemblyai/src/stt.test.ts index f4872caf5..a442156cd 100644 --- a/plugins/assemblyai/src/stt.test.ts +++ b/plugins/assemblyai/src/stt.test.ts @@ -1,11 +1,17 @@ // SPDX-FileCopyrightText: 2026 LiveKit, Inc. // // SPDX-License-Identifier: Apache-2.0 +import { + AgentHandoffItem, + ChatMessage, + createConversationItemAddedEvent, + log, +} from '@livekit/agents'; import { VAD } from '@livekit/agents-plugin-silero'; import { stt } from '@livekit/agents-plugins-test'; import { once } from 'node:events'; import type { AddressInfo } from 'node:net'; -import { describe, expect, it } from 'vitest'; +import { describe, expect, it, vi } from 'vitest'; import { WebSocketServer } from 'ws'; import { STT } from './stt.js'; @@ -61,6 +67,61 @@ describe('AssemblyAI options', () => { ).toThrow(/agentContext/); }); + it('rejects agentContext values longer than 1750 characters', () => { + expect( + () => + new STT({ + apiKey: 'test-key', + agentContext: 'x'.repeat(1751), + }), + ).toThrow(/1750/); + }); + + it('preserves agentContext when an update fails validation', async () => { + const { wss, baseUrl } = await startWebSocketServer(); + let requestUrl = ''; + wss.on('connection', (_ws, req) => { + requestUrl = req.url ?? ''; + }); + + try { + const stt = new STT({ + apiKey: 'test-key', + baseUrl, + agentContext: 'original context', + }); + + expect(() => stt.updateOptions({ agentContext: 'x'.repeat(1751) })).toThrow(/1750/); + + const stream = stt.stream(); + await waitUntil(() => requestUrl !== ''); + stream.close(); + + const url = new URL(`ws://127.0.0.1${requestUrl}`); + expect(url.searchParams.get('agent_context')).toBe('original context'); + } finally { + await closeWebSocketServer(wss); + } + }); + + it('rejects oversized agentContext updates on an active stream', async () => { + const { wss, baseUrl } = await startWebSocketServer(); + let connected = false; + wss.on('connection', () => { + connected = true; + }); + + try { + const stream = new STT({ apiKey: 'test-key', baseUrl }).stream(); + await waitUntil(() => connected); + + expect(() => stream.updateOptions({ agentContext: 'x'.repeat(1751) })).toThrow(/1750/); + stream.close(); + } finally { + await closeWebSocketServer(wss); + } + }); + it('requires a u3-rt-pro model for previousContextNTurns', () => { expect( () => @@ -72,6 +133,129 @@ describe('AssemblyAI options', () => { ).toThrow(/previousContextNTurns/); }); + it('carryover is on by default for U3 Pro family', () => { + for (const speechModel of [ + 'u3-rt-pro', + 'u3-rt-pro-beta-1', + 'universal-3-5-pro', + 'u3-pro', + ] as const) { + const stt = new STT({ apiKey: 'test-key', speechModel }); + expect(stt.capabilities.chatContext).toBe(true); + } + }); + + it('carryover default does not warn', () => { + const warn = vi.spyOn(log(), 'warn'); + + const stt = new STT({ apiKey: 'test-key', speechModel: 'universal-3-5-pro' }); + + expect(stt.capabilities.chatContext).toBe(true); + expect(warn).not.toHaveBeenCalled(); + warn.mockRestore(); + }); + + it('deprecated agentContextCarryover true still enables and warns', () => { + const warn = vi.spyOn(log(), 'warn'); + + const stt = new STT({ + apiKey: 'test-key', + speechModel: 'universal-3-5-pro', + agentContextCarryover: true, + }); + + expect(stt.capabilities.chatContext).toBe(true); + expect(warn).toHaveBeenCalledWith(expect.stringContaining('deprecated')); + warn.mockRestore(); + }); + + it('carryover defaults off for unsupported models without warning', () => { + const warn = vi.spyOn(log(), 'warn'); + + const stt = new STT({ apiKey: 'test-key', speechModel: 'universal-streaming-english' }); + + expect(stt.capabilities.chatContext).toBe(false); + expect(warn).not.toHaveBeenCalled(); + warn.mockRestore(); + }); + + it('deprecated agentContextCarryover true on unsupported model warns and is ignored', () => { + const warn = vi.spyOn(log(), 'warn'); + + const stt = new STT({ + apiKey: 'test-key', + speechModel: 'universal-streaming-english', + agentContextCarryover: true, + }); + + expect(stt.capabilities.chatContext).toBe(false); + expect(warn).toHaveBeenCalledWith(expect.stringContaining('deprecated')); + expect(warn).toHaveBeenCalledWith(expect.stringContaining('does not support it')); + warn.mockRestore(); + }); + + it('deprecated agentContextCarryover false disables and warns', () => { + const warn = vi.spyOn(log(), 'warn'); + + const stt = new STT({ apiKey: 'test-key', agentContextCarryover: false }); + + expect(stt.capabilities.chatContext).toBe(false); + expect(warn).toHaveBeenCalledWith(expect.stringContaining('deprecated')); + warn.mockRestore(); + }); + + it('deprecated agentContextCarryover true wins over previousContextNTurns zero', () => { + const warn = vi.spyOn(log(), 'warn'); + + const stt = new STT({ + apiKey: 'test-key', + previousContextNTurns: 0, + agentContextCarryover: true, + }); + + expect(stt.capabilities.chatContext).toBe(true); + warn.mockRestore(); + }); + + it('forwards short assistant replies as agent context', () => { + const stt = new STT({ apiKey: 'test-key' }); + const update = vi.spyOn(stt, 'updateOptions'); + + stt._pushConversationItem( + createConversationItemAddedEvent( + ChatMessage.create({ role: 'assistant', content: ['Your order is ready.'] }), + ), + ); + + expect(update).toHaveBeenCalledWith({ agentContext: 'Your order is ready.' }); + }); + + it('truncates automatic assistant context to the final 1750 characters', () => { + const stt = new STT({ apiKey: 'test-key' }); + const update = vi.spyOn(stt, 'updateOptions'); + const reply = `${'a'.repeat(500)}${'b'.repeat(1750)}`; + + stt._pushConversationItem( + createConversationItemAddedEvent(ChatMessage.create({ role: 'assistant', content: [reply] })), + ); + + expect(update).toHaveBeenCalledWith({ agentContext: 'b'.repeat(1750) }); + }); + + it('ignores non-assistant messages and non-message items', () => { + const stt = new STT({ apiKey: 'test-key' }); + const update = vi.spyOn(stt, 'updateOptions'); + + stt._pushConversationItem( + createConversationItemAddedEvent(ChatMessage.create({ role: 'user', content: ['hello'] })), + ); + stt._pushConversationItem( + createConversationItemAddedEvent(AgentHandoffItem.create({ newAgentId: 'agent-2' })), + ); + + expect(update).not.toHaveBeenCalled(); + }); + it('forwards inactivity timeout to the streaming query', async () => { const { wss, baseUrl } = await startWebSocketServer(); let requestUrl = ''; diff --git a/plugins/assemblyai/src/stt.ts b/plugins/assemblyai/src/stt.ts index 421116ef3..cb6fe2855 100644 --- a/plugins/assemblyai/src/stt.ts +++ b/plugins/assemblyai/src/stt.ts @@ -24,11 +24,18 @@ import type { STTEncoding, STTModels, VoiceFocus } from './models.js'; // Speech models in the Universal-3 Pro family, which share the same parameter support. const U3_PRO_MODELS = ['u3-rt-pro', 'u3-rt-pro-beta-1', 'universal-3-5-pro'] as const; +const MAX_AGENT_CONTEXT_CHARS = 1750; function isU3ProModel(model: STTModels): boolean { return U3_PRO_MODELS.includes(model as (typeof U3_PRO_MODELS)[number]); } +function validateAgentContext(agentContext: string | undefined): void { + if (agentContext !== undefined && agentContext.length > MAX_AGENT_CONTEXT_CHARS) { + throw new Error(`agentContext must be at most ${MAX_AGENT_CONTEXT_CHARS} characters`); + } +} + // AssemblyAI Universal-Streaming (v3) message envelope. All fields are optional // since we narrow on `type` before reading anything else. interface StreamEventMessage { @@ -121,10 +128,9 @@ export interface STTOptions { */ mode?: 'min_latency' | 'balanced' | 'max_accuracy'; /** - * When the model supports it, let an `AgentSession` push each assistant reply into - * `agentContext` so it is carried into the model's conversation context. Defaults to false; - * set true to enable. Prior user turns are carried automatically by the model regardless of - * this flag. Ignored on models without context support. + * @deprecated Use `new AgentSession({ sttContextOptions: { forwardChatContext: ... } })` + * instead. On the Universal-3 Pro family, assistant replies are carried into `agentContext` by + * default; pass `false` to opt out. On other models it is off. */ agentContextCarryover?: boolean; baseUrl: string; @@ -156,20 +162,27 @@ export class STT extends stt.STT { } constructor(opts: Partial = {}) { - // u3-rt-pro family — "u3-pro" is normalized below — and is opt-in via the user) + validateAgentContext(opts.agentContext); + + // u3-rt-pro family — "u3-pro" is normalized below — supports native chat context. const rawModel = opts.speechModel ?? defaultSTTOptions.speechModel; const supportsCarryover = isU3ProModel(rawModel) || rawModel === 'u3-pro'; - if (opts.agentContextCarryover && !supportsCarryover) { + if (opts.agentContextCarryover !== undefined) { log().warn( - `agentContextCarryover is enabled but model '${rawModel}' does not support it; ignoring`, + 'agentContextCarryover is deprecated, use AgentSession({ sttContextOptions: { forwardChatContext: ... } }) instead', ); + if (opts.agentContextCarryover && !supportsCarryover) { + log().warn( + `agentContextCarryover is enabled but model '${rawModel}' does not support it; ignoring`, + ); + } } super({ streaming: true, interimResults: true, alignedTranscript: 'word', keyterms: true, - chatContext: (opts.agentContextCarryover ?? false) && supportsCarryover, + chatContext: supportsCarryover && (opts.agentContextCarryover ?? true), }); if (opts.speechModel === 'u3-pro') { @@ -220,6 +233,8 @@ export class STT extends stt.STT { } updateOptions(opts: Partial) { + validateAgentContext(opts.agentContext); + // session keyterms so a user update doesn't drop them) const nextOpts = { ...opts }; if (nextOpts.keytermsPrompt !== undefined) { @@ -261,7 +276,9 @@ export class STT extends stt.STT { override _pushConversationItem(ev: ConversationItemAddedEvent): void { const chatItem = ev.item; if (chatItem instanceof ChatMessage && chatItem.role === 'assistant' && chatItem.textContent) { - this.updateOptions({ agentContext: chatItem.textContent }); + this.updateOptions({ + agentContext: chatItem.textContent.slice(-MAX_AGENT_CONTEXT_CHARS), + }); } } @@ -309,6 +326,8 @@ export class SpeechStream extends stt.SpeechStream { } updateOptions(opts: Partial) { + validateAgentContext(opts.agentContext); + this.#opts = { ...this.#opts, ...opts }; const configMsg: Record = { type: 'UpdateConfiguration' };