From 466ba88d378eac7840f53e7f8db872b9488f5d53 Mon Sep 17 00:00:00 2001 From: "rosetta-livekit-bot[bot]" <282703043+rosetta-livekit-bot[bot]@users.noreply.github.com> Date: Sat, 18 Jul 2026 00:54:56 +0000 Subject: [PATCH 1/4] Support AssemblyAI inference context carryover --- .changeset/fresh-pandas-listen.md | 5 ++ agents/src/inference/stt.test.ts | 125 +++++++++++++++++++++++++++++- agents/src/inference/stt.ts | 110 +++++++++++++++++++++----- 3 files changed, 217 insertions(+), 23 deletions(-) create mode 100644 .changeset/fresh-pandas-listen.md diff --git a/.changeset/fresh-pandas-listen.md b/.changeset/fresh-pandas-listen.md new file mode 100644 index 000000000..51e0e558c --- /dev/null +++ b/.changeset/fresh-pandas-listen.md @@ -0,0 +1,5 @@ +--- +'@livekit/agents': patch +--- + +Support AssemblyAI inference STT agent context carryover. diff --git a/agents/src/inference/stt.test.ts b/agents/src/inference/stt.test.ts index 65df44750..d915bc4ee 100644 --- a/agents/src/inference/stt.test.ts +++ b/agents/src/inference/stt.test.ts @@ -1,12 +1,14 @@ // SPDX-FileCopyrightText: 2025 LiveKit, Inc. // // SPDX-License-Identifier: Apache-2.0 -import { beforeAll, describe, expect, it } from 'vitest'; +import { afterEach, beforeAll, describe, expect, it, vi } from 'vitest'; import * as agents from '../index.js'; import { normalizeLanguage } from '../language.js'; -import { initializeLogger } from '../log.js'; +import { AgentHandoffItem, ChatMessage } from '../llm/index.js'; +import { initializeLogger, log } from '../log.js'; import { type APIConnectOptions, DEFAULT_API_CONNECT_OPTIONS } from '../types.js'; import { VAD, type VADStream } from '../vad.js'; +import { createConversationItemAddedEvent } from '../voice/events.js'; import { STT, type STTFallbackModel, @@ -21,6 +23,10 @@ beforeAll(() => { initializeLogger({ level: 'silent', pretty: false }); }); +afterEach(() => { + vi.restoreAllMocks(); +}); + /** Helper to create STT with required credentials. */ function makeStt(overrides: Record = {}) { const defaults = { @@ -32,6 +38,16 @@ function makeStt(overrides: Record = {}) { return new STT({ ...defaults, ...overrides }); } +function makeAssemblyStt(overrides: Record = {}) { + return makeStt({ model: 'assemblyai/universal-3-5-pro', ...overrides }); +} + +function assistantItemEvent(text: string) { + return createConversationItemAddedEvent( + ChatMessage.create({ role: 'assistant', content: [text] }), + ); +} + describe('parseSTTModelString', () => { it('simple model without language', () => { const [model, language] = parseSTTModelString('deepgram'); @@ -329,6 +345,111 @@ describe('STT diarization capabilities', () => { }); }); +describe('STT agent_context carryover', () => { + it('agentContextCarryover defaults to enabled on AssemblyAI U3 Pro family models', () => { + for (const model of ['assemblyai/u3-rt-pro', 'assemblyai/universal-3-5-pro'] as const) { + const stt = makeAssemblyStt({ model }); + expect(stt.capabilities.chatContext).toBe(true); + } + }); + + it('agentContextCarryover defaults off for unsupported models without warning', () => { + const warnSpy = vi.spyOn(log(), 'warn'); + const stt = makeAssemblyStt({ model: 'assemblyai/universal-streaming' }); + + expect(stt.capabilities.chatContext).toBe(false); + expect(warnSpy).not.toHaveBeenCalled(); + }); + + it('agentContextCarryover is off for non-AssemblyAI models', () => { + const stt = makeAssemblyStt({ model: 'deepgram/nova-3' }); + expect(stt.capabilities.chatContext).toBe(false); + }); + + it('explicit true on unsupported model warns and is ignored', () => { + const warnSpy = vi.spyOn(log(), 'warn'); + const stt = makeAssemblyStt({ + model: 'assemblyai/universal-streaming', + agentContextCarryover: true, + }); + + expect(stt.capabilities.chatContext).toBe(false); + expect(warnSpy).toHaveBeenCalledWith( + { model: 'assemblyai/universal-streaming' }, + 'agentContextCarryover is enabled but model does not support it; ignoring', + ); + }); + + it('explicit false disables carryover on a supported model', () => { + const stt = makeAssemblyStt({ agentContextCarryover: false }); + expect(stt.capabilities.chatContext).toBe(false); + }); + + it('previous_context_n_turns=0 disables the default carryover', () => { + const stt = makeAssemblyStt({ modelOptions: { previous_context_n_turns: 0 } }); + expect(stt.capabilities.chatContext).toBe(false); + }); + + it('explicit true wins over previous_context_n_turns=0', () => { + const stt = makeAssemblyStt({ + modelOptions: { previous_context_n_turns: 0 }, + agentContextCarryover: true, + }); + expect(stt.capabilities.chatContext).toBe(true); + }); + + it('forwards short assistant replies verbatim', () => { + const stt = makeAssemblyStt(); + stt._pushConversationItem(assistantItemEvent('Your room is booked for Tuesday.')); + expect(stt['opts'].modelOptions).toHaveProperty( + 'agent_context', + 'Your room is booked for Tuesday.', + ); + }); + + it('truncates oversize replies keeping the tail', () => { + const text = 'a'.repeat(2000) + 'b'.repeat(1750); + const stt = makeAssemblyStt(); + stt._pushConversationItem(assistantItemEvent(text)); + expect(stt['opts'].modelOptions).toHaveProperty('agent_context', 'b'.repeat(1750)); + }); + + it('ignores non-assistant items', () => { + const stt = makeAssemblyStt(); + + stt._pushConversationItem( + createConversationItemAddedEvent(ChatMessage.create({ role: 'user', content: ['hi there'] })), + ); + expect(stt['opts'].modelOptions).not.toHaveProperty('agent_context'); + + stt._pushConversationItem( + createConversationItemAddedEvent(ChatMessage.create({ role: 'assistant', content: [] })), + ); + expect(stt['opts'].modelOptions).not.toHaveProperty('agent_context'); + }); + + it('ignores agent handoff items', () => { + const stt = makeAssemblyStt(); + stt._pushConversationItem( + createConversationItemAddedEvent(AgentHandoffItem.create({ newAgentId: 'agent-2' })), + ); + expect(stt['opts'].modelOptions).not.toHaveProperty('agent_context'); + }); + + it('preserves explicit agent_context and later overwrites it with carryover', () => { + const stt = makeAssemblyStt({ + modelOptions: { agent_context: 'The agent asked for a booking date.' }, + }); + expect(stt['opts'].modelOptions).toHaveProperty( + 'agent_context', + 'The agent asked for a booking date.', + ); + + stt._pushConversationItem(assistantItemEvent('And your zip code?')); + expect(stt['opts'].modelOptions).toHaveProperty('agent_context', 'And your zip code?'); + }); +}); + describe('STT session keyterms', () => { it('updateOptions does not bake session keyterms into the user baseline', () => { const stt = makeStt({ model: 'deepgram/nova-3' }); diff --git a/agents/src/inference/stt.ts b/agents/src/inference/stt.ts index 3cfc8e852..c8e5a0e72 100644 --- a/agents/src/inference/stt.ts +++ b/agents/src/inference/stt.ts @@ -7,6 +7,7 @@ import type { WebSocket } from 'ws'; import { APIError, APIStatusError } from '../_exceptions.js'; import { AudioByteStream } from '../audio.js'; import { type LanguageCode, areLanguagesEquivalent, normalizeLanguage } from '../language.js'; +import { ChatMessage } from '../llm/index.js'; import { log } from '../log.js'; import { createStreamChannel } from '../stream/stream_channel.js'; import { @@ -19,6 +20,7 @@ import { import { type APIConnectOptions, DEFAULT_API_CONNECT_OPTIONS } from '../types.js'; import { type AudioBuffer, Event, Task, cancelAndWait, shortuuid, waitForAbort } from '../utils.js'; import { type VAD, VADEventType, type VADStream } from '../vad.js'; +import type { ConversationItemAddedEvent } from '../voice/events.js'; import { type TimedString, createTimedString } from '../voice/io.js'; import { type SttServerEvent, @@ -120,8 +122,10 @@ export interface AssemblyAIOptions { keyterms_prompt?: string[]; /** Enable speaker diarization. Default: false. */ speaker_labels?: boolean; - /** Context to bias recognition. Only supported with u3-rt-pro. Max 1500 chars. */ + /** Context to bias recognition. Only supported with u3-rt-pro. Max 1750 chars. */ agent_context?: string; + /** Prior turns carried as context; 0 disables carryover. Only supported with u3-rt-pro. */ + previous_context_n_turns?: number; /** Isolate the primary voice. Only supported with u3-rt-pro. */ voice_focus?: 'near-field' | 'far-field'; /** Background suppression strength. Only supported with u3-rt-pro. */ @@ -270,6 +274,28 @@ function keytermsExtraForModel( return { [key]: [...new Set([...existing, ...sessionKeyterms])] }; } +const ASSEMBLYAI_CARRYOVER_MODELS = [ + 'assemblyai/u3-rt-pro', + 'assemblyai/universal-3-5-pro', +] as const; + +const ASSEMBLYAI_MAX_AGENT_CONTEXT_CHARS = 1750; + +function supportsAgentContextCarryover(model: string | undefined): boolean { + return ASSEMBLYAI_CARRYOVER_MODELS.includes( + model as (typeof ASSEMBLYAI_CARRYOVER_MODELS)[number], + ); +} + +function agentContextCarryoverEnabled(opts: { + model: string | undefined; + modelOptions: Record; + agentContextCarryover?: boolean; +}): boolean { + if (!supportsAgentContextCarryover(opts.model)) return false; + return opts.agentContextCarryover ?? opts.modelOptions.previous_context_n_turns !== 0; +} + type _STTModels = | DeepgramModels | DeepgramFluxModels @@ -370,6 +396,7 @@ export interface InferenceSTTOptions { apiKey: string; apiSecret: string; modelOptions: STTOptions; + agentContextCarryover?: boolean; fallback?: STTFallbackModel[]; connOptions?: APIConnectOptions; } @@ -408,23 +435,49 @@ export class STT extends BaseSTT { apiSecret?: string; modelOptions?: STTOptions; fallback?: STTFallbackModelType | STTFallbackModelType[]; + agentContextCarryover?: boolean; connOptions?: APIConnectOptions; vad?: VAD; }) { const modelOptions = (opts?.modelOptions ?? {}) as STTOptions; + // Parse language from model string if provided: "provider/model:language". + let nextModel = opts?.model; + let nextLanguage = opts?.language; + let hasLanguageConflict = false; + if (typeof nextModel === 'string') { + const [parsedModel, parsedLanguage] = parseSTTModelString(nextModel); + if (parsedLanguage !== undefined) { + hasLanguageConflict = + !!nextLanguage && !areLanguagesEquivalent(nextLanguage, parsedLanguage); + if (!hasLanguageConflict) { + nextLanguage = parsedLanguage as STTLanguages; + } + nextModel = parsedModel as TModel; + } + } + const carryoverEnabled = agentContextCarryoverEnabled({ + model: typeof nextModel === 'string' ? nextModel : undefined, + modelOptions: modelOptions as Record, + agentContextCarryover: opts?.agentContextCarryover, + }); super({ streaming: true, interimResults: true, alignedTranscript: 'word', diarization: diarizationEnabled(modelOptions as Record), keyterms: - keytermsExtraForModel(typeof opts?.model === 'string' ? opts.model : undefined) !== - undefined, + keytermsExtraForModel(typeof nextModel === 'string' ? nextModel : undefined) !== undefined, + chatContext: carryoverEnabled, }); + if (opts?.agentContextCarryover && !supportsAgentContextCarryover(nextModel)) { + log().warn( + { model: nextModel }, + 'agentContextCarryover is enabled but model does not support it; ignoring', + ); + } + const { - model, - language, baseURL, encoding = DEFAULT_ENCODING, sampleRate = DEFAULT_SAMPLE_RATE, @@ -447,22 +500,11 @@ export class STT extends BaseSTT { throw new Error('apiSecret is required: pass apiSecret or set LIVEKIT_API_SECRET'); } - // Parse language from model string if provided: "provider/model:language" - let nextModel = model; - let nextLanguage = language; - if (typeof nextModel === 'string') { - const [parsedModel, parsedLanguage] = parseSTTModelString(nextModel); - if (parsedLanguage !== undefined) { - if (nextLanguage && !areLanguagesEquivalent(nextLanguage, parsedLanguage)) { - this.#logger.warn( - '`language` is provided via both argument and model, using the one from the argument', - { language: nextLanguage, model: nextModel }, - ); - } else { - nextLanguage = parsedLanguage as STTLanguages; - } - nextModel = parsedModel as TModel; - } + if (hasLanguageConflict) { + this.#logger.warn( + '`language` is provided via both argument and model, using the one from the argument', + { language: nextLanguage, model: opts?.model }, + ); } const normalizedFallback = fallback ? normalizeSTTFallback(fallback) : undefined; this.vad = resolveVADForModel(nextModel, vad); @@ -476,6 +518,7 @@ export class STT extends BaseSTT { apiKey: lkApiKey, apiSecret: lkApiSecret, modelOptions, + agentContextCarryover: opts?.agentContextCarryover, fallback: normalizedFallback, connOptions: connOptions ?? DEFAULT_API_CONNECT_OPTIONS, }; @@ -531,12 +574,22 @@ export class STT extends BaseSTT { this._vadPromise = undefined; this.updateCapabilities({ keyterms: keytermsExtraForModel(this.opts.model) !== undefined, + chatContext: agentContextCarryoverEnabled({ + model: this.opts.model, + modelOptions: this.opts.modelOptions as Record, + agentContextCarryover: this.opts.agentContextCarryover, + }), }); } if (nextOpts.modelOptions) { this.updateCapabilities({ diarization: diarizationEnabled(this.opts.modelOptions as Record), + chatContext: agentContextCarryoverEnabled({ + model: this.opts.model, + modelOptions: this.opts.modelOptions as Record, + agentContextCarryover: this.opts.agentContextCarryover, + }), }); // re-apply the active session keyterms on top of the update sent to live streams, // so a user extra update doesn't drop them. `this.opts.modelOptions` must stay a @@ -587,6 +640,21 @@ export class STT extends BaseSTT { } } + override _pushConversationItem(ev: ConversationItemAddedEvent): void { + const chatItem = ev.item; + if (chatItem instanceof ChatMessage && chatItem.role === 'assistant' && chatItem.textContent) { + let text = chatItem.textContent; + if (text.length > ASSEMBLYAI_MAX_AGENT_CONTEXT_CHARS) { + this.#logger.debug( + { fromChars: text.length, toChars: ASSEMBLYAI_MAX_AGENT_CONTEXT_CHARS }, + 'truncating agent_context carryover', + ); + text = text.slice(-ASSEMBLYAI_MAX_AGENT_CONTEXT_CHARS); + } + this.updateOptions({ modelOptions: { agent_context: text } as STTOptions }); + } + } + stream(options?: { language?: STTLanguages | string; connOptions?: APIConnectOptions; From ee9e759c3dfe931566c7b3ba3de57e3590693184 Mon Sep 17 00:00:00 2001 From: Toubat Date: Mon, 20 Jul 2026 13:55:25 -0700 Subject: [PATCH 2/4] Fix STT conversation context lifecycle Guard provider updates by live capabilities and propagate model capability changes so sessions and fallback adapters forward context only while supported. Co-authored-by: Cursor --- agents/src/inference/stt.test.ts | 39 +++++++++++++- agents/src/inference/stt.ts | 4 ++ agents/src/stt/fallback_adapter.test.ts | 54 +++++++++++++++++++ agents/src/stt/fallback_adapter.ts | 16 +++++- agents/src/stt/stt.ts | 16 +++++- agents/src/voice/agent_activity.ts | 22 +++++--- .../voice/agent_activity_stt_context.test.ts | 45 ++++++++++++++++ 7 files changed, 186 insertions(+), 10 deletions(-) create mode 100644 agents/src/voice/agent_activity_stt_context.test.ts diff --git a/agents/src/inference/stt.test.ts b/agents/src/inference/stt.test.ts index d915bc4ee..7a17a32d6 100644 --- a/agents/src/inference/stt.test.ts +++ b/agents/src/inference/stt.test.ts @@ -381,8 +381,15 @@ describe('STT agent_context carryover', () => { }); it('explicit false disables carryover on a supported model', () => { - const stt = makeAssemblyStt({ agentContextCarryover: false }); + const stt = makeAssemblyStt({ + agentContextCarryover: false, + modelOptions: { agent_context: 'keep me' }, + }); expect(stt.capabilities.chatContext).toBe(false); + + stt._pushConversationItem(assistantItemEvent('do not forward me')); + + expect(stt['opts'].modelOptions).toHaveProperty('agent_context', 'keep me'); }); it('previous_context_n_turns=0 disables the default carryover', () => { @@ -400,11 +407,19 @@ describe('STT agent_context carryover', () => { it('forwards short assistant replies verbatim', () => { const stt = makeAssemblyStt(); + const stream = stt.stream(); + stt._pushConversationItem(assistantItemEvent('Your room is booked for Tuesday.')); + expect(stt['opts'].modelOptions).toHaveProperty( 'agent_context', 'Your room is booked for Tuesday.', ); + expect(stream['opts'].modelOptions).toHaveProperty( + 'agent_context', + 'Your room is booked for Tuesday.', + ); + stream.close(); }); it('truncates oversize replies keeping the tail', () => { @@ -448,6 +463,28 @@ describe('STT agent_context carryover', () => { stt._pushConversationItem(assistantItemEvent('And your zip code?')); expect(stt['opts'].modelOptions).toHaveProperty('agent_context', 'And your zip code?'); }); + + it('starts forwarding after changing from an unsupported to a supported model', () => { + const stt = makeAssemblyStt({ model: 'assemblyai/universal-streaming' }); + + stt._pushConversationItem(assistantItemEvent('ignored before transition')); + expect(stt['opts'].modelOptions).not.toHaveProperty('agent_context'); + + stt.updateOptions({ model: 'assemblyai/universal-3-5-pro' }); + stt._pushConversationItem(assistantItemEvent('forwarded after transition')); + + expect(stt['opts'].modelOptions).toHaveProperty('agent_context', 'forwarded after transition'); + }); + + it('stops forwarding after changing from a supported to an unsupported model', () => { + const stt = makeAssemblyStt(); + stt._pushConversationItem(assistantItemEvent('last supported context')); + + stt.updateOptions({ model: 'assemblyai/universal-streaming' }); + stt._pushConversationItem(assistantItemEvent('ignored after transition')); + + expect(stt['opts'].modelOptions).toHaveProperty('agent_context', 'last supported context'); + }); }); describe('STT session keyterms', () => { diff --git a/agents/src/inference/stt.ts b/agents/src/inference/stt.ts index c8e5a0e72..80b7cbbdd 100644 --- a/agents/src/inference/stt.ts +++ b/agents/src/inference/stt.ts @@ -641,6 +641,10 @@ export class STT extends BaseSTT { } override _pushConversationItem(ev: ConversationItemAddedEvent): void { + if (!this.capabilities.chatContext) { + return; + } + const chatItem = ev.item; if (chatItem instanceof ChatMessage && chatItem.role === 'assistant' && chatItem.textContent) { let text = chatItem.textContent; diff --git a/agents/src/stt/fallback_adapter.test.ts b/agents/src/stt/fallback_adapter.test.ts index 5773b96b6..6f8a76914 100644 --- a/agents/src/stt/fallback_adapter.test.ts +++ b/agents/src/stt/fallback_adapter.test.ts @@ -4,7 +4,12 @@ import type { EventEmitter } from 'node:events'; import { beforeAll, describe, expect, it, vi } from 'vitest'; import { APIConnectionError, APIError } from '../_exceptions.js'; +import { ChatMessage } from '../llm/index.js'; import { initializeLogger } from '../log.js'; +import { + type ConversationItemAddedEvent, + createConversationItemAddedEvent, +} from '../voice/events.js'; import { FallbackAdapter } from './fallback_adapter.js'; import type { STT, SpeechEvent } from './stt.js'; import { FakeSTT, RecognizeSentinel, emptyAudioFrame } from './testing/fake_stt.js'; @@ -50,6 +55,55 @@ describe('FallbackAdapter', () => { expect(adapter.capabilities.streaming).toBe(true); }); + it('forwards conversation context only to children that support it', () => { + class ContextRecordingSTT extends FakeSTT { + readonly conversationItems: ConversationItemAddedEvent[] = []; + + constructor(label: string, supportsChatContext: boolean) { + super({ label }); + this.updateCapabilities({ chatContext: supportsChatContext }); + } + + override _pushConversationItem(ev: ConversationItemAddedEvent): void { + this.conversationItems.push(ev); + } + } + + const supported = new ContextRecordingSTT('supported', true); + const unsupported = new ContextRecordingSTT('unsupported', false); + const adapter = new FallbackAdapter({ sttInstances: [supported, unsupported] }); + const event = createConversationItemAddedEvent( + ChatMessage.create({ role: 'assistant', content: ['hello'] }), + ); + + adapter._pushConversationItem(event); + + expect(supported.conversationItems).toEqual([event]); + expect(unsupported.conversationItems).toEqual([]); + }); + + it('tracks dynamic child conversation-context capabilities and removes listeners on close', async () => { + class DynamicContextSTT extends FakeSTT { + setChatContext(supported: boolean): void { + this.updateCapabilities({ chatContext: supported }); + } + } + + const child = new DynamicContextSTT(); + const adapter = new FallbackAdapter({ sttInstances: [child] }); + expect(adapter.capabilities.chatContext).toBe(false); + expect(child.listenerCount('capabilities_changed')).toBe(1); + + child.setChatContext(true); + expect(adapter.capabilities.chatContext).toBe(true); + + child.setChatContext(false); + expect(adapter.capabilities.chatContext).toBe(false); + + await adapter.close(); + expect(child.listenerCount('capabilities_changed')).toBe(0); + }); + it('_recognize falls through to the next instance on error', async () => { const primary = new FakeSTT({ label: 'primary', diff --git a/agents/src/stt/fallback_adapter.ts b/agents/src/stt/fallback_adapter.ts index 002a22470..c221cb257 100644 --- a/agents/src/stt/fallback_adapter.ts +++ b/agents/src/stt/fallback_adapter.ts @@ -98,6 +98,7 @@ export class FallbackAdapter extends STT { private _status: STTStatus[] = []; private _logger = log(); private _metricsForwarders = new Map void>(); + private _capabilitiesForwarders = new Map void>(); // Last child that produced output or returned a recognize result. Surfaced // via the dynamic label/model/provider getters so OTel attributes like // `gen_ai.request.model` on `user_turn` (refreshed on every STT event by @@ -197,9 +198,10 @@ export class FallbackAdapter extends STT { } override _pushConversationItem(ev: ConversationItemAddedEvent): void { - // forward to every underlying STT; unsupported ones warn-and-skip internally for (const sttInstance of this.sttInstances) { - sttInstance._pushConversationItem(ev); + if (sttInstance.capabilities.chatContext) { + sttInstance._pushConversationItem(ev); + } } } @@ -213,8 +215,15 @@ export class FallbackAdapter extends STT { // SpeechStream.mainTask emits that on this STT instance naturally. for (const s of this.sttInstances) { const metricsForwarder = (metrics: STTMetrics) => this.emit('metrics_collected', metrics); + const capabilitiesForwarder = () => { + this.updateCapabilities({ + chatContext: this.sttInstances.some((stt) => !!stt.capabilities.chatContext), + }); + }; this._metricsForwarders.set(s, metricsForwarder); + this._capabilitiesForwarders.set(s, capabilitiesForwarder); s.on('metrics_collected', metricsForwarder); + s.on('capabilities_changed', capabilitiesForwarder); } } @@ -325,8 +334,11 @@ export class FallbackAdapter extends STT { for (const s of this.sttInstances) { const m = this._metricsForwarders.get(s); if (m) s.off('metrics_collected' as keyof STTCallbacks, m); + const c = this._capabilitiesForwarders.get(s); + if (c) s.off('capabilities_changed', c); } this._metricsForwarders.clear(); + this._capabilitiesForwarders.clear(); } } diff --git a/agents/src/stt/stt.ts b/agents/src/stt/stt.ts index 923517964..b8d7b45a6 100644 --- a/agents/src/stt/stt.ts +++ b/agents/src/stt/stt.ts @@ -154,6 +154,7 @@ export interface STTError { export type STTCallbacks = { ['metrics_collected']: (metrics: STTMetrics) => void; ['error']: (error: STTError) => void; + ['capabilities_changed']: (capabilities: STTCapabilities) => void; }; /** @@ -180,7 +181,20 @@ export abstract class STT extends (EventEmitter as new () => TypedEmitter): void { - this.#capabilities = { ...this.#capabilities, ...caps }; + const next = { ...this.#capabilities, ...caps }; + const changed = + next.streaming !== this.#capabilities.streaming || + next.interimResults !== this.#capabilities.interimResults || + next.alignedTranscript !== this.#capabilities.alignedTranscript || + next.diarization !== this.#capabilities.diarization || + next.keyterms !== this.#capabilities.keyterms || + next.chatContext !== this.#capabilities.chatContext; + if (!changed) { + return; + } + + this.#capabilities = next; + this.emit('capabilities_changed', next); } /** diff --git a/agents/src/voice/agent_activity.ts b/agents/src/voice/agent_activity.ts index 13cfd5bb3..d92ac9943 100644 --- a/agents/src/voice/agent_activity.ts +++ b/agents/src/voice/agent_activity.ts @@ -621,12 +621,8 @@ export class AgentActivity implements RecognitionHooks { // forward conversation turns to STTs that consume context natively; gated by the // STT's own capability (toggled via the STT's args). stateless and activity-scoped, // so it lives here rather than in the detector. - if (this.stt.capabilities.chatContext) { - this.agentSession.on( - AgentSessionEventTypes.ConversationItemAdded, - this.pushConversationItemToStt, - ); - } + this.stt.on('capabilities_changed', this.syncConversationItemForwarding); + this.syncConversationItemForwarding(); } // Bundled-default VAD is treated as absent when the RealtimeModel does @@ -1216,6 +1212,19 @@ export class AgentActivity implements RecognitionHooks { } }; + private syncConversationItemForwarding = (): void => { + this.agentSession.off( + AgentSessionEventTypes.ConversationItemAdded, + this.pushConversationItemToStt, + ); + if (this.stt instanceof STT && this.stt.capabilities.chatContext) { + this.agentSession.on( + AgentSessionEventTypes.ConversationItemAdded, + this.pushConversationItemToStt, + ); + } + }; + private onError(ev: RealtimeModelError | STTError | TTSError | LLMError): void { if (ev.type === 'realtime_model_error') { const errorEvent = createErrorEvent(ev, this.llm); @@ -4545,6 +4554,7 @@ export class AgentActivity implements RecognitionHooks { if (this.stt instanceof STT) { this.stt.off('metrics_collected', this.onMetricsCollected); this.stt.off('error', this.onModelError); + this.stt.off('capabilities_changed', this.syncConversationItemForwarding); this.agentSession.off( AgentSessionEventTypes.ConversationItemAdded, this.pushConversationItemToStt, diff --git a/agents/src/voice/agent_activity_stt_context.test.ts b/agents/src/voice/agent_activity_stt_context.test.ts new file mode 100644 index 000000000..8ff780d8f --- /dev/null +++ b/agents/src/voice/agent_activity_stt_context.test.ts @@ -0,0 +1,45 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { describe, expect, it, vi } from 'vitest'; +import { STT as InferenceSTT } from '../inference/stt.js'; +import { ChatMessage } from '../llm/index.js'; +import { initializeLogger } from '../log.js'; +import { Agent } from './agent.js'; +import { AgentSession } from './agent_session.js'; +import { AgentSessionEventTypes, createConversationItemAddedEvent } from './events.js'; + +describe('AgentActivity STT conversation-context lifecycle', () => { + initializeLogger({ pretty: false, level: 'silent' }); + + it('tracks model capability transitions and removes its listener on close', async () => { + const stt = new InferenceSTT({ + model: 'assemblyai/universal-streaming', + apiKey: 'test-key', + apiSecret: 'test-secret', + baseURL: 'https://example.livekit.cloud', + }); + const pushSpy = vi.spyOn(stt, '_pushConversationItem'); + const session = new AgentSession({ stt }); + const event = createConversationItemAddedEvent( + ChatMessage.create({ role: 'assistant', content: ['hello'] }), + ); + + await session.start({ agent: new Agent({ instructions: 'test' }) }); + expect(stt.listenerCount('capabilities_changed')).toBe(1); + + session.emit(AgentSessionEventTypes.ConversationItemAdded, event); + expect(pushSpy).not.toHaveBeenCalled(); + + stt.updateOptions({ model: 'assemblyai/universal-3-5-pro' }); + session.emit(AgentSessionEventTypes.ConversationItemAdded, event); + expect(pushSpy).toHaveBeenCalledTimes(1); + + stt.updateOptions({ model: 'assemblyai/universal-streaming' }); + session.emit(AgentSessionEventTypes.ConversationItemAdded, event); + expect(pushSpy).toHaveBeenCalledTimes(1); + + await session.close(); + expect(stt.listenerCount('capabilities_changed')).toBe(0); + }); +}); From 0eebde463c7ab84705ff644327b77bf566de5758 Mon Sep 17 00:00:00 2001 From: Toubat Date: Mon, 20 Jul 2026 14:01:17 -0700 Subject: [PATCH 3/4] Test STT carryover wire lifecycle Cover the emitted session.update payload and the runtime previous_context_n_turns opt-out required by the Rosetta stack contract. Co-authored-by: Cursor --- agents/src/inference/stt.test.ts | 20 ++++++++++++++----- .../voice/agent_activity_stt_context.test.ts | 5 +++-- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/agents/src/inference/stt.test.ts b/agents/src/inference/stt.test.ts index 7a17a32d6..554b3b02c 100644 --- a/agents/src/inference/stt.test.ts +++ b/agents/src/inference/stt.test.ts @@ -405,9 +405,14 @@ describe('STT agent_context carryover', () => { expect(stt.capabilities.chatContext).toBe(true); }); - it('forwards short assistant replies verbatim', () => { + it('emits active carryover shaping in the session.update wire payload', () => { const stt = makeAssemblyStt(); const stream = stt.stream(); + const sent: string[] = []; + Reflect.set(stream, 'activeWs', { + readyState: 1, + send: (payload: string) => sent.push(payload), + }); stt._pushConversationItem(assistantItemEvent('Your room is booked for Tuesday.')); @@ -415,10 +420,15 @@ describe('STT agent_context carryover', () => { 'agent_context', 'Your room is booked for Tuesday.', ); - expect(stream['opts'].modelOptions).toHaveProperty( - 'agent_context', - 'Your room is booked for Tuesday.', - ); + expect(sent).toHaveLength(1); + expect(JSON.parse(sent[0]!)).toMatchObject({ + type: 'session.update', + settings: { + extra: { + agent_context: 'Your room is booked for Tuesday.', + }, + }, + }); stream.close(); }); diff --git a/agents/src/voice/agent_activity_stt_context.test.ts b/agents/src/voice/agent_activity_stt_context.test.ts index 8ff780d8f..82c156d25 100644 --- a/agents/src/voice/agent_activity_stt_context.test.ts +++ b/agents/src/voice/agent_activity_stt_context.test.ts @@ -12,7 +12,7 @@ import { AgentSessionEventTypes, createConversationItemAddedEvent } from './even describe('AgentActivity STT conversation-context lifecycle', () => { initializeLogger({ pretty: false, level: 'silent' }); - it('tracks model capability transitions and removes its listener on close', async () => { + it('stops forwarding when previous_context_n_turns disables carryover', async () => { const stt = new InferenceSTT({ model: 'assemblyai/universal-streaming', apiKey: 'test-key', @@ -35,7 +35,8 @@ describe('AgentActivity STT conversation-context lifecycle', () => { session.emit(AgentSessionEventTypes.ConversationItemAdded, event); expect(pushSpy).toHaveBeenCalledTimes(1); - stt.updateOptions({ model: 'assemblyai/universal-streaming' }); + stt.updateOptions({ modelOptions: { previous_context_n_turns: 0 } }); + expect(stt.capabilities.chatContext).toBe(false); session.emit(AgentSessionEventTypes.ConversationItemAdded, event); expect(pushSpy).toHaveBeenCalledTimes(1); From 9fb45c15332d5e20baf8e6888ac202c541087cfa Mon Sep 17 00:00:00 2001 From: "rosetta-livekit-bot[bot]" <282703043+rosetta-livekit-bot[bot]@users.noreply.github.com> Date: Wed, 22 Jul 2026 12:23:29 -0700 Subject: [PATCH 4/4] feat(stt): add stt context options (#2058) Co-authored-by: rosetta-livekit-bot[bot] <282703043+rosetta-livekit-bot[bot]@users.noreply.github.com> Co-authored-by: Toubat --- .changeset/stt-context-options.md | 6 + agents/src/inference/stt.test.ts | 48 +---- agents/src/inference/stt.ts | 43 +--- agents/src/voice/agent_activity.ts | 13 +- .../voice/agent_activity_stt_context.test.ts | 64 +++++- agents/src/voice/agent_session.test.ts | 42 ++++ agents/src/voice/agent_session.ts | 22 ++- agents/src/voice/index.ts | 1 + agents/src/voice/keyterm_detection.test.ts | 37 ++++ agents/src/voice/keyterm_detection.ts | 61 ++++-- agents/src/voice/turn_config/utils.ts | 13 ++ examples/src/basic_agent.ts | 2 +- plugins/assemblyai/src/stt.test.ts | 186 +++++++++++++++++- plugins/assemblyai/src/stt.ts | 37 +++- 14 files changed, 455 insertions(+), 120 deletions(-) create mode 100644 .changeset/stt-context-options.md diff --git a/.changeset/stt-context-options.md b/.changeset/stt-context-options.md new file mode 100644 index 000000000..b0cfeb39c --- /dev/null +++ b/.changeset/stt-context-options.md @@ -0,0 +1,6 @@ +--- +'@livekit/agents': patch +'@livekit/agents-plugin-assemblyai': patch +--- + +Add STT context options for keyterms and chat context forwarding. diff --git a/agents/src/inference/stt.test.ts b/agents/src/inference/stt.test.ts index 554b3b02c..a41237b86 100644 --- a/agents/src/inference/stt.test.ts +++ b/agents/src/inference/stt.test.ts @@ -5,7 +5,7 @@ import { afterEach, beforeAll, describe, expect, it, vi } from 'vitest'; import * as agents from '../index.js'; import { normalizeLanguage } from '../language.js'; import { AgentHandoffItem, ChatMessage } from '../llm/index.js'; -import { initializeLogger, log } from '../log.js'; +import { initializeLogger } from '../log.js'; import { type APIConnectOptions, DEFAULT_API_CONNECT_OPTIONS } from '../types.js'; import { VAD, type VADStream } from '../vad.js'; import { createConversationItemAddedEvent } from '../voice/events.js'; @@ -345,63 +345,27 @@ describe('STT diarization capabilities', () => { }); }); -describe('STT agent_context carryover', () => { - it('agentContextCarryover defaults to enabled on AssemblyAI U3 Pro family models', () => { +describe('STT agent_context forwarding', () => { + it('enables chat context for AssemblyAI U3 Pro family models', () => { for (const model of ['assemblyai/u3-rt-pro', 'assemblyai/universal-3-5-pro'] as const) { const stt = makeAssemblyStt({ model }); expect(stt.capabilities.chatContext).toBe(true); } }); - it('agentContextCarryover defaults off for unsupported models without warning', () => { - const warnSpy = vi.spyOn(log(), 'warn'); + it('disables chat context for unsupported AssemblyAI models', () => { const stt = makeAssemblyStt({ model: 'assemblyai/universal-streaming' }); expect(stt.capabilities.chatContext).toBe(false); - expect(warnSpy).not.toHaveBeenCalled(); }); - it('agentContextCarryover is off for non-AssemblyAI models', () => { + it('disables chat context for non-AssemblyAI models', () => { const stt = makeAssemblyStt({ model: 'deepgram/nova-3' }); expect(stt.capabilities.chatContext).toBe(false); }); - it('explicit true on unsupported model warns and is ignored', () => { - const warnSpy = vi.spyOn(log(), 'warn'); - const stt = makeAssemblyStt({ - model: 'assemblyai/universal-streaming', - agentContextCarryover: true, - }); - - expect(stt.capabilities.chatContext).toBe(false); - expect(warnSpy).toHaveBeenCalledWith( - { model: 'assemblyai/universal-streaming' }, - 'agentContextCarryover is enabled but model does not support it; ignoring', - ); - }); - - it('explicit false disables carryover on a supported model', () => { - const stt = makeAssemblyStt({ - agentContextCarryover: false, - modelOptions: { agent_context: 'keep me' }, - }); - expect(stt.capabilities.chatContext).toBe(false); - - stt._pushConversationItem(assistantItemEvent('do not forward me')); - - expect(stt['opts'].modelOptions).toHaveProperty('agent_context', 'keep me'); - }); - - it('previous_context_n_turns=0 disables the default carryover', () => { + it('derives chatContext capability from model support only', () => { const stt = makeAssemblyStt({ modelOptions: { previous_context_n_turns: 0 } }); - expect(stt.capabilities.chatContext).toBe(false); - }); - - it('explicit true wins over previous_context_n_turns=0', () => { - const stt = makeAssemblyStt({ - modelOptions: { previous_context_n_turns: 0 }, - agentContextCarryover: true, - }); expect(stt.capabilities.chatContext).toBe(true); }); diff --git a/agents/src/inference/stt.ts b/agents/src/inference/stt.ts index 80b7cbbdd..2e8744375 100644 --- a/agents/src/inference/stt.ts +++ b/agents/src/inference/stt.ts @@ -281,19 +281,8 @@ const ASSEMBLYAI_CARRYOVER_MODELS = [ const ASSEMBLYAI_MAX_AGENT_CONTEXT_CHARS = 1750; -function supportsAgentContextCarryover(model: string | undefined): boolean { - return ASSEMBLYAI_CARRYOVER_MODELS.includes( - model as (typeof ASSEMBLYAI_CARRYOVER_MODELS)[number], - ); -} - -function agentContextCarryoverEnabled(opts: { - model: string | undefined; - modelOptions: Record; - agentContextCarryover?: boolean; -}): boolean { - if (!supportsAgentContextCarryover(opts.model)) return false; - return opts.agentContextCarryover ?? opts.modelOptions.previous_context_n_turns !== 0; +function supportsChatContext(model: string | undefined): boolean { + return model === ASSEMBLYAI_CARRYOVER_MODELS[0] || model === ASSEMBLYAI_CARRYOVER_MODELS[1]; } type _STTModels = @@ -396,7 +385,6 @@ export interface InferenceSTTOptions { apiKey: string; apiSecret: string; modelOptions: STTOptions; - agentContextCarryover?: boolean; fallback?: STTFallbackModel[]; connOptions?: APIConnectOptions; } @@ -435,7 +423,6 @@ export class STT extends BaseSTT { apiSecret?: string; modelOptions?: STTOptions; fallback?: STTFallbackModelType | STTFallbackModelType[]; - agentContextCarryover?: boolean; connOptions?: APIConnectOptions; vad?: VAD; }) { @@ -455,11 +442,6 @@ export class STT extends BaseSTT { nextModel = parsedModel as TModel; } } - const carryoverEnabled = agentContextCarryoverEnabled({ - model: typeof nextModel === 'string' ? nextModel : undefined, - modelOptions: modelOptions as Record, - agentContextCarryover: opts?.agentContextCarryover, - }); super({ streaming: true, interimResults: true, @@ -467,16 +449,9 @@ export class STT extends BaseSTT { diarization: diarizationEnabled(modelOptions as Record), keyterms: keytermsExtraForModel(typeof nextModel === 'string' ? nextModel : undefined) !== undefined, - chatContext: carryoverEnabled, + chatContext: supportsChatContext(typeof nextModel === 'string' ? nextModel : undefined), }); - if (opts?.agentContextCarryover && !supportsAgentContextCarryover(nextModel)) { - log().warn( - { model: nextModel }, - 'agentContextCarryover is enabled but model does not support it; ignoring', - ); - } - const { baseURL, encoding = DEFAULT_ENCODING, @@ -518,7 +493,6 @@ export class STT extends BaseSTT { apiKey: lkApiKey, apiSecret: lkApiSecret, modelOptions, - agentContextCarryover: opts?.agentContextCarryover, fallback: normalizedFallback, connOptions: connOptions ?? DEFAULT_API_CONNECT_OPTIONS, }; @@ -574,22 +548,13 @@ export class STT extends BaseSTT { this._vadPromise = undefined; this.updateCapabilities({ keyterms: keytermsExtraForModel(this.opts.model) !== undefined, - chatContext: agentContextCarryoverEnabled({ - model: this.opts.model, - modelOptions: this.opts.modelOptions as Record, - agentContextCarryover: this.opts.agentContextCarryover, - }), + chatContext: supportsChatContext(this.opts.model), }); } if (nextOpts.modelOptions) { this.updateCapabilities({ diarization: diarizationEnabled(this.opts.modelOptions as Record), - chatContext: agentContextCarryoverEnabled({ - model: this.opts.model, - modelOptions: this.opts.modelOptions as Record, - agentContextCarryover: this.opts.agentContextCarryover, - }), }); // re-apply the active session keyterms on top of the update sent to live streams, // so a user extra update doesn't drop them. `this.opts.modelOptions` must stay a diff --git a/agents/src/voice/agent_activity.ts b/agents/src/voice/agent_activity.ts index d92ac9943..85078541b 100644 --- a/agents/src/voice/agent_activity.ts +++ b/agents/src/voice/agent_activity.ts @@ -615,12 +615,11 @@ export class AgentActivity implements RecognitionHooks { if (this.stt instanceof STT) { // bind the session's keyterm detector to this activity's STT (detection uses its - // own LLM, configured via keytermsOptions, not the agent's) + // own LLM, configured via sttContextOptions, not the agent's) this.agentSession._keytermDetector.start(this.agentSession, this.stt); - // forward conversation turns to STTs that consume context natively; gated by the - // STT's own capability (toggled via the STT's args). stateless and activity-scoped, - // so it lives here rather than in the detector. + // Forward conversation turns to STTs that consume context natively. Keep the + // listener synchronized with dynamic capability changes and the session opt-out. this.stt.on('capabilities_changed', this.syncConversationItemForwarding); this.syncConversationItemForwarding(); } @@ -1217,7 +1216,11 @@ export class AgentActivity implements RecognitionHooks { AgentSessionEventTypes.ConversationItemAdded, this.pushConversationItemToStt, ); - if (this.stt instanceof STT && this.stt.capabilities.chatContext) { + if ( + this.stt instanceof STT && + this.stt.capabilities.chatContext && + this.agentSession.sessionOptions.sttContextOptions.forwardChatContext + ) { this.agentSession.on( AgentSessionEventTypes.ConversationItemAdded, this.pushConversationItemToStt, diff --git a/agents/src/voice/agent_activity_stt_context.test.ts b/agents/src/voice/agent_activity_stt_context.test.ts index 82c156d25..5b874fb61 100644 --- a/agents/src/voice/agent_activity_stt_context.test.ts +++ b/agents/src/voice/agent_activity_stt_context.test.ts @@ -12,7 +12,57 @@ import { AgentSessionEventTypes, createConversationItemAddedEvent } from './even describe('AgentActivity STT conversation-context lifecycle', () => { initializeLogger({ pretty: false, level: 'silent' }); - it('stops forwarding when previous_context_n_turns disables carryover', async () => { + it('forwards assistant context by default and removes listeners on close', async () => { + const stt = new InferenceSTT({ + model: 'assemblyai/universal-3-5-pro', + apiKey: 'test-key', + apiSecret: 'test-secret', + baseURL: 'https://example.livekit.cloud', + }); + const pushSpy = vi.spyOn(stt, '_pushConversationItem'); + const session = new AgentSession({ stt }); + const event = createConversationItemAddedEvent( + ChatMessage.create({ role: 'assistant', content: ['hello'] }), + ); + + await session.start({ agent: new Agent({ instructions: 'test' }) }); + expect(session.listenerCount(AgentSessionEventTypes.ConversationItemAdded)).toBe(1); + + session.emit(AgentSessionEventTypes.ConversationItemAdded, event); + expect(pushSpy).toHaveBeenCalledTimes(1); + + await session.close(); + expect(session.listenerCount(AgentSessionEventTypes.ConversationItemAdded)).toBe(0); + expect(stt.listenerCount('capabilities_changed')).toBe(0); + }); + + it('does not forward context when the session opts out', async () => { + const stt = new InferenceSTT({ + model: 'assemblyai/universal-3-5-pro', + apiKey: 'test-key', + apiSecret: 'test-secret', + baseURL: 'https://example.livekit.cloud', + }); + const pushSpy = vi.spyOn(stt, '_pushConversationItem'); + const session = new AgentSession({ + stt, + sttContextOptions: { forwardChatContext: false }, + }); + + await session.start({ agent: new Agent({ instructions: 'test' }) }); + session.emit( + AgentSessionEventTypes.ConversationItemAdded, + createConversationItemAddedEvent( + ChatMessage.create({ role: 'assistant', content: ['hello'] }), + ), + ); + + expect(pushSpy).not.toHaveBeenCalled(); + expect(session.listenerCount(AgentSessionEventTypes.ConversationItemAdded)).toBe(0); + await session.close(); + }); + + it('tracks model-only capability through dynamic model transitions', async () => { const stt = new InferenceSTT({ model: 'assemblyai/universal-streaming', apiKey: 'test-key', @@ -36,9 +86,19 @@ describe('AgentActivity STT conversation-context lifecycle', () => { expect(pushSpy).toHaveBeenCalledTimes(1); stt.updateOptions({ modelOptions: { previous_context_n_turns: 0 } }); + expect(stt.capabilities.chatContext).toBe(true); + session.emit(AgentSessionEventTypes.ConversationItemAdded, event); + expect(pushSpy).toHaveBeenCalledTimes(2); + + stt.updateOptions({ model: 'assemblyai/universal-streaming' }); expect(stt.capabilities.chatContext).toBe(false); session.emit(AgentSessionEventTypes.ConversationItemAdded, event); - expect(pushSpy).toHaveBeenCalledTimes(1); + expect(pushSpy).toHaveBeenCalledTimes(2); + + stt.updateOptions({ model: 'assemblyai/universal-3-5-pro' }); + expect(stt.capabilities.chatContext).toBe(true); + session.emit(AgentSessionEventTypes.ConversationItemAdded, event); + expect(pushSpy).toHaveBeenCalledTimes(3); await session.close(); expect(stt.listenerCount('capabilities_changed')).toBe(0); diff --git a/agents/src/voice/agent_session.test.ts b/agents/src/voice/agent_session.test.ts index bf4ec27ab..005fe04a0 100644 --- a/agents/src/voice/agent_session.test.ts +++ b/agents/src/voice/agent_session.test.ts @@ -2,6 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 import { describe, expect, it, vi } from 'vitest'; +import { log } from '../log.js'; import { AgentSession, resolveRecordingOptions } from './agent_session.js'; import { SpeechHandle } from './speech_handle.js'; @@ -83,3 +84,44 @@ describe('AgentSession recording state', () => { expect(session._enableRecording).toBe(false); }); }); + +describe('AgentSession STT context options', () => { + it('defaults forwardChatContext on', () => { + const session = new AgentSession({ vad: null }); + + expect(session.sessionOptions.sttContextOptions.forwardChatContext).toBe(true); + }); + + it('passes through sttContextOptions', () => { + const session = new AgentSession({ + vad: null, + sttContextOptions: { keyterms: ['LiveKit'], forwardChatContext: false }, + }); + + expect(session.sessionOptions.sttContextOptions.keyterms).toEqual(['LiveKit']); + expect(session.sessionOptions.sttContextOptions.forwardChatContext).toBe(false); + }); + + it('maps deprecated keytermsOptions to sttContextOptions and warns', () => { + const warn = vi.spyOn(log(), 'warn'); + + const session = new AgentSession({ vad: null, keytermsOptions: { keyterms: ['Acme'] } }); + + expect(session.sessionOptions.sttContextOptions.keyterms).toEqual(['Acme']); + expect(session.sessionOptions.sttContextOptions.forwardChatContext).toBe(true); + expect(warn).toHaveBeenCalledWith( + 'keytermsOptions is deprecated, use sttContextOptions instead', + ); + warn.mockRestore(); + }); + + it('sttContextOptions wins over keytermsOptions', () => { + const session = new AgentSession({ + vad: null, + sttContextOptions: { keyterms: ['new'] }, + keytermsOptions: { keyterms: ['old'] }, + }); + + expect(session.sessionOptions.sttContextOptions.keyterms).toEqual(['new']); + }); +}); diff --git a/agents/src/voice/agent_session.ts b/agents/src/voice/agent_session.ts index c48df175b..562e69266 100644 --- a/agents/src/voice/agent_session.ts +++ b/agents/src/voice/agent_session.ts @@ -95,7 +95,8 @@ import { AgentInput, AgentOutput } from './io.js'; import { KeytermDetector, type KeytermsOptions, - resolveKeytermsOptions, + type ResolvedSTTContextOptions, + type STTContextOptions, } from './keyterm_detection.js'; import { RecorderIO } from './recorder_io/index.js'; import { RoomSessionTransport, SessionHost } from './remote_session.js'; @@ -181,6 +182,7 @@ export function resolveRecordingOptions( export interface InternalSessionOptions extends AgentSessionOptions { turnHandling: InternalTurnHandlingOptions; + sttContextOptions: ResolvedSTTContextOptions; useTtsAlignedTranscript: boolean; maxToolSteps: number; userAwayTimeout: number | null; @@ -310,9 +312,16 @@ export type AgentSessionOptions = { turnHandling?: Partial; /** - * Keyterm biasing for the STT. Holds static `keyterms` plus `keytermDetection` - * (LLM extraction). Applies to STTs that accept a term list; on others it warns - * and is ignored. + * Conversation-aware context for the STT: static `keyterms` plus `keytermDetection` for STTs + * that accept a term list, and `forwardChatContext` (on by default) that forwards conversation + * turns to STTs that consume context directly. Applied where the STT supports it, ignored + * otherwise. + */ + sttContextOptions?: STTContextOptions; + + /** + * @deprecated Use `sttContextOptions` instead. Its `keyterms`/`keytermDetection` keys map onto + * the new option. */ keytermsOptions?: KeytermsOptions; @@ -573,10 +582,9 @@ export class AgentSession< this._chatCtx = ChatContext.empty(); this.sessionOptions = resolvedSessionOptions; - const keytermsOptions = resolveKeytermsOptions(this.sessionOptions.keytermsOptions); this._keytermDetector = new KeytermDetector({ - staticKeyterms: keytermsOptions.keyterms, - options: keytermsOptions.keytermDetection, + staticKeyterms: this.sessionOptions.sttContextOptions.keyterms, + options: this.sessionOptions.sttContextOptions.keytermDetection, }); this.options = legacyVoiceOptions; diff --git a/agents/src/voice/index.ts b/agents/src/voice/index.ts index 3e2ea3ccd..f3b788751 100644 --- a/agents/src/voice/index.ts +++ b/agents/src/voice/index.ts @@ -40,6 +40,7 @@ export { KeytermDetector, type KeytermDetectionOptions, type KeytermsOptions, + type STTContextOptions, } from './keyterm_detection.js'; export { AudioInput, diff --git a/agents/src/voice/keyterm_detection.test.ts b/agents/src/voice/keyterm_detection.test.ts index 8802c7232..64dd4205c 100644 --- a/agents/src/voice/keyterm_detection.test.ts +++ b/agents/src/voice/keyterm_detection.test.ts @@ -21,6 +21,8 @@ import { formatInput, parseToolCall, resolveDetection, + resolveSTTContextOptions, + sttContextFromKeytermsOptions, } from './keyterm_detection.js'; type DetectionResult = [string[], string[], string[]]; @@ -676,4 +678,39 @@ describe('module helpers', () => { expect(resolved.turnInterval).toBe(1); expect(resolved.maxKeyterms).toBeUndefined(); }); + + it('resolve STT context options defaults', () => { + const resolved = resolveSTTContextOptions(undefined); + expect(resolved.keyterms).toEqual([]); + expect(resolved.keytermDetection.enabled).toBe(false); + expect(resolved.forwardChatContext).toBe(true); + }); + + it('resolve STT context options passthrough', () => { + const resolved = resolveSTTContextOptions({ + keyterms: ['LiveKit'], + keytermDetection: { enabled: true, turnInterval: 2 }, + forwardChatContext: false, + }); + + expect(resolved.keyterms).toEqual(['LiveKit']); + expect(resolved.keytermDetection.enabled).toBe(true); + expect(resolved.keytermDetection.turnInterval).toBe(2); + expect(resolved.forwardChatContext).toBe(false); + }); + + it('STT context from keyterms options maps keys', () => { + const mapped = sttContextFromKeytermsOptions({ + keyterms: ['Acme'], + keytermDetection: { enabled: true }, + }); + expect(mapped).toEqual({ keyterms: ['Acme'], keytermDetection: { enabled: true } }); + + const resolved = resolveSTTContextOptions(mapped); + expect(resolved.keyterms).toEqual(['Acme']); + expect(resolved.keytermDetection.enabled).toBe(true); + expect(resolved.forwardChatContext).toBe(true); + + expect(sttContextFromKeytermsOptions(undefined)).toEqual({}); + }); }); diff --git a/agents/src/voice/keyterm_detection.ts b/agents/src/voice/keyterm_detection.ts index 317a3ef66..a38f7447b 100644 --- a/agents/src/voice/keyterm_detection.ts +++ b/agents/src/voice/keyterm_detection.ts @@ -17,16 +17,8 @@ import { AgentSessionEventTypes, type ConversationItemAddedEvent } from './event /** * Keyterm biasing for STTs that accept a term list. * - * Can be passed as a plain object: - * - * ```ts - * new AgentSession({ - * keytermsOptions: { - * keyterms: ['LiveKit', 'Acme Corp'], - * keytermDetection: { enabled: true, turnInterval: 1 }, - * }, - * }); - * ``` + * @deprecated Use {@link STTContextOptions} (`new AgentSession({ sttContextOptions: ... })`) + * instead; its `keyterms` and `keytermDetection` keys are identical. */ export interface KeytermsOptions { /** Static keyterms applied wherever the STT accepts a term list; never touched by detection. */ @@ -63,6 +55,33 @@ export interface KeytermDetectionOptions { timeout?: number; } +/** + * Conversation-aware context for the STT. + * + * Can be passed as a plain object: + * + * ```ts + * new AgentSession({ + * sttContextOptions: { + * keyterms: ['LiveKit', 'Acme Corp'], + * keytermDetection: { enabled: true, turnInterval: 1 }, + * forwardChatContext: true, + * }, + * }); + * ``` + */ +export interface STTContextOptions { + /** Static keyterms applied wherever the STT accepts a term list; never touched by detection. */ + keyterms?: string[]; + /** LLM-based keyterm extraction, for STTs that accept a term list. */ + keytermDetection?: KeytermDetectionOptions; + /** + * Forward conversation turns to STTs that consume context directly (e.g. AssemblyAI U3 Pro). + * Defaults to `true`; only STTs that advertise the `chatContext` capability act on it. + */ + forwardChatContext?: boolean; +} + // bound a single pass so a stuck LLM call can't hold the single-flight guard forever and // stall detection for the rest of the call; a timed-out pass simply makes no change const DETECTION_TIMEOUT = 10_000; @@ -86,10 +105,11 @@ export interface ResolvedKeytermDetectionOptions { timeout: number; } -/** A fully-defaulted keyterms config. @internal */ -export interface ResolvedKeytermsOptions { +/** A fully-defaulted STT-context config. @internal */ +export interface ResolvedSTTContextOptions { keyterms: string[]; keytermDetection: ResolvedKeytermDetectionOptions; + forwardChatContext: boolean; } /** A pending term not confirmed within this many passes is dropped. @internal */ @@ -117,15 +137,28 @@ export function resolveDetection( }; } -/** Return a fully-defaulted keyterms config. @internal */ -export function resolveKeytermsOptions(config?: KeytermsOptions): ResolvedKeytermsOptions { +/** Return a fully-defaulted STT-context config. @internal */ +export function resolveSTTContextOptions(config?: STTContextOptions): ResolvedSTTContextOptions { const cfg = config ?? {}; return { keyterms: [...(cfg.keyterms ?? [])], keytermDetection: resolveDetection(cfg.keytermDetection), + forwardChatContext: cfg.forwardChatContext ?? true, }; } +/** Map deprecated `keytermsOptions` onto the new {@link STTContextOptions} shape. @internal */ +export function sttContextFromKeytermsOptions(config?: KeytermsOptions): STTContextOptions { + const out: STTContextOptions = {}; + if (config?.keyterms !== undefined) { + out.keyterms = config.keyterms; + } + if (config?.keytermDetection !== undefined) { + out.keytermDetection = config.keytermDetection; + } + return out; +} + /** * Resolve the configured detection `llm`: an `LLM` instance is used directly; a * model string (or the default model when unset) is created via the inference gateway. diff --git a/agents/src/voice/turn_config/utils.ts b/agents/src/voice/turn_config/utils.ts index 100cda367..86d8191e1 100644 --- a/agents/src/voice/turn_config/utils.ts +++ b/agents/src/voice/turn_config/utils.ts @@ -9,6 +9,7 @@ import { type TurnDetectionMode, type VoiceOptions, } from '../agent_session.js'; +import { resolveSTTContextOptions, sttContextFromKeytermsOptions } from '../keyterm_detection.js'; import { type EndpointingOptions, defaultEndpointingOptions, @@ -50,6 +51,8 @@ export function migrateLegacyOptions(legacyOptions: AgentSessionOption tts, userData, connOptions, + sttContextOptions, + keytermsOptions, ...sessionOptions } = legacyOptions; @@ -59,6 +62,15 @@ export function migrateLegacyOptions(legacyOptions: AgentSessionOption ); } + if (keytermsOptions !== undefined) { + logger.warn('keytermsOptions is deprecated, use sttContextOptions instead'); + } + + const resolvedSTTContextOptions = + sttContextOptions !== undefined + ? resolveSTTContextOptions(sttContextOptions) + : resolveSTTContextOptions(sttContextFromKeytermsOptions(keytermsOptions)); + const turnHandling: TurnHandlingOptions = { interruption: { discardAudioIfUninterruptible: voiceOptions?.discardAudioIfUninterruptible, @@ -130,6 +142,7 @@ export function migrateLegacyOptions(legacyOptions: AgentSessionOption ...defaultSessionOptions, ...migratedVoiceOptions, ...sessionOptions, + sttContextOptions: resolvedSTTContextOptions, turnHandling: mergeWithDefaults(turnHandling), // repopulate the deprecated voice options with migrated options for backwards compatibility voiceOptions: legacyVoiceOptions, diff --git a/examples/src/basic_agent.ts b/examples/src/basic_agent.ts index fc9b371de..9d03143e9 100644 --- a/examples/src/basic_agent.ts +++ b/examples/src/basic_agent.ts @@ -89,7 +89,7 @@ export default defineAgent({ }, }, // automatically detect keyterms and apply them to the STT per user turn - keytermsOptions: { + sttContextOptions: { keyterms: ['LiveKit'], keytermDetection: { enabled: true, diff --git a/plugins/assemblyai/src/stt.test.ts b/plugins/assemblyai/src/stt.test.ts index f4872caf5..a442156cd 100644 --- a/plugins/assemblyai/src/stt.test.ts +++ b/plugins/assemblyai/src/stt.test.ts @@ -1,11 +1,17 @@ // SPDX-FileCopyrightText: 2026 LiveKit, Inc. // // SPDX-License-Identifier: Apache-2.0 +import { + AgentHandoffItem, + ChatMessage, + createConversationItemAddedEvent, + log, +} from '@livekit/agents'; import { VAD } from '@livekit/agents-plugin-silero'; import { stt } from '@livekit/agents-plugins-test'; import { once } from 'node:events'; import type { AddressInfo } from 'node:net'; -import { describe, expect, it } from 'vitest'; +import { describe, expect, it, vi } from 'vitest'; import { WebSocketServer } from 'ws'; import { STT } from './stt.js'; @@ -61,6 +67,61 @@ describe('AssemblyAI options', () => { ).toThrow(/agentContext/); }); + it('rejects agentContext values longer than 1750 characters', () => { + expect( + () => + new STT({ + apiKey: 'test-key', + agentContext: 'x'.repeat(1751), + }), + ).toThrow(/1750/); + }); + + it('preserves agentContext when an update fails validation', async () => { + const { wss, baseUrl } = await startWebSocketServer(); + let requestUrl = ''; + wss.on('connection', (_ws, req) => { + requestUrl = req.url ?? ''; + }); + + try { + const stt = new STT({ + apiKey: 'test-key', + baseUrl, + agentContext: 'original context', + }); + + expect(() => stt.updateOptions({ agentContext: 'x'.repeat(1751) })).toThrow(/1750/); + + const stream = stt.stream(); + await waitUntil(() => requestUrl !== ''); + stream.close(); + + const url = new URL(`ws://127.0.0.1${requestUrl}`); + expect(url.searchParams.get('agent_context')).toBe('original context'); + } finally { + await closeWebSocketServer(wss); + } + }); + + it('rejects oversized agentContext updates on an active stream', async () => { + const { wss, baseUrl } = await startWebSocketServer(); + let connected = false; + wss.on('connection', () => { + connected = true; + }); + + try { + const stream = new STT({ apiKey: 'test-key', baseUrl }).stream(); + await waitUntil(() => connected); + + expect(() => stream.updateOptions({ agentContext: 'x'.repeat(1751) })).toThrow(/1750/); + stream.close(); + } finally { + await closeWebSocketServer(wss); + } + }); + it('requires a u3-rt-pro model for previousContextNTurns', () => { expect( () => @@ -72,6 +133,129 @@ describe('AssemblyAI options', () => { ).toThrow(/previousContextNTurns/); }); + it('carryover is on by default for U3 Pro family', () => { + for (const speechModel of [ + 'u3-rt-pro', + 'u3-rt-pro-beta-1', + 'universal-3-5-pro', + 'u3-pro', + ] as const) { + const stt = new STT({ apiKey: 'test-key', speechModel }); + expect(stt.capabilities.chatContext).toBe(true); + } + }); + + it('carryover default does not warn', () => { + const warn = vi.spyOn(log(), 'warn'); + + const stt = new STT({ apiKey: 'test-key', speechModel: 'universal-3-5-pro' }); + + expect(stt.capabilities.chatContext).toBe(true); + expect(warn).not.toHaveBeenCalled(); + warn.mockRestore(); + }); + + it('deprecated agentContextCarryover true still enables and warns', () => { + const warn = vi.spyOn(log(), 'warn'); + + const stt = new STT({ + apiKey: 'test-key', + speechModel: 'universal-3-5-pro', + agentContextCarryover: true, + }); + + expect(stt.capabilities.chatContext).toBe(true); + expect(warn).toHaveBeenCalledWith(expect.stringContaining('deprecated')); + warn.mockRestore(); + }); + + it('carryover defaults off for unsupported models without warning', () => { + const warn = vi.spyOn(log(), 'warn'); + + const stt = new STT({ apiKey: 'test-key', speechModel: 'universal-streaming-english' }); + + expect(stt.capabilities.chatContext).toBe(false); + expect(warn).not.toHaveBeenCalled(); + warn.mockRestore(); + }); + + it('deprecated agentContextCarryover true on unsupported model warns and is ignored', () => { + const warn = vi.spyOn(log(), 'warn'); + + const stt = new STT({ + apiKey: 'test-key', + speechModel: 'universal-streaming-english', + agentContextCarryover: true, + }); + + expect(stt.capabilities.chatContext).toBe(false); + expect(warn).toHaveBeenCalledWith(expect.stringContaining('deprecated')); + expect(warn).toHaveBeenCalledWith(expect.stringContaining('does not support it')); + warn.mockRestore(); + }); + + it('deprecated agentContextCarryover false disables and warns', () => { + const warn = vi.spyOn(log(), 'warn'); + + const stt = new STT({ apiKey: 'test-key', agentContextCarryover: false }); + + expect(stt.capabilities.chatContext).toBe(false); + expect(warn).toHaveBeenCalledWith(expect.stringContaining('deprecated')); + warn.mockRestore(); + }); + + it('deprecated agentContextCarryover true wins over previousContextNTurns zero', () => { + const warn = vi.spyOn(log(), 'warn'); + + const stt = new STT({ + apiKey: 'test-key', + previousContextNTurns: 0, + agentContextCarryover: true, + }); + + expect(stt.capabilities.chatContext).toBe(true); + warn.mockRestore(); + }); + + it('forwards short assistant replies as agent context', () => { + const stt = new STT({ apiKey: 'test-key' }); + const update = vi.spyOn(stt, 'updateOptions'); + + stt._pushConversationItem( + createConversationItemAddedEvent( + ChatMessage.create({ role: 'assistant', content: ['Your order is ready.'] }), + ), + ); + + expect(update).toHaveBeenCalledWith({ agentContext: 'Your order is ready.' }); + }); + + it('truncates automatic assistant context to the final 1750 characters', () => { + const stt = new STT({ apiKey: 'test-key' }); + const update = vi.spyOn(stt, 'updateOptions'); + const reply = `${'a'.repeat(500)}${'b'.repeat(1750)}`; + + stt._pushConversationItem( + createConversationItemAddedEvent(ChatMessage.create({ role: 'assistant', content: [reply] })), + ); + + expect(update).toHaveBeenCalledWith({ agentContext: 'b'.repeat(1750) }); + }); + + it('ignores non-assistant messages and non-message items', () => { + const stt = new STT({ apiKey: 'test-key' }); + const update = vi.spyOn(stt, 'updateOptions'); + + stt._pushConversationItem( + createConversationItemAddedEvent(ChatMessage.create({ role: 'user', content: ['hello'] })), + ); + stt._pushConversationItem( + createConversationItemAddedEvent(AgentHandoffItem.create({ newAgentId: 'agent-2' })), + ); + + expect(update).not.toHaveBeenCalled(); + }); + it('forwards inactivity timeout to the streaming query', async () => { const { wss, baseUrl } = await startWebSocketServer(); let requestUrl = ''; diff --git a/plugins/assemblyai/src/stt.ts b/plugins/assemblyai/src/stt.ts index 421116ef3..cb6fe2855 100644 --- a/plugins/assemblyai/src/stt.ts +++ b/plugins/assemblyai/src/stt.ts @@ -24,11 +24,18 @@ import type { STTEncoding, STTModels, VoiceFocus } from './models.js'; // Speech models in the Universal-3 Pro family, which share the same parameter support. const U3_PRO_MODELS = ['u3-rt-pro', 'u3-rt-pro-beta-1', 'universal-3-5-pro'] as const; +const MAX_AGENT_CONTEXT_CHARS = 1750; function isU3ProModel(model: STTModels): boolean { return U3_PRO_MODELS.includes(model as (typeof U3_PRO_MODELS)[number]); } +function validateAgentContext(agentContext: string | undefined): void { + if (agentContext !== undefined && agentContext.length > MAX_AGENT_CONTEXT_CHARS) { + throw new Error(`agentContext must be at most ${MAX_AGENT_CONTEXT_CHARS} characters`); + } +} + // AssemblyAI Universal-Streaming (v3) message envelope. All fields are optional // since we narrow on `type` before reading anything else. interface StreamEventMessage { @@ -121,10 +128,9 @@ export interface STTOptions { */ mode?: 'min_latency' | 'balanced' | 'max_accuracy'; /** - * When the model supports it, let an `AgentSession` push each assistant reply into - * `agentContext` so it is carried into the model's conversation context. Defaults to false; - * set true to enable. Prior user turns are carried automatically by the model regardless of - * this flag. Ignored on models without context support. + * @deprecated Use `new AgentSession({ sttContextOptions: { forwardChatContext: ... } })` + * instead. On the Universal-3 Pro family, assistant replies are carried into `agentContext` by + * default; pass `false` to opt out. On other models it is off. */ agentContextCarryover?: boolean; baseUrl: string; @@ -156,20 +162,27 @@ export class STT extends stt.STT { } constructor(opts: Partial = {}) { - // u3-rt-pro family — "u3-pro" is normalized below — and is opt-in via the user) + validateAgentContext(opts.agentContext); + + // u3-rt-pro family — "u3-pro" is normalized below — supports native chat context. const rawModel = opts.speechModel ?? defaultSTTOptions.speechModel; const supportsCarryover = isU3ProModel(rawModel) || rawModel === 'u3-pro'; - if (opts.agentContextCarryover && !supportsCarryover) { + if (opts.agentContextCarryover !== undefined) { log().warn( - `agentContextCarryover is enabled but model '${rawModel}' does not support it; ignoring`, + 'agentContextCarryover is deprecated, use AgentSession({ sttContextOptions: { forwardChatContext: ... } }) instead', ); + if (opts.agentContextCarryover && !supportsCarryover) { + log().warn( + `agentContextCarryover is enabled but model '${rawModel}' does not support it; ignoring`, + ); + } } super({ streaming: true, interimResults: true, alignedTranscript: 'word', keyterms: true, - chatContext: (opts.agentContextCarryover ?? false) && supportsCarryover, + chatContext: supportsCarryover && (opts.agentContextCarryover ?? true), }); if (opts.speechModel === 'u3-pro') { @@ -220,6 +233,8 @@ export class STT extends stt.STT { } updateOptions(opts: Partial) { + validateAgentContext(opts.agentContext); + // session keyterms so a user update doesn't drop them) const nextOpts = { ...opts }; if (nextOpts.keytermsPrompt !== undefined) { @@ -261,7 +276,9 @@ export class STT extends stt.STT { override _pushConversationItem(ev: ConversationItemAddedEvent): void { const chatItem = ev.item; if (chatItem instanceof ChatMessage && chatItem.role === 'assistant' && chatItem.textContent) { - this.updateOptions({ agentContext: chatItem.textContent }); + this.updateOptions({ + agentContext: chatItem.textContent.slice(-MAX_AGENT_CONTEXT_CHARS), + }); } } @@ -309,6 +326,8 @@ export class SpeechStream extends stt.SpeechStream { } updateOptions(opts: Partial) { + validateAgentContext(opts.agentContext); + this.#opts = { ...this.#opts, ...opts }; const configMsg: Record = { type: 'UpdateConfiguration' };