Support adaptive interruption for realtime models

.changeset/fuzzy-birds-barge.md

@@ -0,0 +1,5 @@
+---
+"@livekit/agents": minor
+---
+
+Support adaptive interruption gating for realtime models without server-side turn detection.

agents/src/inference/interruption/interruption_failover.test.ts

@@ -341,6 +341,7 @@ describe('interruption updateOptions reconnect', () => {
 function createHooks(): RecognitionHooks {
   return {
     onInterruption: vi.fn(),
+    onBackchannelConfirmed: vi.fn(),
     onStartOfSpeech: vi.fn(),
     onVADInferenceDone: vi.fn(),
     onEndOfSpeech: vi.fn(),

agents/src/inference/interruption/interruption_stream.ts

@@ -54,8 +54,8 @@ export class InterruptionStreamSentinel {
     return { type: 'overlap-speech-started', speechDuration, startedAt, userSpeakingSpan };
   }
 
-  static overlapSpeechEnded(endedAt: number): OverlapSpeechEnded {
-    return { type: 'overlap-speech-ended', endedAt };
+  static overlapSpeechEnded(endedAt: number, agentEnded = false): OverlapSpeechEnded {
+    return { type: 'overlap-speech-ended', endedAt, agentEnded };
   }
 
   static flush(): Flush {
@@ -287,6 +287,7 @@ export class InterruptionStreamBase {
                 type: 'overlapping_speech',
                 detectedAt: chunk.endedAt,
                 isInterruption: false,
+                agentEnded: chunk.agentEnded,
                 overlapStartedAt: this.overlapSpeechStartedAt,
                 speechInput: e.speechInput,
                 probabilities: e.probabilities,

agents/src/inference/interruption/types.ts

@@ -7,6 +7,12 @@ export interface OverlappingSpeechEvent {
   type: 'overlapping_speech';
   detectedAt: number;
   isInterruption: boolean;
+  /**
+   * True when the overlap ended because the agent finished speaking rather than the user.
+   * The user may still be talking, so `isInterruption` (always false here) is inconclusive
+   * and must not be treated as a confirmed backchannel verdict.
+   */
+  agentEnded?: boolean;
   totalDurationInS: number;
   predictionDurationInS: number;
   detectionDelayInS: number;
@@ -66,6 +72,8 @@ export interface OverlapSpeechEnded {
   type: 'overlap-speech-ended';
   /** Absolute timestamp (ms) when overlap speech ended, used as the non-interruption event timestamp. */
   endedAt: number;
+  /** Whether the overlap ended because agent speech ended, not because user speech ended. */
+  agentEnded?: boolean;
 }
 
 export interface Flush {

agents/src/inference/interruption/ws_transport.ts

@@ -309,6 +309,7 @@ export function createWsTransport(
             type: 'overlapping_speech',
             detectedAt: Date.now(),
             isInterruption: true,
+            agentEnded: false,
             totalDurationInS: entry.totalDurationInS,
             predictionDurationInS: entry.predictionDurationInS,
             overlapStartedAt: overlapSpeechStartedAt,

agents/src/voice/agent_activity.ts

@@ -296,6 +296,7 @@ export class AgentActivity implements RecognitionHooks {
   private readonly closeAbort = new AbortController();
   private interruptionDetector?: AdaptiveInterruptionDetector;
   private isInterruptionDetectionEnabled: boolean;
+  private interruptionDetected = false;
   private isInterruptionByAudioActivityEnabled: boolean;
   private isDefaultInterruptionByAudioActivityEnabled: boolean;
 
@@ -331,6 +332,7 @@ export class AgentActivity implements RecognitionHooks {
     this.onError(ev);
 
   private readonly onInterruptionOverlappingSpeech = (ev: OverlappingSpeechEvent): void => {
+    this.interruptionDetected = ev.isInterruption;
     this.agentSession.emit(AgentSessionEventTypes.OverlappingSpeech, ev);
   };
 
@@ -1381,6 +1383,7 @@ export class AgentActivity implements RecognitionHooks {
         this.agentSession._userSpeakingSpan,
       );
     }
+    this.interruptionDetected = false;
 
     if (this.falseInterruptionTimer) {
       // cancel the timer when user starts speaking but leave the paused state unchanged
@@ -1445,6 +1448,17 @@ export class AgentActivity implements RecognitionHooks {
     }
   }
 
+  onBackchannelConfirmed(): void {
+    if (
+      this.isInterruptionDetectionEnabled &&
+      this.realtimeSession !== undefined &&
+      this.turnDetection !== 'manual' &&
+      this.turnDetection !== 'realtime_llm'
+    ) {
+      this.realtimeSession.clearAudio();
+    }
+  }
+
   private interruptByAudioActivity(options?: { ignoreUserTranscriptUntil?: number }): void {
     if (!this.isInterruptionByAudioActivityEnabled) {
       return;
@@ -1880,6 +1894,22 @@ export class AgentActivity implements RecognitionHooks {
       }
     }
 
+    if (
+      !this.stt &&
+      this.turnDetection !== 'manual' &&
+      this.llm instanceof RealtimeModel &&
+      !this.llm.capabilities.turnDetection &&
+      this.isInterruptionDetectionEnabled &&
+      (info.backchannelOverAgent ||
+        (!this.interruptionDetected &&
+          this._currentSpeech !== undefined &&
+          !this._currentSpeech.interrupted))
+    ) {
+      this.cancelPreemptiveGeneration();
+      this.realtimeSession?.clearAudio();
+      return false;
+    }
+
     const oldTask = this._userTurnCompletedTask;
     this._userTurnCompletedTask = this.createSpeechTask({
       taskFn: () => this.userTurnCompleted(info, oldTask),
@@ -4371,16 +4401,25 @@ export class AgentActivity implements RecognitionHooks {
   private resolveInterruptionDetector(): AdaptiveInterruptionDetector | undefined {
     const agentInterruptionDetection = this.agent.turnHandling?.interruption?.mode;
     const sessionInterruptionDetection = this.agentSession.interruptionDetection;
-    if (
-      !(
+
+    let canGatekeep: boolean;
+    if (this.llm instanceof RealtimeModel) {
+      // Realtime commits turns manually; barge-in withholds the commit, so no STT is needed.
+      canGatekeep = !this.llm.capabilities.turnDetection;
+    } else {
+      // The STT pipeline gatekeeps by holding and flushing transcripts.
+      canGatekeep = !!(
         this.stt &&
         this.stt.capabilities.alignedTranscript &&
-        this.stt.capabilities.streaming &&
-        this.vad !== undefined &&
-        this.turnDetection !== 'manual' &&
-        this.turnDetection !== 'realtime_llm' &&
-        !(this.llm instanceof RealtimeModel)
-      )
+        this.stt.capabilities.streaming
+      );
+    }
+
+    if (
+      !canGatekeep ||
+      this.vad === undefined ||
+      this.turnDetection === 'manual' ||
+      this.turnDetection === 'realtime_llm'
     ) {
       if (
         agentInterruptionDetection === 'adaptive' ||

agents/src/voice/audio_recognition.ts

@@ -88,6 +88,8 @@ export interface EndOfTurnInfo {
    * caller drives its own `generateReply` (e.g. leaving a voicemail).
    */
   skipReply?: boolean;
+  /** The turn's speech overlapped agent speech and was classified a backchannel. */
+  backchannelOverAgent?: boolean;
 }
 
 type EndOfTurnMetrics = {
@@ -139,6 +141,7 @@ export interface PreemptiveGenerationInfo {
 
 export interface RecognitionHooks {
   onInterruption: (ev: OverlappingSpeechEvent) => void;
+  onBackchannelConfirmed: () => void;
   onStartOfSpeech: (ev: VADEvent) => void;
   onVADInferenceDone: (ev: VADEvent) => void;
   onEndOfSpeech: (ev: VADEvent) => void;
@@ -373,6 +376,8 @@ export class AudioRecognition {
   private isAgentSpeaking: boolean;
   private agentSpeechStartedAt?: number;
   private interruptionDetected?: boolean;
+  private overlapInCurrentTurn = false;
+  private turnBackchannelOverAgent = false;
   private interruptionStreamChannel?: StreamChannel<InterruptionSentinel | AudioFrame>;
   private closed = false;
 
@@ -772,7 +777,7 @@ export class AudioRecognition {
       // so it does not emit a synthetic `isInterruption: false` event following a real
       // interruption.
       if (priorIgnoreUserTranscriptUntil === undefined) {
-        this.onEndOfOverlapSpeech(Date.now());
+        this.onEndOfOverlapSpeech(Date.now(), undefined, true);
       }
       await this.flushHeldTranscripts(endCooldown);
     }
@@ -784,6 +789,8 @@ export class AudioRecognition {
       if (!this.endpointing.overlapping) {
         this.endpointing.onStartOfSpeech(startedAt, true);
       }
+      this.turnBackchannelOverAgent = false;
+      this.overlapInCurrentTurn = true;
       this.trySendInterruptionSentinel(
         InterruptionStreamSentinel.overlapSpeechStarted(
           speechDuration,
@@ -795,15 +802,17 @@ export class AudioRecognition {
   }
 
   /** End interruption inference when overlap speech ends. */
-  async onEndOfOverlapSpeech(endedAt: number, userSpeakingSpan?: Span) {
+  async onEndOfOverlapSpeech(endedAt: number, userSpeakingSpan?: Span, agentEnded = false) {
     if (!this.isInterruptionEnabled) {
       return;
     }
     if (userSpeakingSpan && userSpeakingSpan.isRecording()) {
       userSpeakingSpan.setAttribute(traceTypes.ATTR_IS_INTERRUPTION, 'false');
     }
 
-    return this.trySendInterruptionSentinel(InterruptionStreamSentinel.overlapSpeechEnded(endedAt));
+    return this.trySendInterruptionSentinel(
+      InterruptionStreamSentinel.overlapSpeechEnded(endedAt, agentEnded),
+    );
   }
 
   /**
@@ -1222,6 +1231,8 @@ export class AudioRecognition {
           const ctx = this.userTurnContext(span);
           this.endpointing.onStartOfSpeech(speechStartTime, this.isAgentSpeaking);
           this.interruptionDetected = undefined;
+          this.turnBackchannelOverAgent = false;
+          this.overlapInCurrentTurn = this.isAgentSpeaking;
           otelContext.with(ctx, () => {
             this.hooks.onStartOfSpeech({
               type: VADEventType.START_OF_SPEECH,
@@ -1317,6 +1328,13 @@ export class AudioRecognition {
 
     this.interruptionDetected = ev.isInterruption;
 
+    if (this.overlapInCurrentTurn && !ev.agentEnded) {
+      this.turnBackchannelOverAgent = !ev.isInterruption;
+      if (!ev.isInterruption && !this.speaking) {
+        this.hooks.onBackchannelConfirmed();
+      }
+    }
+
     if (ev.isInterruption) {
       this.hooks.onInterruption(ev);
     }
@@ -1639,6 +1657,7 @@ export class AudioRecognition {
           endOfUtteranceDelay: metrics.endOfUtteranceDelay,
           startedSpeakingAt: metrics.startedSpeakingAt,
           stoppedSpeakingAt: metrics.stoppedSpeakingAt,
+          backchannelOverAgent: this.turnBackchannelOverAgent,
         });
 
         if (committed) {
@@ -1670,6 +1689,8 @@ export class AudioRecognition {
           }
         }
 
+        this.turnBackchannelOverAgent = false;
+        this.overlapInCurrentTurn = false;
         this.userTurnCommitted = false;
       };
 
@@ -1833,6 +1854,8 @@ export class AudioRecognition {
               const ctx = this.userTurnContext(span);
               this.endpointing.onStartOfSpeech(startTime, this.isAgentSpeaking);
               this.interruptionDetected = undefined;
+              this.turnBackchannelOverAgent = false;
+              this.overlapInCurrentTurn = this.isAgentSpeaking;
               otelContext.with(ctx, () => this.hooks.onStartOfSpeech(ev));
             }
             this.speaking = true;

agents/src/voice/audio_recognition_backchannel.test.ts

@@ -14,6 +14,7 @@ import {
 function createHooks(): RecognitionHooks {
   return {
     onInterruption: vi.fn(),
+    onBackchannelConfirmed: vi.fn(),
     onStartOfSpeech: vi.fn(),
     onVADInferenceDone: vi.fn(),
     onEndOfSpeech: vi.fn(),
@@ -48,6 +49,7 @@ function overlapSpeechEvent(isInterruption: boolean): OverlappingSpeechEvent {
     type: 'overlapping_speech',
     detectedAt: Date.now(),
     isInterruption,
+    agentEnded: false,
     totalDurationInS: 0,
     predictionDurationInS: 0,
     detectionDelayInS: 0,

agents/src/voice/audio_recognition_duplicate_commit.test.ts

@@ -67,6 +67,7 @@ describe('AudioRecognition duplicate EOU commit', () => {
 
     const hooks: RecognitionHooks = {
       onInterruption: vi.fn(),
+      onBackchannelConfirmed: vi.fn(),
       onStartOfSpeech: vi.fn(),
       onVADInferenceDone: vi.fn(),
       onEndOfSpeech: vi.fn(),

agents/src/voice/audio_recognition_endpointing.test.ts

@@ -10,6 +10,7 @@ import { BaseEndpointing } from './turn_config/endpointing.js';
 function createHooks(): RecognitionHooks {
   return {
     onInterruption: () => {},
+    onBackchannelConfirmed: () => {},
     onStartOfSpeech: () => {},
     onVADInferenceDone: () => {},
     onEndOfSpeech: () => {},

agents/src/voice/audio_recognition_eou.test.ts

@@ -51,6 +51,7 @@ class SilentVAD extends VAD {
 function createHooks(): RecognitionHooks {
   return {
     onInterruption: vi.fn(),
+    onBackchannelConfirmed: vi.fn(),
     onStartOfSpeech: vi.fn(),
     onVADInferenceDone: vi.fn(),
     onEndOfSpeech: vi.fn(),

agents/src/voice/audio_recognition_handoff.test.ts

@@ -12,6 +12,7 @@ import type { STTNode } from './io.js';
 function createHooks() {
   const hooks: RecognitionHooks = {
     onInterruption: vi.fn(),
+    onBackchannelConfirmed: vi.fn(),
     onStartOfSpeech: vi.fn(),
     onVADInferenceDone: vi.fn(),
     onEndOfSpeech: vi.fn(),

agents/src/voice/audio_recognition_interruption.test.ts

@@ -10,6 +10,7 @@ import { AudioRecognition, type RecognitionHooks } from './audio_recognition.js'
 function createHooks(): RecognitionHooks {
   return {
     onInterruption: vi.fn(),
+    onBackchannelConfirmed: vi.fn(),
     onStartOfSpeech: vi.fn(),
     onVADInferenceDone: vi.fn(),
     onEndOfSpeech: vi.fn(),

agents/src/voice/audio_recognition_push_audio.test.ts

@@ -13,6 +13,7 @@ import { createSilenceFrame, createSilenceFrameLike } from './utils.js';
 function createHooks(): RecognitionHooks {
   return {
     onInterruption: vi.fn(),
+    onBackchannelConfirmed: vi.fn(),
     onStartOfSpeech: vi.fn(),
     onVADInferenceDone: vi.fn(),
     onEndOfSpeech: vi.fn(),

agents/src/voice/audio_recognition_span.test.ts

@@ -104,6 +104,7 @@ describe('AudioRecognition user_turn span', () => {
 
     const hooks: RecognitionHooks = {
       onInterruption: vi.fn(),
+      onBackchannelConfirmed: vi.fn(),
       onStartOfSpeech: vi.fn(),
       onVADInferenceDone: vi.fn(),
       onEndOfSpeech: vi.fn(),
@@ -199,6 +200,7 @@ describe('AudioRecognition user_turn span', () => {
 
     const hooks: RecognitionHooks = {
       onInterruption: vi.fn(),
+      onBackchannelConfirmed: vi.fn(),
       onStartOfSpeech: vi.fn(),
       onVADInferenceDone: vi.fn(),
       onEndOfSpeech: vi.fn(),
@@ -272,6 +274,7 @@ describe('AudioRecognition user_turn span', () => {
 
     const hooks: RecognitionHooks = {
       onInterruption: vi.fn(),
+      onBackchannelConfirmed: vi.fn(),
       onStartOfSpeech: vi.fn(),
       onVADInferenceDone: vi.fn(),
       onEndOfSpeech: vi.fn(),
@@ -373,6 +376,7 @@ describe('AudioRecognition user_turn span', () => {
 
     const hooks: RecognitionHooks = {
       onInterruption: vi.fn(),
+      onBackchannelConfirmed: vi.fn(),
       onStartOfSpeech: vi.fn(),
       onVADInferenceDone: vi.fn(),
       onEndOfSpeech: vi.fn(),

agents/src/voice/audio_recognition_turn_detection.test.ts

@@ -75,6 +75,7 @@ interface RecognitionInternals {
 function makeHooks(): RecognitionHooks {
   return {
     onInterruption: vi.fn(),
+    onBackchannelConfirmed: vi.fn(),
     onStartOfSpeech: vi.fn(),
     onVADInferenceDone: vi.fn(),
     onEndOfSpeech: vi.fn(),

agents/src/voice/audio_recognition_vad_reset.test.ts

@@ -27,6 +27,7 @@ interface RecognitionInternals {
 function makeHooks(): RecognitionHooks {
   return {
     onInterruption: vi.fn(),
+    onBackchannelConfirmed: vi.fn(),
     onStartOfSpeech: vi.fn(),
     onVADInferenceDone: vi.fn(),
     onEndOfSpeech: vi.fn(),