diff --git a/.changeset/merge-frames-linear.md b/.changeset/merge-frames-linear.md new file mode 100644 index 000000000..07490081b --- /dev/null +++ b/.changeset/merge-frames-linear.md @@ -0,0 +1,5 @@ +--- +'@livekit/agents': patch +--- + +Make `mergeFrames` linear instead of quadratic. It previously rebuilt the accumulated buffer with a boxed spread on every frame (`new Int16Array([...data, ...frame.data])`), costing O(total_samples x frame_count) copies. It now sums lengths in one pass and copies each frame once into a preallocated `Int16Array`. Merging a 30s utterance of 10ms frames drops from ~60s of blocked event loop to under a millisecond; callers include the Silero VAD inference loop, STT plugin `recognize` paths, and TTS `ChunkedStream.collect()`. diff --git a/agents/src/utils.test.ts b/agents/src/utils.test.ts index 0e4657baf..b5816fb3a 100644 --- a/agents/src/utils.test.ts +++ b/agents/src/utils.test.ts @@ -13,6 +13,7 @@ import { dedent, delay, isPending, + mergeFrames, resampleStream, toStream, } from '../src/utils.js'; @@ -933,4 +934,43 @@ world expect(outputFrames).toEqual([]); }); }); + + describe('mergeFrames', () => { + const frame = (samples: number[], sampleRate = 16000, channels = 1): AudioFrame => + new AudioFrame(new Int16Array(samples), sampleRate, channels, samples.length / channels); + + it('returns a non-array frame as-is', () => { + const f = frame([1, 2, 3]); + expect(mergeFrames(f)).toBe(f); + }); + + it('concatenates frame data in order', () => { + const merged = mergeFrames([frame([1, 2]), frame([3]), frame([4, 5, 6])]); + expect(Array.from(merged.data)).toEqual([1, 2, 3, 4, 5, 6]); + expect(merged.samplesPerChannel).toBe(6); + expect(merged.sampleRate).toBe(16000); + expect(merged.channels).toBe(1); + }); + + it('handles multi-channel frames', () => { + const merged = mergeFrames([frame([1, 2, 3, 4], 48000, 2), frame([5, 6], 48000, 2)]); + expect(Array.from(merged.data)).toEqual([1, 2, 3, 4, 5, 6]); + expect(merged.samplesPerChannel).toBe(3); + expect(merged.channels).toBe(2); + }); + + it('throws on empty buffer', () => { + expect(() => mergeFrames([])).toThrow(TypeError); + }); + + it('throws on sample rate mismatch', () => { + expect(() => mergeFrames([frame([1]), frame([2], 48000)])).toThrow('sample rate mismatch'); + }); + + it('throws on channel count mismatch', () => { + expect(() => mergeFrames([frame([1, 2], 16000, 1), frame([3, 4], 16000, 2)])).toThrow( + 'channel count mismatch', + ); + }); + }); }); diff --git a/agents/src/utils.ts b/agents/src/utils.ts index 85e351e9c..6430b9563 100644 --- a/agents/src/utils.ts +++ b/agents/src/utils.ts @@ -70,7 +70,7 @@ export const mergeFrames = (buffer: AudioBuffer): AudioFrame => { const sampleRate = buffer[0]!.sampleRate; const channels = buffer[0]!.channels; let samplesPerChannel = 0; - let data = new Int16Array(); + let dataLength = 0; for (const frame of buffer) { if (frame.sampleRate !== sampleRate) { @@ -81,10 +81,17 @@ export const mergeFrames = (buffer: AudioBuffer): AudioFrame => { throw new TypeError('channel count mismatch'); } - data = new Int16Array([...data, ...frame.data]); + dataLength += frame.data.length; samplesPerChannel += frame.samplesPerChannel; } + const data = new Int16Array(dataLength); + let offset = 0; + for (const frame of buffer) { + data.set(frame.data, offset); + offset += frame.data.length; + } + return new AudioFrame(data, sampleRate, channels, samplesPerChannel); }