fix: preserve realtime output audio configuration

Author: jpr5

src/__tests__/multimedia-record.test.ts

@@ -285,7 +285,7 @@ describe("multimedia record: transcription response detection", () => {
   it("persists a typed terminal stream when the client closes before upstream end", async () => {
     const fixturePath = makeTmpDir();
     const terminal =
-      'data: {"type":"transcript.text.done","text":"Typed terminal","languages":[{"code":"en"}]}\n\n';
+      'data: {"type":"transcript.text.done","text":"Typed terminal","languages":[{"code":"en"}],"usage":{"input_tokens":4,"output_tokens":5}}\n\n';
     const { server: upstream, url } = await createUpstream((_req, res) => {
       res.writeHead(200, { "Content-Type": "text/event-stream" });
       res.write(terminal);
@@ -307,11 +307,30 @@ describe("multimedia record: transcription response detection", () => {
       await new Promise((resolve) => setTimeout(resolve, 250));
 
       expect(fixtures).toHaveLength(1);
+      expect(fixtures[0].response).toEqual({
+        transcription: {
+          text: "Typed terminal",
+          languages: [{ code: "en" }],
+          usage: { input_tokens: 4, output_tokens: 5 },
+        },
+      });
+      const savedFixture = JSON.parse(
+        fs.readFileSync(
+          path.join(
+            fixturePath,
+            fs.readdirSync(fixturePath).find((file) => file.endsWith(".json"))!,
+          ),
+          "utf8",
+        ),
+      );
+      expect(savedFixture.fixtures[0].response).toEqual(fixtures[0].response);
       await closeServer(upstream);
 
       const replay = await requestStreamingTranscription(recorder.url);
       expect(replay.status).toBe(200);
-      expect(await replay.text()).toContain('"text":"Typed terminal"');
+      expect(await replay.text()).toContain(
+        '"text":"Typed terminal","languages":[{"code":"en"}],"usage":{"input_tokens":4,"output_tokens":5}',
+      );
     } finally {
       await closeServer(recorder.server);
       if (upstream.listening) await closeServer(upstream);

src/__tests__/ws-api-conformance.test.ts

@@ -690,14 +690,23 @@ describe("GA Realtime conformance", () => {
     ws.send(
       JSON.stringify({
         type: "session.update",
-        session: { audio: { input: { format: { type: "audio/pcm", rate: 24000 } } } },
+        session: {
+          audio: {
+            input: { format: { type: "audio/pcm", rate: 24000 } },
+            output: { voice: "alloy", format: { type: "audio/pcm", rate: 24000 } },
+          },
+        },
       }),
     );
     const raw = await ws.waitForMessages(2);
     ws.close();
     const frame = JSON.parse(raw[1]) as any;
     expect(frame.type).toBe("session.updated");
     expect(frame.session.audio.input.format).toEqual({ type: "audio/pcm", rate: 24000 });
+    expect(frame.session.audio.output).toEqual({
+      voice: "alloy",
+      format: { type: "audio/pcm", rate: 24000 },
+    });
   });
 });
 

src/types.ts

@@ -473,7 +473,7 @@ export type RealtimePhase = "final_answer" | "commentary";
 export interface GASessionAudioConfig {
   voice: string | null;
   input_audio_format: { type: string; rate?: number; [key: string]: unknown } | null;
-  output_audio_format: string | null;
+  output_audio_format: { type: string; rate?: number; [key: string]: unknown } | null;
   input_audio_noise_reduction: { type: string } | null;
   input_audio_transcription: { model: string } | null;
 }

src/ws-realtime.ts

@@ -61,7 +61,7 @@ interface SessionConfig {
   tools: unknown[];
   voice: string | null;
   input_audio_format: Record<string, unknown> | null;
-  output_audio_format: string | null;
+  output_audio_format: Record<string, unknown> | null;
   input_audio_noise_reduction: { type: string } | null;
   input_audio_transcription: { model: string; language?: string; prompt?: string } | null;
   turn_detection: unknown | null;
@@ -93,7 +93,7 @@ function isLiveTranscriptionModel(model: string): boolean {
   );
 }
 
-function normalizeInputAudioFormat(value: unknown): Record<string, unknown> | null {
+function normalizeAudioFormat(value: unknown): Record<string, unknown> | null {
   if (value === null) return null;
   if (typeof value === "string") return { type: value };
   if (
@@ -137,7 +137,7 @@ function serializeSession(session: SessionConfig, sessionId?: string): Record<st
         turn_detection: session.turn_detection,
       },
       output: {
-        format: session.output_audio_format ? { type: session.output_audio_format } : null,
+        format: session.output_audio_format,
         voice: session.voice,
       },
     },
@@ -550,9 +550,9 @@ async function processMessage(
         const audio = (s as Record<string, unknown>).audio as Record<string, unknown>;
         if (audio.voice !== undefined) session.voice = audio.voice as string | null;
         if (audio.input_audio_format !== undefined)
-          session.input_audio_format = normalizeInputAudioFormat(audio.input_audio_format);
+          session.input_audio_format = normalizeAudioFormat(audio.input_audio_format);
         if (audio.output_audio_format !== undefined)
-          session.output_audio_format = audio.output_audio_format as string | null;
+          session.output_audio_format = normalizeAudioFormat(audio.output_audio_format);
         if (audio.input_audio_noise_reduction !== undefined)
           session.input_audio_noise_reduction = audio.input_audio_noise_reduction as {
             type: string;
@@ -577,14 +577,21 @@ async function processMessage(
             session.input_audio_noise_reduction = input.noise_reduction as { type: string } | null;
           if (input.turn_detection !== undefined) session.turn_detection = input.turn_detection;
           if (input.format !== undefined)
-            session.input_audio_format = normalizeInputAudioFormat(input.format);
+            session.input_audio_format = normalizeAudioFormat(input.format);
+        }
+        if (audio.output && typeof audio.output === "object") {
+          const output = audio.output as Record<string, unknown>;
+          if (output.voice !== undefined) session.voice = output.voice as string | null;
+          if (output.format !== undefined)
+            session.output_audio_format = normalizeAudioFormat(output.format);
         }
       }
       // Beta flat fields (backward compat)
       if (s.voice !== undefined) session.voice = s.voice;
       if (s.input_audio_format !== undefined)
-        session.input_audio_format = normalizeInputAudioFormat(s.input_audio_format);
-      if (s.output_audio_format !== undefined) session.output_audio_format = s.output_audio_format;
+        session.input_audio_format = normalizeAudioFormat(s.input_audio_format);
+      if (s.output_audio_format !== undefined)
+        session.output_audio_format = normalizeAudioFormat(s.output_audio_format);
       if (s.input_audio_noise_reduction !== undefined)
         session.input_audio_noise_reduction = s.input_audio_noise_reduction;
       if (s.input_audio_transcription !== undefined)