feat(aio): explain empty generation output when tokens were billed

frontend/src/scenes/session-recordings/player/inspector/components/AIEventItems.tsx

@@ -8,12 +8,13 @@ import { ConversationMessagesDisplay } from 'products/ai_observability/frontend/
 import { useAIData } from 'products/ai_observability/frontend/hooks/useAIData'
 import { LLMInputOutput } from 'products/ai_observability/frontend/LLMInputOutput'
 import { normalizeMessages } from 'products/ai_observability/frontend/messageNormalization'
+import { firstAiOutputWithContent } from 'products/ai_observability/frontend/utils'
 
 export function AIEventExpanded({ event }: { event: Record<string, any> }): JSX.Element {
     const { input, output, tools, isLoading } = useAIData({
         uuid: event.uuid,
         input: event.properties?.$ai_input,
-        output: event.properties?.$ai_output_choices,
+        output: firstAiOutputWithContent(event.properties?.$ai_output_choices, event.properties?.$ai_output),
         tools: event.properties?.$ai_tools,
         traceId: event.properties?.$ai_trace_id,
         timestamp: event.timestamp,
@@ -35,6 +36,10 @@ export function AIEventExpanded({ event }: { event: Record<string, any> }): JSX.
                     errorData={event.properties.$ai_error}
                     httpStatus={event.properties.$ai_http_status}
                     raisedError={raisedError}
+                    outputTokens={event.properties.$ai_output_tokens}
+                    reasoningTokens={event.properties.$ai_reasoning_tokens}
+                    textOutputTokens={event.properties.$ai_text_output_tokens}
+                    stopReason={event.properties.$ai_stop_reason}
                     traceId={event.properties.$ai_trace_id}
                 />
             ) : (

products/ai_observability/frontend/AIObservabilityTraceScene.tsx

@@ -111,6 +111,7 @@ import {
     CostContext,
     costContextFromProperties,
     costContextFromTrace,
+    firstAiOutputWithContent,
     formatLLMCost,
     formatLLMEventTitle,
     formatLLMLatency,
@@ -1777,14 +1778,20 @@ const EventContent = React.memo(
                                                                 traceId={trace.id}
                                                                 timestamp={event.createdAt}
                                                                 rawInput={event.properties.$ai_input}
-                                                                rawOutput={
-                                                                    event.properties.$ai_output_choices ??
+                                                                rawOutput={firstAiOutputWithContent(
+                                                                    event.properties.$ai_output_choices,
                                                                     event.properties.$ai_output
-                                                                }
+                                                                )}
                                                                 tools={event.properties.$ai_tools}
                                                                 errorData={event.properties.$ai_error}
                                                                 httpStatus={event.properties.$ai_http_status}
                                                                 raisedError={event.properties.$ai_is_error}
+                                                                outputTokens={event.properties.$ai_output_tokens}
+                                                                reasoningTokens={event.properties.$ai_reasoning_tokens}
+                                                                textOutputTokens={
+                                                                    event.properties.$ai_text_output_tokens
+                                                                }
+                                                                stopReason={event.properties.$ai_stop_reason}
                                                                 searchQuery={searchQuery}
                                                                 displayOption={displayOption}
                                                                 highlightMessageIndex={highlightMessageIndex}

products/ai_observability/frontend/ConversationDisplay/ConversationDisplay.stories.tsx

@@ -170,6 +170,75 @@ export const Error: Story = {
     },
 }
 
+export const EmptyOutputWithBilledTokens: Story = {
+    args: {
+        eventProperties: {
+            $ai_input: [
+                { role: 'system', content: 'You are a good bot.' },
+                { role: 'user', content: 'Write me a long essay about bots.' },
+            ],
+            $ai_output_choices: [],
+            $ai_input_tokens: 120,
+            $ai_output_tokens: 512,
+            $ai_latency: 8.4,
+            $ai_model: 'gpt-9',
+            $ai_http_status: 200,
+        },
+    },
+}
+
+export const EmptyOutputWithReasoningTokens: Story = {
+    args: {
+        eventProperties: {
+            $ai_input: [
+                { role: 'system', content: 'You are a good bot.' },
+                { role: 'user', content: 'Write me a long essay about bots.' },
+            ],
+            $ai_output_choices: [],
+            $ai_input_tokens: 120,
+            $ai_reasoning_tokens: 442,
+            $ai_latency: 8.4,
+            $ai_model: 'gpt-9',
+            $ai_http_status: 200,
+        },
+    },
+}
+
+export const EmptyOutputHitTokenLimit: Story = {
+    args: {
+        eventProperties: {
+            $ai_input: [
+                { role: 'system', content: 'You are a good bot.' },
+                { role: 'user', content: 'Write me a long essay about bots.' },
+            ],
+            $ai_output_choices: [],
+            $ai_input_tokens: 120,
+            $ai_output_tokens: 2048,
+            $ai_stop_reason: 'MAX_TOKENS',
+            $ai_latency: 8.4,
+            $ai_model: 'gpt-9',
+            $ai_http_status: 200,
+        },
+    },
+}
+
+export const EmptyOutputBlockedByProvider: Story = {
+    args: {
+        eventProperties: {
+            $ai_input: [
+                { role: 'system', content: 'You are a good bot.' },
+                { role: 'user', content: 'Write me a long essay about bots.' },
+            ],
+            $ai_output_choices: [],
+            $ai_input_tokens: 120,
+            $ai_stop_reason: 'PROHIBITED_CONTENT',
+            $ai_latency: 0.4,
+            $ai_model: 'gpt-9',
+            $ai_http_status: 200,
+        },
+    },
+}
+
 export const Anthropic: Story = {
     args: {
         eventProperties: {

products/ai_observability/frontend/ConversationDisplay/ConversationDisplay.test.tsx

@@ -68,6 +68,21 @@ describe('ConversationDisplay', () => {
                 output: 'plain output',
             },
         ],
+        [
+            '$ai_generation whose $ai_output_choices is an empty container',
+            '$ai_generation',
+            {
+                $ai_input: [{ role: 'user', content: 'generation input' }],
+                $ai_output_choices: [],
+                $ai_output: [{ role: 'assistant', content: 'real output' }],
+                $ai_tools: [{ function: { name: 'search' } }],
+                $ai_trace_id: 'trace-1',
+            },
+            {
+                input: [{ role: 'user', content: 'generation input' }],
+                output: [{ role: 'assistant', content: 'real output' }],
+            },
+        ],
         [
             '$ai_embedding',
             '$ai_embedding',

products/ai_observability/frontend/ConversationDisplay/ConversationDisplay.tsx

@@ -10,7 +10,7 @@ import { buildInputSourceIndices } from '../extractSessionTurns'
 import { useAIData } from '../hooks/useAIData'
 import { normalizeMessages } from '../messageNormalization'
 import { openInPlayground } from '../playground/llmPlaygroundPromptsLogic'
-import { costContextFromProperties } from '../utils'
+import { costContextFromProperties, firstAiOutputWithContent } from '../utils'
 import { ConversationMessagesDisplay } from './ConversationMessagesDisplay'
 import { MetadataHeader } from './MetadataHeader'
 
@@ -34,7 +34,7 @@ export function ConversationDisplay({
             : eventProperties.$ai_input_state
     const rawOutput =
         eventName === '$ai_generation'
-            ? (eventProperties.$ai_output_choices ?? eventProperties.$ai_output)
+            ? firstAiOutputWithContent(eventProperties.$ai_output_choices, eventProperties.$ai_output)
             : eventName === '$ai_embedding'
               ? 'Embedding vector generated'
               : eventProperties.$ai_output_state
@@ -100,6 +100,10 @@ export function ConversationDisplay({
                     errorData={eventProperties.$ai_error}
                     httpStatus={eventProperties.$ai_http_status}
                     raisedError={eventProperties.$ai_is_error}
+                    outputTokens={eventProperties.$ai_output_tokens}
+                    reasoningTokens={eventProperties.$ai_reasoning_tokens}
+                    textOutputTokens={eventProperties.$ai_text_output_tokens}
+                    stopReason={eventProperties.$ai_stop_reason}
                     bordered
                     traceId={eventProperties.$ai_trace_id}
                     generationEventId={eventId}

products/ai_observability/frontend/ConversationDisplay/ConversationMessagesDisplay.test.tsx

@@ -470,6 +470,149 @@ describe('ConversationMessagesDisplay', () => {
             expect(screen.queryByText(text)).not.toBeInTheDocument()
         }
     })
+
+    it.each<[string, unknown, unknown, unknown, unknown, string | null]>([
+        [
+            'only output tokens were billed',
+            2048,
+            undefined,
+            undefined,
+            undefined,
+            'The provider reported 2,048 output tokens but no content was captured. The response may have been cut short, or the SDK may not have captured it.',
+        ],
+        [
+            'only reasoning tokens were billed',
+            undefined,
+            442,
+            undefined,
+            undefined,
+            'The provider reported 442 reasoning tokens but no content was captured. The model may have spent its budget on reasoning.',
+        ],
+        [
+            'reasoning matches the billed output exactly',
+            442,
+            442,
+            undefined,
+            undefined,
+            'The provider reported 442 output tokens but no content was captured. All of them may have been reasoning.',
+        ],
+        [
+            'reasoning exceeds the output count, so the output is missing content',
+            400,
+            500,
+            undefined,
+            undefined,
+            'The provider reported 400 output tokens and 500 reasoning tokens but no content was captured. The response may have been cut short, or the SDK may not have captured it.',
+        ],
+        [
+            'a provider reported the count as a string',
+            '512',
+            undefined,
+            undefined,
+            undefined,
+            'The provider reported 512 output tokens but no content was captured. The response may have been cut short, or the SDK may not have captured it.',
+        ],
+        [
+            'a provider spelled the token limit in upper case',
+            2048,
+            undefined,
+            undefined,
+            'MAX_TOKENS',
+            'The provider reported 2,048 output tokens but no content was captured. The response hit its token limit.',
+        ],
+        [
+            'the stop reason outranks the reasoning-token guess',
+            undefined,
+            442,
+            undefined,
+            'length',
+            'The provider reported 442 reasoning tokens but no content was captured. The response hit its token limit.',
+        ],
+        [
+            'the provider blocked the response without billing anything',
+            0,
+            0,
+            undefined,
+            'PROHIBITED_CONTENT',
+            'The provider blocked the response.',
+        ],
+        [
+            'the stop reason describes a normal ending',
+            2048,
+            undefined,
+            undefined,
+            'end_turn',
+            'The provider reported 2,048 output tokens but no content was captured. The response may have been cut short, or the SDK may not have captured it.',
+        ],
+        [
+            'reasoning is only a fraction of the billed output',
+            2048,
+            12,
+            undefined,
+            undefined,
+            'The provider reported 2,048 output tokens and 12 reasoning tokens but no content was captured. The response may have been cut short, or the SDK may not have captured it.',
+        ],
+        [
+            'the text-token split says none of the output was text',
+            169,
+            56,
+            0,
+            undefined,
+            'The provider reported 169 output tokens and 56 reasoning tokens but no content was captured. None of them were text.',
+        ],
+        [
+            'the text-token split says text was billed too',
+            442,
+            442,
+            113,
+            undefined,
+            'The provider reported 442 output tokens and 442 reasoning tokens but no content was captured. The response may have been cut short, or the SDK may not have captured it.',
+        ],
+        [
+            'a provider hyphenated the stop reason',
+            2048,
+            undefined,
+            undefined,
+            'content-filter',
+            'The provider reported 2,048 output tokens but no content was captured. The provider blocked the response.',
+        ],
+        [
+            'an older SDK sent the count as an object',
+            { total: 10585, noCache: 10585, cacheRead: 0 },
+            undefined,
+            undefined,
+            undefined,
+            'The provider reported 10,585 output tokens but no content was captured. The response may have been cut short, or the SDK may not have captured it.',
+        ],
+        ['the provider billed nothing', 0, 0, undefined, undefined, null],
+        ['no token counts arrived', undefined, undefined, undefined, undefined, null],
+    ])(
+        'empty output explains the gap when %s',
+        (_label, outputTokens, reasoningTokens, textOutputTokens, stopReason, expected) => {
+            const { container } = render(
+                <Provider>
+                    <ConversationMessagesDisplay
+                        inputNormalized={inputNormalized}
+                        outputNormalized={[]}
+                        errorData={null}
+                        raisedError={false}
+                        outputTokens={outputTokens}
+                        reasoningTokens={reasoningTokens}
+                        textOutputTokens={textOutputTokens}
+                        stopReason={stopReason}
+                    />
+                </Provider>
+            )
+
+            expect(screen.getByText('No output')).toBeInTheDocument()
+            const explanation = container.querySelector('[data-attr="ai-empty-output-explanation"]')
+            if (expected !== null) {
+                expect(explanation).toHaveTextContent(expected)
+            } else {
+                expect(explanation).toBeNull()
+            }
+        }
+    )
 })
 
 describe('ImageMessageDisplay', () => {