feat(aio): explain empty generation output when tokens were billed

frontend/snapshots.yml

@@ -5692,6 +5692,22 @@ snapshots:
         hash: v1.k794b7964.9ea11684de93cf0bf2fabc01d05a7b4fed8d0e540084b01ec27ab8a29d0f3823.vJCRxMCRSHVE9xVry1hOYTeYimJdH1PyeD0guOSQu4k
     scenes-app-ai-observability-conversation-display--basic--light:
         hash: v1.k794b7964.3cb142d720e0576d3b92310d75f791cf42452f5f32d38674f79d434790589239.7q-Lw22M7R633hrsN-ZnuIgkAWP9l4i9iRbu5jpnOG8
+    scenes-app-ai-observability-conversation-display--empty-output-blocked-by-provider--dark:
+        hash: v1.k794b7964.05125a80c55c9ad20c8edcc84bf05e1f58ff673ca1a2abbef34ced9ec3552b9d.mah6RylH_kokmN1fnEjxZQD4Ps3QUaffSekfCz5N6i4
+    scenes-app-ai-observability-conversation-display--empty-output-blocked-by-provider--light:
+        hash: v1.k794b7964.1f86f5ba9536b50c817ca0a3c0d55548c14b8f4a8ff13b504c3f7b150e012ada.4ZWD_3VA2AijSiEbq-_jE96K3tcgkw3HHuu_Ux3IFfQ
+    scenes-app-ai-observability-conversation-display--empty-output-hit-token-limit--dark:
+        hash: v1.k794b7964.b2548b108edc6f8c958940e1fc0209b999b98438e0a6b6c095f3ac42a4cdd9f1.-411C02o_BaVwQeYWxv1oPT5d_J-6pawteYt4V80BAA
+    scenes-app-ai-observability-conversation-display--empty-output-hit-token-limit--light:
+        hash: v1.k794b7964.f8d093d6d02c6f147e06a6fbff6cbac04aa9cc29e7b1780a67c59b26681869c8.j7EwfprpifYVQrbjfTqvFjMSCeqtwfxhgfeLQQ_2lFk
+    scenes-app-ai-observability-conversation-display--empty-output-with-billed-tokens--dark:
+        hash: v1.k794b7964.42eeaa8552683109e50304840c0fe8907c4c0825b622ecd1d9745bc4f895d955.HGZ_kprIGe14sglkYwj1wNpS3hLSkGbZKw3tuUJTz5s
+    scenes-app-ai-observability-conversation-display--empty-output-with-billed-tokens--light:
+        hash: v1.k794b7964.b9e3adecf8f05139d76126faf07998b0a915fec481915e6a8bad3a97992dae68.4UFV8PthL3Y5pbeU3axDaPZIep6ixfatWlrzqW4JKm0
+    scenes-app-ai-observability-conversation-display--empty-output-with-reasoning-tokens--dark:
+        hash: v1.k794b7964.9f1da3403eb752b5499e096ed36633d7c8fff2e69b7ba3696354e664bde897ff.N7wmsys9CZDuETYl1x-hqLouyIimaxUWLRWc9DZYgjY
+    scenes-app-ai-observability-conversation-display--empty-output-with-reasoning-tokens--light:
+        hash: v1.k794b7964.34eabfa265ecc45640d754647a5ad836ef95a3d32c1885391c94af7865d2fe18.N2i_3GzO4MpTXxY30ebKa5IiVNEIBVSbDgGepsfm4RI
     scenes-app-ai-observability-conversation-display--error--dark:
         hash: v1.k794b7964.477b40a99b407806f5a85839a94004a899be31370ea6861518dd11cf5c883c45.EQEZzP2enfzFprXAhyqSbD8Um6pvTD1cGa3yq6qn4VU
     scenes-app-ai-observability-conversation-display--error--light:

frontend/src/scenes/session-recordings/player/inspector/components/AIEventItems.tsx

@@ -8,12 +8,13 @@ import { ConversationMessagesDisplay } from 'products/ai_observability/frontend/
 import { useAIData } from 'products/ai_observability/frontend/hooks/useAIData'
 import { LLMInputOutput } from 'products/ai_observability/frontend/LLMInputOutput'
 import { normalizeMessages } from 'products/ai_observability/frontend/messageNormalization'
+import { selectAiValue } from 'products/ai_observability/frontend/utils'
 
 export function AIEventExpanded({ event }: { event: Record<string, any> }): JSX.Element {
     const { input, output, tools, isLoading } = useAIData({
         uuid: event.uuid,
         input: event.properties?.$ai_input,
-        output: event.properties?.$ai_output_choices,
+        output: selectAiValue(event.properties?.$ai_output_choices, event.properties?.$ai_output),
         tools: event.properties?.$ai_tools,
         traceId: event.properties?.$ai_trace_id,
         timestamp: event.timestamp,
@@ -35,6 +36,10 @@ export function AIEventExpanded({ event }: { event: Record<string, any> }): JSX.
                     errorData={event.properties.$ai_error}
                     httpStatus={event.properties.$ai_http_status}
                     raisedError={raisedError}
+                    outputTokens={event.properties.$ai_output_tokens}
+                    reasoningTokens={event.properties.$ai_reasoning_tokens}
+                    textOutputTokens={event.properties.$ai_text_output_tokens}
+                    stopReason={event.properties.$ai_stop_reason}
                     traceId={event.properties.$ai_trace_id}
                 />
             ) : (

products/ai_observability/frontend/AIObservabilityTraceScene.tsx

@@ -123,6 +123,7 @@ import {
     isLLMEvent,
     removeMilliseconds,
     sanitizeTraceUrlSearchParams,
+    selectAiValue,
 } from './utils'
 
 interface TraceQueueContext {
@@ -1777,14 +1778,20 @@ const EventContent = React.memo(
                                                                 traceId={trace.id}
                                                                 timestamp={event.createdAt}
                                                                 rawInput={event.properties.$ai_input}
-                                                                rawOutput={
-                                                                    event.properties.$ai_output_choices ??
+                                                                rawOutput={selectAiValue(
+                                                                    event.properties.$ai_output_choices,
                                                                     event.properties.$ai_output
-                                                                }
+                                                                )}
                                                                 tools={event.properties.$ai_tools}
                                                                 errorData={event.properties.$ai_error}
                                                                 httpStatus={event.properties.$ai_http_status}
                                                                 raisedError={event.properties.$ai_is_error}
+                                                                outputTokens={event.properties.$ai_output_tokens}
+                                                                reasoningTokens={event.properties.$ai_reasoning_tokens}
+                                                                textOutputTokens={
+                                                                    event.properties.$ai_text_output_tokens
+                                                                }
+                                                                stopReason={event.properties.$ai_stop_reason}
                                                                 searchQuery={searchQuery}
                                                                 displayOption={displayOption}
                                                                 highlightMessageIndex={highlightMessageIndex}

products/ai_observability/frontend/ConversationDisplay/ConversationDisplay.stories.tsx

@@ -170,6 +170,75 @@ export const Error: Story = {
     },
 }
 
+export const EmptyOutputWithBilledTokens: Story = {
+    args: {
+        eventProperties: {
+            $ai_input: [
+                { role: 'system', content: 'You are a good bot.' },
+                { role: 'user', content: 'Write me a long essay about bots.' },
+            ],
+            $ai_output_choices: [],
+            $ai_input_tokens: 120,
+            $ai_output_tokens: 512,
+            $ai_latency: 8.4,
+            $ai_model: 'gpt-9',
+            $ai_http_status: 200,
+        },
+    },
+}
+
+export const EmptyOutputWithReasoningTokens: Story = {
+    args: {
+        eventProperties: {
+            $ai_input: [
+                { role: 'system', content: 'You are a good bot.' },
+                { role: 'user', content: 'Write me a long essay about bots.' },
+            ],
+            $ai_output_choices: [],
+            $ai_input_tokens: 120,
+            $ai_reasoning_tokens: 442,
+            $ai_latency: 8.4,
+            $ai_model: 'gpt-9',
+            $ai_http_status: 200,
+        },
+    },
+}
+
+export const EmptyOutputHitTokenLimit: Story = {
+    args: {
+        eventProperties: {
+            $ai_input: [
+                { role: 'system', content: 'You are a good bot.' },
+                { role: 'user', content: 'Write me a long essay about bots.' },
+            ],
+            $ai_output_choices: [],
+            $ai_input_tokens: 120,
+            $ai_output_tokens: 2048,
+            $ai_stop_reason: 'MAX_TOKENS',
+            $ai_latency: 8.4,
+            $ai_model: 'gpt-9',
+            $ai_http_status: 200,
+        },
+    },
+}
+
+export const EmptyOutputBlockedByProvider: Story = {
+    args: {
+        eventProperties: {
+            $ai_input: [
+                { role: 'system', content: 'You are a good bot.' },
+                { role: 'user', content: 'Write me a long essay about bots.' },
+            ],
+            $ai_output_choices: [],
+            $ai_input_tokens: 120,
+            $ai_stop_reason: 'PROHIBITED_CONTENT',
+            $ai_latency: 0.4,
+            $ai_model: 'gpt-9',
+            $ai_http_status: 200,
+        },
+    },
+}
+
 export const Anthropic: Story = {
     args: {
         eventProperties: {

products/ai_observability/frontend/ConversationDisplay/ConversationDisplay.test.tsx

@@ -68,6 +68,37 @@ describe('ConversationDisplay', () => {
                 output: 'plain output',
             },
         ],
+        [
+            '$ai_generation whose $ai_output_choices is an empty container',
+            '$ai_generation',
+            {
+                $ai_input: [{ role: 'user', content: 'generation input' }],
+                $ai_output_choices: [],
+                $ai_output: [{ role: 'assistant', content: 'real output' }],
+                $ai_tools: [{ function: { name: 'search' } }],
+                $ai_trace_id: 'trace-1',
+            },
+            {
+                input: [{ role: 'user', content: 'generation input' }],
+                output: [{ role: 'assistant', content: 'real output' }],
+            },
+        ],
+        [
+            '$ai_generation whose output is genuinely empty',
+            '$ai_generation',
+            {
+                $ai_input: [{ role: 'user', content: 'generation input' }],
+                $ai_output_choices: [],
+                $ai_tools: [{ function: { name: 'search' } }],
+                $ai_trace_id: 'trace-1',
+            },
+            {
+                input: [{ role: 'user', content: 'generation input' }],
+                // The empty container still counts as arrived. Passing undefined instead would make
+                // useAIData run the heavy-prop lookup for a generation this PR's notice explains.
+                output: [],
+            },
+        ],
         [
             '$ai_embedding',
             '$ai_embedding',

products/ai_observability/frontend/ConversationDisplay/ConversationDisplay.tsx

@@ -10,7 +10,7 @@ import { buildInputSourceIndices } from '../extractSessionTurns'
 import { useAIData } from '../hooks/useAIData'
 import { normalizeMessages } from '../messageNormalization'
 import { openInPlayground } from '../playground/llmPlaygroundPromptsLogic'
-import { costContextFromProperties } from '../utils'
+import { costContextFromProperties, selectAiValue } from '../utils'
 import { ConversationMessagesDisplay } from './ConversationMessagesDisplay'
 import { MetadataHeader } from './MetadataHeader'
 
@@ -34,7 +34,7 @@ export function ConversationDisplay({
             : eventProperties.$ai_input_state
     const rawOutput =
         eventName === '$ai_generation'
-            ? (eventProperties.$ai_output_choices ?? eventProperties.$ai_output)
+            ? selectAiValue(eventProperties.$ai_output_choices, eventProperties.$ai_output)
             : eventName === '$ai_embedding'
               ? 'Embedding vector generated'
               : eventProperties.$ai_output_state
@@ -100,6 +100,10 @@ export function ConversationDisplay({
                     errorData={eventProperties.$ai_error}
                     httpStatus={eventProperties.$ai_http_status}
                     raisedError={eventProperties.$ai_is_error}
+                    outputTokens={eventProperties.$ai_output_tokens}
+                    reasoningTokens={eventProperties.$ai_reasoning_tokens}
+                    textOutputTokens={eventProperties.$ai_text_output_tokens}
+                    stopReason={eventProperties.$ai_stop_reason}
                     bordered
                     traceId={eventProperties.$ai_trace_id}
                     generationEventId={eventId}

products/ai_observability/frontend/ConversationDisplay/ConversationMessagesDisplay.test.tsx

@@ -470,6 +470,154 @@ describe('ConversationMessagesDisplay', () => {
             expect(screen.queryByText(text)).not.toBeInTheDocument()
         }
     })
+
+    it.each<[string, unknown, unknown, unknown, unknown, string | null]>([
+        [
+            'only output tokens were billed',
+            2048,
+            undefined,
+            undefined,
+            undefined,
+            'The provider reported 2,048 output tokens but no content was captured. The response may have been cut short, or the SDK may not have captured it.',
+        ],
+        [
+            'only reasoning tokens were billed',
+            undefined,
+            442,
+            undefined,
+            undefined,
+            'The provider reported 442 reasoning tokens but no content was captured. The model may have spent its budget on reasoning.',
+        ],
+        [
+            'reasoning matches the billed output exactly',
+            442,
+            442,
+            undefined,
+            undefined,
+            'The provider reported 442 output tokens but no content was captured. All of them may have been reasoning.',
+        ],
+        [
+            'reasoning exceeds the output count, so the output is missing content',
+            400,
+            500,
+            undefined,
+            undefined,
+            'The provider reported 400 output tokens and 500 reasoning tokens but no content was captured. The response may have been cut short, or the SDK may not have captured it.',
+        ],
+        [
+            'a provider reported the count as a string',
+            '512',
+            undefined,
+            undefined,
+            undefined,
+            'The provider reported 512 output tokens but no content was captured. The response may have been cut short, or the SDK may not have captured it.',
+        ],
+        [
+            'a provider spelled the token limit in upper case',
+            2048,
+            undefined,
+            undefined,
+            'MAX_TOKENS',
+            'The provider reported 2,048 output tokens but no content was captured. The response hit its token limit.',
+        ],
+        [
+            'the stop reason outranks the reasoning-token guess',
+            undefined,
+            442,
+            undefined,
+            'length',
+            'The provider reported 442 reasoning tokens but no content was captured. The response hit its token limit.',
+        ],
+        [
+            'the provider blocked the response without billing anything',
+            0,
+            0,
+            undefined,
+            'PROHIBITED_CONTENT',
+            'The provider blocked the response.',
+        ],
+        [
+            'the stop reason describes a normal ending',
+            2048,
+            undefined,
+            undefined,
+            'end_turn',
+            'The provider reported 2,048 output tokens but no content was captured. The response may have been cut short, or the SDK may not have captured it.',
+        ],
+        [
+            'reasoning is only a fraction of the billed output',
+            2048,
+            12,
+            undefined,
+            undefined,
+            'The provider reported 2,048 output tokens and 12 reasoning tokens but no content was captured. The response may have been cut short, or the SDK may not have captured it.',
+        ],
+        [
+            'the text-token split says none of the output was text',
+            169,
+            56,
+            0,
+            undefined,
+            'The provider reported 169 output tokens and 56 reasoning tokens but no content was captured. None of them were text.',
+        ],
+        [
+            'the text-token split says text was billed too',
+            442,
+            442,
+            113,
+            undefined,
+            'The provider reported 442 output tokens and 442 reasoning tokens but no content was captured. The response may have been cut short, or the SDK may not have captured it.',
+        ],
+        [
+            'a provider hyphenated the stop reason',
+            2048,
+            undefined,
+            undefined,
+            'content-filter',
+            'The provider reported 2,048 output tokens but no content was captured. The provider blocked the response.',
+        ],
+        [
+            'an older SDK sent the count as an object',
+            { total: 10585, noCache: 10585, cacheRead: 0 },
+            undefined,
+            undefined,
+            undefined,
+            'The provider reported 10,585 output tokens but no content was captured. The response may have been cut short, or the SDK may not have captured it.',
+        ],
+        ['the provider billed nothing', 0, 0, undefined, undefined, null],
+        ['no token counts arrived', undefined, undefined, undefined, undefined, null],
+    ])(
+        'empty output explains the gap when %s',
+        (_label, outputTokens, reasoningTokens, textOutputTokens, stopReason, expected) => {
+            const { container } = render(
+                <Provider>
+                    <ConversationMessagesDisplay
+                        inputNormalized={inputNormalized}
+                        outputNormalized={[]}
+                        errorData={null}
+                        raisedError={false}
+                        outputTokens={outputTokens}
+                        reasoningTokens={reasoningTokens}
+                        textOutputTokens={textOutputTokens}
+                        stopReason={stopReason}
+                    />
+                </Provider>
+            )
+
+            expect(screen.getByText('No output')).toBeInTheDocument()
+            const explanation = container.querySelector('[data-attr="ai-empty-output-explanation"]')
+            if (expected !== null) {
+                expect(explanation).toHaveTextContent(expected)
+                // The notice names a cause and the link carries the fix. A typo'd anchor dead-ends there.
+                expect(explanation!.querySelector('a')).toHaveAttribute(
+                    'href',
+                    'https://posthog.com/docs/ai-observability/troubleshooting#why-does-my-generation-show-no-output'
+                )
+            } else {
+                expect(explanation).toBeNull()
+            }
+        }
+    )
 })
 
 describe('ImageMessageDisplay', () => {