Tracks: Studio Code chat & instructions events (STU-2120)

apps/cli/commands/ai/index.ts

@@ -5,9 +5,18 @@ import {
 } from '@studio/common/ai/chat-files';
 import { type StudioChatImage } from '@studio/common/ai/chat-images';
 import { getAgentEndFailure } from '@studio/common/ai/json-events';
-import { DEFAULT_MODEL, resolveSessionModel, type AiModelId } from '@studio/common/ai/models';
+import {
+	DEFAULT_MODEL,
+	getAiModelFamily,
+	resolveSessionModel,
+	type AiModelId,
+} from '@studio/common/ai/models';
 import { getAgentEndTurnResult } from '@studio/common/ai/session-events';
-import { buildSkillInvocationPrompt } from '@studio/common/ai/slash-commands';
+import {
+	buildSkillInvocationPrompt,
+	resolveSkillFromPrompt,
+} from '@studio/common/ai/slash-commands';
+import { getAiTracksIdentity } from '@studio/common/ai/tracks-identity';
 import { readAuthToken } from '@studio/common/lib/shared-config';
 import { getSessionsDirectory } from '@studio/common/lib/well-known-paths';
 import { __, sprintf } from '@wordpress/i18n';
@@ -43,6 +52,12 @@ import { findSiteByFolder, findSiteById } from 'cli/lib/cli-config/sites';
 import { disconnectFromDaemon } from 'cli/lib/daemon-client';
 import { isSiteRunning } from 'cli/lib/site-utils';
 import { maybeShowTosNotice } from 'cli/lib/tos-notice';
+import {
+	getTracksOrigin,
+	recordTracksEvent,
+	TRACKS_EVENTS,
+	type TracksEventName,
+} from 'cli/lib/tracks';
 import { Logger, LoggerError, setProgressCallback } from 'cli/logger';
 import { StudioArgv } from 'cli/types';
 import type { SessionManager } from '@earendil-works/pi-coding-agent';
@@ -51,6 +66,7 @@ import type {
 	StudioCustomEntryType,
 } from '@studio/common/ai/sessions/entry-types';
 import type { LoadedAiSession, TurnStatus } from '@studio/common/ai/sessions/types';
+import type { TracksProps } from '@studio/common/lib/record-tracks-event';
 import type { AskUserQuestion } from 'cli/ai/types';
 
 const logger = new Logger< string >();
@@ -66,6 +82,21 @@ function appendStudioEntry< T extends StudioCustomEntryType >(
 	return sm.appendCustomEntry( customType, data );
 }
 
+// Awaited rather than fire-and-forget so JSON mode, which exits right after a turn, doesn't drop the
+// event — the wrapper does async work before the request is even issued. Errors are swallowed: one
+// call sits on the turn's critical path and the other in a `finally`, where a rejection would mask
+// the turn's own error.
+async function recordChatTracksEvent(
+	event: TracksEventName,
+	props: TracksProps
+): Promise< void > {
+	try {
+		await recordTracksEvent( event, props );
+	} catch {
+		// A lost analytics event must never break the chat.
+	}
+}
+
 function isPromptAbortError( error: unknown ): boolean {
 	return (
 		error instanceof Error &&
@@ -528,6 +559,25 @@ export async function runCommand( options: {
 
 		await persistSessionContext();
 
+		// Sole emitter of the chat events: every surface forks this process, and only this layer holds
+		// the provider, model and outcome together. `channel` separates them.
+		const tracksProps = {
+			...getTracksOrigin(),
+			...getAiTracksIdentity( sessionId ),
+			provider: currentProvider,
+			model: currentModel,
+			model_family: getAiModelFamily( currentModel ),
+		};
+		const turnStartedAt = Date.now();
+		await recordChatTracksEvent( TRACKS_EVENTS.CODE_MESSAGE_SENT, {
+			...tracksProps,
+			// Raw prompt, before site context is prepended. Only ever a catalog name.
+			ability_name: resolveSkillFromPrompt( prompt ),
+			has_images: images.length > 0,
+			has_files: files.length > 0,
+			is_resumed: Boolean( options.resumeSession || options.resumeSessionId ),
+		} );
+
 		// Studio marker for the typed prompt; pi appends the real UserMessage.
 		await append( ( s ) =>
 			appendStudioEntry( s, 'studio.user_prompt', {
@@ -596,6 +646,12 @@ export async function runCommand( options: {
 						: {} ),
 				} )
 			);
+			// No `errorMessage`: raw error text can embed paths and site names.
+			await recordChatTracksEvent( TRACKS_EVENTS.CODE_TURN_COMPLETED, {
+				...tracksProps,
+				outcome: turnState.status,
+				duration_ms: Date.now() - turnStartedAt,
+			} );
 			ui.endAgentTurn();
 		}
 

apps/cli/commands/ai/tests/index.test.ts

@@ -12,11 +12,16 @@ import { readCliConfig } from 'cli/lib/cli-config/core';
 import { findSiteByFolder } from 'cli/lib/cli-config/sites';
 import { disconnectFromDaemon } from 'cli/lib/daemon-client';
 import { isSiteRunning } from 'cli/lib/site-utils';
+import { recordTracksEvent, TRACKS_EVENTS } from 'cli/lib/tracks';
 import { runCommand } from '../index';
 
 vi.mock( '@studio/common/lib/shared-config', () => ( {
 	readAuthToken: vi.fn(),
 } ) );
+vi.mock( 'cli/lib/tracks', async ( importActual ) => {
+	const actual = await importActual< typeof import('cli/lib/tracks') >();
+	return { ...actual, recordTracksEvent: vi.fn() };
+} );
 vi.mock( 'cli/ai/auth', () => ( {
 	resolveInitialAiProvider: vi.fn(),
 	saveSelectedAiProvider: vi.fn(),
@@ -281,3 +286,73 @@ describe( 'AI runCommand — active site banner running state', () => {
 		);
 	} );
 } );
+
+describe( 'AI runCommand — Tracks events', () => {
+	beforeEach( () => {
+		vi.clearAllMocks();
+		( createStudioSession as Mock ).mockResolvedValue( {
+			appendCustomEntry: vi.fn( () => 'entry-id' ),
+			getSessionId: () => 'session-id',
+			getEntries: () => [],
+			getSessionFile: () => '/sessions/session-id.jsonl',
+		} );
+		( readCliConfig as Mock ).mockResolvedValue( { aiProvider: 'wpcom' } );
+		( resolveInitialAiProvider as Mock ).mockResolvedValue( 'wpcom' );
+		( recordTracksEvent as Mock ).mockResolvedValue( undefined );
+	} );
+
+	const eventNames = () =>
+		( recordTracksEvent as Mock ).mock.calls.map( ( [ name ] ) => name as string );
+
+	// JSON mode exits right after the turn, so both events must be awaited rather than
+	// fire-and-forget — the wrapper does async work (opt-out check, install id) before the request is
+	// even issued. The mock defers past a microtask so a `void` call would still be in flight when
+	// the command returns, the way it would be lost to process exit in a real headless run.
+	it( 'records both chat events before a headless run returns', async () => {
+		const settled: string[] = [];
+		( recordTracksEvent as Mock ).mockImplementation( async ( name: string ) => {
+			await new Promise( ( resolve ) => setTimeout( resolve, 0 ) );
+			settled.push( name );
+		} );
+
+		await runCommand( { adapter: new JsonAdapter(), initialMessage: 'hello' } );
+
+		expect( eventNames() ).toEqual( [
+			TRACKS_EVENTS.CODE_MESSAGE_SENT,
+			TRACKS_EVENTS.CODE_TURN_COMPLETED,
+		] );
+		expect( settled ).toEqual( [
+			TRACKS_EVENTS.CODE_MESSAGE_SENT,
+			TRACKS_EVENTS.CODE_TURN_COMPLETED,
+		] );
+	} );
+
+	it( 'reports the turn outcome and the resolved provider and model', async () => {
+		await runCommand( { adapter: new JsonAdapter(), initialMessage: 'hello' } );
+
+		const [ , props ] = ( recordTracksEvent as Mock ).mock.calls.find(
+			( [ name ] ) => name === TRACKS_EVENTS.CODE_TURN_COMPLETED
+		) as [ string, Record< string, unknown > ];
+		// `interrupted` because the mocked runtime resolves without an `agent_end` event, which is what
+		// promotes the status — the point here is that the recorded outcome tracks `turnState`.
+		expect( props ).toMatchObject( {
+			outcome: 'interrupted',
+			provider: 'wpcom',
+			model_family: 'anthropic',
+			ai_session_id: 'session-id',
+			client: 'studio-code',
+		} );
+		expect( props.duration_ms ).toBeTypeOf( 'number' );
+	} );
+
+	// One call sits on the turn's critical path, the other in a `finally` where a rejection would
+	// mask the turn's own error. Analytics must never break the chat.
+	it( 'completes the turn even when recording fails', async () => {
+		( recordTracksEvent as Mock ).mockRejectedValue( new Error( 'shared.json is locked' ) );
+
+		await expect(
+			runCommand( { adapter: new JsonAdapter(), initialMessage: 'hello' } )
+		).resolves.toBeUndefined();
+		expect( runStudioAgentTurn ).toHaveBeenCalledTimes( 1 );
+	} );
+} );

apps/local/src/index.ts

@@ -445,6 +445,7 @@ export async function startLocalServer( options: LocalServerOptions ): Promise<
 		} )
 	);
 
+	// No `studio_setting_instructions_change` — this server has no Tracks emitter. See STU-2247.
 	api.post(
 		'/agent-instructions',
 		asyncHandler( async ( req: Request, res: Response ) => {
@@ -1200,7 +1201,11 @@ export async function startLocalServer( options: LocalServerOptions ): Promise<
 					site = { id: found.id, name: found.name, path: found.path };
 				}
 			}
-			res.json( await createOrReuseAiSession( sessionsRoot, { site } ) );
+			// `created` is an analytics signal for the desktop, not part of the session shape.
+			const { created: _created, ...summary } = await createOrReuseAiSession( sessionsRoot, {
+				site,
+			} );
+			res.json( summary );
 		} )
 	);
 

apps/studio/src/ipc-handlers.ts

@@ -41,7 +41,11 @@ import {
 	deleteAiSession as deleteAiSessionFromStore,
 	loadAiSession as loadAiSessionFromStore,
 } from '@studio/common/ai/sessions/store';
-import { getAiSkillCommands, buildSkillInvocationPrompt } from '@studio/common/ai/slash-commands';
+import {
+	buildSkillInvocationPrompt,
+	resolveSkillFromPrompt,
+} from '@studio/common/ai/slash-commands';
+import { getAiTracksIdentity } from '@studio/common/ai/tracks-identity';
 import {
 	installSkillToSite,
 	removeSkillFromSite,
@@ -319,25 +323,32 @@ export async function createAiSession(
 	siteId?: string
 ): Promise< AiSessionSummary > {
 	const sessionsRoot = getSessionsDirectory();
-	if ( ! siteId ) {
-		return createOrReuseAiSession( sessionsRoot );
-	}
-
-	const server = SiteServer.get( siteId );
-	if ( ! server ) {
+	const server = siteId ? SiteServer.get( siteId ) : undefined;
+	if ( siteId && ! server ) {
 		throw new Error( `Site not found: ${ siteId }` );
 	}
 
 	// Binds the session to the site and reuses an existing empty draft for it
 	// instead of piling up orphans — the shared logic the `studio ui` server
 	// uses too.
-	return createOrReuseAiSession( sessionsRoot, {
-		site: {
+	const { created, ...summary } = await createOrReuseAiSession( sessionsRoot, {
+		site: server && {
 			id: server.details.id,
 			name: server.details.name,
 			path: server.details.path,
 		},
 	} );
+
+	// Fires from Main, not the CLI: sessions are created in-process. Reused drafts don't count.
+	// Missing for `studio ui`, which has no Tracks emitter — see STU-2247.
+	if ( created ) {
+		await recordTracksEvent( TRACKS_EVENTS.CODE_SESSION_CREATED, {
+			...getAiTracksIdentity( summary.id ),
+			has_site: Boolean( server ),
+		} );
+	}
+
+	return summary;
 }
 
 export async function updateAiSessionMetadata(
@@ -398,16 +409,8 @@ async function reconcileSessionEnvironmentBeforeRun( sessionId: string ): Promis
 // instruction the agent actually acts on. Mirrors the CLI's interactive main
 // loop so UI clients can send the short form and get the same behaviour.
 function expandSkillCommandPrompt( prompt: string ): string {
-	const trimmed = prompt.trim();
-	if ( ! trimmed.startsWith( '/' ) ) {
-		return prompt;
-	}
-	const name = trimmed.slice( 1 );
-	const match = getAiSkillCommands().find( ( cmd ) => cmd.name === name );
-	if ( ! match ) {
-		return prompt;
-	}
-	return buildSkillInvocationPrompt( name );
+	const name = resolveSkillFromPrompt( prompt );
+	return name ? buildSkillInvocationPrompt( name ) : prompt;
 }
 
 export async function continueAiSession(

apps/studio/src/modules/ai-agent/run-manager.ts

@@ -1,4 +1,5 @@
 import { createAgentRunManager } from '@studio/common/ai/run-manager';
+import { getTracksOriginEnv } from 'src/modules/cli/lib/execute-command';
 import { getBundledNodeBinaryPath, getCliPath } from 'src/storage/paths';
 import type { ActiveAgentRun } from '@studio/common/ai/agent-events';
 import type { StudioChatFileAttachment } from '@studio/common/ai/chat-files';
@@ -23,6 +24,8 @@ const runManager = createAgentRunManager( {
 	cliBinary: getCliPath(),
 	nodeBinary: getBundledNodeBinaryPath(),
 	surface: 'desktop',
+	// Resolved per run; without it the CLI child attributes desktop chat to `channel: studio-cli`.
+	getTracksOrigin: getTracksOriginEnv,
 	emit: ( output ) => {
 		const webContents = runWebContents.get( output.runId );
 		if ( webContents && ! webContents.isDestroyed() ) {

apps/studio/src/modules/cli/lib/execute-command.ts

@@ -8,7 +8,8 @@ import { getBundledNodeBinaryPath, getCliPath } from 'src/storage/paths';
 
 // Origin tag passed to every app-spawned CLI process so its Tracks events are attributed to the
 // active desktop renderer (v1 = legacy, v2 = agentic). Read by the CLI in `apps/cli/lib/tracks.ts`.
-function getTracksOriginEnv(): string {
+// Also used by agent runs, which fork the CLI through `ai/run-manager.ts` instead.
+export function getTracksOriginEnv(): string {
 	return `studio-ui:${ getPreferredUiVersion() }`;
 }
 

apps/studio/src/modules/cli/lib/tests/cli-site-creator.test.ts

@@ -8,6 +8,7 @@ import { createSiteViaCli } from '../cli-site-creator';
 
 vi.mock( 'src/modules/cli/lib/execute-command', () => ( {
 	executeCliCommand: vi.fn(),
+	getTracksOriginEnv: vi.fn( () => 'studio-ui:v1' ),
 } ) );
 
 vi.mock( 'src/ipc-utils', () => ( {

apps/studio/src/modules/user-settings/components/studio-code-tab.tsx

@@ -49,7 +49,10 @@ export function StudioCodeTab() {
 		setIsSaving( true );
 		setError( null );
 		try {
-			await getIpcApi().saveGlobalAgentInstructions( content );
+			// The button only enables while dirty, so every save here closes an edit session.
+			await getIpcApi().saveGlobalAgentInstructions( content, {
+				editSession: { previousContent: savedContent },
+			} );
 			setSavedContent( content );
 			setJustSaved( true );
 		} catch ( err ) {

apps/studio/src/modules/user-settings/lib/ipc-handlers.ts

@@ -3,6 +3,7 @@ import {
 	readGlobalInstructionsFile,
 	writeGlobalInstructions,
 } from '@studio/common/ai/global-instructions';
+import { type TracksInstructionsLengthBucket } from '@studio/common/lib/record-tracks-event';
 import {
 	isAnalyticsOptedOut,
 	readSharedConfig,
@@ -286,11 +287,36 @@ export async function getGlobalAgentInstructions(): Promise< string > {
 	return ( await readGlobalInstructionsFile() ) ?? '';
 }
 
+// Bucketed for `studio_setting_instructions_change`; the text itself is never sent.
+function getInstructionsLengthBucket( content: string ): TracksInstructionsLengthBucket {
+	const length = content.trim().length;
+	if ( length === 0 ) {
+		return 'empty';
+	}
+	if ( length <= 200 ) {
+		return 'short';
+	}
+	return length <= 1000 ? 'medium' : 'long';
+}
+
 export async function saveGlobalAgentInstructions(
 	_event: IpcMainInvokeEvent,
-	content: string
+	content: string,
+	// Set when this save ends an edit session. Only the renderer knows the value it started from,
+	// since the agentic UI autosaves on a debounce. Intermediate autosaves omit it.
+	options: { editSession?: { previousContent: string } } = {}
 ): Promise< void > {
 	await writeGlobalInstructions( content );
+
+	const previous = options.editSession?.previousContent;
+	if ( previous === undefined || previous === content ) {
+		return;
+	}
+	await recordTracksEvent( TRACKS_EVENTS.SETTING_INSTRUCTIONS_CHANGE, {
+		has_content: content.trim().length > 0,
+		length_bucket: getInstructionsLengthBucket( content ),
+		surface: 'settings',
+	} );
 }
 
 export function showUserSettings( event: IpcMainInvokeEvent, tabName?: UserSettingsTabName ) {