diff --git a/src/lib/actions/sandbox/rebuild-gpu-opt-out.ts b/src/lib/actions/sandbox/rebuild-gpu-opt-out.ts index 0814bdc5521..edf092ce6cb 100644 --- a/src/lib/actions/sandbox/rebuild-gpu-opt-out.ts +++ b/src/lib/actions/sandbox/rebuild-gpu-opt-out.ts @@ -17,6 +17,7 @@ import type { PreparedImageRebuildHandoff, } from "../../onboard/prepared-dcode-rebuild"; import type { + ProviderRecoveryReceipt, RebuildProviderReconfigureHandoff, RebuildRouteHandoff, } from "../../onboard/rebuild-route-handoff"; @@ -106,6 +107,7 @@ export type RebuildRecreateOnboardOpts = { preparedDcodeRebuild?: PreparedDcodeRebuildHandoff; rebuildRegistryInferenceRoute?: RebuildRouteHandoff; rebuildProviderReconfigure?: RebuildProviderReconfigureHandoff; + providerRecoveryReceipt?: ProviderRecoveryReceipt; preparedImageRebuild?: PreparedImageRebuildHandoff; autoYes: boolean; toolDisclosure: ToolDisclosure; diff --git a/src/lib/actions/sandbox/rebuild-preflight-target-phase.test.ts b/src/lib/actions/sandbox/rebuild-preflight-target-phase.test.ts new file mode 100644 index 00000000000..f846dd1e6ce --- /dev/null +++ b/src/lib/actions/sandbox/rebuild-preflight-target-phase.test.ts @@ -0,0 +1,55 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import type { + ProviderRecoveryReceipt, + RegistryInferenceRoute, +} from "../../onboard/rebuild-route-handoff"; +import { stageRegistryProviderRecoveryReceipt } from "./rebuild-preflight-target-phase"; + +const target = { + sandboxName: "alpha", + gatewayName: "nemoclaw", + provider: "compatible-endpoint", + model: "nvidia/model", +}; + +const registryRoute: RegistryInferenceRoute = { + provider: target.provider, + model: target.model, + endpointUrl: "https://inference.example.test/v1", + preferredInferenceApi: "openai-completions", + source: "registry", +}; + +describe("stageRegistryProviderRecoveryReceipt", () => { + it("leaves recovery authority absent without a registry-derived route", () => { + const recreateOptions: { providerRecoveryReceipt?: ProviderRecoveryReceipt } = {}; + + stageRegistryProviderRecoveryReceipt(recreateOptions, target, null, { + nonce: "nonce-without-route", + expiresAtMs: 1_000, + }); + + expect(recreateOptions).not.toHaveProperty("providerRecoveryReceipt"); + }); + + it("binds recovery authority to the captured registry route", () => { + const recreateOptions: { providerRecoveryReceipt?: ProviderRecoveryReceipt } = {}; + + stageRegistryProviderRecoveryReceipt(recreateOptions, target, registryRoute, { + nonce: "nonce-with-route", + expiresAtMs: 1_000, + }); + + expect(recreateOptions.providerRecoveryReceipt).toEqual({ + ...target, + route: registryRoute, + nonce: "nonce-with-route", + expiresAtMs: 1_000, + sessionId: null, + }); + }); +}); diff --git a/src/lib/actions/sandbox/rebuild-preflight-target-phase.ts b/src/lib/actions/sandbox/rebuild-preflight-target-phase.ts index de52c49d90a..0954ad42ef9 100644 --- a/src/lib/actions/sandbox/rebuild-preflight-target-phase.ts +++ b/src/lib/actions/sandbox/rebuild-preflight-target-phase.ts @@ -1,11 +1,18 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { randomUUID } from "node:crypto"; import { CLI_NAME } from "../../cli/branding"; import type { SandboxMessagingPlan } from "../../messaging"; import { isSandboxBaseImageRefreshRequested } from "../../onboard/base-image-resolution-flow"; import type { DcodeAutoApprovalMode } from "../../onboard/dcode-auto-approval"; -import { createRebuildProviderReconfigureHandoff } from "../../onboard/rebuild-route-handoff"; + +import { + createRebuildProviderReconfigureHandoff, + mintProviderRecoveryReceipt, + type ProviderRecoveryReceipt, + type RegistryInferenceRoute, +} from "../../onboard/rebuild-route-handoff"; import { readSandboxBaseImageResolutionMetadata } from "../../sandbox-base-image"; import * as registry from "../../state/registry"; import type { ToolDisclosure } from "../../tool-disclosure"; @@ -36,6 +43,31 @@ import { stageRebuildHermesDashboardConfig, } from "./rebuild-target-preflight"; +/** Upper bound on how long a minted provider-recovery receipt stays valid. */ +const PROVIDER_RECOVERY_RECEIPT_TTL_MS = 60 * 60 * 1000; + +/** Stage recovery authority only from a route captured from the registry. */ +export function stageRegistryProviderRecoveryReceipt( + recreateOptions: { providerRecoveryReceipt?: ProviderRecoveryReceipt }, + target: { + sandboxName: string; + gatewayName: string; + provider: string; + model: string; + }, + registryRoute: RegistryInferenceRoute | null, + minting?: { nonce: string; expiresAtMs: number }, +): void { + if (!registryRoute) return; + recreateOptions.providerRecoveryReceipt = mintProviderRecoveryReceipt( + { ...target, route: registryRoute }, + minting ?? { + nonce: randomUUID(), + expiresAtMs: Date.now() + PROVIDER_RECOVERY_RECEIPT_TTL_MS, + }, + ); +} + export interface RebuildPreparedTarget { targetConfig: RebuildTargetConfig; recreateOptions: RebuildRecreateOnboardOpts; @@ -150,6 +182,16 @@ export async function prepareRebuildTargetPreflights(args: { ) { return null; } + stageRegistryProviderRecoveryReceipt( + recreateOptions, + { + sandboxName, + gatewayName: recreateOptions.targetGatewayName, + provider: resumeConfig.provider, + model: resumeConfig.model, + }, + resumeConfig.registryInferenceRoute, + ); if (!(await ensureRebuildTargetGatewaySelected(sandboxName, sandboxEntry, log, bail))) return null; if (!checkRebuildGatewaySchemaPreflight(sandboxName, sandboxEntry, bail)) return null; diff --git a/src/lib/onboard/authoritative-rebuild-target.test.ts b/src/lib/onboard/authoritative-rebuild-target.test.ts index d8ff4f1bac3..ead576b482d 100644 --- a/src/lib/onboard/authoritative-rebuild-target.test.ts +++ b/src/lib/onboard/authoritative-rebuild-target.test.ts @@ -9,6 +9,10 @@ import { rebuildProviderFlowOptions, resolveAuthoritativeOnboardGatewayBinding, } from "./authoritative-rebuild-target"; +import { + mintProviderRecoveryReceipt, + type ProviderRecoveryReceiptTarget, +} from "./rebuild-route-handoff"; const target = { sandboxName: "alpha", @@ -107,11 +111,11 @@ describe("prepared provider reconfiguration handoff", () => { }; it("accepts an exact handoff only for a locked authoritative rebuild resume (#6114)", () => { - expect(rebuildProviderFlowOptions(authorizedOptions, providerTarget)).toEqual({ + expect(rebuildProviderFlowOptions(authorizedOptions, providerTarget)).toMatchObject({ authoritativeResumeConfig: true, forceInferenceSetup: true, }); - expect(rebuildProviderFlowOptions({}, providerTarget)).toEqual({ + expect(rebuildProviderFlowOptions({}, providerTarget)).toMatchObject({ authoritativeResumeConfig: false, forceInferenceSetup: false, }); @@ -119,7 +123,7 @@ describe("prepared provider reconfiguration handoff", () => { it("authorizes incomplete-session recovery only for the locked rebuild context", () => { const recoveryOptions = { ...authorizedOptions, rebuildProviderReconfigure: undefined }; - expect(rebuildProviderFlowOptions(recoveryOptions, providerTarget)).toEqual({ + expect(rebuildProviderFlowOptions(recoveryOptions, providerTarget)).toMatchObject({ authoritativeResumeConfig: true, forceInferenceSetup: false, }); @@ -134,6 +138,47 @@ describe("prepared provider reconfiguration handoff", () => { } }); + it("activates a matching provider-recovery receipt and binds it to the session", () => { + const receiptTarget: ProviderRecoveryReceiptTarget = { + sandboxName: "alpha", + gatewayName: "nemoclaw-8081", + provider: "compatible-endpoint", + model: "nvidia/model", + route: { + provider: "compatible-endpoint", + model: "nvidia/model", + endpointUrl: "https://inference.example.test/v1", + preferredInferenceApi: "openai-completions", + source: "registry", + }, + }; + const receipt = mintProviderRecoveryReceipt(receiptTarget, { + nonce: "n-alpha", + expiresAtMs: Number.MAX_SAFE_INTEGER, + }); + const flowContext = { + ...providerTarget, + preferredInferenceApi: "openai-completions", + session: { sessionId: "sess-alpha" }, + }; + + const activated = rebuildProviderFlowOptions( + { ...authorizedOptions, providerRecoveryReceipt: receipt }, + flowContext, + ); + expect(activated.providerRecoveryReceipt?.sessionId).toBe("sess-alpha"); + + const wrongSandbox = rebuildProviderFlowOptions( + { + ...authorizedOptions, + providerRecoveryReceipt: receipt, + rebuildProviderReconfigure: undefined, + }, + { ...flowContext, sandboxName: "beta" }, + ); + expect(wrongSandbox.providerRecoveryReceipt).toBeNull(); + }); + it("rejects an unauthorized or mismatched handoff (#6114)", () => { expect(() => rebuildProviderFlowOptions( diff --git a/src/lib/onboard/authoritative-rebuild-target.ts b/src/lib/onboard/authoritative-rebuild-target.ts index 6dca8bdc685..402f50ec54a 100644 --- a/src/lib/onboard/authoritative-rebuild-target.ts +++ b/src/lib/onboard/authoritative-rebuild-target.ts @@ -5,7 +5,11 @@ import { findDashboardForwardOwner } from "./dashboard-port"; import { resolveGatewayName } from "./gateway-binding"; import type { PortProbeResult } from "./preflight"; import { assertDashboardPortNotReserved } from "./preflight-ports"; -import { validateRebuildProviderReconfigureHandoff } from "./rebuild-route-handoff"; +import { + createProviderRecoveryReceiptLedger, + type ProviderRecoveryReceipt, + validateRebuildProviderReconfigureHandoff, +} from "./rebuild-route-handoff"; import type { OnboardOptions } from "./types"; export type AuthoritativeOnboardGatewayBinding = { name: string; port: number }; @@ -110,9 +114,19 @@ function validateRebuildHandoff( /** Derive the provider-phase authority from one validated rebuild handoff. */ export function rebuildProviderFlowOptions( opts: OnboardOptions, - target: Parameters[1], -): { authoritativeResumeConfig: boolean; forceInferenceSetup: boolean } { + target: Parameters[1] & { + session?: { sessionId: string } | null; + preferredInferenceApi?: string | null; + }, +): { + authoritativeResumeConfig: boolean; + forceInferenceSetup: boolean; + providerRecoveryReceipt: ProviderRecoveryReceipt | null; + providerRecoveryReceiptLedger: ReturnType; +} { const authoritativeResumeConfig = opts.authoritativeResumeConfig === true; + const providerRecoveryReceiptLedger = createProviderRecoveryReceiptLedger(); + let providerRecoveryReceipt: ProviderRecoveryReceipt | null = null; if (authoritativeResumeConfig) { const gateway = resolveAuthoritativeOnboardGatewayBinding(opts); if ( @@ -128,10 +142,35 @@ export function rebuildProviderFlowOptions( "Authoritative provider recovery requires a preflighted locked rebuild resume.", ); } + const sessionId = target.session?.sessionId ?? null; + if (opts.providerRecoveryReceipt && sessionId) { + providerRecoveryReceipt = providerRecoveryReceiptLedger.activate( + opts.providerRecoveryReceipt, + { + target: { + sandboxName: target.sandboxName, + gatewayName: gateway.name, + provider: target.provider, + model: target.model, + route: { + provider: target.provider, + model: target.model, + endpointUrl: target.endpointUrl ?? null, + preferredInferenceApi: target.preferredInferenceApi ?? "", + source: "registry", + }, + }, + sessionId, + nowMs: Date.now(), + }, + ); + } } return { authoritativeResumeConfig, forceInferenceSetup: validateRebuildHandoff(opts, target), + providerRecoveryReceipt, + providerRecoveryReceiptLedger, }; } diff --git a/src/lib/onboard/machine/core-flow-phases.ts b/src/lib/onboard/machine/core-flow-phases.ts index ae71800572d..ccf8e404add 100644 --- a/src/lib/onboard/machine/core-flow-phases.ts +++ b/src/lib/onboard/machine/core-flow-phases.ts @@ -3,6 +3,10 @@ import type { WebSearchConfig } from "../../inference/web-search"; import type { DcodeAutoApprovalMode } from "../dcode-auto-approval"; +import type { + createProviderRecoveryReceiptLedger, + ProviderRecoveryReceipt, +} from "../rebuild-route-handoff"; import { mergeProviderModelSelectedContext, mergeSandboxCreatedContext, @@ -33,6 +37,8 @@ export interface CoreOnboardFlowPhaseOptions< forceProviderSelection: boolean; forceInferenceSetup?: boolean; authoritativeResumeConfig?: boolean; + providerRecoveryReceipt?: ProviderRecoveryReceipt | null; + providerRecoveryReceiptLedger?: ReturnType; env: NodeJS.ProcessEnv; constants: ProviderInferenceStateOptions["constants"]; providerDeps: ProviderInferenceStateOptions["deps"]; @@ -75,6 +81,8 @@ export function createCoreOnboardFlowPhases< forceProviderSelection: options.forceProviderSelection, forceInferenceSetup: options.forceInferenceSetup, authoritativeResumeConfig: options.authoritativeResumeConfig, + providerRecoveryReceipt: options.providerRecoveryReceipt, + providerRecoveryReceiptLedger: options.providerRecoveryReceiptLedger, initial: { model: context.model, provider: context.provider, diff --git a/src/lib/onboard/machine/handlers/provider-inference-authoritative-recovery.test.ts b/src/lib/onboard/machine/handlers/provider-inference-authoritative-recovery.test.ts index b9b746d5126..2318ff247f7 100644 --- a/src/lib/onboard/machine/handlers/provider-inference-authoritative-recovery.test.ts +++ b/src/lib/onboard/machine/handlers/provider-inference-authoritative-recovery.test.ts @@ -5,7 +5,12 @@ import { describe, expect, it, vi } from "vitest"; import { createSession } from "../../../state/onboard-session"; import { handleProviderInferenceState } from "./provider-inference"; -import { baseOptions, baseSelection, createDeps } from "./provider-inference.test-support"; +import { + activatedRecoveryReceipt, + baseOptions, + baseSelection, + createDeps, +} from "./provider-inference.test-support"; describe("authoritative provider inference recovery", () => { it("stays enabled across messaging revalidation", async () => { @@ -35,10 +40,17 @@ describe("authoritative provider inference recovery", () => { isInferenceRouteReady: vi.fn(() => true), }); + const { receipt, ledger } = activatedRecoveryReceipt({ + sandboxName: "my-assistant", + sessionId: session.sessionId, + }); + const result = await handleProviderInferenceState({ ...baseOptions(deps, session), resume: true, authoritativeResumeConfig: true, + providerRecoveryReceipt: receipt, + providerRecoveryReceiptLedger: ledger, sandboxName: "my-assistant", selectedMessagingChannels: ["telegram"], }); diff --git a/src/lib/onboard/machine/handlers/provider-inference-recovery-gating.test.ts b/src/lib/onboard/machine/handlers/provider-inference-recovery-gating.test.ts index 2de32e5c4c4..889e7a37bd7 100644 --- a/src/lib/onboard/machine/handlers/provider-inference-recovery-gating.test.ts +++ b/src/lib/onboard/machine/handlers/provider-inference-recovery-gating.test.ts @@ -10,7 +10,22 @@ import { createSession } from "../../../state/onboard-session"; import type { SandboxEntry } from "../../../state/registry"; import { classifySandboxRecoveryAuthority } from "../../provider-recovery"; import { handleProviderInferenceState } from "./provider-inference"; -import { baseOptions, createDeps } from "./provider-inference.test-support"; +import { + activatedRecoveryReceipt, + baseOptions, + baseSelection, + createDeps, +} from "./provider-inference.test-support"; + +function captureInLockRecheck() { + let recheck: (() => boolean) | undefined; + const setupInference = vi.fn(async (...args: unknown[]) => { + recheck = (args[7] as { isRecordedProviderRecoveryAuthorized?: () => boolean } | undefined) + ?.isRecordedProviderRecoveryAuthorized; + return { ok: true as const }; + }); + return { setupInference, getRecheck: () => recheck }; +} describe("provider inference recovery gating", () => { it.each([ @@ -69,12 +84,18 @@ describe("provider inference recovery gating", () => { }); session.sandboxName = "dc-after"; const { deps, calls } = createDeps(); + const { receipt, ledger } = activatedRecoveryReceipt({ + sandboxName: "dc-after", + sessionId: session.sessionId, + }); await handleProviderInferenceState({ ...baseOptions(deps, session), resume: true, forceProviderSelection: true, authoritativeResumeConfig: true, + providerRecoveryReceipt: receipt, + providerRecoveryReceiptLedger: ledger, sandboxName: "dc-after", }); @@ -406,4 +427,88 @@ describe("provider inference recovery gating", () => { session.sessionId, ); }); + + it("lets only the reservation owner recover the same sandbox concurrently", async () => { + const owner = createSession(); + const intruder = createSession(); + owner.sandboxName = "dc-after"; + intruder.sandboxName = "dc-after"; + const entry: SandboxEntry = { + name: "dc-after", + pendingRouteReservation: true, + reservationSessionId: owner.sessionId, + }; + const getAuthority = vi.fn((_name: string, sessionId: string | null | undefined) => + classifySandboxRecoveryAuthority(entry, sessionId), + ); + + async function recoverAs(session: typeof owner): Promise { + const { setupInference, getRecheck } = captureInLockRecheck(); + const { deps } = createDeps({ + getSandboxRecoveryAuthority: getAuthority, + setupNim: vi.fn(async () => ({ ...baseSelection, recoveredFromSandbox: true })), + setupInference, + }); + const { receipt, ledger } = activatedRecoveryReceipt({ + sandboxName: "dc-after", + sessionId: session.sessionId, + }); + await handleProviderInferenceState({ + ...baseOptions(deps, session), + resume: true, + forceProviderSelection: true, + authoritativeResumeConfig: true, + providerRecoveryReceipt: receipt, + providerRecoveryReceiptLedger: ledger, + sandboxName: "dc-after", + }); + return getRecheck()?.() ?? false; + } + + const [ownerAuthorized, intruderAuthorized] = await Promise.all([ + recoverAs(owner), + recoverAs(intruder), + ]); + + expect(ownerAuthorized).toBe(true); + expect(intruderAuthorized).toBe(false); + }); + + it("rejects a foreign reservation introduced after selection inside the mutation lock", async () => { + const session = createSession(); + session.sandboxName = "dc-after"; + const entry: SandboxEntry = { + name: "dc-after", + pendingRouteReservation: true, + reservationSessionId: session.sessionId, + }; + const getAuthority = vi.fn((_name: string, sessionId: string | null | undefined) => + classifySandboxRecoveryAuthority(entry, sessionId), + ); + const { setupInference, getRecheck } = captureInLockRecheck(); + const { deps } = createDeps({ + getSandboxRecoveryAuthority: getAuthority, + setupNim: vi.fn(async () => ({ ...baseSelection, recoveredFromSandbox: true })), + setupInference, + }); + const { receipt, ledger } = activatedRecoveryReceipt({ + sandboxName: "dc-after", + sessionId: session.sessionId, + }); + + await handleProviderInferenceState({ + ...baseOptions(deps, session), + resume: true, + forceProviderSelection: true, + authoritativeResumeConfig: true, + providerRecoveryReceipt: receipt, + providerRecoveryReceiptLedger: ledger, + sandboxName: "dc-after", + }); + + const recheck = getRecheck(); + expect(recheck?.()).toBe(true); + entry.reservationSessionId = "session-foreign"; + expect(recheck?.()).toBe(false); + }); }); diff --git a/src/lib/onboard/machine/handlers/provider-inference-recovery.ts b/src/lib/onboard/machine/handlers/provider-inference-recovery.ts index c5cba918bd1..0c0a735603b 100644 --- a/src/lib/onboard/machine/handlers/provider-inference-recovery.ts +++ b/src/lib/onboard/machine/handlers/provider-inference-recovery.ts @@ -2,6 +2,10 @@ // SPDX-License-Identifier: Apache-2.0 import type { Session } from "../../../state/onboard-session"; +import type { + createProviderRecoveryReceiptLedger, + ProviderRecoveryReceipt, +} from "../../rebuild-route-handoff"; import { type SandboxRecoveryAuthority, shouldRecoverRecordedProvider, @@ -9,6 +13,8 @@ import { export type RecoveryAuthority = SandboxRecoveryAuthority; +type ProviderRecoveryReceiptLedger = ReturnType; + interface ProviderRecoveryDeps { getSandboxRecoveryAuthority( sandboxName: string, @@ -23,13 +29,15 @@ interface ProviderRecoverySetupOptions { interface ProviderRecoveryOptions { /** - * Rebuild recreate replaces the durable session after deleting the old sandbox, so its sandbox - * step must remain incomplete until creation succeeds. The locked rebuild pipeline validates the - * target before deletion, then writes that exact identity into the pending session before onboard. - * Remove this exception once #6666 replaces the handoff with a dedicated provider-recovery - * authorization receipt. + * Authorization minted after locked rebuild preflight and activated against + * this onboard session. Its presence lets the recreate path recover the + * recorded provider while the pending sandbox step is still incomplete; the + * mutation-lock recheck below re-binds it to the live reservation owner. */ - authoritativeResumeConfig?: boolean; + recoveryReceipt?: ProviderRecoveryReceipt | null; + recoveryReceiptLedger?: ProviderRecoveryReceiptLedger; + gatewayName?: string; + now?: () => number; } export function createRecovery( @@ -48,6 +56,14 @@ export function createRecovery( ): ProviderRecoverySetupOptions; } { const sessionId = session?.sessionId ?? null; + const receipt = options.recoveryReceipt ?? null; + const receiptAuthorizesIncompleteSession = Boolean( + receipt && + receipt.sessionId && + receipt.sessionId === sessionId && + receipt.sandboxName === sandboxName, + ); + const now = options.now ?? (() => Date.now()); return { sessionId, shouldRecover: () => @@ -58,7 +74,7 @@ export function createRecovery( ? deps.getSandboxRecoveryAuthority(sandboxName, sessionId) : "missing", sessionSandboxName: - session?.steps?.sandbox?.status === "complete" || options.authoritativeResumeConfig + session?.steps?.sandbox?.status === "complete" || receiptAuthorizesIncompleteSession ? (session?.sandboxName ?? null) : null, }), @@ -66,8 +82,20 @@ export function createRecovery( if (!recoveredRecordedProvider) return { reservationSessionId: currentSessionId }; return { reservationSessionId: sessionId ?? undefined, - isRecordedProviderRecoveryAuthorized: () => - deps.getSandboxRecoveryAuthority(selectedSandboxName, sessionId) !== "unauthorized", + isRecordedProviderRecoveryAuthorized: () => { + const reservationOwned = + deps.getSandboxRecoveryAuthority(selectedSandboxName, sessionId) !== "unauthorized"; + if (!receipt || !options.recoveryReceiptLedger || !options.gatewayName) { + return reservationOwned; + } + return options.recoveryReceiptLedger.validateInLock(receipt, { + sandboxName: selectedSandboxName, + gatewayName: options.gatewayName, + sessionId, + nowMs: now(), + reservationOwned, + }); + }, }; }, }; diff --git a/src/lib/onboard/machine/handlers/provider-inference.test-support.ts b/src/lib/onboard/machine/handlers/provider-inference.test-support.ts index 97274c51229..17369dab01f 100644 --- a/src/lib/onboard/machine/handlers/provider-inference.test-support.ts +++ b/src/lib/onboard/machine/handlers/provider-inference.test-support.ts @@ -13,6 +13,11 @@ import type { CurrentGatewayRouteDiscoveryPreflight, } from "../../../inference/gateway-route-compatibility"; import { createSession, type Session, type SessionUpdates } from "../../../state/onboard-session"; +import { + createProviderRecoveryReceiptLedger, + mintProviderRecoveryReceipt, + type ProviderRecoveryReceipt, +} from "../../rebuild-route-handoff"; import type { ProviderInferenceStateOptions, ProviderSelectionResult } from "./provider-inference"; export type Gpu = { type: string } | null; @@ -31,6 +36,48 @@ export const baseSelection: ProviderSelectionResult = { nimContainer: null, }; +/** Mint and activate a provider-recovery receipt bound to one session, as the rebuild assembly would. */ +export function activatedRecoveryReceipt(input: { + sandboxName: string; + sessionId: string; + gatewayName?: string; + provider?: string; + model?: string; + endpointUrl?: string | null; + preferredInferenceApi?: string | null; + nowMs?: number; + ledger?: ReturnType; +}): { + receipt: ProviderRecoveryReceipt; + ledger: ReturnType; +} { + const gatewayName = input.gatewayName ?? "nemoclaw"; + const provider = input.provider ?? "compatible-endpoint"; + const model = input.model ?? "mock/channels-rebuild"; + const nowMs = input.nowMs ?? 0; + const target = { + sandboxName: input.sandboxName, + gatewayName, + provider, + model, + route: { + provider, + model, + endpointUrl: input.endpointUrl ?? "https://compatible.example.test/v1", + preferredInferenceApi: input.preferredInferenceApi ?? "openai-completions", + source: "registry" as const, + }, + }; + const ledger = input.ledger ?? createProviderRecoveryReceiptLedger(); + const minted = mintProviderRecoveryReceipt(target, { + nonce: `nonce-${input.sandboxName}-${input.sessionId}`, + expiresAtMs: Number.MAX_SAFE_INTEGER, + }); + const receipt = ledger.activate(minted, { target, sessionId: input.sessionId, nowMs }); + if (!receipt) throw new Error("test recovery receipt failed to activate"); + return { receipt, ledger }; +} + export function createDeps( overrides: Partial["deps"]> = {}, ) { diff --git a/src/lib/onboard/machine/handlers/provider-inference.test.ts b/src/lib/onboard/machine/handlers/provider-inference.test.ts index 5536a97b535..4c4e0133e5f 100644 --- a/src/lib/onboard/machine/handlers/provider-inference.test.ts +++ b/src/lib/onboard/machine/handlers/provider-inference.test.ts @@ -16,6 +16,7 @@ import { } from "./provider-inference"; import { type Agent, + activatedRecoveryReceipt, baseOptions, baseSelection, createDeps, @@ -769,7 +770,7 @@ describe("handleProviderInferenceState", () => { ); }); - it("revalidates recovered identity before reusing a gateway credential on messaging resume", async () => { + it("uses an activated receipt to recover identity before reusing a gateway credential on messaging resume", async () => { const session = createSession({ sandboxName: "my-assistant", provider: "compatible-endpoint", @@ -803,7 +804,6 @@ describe("handleProviderInferenceState", () => { healthChecks: [], }, }); - session.steps.provider_selection.status = "complete"; const setupNim = vi.fn(async () => ({ ...baseSelection, model: "nvidia/nemotron", @@ -813,22 +813,41 @@ describe("handleProviderInferenceState", () => { preferredInferenceApi: "openai-completions", skipHostInferenceSmoke: true, reuseGatewayCredentialWithoutLocalKey: true, + recoveredFromSandbox: true, })); + let recoveryAuthorization: (() => boolean) | undefined; + const setupInference = vi.fn< + ProviderInferenceStateOptions["deps"]["setupInference"] + >(async (...args) => { + recoveryAuthorization = args[7]?.isRecordedProviderRecoveryAuthorized; + return { ok: true }; + }); const { deps, calls } = createDeps({ setupNim, + setupInference, hydrateCredentialEnv: vi.fn(() => null), isInferenceRouteReady: vi.fn(() => true), }); calls.complete.mockResolvedValue(session); + const { receipt, ledger } = activatedRecoveryReceipt({ + sandboxName: "my-assistant", + sessionId: session.sessionId, + provider: "compatible-endpoint", + model: "nvidia/nemotron", + endpointUrl: "https://integrate.api.nvidia.com/v1", + preferredInferenceApi: "openai-completions", + }); await handleProviderInferenceState({ ...baseOptions(deps, session), resume: true, - authoritativeResumeConfig: true, sandboxName: "my-assistant", + providerRecoveryReceipt: receipt, + providerRecoveryReceiptLedger: ledger, }); expect(setupNim).toHaveBeenCalledOnce(); + expect(calls.skipped).not.toHaveBeenCalledWith("provider_selection", expect.anything()); expect(setupNim).toHaveBeenCalledWith( { type: "nvidia" }, "my-assistant", @@ -839,7 +858,7 @@ describe("handleProviderInferenceState", () => { expect.any(Function), session.sessionId, ); - expect(calls.setupInference).toHaveBeenCalledWith( + expect(setupInference).toHaveBeenCalledWith( "my-assistant", "nvidia/nemotron", "compatible-endpoint", @@ -854,11 +873,10 @@ describe("handleProviderInferenceState", () => { reuseGatewayCredentialWithoutLocalKey: true, preferredInferenceApi: "openai-completions", reservationSessionId: session.sessionId, + isRecordedProviderRecoveryAuthorized: expect.any(Function), }, ); - expect(calls.log).toHaveBeenCalledWith( - " [resume] Revalidating recovered compatible-endpoint identity before reusing its gateway credential.", - ); + expect(recoveryAuthorization?.()).toBe(true); }); it("keeps the compatible-endpoint resume shortcut when no messaging channels are selected", async () => { diff --git a/src/lib/onboard/machine/handlers/provider-inference.ts b/src/lib/onboard/machine/handlers/provider-inference.ts index db794d2d0c3..689fda5c192 100644 --- a/src/lib/onboard/machine/handlers/provider-inference.ts +++ b/src/lib/onboard/machine/handlers/provider-inference.ts @@ -10,6 +10,10 @@ import type { } from "../../../inference/gateway-route-compatibility"; import type { WebSearchConfig } from "../../../inference/web-search"; import type { HermesAuthMethod, Session, SessionUpdates } from "../../../state/onboard-session"; +import type { + createProviderRecoveryReceiptLedger, + ProviderRecoveryReceipt, +} from "../../rebuild-route-handoff"; import { withInferenceTrace, withProviderSelectionTrace } from "../../tracing"; import { advanceTo, type OnboardStateTransitionResult, retryTo } from "../result"; import { createRecovery, type RecoveryAuthority } from "./provider-inference-recovery"; @@ -71,6 +75,9 @@ export interface ProviderInferenceStateOptions { forceInferenceSetup?: boolean; /** Trust the rebuild-preflighted session selection even if its old step marker is incomplete. */ authoritativeResumeConfig?: boolean; + /** One-shot authority, activated at selection, to recover a recorded provider during rebuild. */ + providerRecoveryReceipt?: ProviderRecoveryReceipt | null; + providerRecoveryReceiptLedger?: ReturnType; initial: { model: string | null; provider: string | null; @@ -295,6 +302,8 @@ export async function handleProviderInferenceState({ forceProviderSelection: initialForceProviderSelection = false, forceInferenceSetup: initialForceInferenceSetup = false, authoritativeResumeConfig = false, + providerRecoveryReceipt = null, + providerRecoveryReceiptLedger, initial, selectedMessagingChannels, env, @@ -345,7 +354,9 @@ export async function handleProviderInferenceState({ let forceInferenceSetup = initialForceInferenceSetup; let recoveredRecordedProvider = false; const providerRecovery = createRecovery(fresh, sandboxName, session, deps, { - authoritativeResumeConfig, + recoveryReceipt: providerRecoveryReceipt, + recoveryReceiptLedger: providerRecoveryReceiptLedger, + gatewayName, }); const resumeProviderSelection = !forceProviderSelection && diff --git a/src/lib/onboard/rebuild-route-handoff.test.ts b/src/lib/onboard/rebuild-route-handoff.test.ts index d99ed6f4d21..e6b6c31ac92 100644 --- a/src/lib/onboard/rebuild-route-handoff.test.ts +++ b/src/lib/onboard/rebuild-route-handoff.test.ts @@ -3,7 +3,13 @@ import { describe, expect, expectTypeOf, it } from "vitest"; -import { createRebuildRouteHandoff, type RegistryInferenceRoute } from "./rebuild-route-handoff"; +import { + createProviderRecoveryReceiptLedger, + createRebuildRouteHandoff, + mintProviderRecoveryReceipt, + type ProviderRecoveryReceiptTarget, + type RegistryInferenceRoute, +} from "./rebuild-route-handoff"; function registryRoute(): RegistryInferenceRoute { return { @@ -15,6 +21,19 @@ function registryRoute(): RegistryInferenceRoute { }; } +function receiptTarget( + overrides: Partial = {}, +): ProviderRecoveryReceiptTarget { + return { + sandboxName: "alpha", + gatewayName: "nemoclaw", + provider: "compatible-endpoint", + model: "nvidia/model", + route: registryRoute(), + ...overrides, + }; +} + describe("createRebuildRouteHandoff", () => { it("defensively copies and freezes the complete registry route", () => { const route = registryRoute(); @@ -41,3 +60,146 @@ describe("createRebuildRouteHandoff", () => { ); }); }); + +describe("mintProviderRecoveryReceipt", () => { + it("binds the target and freezes the route with an unbound session", () => { + const target = receiptTarget(); + const receipt = mintProviderRecoveryReceipt(target, { nonce: "n1", expiresAtMs: 1_000 }); + + expect(receipt).toEqual({ ...target, nonce: "n1", expiresAtMs: 1_000, sessionId: null }); + expect(Object.isFrozen(receipt)).toBe(true); + expect(Object.isFrozen(receipt.route)).toBe(true); + expect(receipt.route).not.toBe(target.route); + }); + + it("rejects incomplete targets and a non-registry route", () => { + expect(() => + mintProviderRecoveryReceipt(receiptTarget({ sandboxName: " " }), { + nonce: "n1", + expiresAtMs: 1_000, + }), + ).toThrow("Provider recovery receipt is incomplete"); + expect(() => + mintProviderRecoveryReceipt(receiptTarget(), { nonce: "", expiresAtMs: 1_000 }), + ).toThrow("Provider recovery receipt is incomplete"); + const sessionRoute = { + ...registryRoute(), + source: "session", + } as unknown as RegistryInferenceRoute; + expect(() => + mintProviderRecoveryReceipt(receiptTarget({ route: sessionRoute }), { + nonce: "n1", + expiresAtMs: 1_000, + }), + ).toThrow("Provider recovery receipt requires a registry-derived route"); + }); +}); + +describe("createProviderRecoveryReceiptLedger", () => { + const mint = (overrides: Partial = {}, nonce = "n1") => + mintProviderRecoveryReceipt(receiptTarget(overrides), { nonce, expiresAtMs: 1_000 }); + + it("activates a matching receipt and binds it to the session", () => { + const ledger = createProviderRecoveryReceiptLedger(); + const activated = ledger.activate(mint(), { + target: receiptTarget(), + sessionId: "sess-a", + nowMs: 500, + }); + + expect(activated?.sessionId).toBe("sess-a"); + }); + + it("refuses a replayed activation under a different session (one-shot)", () => { + const ledger = createProviderRecoveryReceiptLedger(); + const receipt = mint(); + expect( + ledger.activate(receipt, { target: receiptTarget(), sessionId: "sess-a", nowMs: 500 }), + ).not.toBeNull(); + expect( + ledger.activate(receipt, { target: receiptTarget(), sessionId: "sess-b", nowMs: 500 }), + ).toBeNull(); + }); + + it("refuses a replayed activation under the same session (one-shot)", () => { + const ledger = createProviderRecoveryReceiptLedger(); + const receipt = mint(); + const context = { target: receiptTarget(), sessionId: "sess-a", nowMs: 500 }; + + expect(ledger.activate(receipt, context)).not.toBeNull(); + expect(ledger.activate(receipt, context)).toBeNull(); + }); + + it("refuses activation for an expired, cross-sandbox, or mismatched-route target", () => { + const ledger = createProviderRecoveryReceiptLedger(); + expect( + ledger.activate(mint({}, "expired"), { + target: receiptTarget(), + sessionId: "sess-a", + nowMs: 2_000, + }), + ).toBeNull(); + expect( + ledger.activate(mint({}, "cross"), { + target: receiptTarget({ sandboxName: "beta" }), + sessionId: "sess-a", + nowMs: 500, + }), + ).toBeNull(); + expect( + ledger.activate(mint({}, "route"), { + target: receiptTarget({ route: { ...registryRoute(), model: "other/model" } }), + sessionId: "sess-a", + nowMs: 500, + }), + ).toBeNull(); + expect( + ledger.activate(mint({}, "api"), { + target: receiptTarget({ + route: { ...registryRoute(), preferredInferenceApi: "openai-responses" }, + }), + sessionId: "sess-a", + nowMs: 500, + }), + ).toBeNull(); + expect( + ledger.activate(null, { target: receiptTarget(), sessionId: "sess-a", nowMs: 500 }), + ).toBeNull(); + }); + + it("passes an in-lock recheck only for the owning session with a held reservation", () => { + const ledger = createProviderRecoveryReceiptLedger(); + const receipt = mint(); + ledger.activate(receipt, { target: receiptTarget(), sessionId: "sess-a", nowMs: 500 }); + const activated = { ...receipt, sessionId: "sess-a" }; + const base = { + sandboxName: "alpha", + gatewayName: "nemoclaw", + sessionId: "sess-a", + nowMs: 500, + reservationOwned: true, + }; + + expect(ledger.validateInLock(activated, base)).toBe(true); + expect(ledger.validateInLock(activated, { ...base, reservationOwned: false })).toBe(false); + expect(ledger.validateInLock(activated, { ...base, sessionId: "sess-b" })).toBe(false); + expect(ledger.validateInLock(activated, { ...base, sandboxName: "beta" })).toBe(false); + expect(ledger.validateInLock(activated, { ...base, nowMs: 2_000 })).toBe(false); + expect(ledger.validateInLock({ ...activated, sessionId: null }, base)).toBe(false); + }); + + it("rejects an in-lock recheck for a receipt that was never activated", () => { + const ledger = createProviderRecoveryReceiptLedger(); + const receipt = { ...mint(), sessionId: "sess-a" }; + + expect( + ledger.validateInLock(receipt, { + sandboxName: "alpha", + gatewayName: "nemoclaw", + sessionId: "sess-a", + nowMs: 500, + reservationOwned: true, + }), + ).toBe(false); + }); +}); diff --git a/src/lib/onboard/rebuild-route-handoff.ts b/src/lib/onboard/rebuild-route-handoff.ts index 977c4af7a6c..68aca9b6226 100644 --- a/src/lib/onboard/rebuild-route-handoff.ts +++ b/src/lib/onboard/rebuild-route-handoff.ts @@ -77,3 +77,158 @@ export function validateRebuildProviderReconfigureHandoff( } return true; } + +/** Exact rebuild identity a provider-recovery receipt is bound to at preflight. */ +export type ProviderRecoveryReceiptTarget = Readonly<{ + sandboxName: string; + gatewayName: string; + provider: string; + model: string; + route: RegistryInferenceRoute; +}>; + +/** + * One-shot authority to recover a recorded provider for an authoritative locked + * rebuild. Minted after preflight validates the target, activated against the + * live onboard session at provider selection, then rechecked inside the sandbox + * and gateway mutation locks. `sessionId` is null until activation binds it. + */ +export type ProviderRecoveryReceipt = Readonly<{ + sandboxName: string; + gatewayName: string; + provider: string; + model: string; + route: RegistryInferenceRoute; + nonce: string; + expiresAtMs: number; + sessionId: string | null; +}>; + +function freezeRoute(route: RegistryInferenceRoute): RegistryInferenceRoute { + if (route.source !== "registry") { + throw new TypeError("Provider recovery receipt requires a registry-derived route"); + } + return Object.freeze({ + provider: route.provider, + model: route.model, + endpointUrl: route.endpointUrl, + preferredInferenceApi: route.preferredInferenceApi, + source: "registry", + }); +} + +function routesMatch(left: RegistryInferenceRoute, right: RegistryInferenceRoute): boolean { + return ( + left.provider === right.provider && + left.model === right.model && + left.endpointUrl === right.endpointUrl && + left.preferredInferenceApi === right.preferredInferenceApi + ); +} + +function receiptMatchesTarget( + receipt: ProviderRecoveryReceipt, + target: ProviderRecoveryReceiptTarget, +): boolean { + return ( + receipt.sandboxName === target.sandboxName && + receipt.gatewayName === target.gatewayName && + receipt.provider === target.provider && + receipt.model === target.model && + routesMatch(receipt.route, target.route) + ); +} + +/** Mint a target-bound, time-boxed recovery receipt after preflight validation. */ +export function mintProviderRecoveryReceipt( + target: ProviderRecoveryReceiptTarget, + minting: { nonce: string; expiresAtMs: number }, +): ProviderRecoveryReceipt { + if ( + !target.sandboxName.trim() || + !target.gatewayName.trim() || + !target.provider.trim() || + !target.model.trim() || + !minting.nonce.trim() || + !Number.isFinite(minting.expiresAtMs) + ) { + throw new TypeError("Provider recovery receipt is incomplete"); + } + return Object.freeze({ + sandboxName: target.sandboxName, + gatewayName: target.gatewayName, + provider: target.provider, + model: target.model, + route: freezeRoute(target.route), + nonce: minting.nonce, + expiresAtMs: minting.expiresAtMs, + sessionId: null, + }); +} + +function receiptIsWellFormed( + receipt: ProviderRecoveryReceipt | null | undefined, +): receipt is ProviderRecoveryReceipt { + return Boolean( + receipt && + typeof receipt.sandboxName === "string" && + receipt.sandboxName && + typeof receipt.gatewayName === "string" && + receipt.gatewayName && + typeof receipt.provider === "string" && + receipt.provider && + typeof receipt.model === "string" && + receipt.model && + typeof receipt.nonce === "string" && + receipt.nonce && + Number.isFinite(receipt.expiresAtMs) && + receipt.route?.source === "registry", + ); +} + +/** + * Single-use ledger binding each minted receipt to exactly one onboard session + * at provider selection, then answering in-lock rechecks for that binding. The + * ledger is what makes recovery authorization one-shot: a second activation of + * the same nonce is a replay and is refused. + */ +export function createProviderRecoveryReceiptLedger(): { + activate( + receipt: ProviderRecoveryReceipt | null | undefined, + context: { target: ProviderRecoveryReceiptTarget; sessionId: string; nowMs: number }, + ): ProviderRecoveryReceipt | null; + validateInLock( + receipt: ProviderRecoveryReceipt | null | undefined, + check: { + sandboxName: string; + gatewayName: string; + sessionId: string | null | undefined; + nowMs: number; + reservationOwned: boolean; + }, + ): boolean; +} { + const activatedSessionByNonce = new Map(); + return { + activate(receipt, context) { + if (!receiptIsWellFormed(receipt)) return null; + if (!context.sessionId) return null; + if (context.nowMs > receipt.expiresAtMs) return null; + if (!receiptMatchesTarget(receipt, context.target)) return null; + if (activatedSessionByNonce.has(receipt.nonce)) return null; + activatedSessionByNonce.set(receipt.nonce, context.sessionId); + return Object.freeze({ ...receipt, sessionId: context.sessionId }); + }, + validateInLock(receipt, check) { + if (!receiptIsWellFormed(receipt)) return false; + if (!check.reservationOwned) return false; + if (!check.sessionId || receipt.sessionId !== check.sessionId) return false; + if (activatedSessionByNonce.get(receipt.nonce) !== check.sessionId) return false; + if (check.nowMs > receipt.expiresAtMs) return false; + if (receipt.sandboxName !== check.sandboxName || receipt.gatewayName !== check.gatewayName) { + return false; + } + return true; + }, + }; +} diff --git a/src/lib/onboard/types.ts b/src/lib/onboard/types.ts index 5c52fbb2ce8..3455d272336 100644 --- a/src/lib/onboard/types.ts +++ b/src/lib/onboard/types.ts @@ -84,6 +84,8 @@ export type OnboardOptions = { rebuildRegistryInferenceRoute?: import("./rebuild-route-handoff").RebuildRouteHandoff | null; /** Internal one-shot authority to upsert a provider observed missing during rebuild preflight. */ rebuildProviderReconfigure?: import("./rebuild-route-handoff").RebuildProviderReconfigureHandoff; + /** Internal one-shot authority to recover the recorded provider during a locked rebuild resume. */ + providerRecoveryReceipt?: import("./rebuild-route-handoff").ProviderRecoveryReceipt; /** Internal one-shot handoff for the exact image context validated before rebuild deletion. */ preparedImageRebuild?: import("./prepared-dcode-rebuild").PreparedImageRebuildHandoff; resume?: boolean;