diff --git a/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx b/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx index 31ded8ba9d8..8fea4c64981 100644 --- a/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx +++ b/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx @@ -103,7 +103,11 @@ The rebuild command preserves the mounted workspace and registered policies whil -The rebuild command preserves Hermes state and registered policies while recreating the container. +The rebuild command preserves Hermes state, registered policies, and managed MCP configuration while recreating the container. +Before post-restore repairs, NemoClaw verifies that the recreated sandbox still identifies as Hermes and exits nonzero if its identity does not match the rebuild target. +After state restore, NemoClaw restores managed MCP configuration through the normal lifecycle, then re-proves or recovers gateway health and performs final MCP reconciliation. +`rebuild` exits nonzero instead of reporting success when it cannot verify final gateway health or managed MCP state. +Follow the printed recovery guidance, using `$$nemoclaw recover` for gateway health and `$$nemoclaw mcp restart` for incomplete managed MCP restoration. diff --git a/src/lib/actions/sandbox/rebuild-hermes-post-restore.test.ts b/src/lib/actions/sandbox/rebuild-hermes-post-restore.test.ts new file mode 100644 index 00000000000..e6aba209276 --- /dev/null +++ b/src/lib/actions/sandbox/rebuild-hermes-post-restore.test.ts @@ -0,0 +1,223 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { + createRebuildFlowHarness, + resetRebuildFlowTestEnvironment, + restoreRebuildFlowTestEnvironment, +} from "../../../../test/helpers/rebuild-flow-harness"; + +describe("Hermes rebuild post-restore verification", () => { + beforeEach(resetRebuildFlowTestEnvironment); + afterEach(restoreRebuildFlowTestEnvironment); + + it("fails instead of reporting readiness when restored state leaves the gateway down (#7084)", async () => { + const mcpEntry = { + server: "blender", + providerName: "nemoclaw-mcp-alpha-blender", + }; + const harness = createRebuildFlowHarness({ + agentName: "hermes", + checkAndRecoverSandboxProcesses: () => ({ + checked: true, + wasRunning: false, + recovered: false, + forwardRecovered: false, + }), + mcpPreparation: { + entries: [mcpEntry], + detachedProviderEntries: [mcpEntry], + scrubbedAdapterEntries: [mcpEntry], + }, + sandboxEntry: { agent: "hermes" }, + }); + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).rejects.toThrow("Hermes post-restore verification failed"); + + const output = harness.logSpy.mock.calls.map((call) => String(call[0])).join("\n"); + expect(output).toContain("rebuilt but some post-restore steps were incomplete"); + expect(output).toContain("Hermes gateway health was not verified after state restore"); + expect(output).not.toContain("MCP bridge definitions were preserved but not fully refreshed"); + expect(output).not.toContain("rebuilt successfully"); + expect(harness.restoreMcpBridgesAfterRebuildSpy).toHaveBeenCalledWith("alpha", [mcpEntry]); + }); + + it("accepts restored MCP configuration only after final gateway recovery (#7084)", async () => { + const mcpEntry = { + server: "blender", + providerName: "nemoclaw-mcp-alpha-blender", + }; + const harness = createRebuildFlowHarness({ + agentName: "hermes", + checkAndRecoverSandboxProcesses: () => ({ + checked: true, + wasRunning: false, + recovered: true, + forwardRecovered: true, + }), + mcpPreparation: { + entries: [mcpEntry], + detachedProviderEntries: [mcpEntry], + scrubbedAdapterEntries: [mcpEntry], + }, + sandboxEntry: { agent: "hermes" }, + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).resolves.toBeUndefined(); + + expect(harness.restoreMcpBridgesAfterRebuildSpy).toHaveBeenCalledWith("alpha", [mcpEntry]); + expect(harness.restoreMcpBridgesAfterRebuildSpy.mock.invocationCallOrder[0]).toBeLessThan( + harness.checkAndRecoverSandboxProcessesSpy.mock.invocationCallOrder[0], + ); + expect(harness.logSpy).toHaveBeenCalledWith( + expect.stringContaining("Hermes gateway recovered after state restore"), + ); + }); + + it("returns a failed rebuild when managed Hermes MCP restoration is incomplete (#7084)", async () => { + const mcpEntry = { + server: "blender", + providerName: "nemoclaw-mcp-alpha-blender", + }; + const harness = createRebuildFlowHarness({ + agentName: "hermes", + mcpPreparation: { + entries: [mcpEntry], + detachedProviderEntries: [mcpEntry], + scrubbedAdapterEntries: [mcpEntry], + }, + sandboxEntry: { agent: "hermes" }, + }); + harness.restoreMcpBridgesAfterRebuildSpy.mockRejectedValueOnce(new Error("reload failed")); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).rejects.toThrow("Hermes post-restore verification failed"); + + const output = harness.logSpy.mock.calls.map((call) => String(call[0])).join("\n"); + expect(output).toContain("MCP bridge definitions were preserved but not fully refreshed"); + expect(output).not.toContain("rebuilt successfully"); + }); + + it("fails when the final gateway check refuses MCP reconciliation (#7084)", async () => { + const mcpEntry = { + server: "blender", + providerName: "nemoclaw-mcp-alpha-blender", + }; + const harness = createRebuildFlowHarness({ + agentName: "hermes", + checkAndRecoverSandboxProcesses: () => ({ + checked: true, + wasRunning: true, + recovered: false, + forwardRecovered: false, + mcpReconciliationRefused: true, + }), + mcpPreparation: { + entries: [mcpEntry], + detachedProviderEntries: [mcpEntry], + scrubbedAdapterEntries: [mcpEntry], + }, + sandboxEntry: { agent: "hermes" }, + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).rejects.toThrow("Hermes post-restore verification failed"); + + expect(harness.restoreMcpBridgesAfterRebuildSpy).toHaveBeenCalledWith("alpha", [mcpEntry]); + expect(harness.logSpy).not.toHaveBeenCalledWith( + expect.stringContaining("rebuilt successfully"), + ); + }); + + it.each([ + "forwardRecoveryFailed", + "secretBoundaryRefused", + ] as const)("fails when the final gateway check reports %s (#7084)", async (failureFlag) => { + const harness = createRebuildFlowHarness({ + agentName: "hermes", + checkAndRecoverSandboxProcesses: () => ({ + checked: true, + wasRunning: true, + recovered: false, + forwardRecovered: false, + [failureFlag]: true, + }), + sandboxEntry: { agent: "hermes" }, + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).rejects.toThrow("Hermes post-restore verification failed"); + + expect(harness.logSpy).not.toHaveBeenCalledWith( + expect.stringContaining("rebuilt successfully"), + ); + }); + + it("fails when the final gateway health probe is unavailable (#7084)", async () => { + const harness = createRebuildFlowHarness({ + agentName: "hermes", + checkAndRecoverSandboxProcesses: () => ({ + checked: false, + wasRunning: null, + recovered: false, + forwardRecovered: false, + }), + sandboxEntry: { agent: "hermes" }, + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).rejects.toThrow("Hermes post-restore verification failed"); + + expect(harness.logSpy).not.toHaveBeenCalledWith( + expect.stringContaining("rebuilt successfully"), + ); + }); + + it("fails before recovery when recreated Hermes identity is missing (#7084)", async () => { + const harness = createRebuildFlowHarness({ + agentName: "hermes", + sessionAgentName: null, + sandboxEntry: { agent: "hermes" }, + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).rejects.toThrow( + "Recreated sandbox agent identity did not match the authoritative rebuild target", + ); + + expect(harness.checkAndRecoverSandboxProcessesSpy).not.toHaveBeenCalled(); + expect(harness.restoreMcpBridgesAfterRebuildSpy).not.toHaveBeenCalled(); + expect(harness.logSpy).not.toHaveBeenCalledWith( + expect.stringContaining("rebuilt successfully"), + ); + }); + + it("fails before recovery when recreated Hermes identity mismatches (#7084)", async () => { + const harness = createRebuildFlowHarness({ + agentName: "hermes", + sessionAgentName: "langchain-deepagents-code", + sandboxEntry: { agent: "hermes" }, + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).rejects.toThrow( + "Recreated sandbox agent identity did not match the authoritative rebuild target", + ); + + expect(harness.checkAndRecoverSandboxProcessesSpy).not.toHaveBeenCalled(); + expect(harness.restoreMcpBridgesAfterRebuildSpy).not.toHaveBeenCalled(); + expect(harness.logSpy).not.toHaveBeenCalledWith( + expect.stringContaining("rebuilt successfully"), + ); + }); +}); diff --git a/src/lib/actions/sandbox/rebuild-hermes-post-restore.ts b/src/lib/actions/sandbox/rebuild-hermes-post-restore.ts new file mode 100644 index 00000000000..5362f592a7c --- /dev/null +++ b/src/lib/actions/sandbox/rebuild-hermes-post-restore.ts @@ -0,0 +1,67 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { CLI_NAME } from "../../cli/branding"; +import * as processRecovery from "./process-recovery"; + +export type HermesPostRestoreGatewayState = + | "not-applicable" + | "healthy" + | "recovered" + | "unverified"; + +type GatewayRecoveryObservation = { + checked: boolean; + wasRunning: boolean | null; + recovered: boolean; + forwardRecoveryFailed?: boolean; + secretBoundaryRefused?: boolean; + mcpReconciliationRefused?: boolean; +}; + +interface HermesPostRestoreGatewayDeps { + checkAndRecoverSandboxProcesses?: ( + sandboxName: string, + options: { quiet: boolean }, + ) => GatewayRecoveryObservation; +} + +/** + * Re-prove Hermes gateway health after workspace state restoration. + * + * Inner onboarding verifies the fresh image before rebuild restores the prior + * state. That restore can still stop or wedge the gateway, so its earlier + * readiness message is not authoritative for rebuild completion. + */ +export function ensureHermesGatewayAfterStateRestore( + sandboxName: string, + agentName: string, + deps: HermesPostRestoreGatewayDeps = {}, +): HermesPostRestoreGatewayState { + if (agentName !== "hermes") return "not-applicable"; + const checkAndRecover = + deps.checkAndRecoverSandboxProcesses ?? processRecovery.checkAndRecoverSandboxProcesses; + const observation: GatewayRecoveryObservation = checkAndRecover(sandboxName, { quiet: true }); + if ( + !observation.checked || + observation.forwardRecoveryFailed === true || + observation.secretBoundaryRefused === true || + observation.mcpReconciliationRefused === true + ) { + return "unverified"; + } + if (observation.wasRunning === true) return "healthy"; + if (observation.recovered) return "recovered"; + return "unverified"; +} + +export function printHermesGatewayRestoreRecovery( + sandboxName: string, + state: HermesPostRestoreGatewayState, + writeLine: (message: string) => void = console.log, +): void { + if (state !== "unverified") return; + writeLine( + ` Hermes gateway health was not verified after state restore — run \`${CLI_NAME} ${sandboxName} recover\` before relying on this sandbox`, + ); +} diff --git a/src/lib/actions/sandbox/rebuild-mcp-phase.ts b/src/lib/actions/sandbox/rebuild-mcp-phase.ts index f24f93a407e..7c25568d3db 100644 --- a/src/lib/actions/sandbox/rebuild-mcp-phase.ts +++ b/src/lib/actions/sandbox/rebuild-mcp-phase.ts @@ -126,6 +126,7 @@ export async function restoreMcpAfterRebuild( } export function postRestoreCompleted(status: { + hermesGatewayRestoreUnverified: boolean; messagingHostForwardUnverified: boolean; mcpBridgeRestoreUnverified: boolean; mutableConfigHashRefreshUnverified: boolean; @@ -135,6 +136,7 @@ export function postRestoreCompleted(status: { }): boolean { return ( status.restoreSucceeded && + !status.hermesGatewayRestoreUnverified && !status.mutablePermsRepairUnverified && !status.mutableConfigHashRefreshUnverified && !status.messagingHostForwardUnverified && diff --git a/src/lib/actions/sandbox/rebuild-pipeline.ts b/src/lib/actions/sandbox/rebuild-pipeline.ts index c3c2a87042c..d2f4942f779 100644 --- a/src/lib/actions/sandbox/rebuild-pipeline.ts +++ b/src/lib/actions/sandbox/rebuild-pipeline.ts @@ -312,6 +312,7 @@ async function rebuildSandboxUnlocked( await runRebuildPostRestorePhase({ sandboxName, sandboxEntry, + targetAgentName: rebuildAgent || "openclaw", messagingPlan, backupManifest: backup.backupManifest, mcpEntries: mcpPreparation.entries, diff --git a/src/lib/actions/sandbox/rebuild-post-restore-phase.test.ts b/src/lib/actions/sandbox/rebuild-post-restore-phase.test.ts index 1d878cf04d9..377cd9a15b3 100644 --- a/src/lib/actions/sandbox/rebuild-post-restore-phase.test.ts +++ b/src/lib/actions/sandbox/rebuild-post-restore-phase.test.ts @@ -9,6 +9,7 @@ import * as registry from "../../state/registry"; import * as messagingHostForward from "./messaging-host-forward-lifecycle"; import * as processRecovery from "./process-recovery"; import * as rebuildConfigHash from "./rebuild-config-hash"; +import * as rebuildHermesPostRestore from "./rebuild-hermes-post-restore"; import * as rebuildMcp from "./rebuild-mcp-phase"; import * as rebuildMessaging from "./rebuild-messaging-phase"; import { runRebuildPostRestorePhase } from "./rebuild-post-restore-phase"; @@ -23,8 +24,8 @@ describe("rebuild post-restore session model reconciliation (#7102)", () => { order = []; vi.spyOn(console, "log").mockImplementation(() => undefined); vi.spyOn(console, "error").mockImplementation(() => undefined); - vi.spyOn(agentRuntime, "getSessionAgent").mockImplementation( - () => ({ name: agentName }) as never, + vi.spyOn(agentRuntime, "getSessionAgent").mockImplementation(() => + agentName === "openclaw" ? null : ({ name: agentName } as never), ); vi.spyOn(agentRuntime, "getAgentDisplayName").mockReturnValue("test agent"); vi.spyOn(agentDefs, "loadAgent").mockImplementation( @@ -57,6 +58,13 @@ describe("rebuild post-restore session model reconciliation (#7102)", () => { skipReason: "not-needed", } as never); vi.spyOn(rebuildMcp, "restoreMcpAfterRebuild").mockResolvedValue(true); + vi.spyOn(rebuildHermesPostRestore, "ensureHermesGatewayAfterStateRestore").mockImplementation( + (_sandboxName, targetAgentName) => + targetAgentName === "hermes" ? "healthy" : "not-applicable", + ); + vi.spyOn(registry, "getSandbox").mockImplementation( + () => ({ agent: agentName === "openclaw" ? null : agentName }) as never, + ); vi.spyOn(registry, "updateSandbox").mockReturnValue(true); vi.spyOn(messagingHostForward, "ensureMessagingHostForwardAfterRebuild").mockReturnValue(true); }); @@ -68,6 +76,7 @@ describe("rebuild post-restore session model reconciliation (#7102)", () => { function input() { return { sandboxName: "alpha", + targetAgentName: agentName, sandboxEntry: {} as never, messagingPlan: null, backupManifest: null, @@ -97,9 +106,11 @@ describe("rebuild post-restore session model reconciliation (#7102)", () => { it("does not run OpenClaw session reconciliation for another agent", async () => { agentName = "hermes"; + const args = input(); - await runRebuildPostRestorePhase(input()); + await runRebuildPostRestorePhase(args); + expect(args.bail).not.toHaveBeenCalled(); expect(sessionModels.reconcileStalePinnedSessionModelsAfterRebuild).not.toHaveBeenCalled(); expect(processRecovery.executeSandboxCommand).not.toHaveBeenCalled(); }); diff --git a/src/lib/actions/sandbox/rebuild-post-restore-phase.ts b/src/lib/actions/sandbox/rebuild-post-restore-phase.ts index 296c46576c5..32c8c02b5e4 100644 --- a/src/lib/actions/sandbox/rebuild-post-restore-phase.ts +++ b/src/lib/actions/sandbox/rebuild-post-restore-phase.ts @@ -16,6 +16,10 @@ import type { RebuildBackupManifest } from "./rebuild-backup-phase"; import { refreshMutableOpenClawConfigHashAfterPostRestoreWrites } from "./rebuild-config-hash"; import type { RebuildBail, RebuildLog } from "./rebuild-credential-preflight"; import type { RebuildSandboxEntry } from "./rebuild-flow-helpers"; +import { + ensureHermesGatewayAfterStateRestore, + printHermesGatewayRestoreRecovery, +} from "./rebuild-hermes-post-restore"; import { type McpRebuildPreparation, postRestoreCompleted, @@ -28,6 +32,7 @@ import { reconcileStalePinnedSessionModelsAfterRebuild } from "./reconcile-sessi export interface RebuildPostRestorePhaseInput { sandboxName: string; sandboxEntry: RebuildSandboxEntry; + targetAgentName: string; messagingPlan: SandboxMessagingPlan | null; backupManifest: RebuildBackupManifest; mcpEntries: McpRebuildPreparation["entries"]; @@ -103,6 +108,7 @@ export async function runRebuildPostRestorePhase( const { sandboxName, sandboxEntry: sb, + targetAgentName, messagingPlan, backupManifest, mcpEntries, @@ -121,9 +127,24 @@ export async function runRebuildPostRestorePhase( log, bail, } = input; - const rebuiltAgent = agentRuntime.getSessionAgent(sandboxName); - const rebuiltAgentName = agentRuntime.getAgentDisplayName(rebuiltAgent); - const agentDef = rebuiltAgent ? loadAgent(rebuiltAgent.name) : loadAgent("openclaw"); + const recreatedEntry = registry.getSandbox(sandboxName); + const recreatedAgent = agentRuntime.getSessionAgent(sandboxName); + // OpenClaw is represented by a null registry agent and a null runtime definition. + const recreatedRegistryAgentName = recreatedEntry?.agent ?? "openclaw"; + const recreatedRuntimeAgentName = recreatedAgent?.name ?? "openclaw"; + if ( + !recreatedEntry || + recreatedRegistryAgentName !== targetAgentName || + recreatedRuntimeAgentName !== targetAgentName + ) { + console.error( + ` ${YW}\u26a0${R} Recreated sandbox agent identity could not be verified against the rebuild target.`, + ); + bail("Recreated sandbox agent identity did not match the authoritative rebuild target."); + return; + } + const agentDef = loadAgent(targetAgentName); + const rebuiltAgentName = agentDef.displayName; let mutablePermsRepairUnverified = false; let mutableConfigHashRefreshUnverified = false; let messagingHostForwardUnverified = false; @@ -132,7 +153,7 @@ export async function runRebuildPostRestorePhase( failedPresetRemovals.length > 0 || !policyPresetReconciliationVerified; - if (agentDef.name === "openclaw") { + if (targetAgentName === "openclaw") { log("Running openclaw doctor --fix inside sandbox for post-upgrade structure repair"); const doctorResult = executeSandboxCommand(sandboxName, "openclaw doctor --fix"); log( @@ -188,6 +209,16 @@ export async function runRebuildPostRestorePhase( } const mcpBridgeRestoreUnverified = !(await restoreMcpAfterRebuild(sandboxName, mcpEntries)); + const hermesGatewayRestoreState = ensureHermesGatewayAfterStateRestore( + sandboxName, + targetAgentName, + ); + const hermesGatewayRestoreUnverified = hermesGatewayRestoreState === "unverified"; + if (hermesGatewayRestoreState === "healthy") { + console.log(` ${G}\u2713${R} Hermes gateway health verified after state restore`); + } else if (hermesGatewayRestoreState === "recovered") { + console.log(` ${G}\u2713${R} Hermes gateway recovered after state restore`); + } const { policies: restoredBuiltinPresets, policyPresetsFinalized } = resolveRestoredPolicyRegistryState( { @@ -217,6 +248,7 @@ export async function runRebuildPostRestorePhase( console.log(""); const postRestoreComplete = postRestoreCompleted({ + hermesGatewayRestoreUnverified, messagingHostForwardUnverified, mcpBridgeRestoreUnverified, mutableConfigHashRefreshUnverified, @@ -257,6 +289,7 @@ export async function runRebuildPostRestorePhase( ` Messaging webhook forward was not verified \u2014 run \`${CLI_NAME} ${sandboxName} connect\` after resolving the port conflict`, ); } + printHermesGatewayRestoreRecovery(sandboxName, hermesGatewayRestoreState); printMcpRestoreRecovery(sandboxName, mcpBridgeRestoreUnverified); if (policyPresetRestoreIncomplete) { if (failedPresets.length > 0) { @@ -280,6 +313,13 @@ export async function runRebuildPostRestorePhase( bail(`Rebuild completed with unverified live policy reconciliation for '${sandboxName}'.`); return; } + if ( + targetAgentName === "hermes" && + (hermesGatewayRestoreUnverified || mcpBridgeRestoreUnverified) + ) { + bail(`Hermes post-restore verification failed for '${sandboxName}'.`); + return; + } if (preparedBackupRecovery && !postRestoreComplete) { bail( `Prepared backup recovery for '${sandboxName}' completed with unverified post-restore state.`, diff --git a/test/helpers/rebuild-flow-harness.ts b/test/helpers/rebuild-flow-harness.ts index 1f225534c74..6a23fda807e 100644 --- a/test/helpers/rebuild-flow-harness.ts +++ b/test/helpers/rebuild-flow-harness.ts @@ -75,8 +75,18 @@ export type RebuildFlowSession = Record & { export type RebuildFlowOverrides = { agentName?: string; + sessionAgentName?: string | null; applyPreset?: (presetName: string) => boolean; executeSandboxCommand?: () => { status: number; stdout: string; stderr: string } | null; + checkAndRecoverSandboxProcesses?: () => { + checked: boolean; + wasRunning: boolean | null; + recovered: boolean; + forwardRecovered: boolean; + forwardRecoveryFailed?: boolean; + secretBoundaryRefused?: boolean; + mcpReconciliationRefused?: boolean; + }; onboard?: (session: RebuildFlowSession) => Promise | void; repairMutableConfigPerms?: () => | { applied: false; skipReason: "agent" | "locked" | "unreadable"; reason: string } @@ -132,6 +142,7 @@ export type RebuildFlowHarness = { errorSpy: MockInstance; ensureAgentBaseImageSpy: MockInstance; executeSandboxCommandSpy: MockInstance; + checkAndRecoverSandboxProcessesSpy: MockInstance; ensureMessagingHostForwardAfterRebuildSpy: MockInstance; logSpy: MockInstance; finalizeIncompleteOnboardStepSpy: MockInstance; @@ -295,9 +306,19 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): imageTag: `nemoclaw-${agentName}-base:test`, built: true, }); - vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue({ name: agentName }); + const sessionAgentName = + overrides.sessionAgentName === undefined ? agentName : overrides.sessionAgentName; + vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue( + sessionAgentName === null || sessionAgentName === "openclaw" + ? null + : ({ name: sessionAgentName } as never), + ); vi.spyOn(agentRuntime, "getAgentDisplayName").mockReturnValue( - agentName === "langchain-deepagents-code" ? "Deep Agents Code" : "OpenClaw", + agentName === "langchain-deepagents-code" + ? "Deep Agents Code" + : agentName === "hermes" + ? "Hermes Agent" + : "OpenClaw", ); vi.spyOn(gatewayRuntime, "recoverNamedGatewayRuntime").mockImplementation( async (...args: unknown[]) => { @@ -579,6 +600,17 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): .mockImplementation( overrides.executeSandboxCommand ?? (() => ({ status: 0, stdout: "doctor ok", stderr: "" })), ); + const checkAndRecoverSandboxProcessesSpy = vi + .spyOn(processRecovery, "checkAndRecoverSandboxProcesses") + .mockImplementation( + overrides.checkAndRecoverSandboxProcesses ?? + (() => ({ + checked: true, + wasRunning: true, + recovered: false, + forwardRecovered: false, + })), + ); vi.spyOn(shields, "repairMutableConfigPerms").mockImplementation( overrides.repairMutableConfigPerms ?? (() => ({ applied: true, verified: true, errors: [] })), ); @@ -629,6 +661,7 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): errorSpy, ensureAgentBaseImageSpy, executeSandboxCommandSpy, + checkAndRecoverSandboxProcessesSpy, ensureMessagingHostForwardAfterRebuildSpy, logSpy, finalizeIncompleteOnboardStepSpy, diff --git a/test/helpers/rebuild-flow-test-harness.ts b/test/helpers/rebuild-flow-test-harness.ts index d9164c137d6..82082ce578c 100644 --- a/test/helpers/rebuild-flow-test-harness.ts +++ b/test/helpers/rebuild-flow-test-harness.ts @@ -140,8 +140,16 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): .mockImplementation(() => undefined); vi.spyOn(resolve, "resolveOpenshell").mockReturnValue(null); vi.spyOn(agentDefs, "loadAgent").mockReturnValue(agentDef); - vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue({ name: "openclaw" }); - vi.spyOn(agentRuntime, "getAgentDisplayName").mockReturnValue("OpenClaw"); + vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue( + agentDef.name === "openclaw" ? null : ({ name: agentDef.name } as never), + ); + vi.spyOn(agentRuntime, "getAgentDisplayName").mockReturnValue( + agentDef.name === "hermes" + ? "Hermes Agent" + : agentDef.name === "langchain-deepagents-code" + ? "Deep Agents Code" + : "OpenClaw", + ); const defaultHydrateCredentialEnv = onboardCredentialEnv.hydrateCredentialEnv.bind(onboardCredentialEnv); const hydrateCredentialEnvSpy = vi @@ -449,6 +457,17 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): .mockImplementation( overrides.executeSandboxCommand ?? (() => ({ status: 0, stdout: "doctor ok", stderr: "" })), ); + const checkAndRecoverSandboxProcessesSpy = vi + .spyOn(processRecovery, "checkAndRecoverSandboxProcesses") + .mockImplementation( + overrides.checkAndRecoverSandboxProcesses ?? + (() => ({ + checked: true, + wasRunning: true, + recovered: false, + forwardRecovered: false, + })), + ); vi.spyOn(shields, "repairMutableConfigPerms").mockImplementation( overrides.repairMutableConfigPerms ?? (() => ({ applied: true, verified: true, errors: [] })), ); @@ -494,6 +513,7 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): rebuildSandbox: requireDist(rebuildModulePath).rebuildSandbox, applyPresetSpy, backupSandboxStateSpy, + checkAndRecoverSandboxProcessesSpy, errorSpy, executeSandboxCommandSpy, ensureMessagingHostForwardAfterRebuildSpy, diff --git a/test/helpers/rebuild-flow-test-support.ts b/test/helpers/rebuild-flow-test-support.ts index bd1b57993a1..fcf6d151256 100644 --- a/test/helpers/rebuild-flow-test-support.ts +++ b/test/helpers/rebuild-flow-test-support.ts @@ -37,6 +37,15 @@ export type RebuildFlowOverrides = { overrideEnvVar: string | null; }; executeSandboxCommand?: () => { status: number; stdout: string; stderr: string } | null; + checkAndRecoverSandboxProcesses?: () => { + checked: boolean; + wasRunning: boolean | null; + recovered: boolean; + forwardRecovered: boolean; + forwardRecoveryFailed?: boolean; + secretBoundaryRefused?: boolean; + mcpReconciliationRefused?: boolean; + }; onboard?: ( session: RebuildFlowSession, options: RebuildRecreateOnboardOpts, @@ -97,6 +106,7 @@ export type RebuildFlowHarness = { rebuildSandbox: RebuildSandbox; applyPresetSpy: MockInstance; backupSandboxStateSpy: MockInstance; + checkAndRecoverSandboxProcessesSpy: MockInstance; errorSpy: MockInstance; executeSandboxCommandSpy: MockInstance; ensureMessagingHostForwardAfterRebuildSpy: MockInstance;