From 595ace926076789e5b19a39ed23f09d5d4dff055 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Fri, 17 Jul 2026 10:46:33 -0700 Subject: [PATCH 1/7] fix(rebuild): verify Hermes post-restore health Signed-off-by: Apurv Kumaria --- .../recover-rebuild-sandboxes.mdx | 5 +- .../rebuild-hermes-post-restore.test.ts | 181 ++++++++++++++++++ .../sandbox/rebuild-hermes-post-restore.ts | 67 +++++++ src/lib/actions/sandbox/rebuild-mcp-phase.ts | 2 + src/lib/actions/sandbox/rebuild-pipeline.ts | 1 + .../sandbox/rebuild-post-restore-phase.ts | 47 ++++- test/helpers/rebuild-flow-harness.ts | 37 +++- test/helpers/rebuild-flow-test-harness.ts | 24 ++- test/helpers/rebuild-flow-test-support.ts | 10 + 9 files changed, 365 insertions(+), 9 deletions(-) create mode 100644 src/lib/actions/sandbox/rebuild-hermes-post-restore.test.ts create mode 100644 src/lib/actions/sandbox/rebuild-hermes-post-restore.ts diff --git a/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx b/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx index 31ded8ba9d8..211235582a2 100644 --- a/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx +++ b/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx @@ -103,7 +103,10 @@ The rebuild command preserves the mounted workspace and registered policies whil -The rebuild command preserves Hermes state and registered policies while recreating the container. +The rebuild command preserves Hermes state, registered policies, and managed MCP configuration while recreating the container. +After state restore, NemoClaw restores managed MCP configuration through the normal lifecycle, then re-proves or recovers gateway health and performs final MCP reconciliation. +`rebuild` exits nonzero instead of reporting success when it cannot verify final gateway health or managed MCP state. +Follow the printed recovery guidance, using `$$nemoclaw recover` for gateway health and `$$nemoclaw mcp restart` for incomplete managed MCP restoration. diff --git a/src/lib/actions/sandbox/rebuild-hermes-post-restore.test.ts b/src/lib/actions/sandbox/rebuild-hermes-post-restore.test.ts new file mode 100644 index 00000000000..5e188a1eb1f --- /dev/null +++ b/src/lib/actions/sandbox/rebuild-hermes-post-restore.test.ts @@ -0,0 +1,181 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { + createRebuildFlowHarness, + resetRebuildFlowTestEnvironment, + restoreRebuildFlowTestEnvironment, +} from "../../../../test/helpers/rebuild-flow-harness"; + +describe("Hermes rebuild post-restore verification", () => { + beforeEach(resetRebuildFlowTestEnvironment); + afterEach(restoreRebuildFlowTestEnvironment); + + it("fails instead of reporting readiness when restored state leaves the gateway down (#7084)", async () => { + const mcpEntry = { + server: "blender", + providerName: "nemoclaw-mcp-alpha-blender", + }; + const harness = createRebuildFlowHarness({ + agentName: "hermes", + checkAndRecoverSandboxProcesses: () => ({ + checked: true, + wasRunning: false, + recovered: false, + forwardRecovered: false, + }), + mcpPreparation: { + entries: [mcpEntry], + detachedProviderEntries: [mcpEntry], + scrubbedAdapterEntries: [mcpEntry], + }, + sandboxEntry: { agent: "hermes" }, + }); + harness.restoreMcpBridgesAfterRebuildSpy.mockRejectedValueOnce( + new Error("gateway unavailable"), + ); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).rejects.toThrow("Hermes post-restore verification failed"); + + const output = harness.logSpy.mock.calls.map((call) => String(call[0])).join("\n"); + expect(output).toContain("rebuilt but some post-restore steps were incomplete"); + expect(output).toContain("Hermes gateway health was not verified after state restore"); + expect(output).toContain("MCP bridge definitions were preserved but not fully refreshed"); + expect(output).not.toContain("rebuilt successfully"); + expect(harness.restoreMcpBridgesAfterRebuildSpy).toHaveBeenCalledWith("alpha", [mcpEntry]); + }); + + it("accepts restored MCP configuration only after final gateway recovery (#7084)", async () => { + const mcpEntry = { + server: "blender", + providerName: "nemoclaw-mcp-alpha-blender", + }; + const harness = createRebuildFlowHarness({ + agentName: "hermes", + checkAndRecoverSandboxProcesses: () => ({ + checked: true, + wasRunning: false, + recovered: true, + forwardRecovered: true, + }), + mcpPreparation: { + entries: [mcpEntry], + detachedProviderEntries: [mcpEntry], + scrubbedAdapterEntries: [mcpEntry], + }, + sandboxEntry: { agent: "hermes" }, + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).resolves.toBeUndefined(); + + expect(harness.restoreMcpBridgesAfterRebuildSpy).toHaveBeenCalledWith("alpha", [mcpEntry]); + expect(harness.restoreMcpBridgesAfterRebuildSpy.mock.invocationCallOrder[0]).toBeLessThan( + harness.checkAndRecoverSandboxProcessesSpy.mock.invocationCallOrder[0], + ); + expect(harness.logSpy).toHaveBeenCalledWith( + expect.stringContaining("Hermes gateway recovered after state restore"), + ); + }); + + it("returns a failed rebuild when managed Hermes MCP restoration is incomplete (#7084)", async () => { + const mcpEntry = { + server: "blender", + providerName: "nemoclaw-mcp-alpha-blender", + }; + const harness = createRebuildFlowHarness({ + agentName: "hermes", + mcpPreparation: { + entries: [mcpEntry], + detachedProviderEntries: [mcpEntry], + scrubbedAdapterEntries: [mcpEntry], + }, + sandboxEntry: { agent: "hermes" }, + }); + harness.restoreMcpBridgesAfterRebuildSpy.mockRejectedValueOnce(new Error("reload failed")); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).rejects.toThrow("Hermes post-restore verification failed"); + + const output = harness.logSpy.mock.calls.map((call) => String(call[0])).join("\n"); + expect(output).toContain("MCP bridge definitions were preserved but not fully refreshed"); + expect(output).not.toContain("rebuilt successfully"); + }); + + it("fails when the final gateway check refuses MCP reconciliation (#7084)", async () => { + const mcpEntry = { + server: "blender", + providerName: "nemoclaw-mcp-alpha-blender", + }; + const harness = createRebuildFlowHarness({ + agentName: "hermes", + checkAndRecoverSandboxProcesses: () => ({ + checked: true, + wasRunning: true, + recovered: false, + forwardRecovered: false, + mcpReconciliationRefused: true, + }), + mcpPreparation: { + entries: [mcpEntry], + detachedProviderEntries: [mcpEntry], + scrubbedAdapterEntries: [mcpEntry], + }, + sandboxEntry: { agent: "hermes" }, + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).rejects.toThrow("Hermes post-restore verification failed"); + + expect(harness.restoreMcpBridgesAfterRebuildSpy).toHaveBeenCalledWith("alpha", [mcpEntry]); + expect(harness.logSpy).not.toHaveBeenCalledWith( + expect.stringContaining("rebuilt successfully"), + ); + }); + + it("fails before recovery when recreated Hermes identity is missing (#7084)", async () => { + const harness = createRebuildFlowHarness({ + agentName: "hermes", + sessionAgentName: null, + sandboxEntry: { agent: "hermes" }, + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).rejects.toThrow( + "Recreated sandbox agent identity did not match the authoritative rebuild target", + ); + + expect(harness.checkAndRecoverSandboxProcessesSpy).not.toHaveBeenCalled(); + expect(harness.restoreMcpBridgesAfterRebuildSpy).not.toHaveBeenCalled(); + expect(harness.logSpy).not.toHaveBeenCalledWith( + expect.stringContaining("rebuilt successfully"), + ); + }); + + it("fails before recovery when recreated Hermes identity mismatches (#7084)", async () => { + const harness = createRebuildFlowHarness({ + agentName: "hermes", + sessionAgentName: "langchain-deepagents-code", + sandboxEntry: { agent: "hermes" }, + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).rejects.toThrow( + "Recreated sandbox agent identity did not match the authoritative rebuild target", + ); + + expect(harness.checkAndRecoverSandboxProcessesSpy).not.toHaveBeenCalled(); + expect(harness.restoreMcpBridgesAfterRebuildSpy).not.toHaveBeenCalled(); + expect(harness.logSpy).not.toHaveBeenCalledWith( + expect.stringContaining("rebuilt successfully"), + ); + }); +}); diff --git a/src/lib/actions/sandbox/rebuild-hermes-post-restore.ts b/src/lib/actions/sandbox/rebuild-hermes-post-restore.ts new file mode 100644 index 00000000000..5362f592a7c --- /dev/null +++ b/src/lib/actions/sandbox/rebuild-hermes-post-restore.ts @@ -0,0 +1,67 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { CLI_NAME } from "../../cli/branding"; +import * as processRecovery from "./process-recovery"; + +export type HermesPostRestoreGatewayState = + | "not-applicable" + | "healthy" + | "recovered" + | "unverified"; + +type GatewayRecoveryObservation = { + checked: boolean; + wasRunning: boolean | null; + recovered: boolean; + forwardRecoveryFailed?: boolean; + secretBoundaryRefused?: boolean; + mcpReconciliationRefused?: boolean; +}; + +interface HermesPostRestoreGatewayDeps { + checkAndRecoverSandboxProcesses?: ( + sandboxName: string, + options: { quiet: boolean }, + ) => GatewayRecoveryObservation; +} + +/** + * Re-prove Hermes gateway health after workspace state restoration. + * + * Inner onboarding verifies the fresh image before rebuild restores the prior + * state. That restore can still stop or wedge the gateway, so its earlier + * readiness message is not authoritative for rebuild completion. + */ +export function ensureHermesGatewayAfterStateRestore( + sandboxName: string, + agentName: string, + deps: HermesPostRestoreGatewayDeps = {}, +): HermesPostRestoreGatewayState { + if (agentName !== "hermes") return "not-applicable"; + const checkAndRecover = + deps.checkAndRecoverSandboxProcesses ?? processRecovery.checkAndRecoverSandboxProcesses; + const observation: GatewayRecoveryObservation = checkAndRecover(sandboxName, { quiet: true }); + if ( + !observation.checked || + observation.forwardRecoveryFailed === true || + observation.secretBoundaryRefused === true || + observation.mcpReconciliationRefused === true + ) { + return "unverified"; + } + if (observation.wasRunning === true) return "healthy"; + if (observation.recovered) return "recovered"; + return "unverified"; +} + +export function printHermesGatewayRestoreRecovery( + sandboxName: string, + state: HermesPostRestoreGatewayState, + writeLine: (message: string) => void = console.log, +): void { + if (state !== "unverified") return; + writeLine( + ` Hermes gateway health was not verified after state restore — run \`${CLI_NAME} ${sandboxName} recover\` before relying on this sandbox`, + ); +} diff --git a/src/lib/actions/sandbox/rebuild-mcp-phase.ts b/src/lib/actions/sandbox/rebuild-mcp-phase.ts index f24f93a407e..7c25568d3db 100644 --- a/src/lib/actions/sandbox/rebuild-mcp-phase.ts +++ b/src/lib/actions/sandbox/rebuild-mcp-phase.ts @@ -126,6 +126,7 @@ export async function restoreMcpAfterRebuild( } export function postRestoreCompleted(status: { + hermesGatewayRestoreUnverified: boolean; messagingHostForwardUnverified: boolean; mcpBridgeRestoreUnverified: boolean; mutableConfigHashRefreshUnverified: boolean; @@ -135,6 +136,7 @@ export function postRestoreCompleted(status: { }): boolean { return ( status.restoreSucceeded && + !status.hermesGatewayRestoreUnverified && !status.mutablePermsRepairUnverified && !status.mutableConfigHashRefreshUnverified && !status.messagingHostForwardUnverified && diff --git a/src/lib/actions/sandbox/rebuild-pipeline.ts b/src/lib/actions/sandbox/rebuild-pipeline.ts index c3c2a87042c..d2f4942f779 100644 --- a/src/lib/actions/sandbox/rebuild-pipeline.ts +++ b/src/lib/actions/sandbox/rebuild-pipeline.ts @@ -312,6 +312,7 @@ async function rebuildSandboxUnlocked( await runRebuildPostRestorePhase({ sandboxName, sandboxEntry, + targetAgentName: rebuildAgent || "openclaw", messagingPlan, backupManifest: backup.backupManifest, mcpEntries: mcpPreparation.entries, diff --git a/src/lib/actions/sandbox/rebuild-post-restore-phase.ts b/src/lib/actions/sandbox/rebuild-post-restore-phase.ts index bd4551354a3..71928a919ac 100644 --- a/src/lib/actions/sandbox/rebuild-post-restore-phase.ts +++ b/src/lib/actions/sandbox/rebuild-post-restore-phase.ts @@ -16,6 +16,10 @@ import type { RebuildBackupManifest } from "./rebuild-backup-phase"; import { refreshMutableOpenClawConfigHashAfterPostRestoreWrites } from "./rebuild-config-hash"; import type { RebuildBail, RebuildLog } from "./rebuild-credential-preflight"; import type { RebuildSandboxEntry } from "./rebuild-flow-helpers"; +import { + ensureHermesGatewayAfterStateRestore, + printHermesGatewayRestoreRecovery, +} from "./rebuild-hermes-post-restore"; import { type McpRebuildPreparation, postRestoreCompleted, @@ -27,6 +31,7 @@ import { reapplyMessagingManifestAfterOpenClawDoctor } from "./rebuild-messaging export interface RebuildPostRestorePhaseInput { sandboxName: string; sandboxEntry: RebuildSandboxEntry; + targetAgentName: string; messagingPlan: SandboxMessagingPlan | null; backupManifest: RebuildBackupManifest; mcpEntries: McpRebuildPreparation["entries"]; @@ -102,6 +107,7 @@ export async function runRebuildPostRestorePhase( const { sandboxName, sandboxEntry: sb, + targetAgentName, messagingPlan, backupManifest, mcpEntries, @@ -120,9 +126,23 @@ export async function runRebuildPostRestorePhase( log, bail, } = input; - const rebuiltAgent = agentRuntime.getSessionAgent(sandboxName); - const rebuiltAgentName = agentRuntime.getAgentDisplayName(rebuiltAgent); - const agentDef = rebuiltAgent ? loadAgent(rebuiltAgent.name) : loadAgent("openclaw"); + const recreatedEntry = registry.getSandbox(sandboxName); + const recreatedAgent = agentRuntime.getSessionAgent(sandboxName); + const recreatedRegistryAgentName = recreatedEntry?.agent ?? "openclaw"; + const recreatedRuntimeAgentName = recreatedAgent?.name ?? "openclaw"; + if ( + !recreatedEntry || + recreatedRegistryAgentName !== targetAgentName || + recreatedRuntimeAgentName !== targetAgentName + ) { + console.error( + ` ${YW}\u26a0${R} Recreated sandbox agent identity could not be verified against the rebuild target.`, + ); + bail("Recreated sandbox agent identity did not match the authoritative rebuild target."); + return; + } + const agentDef = loadAgent(targetAgentName); + const rebuiltAgentName = agentDef.displayName; let mutablePermsRepairUnverified = false; let mutableConfigHashRefreshUnverified = false; let messagingHostForwardUnverified = false; @@ -131,7 +151,7 @@ export async function runRebuildPostRestorePhase( failedPresetRemovals.length > 0 || !policyPresetReconciliationVerified; - if (agentDef.name === "openclaw") { + if (targetAgentName === "openclaw") { log("Running openclaw doctor --fix inside sandbox for post-upgrade structure repair"); const doctorResult = executeSandboxCommand(sandboxName, "openclaw doctor --fix"); log( @@ -183,6 +203,16 @@ export async function runRebuildPostRestorePhase( } const mcpBridgeRestoreUnverified = !(await restoreMcpAfterRebuild(sandboxName, mcpEntries)); + const hermesGatewayRestoreState = ensureHermesGatewayAfterStateRestore( + sandboxName, + targetAgentName, + ); + const hermesGatewayRestoreUnverified = hermesGatewayRestoreState === "unverified"; + if (hermesGatewayRestoreState === "healthy") { + console.log(` ${G}\u2713${R} Hermes gateway health verified after state restore`); + } else if (hermesGatewayRestoreState === "recovered") { + console.log(` ${G}\u2713${R} Hermes gateway recovered after state restore`); + } const { policies: restoredBuiltinPresets, policyPresetsFinalized } = resolveRestoredPolicyRegistryState( { @@ -212,6 +242,7 @@ export async function runRebuildPostRestorePhase( console.log(""); const postRestoreComplete = postRestoreCompleted({ + hermesGatewayRestoreUnverified, messagingHostForwardUnverified, mcpBridgeRestoreUnverified, mutableConfigHashRefreshUnverified, @@ -252,6 +283,7 @@ export async function runRebuildPostRestorePhase( ` Messaging webhook forward was not verified \u2014 run \`${CLI_NAME} ${sandboxName} connect\` after resolving the port conflict`, ); } + printHermesGatewayRestoreRecovery(sandboxName, hermesGatewayRestoreState); printMcpRestoreRecovery(sandboxName, mcpBridgeRestoreUnverified); if (policyPresetRestoreIncomplete) { if (failedPresets.length > 0) { @@ -275,6 +307,13 @@ export async function runRebuildPostRestorePhase( bail(`Rebuild completed with unverified live policy reconciliation for '${sandboxName}'.`); return; } + if ( + targetAgentName === "hermes" && + (hermesGatewayRestoreUnverified || mcpBridgeRestoreUnverified) + ) { + bail(`Hermes post-restore verification failed for '${sandboxName}'.`); + return; + } if (preparedBackupRecovery && !postRestoreComplete) { bail( `Prepared backup recovery for '${sandboxName}' completed with unverified post-restore state.`, diff --git a/test/helpers/rebuild-flow-harness.ts b/test/helpers/rebuild-flow-harness.ts index 1f225534c74..6a23fda807e 100644 --- a/test/helpers/rebuild-flow-harness.ts +++ b/test/helpers/rebuild-flow-harness.ts @@ -75,8 +75,18 @@ export type RebuildFlowSession = Record & { export type RebuildFlowOverrides = { agentName?: string; + sessionAgentName?: string | null; applyPreset?: (presetName: string) => boolean; executeSandboxCommand?: () => { status: number; stdout: string; stderr: string } | null; + checkAndRecoverSandboxProcesses?: () => { + checked: boolean; + wasRunning: boolean | null; + recovered: boolean; + forwardRecovered: boolean; + forwardRecoveryFailed?: boolean; + secretBoundaryRefused?: boolean; + mcpReconciliationRefused?: boolean; + }; onboard?: (session: RebuildFlowSession) => Promise | void; repairMutableConfigPerms?: () => | { applied: false; skipReason: "agent" | "locked" | "unreadable"; reason: string } @@ -132,6 +142,7 @@ export type RebuildFlowHarness = { errorSpy: MockInstance; ensureAgentBaseImageSpy: MockInstance; executeSandboxCommandSpy: MockInstance; + checkAndRecoverSandboxProcessesSpy: MockInstance; ensureMessagingHostForwardAfterRebuildSpy: MockInstance; logSpy: MockInstance; finalizeIncompleteOnboardStepSpy: MockInstance; @@ -295,9 +306,19 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): imageTag: `nemoclaw-${agentName}-base:test`, built: true, }); - vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue({ name: agentName }); + const sessionAgentName = + overrides.sessionAgentName === undefined ? agentName : overrides.sessionAgentName; + vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue( + sessionAgentName === null || sessionAgentName === "openclaw" + ? null + : ({ name: sessionAgentName } as never), + ); vi.spyOn(agentRuntime, "getAgentDisplayName").mockReturnValue( - agentName === "langchain-deepagents-code" ? "Deep Agents Code" : "OpenClaw", + agentName === "langchain-deepagents-code" + ? "Deep Agents Code" + : agentName === "hermes" + ? "Hermes Agent" + : "OpenClaw", ); vi.spyOn(gatewayRuntime, "recoverNamedGatewayRuntime").mockImplementation( async (...args: unknown[]) => { @@ -579,6 +600,17 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): .mockImplementation( overrides.executeSandboxCommand ?? (() => ({ status: 0, stdout: "doctor ok", stderr: "" })), ); + const checkAndRecoverSandboxProcessesSpy = vi + .spyOn(processRecovery, "checkAndRecoverSandboxProcesses") + .mockImplementation( + overrides.checkAndRecoverSandboxProcesses ?? + (() => ({ + checked: true, + wasRunning: true, + recovered: false, + forwardRecovered: false, + })), + ); vi.spyOn(shields, "repairMutableConfigPerms").mockImplementation( overrides.repairMutableConfigPerms ?? (() => ({ applied: true, verified: true, errors: [] })), ); @@ -629,6 +661,7 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): errorSpy, ensureAgentBaseImageSpy, executeSandboxCommandSpy, + checkAndRecoverSandboxProcessesSpy, ensureMessagingHostForwardAfterRebuildSpy, logSpy, finalizeIncompleteOnboardStepSpy, diff --git a/test/helpers/rebuild-flow-test-harness.ts b/test/helpers/rebuild-flow-test-harness.ts index d9164c137d6..82082ce578c 100644 --- a/test/helpers/rebuild-flow-test-harness.ts +++ b/test/helpers/rebuild-flow-test-harness.ts @@ -140,8 +140,16 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): .mockImplementation(() => undefined); vi.spyOn(resolve, "resolveOpenshell").mockReturnValue(null); vi.spyOn(agentDefs, "loadAgent").mockReturnValue(agentDef); - vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue({ name: "openclaw" }); - vi.spyOn(agentRuntime, "getAgentDisplayName").mockReturnValue("OpenClaw"); + vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue( + agentDef.name === "openclaw" ? null : ({ name: agentDef.name } as never), + ); + vi.spyOn(agentRuntime, "getAgentDisplayName").mockReturnValue( + agentDef.name === "hermes" + ? "Hermes Agent" + : agentDef.name === "langchain-deepagents-code" + ? "Deep Agents Code" + : "OpenClaw", + ); const defaultHydrateCredentialEnv = onboardCredentialEnv.hydrateCredentialEnv.bind(onboardCredentialEnv); const hydrateCredentialEnvSpy = vi @@ -449,6 +457,17 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): .mockImplementation( overrides.executeSandboxCommand ?? (() => ({ status: 0, stdout: "doctor ok", stderr: "" })), ); + const checkAndRecoverSandboxProcessesSpy = vi + .spyOn(processRecovery, "checkAndRecoverSandboxProcesses") + .mockImplementation( + overrides.checkAndRecoverSandboxProcesses ?? + (() => ({ + checked: true, + wasRunning: true, + recovered: false, + forwardRecovered: false, + })), + ); vi.spyOn(shields, "repairMutableConfigPerms").mockImplementation( overrides.repairMutableConfigPerms ?? (() => ({ applied: true, verified: true, errors: [] })), ); @@ -494,6 +513,7 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): rebuildSandbox: requireDist(rebuildModulePath).rebuildSandbox, applyPresetSpy, backupSandboxStateSpy, + checkAndRecoverSandboxProcessesSpy, errorSpy, executeSandboxCommandSpy, ensureMessagingHostForwardAfterRebuildSpy, diff --git a/test/helpers/rebuild-flow-test-support.ts b/test/helpers/rebuild-flow-test-support.ts index bd1b57993a1..fcf6d151256 100644 --- a/test/helpers/rebuild-flow-test-support.ts +++ b/test/helpers/rebuild-flow-test-support.ts @@ -37,6 +37,15 @@ export type RebuildFlowOverrides = { overrideEnvVar: string | null; }; executeSandboxCommand?: () => { status: number; stdout: string; stderr: string } | null; + checkAndRecoverSandboxProcesses?: () => { + checked: boolean; + wasRunning: boolean | null; + recovered: boolean; + forwardRecovered: boolean; + forwardRecoveryFailed?: boolean; + secretBoundaryRefused?: boolean; + mcpReconciliationRefused?: boolean; + }; onboard?: ( session: RebuildFlowSession, options: RebuildRecreateOnboardOpts, @@ -97,6 +106,7 @@ export type RebuildFlowHarness = { rebuildSandbox: RebuildSandbox; applyPresetSpy: MockInstance; backupSandboxStateSpy: MockInstance; + checkAndRecoverSandboxProcessesSpy: MockInstance; errorSpy: MockInstance; executeSandboxCommandSpy: MockInstance; ensureMessagingHostForwardAfterRebuildSpy: MockInstance; From f3b14295d919be1efbb2ef39af41da7611fc4e35 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 17 Jul 2026 16:04:06 -0400 Subject: [PATCH 2/7] test(rebuild): isolate gateway verification Signed-off-by: Julie Yaunches --- src/lib/actions/sandbox/rebuild-hermes-post-restore.test.ts | 6 +----- src/lib/actions/sandbox/rebuild-post-restore-phase.ts | 1 + 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/lib/actions/sandbox/rebuild-hermes-post-restore.test.ts b/src/lib/actions/sandbox/rebuild-hermes-post-restore.test.ts index 5e188a1eb1f..12f5d45334d 100644 --- a/src/lib/actions/sandbox/rebuild-hermes-post-restore.test.ts +++ b/src/lib/actions/sandbox/rebuild-hermes-post-restore.test.ts @@ -32,10 +32,6 @@ describe("Hermes rebuild post-restore verification", () => { }, sandboxEntry: { agent: "hermes" }, }); - harness.restoreMcpBridgesAfterRebuildSpy.mockRejectedValueOnce( - new Error("gateway unavailable"), - ); - await expect( harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), ).rejects.toThrow("Hermes post-restore verification failed"); @@ -43,7 +39,7 @@ describe("Hermes rebuild post-restore verification", () => { const output = harness.logSpy.mock.calls.map((call) => String(call[0])).join("\n"); expect(output).toContain("rebuilt but some post-restore steps were incomplete"); expect(output).toContain("Hermes gateway health was not verified after state restore"); - expect(output).toContain("MCP bridge definitions were preserved but not fully refreshed"); + expect(output).not.toContain("MCP bridge definitions were preserved but not fully refreshed"); expect(output).not.toContain("rebuilt successfully"); expect(harness.restoreMcpBridgesAfterRebuildSpy).toHaveBeenCalledWith("alpha", [mcpEntry]); }); diff --git a/src/lib/actions/sandbox/rebuild-post-restore-phase.ts b/src/lib/actions/sandbox/rebuild-post-restore-phase.ts index 71928a919ac..c92b4b88b0e 100644 --- a/src/lib/actions/sandbox/rebuild-post-restore-phase.ts +++ b/src/lib/actions/sandbox/rebuild-post-restore-phase.ts @@ -128,6 +128,7 @@ export async function runRebuildPostRestorePhase( } = input; const recreatedEntry = registry.getSandbox(sandboxName); const recreatedAgent = agentRuntime.getSessionAgent(sandboxName); + // OpenClaw is represented by a null registry agent and a null runtime definition. const recreatedRegistryAgentName = recreatedEntry?.agent ?? "openclaw"; const recreatedRuntimeAgentName = recreatedAgent?.name ?? "openclaw"; if ( From 8b1e31c94243e5ddd2460c7553d7a1c5a02598a9 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 17 Jul 2026 16:06:23 -0400 Subject: [PATCH 3/7] docs(rebuild): clarify identity verification Signed-off-by: Julie Yaunches --- docs/manage-sandboxes/recover-rebuild-sandboxes.mdx | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx b/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx index 211235582a2..8fea4c64981 100644 --- a/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx +++ b/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx @@ -104,6 +104,7 @@ The rebuild command preserves the mounted workspace and registered policies whil The rebuild command preserves Hermes state, registered policies, and managed MCP configuration while recreating the container. +Before post-restore repairs, NemoClaw verifies that the recreated sandbox still identifies as Hermes and exits nonzero if its identity does not match the rebuild target. After state restore, NemoClaw restores managed MCP configuration through the normal lifecycle, then re-proves or recovers gateway health and performs final MCP reconciliation. `rebuild` exits nonzero instead of reporting success when it cannot verify final gateway health or managed MCP state. Follow the printed recovery guidance, using `$$nemoclaw recover` for gateway health and `$$nemoclaw mcp restart` for incomplete managed MCP restoration. From 6b80397c9e8b37c9cf28b557dad0ffa25355f648 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 17 Jul 2026 16:29:34 -0400 Subject: [PATCH 4/7] test(rebuild): cover gateway refusal states Signed-off-by: Julie Yaunches --- .../rebuild-hermes-post-restore.test.ts | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/lib/actions/sandbox/rebuild-hermes-post-restore.test.ts b/src/lib/actions/sandbox/rebuild-hermes-post-restore.test.ts index 12f5d45334d..f69a05879bd 100644 --- a/src/lib/actions/sandbox/rebuild-hermes-post-restore.test.ts +++ b/src/lib/actions/sandbox/rebuild-hermes-post-restore.test.ts @@ -135,6 +135,31 @@ describe("Hermes rebuild post-restore verification", () => { ); }); + it.each([ + "forwardRecoveryFailed", + "secretBoundaryRefused", + ] as const)("fails when the final gateway check reports %s (#7084)", async (failureFlag) => { + const harness = createRebuildFlowHarness({ + agentName: "hermes", + checkAndRecoverSandboxProcesses: () => ({ + checked: true, + wasRunning: true, + recovered: false, + forwardRecovered: false, + [failureFlag]: true, + }), + sandboxEntry: { agent: "hermes" }, + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).rejects.toThrow("Hermes post-restore verification failed"); + + expect(harness.logSpy).not.toHaveBeenCalledWith( + expect.stringContaining("rebuilt successfully"), + ); + }); + it("fails before recovery when recreated Hermes identity is missing (#7084)", async () => { const harness = createRebuildFlowHarness({ agentName: "hermes", From f12c7f70d5583421247e269bf9a8dfaaad9bae03 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 17 Jul 2026 16:57:06 -0400 Subject: [PATCH 5/7] test(rebuild): cover unavailable health probe Signed-off-by: Julie Yaunches --- .../rebuild-hermes-post-restore.test.ts | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/lib/actions/sandbox/rebuild-hermes-post-restore.test.ts b/src/lib/actions/sandbox/rebuild-hermes-post-restore.test.ts index f69a05879bd..e6aba209276 100644 --- a/src/lib/actions/sandbox/rebuild-hermes-post-restore.test.ts +++ b/src/lib/actions/sandbox/rebuild-hermes-post-restore.test.ts @@ -160,6 +160,27 @@ describe("Hermes rebuild post-restore verification", () => { ); }); + it("fails when the final gateway health probe is unavailable (#7084)", async () => { + const harness = createRebuildFlowHarness({ + agentName: "hermes", + checkAndRecoverSandboxProcesses: () => ({ + checked: false, + wasRunning: null, + recovered: false, + forwardRecovered: false, + }), + sandboxEntry: { agent: "hermes" }, + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).rejects.toThrow("Hermes post-restore verification failed"); + + expect(harness.logSpy).not.toHaveBeenCalledWith( + expect.stringContaining("rebuilt successfully"), + ); + }); + it("fails before recovery when recreated Hermes identity is missing (#7084)", async () => { const harness = createRebuildFlowHarness({ agentName: "hermes", From c21c793a59d3e86423fb3b0a3ccea4fa69ad26fa Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 17 Jul 2026 17:31:55 -0400 Subject: [PATCH 6/7] test(rebuild): update post-restore fixture Signed-off-by: Julie Yaunches --- .../actions/sandbox/rebuild-post-restore-phase.test.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/lib/actions/sandbox/rebuild-post-restore-phase.test.ts b/src/lib/actions/sandbox/rebuild-post-restore-phase.test.ts index 1d878cf04d9..641ef1abc12 100644 --- a/src/lib/actions/sandbox/rebuild-post-restore-phase.test.ts +++ b/src/lib/actions/sandbox/rebuild-post-restore-phase.test.ts @@ -23,8 +23,8 @@ describe("rebuild post-restore session model reconciliation (#7102)", () => { order = []; vi.spyOn(console, "log").mockImplementation(() => undefined); vi.spyOn(console, "error").mockImplementation(() => undefined); - vi.spyOn(agentRuntime, "getSessionAgent").mockImplementation( - () => ({ name: agentName }) as never, + vi.spyOn(agentRuntime, "getSessionAgent").mockImplementation(() => + agentName === "openclaw" ? null : ({ name: agentName } as never), ); vi.spyOn(agentRuntime, "getAgentDisplayName").mockReturnValue("test agent"); vi.spyOn(agentDefs, "loadAgent").mockImplementation( @@ -57,6 +57,9 @@ describe("rebuild post-restore session model reconciliation (#7102)", () => { skipReason: "not-needed", } as never); vi.spyOn(rebuildMcp, "restoreMcpAfterRebuild").mockResolvedValue(true); + vi.spyOn(registry, "getSandbox").mockImplementation( + () => ({ agent: agentName === "openclaw" ? null : agentName }) as never, + ); vi.spyOn(registry, "updateSandbox").mockReturnValue(true); vi.spyOn(messagingHostForward, "ensureMessagingHostForwardAfterRebuild").mockReturnValue(true); }); @@ -68,6 +71,7 @@ describe("rebuild post-restore session model reconciliation (#7102)", () => { function input() { return { sandboxName: "alpha", + targetAgentName: agentName, sandboxEntry: {} as never, messagingPlan: null, backupManifest: null, From 8afd8362e1a2290351d3037b15afa16eb23a3dfe Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Fri, 17 Jul 2026 14:49:01 -0700 Subject: [PATCH 7/7] test(rebuild): prove non-OpenClaw path succeeds Co-authored-by: Julie Yaunches Signed-off-by: Apurv Kumaria --- .../actions/sandbox/rebuild-post-restore-phase.test.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/lib/actions/sandbox/rebuild-post-restore-phase.test.ts b/src/lib/actions/sandbox/rebuild-post-restore-phase.test.ts index 641ef1abc12..377cd9a15b3 100644 --- a/src/lib/actions/sandbox/rebuild-post-restore-phase.test.ts +++ b/src/lib/actions/sandbox/rebuild-post-restore-phase.test.ts @@ -9,6 +9,7 @@ import * as registry from "../../state/registry"; import * as messagingHostForward from "./messaging-host-forward-lifecycle"; import * as processRecovery from "./process-recovery"; import * as rebuildConfigHash from "./rebuild-config-hash"; +import * as rebuildHermesPostRestore from "./rebuild-hermes-post-restore"; import * as rebuildMcp from "./rebuild-mcp-phase"; import * as rebuildMessaging from "./rebuild-messaging-phase"; import { runRebuildPostRestorePhase } from "./rebuild-post-restore-phase"; @@ -57,6 +58,10 @@ describe("rebuild post-restore session model reconciliation (#7102)", () => { skipReason: "not-needed", } as never); vi.spyOn(rebuildMcp, "restoreMcpAfterRebuild").mockResolvedValue(true); + vi.spyOn(rebuildHermesPostRestore, "ensureHermesGatewayAfterStateRestore").mockImplementation( + (_sandboxName, targetAgentName) => + targetAgentName === "hermes" ? "healthy" : "not-applicable", + ); vi.spyOn(registry, "getSandbox").mockImplementation( () => ({ agent: agentName === "openclaw" ? null : agentName }) as never, ); @@ -101,9 +106,11 @@ describe("rebuild post-restore session model reconciliation (#7102)", () => { it("does not run OpenClaw session reconciliation for another agent", async () => { agentName = "hermes"; + const args = input(); - await runRebuildPostRestorePhase(input()); + await runRebuildPostRestorePhase(args); + expect(args.bail).not.toHaveBeenCalled(); expect(sessionModels.reconcileStalePinnedSessionModelsAfterRebuild).not.toHaveBeenCalled(); expect(processRecovery.executeSandboxCommand).not.toHaveBeenCalled(); });