From 9648fbd96da2102198c656875360107109cda74b Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 29 Jul 2026 09:34:53 -0400 Subject: [PATCH 1/4] fix(status): recover visible post-reboot sandbox --- docs/reference/commands.mdx | 4 ++ src/lib/actions/sandbox/status-flow.test.ts | 23 +++++++++- .../sandbox/status-lookup-rendering.ts | 5 ++- .../sandbox/status-snapshot-recovery.test.ts | 45 +++++++++++++++++++ src/lib/actions/sandbox/status-snapshot.ts | 18 +++++--- 5 files changed, 86 insertions(+), 9 deletions(-) diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 52cf1dd02a4..149262e32e7 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -1366,6 +1366,10 @@ When the host Docker daemon is reachable but the per-sandbox container is stoppe If the owning OpenShell gateway is healthy but no longer lists the registered Docker-driver sandbox, status attempts post-reboot recovery from the labeled container. It waits for Docker readiness, restores the in-sandbox gateway and host forwards, and refreshes preflight before probing inference. A successful recovery clears the stale stopped-container failure. + +If OpenShell already reports the registered Docker-driver sandbox as present, status verifies the OpenClaw gateway and host forward. +It recovers either component when the verification reports it absent. + If Docker readiness or the agent delivery chain cannot be proven, status exits non-zero and reports the failed recovery layer. If the sandbox's recorded dashboard port is also held by a foreign listener, the header escalates to the `sandbox_dashboard_port_conflict` failure layer with the message `sandbox container is stopped and the dashboard port is held by a foreign listener.` so the operator can recover the port before restarting the sandbox. diff --git a/src/lib/actions/sandbox/status-flow.test.ts b/src/lib/actions/sandbox/status-flow.test.ts index 57b9a9508a3..e7336db919a 100644 --- a/src/lib/actions/sandbox/status-flow.test.ts +++ b/src/lib/actions/sandbox/status-flow.test.ts @@ -557,7 +557,7 @@ describe("showSandboxStatus flow", () => { lookup: { state: "sandbox_recovery_failed", output: - " Docker restored sandbox 'alpha', but its agent delivery chain is not ready " + + " Sandbox 'alpha' is present, but its agent delivery chain could not be proven " + "(forward-recovery: OpenShell forward state unavailable).", recoveredSandbox: true, }, @@ -567,12 +567,31 @@ describe("showSandboxStatus flow", () => { const output = harness.logSpy.mock.calls.flat().join("\n"); expect(output).toContain("restored from Docker"); - expect(output).toContain("agent delivery chain could not be recovered safely"); + expect(output).toContain("agent delivery chain could not be proven"); expect(output).toContain("forward-recovery: OpenShell forward state unavailable"); expect(output).toContain("Retry `nemoclaw alpha recover`"); expect(output).not.toContain("Could not verify against live gateway"); }); + it("does not claim Docker restoration when a visible sandbox fails delivery recovery", async () => { + const harness = createStatusFlowHarness({ + inferenceHealth: null, + lookup: { + state: "sandbox_recovery_failed", + output: + " Sandbox 'alpha' is present, but its agent delivery chain could not be proven " + + "(gateway-recovery: the managed agent gateway could not be restarted).", + }, + }); + + await expect(harness.showSandboxStatus("alpha")).rejects.toThrow("process.exit(1)"); + + const output = harness.logSpy.mock.calls.flat().join("\n"); + expect(output).toContain("Sandbox 'alpha' is present"); + expect(output).toContain("agent delivery chain could not be proven"); + expect(output).not.toContain("restored from Docker"); + }); + it("renders missing gateway metadata after restart without claiming recovery", async () => { const harness = createStatusFlowHarness({ inferenceHealth: null, diff --git a/src/lib/actions/sandbox/status-lookup-rendering.ts b/src/lib/actions/sandbox/status-lookup-rendering.ts index 7343d944307..a028ad3e454 100644 --- a/src/lib/actions/sandbox/status-lookup-rendering.ts +++ b/src/lib/actions/sandbox/status-lookup-rendering.ts @@ -60,8 +60,11 @@ function printSandboxRecoveryFailedLookupStatus({ lookup, }: SandboxGatewayLookupStatusContext): void { console.log(""); + const recoveredFromDocker = "recoveredSandbox" in lookup && lookup.recoveredSandbox === true; console.log( - ` Sandbox '${sandboxName}' was restored from Docker, but its agent delivery chain could not be recovered safely.`, + recoveredFromDocker + ? ` Sandbox '${sandboxName}' was restored from Docker, but its agent delivery chain could not be proven.` + : ` Sandbox '${sandboxName}' is present, but its agent delivery chain could not be proven.`, ); if (lookup.output) console.log(lookup.output); console.log( diff --git a/src/lib/actions/sandbox/status-snapshot-recovery.test.ts b/src/lib/actions/sandbox/status-snapshot-recovery.test.ts index 629f3600c61..68294fd10c9 100644 --- a/src/lib/actions/sandbox/status-snapshot-recovery.test.ts +++ b/src/lib/actions/sandbox/status-snapshot-recovery.test.ts @@ -78,6 +78,51 @@ function snapshotDeps(recoveryResult: unknown) { } describe("collectSandboxStatusSnapshot Docker recovery", () => { + it("recovers the delivery chain when OpenShell already reports the restarted container (#7824)", async () => { + const deps = { + ...snapshotDeps({ + checked: true, + wasRunning: false, + recovered: true, + forwardRecovered: true, + }), + reconcile: () => + Promise.resolve({ + state: "present" as const, + output: "Phase: Ready", + }), + }; + + const snapshot = await collectSandboxStatusSnapshot("alpha", { deps }); + + expect(deps.recoverSandboxProcesses).toHaveBeenCalledWith("alpha", { quiet: true }); + expect(snapshot.lookup.state).toBe("present"); + }); + + it("fails closed when the visible restarted container cannot recover OpenClaw (#7824)", async () => { + const deps = { + ...snapshotDeps({ + checked: true, + wasRunning: false, + recovered: false, + forwardRecovered: false, + }), + reconcile: () => + Promise.resolve({ + state: "present" as const, + output: "Phase: Ready", + }), + }; + + const snapshot = await collectSandboxStatusSnapshot("alpha", { deps }); + + expect(snapshot.lookup.state).toBe("sandbox_recovery_failed"); + expect(snapshot.lookup.output).toContain( + "Sandbox 'alpha' is present, but its agent delivery chain could not be proven", + ); + expect(deps.probeSandboxInferenceGatewayHealthImpl).not.toHaveBeenCalled(); + }); + it.each([ [ "inspection", diff --git a/src/lib/actions/sandbox/status-snapshot.ts b/src/lib/actions/sandbox/status-snapshot.ts index 7cf8e5db7bb..10119c9843e 100644 --- a/src/lib/actions/sandbox/status-snapshot.ts +++ b/src/lib/actions/sandbox/status-snapshot.ts @@ -414,13 +414,19 @@ export async function collectSandboxStatusSnapshot( }; } const dockerRecovered = lookup.recoveredSandbox === true; - if (lookup.state === "present" && lookup.recoveredSandbox) { + const managedOpenClawDeliveryMustBeProven = + lookup.state === "present" && + sb?.openshellDriver === "docker" && + (sb.agent ?? "openclaw") === "openclaw"; + if ( + lookup.state === "present" && + (lookup.recoveredSandbox || managedOpenClawDeliveryMustBeProven) + ) { let failure: SandboxProcessRecoveryFailure | null; try { - // Docker recovery makes the sandbox visible to OpenShell again, but a - // host reboot also tears down the managed agent process and port-forward. - // Reuse the guarded connect recovery only for this explicit mutation - // path, before status probes the delivery chain. + // The managed gateway service can restart a Docker sandbox before status + // runs. OpenShell then reports Ready without a recoveredSandbox marker, + // while the OpenClaw gateway and host forward can still be absent. const recovery = (opts.deps?.recoverSandboxProcesses ?? loadRecoverSandboxProcesses())( sandboxName, { @@ -439,7 +445,7 @@ export async function collectSandboxStatusSnapshot( ...lookup, state: "sandbox_recovery_failed", output: - ` Docker restored sandbox '${sandboxName}', but its agent delivery chain is not ready ` + + ` Sandbox '${sandboxName}' is present, but its agent delivery chain could not be proven ` + `(${failure.layer}: ${failure.detail}).`, }; } From e8d2f590c51be4890648e591f877206feb8bfb0d Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 29 Jul 2026 16:28:23 -0400 Subject: [PATCH 2/4] fix(status): preserve status failure precedence --- docs/reference/commands.mdx | 2 +- .../sandbox/status-snapshot-recovery.test.ts | 48 +++++++++++++++++++ src/lib/actions/sandbox/status-snapshot.ts | 4 +- 3 files changed, 52 insertions(+), 2 deletions(-) diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 149262e32e7..c9e83b14dd5 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -1367,7 +1367,7 @@ If the owning OpenShell gateway is healthy but no longer lists the registered Do It waits for Docker readiness, restores the in-sandbox gateway and host forwards, and refreshes preflight before probing inference. A successful recovery clears the stale stopped-container failure. -If OpenShell already reports the registered Docker-driver sandbox as present, status verifies the OpenClaw gateway and host forward. +If OpenShell already reports the registered Docker-driver sandbox as present and `Ready`, status verifies the OpenClaw gateway and host forward. It recovers either component when the verification reports it absent. If Docker readiness or the agent delivery chain cannot be proven, status exits non-zero and reports the failed recovery layer. diff --git a/src/lib/actions/sandbox/status-snapshot-recovery.test.ts b/src/lib/actions/sandbox/status-snapshot-recovery.test.ts index 68294fd10c9..47008ef8e7f 100644 --- a/src/lib/actions/sandbox/status-snapshot-recovery.test.ts +++ b/src/lib/actions/sandbox/status-snapshot-recovery.test.ts @@ -123,6 +123,54 @@ describe("collectSandboxStatusSnapshot Docker recovery", () => { expect(deps.probeSandboxInferenceGatewayHealthImpl).not.toHaveBeenCalled(); }); + it.each([ + "Provisioning", + "Failed", + ])("keeps the existing %s phase diagnosis ahead of markerless recovery (#7824)", async (phase) => { + const deps = { + ...snapshotDeps({ + checked: true, + wasRunning: false, + recovered: false, + forwardRecovered: false, + }), + reconcile: () => + Promise.resolve({ + state: "present" as const, + output: `Phase: ${phase}`, + }), + }; + + const snapshot = await collectSandboxStatusSnapshot("alpha", { deps }); + + expect(deps.recoverSandboxProcesses).not.toHaveBeenCalled(); + expect(snapshot.lookup.state).toBe("present"); + }); + + it("keeps a host preflight failure ahead of markerless recovery (#7824)", async () => { + const deps = { + ...snapshotDeps({ + checked: true, + wasRunning: false, + recovered: false, + forwardRecovered: false, + }), + reconcile: () => + Promise.resolve({ + state: "present" as const, + output: "Phase: Ready", + }), + }; + + const snapshot = await collectSandboxStatusSnapshot("alpha", { + deps, + preflight: stoppedPreflight, + }); + + expect(deps.recoverSandboxProcesses).not.toHaveBeenCalled(); + expect(snapshot.lookup.state).toBe("present"); + }); + it.each([ [ "inspection", diff --git a/src/lib/actions/sandbox/status-snapshot.ts b/src/lib/actions/sandbox/status-snapshot.ts index 10119c9843e..e536b727b31 100644 --- a/src/lib/actions/sandbox/status-snapshot.ts +++ b/src/lib/actions/sandbox/status-snapshot.ts @@ -417,7 +417,9 @@ export async function collectSandboxStatusSnapshot( const managedOpenClawDeliveryMustBeProven = lookup.state === "present" && sb?.openshellDriver === "docker" && - (sb.agent ?? "openclaw") === "openclaw"; + (sb.agent ?? "openclaw") === "openclaw" && + parseSandboxPhase(lookup.output || "") === "Ready" && + !opts.preflight?.failure; if ( lookup.state === "present" && (lookup.recoveredSandbox || managedOpenClawDeliveryMustBeProven) From b7c47f87882e1568e54c86c28b99baf81c417f06 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 29 Jul 2026 16:40:49 -0400 Subject: [PATCH 3/4] test(status): isolate inference route fixture --- test/cli/sandbox-status-json.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/cli/sandbox-status-json.test.ts b/test/cli/sandbox-status-json.test.ts index 246c67fa1e2..f80cd5075a9 100644 --- a/test/cli/sandbox-status-json.test.ts +++ b/test/cli/sandbox-status-json.test.ts @@ -28,7 +28,9 @@ function createInferenceRouteStatusSetup(options: { writeSandboxRegistry(home, sandboxName, { model: "nvidia/nemotron", provider: "nvidia-prod", - openshellDriver: "docker", + // These cases test only inference.local classification. Use the VM driver + // so Docker post-reboot delivery recovery does not affect their assertions. + openshellDriver: "vm", }); fs.writeFileSync( path.join(localBin, "docker"), From caeacc131b387d7db81d4e0f0d15ea4706944fe2 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 29 Jul 2026 17:18:05 -0400 Subject: [PATCH 4/4] fix(status): retry exact OpenShell registration refusal Signed-off-by: Julie Yaunches --- .../actions/sandbox/process-recovery.test.ts | 26 +++++++++++++++++++ src/lib/actions/sandbox/process-recovery.ts | 18 ++++++------- 2 files changed, 35 insertions(+), 9 deletions(-) diff --git a/src/lib/actions/sandbox/process-recovery.test.ts b/src/lib/actions/sandbox/process-recovery.test.ts index 4a757c15dbd..6c2f5d77622 100644 --- a/src/lib/actions/sandbox/process-recovery.test.ts +++ b/src/lib/actions/sandbox/process-recovery.test.ts @@ -109,6 +109,32 @@ describe("recreated sandbox OpenShell readiness", () => { expect(sleeps).toEqual([3]); }); + it("retries the exact same-sandbox Error phase when OpenShell also emits informational stdout", () => { + const captureOpenshellImpl = vi + .fn() + .mockReturnValueOnce({ + status: 1, + output: + `Waiting for sandbox registration\n${OPENSHELL_TRANSIENT_ERROR_PHASE_STDERR}`.trim(), + stdout: "Waiting for sandbox registration\n", + stderr: OPENSHELL_TRANSIENT_ERROR_PHASE_STDERR, + }) + .mockReturnValueOnce({ status: 0, output: "", stdout: "", stderr: "" }); + const sleeps: number[] = []; + + expect( + waitForRecreatedSandboxOpenShellReady("recreated-box", { + beforeProbe: () => true, + captureOpenshellImpl, + intervalSeconds: 3, + sleepImpl: (seconds) => sleeps.push(seconds), + timeoutSeconds: 30, + }), + ).toBe(true); + expect(captureOpenshellImpl).toHaveBeenCalledTimes(2); + expect(sleeps).toEqual([3]); + }); + it("rides out a transient Error phase past the old 30s budget by default (#7227)", () => { // No timeoutSeconds option and no env override: the default recovery budget // must be large enough (120s, aligned with connect's readiness wait) to keep diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index 9497ee37854..9bc28614aee 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -599,30 +599,30 @@ function normalizeOpenshellStructuredError(value: string): string { return stripAnsi(value).replace(/[×│]/gu, " ").replace(/\s+/gu, " ").trim(); } -function hasRetryableOpenshellResultShape(result: ReturnType): boolean { - return ( - result.status === 1 && - !result.error && - String(result.stdout ?? "").trim() === "" && - String(result.stderr ?? "").trim() !== "" - ); +function hasRetryableOpenshellFailureShape(result: ReturnType): boolean { + return result.status === 1 && !result.error && String(result.stderr ?? "").trim() !== ""; } function isRetryableOpenshellReRegistrationState( result: ReturnType, sandboxName: string, ): boolean { - if (!hasRetryableOpenshellResultShape(result)) return false; + if (!hasRetryableOpenshellFailureShape(result)) return false; const error = normalizeOpenshellStructuredError(String(result.stderr)); - if (error === OPENSHELL_SANDBOX_NOT_READY) return true; // OpenShell can publish Ready before replacement registration settles. // Retry only if the readiness probe reports phase Error for this sandbox. + // The CLI can emit informational stdout before this exact stderr refusal; + // stdout does not change the result of the read-only `true` probe. if ( error === `Error: sandbox '${sandboxName}' is not ready (phase: Error); wait for it to reach Ready state.` ) { return true; } + // All less-specific transient signatures remain constrained to an otherwise + // empty stdout stream so unrelated command output cannot be reclassified. + if (String(result.stdout ?? "").trim() !== "") return false; + if (error === OPENSHELL_SANDBOX_NOT_READY) return true; // OpenShell 0.0.85 can keep the recreated sandbox's cached phase at Ready // while its replacement supervisor session is still registering. The exec