From db52f208c48283fc1b10cf15570fb5e406c6d81e Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 29 Jul 2026 19:43:15 -0400 Subject: [PATCH 01/30] fix(recovery): retry blank OpenShell registration follow-up Signed-off-by: Julie Yaunches --- .../actions/sandbox/process-recovery.test.ts | 50 +++++++++++++++++++ src/lib/actions/sandbox/process-recovery.ts | 25 ++++++++-- 2 files changed, 71 insertions(+), 4 deletions(-) diff --git a/src/lib/actions/sandbox/process-recovery.test.ts b/src/lib/actions/sandbox/process-recovery.test.ts index 6c2f5d77622..7cc40b24e51 100644 --- a/src/lib/actions/sandbox/process-recovery.test.ts +++ b/src/lib/actions/sandbox/process-recovery.test.ts @@ -135,6 +135,56 @@ describe("recreated sandbox OpenShell readiness", () => { expect(sleeps).toEqual([3]); }); + it("retries a blank OpenShell failure after an exact re-registration state", () => { + const captureOpenshellImpl = vi + .fn() + .mockReturnValueOnce({ + status: 1, + output: OPENSHELL_TRANSIENT_ERROR_PHASE_STDERR.trim(), + stdout: "", + stderr: OPENSHELL_TRANSIENT_ERROR_PHASE_STDERR, + }) + .mockReturnValueOnce({ status: 1, output: "", stdout: "", stderr: "" }) + .mockReturnValueOnce({ status: 0, output: "", stdout: "", stderr: "" }); + const beforeProbe = vi.fn(() => true); + const sleeps: number[] = []; + + expect( + waitForRecreatedSandboxOpenShellReady("recreated-box", { + beforeProbe, + captureOpenshellImpl, + intervalSeconds: 3, + sleepImpl: (seconds) => sleeps.push(seconds), + timeoutSeconds: 6, + }), + ).toBe(true); + expect(beforeProbe).toHaveBeenCalledTimes(3); + expect(captureOpenshellImpl).toHaveBeenCalledTimes(3); + expect(sleeps).toEqual([3, 3]); + }); + + it("keeps an isolated blank OpenShell failure terminal", () => { + const captureOpenshellImpl = vi.fn(() => ({ + status: 1, + output: "", + stdout: "", + stderr: "", + })); + const sleeps: number[] = []; + + expect( + waitForRecreatedSandboxOpenShellReady("recreated-box", { + beforeProbe: () => true, + captureOpenshellImpl, + intervalSeconds: 3, + sleepImpl: (seconds) => sleeps.push(seconds), + timeoutSeconds: 30, + }), + ).toBe(false); + expect(captureOpenshellImpl).toHaveBeenCalledOnce(); + expect(sleeps).toEqual([]); + }); + it("rides out a transient Error phase past the old 30s budget by default (#7227)", () => { // No timeoutSeconds option and no env override: the default recovery budget // must be large enough (120s, aligned with connect's readiness wait) to keep diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index 9bc28614aee..dd5bc93a9c9 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -742,6 +742,7 @@ function waitForRecreatedSandboxOpenShellReadyResult( ? Math.max(1, Math.floor(timeoutSeconds / intervalSeconds) + 1) : Math.max(1, Math.floor(timeoutSeconds) + 1); let lastOpenshellError: string | undefined; + let sawRetryableReRegistrationState = false; for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { const preGuardRemainingMs = deadlineMs - now(); @@ -785,10 +786,26 @@ function waitForRecreatedSandboxOpenShellReadyResult( // mutation outcome to reconcile. Treat that exact timeout as inconclusive // and retry behind the pinned managed-health guard on the next iteration. // All other unexpected OpenShell failures remain definitive. - if ( - !isRetryableOpenshellReRegistrationState(result, sandboxName) && - !isCommandTimeout(result) - ) { + const retryableReRegistrationState = isRetryableOpenshellReRegistrationState( + result, + sandboxName, + ); + if (retryableReRegistrationState) { + sawRetryableReRegistrationState = true; + } + // OpenShell 0.0.85 can follow its exact phase/session transition response + // with one or more blank exit-1 results while the replacement registration + // settles. A blank failure is inconclusive only after this loop observed a + // known retryable state and while the pinned managed-health guard continues + // to pass. Remove this follow-up when supported OpenShell versions keep + // returning a structured transition response until exec is available. + const retryableEmptyFollowUp = + sawRetryableReRegistrationState && + result.status === 1 && + !result.error && + String(result.stdout ?? "").trim() === "" && + String(result.stderr ?? "").trim() === ""; + if (!retryableReRegistrationState && !retryableEmptyFollowUp && !isCommandTimeout(result)) { return { failure: "openshell-readiness-failure", openshellError: lastOpenshellError, From 24681d5d3a645407740e1e228c48c5764e474857 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 29 Jul 2026 20:01:36 -0400 Subject: [PATCH 02/30] test(e2e): prove gateway health after restart Signed-off-by: Julie Yaunches --- test/e2e/live/sandbox-survival.test.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/e2e/live/sandbox-survival.test.ts b/test/e2e/live/sandbox-survival.test.ts index 6b2d6f753cf..ecee97503f6 100644 --- a/test/e2e/live/sandbox-survival.test.ts +++ b/test/e2e/live/sandbox-survival.test.ts @@ -5,8 +5,8 @@ * * Preserves the real boundaries: install.sh/onboard, Docker, OpenShell * gateway stop/start, NemoClaw registry/list/status, sandbox SSH/exec, durable - * /sandbox/.openclaw state markers, and inference.local chat completion before - * and after gateway restart. + * /sandbox/.openclaw state markers, OpenClaw gateway health after restart, and + * inference.local chat completion before and after gateway restart. */ import fs from "node:fs"; @@ -115,6 +115,7 @@ test( "NemoClaw registry, nemoclaw list/status, and openshell sandbox list discover the sandbox", "OpenShell version supports gateway resume and state persistence", "sandbox exec/SSH-equivalent access works before and after gateway restart", + "OpenClaw gateway health is restored after the OpenShell gateway restart", "inference.local returns a live PONG before and after gateway restart", "markers under /sandbox/.openclaw survive the gateway stop/start cycle", "final destroy removes the sandbox from NemoClaw registry/list state", @@ -324,6 +325,7 @@ test( artifactName: "post-restart-nemoclaw-status", timeoutMs: 120_000, }); + await stateValidation.from("cloud-openclaw-ready", instance); await expectSandboxExecAlive(SANDBOX_NAME, execShell, "post-restart-sandbox-exec-alive"); await stateValidation.expectSandboxMarkers(instance, markers, "post-restart-marker-read"); await stateValidation.expectSandboxDirectoryPopulated( @@ -360,6 +362,7 @@ test( assertions: { installCompleted: install.exitCode === 0, registryListedBeforeRestart: true, + openClawGatewayHealthyAfterRestart: true, inferenceLocalBeforeRestart: true, markersPersistedAfterRestart: true, inferenceLocalAfterRestart: true, From ae26a2b0797dbbd4cf59763e29356e64f6f5b903 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 29 Jul 2026 20:25:25 -0400 Subject: [PATCH 03/30] test(recovery): cover repeated blank follow-ups Signed-off-by: Julie Yaunches --- .../actions/sandbox/process-recovery.test.ts | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/lib/actions/sandbox/process-recovery.test.ts b/src/lib/actions/sandbox/process-recovery.test.ts index 7cc40b24e51..af8dbe5449b 100644 --- a/src/lib/actions/sandbox/process-recovery.test.ts +++ b/src/lib/actions/sandbox/process-recovery.test.ts @@ -135,7 +135,7 @@ describe("recreated sandbox OpenShell readiness", () => { expect(sleeps).toEqual([3]); }); - it("retries a blank OpenShell failure after an exact re-registration state", () => { + it("retries repeated blank OpenShell failures after an exact re-registration state", () => { const captureOpenshellImpl = vi .fn() .mockReturnValueOnce({ @@ -145,22 +145,28 @@ describe("recreated sandbox OpenShell readiness", () => { stderr: OPENSHELL_TRANSIENT_ERROR_PHASE_STDERR, }) .mockReturnValueOnce({ status: 1, output: "", stdout: "", stderr: "" }) + .mockReturnValueOnce({ status: 1, output: "", stdout: "", stderr: "" }) .mockReturnValueOnce({ status: 0, output: "", stdout: "", stderr: "" }); const beforeProbe = vi.fn(() => true); const sleeps: number[] = []; + let nowMs = 0; expect( waitForRecreatedSandboxOpenShellReady("recreated-box", { beforeProbe, captureOpenshellImpl, intervalSeconds: 3, - sleepImpl: (seconds) => sleeps.push(seconds), - timeoutSeconds: 6, + nowImpl: () => nowMs, + sleepImpl: (seconds) => { + sleeps.push(seconds); + nowMs += seconds * 1000; + }, + timeoutSeconds: 10, }), ).toBe(true); - expect(beforeProbe).toHaveBeenCalledTimes(3); - expect(captureOpenshellImpl).toHaveBeenCalledTimes(3); - expect(sleeps).toEqual([3, 3]); + expect(beforeProbe).toHaveBeenCalledTimes(4); + expect(captureOpenshellImpl).toHaveBeenCalledTimes(4); + expect(sleeps).toEqual([3, 3, 3]); }); it("keeps an isolated blank OpenShell failure terminal", () => { From 6609e6a0ff368a64d24e3d6e0ecb230c97ff84d5 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 29 Jul 2026 20:54:25 -0400 Subject: [PATCH 04/30] fix(recovery): retry opaque registration follow-ups Signed-off-by: Julie Yaunches --- .../actions/sandbox/process-recovery.test.ts | 36 +++++++++++++++++++ src/lib/actions/sandbox/process-recovery.ts | 14 ++++---- 2 files changed, 43 insertions(+), 7 deletions(-) diff --git a/src/lib/actions/sandbox/process-recovery.test.ts b/src/lib/actions/sandbox/process-recovery.test.ts index af8dbe5449b..4add83ec132 100644 --- a/src/lib/actions/sandbox/process-recovery.test.ts +++ b/src/lib/actions/sandbox/process-recovery.test.ts @@ -169,6 +169,42 @@ describe("recreated sandbox OpenShell readiness", () => { expect(sleeps).toEqual([3, 3, 3]); }); + it("retries opaque OpenShell follow-ups after an exact re-registration state", () => { + const bufferError = Object.assign(new Error("spawnSync openshell ENOBUFS"), { + code: "ENOBUFS", + }); + const captureOpenshellImpl = vi + .fn() + .mockReturnValueOnce({ + status: 1, + output: OPENSHELL_TRANSIENT_ERROR_PHASE_STDERR.trim(), + stdout: "", + stderr: OPENSHELL_TRANSIENT_ERROR_PHASE_STDERR, + }) + .mockReturnValueOnce({ status: 101, output: "", stdout: "", stderr: "" }) + .mockReturnValueOnce({ + status: null, + output: "", + stdout: "", + stderr: "", + error: bufferError, + }) + .mockReturnValueOnce({ status: 0, output: "", stdout: "", stderr: "" }); + const sleeps: number[] = []; + + expect( + waitForRecreatedSandboxOpenShellReady("recreated-box", { + beforeProbe: () => true, + captureOpenshellImpl, + intervalSeconds: 3, + sleepImpl: (seconds) => sleeps.push(seconds), + timeoutSeconds: 30, + }), + ).toBe(true); + expect(captureOpenshellImpl).toHaveBeenCalledTimes(4); + expect(sleeps).toEqual([3, 3, 3]); + }); + it("keeps an isolated blank OpenShell failure terminal", () => { const captureOpenshellImpl = vi.fn(() => ({ status: 1, diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index dd5bc93a9c9..ff82bdaa208 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -794,15 +794,15 @@ function waitForRecreatedSandboxOpenShellReadyResult( sawRetryableReRegistrationState = true; } // OpenShell 0.0.85 can follow its exact phase/session transition response - // with one or more blank exit-1 results while the replacement registration - // settles. A blank failure is inconclusive only after this loop observed a - // known retryable state and while the pinned managed-health guard continues - // to pass. Remove this follow-up when supported OpenShell versions keep - // returning a structured transition response until exec is available. + // with one or more output-free non-success results while the replacement + // registration settles. The process status and spawn error are not stable + // across those follow-ups. An opaque result is inconclusive only after this + // loop observed a known retryable state and while the pinned managed-health + // guard continues to pass. Remove this follow-up when supported OpenShell + // versions keep returning a structured transition response until exec is + // available. const retryableEmptyFollowUp = sawRetryableReRegistrationState && - result.status === 1 && - !result.error && String(result.stdout ?? "").trim() === "" && String(result.stderr ?? "").trim() === ""; if (!retryableReRegistrationState && !retryableEmptyFollowUp && !isCommandTimeout(result)) { From 31c30def94c5a4a2ed9187d13f6926678cc599f8 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 29 Jul 2026 21:21:37 -0400 Subject: [PATCH 05/30] fix(recovery): retry informational registration follow-ups Signed-off-by: Julie Yaunches --- .../actions/sandbox/process-recovery.test.ts | 9 +++++-- src/lib/actions/sandbox/process-recovery.ts | 26 ++++++++++--------- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/src/lib/actions/sandbox/process-recovery.test.ts b/src/lib/actions/sandbox/process-recovery.test.ts index 4add83ec132..a594e8fa89a 100644 --- a/src/lib/actions/sandbox/process-recovery.test.ts +++ b/src/lib/actions/sandbox/process-recovery.test.ts @@ -169,7 +169,7 @@ describe("recreated sandbox OpenShell readiness", () => { expect(sleeps).toEqual([3, 3, 3]); }); - it("retries opaque OpenShell follow-ups after an exact re-registration state", () => { + it("retries unstructured OpenShell follow-ups after an exact re-registration state", () => { const bufferError = Object.assign(new Error("spawnSync openshell ENOBUFS"), { code: "ENOBUFS", }); @@ -181,7 +181,12 @@ describe("recreated sandbox OpenShell readiness", () => { stdout: "", stderr: OPENSHELL_TRANSIENT_ERROR_PHASE_STDERR, }) - .mockReturnValueOnce({ status: 101, output: "", stdout: "", stderr: "" }) + .mockReturnValueOnce({ + status: 101, + output: "Waiting for sandbox registration", + stdout: "Waiting for sandbox registration\n", + stderr: "", + }) .mockReturnValueOnce({ status: null, output: "", diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index ff82bdaa208..8d0d331aa47 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -794,18 +794,20 @@ function waitForRecreatedSandboxOpenShellReadyResult( sawRetryableReRegistrationState = true; } // OpenShell 0.0.85 can follow its exact phase/session transition response - // with one or more output-free non-success results while the replacement - // registration settles. The process status and spawn error are not stable - // across those follow-ups. An opaque result is inconclusive only after this - // loop observed a known retryable state and while the pinned managed-health - // guard continues to pass. Remove this follow-up when supported OpenShell - // versions keep returning a structured transition response until exec is - // available. - const retryableEmptyFollowUp = - sawRetryableReRegistrationState && - String(result.stdout ?? "").trim() === "" && - String(result.stderr ?? "").trim() === ""; - if (!retryableReRegistrationState && !retryableEmptyFollowUp && !isCommandTimeout(result)) { + // with one or more non-success results that have no structured stderr while + // the replacement registration settles. The process status, spawn error, + // and informational stdout are not stable across those follow-ups. An + // unstructured result is inconclusive only after this loop observed a known + // retryable state and while the pinned managed-health guard continues to + // pass. Remove this follow-up when supported OpenShell versions keep + // returning a structured transition response until exec is available. + const retryableUnstructuredFollowUp = + sawRetryableReRegistrationState && String(result.stderr ?? "").trim() === ""; + if ( + !retryableReRegistrationState && + !retryableUnstructuredFollowUp && + !isCommandTimeout(result) + ) { return { failure: "openshell-readiness-failure", openshellError: lastOpenshellError, From 0e531dd74b4b39d32736c39b6f8df7b6abd4c733 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 29 Jul 2026 21:56:47 -0400 Subject: [PATCH 06/30] fix(recovery): retry exact registration error metadata Signed-off-by: Julie Yaunches --- .../actions/sandbox/process-recovery.test.ts | 43 +++++++++++++++++++ src/lib/actions/sandbox/process-recovery.ts | 10 ++--- 2 files changed, 48 insertions(+), 5 deletions(-) diff --git a/src/lib/actions/sandbox/process-recovery.test.ts b/src/lib/actions/sandbox/process-recovery.test.ts index a594e8fa89a..77a023eb720 100644 --- a/src/lib/actions/sandbox/process-recovery.test.ts +++ b/src/lib/actions/sandbox/process-recovery.test.ts @@ -210,6 +210,49 @@ describe("recreated sandbox OpenShell readiness", () => { expect(sleeps).toEqual([3, 3, 3]); }); + it("retries the exact Error phase when capture metadata is unstable", () => { + const bufferError = Object.assign(new Error("spawnSync openshell ENOBUFS"), { + code: "ENOBUFS", + }); + const captureOpenshellImpl = vi + .fn() + .mockReturnValueOnce({ + status: 1, + output: OPENSHELL_TRANSIENT_ERROR_PHASE_STDERR.trim(), + stdout: "", + stderr: OPENSHELL_TRANSIENT_ERROR_PHASE_STDERR, + }) + .mockReturnValueOnce({ + status: null, + output: + `Waiting for sandbox registration\n${OPENSHELL_TRANSIENT_ERROR_PHASE_STDERR}`.trim(), + stdout: "Waiting for sandbox registration\n", + stderr: OPENSHELL_TRANSIENT_ERROR_PHASE_STDERR, + error: bufferError, + }) + .mockReturnValueOnce({ + status: null, + output: "× │", + stdout: "", + stderr: "× │", + error: bufferError, + }) + .mockReturnValueOnce({ status: 0, output: "", stdout: "", stderr: "" }); + const sleeps: number[] = []; + + expect( + waitForRecreatedSandboxOpenShellReady("recreated-box", { + beforeProbe: () => true, + captureOpenshellImpl, + intervalSeconds: 3, + sleepImpl: (seconds) => sleeps.push(seconds), + timeoutSeconds: 30, + }), + ).toBe(true); + expect(captureOpenshellImpl).toHaveBeenCalledTimes(4); + expect(sleeps).toEqual([3, 3, 3]); + }); + it("keeps an isolated blank OpenShell failure terminal", () => { const captureOpenshellImpl = vi.fn(() => ({ status: 1, diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index 8d0d331aa47..d4f9f945e28 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -607,18 +607,19 @@ function isRetryableOpenshellReRegistrationState( result: ReturnType, sandboxName: string, ): boolean { - if (!hasRetryableOpenshellFailureShape(result)) return false; const error = normalizeOpenshellStructuredError(String(result.stderr)); // OpenShell can publish Ready before replacement registration settles. // Retry only if the readiness probe reports phase Error for this sandbox. - // The CLI can emit informational stdout before this exact stderr refusal; - // stdout does not change the result of the read-only `true` probe. + // The CLI can emit informational stdout or return unstable process metadata + // with this exact stderr refusal; neither changes the result of the read-only + // `true` probe. if ( error === `Error: sandbox '${sandboxName}' is not ready (phase: Error); wait for it to reach Ready state.` ) { return true; } + if (!hasRetryableOpenshellFailureShape(result)) return false; // All less-specific transient signatures remain constrained to an otherwise // empty stdout stream so unrelated command output cannot be reclassified. if (String(result.stdout ?? "").trim() !== "") return false; @@ -801,8 +802,7 @@ function waitForRecreatedSandboxOpenShellReadyResult( // retryable state and while the pinned managed-health guard continues to // pass. Remove this follow-up when supported OpenShell versions keep // returning a structured transition response until exec is available. - const retryableUnstructuredFollowUp = - sawRetryableReRegistrationState && String(result.stderr ?? "").trim() === ""; + const retryableUnstructuredFollowUp = sawRetryableReRegistrationState && !openshellError; if ( !retryableReRegistrationState && !retryableUnstructuredFollowUp && From 9faffab79d8a62ad026d756f5ed1b9b8df29d6f7 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Wed, 29 Jul 2026 19:12:12 -0700 Subject: [PATCH 07/30] fix(recovery): preserve supervisor during container handoff Signed-off-by: Prekshi Vyas --- .../sandbox/supervisor-relaunch.test.ts | 1 + .../actions/sandbox/supervisor-relaunch.ts | 1 + .../onboard/docker-gpu-patch-finalize.test.ts | 62 +++++++++++++ src/lib/onboard/docker-gpu-patch-finalize.ts | 16 +++- src/lib/onboard/docker-gpu-patch-recreate.ts | 53 +++++++---- src/lib/onboard/docker-gpu-patch-rollback.ts | 18 +++- src/lib/onboard/docker-gpu-patch-types.ts | 6 ++ .../docker-startup-command-patch.test.ts | 93 +++++++++++++++++++ .../onboard/docker-startup-command-patch.ts | 1 + test/e2e/live/sandbox-survival.test.ts | 2 +- 10 files changed, 227 insertions(+), 26 deletions(-) diff --git a/src/lib/actions/sandbox/supervisor-relaunch.test.ts b/src/lib/actions/sandbox/supervisor-relaunch.test.ts index f66776e636d..50937047195 100644 --- a/src/lib/actions/sandbox/supervisor-relaunch.test.ts +++ b/src/lib/actions/sandbox/supervisor-relaunch.test.ts @@ -114,6 +114,7 @@ describe("relaunchManagedSupervisorSession", () => { expect(options).toMatchObject({ sandboxName: "alpha", expectedOldContainerId: "old-container-id", + keepOriginalRunningUntilFinalize: true, waitForSupervisor: false, }); const serialized = options?.openshellSandboxCommand.join(" ") ?? ""; diff --git a/src/lib/actions/sandbox/supervisor-relaunch.ts b/src/lib/actions/sandbox/supervisor-relaunch.ts index 729827fa866..fecf06465b7 100644 --- a/src/lib/actions/sandbox/supervisor-relaunch.ts +++ b/src/lib/actions/sandbox/supervisor-relaunch.ts @@ -141,6 +141,7 @@ export function relaunchManagedSupervisorSession( sandboxName, openshellSandboxCommand: startupCommand, expectedOldContainerId: containerId, + keepOriginalRunningUntilFinalize: true, waitForSupervisor: false, }); let completed: { supervisorReady: boolean; outcome: DockerGpuPatchFinalizeOutcome } | null = diff --git a/src/lib/onboard/docker-gpu-patch-finalize.test.ts b/src/lib/onboard/docker-gpu-patch-finalize.test.ts index 5a691435ce4..fa53461e1f6 100644 --- a/src/lib/onboard/docker-gpu-patch-finalize.test.ts +++ b/src/lib/onboard/docker-gpu-patch-finalize.test.ts @@ -37,6 +37,42 @@ describe("finalizeDockerGpuPatchBackup", () => { ); }); + it("force-removes the running original container when supervisor readiness is confirmed", () => { + const dockerForceRm = vi.fn((_name: string) => ({ status: 0 })); + const dockerRm = vi.fn((_name: string) => ({ status: 0 })); + const outcome = finalizeDockerGpuPatchBackup( + { + result: { ...deferredCreateResult(), backupWasRunning: true }, + supervisorReady: true, + }, + { dockerForceRm, dockerRm }, + ); + + expect(outcome).toEqual({ backupRemoved: true, rolledBack: false }); + expect(dockerForceRm).toHaveBeenCalledWith( + "openshell-alpha-nemoclaw-gpu-backup-1780491860342", + expect.objectContaining({ ignoreError: true }), + ); + expect(dockerRm).not.toHaveBeenCalled(); + }); + + it("reports backupRemoved false when force-removal of the running original container fails", () => { + const outcome = finalizeDockerGpuPatchBackup( + { + result: { ...deferredCreateResult(), backupWasRunning: true }, + supervisorReady: true, + }, + { + dockerForceRm: vi.fn(() => ({ + status: 1, + stderr: "container removal failed", + })), + }, + ); + + expect(outcome).toEqual({ backupRemoved: false, rolledBack: false }); + }); + it("rolls back to the backup container when supervisor reconnect failed", () => { const dockerStop = vi.fn(() => ({ status: 0 })); const dockerRm = vi.fn((_name: string) => ({ status: 0 })); @@ -65,6 +101,32 @@ describe("finalizeDockerGpuPatchBackup", () => { ).toBe(false); }); + it("rolls back to the still-running original container without restarting it", () => { + const dockerStop = vi.fn(() => ({ status: 0 })); + const dockerRm = vi.fn((_name: string) => ({ status: 0 })); + const dockerRename = vi.fn((_old: string, _next: string) => ({ status: 0 })); + const dockerStart = vi.fn(() => ({ status: 0 })); + const outcome = finalizeDockerGpuPatchBackup( + { + result: { ...deferredCreateResult(), backupWasRunning: true }, + supervisorReady: false, + }, + { dockerStop, dockerRm, dockerRename, dockerStart }, + ); + + expect(outcome).toEqual({ backupRemoved: false, rolledBack: true }); + expect(dockerStop).toHaveBeenCalledWith( + "new-container-id", + expect.objectContaining({ ignoreError: true }), + ); + expect(dockerRename).toHaveBeenCalledWith( + "openshell-alpha-nemoclaw-gpu-backup-1780491860342", + "openshell-alpha", + expect.objectContaining({ ignoreError: true }), + ); + expect(dockerStart).not.toHaveBeenCalled(); + }); + it("reports rolledBack=false when restoring the backup fails", () => { const dockerStop = vi.fn(() => ({ status: 0 })); const dockerRm = vi.fn((_name: string) => ({ status: 0 })); diff --git a/src/lib/onboard/docker-gpu-patch-finalize.ts b/src/lib/onboard/docker-gpu-patch-finalize.ts index 4eeeedf399b..e3c0dc65ff6 100644 --- a/src/lib/onboard/docker-gpu-patch-finalize.ts +++ b/src/lib/onboard/docker-gpu-patch-finalize.ts @@ -65,7 +65,9 @@ export function finalizeDockerGpuPatchBackup( // even if `docker rm` cannot delete it (e.g. concurrent admin action, // daemon timeout). Reflect the actual rm status in the outcome so // diagnostics can flag a leaked backup container. - const rmResult = resolved.dockerRm(options.result.backupContainerName, containerOpts); + const rmResult = options.result.backupWasRunning + ? resolved.dockerForceRm(options.result.backupContainerName, containerOpts) + : resolved.dockerRm(options.result.backupContainerName, containerOpts); return { backupRemoved: hasZeroDockerExitStatus(rmResult), rolledBack: false }; } const rolledBack = rollbackToBackupContainer( @@ -73,6 +75,7 @@ export function finalizeDockerGpuPatchBackup( newContainerId: options.result.newContainerId, backupContainerName: options.result.backupContainerName, originalName: options.result.originalName, + backupWasRunning: options.result.backupWasRunning, }, resolved, ); @@ -85,7 +88,12 @@ export type SupervisorReconnectOutcome = export function reconcileSupervisorReconnect( execReady: boolean, - refs: { newContainerId: string; backupContainerName: string; originalName: string }, + refs: { + newContainerId: string; + backupContainerName: string; + originalName: string; + backupWasRunning?: boolean; + }, deps: DockerGpuPatchDeps, ): SupervisorReconnectOutcome { const resolved = resolveDockerGpuPatchRollbackDeps(deps); @@ -100,7 +108,9 @@ export function reconcileSupervisorReconnect( // leaked backup container but the user-visible sandbox is healthy. // Surface the actual rm status so callers can fold it into diagnostics // alongside the deferred-finalize path in `finalizeDockerGpuPatchBackup`. - const rmResult = resolved.dockerRm(refs.backupContainerName, containerOpts); + const rmResult = refs.backupWasRunning + ? resolved.dockerForceRm(refs.backupContainerName, containerOpts) + : resolved.dockerRm(refs.backupContainerName, containerOpts); return { execReady: true, backupRemoved: hasZeroDockerExitStatus(rmResult) }; } const rolledBack = rollbackToBackupContainer(refs, resolved); diff --git a/src/lib/onboard/docker-gpu-patch-recreate.ts b/src/lib/onboard/docker-gpu-patch-recreate.ts index b205c144af8..7587437378a 100644 --- a/src/lib/onboard/docker-gpu-patch-recreate.ts +++ b/src/lib/onboard/docker-gpu-patch-recreate.ts @@ -3,6 +3,7 @@ import { dockerCapture, + dockerForceRm, dockerRename, dockerRm, dockerRun, @@ -48,6 +49,7 @@ type RecreateDeps = Required< Pick< DockerGpuPatchDeps, | "dockerCapture" + | "dockerForceRm" | "dockerRun" | "dockerRunDetached" | "dockerRename" @@ -66,6 +68,7 @@ type RecreateDeps = Required< function recreateDeps(deps: DockerGpuPatchDeps): RecreateDeps { return { dockerCapture, + dockerForceRm, dockerRun, dockerRunDetached, dockerRename, @@ -155,6 +158,7 @@ export function recreateOpenShellDockerSandboxContainer( gpuDevice?: string | null; timeoutSecs?: number; waitForSupervisor?: boolean; + keepOriginalRunningUntilFinalize?: boolean; openshellSandboxCommand?: readonly string[] | null; requiredUlimits?: readonly import("./docker-gpu-patch-types").DockerUlimit[] | null; expectedOldContainerId?: string | null; @@ -171,6 +175,11 @@ export function recreateOpenShellDockerSandboxContainer( }; try { validateRequiredDockerUlimits(options.requiredUlimits); + if (options.keepOriginalRunningUntilFinalize && options.waitForSupervisor !== false) { + throw new Error( + "Keeping the original OpenShell supervisor running requires deferred supervisor finalization.", + ); + } const containerIds = findOpenShellDockerSandboxContainerIds(options.sandboxName, deps); const oldContainerId = containerIds[0]; if (!oldContainerId) { @@ -284,21 +293,24 @@ export function recreateOpenShellDockerSandboxContainer( suppressOutput: true, timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, }; - const stopResult = d.dockerStop(oldContainerId, { - ...containerMutationOptions, - timeout: DOCKER_GPU_PATCH_STOP_TIMEOUT_MS, - }); - if (!hasZeroDockerExitStatus(stopResult)) { - context.rolledBack = hasZeroDockerExitStatus( - d.dockerStart(oldContainerId, containerMutationOptions), - ); - throw new Error( - `Could not stop original sandbox container: ${resultText(stopResult)}; ${ - context.rolledBack - ? "original sandbox container confirmed running" - : "restart failed; original sandbox container may be stopped" - }`, - ); + const backupWasRunning = options.keepOriginalRunningUntilFinalize === true; + if (!backupWasRunning) { + const stopResult = d.dockerStop(oldContainerId, { + ...containerMutationOptions, + timeout: DOCKER_GPU_PATCH_STOP_TIMEOUT_MS, + }); + if (!hasZeroDockerExitStatus(stopResult)) { + context.rolledBack = hasZeroDockerExitStatus( + d.dockerStart(oldContainerId, containerMutationOptions), + ); + throw new Error( + `Could not stop original sandbox container: ${resultText(stopResult)}; ${ + context.rolledBack + ? "original sandbox container confirmed running" + : "restart failed; original sandbox container may be stopped" + }`, + ); + } } const renameResult = d.dockerRename( oldContainerId, @@ -307,9 +319,9 @@ export function recreateOpenShellDockerSandboxContainer( ); if (!hasZeroDockerExitStatus(renameResult)) { d.dockerRename(backupContainerName, originalName, containerMutationOptions); - const restarted = hasZeroDockerExitStatus( - d.dockerStart(oldContainerId, containerMutationOptions), - ); + const restarted = + backupWasRunning || + hasZeroDockerExitStatus(d.dockerStart(oldContainerId, containerMutationOptions)); let originalNameRestored = false; try { originalNameRestored = @@ -334,7 +346,7 @@ export function recreateOpenShellDockerSandboxContainer( }); if (!hasZeroDockerExitStatus(runResult)) { context.rolledBack = restoreDockerGpuPatchBackupAfterRecreateFailure( - { newContainerId: originalName, backupContainerName, originalName }, + { newContainerId: originalName, backupContainerName, originalName, backupWasRunning }, deps, ); const containerDescription = @@ -360,7 +372,7 @@ export function recreateOpenShellDockerSandboxContainer( ); if (!newContainerId) { context.rolledBack = restoreDockerGpuPatchBackupAfterRecreateFailure( - { newContainerId: originalName, backupContainerName, originalName }, + { newContainerId: originalName, backupContainerName, originalName, backupWasRunning }, deps, ); const containerDescription = @@ -384,6 +396,7 @@ export function recreateOpenShellDockerSandboxContainer( originalName, backupContainerName, mode: selectedMode, + backupWasRunning, backupRemoved, }); if (options.waitForSupervisor === false) return result(false); diff --git a/src/lib/onboard/docker-gpu-patch-rollback.ts b/src/lib/onboard/docker-gpu-patch-rollback.ts index 81532529503..a18ae91febc 100644 --- a/src/lib/onboard/docker-gpu-patch-rollback.ts +++ b/src/lib/onboard/docker-gpu-patch-rollback.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { + dockerForceRm as defaultDockerForceRm, dockerRename as defaultDockerRename, dockerRm as defaultDockerRm, dockerStart as defaultDockerStart, @@ -26,6 +27,7 @@ type DockerRenameFn = ( ) => DockerRunResult; export type ResolvedDockerGpuPatchRollbackDeps = { + dockerForceRm: DockerContainerFn; dockerStop: DockerContainerFn; dockerRm: DockerContainerFn; dockerRename: DockerRenameFn; @@ -36,6 +38,7 @@ export function resolveDockerGpuPatchRollbackDeps( deps: DockerGpuPatchDeps, ): ResolvedDockerGpuPatchRollbackDeps { return { + dockerForceRm: deps.dockerForceRm ?? defaultDockerForceRm, dockerStop: deps.dockerStop ?? defaultDockerStop, dockerRm: deps.dockerRm ?? defaultDockerRm, dockerRename: deps.dockerRename ?? defaultDockerRename, @@ -44,7 +47,12 @@ export function resolveDockerGpuPatchRollbackDeps( } export function rollbackToBackupContainer( - refs: { newContainerId: string; backupContainerName: string; originalName: string }, + refs: { + newContainerId: string; + backupContainerName: string; + originalName: string; + backupWasRunning?: boolean; + }, deps: ResolvedDockerGpuPatchRollbackDeps, ): boolean { const containerOpts = { @@ -56,13 +64,19 @@ export function rollbackToBackupContainer( deps.dockerRm(refs.newContainerId, containerOpts); const restored = deps.dockerRename(refs.backupContainerName, refs.originalName, containerOpts); if (!hasZeroDockerExitStatus(restored)) return false; + if (refs.backupWasRunning) return true; const started = deps.dockerStart(refs.originalName, containerOpts); return hasZeroDockerExitStatus(started); } /** Restore the original sandbox after `docker run` fails during GPU recreation. */ export function restoreDockerGpuPatchBackupAfterRecreateFailure( - refs: { newContainerId: string; backupContainerName: string; originalName: string }, + refs: { + newContainerId: string; + backupContainerName: string; + originalName: string; + backupWasRunning?: boolean; + }, deps: DockerGpuPatchDeps = {}, ): boolean { return rollbackToBackupContainer(refs, resolveDockerGpuPatchRollbackDeps(deps)); diff --git a/src/lib/onboard/docker-gpu-patch-types.ts b/src/lib/onboard/docker-gpu-patch-types.ts index 32be0afe46f..a935972e242 100644 --- a/src/lib/onboard/docker-gpu-patch-types.ts +++ b/src/lib/onboard/docker-gpu-patch-types.ts @@ -27,6 +27,7 @@ export type DockerGpuPatchDeps = { dockerRun?: DockerRunFn; dockerRunDetached?: DockerRunFn; dockerRename?: DockerRenameFn; + dockerForceRm?: DockerContainerFn; dockerRm?: DockerContainerFn; dockerStart?: DockerContainerFn; dockerStop?: DockerContainerFn; @@ -91,6 +92,11 @@ export type DockerGpuPatchResult = { originalName: string; backupContainerName: string; mode: DockerGpuPatchMode; + // True when a deferred startup-command recreation kept the original + // OpenShell supervisor running while the replacement established managed + // health. Finalization attempts to force-remove that original container on + // success and does not try to restart it during rollback. + backupWasRunning?: boolean; // True when the patch path also confirmed supervisor reconnect AND removed // the backup container. False when the caller deferred the reconnect wait // (via `waitForSupervisor: false`); the backup is still in place and the diff --git a/src/lib/onboard/docker-startup-command-patch.test.ts b/src/lib/onboard/docker-startup-command-patch.test.ts index 71c5c9d12c2..b4557a7302b 100644 --- a/src/lib/onboard/docker-startup-command-patch.test.ts +++ b/src/lib/onboard/docker-startup-command-patch.test.ts @@ -43,6 +43,99 @@ function inspectFixture(): DockerContainerInspect { } describe("Docker startup-command patch", () => { + it("keeps the registered supervisor running until deferred recovery finalizes", () => { + const dockerCapture = vi.fn((args: readonly string[]) => + args[0] === "ps" + ? "old-container-id\n" + : args[0] === "inspect" + ? JSON.stringify([inspectFixture()]) + : "", + ); + const dockerStop = vi.fn(() => ({ status: 0 })); + const dockerRename = vi.fn(() => ({ status: 0 })); + const dockerRunDetached = vi.fn(() => ({ status: 0, stdout: "new-container-id\n" })); + + const result = recreateStartupCommandForTest( + { + sandboxName: "alpha", + keepOriginalRunningUntilFinalize: true, + waitForSupervisor: false, + openshellSandboxCommand: ["env", "nemoclaw-start"], + }, + { + dockerCapture, + dockerRunDetached, + dockerRename, + dockerStop, + now: () => new Date("2026-07-10T00:00:00Z"), + }, + ); + + expect(result).toMatchObject({ + newContainerId: "new-container-id", + backupWasRunning: true, + backupRemoved: false, + }); + expect(dockerStop).not.toHaveBeenCalled(); + expect(dockerRename.mock.invocationCallOrder[0]).toBeLessThan( + dockerRunDetached.mock.invocationCallOrder[0] ?? Number.POSITIVE_INFINITY, + ); + }); + + it("requires deferred finalization when preserving the registered supervisor", () => { + const dockerCapture = vi.fn(); + + expect(() => + recreateStartupCommandForTest( + { + sandboxName: "alpha", + keepOriginalRunningUntilFinalize: true, + openshellSandboxCommand: ["env", "nemoclaw-start"], + }, + { dockerCapture }, + ), + ).toThrow(/requires deferred supervisor finalization/); + expect(dockerCapture).not.toHaveBeenCalled(); + }); + + it("rolls back to the running original container without restarting it when replacement creation fails", () => { + const dockerRename = vi.fn(() => ({ status: 0 })); + const dockerStart = vi.fn(() => ({ status: 0 })); + + expect(() => + recreateStartupCommandForTest( + { + sandboxName: "alpha", + keepOriginalRunningUntilFinalize: true, + waitForSupervisor: false, + openshellSandboxCommand: ["env", "nemoclaw-start"], + }, + { + dockerCapture: vi.fn((args: readonly string[]) => + args[0] === "ps" + ? "old-container-id\n" + : args[0] === "inspect" + ? JSON.stringify([inspectFixture()]) + : "", + ), + dockerRunDetached: vi.fn(() => ({ status: 1, stderr: "boom" })), + dockerRename, + dockerRm: vi.fn(() => ({ status: 0 })), + dockerStart, + dockerStop: vi.fn(() => ({ status: 0 })), + now: () => new Date("2026-07-10T00:00:00Z"), + }, + ), + ).toThrow(/Could not start recreated sandbox container: boom; pre-patch sandbox restored/); + + expect(dockerRename).toHaveBeenLastCalledWith( + expect.stringContaining("openshell-alpha-nemoclaw-gpu-backup-"), + "openshell-alpha", + expect.objectContaining({ ignoreError: true }), + ); + expect(dockerStart).not.toHaveBeenCalled(); + }); + it("persists the startup command without adding GPU-only container privileges", () => { const dockerCaptureOutput: Record = { ps: "old-container-id\n", diff --git a/src/lib/onboard/docker-startup-command-patch.ts b/src/lib/onboard/docker-startup-command-patch.ts index 2b5b6f761ef..6541152e2b7 100644 --- a/src/lib/onboard/docker-startup-command-patch.ts +++ b/src/lib/onboard/docker-startup-command-patch.ts @@ -15,6 +15,7 @@ export function recreateOpenShellDockerSandboxWithStartupCommand( sandboxName: string; timeoutSecs?: number; waitForSupervisor?: boolean; + keepOriginalRunningUntilFinalize?: boolean; openshellSandboxCommand: readonly string[]; requiredUlimits?: readonly import("./docker-gpu-patch-types").DockerUlimit[] | null; expectedOldContainerId?: string | null; diff --git a/test/e2e/live/sandbox-survival.test.ts b/test/e2e/live/sandbox-survival.test.ts index ecee97503f6..6a26b3db67e 100644 --- a/test/e2e/live/sandbox-survival.test.ts +++ b/test/e2e/live/sandbox-survival.test.ts @@ -115,7 +115,7 @@ test( "NemoClaw registry, nemoclaw list/status, and openshell sandbox list discover the sandbox", "OpenShell version supports gateway resume and state persistence", "sandbox exec/SSH-equivalent access works before and after gateway restart", - "OpenClaw gateway health is restored after the OpenShell gateway restart", + "OpenClaw gateway passes its health check after the OpenShell gateway restart", "inference.local returns a live PONG before and after gateway restart", "markers under /sandbox/.openclaw survive the gateway stop/start cycle", "final destroy removes the sandbox from NemoClaw registry/list state", From b2b66c5df34d1b9500347da1ebf44450ce16d3a2 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 29 Jul 2026 22:29:46 -0400 Subject: [PATCH 08/30] test(recovery): cover supervisor handoff lifecycle Signed-off-by: Julie Yaunches --- .../sandbox/supervisor-relaunch.test.ts | 107 +++++++++++++++++- src/lib/onboard/docker-gpu-patch-finalize.ts | 35 +++--- 2 files changed, 127 insertions(+), 15 deletions(-) diff --git a/src/lib/actions/sandbox/supervisor-relaunch.test.ts b/src/lib/actions/sandbox/supervisor-relaunch.test.ts index 50937047195..6ae2ed96211 100644 --- a/src/lib/actions/sandbox/supervisor-relaunch.test.ts +++ b/src/lib/actions/sandbox/supervisor-relaunch.test.ts @@ -2,7 +2,9 @@ // SPDX-License-Identifier: Apache-2.0 import { afterEach, describe, expect, it, vi } from "vitest"; -import type { DockerGpuPatchResult } from "../../onboard/docker-gpu-patch"; +import type { DockerGpuPatchDeps, DockerGpuPatchResult } from "../../onboard/docker-gpu-patch"; +import { finalizeDockerGpuPatchBackup } from "../../onboard/docker-gpu-patch-finalize"; +import { recreateOpenShellDockerSandboxWithStartupCommand } from "../../onboard/docker-startup-command-patch"; import { type ManagedSupervisorRelaunchDeps, relaunchManagedSupervisorSession, @@ -30,6 +32,60 @@ function patchResult(): DockerGpuPatchResult { }; } +function composedHandoffDeps() { + const inspect = { + Id: "old-container-id", + Image: `sha256:${"c".repeat(64)}`, + Name: "/openshell-alpha", + Config: { + Image: "openshell/sandbox:abc", + Env: ["OPENSHELL_SANDBOX_COMMAND=sleep infinity"], + Labels: { + "openshell.ai/managed-by": "openshell", + "openshell.ai/sandbox-name": "alpha", + }, + Entrypoint: ["/opt/openshell/bin/openshell-sandbox"], + Cmd: [], + User: "0", + WorkingDir: "/workspace", + }, + HostConfig: { + NetworkMode: "openshell-docker", + RestartPolicy: { Name: "unless-stopped" }, + CapAdd: [], + SecurityOpt: [], + }, + }; + const dockerCapture = vi.fn((args: readonly string[]) => + args[0] === "ps" ? "old-container-id\n" : JSON.stringify([inspect]), + ); + const dockerForceRm = vi.fn(() => ({ status: 0 })); + const dockerRm = vi.fn(() => ({ status: 0 })); + const dockerStart = vi.fn(() => ({ status: 0 })); + const dockerStop = vi.fn(() => ({ status: 0 })); + const dockerRename = vi.fn(() => ({ status: 0 })); + const dockerRunDetached = vi.fn(() => ({ status: 0, stdout: "new-container-id\n" })); + const dockerDeps: DockerGpuPatchDeps = { + detectSandboxFallbackDns: () => null, + dockerCapture, + dockerForceRm, + dockerRename, + dockerRm, + dockerRunDetached, + dockerStart, + dockerStop, + now: () => new Date("2026-07-10T00:00:00Z"), + }; + return { + dockerDeps, + dockerForceRm, + dockerRename, + dockerRm, + dockerStart, + dockerStop, + }; +} + function baseDeps(overrides: ManagedSupervisorRelaunchDeps = {}) { return { getSandbox: vi.fn(() => ({ @@ -142,6 +198,55 @@ describe("relaunchManagedSupervisorSession", () => { }); }); + it("removes the running original only after the handoff is finalized as ready", () => { + const handoff = composedHandoffDeps(); + const deps = baseDeps({ + recreate: (options) => + recreateOpenShellDockerSandboxWithStartupCommand(options, handoff.dockerDeps), + finalize: (options) => finalizeDockerGpuPatchBackup(options, handoff.dockerDeps), + }); + + const relaunch = relaunchManagedSupervisorSession("alpha", { quiet: true, deps }); + + expect(relaunch?.containerId).toBe("new-container-id"); + expect(handoff.dockerStop).not.toHaveBeenCalled(); + expect(handoff.dockerForceRm).not.toHaveBeenCalled(); + + expect(relaunch?.finalize(true)).toEqual({ backupRemoved: true, rolledBack: false }); + expect(handoff.dockerForceRm).toHaveBeenCalledWith( + expect.stringContaining("openshell-alpha-nemoclaw-gpu-backup-"), + expect.objectContaining({ ignoreError: true }), + ); + expect(handoff.dockerStart).not.toHaveBeenCalled(); + }); + + it("restores the running original without restarting it when the handoff is finalized as not ready", () => { + const handoff = composedHandoffDeps(); + const deps = baseDeps({ + recreate: (options) => + recreateOpenShellDockerSandboxWithStartupCommand(options, handoff.dockerDeps), + finalize: (options) => finalizeDockerGpuPatchBackup(options, handoff.dockerDeps), + }); + const relaunch = relaunchManagedSupervisorSession("alpha", { quiet: true, deps }); + + expect(relaunch?.finalize(false)).toEqual({ backupRemoved: false, rolledBack: true }); + expect(handoff.dockerStop).toHaveBeenCalledWith( + "new-container-id", + expect.objectContaining({ ignoreError: true }), + ); + expect(handoff.dockerRm).toHaveBeenCalledWith( + "new-container-id", + expect.objectContaining({ ignoreError: true }), + ); + expect(handoff.dockerRename).toHaveBeenLastCalledWith( + expect.stringContaining("openshell-alpha-nemoclaw-gpu-backup-"), + "openshell-alpha", + expect.objectContaining({ ignoreError: true }), + ); + expect(handoff.dockerStart).not.toHaveBeenCalled(); + expect(handoff.dockerForceRm).not.toHaveBeenCalled(); + }); + it("returns null when the pinned recreation fails", () => { const deps = baseDeps({ recreate: vi.fn(() => { diff --git a/src/lib/onboard/docker-gpu-patch-finalize.ts b/src/lib/onboard/docker-gpu-patch-finalize.ts index e3c0dc65ff6..afed03ad355 100644 --- a/src/lib/onboard/docker-gpu-patch-finalize.ts +++ b/src/lib/onboard/docker-gpu-patch-finalize.ts @@ -1,16 +1,20 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 // -// Source-of-truth: this module is a NemoClaw-side workaround. The invalid -// state it recovers from is "OpenShell Docker-driver GPU patch left the -// sandbox in a deleted-backup / failed-new state when the post-recreate -// supervisor reconnect could not confirm the GPU container". The preferred -// source boundary for the fix is OpenShell: a Docker-driver sandbox create -// that natively accepts NVIDIA GPU access would remove the need for the -// post-create container recreation NemoClaw performs here. Until OpenShell -// supports that natively, NemoClaw recreates the container with GPU access -// and uses this module to either confirm the new container or restore the -// pre-patch backup. Regression coverage: +// Source-of-truth: this module finalizes two NemoClaw-side Docker recreation +// workarounds. For GPU patching, the invalid state is "OpenShell Docker-driver +// GPU patch left the sandbox in a deleted-backup / failed-new state when the +// post-recreate supervisor reconnect could not confirm the GPU container". +// For legacy supervisor relaunch on OpenShell 0.0.85, the invalid state is +// "stopping the registered keepalive container lets the Docker watcher publish +// a terminal Error phase before the replacement supervisor registers". That +// caller keeps the renamed original container running until pinned managed +// health confirms the replacement, then force-removes it; failed health rolls +// back to the still-running original without restarting it. +// +// The preferred source boundaries are OpenShell's Docker create and watcher: +// native NVIDIA GPU access would remove GPU recreation, while replacement-aware +// registration would remove the running-original handoff. Regression coverage: // * src/lib/onboard/docker-gpu-patch-finalize.test.ts — direct unit tests // for finalize success / rollback / no-op / rollback failure outcomes. // * src/lib/onboard/docker-gpu-patch-rollback.test.ts — composed @@ -18,10 +22,13 @@ // * src/lib/onboard/docker-gpu-sandbox-create.test.ts — composed create // flow driving maybeApplyDuringCreate → waitForSupervisorReconnect → // finalizeBackup. -// Removal condition: when OpenShell supports native Docker-driver GPU -// creation/reconnect, drop the NemoClaw post-create container recreation -// and delete this module along with its callers in docker-gpu-patch.ts and -// docker-gpu-sandbox-create.ts. +// * src/lib/actions/sandbox/supervisor-relaunch.test.ts — composed legacy +// relaunch handoff, successful finalization, and rollback without restart. +// Removal conditions: delete the GPU callers when OpenShell supports native +// Docker-driver GPU creation/reconnect. Delete the supervisor handoff branch +// when the legacy relaunch compatibility path is removed or every supported +// OpenShell version keeps direct container replacement non-terminal until +// registration settles. Delete this module when neither caller remains. import { hasZeroDockerExitStatus } from "./docker-command-result"; import { DOCKER_GPU_PATCH_TIMEOUT_MS } from "./docker-gpu-patch-constants"; From b8e273b9f61796bec141543d7f602ab014abec6b Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Wed, 29 Jul 2026 19:41:32 -0700 Subject: [PATCH 09/30] test(recovery): clarify supervisor handoff lifecycle Signed-off-by: Prekshi Vyas --- src/lib/actions/sandbox/supervisor-relaunch.test.ts | 4 ++-- src/lib/onboard/docker-gpu-patch-finalize.ts | 13 +++++++------ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/lib/actions/sandbox/supervisor-relaunch.test.ts b/src/lib/actions/sandbox/supervisor-relaunch.test.ts index 6ae2ed96211..38858002b2d 100644 --- a/src/lib/actions/sandbox/supervisor-relaunch.test.ts +++ b/src/lib/actions/sandbox/supervisor-relaunch.test.ts @@ -198,7 +198,7 @@ describe("relaunchManagedSupervisorSession", () => { }); }); - it("removes the running original only after the handoff is finalized as ready", () => { + it("removes the running original container only after supervisor readiness is confirmed", () => { const handoff = composedHandoffDeps(); const deps = baseDeps({ recreate: (options) => @@ -220,7 +220,7 @@ describe("relaunchManagedSupervisorSession", () => { expect(handoff.dockerStart).not.toHaveBeenCalled(); }); - it("restores the running original without restarting it when the handoff is finalized as not ready", () => { + it("rolls back to the running original container without restarting it when supervisor readiness is not confirmed", () => { const handoff = composedHandoffDeps(); const deps = baseDeps({ recreate: (options) => diff --git a/src/lib/onboard/docker-gpu-patch-finalize.ts b/src/lib/onboard/docker-gpu-patch-finalize.ts index afed03ad355..25890c0e89b 100644 --- a/src/lib/onboard/docker-gpu-patch-finalize.ts +++ b/src/lib/onboard/docker-gpu-patch-finalize.ts @@ -8,9 +8,10 @@ // For legacy supervisor relaunch on OpenShell 0.0.85, the invalid state is // "stopping the registered keepalive container lets the Docker watcher publish // a terminal Error phase before the replacement supervisor registers". That -// caller keeps the renamed original container running until pinned managed -// health confirms the replacement, then force-removes it; failed health rolls -// back to the still-running original without restarting it. +// caller keeps the renamed original container running until the pinned managed +// health check confirms the replacement, then force-removes it; a failed +// managed health check rolls back to the still-running original without +// restarting it. // // The preferred source boundaries are OpenShell's Docker create and watcher: // native NVIDIA GPU access would remove GPU recreation, while replacement-aware @@ -26,9 +27,9 @@ // relaunch handoff, successful finalization, and rollback without restart. // Removal conditions: delete the GPU callers when OpenShell supports native // Docker-driver GPU creation/reconnect. Delete the supervisor handoff branch -// when the legacy relaunch compatibility path is removed or every supported -// OpenShell version keeps direct container replacement non-terminal until -// registration settles. Delete this module when neither caller remains. +// when the legacy relaunch compatibility path is removed or no supported +// OpenShell version publishes phase Error before replacement registration +// settles. Delete this module when neither caller remains. import { hasZeroDockerExitStatus } from "./docker-command-result"; import { DOCKER_GPU_PATCH_TIMEOUT_MS } from "./docker-gpu-patch-constants"; From 297ade9b8a04b844cb97ae0f77f318838a25d5ca Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Wed, 29 Jul 2026 20:18:01 -0700 Subject: [PATCH 10/30] fix(recovery): relaunch legacy supervisor in place Signed-off-by: Prekshi Vyas --- .../configure-inference-timeouts.mdx | 6 +- .../recover-rebuild-sandboxes.mdx | 15 +- docs/reference/commands.mdx | 19 +- docs/reference/troubleshooting.mdx | 10 +- docs/security/tcb-boundary.mdx | 4 +- scripts/gateway-control.sh | 8 +- scripts/managed-gateway-control.py | 350 ++++++++++++++++-- .../actions/sandbox/process-recovery.test.ts | 56 +-- src/lib/actions/sandbox/process-recovery.ts | 133 +++---- .../sandbox/supervisor-relaunch.test.ts | 251 +++++-------- .../actions/sandbox/supervisor-relaunch.ts | 90 ++--- .../onboard/docker-gpu-patch-finalize.test.ts | 62 ---- src/lib/onboard/docker-gpu-patch-finalize.ts | 52 +-- src/lib/onboard/docker-gpu-patch-recreate.ts | 53 +-- src/lib/onboard/docker-gpu-patch-rollback.ts | 18 +- src/lib/onboard/docker-gpu-patch-types.ts | 6 - .../docker-startup-command-patch.test.ts | 93 ----- .../onboard/docker-startup-command-patch.ts | 1 - src/lib/onboard/finalization-deps.ts | 2 +- test/gateway-supervisor-control.test.ts | 6 + test/managed-supervisor-launch.test.ts | 108 ++++++ ...ocess-recovery-supervisor-relaunch.test.ts | 78 ++-- 22 files changed, 753 insertions(+), 668 deletions(-) create mode 100644 test/managed-supervisor-launch.test.ts diff --git a/docs/inference/configure-inference-timeouts.mdx b/docs/inference/configure-inference-timeouts.mdx index 643b65efc17..26c9fac65f6 100644 --- a/docs/inference/configure-inference-timeouts.mdx +++ b/docs/inference/configure-inference-timeouts.mdx @@ -20,7 +20,7 @@ Use the error location to select the correct setting. |---|---|---| | `NEMOCLAW_AGENT_TIMEOUT` | OpenClaw per-request inference | `600` seconds | | `NEMOCLAW_LOCAL_INFERENCE_TIMEOUT` | Ollama, vLLM, NIM, and compatible-endpoint validation during onboarding | `180` seconds | -| `NEMOCLAW_SANDBOX_READY_TIMEOUT` | Image build, gateway upload, and in-sandbox boot after creation; OpenShell command re-registration after policy application or after OpenClaw or Hermes managed recovery recreates the sandbox | `180` seconds | +| `NEMOCLAW_SANDBOX_READY_TIMEOUT` | Image build, gateway upload, and in-sandbox boot after creation; OpenShell command re-registration after policy application; OpenShell readiness after OpenClaw or Hermes managed recovery | `180` seconds | The readiness timeout does not govern inference requests or provider validation. @@ -68,7 +68,7 @@ Onboarding also uses this budget to confirm that the sandbox can execute command -The same budget applies when `start` or `recover` transactionally recreates a managed sandbox and waits for OpenShell to re-register it. +The same budget applies when `start` or `recover` launches a missing supervisor in a legacy keepalive sandbox and proves OpenShell readiness. @@ -79,7 +79,7 @@ $$nemoclaw onboard -For an existing sandbox, export the variable before the `start` or `recover` command that performs the recreation. +For an existing sandbox, export the variable before the `start` or `recover` command that performs the launch. diff --git a/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx b/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx index 4d1ec536979..a1b7cdc40dd 100644 --- a/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx +++ b/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx @@ -80,12 +80,15 @@ Refer to [`$$nemoclaw recover`](../../reference/commands#$$nemoclaw-name- Recovery uses registry-scoped privileged direct-container control and does not fall back to ordinary `openshell sandbox exec` or a manual in-sandbox relaunch. -For a local Docker-driver sandbox whose container still uses the legacy keepalive startup, `recover` can transactionally recreate the registered container with a credential-free managed startup command. - -NemoClaw keeps the previous container available until the managed controller proves the supervisor topology, gateway health, and settle check, and attempts to restore it if that proof fails. -The recreation preserves mounted sandbox state, but a committed swap does not retain changes stored only in the previous container's writable layer. -After a transactional recreation, NemoClaw uses the `NEMOCLAW_SANDBOX_READY_TIMEOUT` budget (180 seconds by default) for OpenShell to re-register the sandbox before starting the primary dashboard or API host forward. -A definitive managed-health failure still stops immediately; if re-registration does not complete within the budget, the forward stays stopped. +For a local Docker-driver sandbox whose container still uses the legacy keepalive startup, `recover` can launch the managed supervisor in the registered container with credential-free host runtime overrides. + +The controller repeats the stable supervisor-absence proof under its root-only lifecycle lock and starts the fixed entrypoint under the `sandbox` UID. +It opens a pidfd before the short-lived parent exits and proves that OpenShell PID 1 adopted exactly one supervisor. +The launch keeps the registered container in place and does not change its persisted legacy startup command. +A later full container restart can therefore require `recover` again. +After the launch, NemoClaw requires the managed gateway health and settle checks. +It then uses the `NEMOCLAW_SANDBOX_READY_TIMEOUT` budget (180 seconds by default) to prove OpenShell readiness before starting the primary dashboard or API host forward. +A definitive managed-health failure still stops immediately; if readiness cannot be proved within the budget, the forward stays stopped. For the controller topology, trust boundary, and fail-closed conditions, refer to [Understand Gateway Lifecycle Control](../configure-sandboxes/understand-gateway-lifecycle-control). If recovery cannot repair a sandbox that needs credentials or a current controller contract, rebuild it. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 7468c15e23e..1bc70ff6f4f 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -1161,9 +1161,14 @@ The host selects the controller from the live container topology. In a direct root-entrypoint container, the request reaches the root PID 1 supervisor. In an OpenShell-managed container, the request enters the root-owned mode `0500` managed controller through a sanitized root exec while OpenShell remains PID 1. It does not use ordinary `openshell sandbox exec` or an in-sandbox manual relaunch as a fallback. -When the root-owned managed controller attests two unchanged zero-supervisor process scans with a stable PID 1 and reports `SUPERVISOR_NOT_RUNNING`, a local Docker-driver sandbox with the legacy keepalive startup can enter a transactional container recreation. -The recreation uses a credential-free managed startup command, pins the registered container identity, retains the previous container for rollback, and commits only after managed gateway health and the settle check pass. -The recreation preserves mounted sandbox state, but a committed swap does not retain changes stored only in the previous container's writable layer. +The root-owned managed controller reports `SUPERVISOR_NOT_RUNNING` only after two unchanged zero-supervisor process scans with a stable PID 1. +A local Docker-driver sandbox with the legacy keepalive startup can then enter an in-place supervisor launch. +The host pins the registered container identity and asks that controller to launch the fixed `nemoclaw-start` entrypoint with allowlisted, credential-free host runtime overrides. +The controller holds its root-only lifecycle lock, repeats the stable absence proof, and starts the entrypoint under the `sandbox` UID. +It opens a pidfd before the short-lived parent exits and proves that OpenShell PID 1 adopted exactly one supervisor. +The launch keeps the registered container and its writable layer in place. +It does not change the persisted legacy startup command, so a later full container restart can require `recover` again. +Recovery succeeds only after the managed gateway health and settle checks pass. It is idempotent. When `recover` repairs a stopped built-in OpenClaw or Hermes gateway, it repeats the recovery action only for an exit status of `1` with blank stdout and a sole nonblank stderr line equal to `SUPERVISOR_BUSY`, with at most three controller attempts. The same result is inconclusive during managed settle confirmation and can be probed again only within the configured settle window. @@ -3934,7 +3939,7 @@ The following environment variables tune onboard-time wall-clock limits. `NEMOCLAW_SANDBOX_READY_TIMEOUT` also covers OpenShell command re-registration after onboarding applies policy presets. -`NEMOCLAW_SANDBOX_READY_TIMEOUT` also applies when managed recovery transactionally recreates an existing sandbox. +`NEMOCLAW_SANDBOX_READY_TIMEOUT` also applies when managed recovery launches a missing supervisor in a legacy keepalive sandbox. Set them before running `$$nemoclaw onboard` if a slow connection or large model pull risks tripping the default. @@ -3947,7 +3952,7 @@ Set them before running `$$nemoclaw onboard` if a slow connection or large model -For managed recovery, the same timeout covers OpenShell re-registration after transactional recreation. +For managed recovery, the same timeout covers the post-relaunch OpenShell readiness check. When the deadline expires, the primary dashboard or API host forward stays stopped. @@ -3975,7 +3980,9 @@ The following flags change defaults for commands that manage existing sandboxes. | `NEMOCLAW_CONFIRM_LEGACY_MANAGED_RECREATE` | Exact JSON array of sandbox names | Confirms to the installer that the exact listed set of pre-fingerprint OpenClaw or Hermes sandboxes used NemoClaw-managed images, allowing recovery onto the current managed image. The normalized names must exactly match the installer's printed array. Set it only after verifying every named sandbox. Recorded custom-image evidence remains blocked. | | `NEMOCLAW_DISABLE_INFERENCE_ROUTE_REPAIR` | `1` to enable | Skips the automatic DNS-proxy repair for stale `inference.local` routes during `$$nemoclaw connect` and `$$nemoclaw connect --probe-only`. Use only as a troubleshooting escape hatch. | -| `NEMOCLAW_DISABLE_SUPERVISOR_RELAUNCH` | `1` to enable | Skips the automatic trusted container recreation during `$$nemoclaw recover` when two managed scans find no supervisor while PID 1 remains stable. Use only as a troubleshooting escape hatch; recovery then falls back to the rebuild or re-onboard guidance. | + +| `NEMOCLAW_DISABLE_SUPERVISOR_RELAUNCH` | `1` to enable | Skips the automatic trusted in-place supervisor launch during `$$nemoclaw recover` when two managed scans find no supervisor while PID 1 remains stable. Use only as a troubleshooting escape hatch; recovery then falls back to the rebuild or re-onboard guidance. | + | `NEMOCLAW_SHIELDS_ACCEPT_LEGACY_BASELINE` | `1` to opt in | Allows advanced immutable-config verification to trust the current on-disk bytes for older or partial content baselines. Use only after you have rebuilt or manually inspected the sandbox state and accepted that the baseline is operator-approved. | | `NEMOCLAW_SHIELDS_SETTLE_MS` | milliseconds (default `750`, clamped to `0` to `10000`) | Settle window NemoClaw waits after re-applying a config lockdown (during shields auto-restore and `$$nemoclaw shields up` drift remediation) before re-confirming the lock still holds. Detects when an in-sandbox reconciler changes config file permissions after lockdown and re-applies the lock; if NemoClaw cannot re-confirm the lock within the retry budget, shields stay down. This narrows the window in which a reconciler can revert permissions rather than eliminating it. The best-effort `chattr +i` immutable bit remains the only fully durable lock. Raise it on hosts where the gateway settles slowly. | | `NEMOCLAW_SKIP_UNREACHABLE_SANDBOX_BACKUP` | Exactly `1` to opt in (`true`, `yes`, `0` are not accepted) | Applies to standalone `$$nemoclaw backup-all` runs. Skips running sandboxes whose in-sandbox SSH endpoint does not answer. It does not relax the installer's strict pre-upgrade backup, which still aborts if any registered sandbox is skipped or fails. Any uncommitted state since the last successful backup is not included in the skipped backup. | diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index 6cad47f36ff..0c255e6930a 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -1019,10 +1019,14 @@ The same result is inconclusive during managed settle confirmation and can be pr NemoClaw treats `SUPERVISOR_UNAVAILABLE` as terminal because it can report unreadable or untrusted supervisor state, ambiguous discovery, or a process-identity change. It does not retry other status or output combinations. `SUPERVISOR_NOT_RUNNING` is a separate result that requires two zero-supervisor scans with a stable PID 1 and does not enter that retry loop. -On a supported local Docker-driver sandbox with the legacy keepalive startup, it can authorize a container-identity-pinned recreation that commits only after managed health and settle checks pass. -To bypass that trusted recreation while troubleshooting, run `NEMOCLAW_DISABLE_SUPERVISOR_RELAUNCH=1 $$nemoclaw recover`; NemoClaw leaves the container unchanged and returns rebuild or re-onboard guidance. +On a supported local Docker-driver sandbox with the legacy keepalive startup, this result can authorize a container-identity-pinned in-place launch. +The root-owned controller repeats the absence proof and starts the fixed entrypoint under the `sandbox` UID. +It then proves that OpenShell PID 1 adopted exactly one supervisor. +NemoClaw still requires managed gateway health, settle, and OpenShell readiness before it re-establishes the primary port forward. +To bypass that trusted launch while troubleshooting, run `NEMOCLAW_DISABLE_SUPERVISOR_RELAUNCH=1 $$nemoclaw recover`; NemoClaw leaves the container unchanged and returns rebuild or re-onboard guidance. If that bounded retry is exhausted, or if `gateway restart` reports `SUPERVISOR_BUSY`, wait for the active request to finish and retry the command. -If the error mentions `SUPERVISOR_NOT_RUNNING` and trusted recreation could not proceed, `SUPERVISOR_REBUILD_REQUIRED`, a missing `nemoclaw-gateway-control` helper, or a missing managed controller, the sandbox image may predate the current lifecycle contract. +The sandbox image can predate the current lifecycle contract when the trusted launch cannot proceed after `SUPERVISOR_NOT_RUNNING`. +The same guidance applies to `SUPERVISOR_REBUILD_REQUIRED`, a missing `nemoclaw-gateway-control` helper, or a missing managed controller. An exact `SUPERVISOR_UNAVAILABLE` result instead means the managed controller refused the current supervisor state rather than guessing which same-UID process is the gateway. The current recovery action and any managed settle confirmation stop immediately. If `recover` reports this result, follow its host-side `gateway restart` guidance. diff --git a/docs/security/tcb-boundary.mdx b/docs/security/tcb-boundary.mdx index a725149fdb2..ab2390efcfb 100644 --- a/docs/security/tcb-boundary.mdx +++ b/docs/security/tcb-boundary.mdx @@ -44,7 +44,7 @@ A successful build does not replace review of privilege, process identity, descr | `scripts/state-dir-guard.py` | The installed copy is root-owned and mode `0500`; the host reaches it through the shields transaction. | Fixed paths, a bounded action contract, and a lock token from the host coordinator. | Applies descriptor-rooted state-directory posture changes, rejects link and mount substitution, bounds traversal, and verifies the committed modes and ownership. | | `scripts/lib/normalize_mutable_config_perms.py` | The installed copy is root-owned and mode `0555`; startup invokes it under the entrypoint identity, and only root can reclaim a root-owned tree. | The fixed OpenClaw config path, the resolved sandbox identity, and an exact `root:root 0700/0600` mutable-drift signature under the expected sandbox-owned parent. | Restores the mutable `2770/660` contract, pins every privileged handoff by descriptor, and rejects ambiguous posture, links, mount substitution, metadata races, and sealed config. | | `scripts/openclaw-config-guard.py` | The installed copy is root-owned and mode `0500`; direct root PID 1 or the authenticated host transaction invokes it. | Bounded strict JSON for writes, stable captured config bytes for restart validation, and fixed installed parser paths for existing JSON5 config. | Seals and unseals OpenClaw config with no-follow descriptors, stable inode checks, atomic replacement, hash coherence, and recoverable transaction journals. | -| `scripts/managed-gateway-control.py` | The installed copy is root-owned and mode `0500`; the host invokes it through sanitized registry-scoped direct-container execution. | A fixed action, a 64-character nonce, fixed installed helpers, and a live OpenShell process tree observed through `/proc`. | Authenticates the host action, proves the managed supervisor and gateway identity, holds a root-owned mode `0600` lifecycle lock, publishes one root-owned mode `0444` exact-exit authorization bound to the gateway and live root controller identities, signals through a pidfd, waits for the normal respawn loop, and verifies listener and HTTP health. | +| `scripts/managed-gateway-control.py` | The installed copy is root-owned and mode `0500`; the host invokes it through sanitized registry-scoped direct-container execution. | A fixed action, a 64-character nonce, fixed installed helpers, allowlisted credential-free host overrides, and a live OpenShell process tree observed through `/proc`. | Authenticates the host action, proves the managed supervisor and gateway identity, holds a root-owned mode `0600` lifecycle lock, publishes one root-owned mode `0444` exact-exit authorization bound to the gateway and live root controller identities, signals through pidfds, waits for the normal respawn loop, and verifies listener and HTTP health. For legacy keepalive recovery, it repeats the supervisor-absence proof, launches the fixed entrypoint under the sandbox UID with inherited loader and interpreter startup variables removed, pins that child before adoption, and proves that OpenShell PID 1 adopted exactly one supervisor. | | `src/lib/shields/transition-lock.ts` | Runs in the host CLI under the operator account and owns the canonical per-sandbox transition lock. | Host state directory entries whose owner PID and start identity match the live lock owner, or prove that the recorded owner is definitively dead or PID-reused. | Serializes shields mutations, recovers definitively stale owners through inode-checked quarantine, rejects ambiguous owners, and allows token-gated takeover only through the explicit recovery contract. | | `src/lib/shields/timer-bound-lock.ts` | Runs in the host CLI and composes the transition lock with the recorded auto-restore generation. | A validated timer marker and transition owner from the host state directory. | Prevents an expired or replaced timer from authorizing a later mutation and keeps restore authority bound to one generation. | | `src/lib/shields/verify-lock.ts` | Runs in the host CLI and delegates sandbox inspection through the privileged execution adapter. | Resolved built-in agent paths and the expected locked posture recorded by the host. | Verifies modes, ownership, immutable flags, layout, and recorded content hashes before NemoClaw reports shields as locked. | @@ -75,6 +75,8 @@ flowchart LR The host CLI first resolves the sandbox from host-owned registry state and selects the built-in agent topology. Gateway restart generates a fresh nonce and enters `nemoclaw-gateway-control` as root with injection-capable environment variables cleared. The direct topology publishes a root-owned request to PID 1, while the OpenShell-managed topology executes `managed-gateway-control.py` directly. +The legacy `launch-supervisor` action additionally requires the host-pinned registered container and legacy keepalive evidence. +The controller then repeats the absence proof before it launches anything. Both paths prove the exact replacement gateway and health state before the host repairs port forwards or reports success. Shields mutations acquire the host transition lock before changing network policy, config posture, timer authority, or host state. diff --git a/scripts/gateway-control.sh b/scripts/gateway-control.sh index bcfca804bfb..e8e2ee79480 100755 --- a/scripts/gateway-control.sh +++ b/scripts/gateway-control.sh @@ -36,11 +36,12 @@ fail() { exit 1 } -[ "$#" -eq 2 ] || fail "SUPERVISOR_INVALID_REQUEST" +[ "$#" -ge 2 ] || fail "SUPERVISOR_INVALID_REQUEST" ACTION="$1" NONCE="$2" case "$ACTION" in - restart | recover | probe) ;; + restart | recover | probe) [ "$#" -eq 2 ] || fail "SUPERVISOR_INVALID_REQUEST" ;; + launch-supervisor) [ "$#" -le 34 ] || fail "SUPERVISOR_INVALID_REQUEST" ;; *) fail "SUPERVISOR_INVALID_ACTION" ;; esac case "$NONCE" in @@ -55,8 +56,9 @@ if [ "$PID1_ARGV0" = "/opt/openshell/bin/openshell-sandbox" ]; then [ -x "$CONTROL_MANAGED_HELPER" ] || fail "SUPERVISOR_REBUILD_REQUIRED" # Isolated mode ignores Python startup hooks, user-site packages, and # PYTHON* environment variables before the root helper imports anything. - exec python3 -I "$CONTROL_MANAGED_HELPER" "$ACTION" "$NONCE" + exec python3 -I "$CONTROL_MANAGED_HELPER" "$@" fi +[ "$ACTION" != "launch-supervisor" ] || fail "SUPERVISOR_INVALID_ACTION" case "$PID1_CMDLINE" in *nemoclaw-start*) ;; *) fail "SUPERVISOR_UNAVAILABLE" ;; diff --git a/scripts/managed-gateway-control.py b/scripts/managed-gateway-control.py index 884bc0d99fd..547dd3ca2ef 100755 --- a/scripts/managed-gateway-control.py +++ b/scripts/managed-gateway-control.py @@ -6,14 +6,21 @@ OpenShell is container PID 1 in its managed topology and starts ``nemoclaw-start`` as the unprivileged ``sandbox`` user. The shell entrypoint -still owns and reaps the gateway child, so this helper never launches a second -gateway and never trusts a status file writable by that user. Instead it: +still owns and reaps the gateway child, so ordinary control never launches a +second gateway and never trusts a status file writable by that user. Instead +it: * verifies a stable OpenShell -> nemoclaw-start -> gateway process tree; * signals the already-proven gateway through a pidfd; * waits for the entrypoint's normal respawn loop; and * independently proves the replacement process, listener, and HTTP health. +One legacy recovery action is available only when two complete process-table +scans prove that the supervisor is absent. It holds the root lifecycle lock, +launches the fixed entrypoint under the sandbox UID through a short-lived child, +and proves that OpenShell PID 1 adopted exactly that supervisor. The normal +managed health probe must still prove its gateway before host recovery succeeds. + The host enters this helper through registry-scoped ``docker exec --user root``. The installed copy is root-owned and mode 0500, which is the host request authentication boundary. No same-UID request or completion channel exists. @@ -89,6 +96,58 @@ NONCE_RE = re.compile(r"[0-9a-f]{64}\Z") ENV_KEY_RE = re.compile(rb"[A-Za-z_][A-Za-z0-9_]*\Z") SHA256_RE = re.compile(r"[0-9a-f]{64}\Z") +SUPERVISOR_LAUNCH_ACTION = "launch-supervisor" +SUPERVISOR_LAUNCH_ENV_KEYS = frozenset( + { + "CHAT_UI_URL", + "HTTP_PROXY", + "HTTPS_PROXY", + "NEMOCLAW_AUTO_PAIR_DEADLINE_SECS", + "NEMOCLAW_AUTO_PAIR_FAST_DEADLINE_SECS", + "NEMOCLAW_AUTO_PAIR_RUN_TIMEOUT_SECS", + "NEMOCLAW_AUTO_PAIR_SLOW_INTERVAL_SECS", + "NEMOCLAW_DASHBOARD_BIND", + "NEMOCLAW_DASHBOARD_PORT", + "NEMOCLAW_HERMES_DASHBOARD", + "NEMOCLAW_HERMES_DASHBOARD_INTERNAL_PORT", + "NEMOCLAW_HERMES_DASHBOARD_PORT", + "NEMOCLAW_HERMES_DASHBOARD_TUI", + "NEMOCLAW_MINIMAL_BOOTSTRAP", + "NEMOCLAW_PROXY_HOST", + "NEMOCLAW_PROXY_PORT", + "NO_PROXY", + "OPENCLAW_HOME", + "OPENCLAW_STATE_DIR", + "OPENCLAW_WORKSPACE_DIR", + "http_proxy", + "https_proxy", + "no_proxy", + } +) +SUPERVISOR_LAUNCH_STRIPPED_ENV_KEYS = frozenset( + { + "BASH_ENV", + "ENV", + "GCONV_PATH", + "GLIBC_TUNABLES", + "LD_AUDIT", + "LD_LIBRARY_PATH", + "LD_PRELOAD", + "LOCPATH", + "NODE_OPTIONS", + "PERL5OPT", + "PYTHONHOME", + "PYTHONINSPECT", + "PYTHONPATH", + "PYTHONSTARTUP", + "PYTHONUSERBASE", + "RUBYOPT", + } +) +MAX_SUPERVISOR_LAUNCH_ENV_BYTES = 64 * 1024 +SUPERVISOR_LAUNCH_HANDSHAKE_SECONDS = 10 +SUPERVISOR_LAUNCH_PROOF_SECONDS = 5.0 +TRUSTED_RUNTIME_PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" HERMES_MCP_STATE_RE = re.compile( r"# nemoclaw-hermes-mcp-state-v1 " r"intended=[0-9a-f]{64} applied=[0-9a-f]{64}\Z" @@ -941,6 +1000,43 @@ def _supervisor_candidates( return matches, inconclusive +def _confirm_supervisor_absence( + reader: ProcReader, pid1: ProcessIdentity, sandbox_uid: int +) -> bool: + """Prove two complete zero-match scans around one exact OpenShell PID 1.""" + + between_pid1 = reader.capture(1) + second_matches, second_inconclusive = _supervisor_candidates( + reader, pid1, sandbox_uid + ) + after_pid1 = reader.capture(1) + return bool( + between_pid1.stable_key() == pid1.stable_key() + and after_pid1.stable_key() == pid1.stable_key() + and not second_inconclusive + and len(second_matches) == 0 + ) + + +def _prove_supervisor_absent( + reader: ProcReader, +) -> tuple[ProcessIdentity, int]: + """Return the exact OpenShell identity only for stable supervisor absence.""" + + pid1 = reader.capture(1) + if not _is_openshell(pid1): + raise ControlError("SUPERVISOR_UNAVAILABLE") + sandbox_uid = _sandbox_uid() + matches, inconclusive = _supervisor_candidates(reader, pid1, sandbox_uid) + if ( + inconclusive + or len(matches) != 0 + or not _confirm_supervisor_absence(reader, pid1, sandbox_uid) + ): + raise ControlError("SUPERVISOR_UNAVAILABLE") + return pid1, sandbox_uid + + def _discover_supervisor(reader: ProcReader) -> ProcessIdentity: pid1 = reader.capture(1) if not _is_openshell(pid1): @@ -967,22 +1063,12 @@ def _discover_supervisor(reader: ProcReader) -> ProcessIdentity: raise ControlError("SUPERVISOR_UNAVAILABLE") if len(matches) == 0: # A zero-match scan is the only absence signal that may authorize the - # host to recreate a legacy Docker container with its managed startup - # command. Re-scan the complete process table and pin PID 1 around both - # observations so ambiguity, process churn, and supervisor startup - # races remain generic unavailability rather than destructive-recovery + # host to launch the managed supervisor in a legacy keepalive + # container. Re-scan the complete process table and pin PID 1 around + # both observations so ambiguity, process churn, and supervisor + # startup races remain generic unavailability rather than launch # authorization. - between_pid1 = reader.capture(1) - second_matches, second_inconclusive = _supervisor_candidates( - reader, pid1, sandbox_uid - ) - after_pid1 = reader.capture(1) - if ( - between_pid1.stable_key() == pid1.stable_key() - and after_pid1.stable_key() == pid1.stable_key() - and not second_inconclusive - and len(second_matches) == 0 - ): + if _confirm_supervisor_absence(reader, pid1, sandbox_uid): raise ControlError("SUPERVISOR_NOT_RUNNING") raise ControlError("SUPERVISOR_UNAVAILABLE") if len(matches) != 1: @@ -1510,6 +1596,198 @@ def _terminate_gateway(reader: ProcReader, identity: ProcessIdentity) -> None: os.close(pidfd) +def _supervisor_launch_environment( + runtime_environment: dict[str, str], +) -> dict[str, str]: + """Build the cold-start-compatible environment without loader hooks.""" + + environment = { + key: value + for key, value in os.environ.items() + if key not in SUPERVISOR_LAUNCH_STRIPPED_ENV_KEYS + and not key.startswith("NEMOCLAW_TEST_") + and not key.startswith("NEMOCLAW_MANAGED_CONTROL_") + } + environment.update(runtime_environment) + environment.update( + { + "HOME": "/sandbox", + "LOGNAME": "sandbox", + "PATH": TRUSTED_RUNTIME_PATH, + "PYTHONNOUSERSITE": "1", + "SHELL": "/bin/bash", + "USER": "sandbox", + } + ) + return environment + + +def _spawn_supervisor_as_orphan(environment: dict[str, str]) -> tuple[int, int]: + """Pin a launched child before its short-lived parent permits adoption.""" + + status_read_fd, status_write_fd = os.pipe() + adoption_read_fd, adoption_write_fd = os.pipe() + try: + intermediate_pid = os.fork() + except OSError as exc: + os.close(status_read_fd) + os.close(status_write_fd) + os.close(adoption_read_fd) + os.close(adoption_write_fd) + raise ControlError("SUPERVISOR_UNAVAILABLE") from exc + + if intermediate_pid == 0: + os.close(status_read_fd) + os.close(adoption_write_fd) + try: + signal.alarm(SUPERVISOR_LAUNCH_HANDSHAKE_SECONDS) + account = pwd.getpwnam("sandbox") + groups = os.getgrouplist(account.pw_name, account.pw_gid) + supervisor = subprocess.Popen( + [NEMOCLAW_START_PATH.decode("ascii")], + close_fds=True, + cwd="/sandbox", + env=environment, + extra_groups=groups, + group=account.pw_gid, + start_new_session=True, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + umask=0o077, + user=account.pw_uid, + ) + signal.alarm(0) + os.write(status_write_fd, f"{supervisor.pid}\n".encode("ascii")) + if os.read(adoption_read_fd, 1) != b"1": + # The child is still ours and cannot be PID-reused before it is + # reaped, so Popen can stop and reap this exact failed child. + supervisor.kill() + supervisor.wait() + raise ControlError("SUPERVISOR_UNAVAILABLE") + os.close(status_write_fd) + os.close(adoption_read_fd) + os._exit(0) + except BaseException: + try: + os.write(status_write_fd, b"SUPERVISOR_LAUNCH_FAILED\n") + os.close(status_write_fd) + os.close(adoption_read_fd) + except OSError: + pass + os._exit(1) + + os.close(status_write_fd) + os.close(adoption_read_fd) + payload = b"" + try: + while len(payload) <= 64 and not payload.endswith(b"\n"): + chunk = os.read(status_read_fd, 65 - len(payload)) + if not chunk: + break + payload += chunk + finally: + os.close(status_read_fd) + + supervisor_pidfd = -1 + launch_error: ControlError | None = None + if re.fullmatch(rb"[1-9][0-9]*\n", payload): + supervisor_pid = int(payload, 10) + try: + opened_pidfd = _pidfd_open(supervisor_pid) + if opened_pidfd is None: + raise ControlError("SUPERVISOR_UNAVAILABLE") + supervisor_pidfd = opened_pidfd + except ControlError as error: + launch_error = error + else: + supervisor_pid = 0 + launch_error = ControlError("SUPERVISOR_UNAVAILABLE") + + try: + os.write(adoption_write_fd, b"1" if launch_error is None else b"0") + except OSError: + launch_error = launch_error or ControlError("SUPERVISOR_UNAVAILABLE") + finally: + os.close(adoption_write_fd) + + _completed_pid, status = os.waitpid(intermediate_pid, 0) + if launch_error is not None or not os.WIFEXITED(status) or os.WEXITSTATUS(status) != 0: + if supervisor_pidfd >= 0: + os.close(supervisor_pidfd) + if launch_error is not None: + raise launch_error + raise ControlError("SUPERVISOR_UNAVAILABLE") + return supervisor_pid, supervisor_pidfd + + +def _wait_for_launched_supervisor( + supervisor_pid: int, + expected_pid1: ProcessIdentity, + sandbox_uid: int, +) -> ProcessIdentity: + """Prove the launched process was adopted into the one managed topology.""" + + deadline = time.monotonic() + SUPERVISOR_LAUNCH_PROOF_SECONDS + while time.monotonic() < deadline: + try: + with ProcReader() as reader: + current_pid1 = reader.capture(1) + if current_pid1.stable_key() != expected_pid1.stable_key(): + raise ControlError("SUPERVISOR_UNAVAILABLE") + supervisor = reader.capture(supervisor_pid) + matches, inconclusive = _supervisor_candidates( + reader, current_pid1, sandbox_uid + ) + if ( + not inconclusive + and _is_nemoclaw_start(supervisor, sandbox_uid) + and len(matches) == 1 + and matches[0].stable_key() == supervisor.stable_key() + ): + return supervisor + except ( + ControlError, + FileNotFoundError, + ProcessLookupError, + PermissionError, + ): + pass + time.sleep(POLL_SECONDS) + raise ControlError("SUPERVISOR_UNAVAILABLE") + + +def _launch_managed_supervisor(runtime_environment: dict[str, str]) -> int: + """Serialize, re-prove absence, launch, and pin one adopted supervisor.""" + + _detect_agent() + _validate_trusted_regular(NEMOCLAW_START_PATH.decode("ascii")) + directory_fd = _open_managed_runtime_directory() + lock_fd = -1 + supervisor_pidfd = -1 + try: + lock_fd = _open_expected_exit_lock(directory_fd) + with ProcReader() as reader: + pid1, sandbox_uid = _prove_supervisor_absent(reader) + supervisor_pid, supervisor_pidfd = _spawn_supervisor_as_orphan( + _supervisor_launch_environment(runtime_environment) + ) + try: + supervisor = _wait_for_launched_supervisor( + supervisor_pid, pid1, sandbox_uid + ) + except Exception: + _send_pidfd(supervisor_pidfd, signal.SIGKILL) + raise + return supervisor.pid + finally: + if supervisor_pidfd >= 0: + os.close(supervisor_pidfd) + if lock_fd >= 0: + os.close(lock_fd) + os.close(directory_fd) + + def _wait_for_healthy_gateway( reader: ProcReader, supervisor: ProcessIdentity, @@ -1807,22 +2085,48 @@ def _managed_failure_diagnostics() -> tuple[str, ...]: return tuple(diagnostics) -def _validate_request(argv: list[str]) -> tuple[str, str]: - if len(argv) != 2: +def _parse_supervisor_launch_environment(argv: list[str]) -> dict[str, str]: + environment: dict[str, str] = {} + total_bytes = 0 + for assignment in argv: + key, separator, value = assignment.partition("=") + total_bytes += len(assignment.encode("utf-8", errors="surrogateescape")) + if ( + not separator + or key not in SUPERVISOR_LAUNCH_ENV_KEYS + or key in environment + or total_bytes > MAX_SUPERVISOR_LAUNCH_ENV_BYTES + ): + raise ControlError("SUPERVISOR_INVALID_REQUEST") + environment[key] = value + return environment + + +def _validate_request(argv: list[str]) -> tuple[str, str, dict[str, str]]: + if len(argv) < 2: raise ControlError("SUPERVISOR_INVALID_REQUEST") - action, nonce = argv - if action not in ("restart", "recover", "probe"): + action, nonce, *arguments = argv + if action not in ("restart", "recover", "probe", SUPERVISOR_LAUNCH_ACTION): raise ControlError("SUPERVISOR_INVALID_ACTION") if not NONCE_RE.fullmatch(nonce): raise ControlError("SUPERVISOR_INVALID_NONCE") - return action, nonce + if action == SUPERVISOR_LAUNCH_ACTION: + return action, nonce, _parse_supervisor_launch_environment(arguments) + if arguments: + raise ControlError("SUPERVISOR_INVALID_REQUEST") + return action, nonce, {} def main(argv: list[str]) -> int: try: - action, nonce = _validate_request(argv) + action, nonce, runtime_environment = _validate_request(argv) _require_root() _require_installed_helper_trust() + if action == SUPERVISOR_LAUNCH_ACTION: + supervisor_pid = _launch_managed_supervisor(runtime_environment) + print(f"v1 {nonce} complete launched 0 {supervisor_pid}") + print(f"SUPERVISOR_PID={supervisor_pid}") + return 0 result, old_pid, new_pid = _control(action, nonce) print(f"v1 {nonce} complete {result} {old_pid} {new_pid}") print(f"GATEWAY_PID={new_pid}") diff --git a/src/lib/actions/sandbox/process-recovery.test.ts b/src/lib/actions/sandbox/process-recovery.test.ts index 77a023eb720..fff32d68513 100644 --- a/src/lib/actions/sandbox/process-recovery.test.ts +++ b/src/lib/actions/sandbox/process-recovery.test.ts @@ -7,7 +7,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { confirmRecoveredSandboxGatewayManaged, waitForRecoveredSandboxGateway, - waitForRecreatedSandboxOpenShellReady, + waitForRecoveredSandboxOpenShellReady, } from "./process-recovery"; const OPENSHELL_SANDBOX_NOT_READY_STDERR = `Error: × code: 'The system is not in a state required for the operation's @@ -38,9 +38,9 @@ const OPENSHELL_RELAY_TARGET_REFUSED_STDERR = `Error: × code: 'The service is │ refused (os error 111)" `; const OPENSHELL_TRANSIENT_ERROR_PHASE_STDERR = - "Error: sandbox 'recreated-box' is not ready (phase: Error); wait for it to reach Ready state.\n"; + "Error: sandbox 'recovered-box' is not ready (phase: Error); wait for it to reach Ready state.\n"; -describe("recreated sandbox OpenShell readiness", () => { +describe("recovered sandbox OpenShell readiness", () => { afterEach(() => { vi.unstubAllEnvs(); }); @@ -61,7 +61,7 @@ describe("recreated sandbox OpenShell readiness", () => { const sleeps: number[] = []; expect( - waitForRecreatedSandboxOpenShellReady("recreated-box", { + waitForRecoveredSandboxOpenShellReady("recovered-box", { beforeProbe, captureOpenshellImpl, intervalSeconds: 3, @@ -71,7 +71,7 @@ describe("recreated sandbox OpenShell readiness", () => { ).toBe(true); expect(captureOpenshellImpl).toHaveBeenCalledTimes(3); expect(captureOpenshellImpl).toHaveBeenCalledWith( - ["sandbox", "exec", "--name", "recreated-box", "--", "true"], + ["sandbox", "exec", "--name", "recovered-box", "--", "true"], expect.objectContaining({ ignoreError: true, includeStderr: true, @@ -96,7 +96,7 @@ describe("recreated sandbox OpenShell readiness", () => { const sleeps: number[] = []; expect( - waitForRecreatedSandboxOpenShellReady("recreated-box", { + waitForRecoveredSandboxOpenShellReady("recovered-box", { beforeProbe, captureOpenshellImpl, intervalSeconds: 3, @@ -123,7 +123,7 @@ describe("recreated sandbox OpenShell readiness", () => { const sleeps: number[] = []; expect( - waitForRecreatedSandboxOpenShellReady("recreated-box", { + waitForRecoveredSandboxOpenShellReady("recovered-box", { beforeProbe: () => true, captureOpenshellImpl, intervalSeconds: 3, @@ -152,7 +152,7 @@ describe("recreated sandbox OpenShell readiness", () => { let nowMs = 0; expect( - waitForRecreatedSandboxOpenShellReady("recreated-box", { + waitForRecoveredSandboxOpenShellReady("recovered-box", { beforeProbe, captureOpenshellImpl, intervalSeconds: 3, @@ -198,7 +198,7 @@ describe("recreated sandbox OpenShell readiness", () => { const sleeps: number[] = []; expect( - waitForRecreatedSandboxOpenShellReady("recreated-box", { + waitForRecoveredSandboxOpenShellReady("recovered-box", { beforeProbe: () => true, captureOpenshellImpl, intervalSeconds: 3, @@ -241,7 +241,7 @@ describe("recreated sandbox OpenShell readiness", () => { const sleeps: number[] = []; expect( - waitForRecreatedSandboxOpenShellReady("recreated-box", { + waitForRecoveredSandboxOpenShellReady("recovered-box", { beforeProbe: () => true, captureOpenshellImpl, intervalSeconds: 3, @@ -263,7 +263,7 @@ describe("recreated sandbox OpenShell readiness", () => { const sleeps: number[] = []; expect( - waitForRecreatedSandboxOpenShellReady("recreated-box", { + waitForRecoveredSandboxOpenShellReady("recovered-box", { beforeProbe: () => true, captureOpenshellImpl, intervalSeconds: 3, @@ -300,7 +300,7 @@ describe("recreated sandbox OpenShell readiness", () => { }); expect( - waitForRecreatedSandboxOpenShellReady("recreated-box", { + waitForRecoveredSandboxOpenShellReady("recovered-box", { beforeProbe: () => true, captureOpenshellImpl, intervalSeconds: 3, @@ -312,7 +312,7 @@ describe("recreated sandbox OpenShell readiness", () => { expect(captureOpenshellImpl).toHaveBeenCalledTimes(12); }); - it("retries the exact supervisor reconnect states exposed during direct recreation", () => { + it("retries the exact supervisor reconnect states exposed during direct recovery", () => { const reconnecting = [ OPENSHELL_SUPERVISOR_NOT_CONNECTED_STDERR, OPENSHELL_SUPERVISOR_DISCONNECTED_STDERR, @@ -331,7 +331,7 @@ describe("recreated sandbox OpenShell readiness", () => { const sleeps: number[] = []; expect( - waitForRecreatedSandboxOpenShellReady("recreated-box", { + waitForRecoveredSandboxOpenShellReady("recovered-box", { beforeProbe, captureOpenshellImpl, intervalSeconds: 3, @@ -361,7 +361,7 @@ describe("recreated sandbox OpenShell readiness", () => { const sleeps: number[] = []; expect( - waitForRecreatedSandboxOpenShellReady("recreated-box", { + waitForRecoveredSandboxOpenShellReady("recovered-box", { beforeProbe, captureOpenshellImpl, intervalSeconds: 3, @@ -374,7 +374,7 @@ describe("recreated sandbox OpenShell readiness", () => { expect(sleeps).toEqual([3]); }); - it("retries when OpenShell drops the replacement supervisor's reverse relay", () => { + it("retries when OpenShell drops the recovered supervisor's reverse relay", () => { const captureOpenshellImpl = vi .fn() .mockReturnValueOnce({ @@ -388,7 +388,7 @@ describe("recreated sandbox OpenShell readiness", () => { const sleeps: number[] = []; expect( - waitForRecreatedSandboxOpenShellReady("recreated-box", { + waitForRecoveredSandboxOpenShellReady("recovered-box", { beforeProbe, captureOpenshellImpl, intervalSeconds: 3, @@ -404,7 +404,7 @@ describe("recreated sandbox OpenShell readiness", () => { it.each([ OPENSHELL_RELAY_TARGET_NOT_FOUND_STDERR, OPENSHELL_RELAY_TARGET_REFUSED_STDERR, - ])("retries while the replacement supervisor's local relay target starts (#7273)", (stderr) => { + ])("retries while the recovered supervisor's local relay target starts (#7273)", (stderr) => { const captureOpenshellImpl = vi .fn() .mockReturnValueOnce({ @@ -418,7 +418,7 @@ describe("recreated sandbox OpenShell readiness", () => { const sleeps: number[] = []; expect( - waitForRecreatedSandboxOpenShellReady("recreated-box", { + waitForRecoveredSandboxOpenShellReady("recovered-box", { beforeProbe, captureOpenshellImpl, intervalSeconds: 3, @@ -438,7 +438,7 @@ describe("recreated sandbox OpenShell readiness", () => { │ out\\", details: [], metadata: MetadataMap { headers: {} }"`, `Error: × code: 'The service is currently unavailable', message: "permission denied"`, "Error: sandbox 'other-box' is not ready (phase: Error); wait for it to reach Ready state.", - "Error: sandbox 'recreated-box' is not ready (phase: Failed); wait for it to reach Ready state.", + "Error: sandbox 'recovered-box' is not ready (phase: Failed); wait for it to reach Ready state.", ])("does not retry an unrelated OpenShell error", (stderr) => { const captureOpenshellImpl = vi.fn(() => ({ status: 1, @@ -449,7 +449,7 @@ describe("recreated sandbox OpenShell readiness", () => { const sleeps: number[] = []; expect( - waitForRecreatedSandboxOpenShellReady("recreated-box", { + waitForRecoveredSandboxOpenShellReady("recovered-box", { captureOpenshellImpl, intervalSeconds: 3, sleepImpl: (seconds) => sleeps.push(seconds), @@ -470,7 +470,7 @@ describe("recreated sandbox OpenShell readiness", () => { const sleeps: number[] = []; expect( - waitForRecreatedSandboxOpenShellReady("recreated-box", { + waitForRecoveredSandboxOpenShellReady("recovered-box", { captureOpenshellImpl, intervalSeconds: 3, sleepImpl: (seconds) => sleeps.push(seconds), @@ -497,7 +497,7 @@ describe("recreated sandbox OpenShell readiness", () => { const sleeps: number[] = []; expect( - waitForRecreatedSandboxOpenShellReady("recreated-box", { + waitForRecoveredSandboxOpenShellReady("recovered-box", { beforeProbe, captureOpenshellImpl, intervalSeconds: 3, @@ -521,7 +521,7 @@ describe("recreated sandbox OpenShell readiness", () => { const sleeps: number[] = []; expect( - waitForRecreatedSandboxOpenShellReady("recreated-box", { + waitForRecoveredSandboxOpenShellReady("recovered-box", { beforeProbe, captureOpenshellImpl, intervalSeconds: 3, @@ -545,7 +545,7 @@ describe("recreated sandbox OpenShell readiness", () => { const sleeps: number[] = []; expect( - waitForRecreatedSandboxOpenShellReady("recreated-box", { + waitForRecoveredSandboxOpenShellReady("recovered-box", { beforeProbe, captureOpenshellImpl, intervalSeconds: 3, @@ -569,7 +569,7 @@ describe("recreated sandbox OpenShell readiness", () => { const sleeps: number[] = []; expect( - waitForRecreatedSandboxOpenShellReady("recreated-box", { + waitForRecoveredSandboxOpenShellReady("recovered-box", { beforeProbe, captureOpenshellImpl, intervalSeconds: 3, @@ -593,7 +593,7 @@ describe("recreated sandbox OpenShell readiness", () => { const sleeps: number[] = []; expect( - waitForRecreatedSandboxOpenShellReady("recreated-box", { + waitForRecoveredSandboxOpenShellReady("recovered-box", { beforeProbe, captureOpenshellImpl, intervalSeconds: 3, @@ -618,7 +618,7 @@ describe("recreated sandbox OpenShell readiness", () => { const sleeps: number[] = []; expect( - waitForRecreatedSandboxOpenShellReady("recreated-box", { + waitForRecoveredSandboxOpenShellReady("recovered-box", { captureOpenshellImpl, intervalSeconds: 3, sleepImpl: (seconds) => sleeps.push(seconds), diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index d4f9f945e28..759b6539c9f 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -431,8 +431,8 @@ export async function isSandboxGatewayRunningForStatus( /** * Recover a gateway through the registered agent's managed control boundary. * Legacy custom agents retain their SSH-owned compatibility path. Built-in - * agents may return a transactional supervisor relaunch that the caller must - * commit or roll back after the managed health gate. + * agents may return an identity-pinned in-place supervisor relaunch that the + * caller must verify through the managed health gate. */ type SandboxProcessRecovery = | { kind: "managed" | "custom" } @@ -608,7 +608,7 @@ function isRetryableOpenshellReRegistrationState( sandboxName: string, ): boolean { const error = normalizeOpenshellStructuredError(String(result.stderr)); - // OpenShell can publish Ready before replacement registration settles. + // OpenShell can publish Ready before recovered supervisor registration settles. // Retry only if the readiness probe reports phase Error for this sandbox. // The CLI can emit informational stdout or return unstable process metadata // with this exact stderr refusal; neither changes the result of the read-only @@ -625,10 +625,10 @@ function isRetryableOpenshellReRegistrationState( if (String(result.stdout ?? "").trim() !== "") return false; if (error === OPENSHELL_SANDBOX_NOT_READY) return true; - // OpenShell 0.0.85 can keep the recreated sandbox's cached phase at Ready - // while its replacement supervisor session is still registering. The exec + // OpenShell 0.0.85 can keep the recovered sandbox's cached phase at Ready + // while its recovered supervisor session is still registering. The exec // RPC can fail before a session connects, after a session disconnects, while - // the replacement supervisor's local SSH relay target is starting, or after + // the recovered supervisor's local SSH relay target is starting, or after // the session connects but does not claim its reverse relay within OpenShell's // 10-second relay deadline. These exact results are control-plane // re-registration states; all other OpenShell failures remain terminal. @@ -662,20 +662,20 @@ function isRetryableOpenshellReRegistrationState( ); } -type RecreatedSandboxOpenShellReadinessFailure = +type RecoveredSandboxOpenShellReadinessFailure = | "managed-health-definitive-failure" | "managed-health-inconclusive-timeout" | "openshell-readiness-failure"; -type RecreatedSandboxOpenShellReadinessResult = +type RecoveredSandboxOpenShellReadinessResult = | { ready: true } | { - failure: RecreatedSandboxOpenShellReadinessFailure; + failure: RecoveredSandboxOpenShellReadinessFailure; openshellError?: string; ready: false; }; -type RecreatedSandboxOpenShellReadyOptions = { +type RecoveredSandboxOpenShellReadyOptions = { captureOpenshellImpl?: typeof captureOpenshell; beforeProbe?: (timeoutMs: number) => boolean | null; intervalSeconds?: number; @@ -684,27 +684,27 @@ type RecreatedSandboxOpenShellReadyOptions = { timeoutSeconds?: number; }; -function recreatedSandboxOpenShellReadinessFailureDetail( - failure: RecreatedSandboxOpenShellReadinessFailure, +function recoveredSandboxOpenShellReadinessFailureDetail( + failure: RecoveredSandboxOpenShellReadinessFailure, openshellError?: string, ): string { const detail = (() => { switch (failure) { case "managed-health-definitive-failure": - return "the recreated sandbox failed the managed health guard, so the primary dashboard/API host forward was not started"; + return "the recovered sandbox failed the managed health guard, so the primary dashboard/API host forward was not started"; case "managed-health-inconclusive-timeout": - return "the recreated sandbox managed health guard stayed inconclusive within the readiness deadline, so the primary dashboard/API host forward was not started"; + return "the recovered sandbox managed health guard stayed inconclusive within the readiness deadline, so the primary dashboard/API host forward was not started"; case "openshell-readiness-failure": - return "the recreated sandbox did not become ready in OpenShell, so the primary dashboard/API host forward was not started"; + return "the recovered sandbox did not become ready in OpenShell, so the primary dashboard/API host forward was not started"; } })(); return openshellError ? `${detail} Last OpenShell readiness error: ${openshellError}` : detail; } -// Default seconds to wait for OpenShell to re-register a recreated sandbox as -// Ready before giving up and surfacing the manual-recover hint. Aligned with +// Default seconds to prove OpenShell readiness after direct-container recovery +// before giving up and surfacing the manual-recover hint. Aligned with // `connect`'s readiness budget (`waitForSandboxReadyOrExit` defaults to 120s): -// both prove the same post-recreate sandbox readiness, but this path used to +// both prove the same post-recovery sandbox readiness, but this path used to // give up 4x sooner (30s), so a cold-start `phase: Error` settling window that // exceeded 30s but was within `connect`'s 120s left the primary dashboard/API // forward unstarted — exactly why `connect --probe-only` recovers what `start` @@ -712,15 +712,14 @@ function recreatedSandboxOpenShellReadinessFailureDetail( const GATEWAY_RECOVERY_WAIT_DEFAULT_SECONDS = 120; /** - * Wait until OpenShell has re-registered a directly recreated sandbox as - * ready. This probe deliberately has no direct-Docker or SSH fallback: it is - * proving control-plane readiness, not authorizing the already completed - * replacement-container recovery. + * Wait until OpenShell proves a directly recovered sandbox is ready. This + * probe deliberately has no direct-Docker or SSH fallback: it is proving + * control-plane readiness, not authorizing the already completed recovery. */ -function waitForRecreatedSandboxOpenShellReadyResult( +function waitForRecoveredSandboxOpenShellReadyResult( sandboxName: string, - options: RecreatedSandboxOpenShellReadyOptions = {}, -): RecreatedSandboxOpenShellReadinessResult { + options: RecoveredSandboxOpenShellReadyOptions = {}, +): RecoveredSandboxOpenShellReadinessResult { const capture = options.captureOpenshellImpl ?? captureOpenshell; const now = options.nowImpl ?? Date.now; const sleep = options.sleepImpl ?? sleepSeconds; @@ -796,7 +795,7 @@ function waitForRecreatedSandboxOpenShellReadyResult( } // OpenShell 0.0.85 can follow its exact phase/session transition response // with one or more non-success results that have no structured stderr while - // the replacement registration settles. The process status, spawn error, + // recovered supervisor registration settles. The process status, spawn error, // and informational stdout are not stable across those follow-ups. An // unstructured result is inconclusive only after this loop observed a known // retryable state and while the pinned managed-health guard continues to @@ -838,11 +837,11 @@ function waitForRecreatedSandboxOpenShellReadyResult( }; } -export function waitForRecreatedSandboxOpenShellReady( +export function waitForRecoveredSandboxOpenShellReady( sandboxName: string, - options: RecreatedSandboxOpenShellReadyOptions = {}, + options: RecoveredSandboxOpenShellReadyOptions = {}, ): boolean { - return waitForRecreatedSandboxOpenShellReadyResult(sandboxName, options).ready; + return waitForRecoveredSandboxOpenShellReadyResult(sandboxName, options).ready; } function gatewayRecoveryTimeoutSeconds( @@ -1052,7 +1051,7 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( requestPinnedGatewaySupervisorAction = executeGatewaySupervisorActionPinned, relaunchManagedSupervisorSessionImpl = relaunchManagedSupervisorSession, isSandboxGatewayRunningImpl = isSandboxGatewayRunning, - waitForRecreatedSandboxOpenShellReadyImpl = waitForRecreatedSandboxOpenShellReady, + waitForRecoveredSandboxOpenShellReadyImpl = waitForRecoveredSandboxOpenShellReady, isWsl: isWslOverride, }: { quiet?: boolean; @@ -1060,7 +1059,7 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( requestPinnedGatewaySupervisorAction?: RequestPinnedGatewaySupervisorAction; relaunchManagedSupervisorSessionImpl?: typeof relaunchManagedSupervisorSession; isSandboxGatewayRunningImpl?: typeof isSandboxGatewayRunning; - waitForRecreatedSandboxOpenShellReadyImpl?: typeof waitForRecreatedSandboxOpenShellReady; + waitForRecoveredSandboxOpenShellReadyImpl?: typeof waitForRecoveredSandboxOpenShellReady; isWsl?: boolean; } = {}, ) { @@ -1271,45 +1270,21 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( : null; // Wait for gateway to bind its HTTP port before declaring success. The // recovered process can be alive before the OpenAI-compatible API is ready. - let gatewayReady = false; - try { - gatewayReady = waitForRecoveredSandboxGateway(sandboxName, { - quiet, - initialManagedHealthPassed: recovery.kind === "managed", - requireManagedProbe: recovery.kind === "relaunched", - timeoutSeconds: gatewayRecoveryTimeoutSeconds(recoveryAgent), - managedProbeImpl: (name) => - confirmRecoveredSandboxGatewayManaged(name, { - requestGatewaySupervisorActionImpl: requestManagedProbe, - }), - }); - } catch (error) { - try { - relaunch?.finalize(false); - } catch { - // Preserve the original recovery error; the failure path below will - // direct the operator to inspect/rebuild the sandbox. - } - throw error; - } + const gatewayReady = waitForRecoveredSandboxGateway(sandboxName, { + quiet, + initialManagedHealthPassed: recovery.kind === "managed", + requireManagedProbe: recovery.kind === "relaunched", + timeoutSeconds: gatewayRecoveryTimeoutSeconds(recoveryAgent), + managedProbeImpl: (name) => + confirmRecoveredSandboxGatewayManaged(name, { + requestGatewaySupervisorActionImpl: requestManagedProbe, + }), + }); if (!gatewayReady) { - let rolledBack = true; - if (relaunch) { - try { - rolledBack = relaunch.finalize(false).rolledBack; - } catch { - rolledBack = false; - } - } if (!quiet) { console.error(" Gateway process started but is not responding."); printGatewayWedgeDiagnostics(sandboxName, executeSandboxExecCommand); console.error(" Check /tmp/gateway.log inside the sandbox for details."); - if (!rolledBack) { - console.error( - " Automatic rollback of the previous sandbox container failed; inspect Docker state before retrying.", - ); - } printHostManagedGatewayRecoveryHints( sandboxName, recoveryAgent, @@ -1318,37 +1293,21 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( } return { checked: true, wasRunning: false, recovered: false, forwardRecovered: false }; } - if (relaunch) { - try { - const completion = relaunch.finalize(true); - if (!completion.backupRemoved && !quiet) { - console.error( - " Warning: the recovered sandbox is healthy, but its previous container backup could not be removed.", - ); - } - } catch { - if (!quiet) { - console.error( - " Warning: the recovered sandbox is healthy, but container transaction cleanup could not be confirmed.", - ); - } - } - } const readinessFailureDetail = relaunch ? (() => { - const readinessOptions: RecreatedSandboxOpenShellReadyOptions = { + const readinessOptions: RecoveredSandboxOpenShellReadyOptions = { beforeProbe: (timeoutMs) => confirmRelaunchedManagedHealth?.(timeoutMs) ?? null, timeoutSeconds: SANDBOX_READY_TIMEOUT_SECS, }; const readiness = - waitForRecreatedSandboxOpenShellReadyImpl === waitForRecreatedSandboxOpenShellReady - ? waitForRecreatedSandboxOpenShellReadyResult(sandboxName, readinessOptions) - : waitForRecreatedSandboxOpenShellReadyImpl(sandboxName, readinessOptions) + waitForRecoveredSandboxOpenShellReadyImpl === waitForRecoveredSandboxOpenShellReady + ? waitForRecoveredSandboxOpenShellReadyResult(sandboxName, readinessOptions) + : waitForRecoveredSandboxOpenShellReadyImpl(sandboxName, readinessOptions) ? ({ ready: true } as const) : ({ failure: "openshell-readiness-failure", ready: false } as const); return readiness.ready ? null - : recreatedSandboxOpenShellReadinessFailureDetail( + : recoveredSandboxOpenShellReadinessFailureDetail( readiness.failure, "openshellError" in readiness ? readiness.openshellError : undefined, ); @@ -1449,7 +1408,7 @@ export function checkAndRecoverSandboxProcesses( requestPinnedGatewaySupervisorAction?: RequestPinnedGatewaySupervisorAction; relaunchManagedSupervisorSessionImpl?: typeof relaunchManagedSupervisorSession; isSandboxGatewayRunningImpl?: typeof isSandboxGatewayRunning; - waitForRecreatedSandboxOpenShellReadyImpl?: typeof waitForRecreatedSandboxOpenShellReady; + waitForRecoveredSandboxOpenShellReadyImpl?: typeof waitForRecoveredSandboxOpenShellReady; isWsl?: boolean; } = {}, ) { diff --git a/src/lib/actions/sandbox/supervisor-relaunch.test.ts b/src/lib/actions/sandbox/supervisor-relaunch.test.ts index 38858002b2d..cfdb41095c4 100644 --- a/src/lib/actions/sandbox/supervisor-relaunch.test.ts +++ b/src/lib/actions/sandbox/supervisor-relaunch.test.ts @@ -2,9 +2,6 @@ // SPDX-License-Identifier: Apache-2.0 import { afterEach, describe, expect, it, vi } from "vitest"; -import type { DockerGpuPatchDeps, DockerGpuPatchResult } from "../../onboard/docker-gpu-patch"; -import { finalizeDockerGpuPatchBackup } from "../../onboard/docker-gpu-patch-finalize"; -import { recreateOpenShellDockerSandboxWithStartupCommand } from "../../onboard/docker-startup-command-patch"; import { type ManagedSupervisorRelaunchDeps, relaunchManagedSupervisorSession, @@ -15,74 +12,14 @@ afterEach(() => { vi.unstubAllEnvs(); }); -function patchResult(): DockerGpuPatchResult { +function dockerResult(status: number) { return { - applied: true, - oldContainerId: "old-container-id", - newContainerId: "new-container-id", - originalName: "openshell-alpha", - backupContainerName: "openshell-alpha-nemoclaw-backup", - mode: { - kind: "startup-command", - label: "persistent sandbox startup command", - device: "", - args: [], - }, - backupRemoved: false, - }; -} - -function composedHandoffDeps() { - const inspect = { - Id: "old-container-id", - Image: `sha256:${"c".repeat(64)}`, - Name: "/openshell-alpha", - Config: { - Image: "openshell/sandbox:abc", - Env: ["OPENSHELL_SANDBOX_COMMAND=sleep infinity"], - Labels: { - "openshell.ai/managed-by": "openshell", - "openshell.ai/sandbox-name": "alpha", - }, - Entrypoint: ["/opt/openshell/bin/openshell-sandbox"], - Cmd: [], - User: "0", - WorkingDir: "/workspace", - }, - HostConfig: { - NetworkMode: "openshell-docker", - RestartPolicy: { Name: "unless-stopped" }, - CapAdd: [], - SecurityOpt: [], - }, - }; - const dockerCapture = vi.fn((args: readonly string[]) => - args[0] === "ps" ? "old-container-id\n" : JSON.stringify([inspect]), - ); - const dockerForceRm = vi.fn(() => ({ status: 0 })); - const dockerRm = vi.fn(() => ({ status: 0 })); - const dockerStart = vi.fn(() => ({ status: 0 })); - const dockerStop = vi.fn(() => ({ status: 0 })); - const dockerRename = vi.fn(() => ({ status: 0 })); - const dockerRunDetached = vi.fn(() => ({ status: 0, stdout: "new-container-id\n" })); - const dockerDeps: DockerGpuPatchDeps = { - detectSandboxFallbackDns: () => null, - dockerCapture, - dockerForceRm, - dockerRename, - dockerRm, - dockerRunDetached, - dockerStart, - dockerStop, - now: () => new Date("2026-07-10T00:00:00Z"), - }; - return { - dockerDeps, - dockerForceRm, - dockerRename, - dockerRm, - dockerStart, - dockerStop, + pid: 1, + output: [], + stdout: "", + stderr: "", + status, + signal: null, }; } @@ -103,17 +40,22 @@ function baseDeps(overrides: ManagedSupervisorRelaunchDeps = {}) { }) as never, ), resolveDashboardPort: vi.fn(() => 18789), - resolveContainer: vi.fn(() => "old-container-id"), + resolveContainer: vi.fn(() => "registered-container-id"), inspectContainer: vi.fn(() => ({ Config: { Env: ["OPENSHELL_SANDBOX_COMMAND=sleep infinity"] }, })), confirmMissingSupervisor: vi.fn(() => true), - recreate: vi.fn(() => patchResult()), - finalize: vi.fn(({ supervisorReady }) => - supervisorReady - ? { backupRemoved: true, rolledBack: false } - : { backupRemoved: false, rolledBack: true }, - ), + createNonce: vi.fn(() => "a".repeat(64)), + privilegedExecArgv: vi.fn(() => [ + "exec", + "--user", + "root", + "registered-container-id", + "/usr/local/bin/nemoclaw-gateway-control", + "launch-supervisor", + "a".repeat(64), + ]), + runDocker: vi.fn(() => dockerResult(0)), ...overrides, } satisfies ManagedSupervisorRelaunchDeps; } @@ -124,7 +66,7 @@ describe("relaunchManagedSupervisorSession", () => { expect(relaunchManagedSupervisorSession("missing-box", { quiet: true, deps })).toBeNull(); expect(deps.resolveContainer).not.toHaveBeenCalled(); - expect(deps.recreate).not.toHaveBeenCalled(); + expect(deps.runDocker).not.toHaveBeenCalled(); }); it("honors the troubleshooting kill switch without mutating Docker", () => { @@ -133,7 +75,7 @@ describe("relaunchManagedSupervisorSession", () => { expect(relaunchManagedSupervisorSession("alpha", { quiet: true, deps })).toBeNull(); expect(deps.resolveContainer).not.toHaveBeenCalled(); - expect(deps.recreate).not.toHaveBeenCalled(); + expect(deps.runDocker).not.toHaveBeenCalled(); }); it("refuses a container that no longer has the legacy keepalive startup", () => { @@ -144,18 +86,18 @@ describe("relaunchManagedSupervisorSession", () => { }); expect(relaunchManagedSupervisorSession("alpha", { quiet: true, deps })).toBeNull(); - expect(deps.recreate).not.toHaveBeenCalled(); + expect(deps.runDocker).not.toHaveBeenCalled(); }); - it("refuses recreation when the pinned container no longer proves supervisor absence", () => { + it("refuses launch when the pinned container no longer proves supervisor absence", () => { const deps = baseDeps({ confirmMissingSupervisor: vi.fn(() => false) }); expect(relaunchManagedSupervisorSession("alpha", { quiet: true, deps })).toBeNull(); - expect(deps.confirmMissingSupervisor).toHaveBeenCalledWith("old-container-id"); - expect(deps.recreate).not.toHaveBeenCalled(); + expect(deps.confirmMissingSupervisor).toHaveBeenCalledWith("registered-container-id"); + expect(deps.runDocker).not.toHaveBeenCalled(); }); - it("pins the selected container and persists only a credential-free startup command", () => { + it("requests a credential-free managed launch in the registered keepalive", () => { vi.stubEnv("NEMOCLAW_EXTRA_PLACEHOLDER_KEYS", "CUSTOM_PROVIDER_CREDENTIAL"); vi.stubEnv("CUSTOM_PROVIDER_CREDENTIAL", "s3cr3t-token"); vi.stubEnv("HTTPS_PROXY", "http://proxyuser:proxypass@proxy.example:8080"); @@ -163,105 +105,92 @@ describe("relaunchManagedSupervisorSession", () => { const relaunch = relaunchManagedSupervisorSession("alpha", { quiet: true, deps }); - expect(relaunch).not.toBeNull(); - expect(relaunch?.containerId).toBe("new-container-id"); - expect(deps.recreate).toHaveBeenCalledOnce(); - const options = vi.mocked(deps.recreate).mock.calls[0]?.[0]; - expect(options).toMatchObject({ + expect(relaunch).toEqual({ containerId: "registered-container-id" }); + expect(deps.privilegedExecArgv).toHaveBeenCalledOnce(); + const [sandboxName, launchCommand, stdin, sanitizeEnvironment, expectedContainerId] = + vi.mocked(deps.privilegedExecArgv).mock.calls[0] ?? []; + expect({ + expectedContainerId, + sandboxName, + sanitizeEnvironment, + stdin, + }).toEqual({ + expectedContainerId: "registered-container-id", sandboxName: "alpha", - expectedOldContainerId: "old-container-id", - keepOriginalRunningUntilFinalize: true, - waitForSupervisor: false, + sanitizeEnvironment: true, + stdin: false, }); - const serialized = options?.openshellSandboxCommand.join(" ") ?? ""; + expect(launchCommand?.slice(0, 3)).toEqual([ + "/usr/local/bin/nemoclaw-gateway-control", + "launch-supervisor", + "a".repeat(64), + ]); + const serialized = launchCommand?.join(" ") ?? ""; expect(serialized).toContain("NEMOCLAW_DASHBOARD_PORT=18789"); - expect(serialized).toMatch(/nemoclaw-start$/); expect(serialized).not.toContain("s3cr3t-token"); expect(serialized).not.toContain("CUSTOM_PROVIDER_CREDENTIAL"); expect(serialized).not.toContain("proxypass"); - - expect(relaunch?.finalize(true)).toEqual({ backupRemoved: true, rolledBack: false }); - expect(deps.finalize).toHaveBeenCalledWith({ - result: expect.objectContaining({ newContainerId: "new-container-id" }), - supervisorReady: true, - }); - }); - - it("rolls the container transaction back when managed readiness is not proven", () => { - const deps = baseDeps(); - const relaunch = relaunchManagedSupervisorSession("alpha", { quiet: true, deps }); - - expect(relaunch?.finalize(false)).toEqual({ backupRemoved: false, rolledBack: true }); - expect(deps.finalize).toHaveBeenCalledWith({ - result: expect.objectContaining({ backupContainerName: expect.any(String) }), - supervisorReady: false, - }); - }); - - it("removes the running original container only after supervisor readiness is confirmed", () => { - const handoff = composedHandoffDeps(); - const deps = baseDeps({ - recreate: (options) => - recreateOpenShellDockerSandboxWithStartupCommand(options, handoff.dockerDeps), - finalize: (options) => finalizeDockerGpuPatchBackup(options, handoff.dockerDeps), - }); - - const relaunch = relaunchManagedSupervisorSession("alpha", { quiet: true, deps }); - - expect(relaunch?.containerId).toBe("new-container-id"); - expect(handoff.dockerStop).not.toHaveBeenCalled(); - expect(handoff.dockerForceRm).not.toHaveBeenCalled(); - - expect(relaunch?.finalize(true)).toEqual({ backupRemoved: true, rolledBack: false }); - expect(handoff.dockerForceRm).toHaveBeenCalledWith( - expect.stringContaining("openshell-alpha-nemoclaw-gpu-backup-"), - expect.objectContaining({ ignoreError: true }), + expect(deps.runDocker).toHaveBeenCalledWith( + expect.arrayContaining(["exec", "registered-container-id"]), + expect.objectContaining({ + ignoreError: true, + suppressOutput: true, + timeout: 30000, + }), ); - expect(handoff.dockerStart).not.toHaveBeenCalled(); }); - it("rolls back to the running original container without restarting it when supervisor readiness is not confirmed", () => { - const handoff = composedHandoffDeps(); + it("reconstructs the persisted Hermes dashboard environment", () => { const deps = baseDeps({ - recreate: (options) => - recreateOpenShellDockerSandboxWithStartupCommand(options, handoff.dockerDeps), - finalize: (options) => finalizeDockerGpuPatchBackup(options, handoff.dockerDeps), + getSandbox: vi.fn(() => ({ + name: "alpha", + agent: "hermes", + dashboardPort: 18790, + hermesDashboardEnabled: true, + hermesDashboardInternalPort: 19119, + hermesDashboardPort: 18790, + hermesDashboardTui: true, + openshellDriver: "docker", + })), + getSessionAgent: vi.fn( + () => + ({ + name: "hermes", + displayName: "Hermes", + forwardPort: 18790, + }) as never, + ), + resolveDashboardPort: vi.fn(() => 18790), }); - const relaunch = relaunchManagedSupervisorSession("alpha", { quiet: true, deps }); - expect(relaunch?.finalize(false)).toEqual({ backupRemoved: false, rolledBack: true }); - expect(handoff.dockerStop).toHaveBeenCalledWith( - "new-container-id", - expect.objectContaining({ ignoreError: true }), - ); - expect(handoff.dockerRm).toHaveBeenCalledWith( - "new-container-id", - expect.objectContaining({ ignoreError: true }), - ); - expect(handoff.dockerRename).toHaveBeenLastCalledWith( - expect.stringContaining("openshell-alpha-nemoclaw-gpu-backup-"), - "openshell-alpha", - expect.objectContaining({ ignoreError: true }), + expect(relaunchManagedSupervisorSession("alpha", { quiet: true, deps })).toEqual({ + containerId: "registered-container-id", + }); + const launchCommand = vi.mocked(deps.privilegedExecArgv).mock.calls[0]?.[1] ?? []; + expect(launchCommand).toEqual( + expect.arrayContaining([ + "CHAT_UI_URL=http://127.0.0.1:18790", + "NEMOCLAW_DASHBOARD_PORT=18790", + "NEMOCLAW_HERMES_DASHBOARD=1", + "NEMOCLAW_HERMES_DASHBOARD_PORT=18790", + "NEMOCLAW_HERMES_DASHBOARD_INTERNAL_PORT=19119", + "NEMOCLAW_HERMES_DASHBOARD_TUI=1", + ]), ); - expect(handoff.dockerStart).not.toHaveBeenCalled(); - expect(handoff.dockerForceRm).not.toHaveBeenCalled(); + expect(launchCommand.some((value) => value.startsWith("OPENCLAW_"))).toBe(false); }); - it("returns null when the pinned recreation fails", () => { - const deps = baseDeps({ - recreate: vi.fn(() => { - throw new Error("container identity changed"); - }), - }); + it("returns null when the registered container refuses managed launch", () => { + const deps = baseDeps({ runDocker: vi.fn(() => dockerResult(1)) }); expect(relaunchManagedSupervisorSession("alpha", { quiet: true, deps })).toBeNull(); }); - it("redacts diagnostics when trusted recreation fails", () => { + it("redacts diagnostics when trusted in-place launch fails", () => { vi.spyOn(console, "log").mockImplementation(() => undefined); const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); const deps = baseDeps({ - recreate: vi.fn(() => { + runDocker: vi.fn(() => { throw new Error( "OPENAI_API_KEY=sk-recovery-secret HTTPS_PROXY=http://proxyuser:proxypass@proxy.example:8080", ); diff --git a/src/lib/actions/sandbox/supervisor-relaunch.ts b/src/lib/actions/sandbox/supervisor-relaunch.ts index fecf06465b7..8e000d4754b 100644 --- a/src/lib/actions/sandbox/supervisor-relaunch.ts +++ b/src/lib/actions/sandbox/supervisor-relaunch.ts @@ -1,20 +1,20 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { dockerCapture } from "../../adapters/docker"; +import { randomBytes } from "node:crypto"; +import { dockerCapture, dockerRun } from "../../adapters/docker"; import * as agentRuntime from "../../agent/runtime"; import { shouldManageDashboardForAgent } from "../../onboard/dashboard-runtime"; import { type DockerContainerInspect, parseDockerInspectJson, } from "../../onboard/docker-gpu-patch"; -import { - type DockerGpuPatchFinalizeOutcome, - finalizeDockerGpuPatchBackup, -} from "../../onboard/docker-gpu-patch-finalize"; -import { recreateOpenShellDockerSandboxWithStartupCommand } from "../../onboard/docker-startup-command-patch"; +import { hasZeroDockerExitStatus } from "../../onboard/docker-command-result"; import { buildSandboxRuntimeEnvArgs } from "../../onboard/sandbox-create-launch"; -import { resolveDirectSandboxContainer } from "../../sandbox/privileged-exec"; +import { + privilegedSandboxExecArgv, + resolveDirectSandboxContainer, +} from "../../sandbox/privileged-exec"; import { redact, redactFull } from "../../security/redact"; import * as registry from "../../state/registry"; import { resolveSandboxDashboardPort } from "./forward-recovery"; @@ -23,17 +23,22 @@ import { resolveSandboxDashboardPort } from "./forward-recovery"; * Compatibility boundary for OpenShell 0.0.71's Docker driver: legacy * sandboxes persist `OPENSHELL_SANDBOX_COMMAND=sleep infinity` while * `scripts/nemoclaw-start.sh` owns the managed workload as a sibling process. - * Only that inspected value authorizes this migration. Regression coverage is - * named in `supervisor-relaunch.test.ts` and `gateway-guard-recovery.test.ts`. + * Relaunch requires that inspected legacy value, the registered exact container + * identity, and the managed controller's exact pinned proof that no supervisor + * is running. The root-owned controller then rechecks absence before launching + * one sandbox-UID supervisor that is adopted by OpenShell PID 1. Replacing the + * container while its OpenShell supervisor is registered either publishes + * terminal phase Error or blocks duplicate supervisor registration. Regression + * coverage is named in `supervisor-relaunch.test.ts`, + * `process-recovery-supervisor-relaunch.test.ts`, and `sandbox-survival.test.ts`. * Remove this path after supported upgrades rebuild every legacy keepalive * container with `nemoclaw-start` as its persisted startup command. */ const LEGACY_OPENSHELL_KEEPALIVE = "sleep infinity"; -const DOCKER_INSPECT_TIMEOUT_MS = 15000; +const DOCKER_CONTROL_TIMEOUT_MS = 30000; export type ManagedSupervisorRelaunch = { containerId: string; - finalize(supervisorReady: boolean): DockerGpuPatchFinalizeOutcome; }; export type ManagedSupervisorRelaunchDeps = { @@ -43,15 +48,16 @@ export type ManagedSupervisorRelaunchDeps = { resolveContainer?: typeof resolveDirectSandboxContainer; inspectContainer?: (containerId: string) => DockerContainerInspect; confirmMissingSupervisor?: (containerId: string) => boolean; - recreate?: typeof recreateOpenShellDockerSandboxWithStartupCommand; - finalize?: typeof finalizeDockerGpuPatchBackup; + createNonce?: () => string; + privilegedExecArgv?: typeof privilegedSandboxExecArgv; + runDocker?: typeof dockerRun; }; function inspectContainer(containerId: string): DockerContainerInspect { return parseDockerInspectJson( dockerCapture(["inspect", "--type", "container", containerId], { ignoreError: true, - timeout: DOCKER_INSPECT_TIMEOUT_MS, + timeout: DOCKER_CONTROL_TIMEOUT_MS, }), ); } @@ -64,7 +70,7 @@ function hasLegacyKeepaliveStartup(inspect: DockerContainerInspect): boolean { return values.length === 1 && values[0] === LEGACY_OPENSHELL_KEEPALIVE; } -function reconstructSupervisorLaunchCommand( +function reconstructSupervisorRuntimeEnvironment( sandboxName: string, entry: NonNullable>, deps: ManagedSupervisorRelaunchDeps, @@ -103,7 +109,7 @@ function reconstructSupervisorLaunchCommand( env: process.env, omitCredentialEnv: true, }); - return ["env", ...envArgs, "nemoclaw-start"]; + return envArgs; } export function relaunchManagedSupervisorSession( @@ -122,46 +128,40 @@ export function relaunchManagedSupervisorSession( if (!entry) return null; const driver = entry.openshellDriver?.trim().toLowerCase() ?? null; if (driver !== null && driver !== "docker" && driver !== "vm") return null; - const startupCommand = reconstructSupervisorLaunchCommand(sandboxName, entry, deps); - if (startupCommand === null) return null; + const runtimeEnvironment = reconstructSupervisorRuntimeEnvironment(sandboxName, entry, deps); + if (runtimeEnvironment === null) return null; const resolveContainer = deps.resolveContainer ?? resolveDirectSandboxContainer; const inspect = deps.inspectContainer ?? inspectContainer; const confirmMissingSupervisor = deps.confirmMissingSupervisor; - const recreate = deps.recreate ?? recreateOpenShellDockerSandboxWithStartupCommand; - const finalize = deps.finalize ?? finalizeDockerGpuPatchBackup; + const privilegedExecArgv = deps.privilegedExecArgv ?? privilegedSandboxExecArgv; + const runDocker = deps.runDocker ?? dockerRun; try { const containerId = resolveContainer(sandboxName, driver); if (!hasLegacyKeepaliveStartup(inspect(containerId))) return null; if (!confirmMissingSupervisor?.(containerId)) return null; if (!quiet) { - console.log(" Recreating the sandbox container with its managed startup command..."); + console.log(" Launching the managed supervisor in the registered sandbox container..."); } - const result = recreate({ - sandboxName, - openshellSandboxCommand: startupCommand, - expectedOldContainerId: containerId, - keepOriginalRunningUntilFinalize: true, - waitForSupervisor: false, - }); - let completed: { supervisorReady: boolean; outcome: DockerGpuPatchFinalizeOutcome } | null = - null; - return { - containerId: result.newContainerId, - finalize(supervisorReady) { - if (completed) { - if (completed.supervisorReady !== supervisorReady) { - throw new Error( - "Supervisor relaunch transaction was finalized with conflicting state.", - ); - } - return completed.outcome; - } - const outcome = finalize({ result, supervisorReady }); - completed = { supervisorReady, outcome }; - return outcome; + const nonce = (deps.createNonce ?? (() => randomBytes(32).toString("hex")))(); + const launchCommand = [ + "/usr/local/bin/nemoclaw-gateway-control", + "launch-supervisor", + nonce, + ...runtimeEnvironment, + ]; + const launchResult = runDocker( + privilegedExecArgv(sandboxName, launchCommand, false, true, containerId), + { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_CONTROL_TIMEOUT_MS, }, - }; + ); + if (!hasZeroDockerExitStatus(launchResult)) { + throw new Error("The registered container refused the managed supervisor launch."); + } + return { containerId }; } catch (error) { if (!quiet) { const detail = error instanceof Error ? error.message : String(error); diff --git a/src/lib/onboard/docker-gpu-patch-finalize.test.ts b/src/lib/onboard/docker-gpu-patch-finalize.test.ts index fa53461e1f6..5a691435ce4 100644 --- a/src/lib/onboard/docker-gpu-patch-finalize.test.ts +++ b/src/lib/onboard/docker-gpu-patch-finalize.test.ts @@ -37,42 +37,6 @@ describe("finalizeDockerGpuPatchBackup", () => { ); }); - it("force-removes the running original container when supervisor readiness is confirmed", () => { - const dockerForceRm = vi.fn((_name: string) => ({ status: 0 })); - const dockerRm = vi.fn((_name: string) => ({ status: 0 })); - const outcome = finalizeDockerGpuPatchBackup( - { - result: { ...deferredCreateResult(), backupWasRunning: true }, - supervisorReady: true, - }, - { dockerForceRm, dockerRm }, - ); - - expect(outcome).toEqual({ backupRemoved: true, rolledBack: false }); - expect(dockerForceRm).toHaveBeenCalledWith( - "openshell-alpha-nemoclaw-gpu-backup-1780491860342", - expect.objectContaining({ ignoreError: true }), - ); - expect(dockerRm).not.toHaveBeenCalled(); - }); - - it("reports backupRemoved false when force-removal of the running original container fails", () => { - const outcome = finalizeDockerGpuPatchBackup( - { - result: { ...deferredCreateResult(), backupWasRunning: true }, - supervisorReady: true, - }, - { - dockerForceRm: vi.fn(() => ({ - status: 1, - stderr: "container removal failed", - })), - }, - ); - - expect(outcome).toEqual({ backupRemoved: false, rolledBack: false }); - }); - it("rolls back to the backup container when supervisor reconnect failed", () => { const dockerStop = vi.fn(() => ({ status: 0 })); const dockerRm = vi.fn((_name: string) => ({ status: 0 })); @@ -101,32 +65,6 @@ describe("finalizeDockerGpuPatchBackup", () => { ).toBe(false); }); - it("rolls back to the still-running original container without restarting it", () => { - const dockerStop = vi.fn(() => ({ status: 0 })); - const dockerRm = vi.fn((_name: string) => ({ status: 0 })); - const dockerRename = vi.fn((_old: string, _next: string) => ({ status: 0 })); - const dockerStart = vi.fn(() => ({ status: 0 })); - const outcome = finalizeDockerGpuPatchBackup( - { - result: { ...deferredCreateResult(), backupWasRunning: true }, - supervisorReady: false, - }, - { dockerStop, dockerRm, dockerRename, dockerStart }, - ); - - expect(outcome).toEqual({ backupRemoved: false, rolledBack: true }); - expect(dockerStop).toHaveBeenCalledWith( - "new-container-id", - expect.objectContaining({ ignoreError: true }), - ); - expect(dockerRename).toHaveBeenCalledWith( - "openshell-alpha-nemoclaw-gpu-backup-1780491860342", - "openshell-alpha", - expect.objectContaining({ ignoreError: true }), - ); - expect(dockerStart).not.toHaveBeenCalled(); - }); - it("reports rolledBack=false when restoring the backup fails", () => { const dockerStop = vi.fn(() => ({ status: 0 })); const dockerRm = vi.fn((_name: string) => ({ status: 0 })); diff --git a/src/lib/onboard/docker-gpu-patch-finalize.ts b/src/lib/onboard/docker-gpu-patch-finalize.ts index 25890c0e89b..4eeeedf399b 100644 --- a/src/lib/onboard/docker-gpu-patch-finalize.ts +++ b/src/lib/onboard/docker-gpu-patch-finalize.ts @@ -1,21 +1,16 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 // -// Source-of-truth: this module finalizes two NemoClaw-side Docker recreation -// workarounds. For GPU patching, the invalid state is "OpenShell Docker-driver -// GPU patch left the sandbox in a deleted-backup / failed-new state when the -// post-recreate supervisor reconnect could not confirm the GPU container". -// For legacy supervisor relaunch on OpenShell 0.0.85, the invalid state is -// "stopping the registered keepalive container lets the Docker watcher publish -// a terminal Error phase before the replacement supervisor registers". That -// caller keeps the renamed original container running until the pinned managed -// health check confirms the replacement, then force-removes it; a failed -// managed health check rolls back to the still-running original without -// restarting it. -// -// The preferred source boundaries are OpenShell's Docker create and watcher: -// native NVIDIA GPU access would remove GPU recreation, while replacement-aware -// registration would remove the running-original handoff. Regression coverage: +// Source-of-truth: this module is a NemoClaw-side workaround. The invalid +// state it recovers from is "OpenShell Docker-driver GPU patch left the +// sandbox in a deleted-backup / failed-new state when the post-recreate +// supervisor reconnect could not confirm the GPU container". The preferred +// source boundary for the fix is OpenShell: a Docker-driver sandbox create +// that natively accepts NVIDIA GPU access would remove the need for the +// post-create container recreation NemoClaw performs here. Until OpenShell +// supports that natively, NemoClaw recreates the container with GPU access +// and uses this module to either confirm the new container or restore the +// pre-patch backup. Regression coverage: // * src/lib/onboard/docker-gpu-patch-finalize.test.ts — direct unit tests // for finalize success / rollback / no-op / rollback failure outcomes. // * src/lib/onboard/docker-gpu-patch-rollback.test.ts — composed @@ -23,13 +18,10 @@ // * src/lib/onboard/docker-gpu-sandbox-create.test.ts — composed create // flow driving maybeApplyDuringCreate → waitForSupervisorReconnect → // finalizeBackup. -// * src/lib/actions/sandbox/supervisor-relaunch.test.ts — composed legacy -// relaunch handoff, successful finalization, and rollback without restart. -// Removal conditions: delete the GPU callers when OpenShell supports native -// Docker-driver GPU creation/reconnect. Delete the supervisor handoff branch -// when the legacy relaunch compatibility path is removed or no supported -// OpenShell version publishes phase Error before replacement registration -// settles. Delete this module when neither caller remains. +// Removal condition: when OpenShell supports native Docker-driver GPU +// creation/reconnect, drop the NemoClaw post-create container recreation +// and delete this module along with its callers in docker-gpu-patch.ts and +// docker-gpu-sandbox-create.ts. import { hasZeroDockerExitStatus } from "./docker-command-result"; import { DOCKER_GPU_PATCH_TIMEOUT_MS } from "./docker-gpu-patch-constants"; @@ -73,9 +65,7 @@ export function finalizeDockerGpuPatchBackup( // even if `docker rm` cannot delete it (e.g. concurrent admin action, // daemon timeout). Reflect the actual rm status in the outcome so // diagnostics can flag a leaked backup container. - const rmResult = options.result.backupWasRunning - ? resolved.dockerForceRm(options.result.backupContainerName, containerOpts) - : resolved.dockerRm(options.result.backupContainerName, containerOpts); + const rmResult = resolved.dockerRm(options.result.backupContainerName, containerOpts); return { backupRemoved: hasZeroDockerExitStatus(rmResult), rolledBack: false }; } const rolledBack = rollbackToBackupContainer( @@ -83,7 +73,6 @@ export function finalizeDockerGpuPatchBackup( newContainerId: options.result.newContainerId, backupContainerName: options.result.backupContainerName, originalName: options.result.originalName, - backupWasRunning: options.result.backupWasRunning, }, resolved, ); @@ -96,12 +85,7 @@ export type SupervisorReconnectOutcome = export function reconcileSupervisorReconnect( execReady: boolean, - refs: { - newContainerId: string; - backupContainerName: string; - originalName: string; - backupWasRunning?: boolean; - }, + refs: { newContainerId: string; backupContainerName: string; originalName: string }, deps: DockerGpuPatchDeps, ): SupervisorReconnectOutcome { const resolved = resolveDockerGpuPatchRollbackDeps(deps); @@ -116,9 +100,7 @@ export function reconcileSupervisorReconnect( // leaked backup container but the user-visible sandbox is healthy. // Surface the actual rm status so callers can fold it into diagnostics // alongside the deferred-finalize path in `finalizeDockerGpuPatchBackup`. - const rmResult = refs.backupWasRunning - ? resolved.dockerForceRm(refs.backupContainerName, containerOpts) - : resolved.dockerRm(refs.backupContainerName, containerOpts); + const rmResult = resolved.dockerRm(refs.backupContainerName, containerOpts); return { execReady: true, backupRemoved: hasZeroDockerExitStatus(rmResult) }; } const rolledBack = rollbackToBackupContainer(refs, resolved); diff --git a/src/lib/onboard/docker-gpu-patch-recreate.ts b/src/lib/onboard/docker-gpu-patch-recreate.ts index 7587437378a..b205c144af8 100644 --- a/src/lib/onboard/docker-gpu-patch-recreate.ts +++ b/src/lib/onboard/docker-gpu-patch-recreate.ts @@ -3,7 +3,6 @@ import { dockerCapture, - dockerForceRm, dockerRename, dockerRm, dockerRun, @@ -49,7 +48,6 @@ type RecreateDeps = Required< Pick< DockerGpuPatchDeps, | "dockerCapture" - | "dockerForceRm" | "dockerRun" | "dockerRunDetached" | "dockerRename" @@ -68,7 +66,6 @@ type RecreateDeps = Required< function recreateDeps(deps: DockerGpuPatchDeps): RecreateDeps { return { dockerCapture, - dockerForceRm, dockerRun, dockerRunDetached, dockerRename, @@ -158,7 +155,6 @@ export function recreateOpenShellDockerSandboxContainer( gpuDevice?: string | null; timeoutSecs?: number; waitForSupervisor?: boolean; - keepOriginalRunningUntilFinalize?: boolean; openshellSandboxCommand?: readonly string[] | null; requiredUlimits?: readonly import("./docker-gpu-patch-types").DockerUlimit[] | null; expectedOldContainerId?: string | null; @@ -175,11 +171,6 @@ export function recreateOpenShellDockerSandboxContainer( }; try { validateRequiredDockerUlimits(options.requiredUlimits); - if (options.keepOriginalRunningUntilFinalize && options.waitForSupervisor !== false) { - throw new Error( - "Keeping the original OpenShell supervisor running requires deferred supervisor finalization.", - ); - } const containerIds = findOpenShellDockerSandboxContainerIds(options.sandboxName, deps); const oldContainerId = containerIds[0]; if (!oldContainerId) { @@ -293,24 +284,21 @@ export function recreateOpenShellDockerSandboxContainer( suppressOutput: true, timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, }; - const backupWasRunning = options.keepOriginalRunningUntilFinalize === true; - if (!backupWasRunning) { - const stopResult = d.dockerStop(oldContainerId, { - ...containerMutationOptions, - timeout: DOCKER_GPU_PATCH_STOP_TIMEOUT_MS, - }); - if (!hasZeroDockerExitStatus(stopResult)) { - context.rolledBack = hasZeroDockerExitStatus( - d.dockerStart(oldContainerId, containerMutationOptions), - ); - throw new Error( - `Could not stop original sandbox container: ${resultText(stopResult)}; ${ - context.rolledBack - ? "original sandbox container confirmed running" - : "restart failed; original sandbox container may be stopped" - }`, - ); - } + const stopResult = d.dockerStop(oldContainerId, { + ...containerMutationOptions, + timeout: DOCKER_GPU_PATCH_STOP_TIMEOUT_MS, + }); + if (!hasZeroDockerExitStatus(stopResult)) { + context.rolledBack = hasZeroDockerExitStatus( + d.dockerStart(oldContainerId, containerMutationOptions), + ); + throw new Error( + `Could not stop original sandbox container: ${resultText(stopResult)}; ${ + context.rolledBack + ? "original sandbox container confirmed running" + : "restart failed; original sandbox container may be stopped" + }`, + ); } const renameResult = d.dockerRename( oldContainerId, @@ -319,9 +307,9 @@ export function recreateOpenShellDockerSandboxContainer( ); if (!hasZeroDockerExitStatus(renameResult)) { d.dockerRename(backupContainerName, originalName, containerMutationOptions); - const restarted = - backupWasRunning || - hasZeroDockerExitStatus(d.dockerStart(oldContainerId, containerMutationOptions)); + const restarted = hasZeroDockerExitStatus( + d.dockerStart(oldContainerId, containerMutationOptions), + ); let originalNameRestored = false; try { originalNameRestored = @@ -346,7 +334,7 @@ export function recreateOpenShellDockerSandboxContainer( }); if (!hasZeroDockerExitStatus(runResult)) { context.rolledBack = restoreDockerGpuPatchBackupAfterRecreateFailure( - { newContainerId: originalName, backupContainerName, originalName, backupWasRunning }, + { newContainerId: originalName, backupContainerName, originalName }, deps, ); const containerDescription = @@ -372,7 +360,7 @@ export function recreateOpenShellDockerSandboxContainer( ); if (!newContainerId) { context.rolledBack = restoreDockerGpuPatchBackupAfterRecreateFailure( - { newContainerId: originalName, backupContainerName, originalName, backupWasRunning }, + { newContainerId: originalName, backupContainerName, originalName }, deps, ); const containerDescription = @@ -396,7 +384,6 @@ export function recreateOpenShellDockerSandboxContainer( originalName, backupContainerName, mode: selectedMode, - backupWasRunning, backupRemoved, }); if (options.waitForSupervisor === false) return result(false); diff --git a/src/lib/onboard/docker-gpu-patch-rollback.ts b/src/lib/onboard/docker-gpu-patch-rollback.ts index a18ae91febc..81532529503 100644 --- a/src/lib/onboard/docker-gpu-patch-rollback.ts +++ b/src/lib/onboard/docker-gpu-patch-rollback.ts @@ -2,7 +2,6 @@ // SPDX-License-Identifier: Apache-2.0 import { - dockerForceRm as defaultDockerForceRm, dockerRename as defaultDockerRename, dockerRm as defaultDockerRm, dockerStart as defaultDockerStart, @@ -27,7 +26,6 @@ type DockerRenameFn = ( ) => DockerRunResult; export type ResolvedDockerGpuPatchRollbackDeps = { - dockerForceRm: DockerContainerFn; dockerStop: DockerContainerFn; dockerRm: DockerContainerFn; dockerRename: DockerRenameFn; @@ -38,7 +36,6 @@ export function resolveDockerGpuPatchRollbackDeps( deps: DockerGpuPatchDeps, ): ResolvedDockerGpuPatchRollbackDeps { return { - dockerForceRm: deps.dockerForceRm ?? defaultDockerForceRm, dockerStop: deps.dockerStop ?? defaultDockerStop, dockerRm: deps.dockerRm ?? defaultDockerRm, dockerRename: deps.dockerRename ?? defaultDockerRename, @@ -47,12 +44,7 @@ export function resolveDockerGpuPatchRollbackDeps( } export function rollbackToBackupContainer( - refs: { - newContainerId: string; - backupContainerName: string; - originalName: string; - backupWasRunning?: boolean; - }, + refs: { newContainerId: string; backupContainerName: string; originalName: string }, deps: ResolvedDockerGpuPatchRollbackDeps, ): boolean { const containerOpts = { @@ -64,19 +56,13 @@ export function rollbackToBackupContainer( deps.dockerRm(refs.newContainerId, containerOpts); const restored = deps.dockerRename(refs.backupContainerName, refs.originalName, containerOpts); if (!hasZeroDockerExitStatus(restored)) return false; - if (refs.backupWasRunning) return true; const started = deps.dockerStart(refs.originalName, containerOpts); return hasZeroDockerExitStatus(started); } /** Restore the original sandbox after `docker run` fails during GPU recreation. */ export function restoreDockerGpuPatchBackupAfterRecreateFailure( - refs: { - newContainerId: string; - backupContainerName: string; - originalName: string; - backupWasRunning?: boolean; - }, + refs: { newContainerId: string; backupContainerName: string; originalName: string }, deps: DockerGpuPatchDeps = {}, ): boolean { return rollbackToBackupContainer(refs, resolveDockerGpuPatchRollbackDeps(deps)); diff --git a/src/lib/onboard/docker-gpu-patch-types.ts b/src/lib/onboard/docker-gpu-patch-types.ts index a935972e242..32be0afe46f 100644 --- a/src/lib/onboard/docker-gpu-patch-types.ts +++ b/src/lib/onboard/docker-gpu-patch-types.ts @@ -27,7 +27,6 @@ export type DockerGpuPatchDeps = { dockerRun?: DockerRunFn; dockerRunDetached?: DockerRunFn; dockerRename?: DockerRenameFn; - dockerForceRm?: DockerContainerFn; dockerRm?: DockerContainerFn; dockerStart?: DockerContainerFn; dockerStop?: DockerContainerFn; @@ -92,11 +91,6 @@ export type DockerGpuPatchResult = { originalName: string; backupContainerName: string; mode: DockerGpuPatchMode; - // True when a deferred startup-command recreation kept the original - // OpenShell supervisor running while the replacement established managed - // health. Finalization attempts to force-remove that original container on - // success and does not try to restart it during rollback. - backupWasRunning?: boolean; // True when the patch path also confirmed supervisor reconnect AND removed // the backup container. False when the caller deferred the reconnect wait // (via `waitForSupervisor: false`); the backup is still in place and the diff --git a/src/lib/onboard/docker-startup-command-patch.test.ts b/src/lib/onboard/docker-startup-command-patch.test.ts index b4557a7302b..71c5c9d12c2 100644 --- a/src/lib/onboard/docker-startup-command-patch.test.ts +++ b/src/lib/onboard/docker-startup-command-patch.test.ts @@ -43,99 +43,6 @@ function inspectFixture(): DockerContainerInspect { } describe("Docker startup-command patch", () => { - it("keeps the registered supervisor running until deferred recovery finalizes", () => { - const dockerCapture = vi.fn((args: readonly string[]) => - args[0] === "ps" - ? "old-container-id\n" - : args[0] === "inspect" - ? JSON.stringify([inspectFixture()]) - : "", - ); - const dockerStop = vi.fn(() => ({ status: 0 })); - const dockerRename = vi.fn(() => ({ status: 0 })); - const dockerRunDetached = vi.fn(() => ({ status: 0, stdout: "new-container-id\n" })); - - const result = recreateStartupCommandForTest( - { - sandboxName: "alpha", - keepOriginalRunningUntilFinalize: true, - waitForSupervisor: false, - openshellSandboxCommand: ["env", "nemoclaw-start"], - }, - { - dockerCapture, - dockerRunDetached, - dockerRename, - dockerStop, - now: () => new Date("2026-07-10T00:00:00Z"), - }, - ); - - expect(result).toMatchObject({ - newContainerId: "new-container-id", - backupWasRunning: true, - backupRemoved: false, - }); - expect(dockerStop).not.toHaveBeenCalled(); - expect(dockerRename.mock.invocationCallOrder[0]).toBeLessThan( - dockerRunDetached.mock.invocationCallOrder[0] ?? Number.POSITIVE_INFINITY, - ); - }); - - it("requires deferred finalization when preserving the registered supervisor", () => { - const dockerCapture = vi.fn(); - - expect(() => - recreateStartupCommandForTest( - { - sandboxName: "alpha", - keepOriginalRunningUntilFinalize: true, - openshellSandboxCommand: ["env", "nemoclaw-start"], - }, - { dockerCapture }, - ), - ).toThrow(/requires deferred supervisor finalization/); - expect(dockerCapture).not.toHaveBeenCalled(); - }); - - it("rolls back to the running original container without restarting it when replacement creation fails", () => { - const dockerRename = vi.fn(() => ({ status: 0 })); - const dockerStart = vi.fn(() => ({ status: 0 })); - - expect(() => - recreateStartupCommandForTest( - { - sandboxName: "alpha", - keepOriginalRunningUntilFinalize: true, - waitForSupervisor: false, - openshellSandboxCommand: ["env", "nemoclaw-start"], - }, - { - dockerCapture: vi.fn((args: readonly string[]) => - args[0] === "ps" - ? "old-container-id\n" - : args[0] === "inspect" - ? JSON.stringify([inspectFixture()]) - : "", - ), - dockerRunDetached: vi.fn(() => ({ status: 1, stderr: "boom" })), - dockerRename, - dockerRm: vi.fn(() => ({ status: 0 })), - dockerStart, - dockerStop: vi.fn(() => ({ status: 0 })), - now: () => new Date("2026-07-10T00:00:00Z"), - }, - ), - ).toThrow(/Could not start recreated sandbox container: boom; pre-patch sandbox restored/); - - expect(dockerRename).toHaveBeenLastCalledWith( - expect.stringContaining("openshell-alpha-nemoclaw-gpu-backup-"), - "openshell-alpha", - expect.objectContaining({ ignoreError: true }), - ); - expect(dockerStart).not.toHaveBeenCalled(); - }); - it("persists the startup command without adding GPU-only container privileges", () => { const dockerCaptureOutput: Record = { ps: "old-container-id\n", diff --git a/src/lib/onboard/docker-startup-command-patch.ts b/src/lib/onboard/docker-startup-command-patch.ts index 6541152e2b7..2b5b6f761ef 100644 --- a/src/lib/onboard/docker-startup-command-patch.ts +++ b/src/lib/onboard/docker-startup-command-patch.ts @@ -15,7 +15,6 @@ export function recreateOpenShellDockerSandboxWithStartupCommand( sandboxName: string; timeoutSecs?: number; waitForSupervisor?: boolean; - keepOriginalRunningUntilFinalize?: boolean; openshellSandboxCommand: readonly string[]; requiredUlimits?: readonly import("./docker-gpu-patch-types").DockerUlimit[] | null; expectedOldContainerId?: string | null; diff --git a/src/lib/onboard/finalization-deps.ts b/src/lib/onboard/finalization-deps.ts index 750779bccd0..08831e66205 100644 --- a/src/lib/onboard/finalization-deps.ts +++ b/src/lib/onboard/finalization-deps.ts @@ -11,7 +11,7 @@ export const finalizationHandlerDeps = { const processRecovery: typeof import("../actions/sandbox/process-recovery") = require("../actions/sandbox/process-recovery"); const { SANDBOX_READY_TIMEOUT_SECS }: typeof import("./env") = require("./env"); - return processRecovery.waitForRecreatedSandboxOpenShellReady(name, { + return processRecovery.waitForRecoveredSandboxOpenShellReady(name, { timeoutSeconds: SANDBOX_READY_TIMEOUT_SECS, }); }, diff --git a/test/gateway-supervisor-control.test.ts b/test/gateway-supervisor-control.test.ts index 8a9e429fd96..0a3f292fdc1 100644 --- a/test/gateway-supervisor-control.test.ts +++ b/test/gateway-supervisor-control.test.ts @@ -278,6 +278,7 @@ describe("root-only gateway control helper", () => { it.each([ "restart", "probe", + "launch-supervisor", ])("enters managed %s control with isolated Python before user-site startup hooks", (action) => { const root = temporaryDirectory("nemoclaw-managed-python-isolation-"); const userBase = join(root, "attacker-userbase"); @@ -339,6 +340,11 @@ describe("root-only gateway control helper", () => { ["bad action", ["replace", VALID_NONCE], "SUPERVISOR_INVALID_ACTION"], ["short nonce", ["restart", "abcd"], "SUPERVISOR_INVALID_NONCE"], ["uppercase nonce", ["recover", "B".repeat(64)], "SUPERVISOR_INVALID_NONCE"], + [ + "extra restart argument", + ["restart", VALID_NONCE, "CHAT_UI_URL=http://127.0.0.1:18789"], + "SUPERVISOR_INVALID_REQUEST", + ], ])("rejects %s before touching the control directory", (_label, args, marker) => { const result = spawnSync(CONTROL_HELPER, args, { encoding: "utf-8", diff --git a/test/managed-supervisor-launch.test.ts b/test/managed-supervisor-launch.test.ts new file mode 100644 index 00000000000..5a5a87f0dfa --- /dev/null +++ b/test/managed-supervisor-launch.test.ts @@ -0,0 +1,108 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; + +const HELPER = path.join(import.meta.dirname, "..", "scripts", "managed-gateway-control.py"); +const NONCE = "a".repeat(64); + +const SUPERVISOR_LAUNCH_ENV_HARNESS = String.raw` +import importlib.util +import json +import os +import sys + +spec = importlib.util.spec_from_file_location("managed_control_launch", sys.argv[1]) +control = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = control +spec.loader.exec_module(control) + +os.environ.update({ + "LD_PRELOAD": "/attacker/preload.so", + "NODE_OPTIONS": "--require=/attacker/hook.js", + "NEMOCLAW_TEST_ESCAPE": "attacker", + "NVIDIA_INFERENCE_API_KEY": "container-owned-placeholder", +}) +action, nonce, runtime = control._validate_request([ + "launch-supervisor", + "a" * 64, + "CHAT_UI_URL=http://127.0.0.1:18789", + "NEMOCLAW_DASHBOARD_PORT=18789", + "HTTPS_PROXY=https://proxy.example/path?token=a=b", +]) +environment = control._supervisor_launch_environment(runtime) +print(json.dumps({ + "action": action, + "nonce": nonce, + "runtime": runtime, + "identity": { + key: environment.get(key) + for key in ("HOME", "LOGNAME", "PATH", "SHELL", "USER") + }, + "python_no_user_site": environment.get("PYTHONNOUSERSITE"), + "stripped": { + key: key in environment + for key in ("LD_PRELOAD", "NODE_OPTIONS", "NEMOCLAW_TEST_ESCAPE") + }, + "container_environment": environment.get("NVIDIA_INFERENCE_API_KEY"), +}, sort_keys=True)) +`; + +describe("managed supervisor launch", () => { + it("allowlists launch inputs and strips loader hooks before sandbox UID launch", () => { + const result = spawnSync("python3", ["-c", SUPERVISOR_LAUNCH_ENV_HARNESS, HELPER], { + encoding: "utf-8", + timeout: 5000, + }); + + expect(result.status, result.stderr).toBe(0); + expect(JSON.parse(result.stdout)).toEqual({ + action: "launch-supervisor", + nonce: NONCE, + runtime: { + CHAT_UI_URL: "http://127.0.0.1:18789", + HTTPS_PROXY: "https://proxy.example/path?token=a=b", + NEMOCLAW_DASHBOARD_PORT: "18789", + }, + identity: { + HOME: "/sandbox", + LOGNAME: "sandbox", + PATH: "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + SHELL: "/bin/bash", + USER: "sandbox", + }, + python_no_user_site: "1", + stripped: { + LD_PRELOAD: false, + NEMOCLAW_TEST_ESCAPE: false, + NODE_OPTIONS: false, + }, + container_environment: "container-owned-placeholder", + }); + }); + + it.each([ + [["launch-supervisor", NONCE, "UNREVIEWED=value"], "SUPERVISOR_INVALID_REQUEST"], + [ + [ + "launch-supervisor", + NONCE, + "CHAT_UI_URL=http://127.0.0.1:18789", + "CHAT_UI_URL=http://127.0.0.1:18790", + ], + "SUPERVISOR_INVALID_REQUEST", + ], + [["restart", NONCE, "CHAT_UI_URL=http://127.0.0.1:18789"], "SUPERVISOR_INVALID_REQUEST"], + ])("rejects disallowed or duplicate request arguments before privilege use", (args, marker) => { + const result = spawnSync("python3", [HELPER, ...args], { + encoding: "utf-8", + timeout: 5000, + }); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(""); + expect(result.stderr.trim()).toBe(marker); + }); +}); diff --git a/test/process-recovery-supervisor-relaunch.test.ts b/test/process-recovery-supervisor-relaunch.test.ts index cb0fe7ac52e..3cf53a7deb7 100644 --- a/test/process-recovery-supervisor-relaunch.test.ts +++ b/test/process-recovery-supervisor-relaunch.test.ts @@ -99,15 +99,15 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { stderr: "SUPERVISOR_NOT_RUNNING", })); const resolveContainer = vi.fn(() => "old-container-id"); - const recreate = vi.fn(() => { - throw new Error("kill switch allowed container mutation"); + const runDocker = vi.fn(() => { + throw new Error("kill switch allowed supervisor launch"); }); const requestPinnedGatewaySupervisorAction = vi.fn(() => null); const relaunchManagedSupervisorSessionImpl = vi.fn( (sandboxName: string, options: Parameters[1]) => relaunchManagedSupervisorSession(sandboxName, { quiet: options.quiet, - deps: { ...options.deps, resolveContainer, recreate }, + deps: { ...options.deps, resolveContainer, runDocker }, }), ); const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); @@ -129,7 +129,7 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { ); expect(resolveContainer).not.toHaveBeenCalled(); expect(requestPinnedGatewaySupervisorAction).not.toHaveBeenCalled(); - expect(recreate).not.toHaveBeenCalled(); + expect(runDocker).not.toHaveBeenCalled(); const errorLines = errorSpy.mock.calls.map((call) => String(call[0])); expect(errorLines).toContainEqual( expect.stringContaining("Failure layer: supervisor not running"), @@ -141,13 +141,11 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { ); }); - it("rolls back when recreation starts but managed control never accepts it", () => { + it("keeps the registered container when in-place managed control never accepts it", () => { mockOpenClawSandbox("rejected-box"); setImmediateRecoveryPolling(); - const finalize = vi.fn(() => ({ backupRemoved: false, rolledBack: true })); const relaunchManagedSupervisorSessionImpl = vi.fn(() => ({ - containerId: "replacement-container-id", - finalize, + containerId: "registered-container-id", })); const requestGatewaySupervisorAction = vi.fn((_name: string, action: string) => action === "recover" ? { status: 1, stdout: "", stderr: "SUPERVISOR_NOT_RUNNING" } : null, @@ -167,23 +165,15 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { "rejected-box", "probe", 210000, - "replacement-container-id", + "registered-container-id", ); - expect(finalize).toHaveBeenCalledOnce(); - expect(finalize).toHaveBeenCalledWith(false); }); - it("commits only after managed health accepts the recreated supervisor", () => { + it("accepts the in-place supervisor only after managed health passes", () => { mockOpenClawSandbox("recovered-box"); setImmediateRecoveryPolling(); - const finalize = vi.fn((supervisorReady: boolean) => - supervisorReady - ? { backupRemoved: true, rolledBack: false } - : { backupRemoved: false, rolledBack: true }, - ); const relaunchManagedSupervisorSessionImpl = vi.fn(() => ({ - containerId: "replacement-container-id", - finalize, + containerId: "registered-container-id", })); const requestGatewaySupervisorAction = vi.fn((_name: string, action: string) => action === "recover" ? { status: 1, stdout: "", stderr: "SUPERVISOR_NOT_RUNNING" } : null, @@ -214,26 +204,18 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { "recovered-box", "probe", 210000, - "replacement-container-id", + "registered-container-id", ); - expect(finalize).toHaveBeenCalledOnce(); - expect(finalize).toHaveBeenCalledWith(true); }); - it("retries a busy pinned managed probe before starting the replacement forward", () => { + it("retries a busy pinned managed probe before starting the recovered forward", () => { mockOpenClawSandbox("busy-recovered-box"); vi.stubEnv("NEMOCLAW_GATEWAY_RECOVERY_POLL_INTERVAL_SECONDS", "0"); vi.stubEnv("NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS", "1"); vi.stubEnv("NEMOCLAW_GATEWAY_RECOVERY_SETTLE_SECONDS", "0"); vi.stubEnv("NEMOCLAW_FORWARD_RECOVERY_WAIT_MS", "0"); - const finalize = vi.fn((supervisorReady: boolean) => - supervisorReady - ? { backupRemoved: true, rolledBack: false } - : { backupRemoved: false, rolledBack: true }, - ); const relaunchManagedSupervisorSessionImpl = vi.fn(() => ({ - containerId: "replacement-container-id", - finalize, + containerId: "registered-container-id", })); const requestGatewaySupervisorAction = vi.fn((_name: string, action: string) => action === "recover" ? { status: 1, stdout: "", stderr: "SUPERVISOR_NOT_RUNNING" } : null, @@ -301,7 +283,6 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { ["sandbox", "exec", "--name", "busy-recovered-box", "--", "true"], expect.objectContaining({ ignoreError: true }), ); - expect(finalize).toHaveBeenCalledWith(true); expect(runOpenshell).toHaveBeenCalledWith( ["forward", "start", "--background", "18789", "busy-recovered-box"], expect.objectContaining({ ignoreError: true }), @@ -311,10 +292,8 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { it("uses the sandbox readiness budget after a longer gateway health wait (#7273)", () => { mockOpenClawSandbox("unready-box", 600); setImmediateRecoveryPolling(); - const finalize = vi.fn(() => ({ backupRemoved: true, rolledBack: false })); const relaunchManagedSupervisorSessionImpl = vi.fn(() => ({ - containerId: "replacement-container-id", - finalize, + containerId: "registered-container-id", })); const requestGatewaySupervisorAction = vi.fn(() => ({ status: 1, @@ -326,7 +305,7 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { stdout: "GATEWAY_PID=4242\n", stderr: "", })); - const waitForRecreatedSandboxOpenShellReadyImpl = vi.fn(() => false); + const waitForRecoveredSandboxOpenShellReadyImpl = vi.fn(() => false); const runOpenshell = vi.spyOn(openshellRuntime, "runOpenshell"); const result = checkAndRecoverSandboxProcesses("unready-box", { @@ -335,7 +314,7 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { requestGatewaySupervisorAction, requestPinnedGatewaySupervisorAction, relaunchManagedSupervisorSessionImpl, - waitForRecreatedSandboxOpenShellReadyImpl, + waitForRecoveredSandboxOpenShellReadyImpl, }); expect(result).toMatchObject({ @@ -346,9 +325,7 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { forwardRecoveryFailed: true, forwardRecoveryFailureDetail: expect.stringContaining("did not become ready in OpenShell"), }); - expect(finalize).toHaveBeenCalledOnce(); - expect(finalize).toHaveBeenCalledWith(true); - expect(waitForRecreatedSandboxOpenShellReadyImpl).toHaveBeenCalledWith( + expect(waitForRecoveredSandboxOpenShellReadyImpl).toHaveBeenCalledWith( "unready-box", expect.objectContaining({ beforeProbe: expect.any(Function), timeoutSeconds: 180 }), ); @@ -358,10 +335,8 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { it("reports the last structured OpenShell error when readiness times out", () => { mockOpenClawSandbox("relay-dropped-box"); setImmediateRecoveryPolling(); - const finalize = vi.fn(() => ({ backupRemoved: true, rolledBack: false })); const relaunchManagedSupervisorSessionImpl = vi.fn(() => ({ - containerId: "replacement-container-id", - finalize, + containerId: "registered-container-id", })); const requestGatewaySupervisorAction = vi.fn(() => ({ status: 1, @@ -410,17 +385,14 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { ), }); expect(captureOpenshell).toHaveBeenCalled(); - expect(finalize).toHaveBeenCalledWith(true); expect(runOpenshell).not.toHaveBeenCalled(); }); it("reports a definitive managed health failure separately from OpenShell readiness", () => { mockOpenClawSandbox("managed-failed-box"); setImmediateRecoveryPolling(); - const finalize = vi.fn(() => ({ backupRemoved: true, rolledBack: false })); const relaunchManagedSupervisorSessionImpl = vi.fn(() => ({ - containerId: "replacement-container-id", - finalize, + containerId: "registered-container-id", })); const requestGatewaySupervisorAction = vi.fn(() => ({ status: 1, @@ -458,11 +430,10 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { forwardRecoveryFailed: true, forwardRecoveryFailureDetail: expect.stringContaining("failed the managed health guard"), }); - expect(finalize).toHaveBeenCalledWith(true); expect(captureOpenshell).not.toHaveBeenCalled(); }); - it("rejects a healthy forward when the replacement identity changes after readiness", () => { + it("rejects a healthy forward when the registered container identity changes", () => { mockOpenClawSandbox("drifted-box"); vi.mocked(agentRuntime.getSessionAgent).mockReturnValue({ name: "openclaw", @@ -472,10 +443,8 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { healthProbe: { url: "http://127.0.0.1:18789/health", port: 18789, timeout_seconds: 30 }, } as never); setImmediateRecoveryPolling(); - const finalize = vi.fn(() => ({ backupRemoved: true, rolledBack: false })); const relaunchManagedSupervisorSessionImpl = vi.fn(() => ({ - containerId: "replacement-container-id", - finalize, + containerId: "registered-container-id", })); const requestGatewaySupervisorAction = vi.fn(() => ({ status: 1, @@ -495,7 +464,7 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { throw new Error("replacement identity changed"); }) .mockReturnValue(acceptedProbe); - const waitForRecreatedSandboxOpenShellReadyImpl = vi.fn( + const waitForRecoveredSandboxOpenShellReadyImpl = vi.fn( (_name, options) => options.beforeProbe?.(1000) === true, ); vi.spyOn(forwardHealth, "isLocalForwardReachable").mockReturnValue(true); @@ -513,7 +482,7 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { requestGatewaySupervisorAction, requestPinnedGatewaySupervisorAction, relaunchManagedSupervisorSessionImpl, - waitForRecreatedSandboxOpenShellReadyImpl, + waitForRecoveredSandboxOpenShellReadyImpl, }); expect(result).toMatchObject({ @@ -528,9 +497,8 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { "drifted-box", "probe", 15000, - "replacement-container-id", + "registered-container-id", ); - expect(finalize).toHaveBeenCalledWith(true); expect(runOpenshell).toHaveBeenCalledOnce(); expect(runOpenshell).toHaveBeenCalledWith(["forward", "stop", "18789", "drifted-box"], { ignoreError: true, From ab4fca5ff119e7164a205d1c6a616fb08552a18e Mon Sep 17 00:00:00 2001 From: San Dang Date: Thu, 30 Jul 2026 11:44:01 +0530 Subject: [PATCH 11/30] fix(recovery): harden supervisor launch cleanup --- scripts/managed-gateway-control.py | 9 +- test/managed-supervisor-launch.test.ts | 141 +++++++++++++++++++++++++ 2 files changed, 147 insertions(+), 3 deletions(-) diff --git a/scripts/managed-gateway-control.py b/scripts/managed-gateway-control.py index 3ae6cd06772..b21c949202b 100755 --- a/scripts/managed-gateway-control.py +++ b/scripts/managed-gateway-control.py @@ -1671,13 +1671,14 @@ def _spawn_supervisor_as_orphan(environment: dict[str, str]) -> tuple[int, int]: os.close(status_write_fd) os.close(adoption_read_fd) os._exit(0) - except BaseException: + except Exception: try: os.write(status_write_fd, b"SUPERVISOR_LAUNCH_FAILED\n") os.close(status_write_fd) os.close(adoption_read_fd) except OSError: - pass + # The parent rejects a failed handshake from the intermediate process. + os._exit(1) os._exit(1) os.close(status_write_fd) @@ -1755,7 +1756,9 @@ def _wait_for_launched_supervisor( ProcessLookupError, PermissionError, ): - pass + # Process evidence can change until the bounded adoption proof expires. + time.sleep(POLL_SECONDS) + continue time.sleep(POLL_SECONDS) raise ControlError("SUPERVISOR_UNAVAILABLE") diff --git a/test/managed-supervisor-launch.test.ts b/test/managed-supervisor-launch.test.ts index 5a5a87f0dfa..d9fc1923891 100644 --- a/test/managed-supervisor-launch.test.ts +++ b/test/managed-supervisor-launch.test.ts @@ -50,6 +50,125 @@ print(json.dumps({ }, sort_keys=True)) `; +const SUPERVISOR_ADOPTION_HARNESS = String.raw` +import importlib.util +import json +import os +import sys +import tempfile + +spec = importlib.util.spec_from_file_location("managed_control_adoption", sys.argv[1]) +control = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = control +spec.loader.exec_module(control) + +def write_process(proc_root, namespace_path, pid, start_time, parent_pid, uid, cmdline): + process_root = os.path.join(proc_root, str(pid)) + os.makedirs(os.path.join(process_root, "ns")) + fields = ["S", str(parent_pid)] + (["0"] * 15) + ["1", "0", str(start_time)] + with open(os.path.join(process_root, "stat"), "w", encoding="ascii") as stream: + stream.write(f"{pid} (managed) {' '.join(fields)}\n") + with open(os.path.join(process_root, "status"), "w", encoding="ascii") as stream: + stream.write(f"Uid:\t{uid}\t{uid}\t{uid}\t{uid}\nNSpid:\t{pid}\n") + with open(os.path.join(process_root, "cmdline"), "wb") as stream: + stream.write(cmdline) + os.link(namespace_path, os.path.join(process_root, "ns", "pid")) + +def run_case(supervisor_parent_pid): + with tempfile.TemporaryDirectory() as root: + proc_root = os.path.join(root, "proc") + system_root = os.path.join(root, "system") + os.makedirs(proc_root) + os.makedirs(os.path.join(system_root, "run")) + namespace_path = os.path.join(root, "pid-namespace") + with open(namespace_path, "wb") as stream: + stream.write(b"namespace") + write_process( + proc_root, + namespace_path, + 1, + 111, + 0, + 0, + b"/opt/openshell/bin/openshell-sandbox\0--managed\0", + ) + + os.environ["NEMOCLAW_MANAGED_CONTROL_ALLOW_NONROOT_TEST"] = "1" + os.environ["NEMOCLAW_MANAGED_CONTROL_PROC_ROOT"] = proc_root + os.environ["NEMOCLAW_MANAGED_CONTROL_SYSTEM_ROOT"] = system_root + control._detect_agent = lambda: "openclaw" + control._validate_trusted_regular = lambda _path: None + control._sandbox_uid = lambda: 1000 + + read_fd, write_fd = os.pipe() + spawned_pidfd = [-1] + signals = [] + clock = [0.0] + + def spawn_supervisor(_environment): + write_process( + proc_root, + namespace_path, + 42, + 222, + supervisor_parent_pid, + 1000, + b"bash\0/usr/local/bin/nemoclaw-start\0", + ) + spawned_pidfd[0] = os.dup(read_fd) + return 42, spawned_pidfd[0] + + def send_pidfd(pidfd, signum): + os.fstat(pidfd) + signals.append({ + "signum": int(signum), + "used_spawned_pidfd": pidfd == spawned_pidfd[0], + }) + return True + + control._spawn_supervisor_as_orphan = spawn_supervisor + control._send_pidfd = send_pidfd + control.SUPERVISOR_LAUNCH_PROOF_SECONDS = 0.4 + control.time.monotonic = lambda: clock[0] + control.time.sleep = lambda seconds: clock.__setitem__(0, clock[0] + seconds) + + result = None + error = None + try: + result = control._launch_managed_supervisor({}) + except control.ControlError as caught: + error = caught.code + + try: + os.fstat(spawned_pidfd[0]) + pidfd_closed = False + except OSError: + pidfd_closed = True + os.close(read_fd) + os.close(write_fd) + return { + "result": result, + "error": error, + "signals": signals, + "pidfd_closed": pidfd_closed, + } + +print(json.dumps({ + "adopted": run_case(1), + "not_adopted": run_case(2), +}, sort_keys=True)) +`; + +function runSupervisorAdoptionHarness() { + const result = spawnSync("python3", ["-c", SUPERVISOR_ADOPTION_HARNESS, HELPER], { + encoding: "utf-8", + timeout: 5000, + }); + + expect(result.status, result.stderr).toBe(0); + return JSON.parse(result.stdout); +} + describe("managed supervisor launch", () => { it("allowlists launch inputs and strips loader hooks before sandbox UID launch", () => { const result = spawnSync("python3", ["-c", SUPERVISOR_LAUNCH_ENV_HARNESS, HELPER], { @@ -105,4 +224,26 @@ describe("managed supervisor launch", () => { expect(result.stdout).toBe(""); expect(result.stderr.trim()).toBe(marker); }); + + it("accepts the spawned supervisor after stable OpenShell PID 1 adopts it", () => { + const observed = runSupervisorAdoptionHarness(); + + expect(observed.adopted).toEqual({ + result: 42, + error: null, + signals: [], + pidfd_closed: true, + }); + }); + + it("terminates the pidfd-pinned child when OpenShell does not adopt it", () => { + const observed = runSupervisorAdoptionHarness(); + + expect(observed.not_adopted).toEqual({ + result: null, + error: "SUPERVISOR_UNAVAILABLE", + signals: [{ signum: 9, used_spawned_pidfd: true }], + pidfd_closed: true, + }); + }); }); From 49cf4c135c5c0e6c248590069f84bf6b0d12f2f8 Mon Sep 17 00:00:00 2001 From: San Dang Date: Thu, 30 Jul 2026 13:24:53 +0530 Subject: [PATCH 12/30] fix(recovery): stabilize restored dashboard forward Signed-off-by: San Dang --- src/lib/actions/sandbox/forward-recovery.ts | 70 ++++++++++++++----- test/process-recovery-forward-failure.test.ts | 48 +++++++++++++ 2 files changed, 102 insertions(+), 16 deletions(-) diff --git a/src/lib/actions/sandbox/forward-recovery.ts b/src/lib/actions/sandbox/forward-recovery.ts index bcf4adcc366..69857d455e5 100644 --- a/src/lib/actions/sandbox/forward-recovery.ts +++ b/src/lib/actions/sandbox/forward-recovery.ts @@ -11,7 +11,7 @@ import { } from "../../adapters/openshell/timeouts"; import * as agentRuntime from "../../agent/runtime"; import { DASHBOARD_PORT } from "../../core/ports"; -import { waitUntil } from "../../core/wait"; +import { sleepMs, waitUntil } from "../../core/wait"; import { getActiveMessagingHostForward } from "../../messaging/host-forward"; import { hydrateDerivedSandboxMessagingPlanFields } from "../../messaging/hydration"; import type { SandboxMessagingHostForwardPlan } from "../../messaging/manifest"; @@ -47,6 +47,14 @@ type SandboxForwardRecoveryOptions = { isWsl?: boolean; }; +type SandboxPortForwardRecoveryOptions = { + afterSuccess?: () => boolean; + forwardTarget?: string; + forceRestart?: boolean; + expectedBind?: string; + beforeStart?: () => boolean; +}; + type DashboardForwardStopRunner = ( args: string[], options: { ignoreError: true; stdio: "ignore"; timeout: number }, @@ -237,16 +245,11 @@ export function isSandboxPortForwardHealthy( ); } -export function ensureSandboxPortForwardForPort( +function ensureSandboxPortForwardForPortWithRetries( sandboxName: string, port: number, - options: { - afterSuccess?: () => boolean; - forwardTarget?: string; - forceRestart?: boolean; - expectedBind?: string; - beforeStart?: () => boolean; - } = {}, + options: SandboxPortForwardRecoveryOptions, + guardedDropRetries: number, ): boolean { const { afterSuccess = () => true, @@ -255,25 +258,47 @@ export function ensureSandboxPortForwardForPort( expectedBind, beforeStart = () => true, } = options; + const configuredWaitMs = Number(process.env.NEMOCLAW_FORWARD_RECOVERY_WAIT_MS ?? "3000"); + const waitMs = Number.isFinite(configuredWaitMs) ? Math.max(0, configuredWaitMs) : 3000; + const guardedSettleMs = Math.min(waitMs, 500); const acceptSuccessfulForward = () => { - let accepted = false; + let guardAccepted = false; try { - accepted = afterSuccess(); + guardAccepted = afterSuccess(); } catch { - accepted = false; + guardAccepted = false; + } + if (!guardAccepted) { + runOpenshell(["forward", "stop", String(port), sandboxName], { + ignoreError: true, + stdio: "ignore", + }); + return false; + } + let forwardAccepted = true; + if (options.afterSuccess) { + // A newly registered relay can report running and then drop while the + // pinned managed-health guard executes. Re-prove the authoritative owner + // and listener after that guard before status reports recovery success. + sleepMs(guardedSettleMs); + forwardAccepted = isSandboxPortForwardHealthy(sandboxName, port, expectedBind) === true; } - if (accepted) return true; + if (forwardAccepted) return true; runOpenshell(["forward", "stop", String(port), sandboxName], { ignoreError: true, stdio: "ignore", }); - return false; + if (!options.afterSuccess || guardedDropRetries <= 0) return false; + return ensureSandboxPortForwardForPortWithRetries( + sandboxName, + port, + options, + guardedDropRetries - 1, + ); }; let forwardHealth = isSandboxPortForwardHealthy(sandboxName, port, expectedBind); if (forwardHealth === true && !forceRestart) return acceptSuccessfulForward(); if (forwardHealth === "occupied") return false; - const configuredWaitMs = Number(process.env.NEMOCLAW_FORWARD_RECOVERY_WAIT_MS ?? "3000"); - const waitMs = Number.isFinite(configuredWaitMs) ? Math.max(0, configuredWaitMs) : 3000; const stopResult = runOpenshell(["forward", "stop", String(port), sandboxName], { ignoreError: true, @@ -379,6 +404,19 @@ export function ensureSandboxPortForwardForPort( return settled && !occupied && acceptSuccessfulForward(); } +export function ensureSandboxPortForwardForPort( + sandboxName: string, + port: number, + options: SandboxPortForwardRecoveryOptions = {}, +): boolean { + return ensureSandboxPortForwardForPortWithRetries( + sandboxName, + port, + options, + options.afterSuccess ? 1 : 0, + ); +} + export function ensureHermesDashboardPortForwardIfEnabled(sandboxName: string): boolean | null { return ensureHermesDashboardPortForward(sandboxName, { isPortForwardHealthy: isSandboxPortForwardHealthy, diff --git a/test/process-recovery-forward-failure.test.ts b/test/process-recovery-forward-failure.test.ts index 1ea8f506d6e..2c68fab3e8f 100644 --- a/test/process-recovery-forward-failure.test.ts +++ b/test/process-recovery-forward-failure.test.ts @@ -232,6 +232,54 @@ beta 127.0.0.1 18789 12345 dead`, }); describe("ensureSandboxPortForwardForPort already-forwarded idempotency (#7085)", () => { + it("retries when a newly started forward drops during the managed health guard", () => { + vi.stubEnv("NEMOCLAW_FORWARD_RECOVERY_WAIT_MS", "0"); + let forwardLive = false; + let startCount = 0; + let guardCount = 0; + + vi.spyOn(forwardHealth, "isLocalForwardReachable").mockImplementation(() => forwardLive); + vi.spyOn(openshellRuntime, "captureOpenshell").mockImplementation(() => ({ + status: 0, + output: forwardLive + ? `SANDBOX BIND PORT PID STATUS +beta 127.0.0.1 18791 12345 running` + : "SANDBOX BIND PORT PID STATUS", + })); + const runOpenshell = vi + .spyOn(openshellRuntime, "runOpenshell") + .mockImplementation((rawArgs: unknown) => { + const args = Array.isArray(rawArgs) ? rawArgs.map(String) : []; + if (args[0] === "forward" && args[1] === "stop") { + forwardLive = false; + } + if (args[0] === "forward" && args[1] === "start") { + startCount += 1; + forwardLive = true; + } + return { status: 0 } as never; + }); + + expect( + withFakeOpenshellBinary(() => + ensureSandboxPortForwardForPort("beta", 18791, { + afterSuccess: () => { + guardCount += 1; + if (guardCount === 1) forwardLive = false; + return true; + }, + expectedBind: "127.0.0.1", + }), + ), + ).toBe(true); + expect(startCount).toBe(2); + expect(guardCount).toBe(2); + expect(runOpenshell).toHaveBeenCalledWith( + ["forward", "start", "--background", "18791", "beta"], + expect.objectContaining({ ignoreError: true }), + ); + }); + it("reconciles a reachable ownerless listener with a nonzero recovery wait", () => { vi.stubEnv("NEMOCLAW_FORWARD_RECOVERY_WAIT_MS", "25"); let started = false; From 5b295a573dc0575d0d60208341017ed962dc83b9 Mon Sep 17 00:00:00 2001 From: San Dang Date: Thu, 30 Jul 2026 14:28:56 +0530 Subject: [PATCH 13/30] test(recovery): keep forward retry fixture linear Signed-off-by: San Dang --- test/process-recovery-forward-failure.test.ts | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/test/process-recovery-forward-failure.test.ts b/test/process-recovery-forward-failure.test.ts index 2c68fab3e8f..37b6476474b 100644 --- a/test/process-recovery-forward-failure.test.ts +++ b/test/process-recovery-forward-failure.test.ts @@ -250,13 +250,10 @@ beta 127.0.0.1 18791 12345 running` .spyOn(openshellRuntime, "runOpenshell") .mockImplementation((rawArgs: unknown) => { const args = Array.isArray(rawArgs) ? rawArgs.map(String) : []; - if (args[0] === "forward" && args[1] === "stop") { - forwardLive = false; - } - if (args[0] === "forward" && args[1] === "start") { - startCount += 1; - forwardLive = true; - } + const forwardAction = args[0] === "forward" ? args[1] : ""; + const startsForward = Number(forwardAction === "start"); + startCount += startsForward; + forwardLive = startsForward > 0 || (forwardLive && forwardAction !== "stop"); return { status: 0 } as never; }); @@ -265,7 +262,7 @@ beta 127.0.0.1 18791 12345 running` ensureSandboxPortForwardForPort("beta", 18791, { afterSuccess: () => { guardCount += 1; - if (guardCount === 1) forwardLive = false; + forwardLive = guardCount !== 1; return true; }, expectedBind: "127.0.0.1", From 062820e4392c4ac47b671b271d1e3557751db080 Mon Sep 17 00:00:00 2001 From: San Dang Date: Thu, 30 Jul 2026 18:14:33 +0530 Subject: [PATCH 14/30] fix(recovery): explain temporary cleanup Signed-off-by: San Dang --- scripts/managed-gateway-control.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/managed-gateway-control.py b/scripts/managed-gateway-control.py index 849d74fa237..5e9a8dc062e 100755 --- a/scripts/managed-gateway-control.py +++ b/scripts/managed-gateway-control.py @@ -1714,6 +1714,7 @@ def _refresh_supervisor_ca_bundle() -> str: try: os.unlink(temporary_name, dir_fd=directory_fd) except OSError: + # Best-effort cleanup must not mask the original control failure. pass os.close(directory_fd) From 035852339c3cd2bc2ada1ccb019344a8a2471e49 Mon Sep 17 00:00:00 2001 From: San Dang Date: Thu, 30 Jul 2026 19:13:10 +0530 Subject: [PATCH 15/30] test(e2e): recover sandbox before survival checks Signed-off-by: San Dang --- test/e2e/live/gateway-guard-recovery.test.ts | 8 ++++---- test/e2e/live/sandbox-survival.test.ts | 10 ++++++++-- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/test/e2e/live/gateway-guard-recovery.test.ts b/test/e2e/live/gateway-guard-recovery.test.ts index ab658d9c662..ee2a4d65555 100644 --- a/test/e2e/live/gateway-guard-recovery.test.ts +++ b/test/e2e/live/gateway-guard-recovery.test.ts @@ -287,8 +287,8 @@ test("gateway recovery restores /tmp guard chain after pod-recreate wipe (#2701) // Fresh non-GPU OpenClaw containers on this OpenShell floor still carry the // legacy keepalive. Restarting the container therefore kills the initial // OpenShell workload session and deterministically leaves no managed - // supervisor. Recovery must upgrade that container through the host-side - // transaction and commit only after managed control accepts the new tree. + // supervisor. Recovery must launch the supervisor in the registered + // container and prove managed health without replacing that container. const originalContainerId = await findSandboxContainer(host, "legacy-restart-container-before"); expect( await inspectStartupCommand(host, originalContainerId, "legacy-restart-command-before"), @@ -321,13 +321,13 @@ test("gateway recovery restores /tmp guard chain after pod-recreate wipe (#2701) expect(resultText(trustedRecovery)).toContain("Probe complete: recovered OpenClaw gateway"); const recoveredContainerId = await findSandboxContainer(host, "legacy-restart-container-after"); - expect(recoveredContainerId).not.toBe(originalContainerId); + expect(recoveredContainerId).toBe(originalContainerId); const recoveredStartupCommand = await inspectStartupCommand( host, recoveredContainerId, "legacy-restart-command-after", ); - expect(recoveredStartupCommand).toMatch(/(?:^| )nemoclaw-start$/); + expect(recoveredStartupCommand).toBe("sleep infinity"); expect(recoveredStartupCommand).not.toContain("CUSTOM_PROVIDER_CREDENTIAL"); expect(recoveredStartupCommand).not.toContain(credentialCanary); diff --git a/test/e2e/live/sandbox-survival.test.ts b/test/e2e/live/sandbox-survival.test.ts index 6a26b3db67e..af051d51d4f 100644 --- a/test/e2e/live/sandbox-survival.test.ts +++ b/test/e2e/live/sandbox-survival.test.ts @@ -85,7 +85,7 @@ test( "install and register the OpenClaw sandbox", "prove baseline sandbox access and inference", "write persistent OpenClaw markers", - "restart the gateway and reconnect the sandbox", + "restart the gateway and recover the sandbox", "recheck state and inference after restart", "destroy the sandbox and confirm registry removal", ], @@ -303,7 +303,7 @@ test( await stateValidation.writeSandboxMarkers(instance, markers); await stateValidation.expectSandboxMarkers(instance, markers, "pre-restart-marker-read"); - progress.phase("restart the gateway and reconnect the sandbox"); + progress.phase("restart the gateway and recover the sandbox"); await lifecycle.restartGatewayRuntime({ delayMs: 5_000, sandboxName: SANDBOX_NAME, @@ -312,6 +312,12 @@ test( attempts: 60, intervalMs: 5_000, }); + const recovery = await host.nemoclaw([SANDBOX_NAME, "recover"], { + artifactName: "post-restart-nemoclaw-recover", + env: buildAvailabilityProbeEnv(), + timeoutMs: 240_000, + }); + assertExitZero(recovery, `recover sandbox ${SANDBOX_NAME}`); progress.phase("recheck state and inference after restart"); await sandbox.expectListed(SANDBOX_NAME, { From 0e1f36382d19820f37933762bf2fe73af2ca7c38 Mon Sep 17 00:00:00 2001 From: San Dang Date: Thu, 30 Jul 2026 20:24:09 +0530 Subject: [PATCH 16/30] test(e2e): focus sandbox survival regression --- .../configure-inference-timeouts.mdx | 6 +- .../recover-rebuild-sandboxes.mdx | 10 +- docs/reference/commands.mdx | 15 +- docs/reference/troubleshooting.mdx | 7 +- docs/security/tcb-boundary.mdx | 2 +- scripts/gateway-control.sh | 8 +- scripts/managed-gateway-control.py | 323 ++---------------- src/lib/actions/sandbox/forward-recovery.ts | 70 +--- src/lib/actions/sandbox/process-recovery.ts | 65 +++- .../sandbox/supervisor-relaunch.test.ts | 151 ++++---- .../actions/sandbox/supervisor-relaunch.ts | 90 +++-- test/e2e/live/gateway-guard-recovery.test.ts | 8 +- test/gateway-supervisor-control.test.ts | 6 - test/managed-supervisor-launch.test.ts | 181 ---------- test/process-recovery-forward-failure.test.ts | 45 --- ...ocess-recovery-supervisor-relaunch.test.ts | 68 +++- 16 files changed, 268 insertions(+), 787 deletions(-) delete mode 100644 test/managed-supervisor-launch.test.ts diff --git a/docs/inference/configure-inference-timeouts.mdx b/docs/inference/configure-inference-timeouts.mdx index 26c9fac65f6..643b65efc17 100644 --- a/docs/inference/configure-inference-timeouts.mdx +++ b/docs/inference/configure-inference-timeouts.mdx @@ -20,7 +20,7 @@ Use the error location to select the correct setting. |---|---|---| | `NEMOCLAW_AGENT_TIMEOUT` | OpenClaw per-request inference | `600` seconds | | `NEMOCLAW_LOCAL_INFERENCE_TIMEOUT` | Ollama, vLLM, NIM, and compatible-endpoint validation during onboarding | `180` seconds | -| `NEMOCLAW_SANDBOX_READY_TIMEOUT` | Image build, gateway upload, and in-sandbox boot after creation; OpenShell command re-registration after policy application; OpenShell readiness after OpenClaw or Hermes managed recovery | `180` seconds | +| `NEMOCLAW_SANDBOX_READY_TIMEOUT` | Image build, gateway upload, and in-sandbox boot after creation; OpenShell command re-registration after policy application or after OpenClaw or Hermes managed recovery recreates the sandbox | `180` seconds | The readiness timeout does not govern inference requests or provider validation. @@ -68,7 +68,7 @@ Onboarding also uses this budget to confirm that the sandbox can execute command -The same budget applies when `start` or `recover` launches a missing supervisor in a legacy keepalive sandbox and proves OpenShell readiness. +The same budget applies when `start` or `recover` transactionally recreates a managed sandbox and waits for OpenShell to re-register it. @@ -79,7 +79,7 @@ $$nemoclaw onboard -For an existing sandbox, export the variable before the `start` or `recover` command that performs the launch. +For an existing sandbox, export the variable before the `start` or `recover` command that performs the recreation. diff --git a/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx b/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx index 5ee8f10d645..4d1ec536979 100644 --- a/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx +++ b/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx @@ -80,12 +80,12 @@ Refer to [`$$nemoclaw recover`](../../reference/commands#$$nemoclaw-name- Recovery uses registry-scoped privileged direct-container control and does not fall back to ordinary `openshell sandbox exec` or a manual in-sandbox relaunch. -For a local Docker-driver sandbox whose registered container uses the legacy keepalive startup, `recover` can launch the managed supervisor in that container. +For a local Docker-driver sandbox whose container still uses the legacy keepalive startup, `recover` can transactionally recreate the registered container with a credential-free managed startup command. -The registered container, its writable layer, its mounts, and its persisted startup command remain unchanged. -A later full container restart can require `recover` again. -NemoClaw requires managed gateway health and uses `NEMOCLAW_SANDBOX_READY_TIMEOUT` (180 seconds by default) to prove OpenShell readiness before starting the primary dashboard or API host forward. -A definitive managed-health failure still stops immediately; if readiness cannot be proved within the budget, the forward stays stopped. +NemoClaw keeps the previous container available until the managed controller proves the supervisor topology, gateway health, and settle check, and attempts to restore it if that proof fails. +The recreation preserves mounted sandbox state, but a committed swap does not retain changes stored only in the previous container's writable layer. +After a transactional recreation, NemoClaw uses the `NEMOCLAW_SANDBOX_READY_TIMEOUT` budget (180 seconds by default) for OpenShell to re-register the sandbox before starting the primary dashboard or API host forward. +A definitive managed-health failure still stops immediately; if re-registration does not complete within the budget, the forward stays stopped. For the controller topology, trust boundary, and fail-closed conditions, refer to [Understand Gateway Lifecycle Control](../configure-sandboxes/understand-gateway-lifecycle-control). If recovery cannot repair a sandbox that needs credentials or a current controller contract, rebuild it. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 947a3224860..f4d83aee124 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -1163,10 +1163,9 @@ The host selects the controller from the live container topology. In a direct root-entrypoint container, the request reaches the root PID 1 supervisor. In an OpenShell-managed container, the request enters the root-owned mode `0500` managed controller through a sanitized root exec while OpenShell remains PID 1. It does not use ordinary `openshell sandbox exec` or an in-sandbox manual relaunch as a fallback. -When the root-owned managed controller reports `SUPERVISOR_NOT_RUNNING` after two complete zero-supervisor scans with a stable PID 1, a local Docker-driver sandbox with the legacy keepalive startup can enter an in-place supervisor launch. -The registered container, its writable layer, its mounts, and its persisted startup command remain unchanged. -A later full container restart can require `recover` again. -Recovery succeeds only after managed gateway health and the settle check pass. +When the root-owned managed controller attests two unchanged zero-supervisor process scans with a stable PID 1 and reports `SUPERVISOR_NOT_RUNNING`, a local Docker-driver sandbox with the legacy keepalive startup can enter a transactional container recreation. +The recreation uses a credential-free managed startup command, pins the registered container identity, retains the previous container for rollback, and commits only after managed gateway health and the settle check pass. +The recreation preserves mounted sandbox state, but a committed swap does not retain changes stored only in the previous container's writable layer. It is idempotent. When `recover` repairs a stopped built-in OpenClaw or Hermes gateway, it repeats the recovery action only for an exit status of `1` with blank stdout and a sole nonblank stderr line equal to `SUPERVISOR_BUSY`, with at most three controller attempts. The same result is inconclusive during managed settle confirmation and can be probed again only within the configured settle window. @@ -3937,7 +3936,7 @@ The following environment variables tune onboard-time wall-clock limits. `NEMOCLAW_SANDBOX_READY_TIMEOUT` also covers OpenShell command re-registration after onboarding applies policy presets. -`NEMOCLAW_SANDBOX_READY_TIMEOUT` also applies when managed recovery launches a missing supervisor in a legacy keepalive sandbox. +`NEMOCLAW_SANDBOX_READY_TIMEOUT` also applies when managed recovery transactionally recreates an existing sandbox. Set them before running `$$nemoclaw onboard` if a slow connection or large model pull risks tripping the default. @@ -3950,7 +3949,7 @@ Set them before running `$$nemoclaw onboard` if a slow connection or large model -For managed recovery, the same timeout covers the OpenShell readiness check after an in-place supervisor launch. +For managed recovery, the same timeout covers OpenShell re-registration after transactional recreation. When the deadline expires, the primary dashboard or API host forward stays stopped. @@ -3978,9 +3977,7 @@ The following flags change defaults for commands that manage existing sandboxes. | `NEMOCLAW_CONFIRM_LEGACY_MANAGED_RECREATE` | Exact JSON array of sandbox names | Confirms to the installer that the exact listed set of pre-fingerprint OpenClaw or Hermes sandboxes used NemoClaw-managed images, allowing recovery onto the current managed image. The normalized names must exactly match the installer's printed array. Set it only after verifying every named sandbox. Recorded custom-image evidence remains blocked. | | `NEMOCLAW_DISABLE_INFERENCE_ROUTE_REPAIR` | `1` to enable | Skips the automatic DNS-proxy repair for stale `inference.local` routes during `$$nemoclaw connect` and `$$nemoclaw connect --probe-only`. Use only as a troubleshooting escape hatch. | - -| `NEMOCLAW_DISABLE_SUPERVISOR_RELAUNCH` | `1` to enable | Skips the automatic in-place supervisor launch during `$$nemoclaw recover` when two managed scans find no supervisor while PID 1 remains stable. Use only as a troubleshooting escape hatch; recovery then falls back to the rebuild or re-onboard guidance. | - +| `NEMOCLAW_DISABLE_SUPERVISOR_RELAUNCH` | `1` to enable | Skips the automatic trusted container recreation during `$$nemoclaw recover` when two managed scans find no supervisor while PID 1 remains stable. Use only as a troubleshooting escape hatch; recovery then falls back to the rebuild or re-onboard guidance. | | `NEMOCLAW_SHIELDS_ACCEPT_LEGACY_BASELINE` | `1` to opt in | Allows advanced immutable-config verification to trust the current on-disk bytes for older or partial content baselines. Use only after you have rebuilt or manually inspected the sandbox state and accepted that the baseline is operator-approved. | | `NEMOCLAW_SHIELDS_SETTLE_MS` | milliseconds (default `750`, clamped to `0` to `10000`) | Settle window NemoClaw waits after re-applying a config lockdown (during shields auto-restore and `$$nemoclaw shields up` drift remediation) before re-confirming the lock still holds. Detects when an in-sandbox reconciler changes config file permissions after lockdown and re-applies the lock; if NemoClaw cannot re-confirm the lock within the retry budget, shields stay down. This narrows the window in which a reconciler can revert permissions rather than eliminating it. The best-effort `chattr +i` immutable bit remains the only fully durable lock. Raise it on hosts where the gateway settles slowly. | | `NEMOCLAW_SKIP_UNREACHABLE_SANDBOX_BACKUP` | Exactly `1` to opt in (`true`, `yes`, `0` are not accepted) | Applies to standalone `$$nemoclaw backup-all` runs. Skips running sandboxes whose in-sandbox SSH endpoint does not answer. It does not relax the installer's strict pre-upgrade backup, which still aborts if any registered sandbox is skipped or fails. Any uncommitted state since the last successful backup is not included in the skipped backup. | diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index 4d93acf187e..6cad47f36ff 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -1019,11 +1019,10 @@ The same result is inconclusive during managed settle confirmation and can be pr NemoClaw treats `SUPERVISOR_UNAVAILABLE` as terminal because it can report unreadable or untrusted supervisor state, ambiguous discovery, or a process-identity change. It does not retry other status or output combinations. `SUPERVISOR_NOT_RUNNING` is a separate result that requires two zero-supervisor scans with a stable PID 1 and does not enter that retry loop. -On a supported local Docker-driver sandbox with the legacy keepalive startup, it can authorize a container-identity-pinned in-place supervisor launch. -NemoClaw requires managed gateway health, the settle check, and OpenShell readiness before it re-establishes the primary port forward. -To bypass that trusted launch while troubleshooting, run `NEMOCLAW_DISABLE_SUPERVISOR_RELAUNCH=1 $$nemoclaw recover`; NemoClaw leaves the container unchanged and returns rebuild or re-onboard guidance. +On a supported local Docker-driver sandbox with the legacy keepalive startup, it can authorize a container-identity-pinned recreation that commits only after managed health and settle checks pass. +To bypass that trusted recreation while troubleshooting, run `NEMOCLAW_DISABLE_SUPERVISOR_RELAUNCH=1 $$nemoclaw recover`; NemoClaw leaves the container unchanged and returns rebuild or re-onboard guidance. If that bounded retry is exhausted, or if `gateway restart` reports `SUPERVISOR_BUSY`, wait for the active request to finish and retry the command. -If the error mentions `SUPERVISOR_NOT_RUNNING` and the trusted launch could not proceed, `SUPERVISOR_REBUILD_REQUIRED`, a missing `nemoclaw-gateway-control` helper, or a missing managed controller, the sandbox image may predate the current lifecycle contract. +If the error mentions `SUPERVISOR_NOT_RUNNING` and trusted recreation could not proceed, `SUPERVISOR_REBUILD_REQUIRED`, a missing `nemoclaw-gateway-control` helper, or a missing managed controller, the sandbox image may predate the current lifecycle contract. An exact `SUPERVISOR_UNAVAILABLE` result instead means the managed controller refused the current supervisor state rather than guessing which same-UID process is the gateway. The current recovery action and any managed settle confirmation stop immediately. If `recover` reports this result, follow its host-side `gateway restart` guidance. diff --git a/docs/security/tcb-boundary.mdx b/docs/security/tcb-boundary.mdx index d6636754da1..a725149fdb2 100644 --- a/docs/security/tcb-boundary.mdx +++ b/docs/security/tcb-boundary.mdx @@ -44,7 +44,7 @@ A successful build does not replace review of privilege, process identity, descr | `scripts/state-dir-guard.py` | The installed copy is root-owned and mode `0500`; the host reaches it through the shields transaction. | Fixed paths, a bounded action contract, and a lock token from the host coordinator. | Applies descriptor-rooted state-directory posture changes, rejects link and mount substitution, bounds traversal, and verifies the committed modes and ownership. | | `scripts/lib/normalize_mutable_config_perms.py` | The installed copy is root-owned and mode `0555`; startup invokes it under the entrypoint identity, and only root can reclaim a root-owned tree. | The fixed OpenClaw config path, the resolved sandbox identity, and an exact `root:root 0700/0600` mutable-drift signature under the expected sandbox-owned parent. | Restores the mutable `2770/660` contract, pins every privileged handoff by descriptor, and rejects ambiguous posture, links, mount substitution, metadata races, and sealed config. | | `scripts/openclaw-config-guard.py` | The installed copy is root-owned and mode `0500`; direct root PID 1 or the authenticated host transaction invokes it. | Bounded strict JSON for writes, stable captured config bytes for restart validation, and fixed installed parser paths for existing JSON5 config. | Seals and unseals OpenClaw config with no-follow descriptors, stable inode checks, atomic replacement, hash coherence, and recoverable transaction journals. | -| `scripts/managed-gateway-control.py` | The installed copy is root-owned and mode `0500`; the host invokes it through sanitized registry-scoped direct-container execution. | A fixed action, a 64-character nonce, fixed installed helpers, allowlisted credential-free runtime overrides, trusted-owner CA bundles, and a live OpenShell process tree observed through `/proc`. | Authenticates the host action, proves the managed supervisor and gateway identity, holds a root-owned mode `0600` lifecycle lock, publishes one root-owned mode `0444` exact-exit authorization bound to the gateway and live root controller identities, signals through a pidfd, waits for the normal respawn loop, and verifies listener and HTTP health. For legacy keepalive recovery, it confirms supervisor absence under the lifecycle lock, refreshes CA trust, and starts the fixed entrypoint under the sandbox identity. | +| `scripts/managed-gateway-control.py` | The installed copy is root-owned and mode `0500`; the host invokes it through sanitized registry-scoped direct-container execution. | A fixed action, a 64-character nonce, fixed installed helpers, and a live OpenShell process tree observed through `/proc`. | Authenticates the host action, proves the managed supervisor and gateway identity, holds a root-owned mode `0600` lifecycle lock, publishes one root-owned mode `0444` exact-exit authorization bound to the gateway and live root controller identities, signals through a pidfd, waits for the normal respawn loop, and verifies listener and HTTP health. | | `src/lib/shields/transition-lock.ts` | Runs in the host CLI under the operator account and owns the canonical per-sandbox transition lock. | Host state directory entries whose owner PID and start identity match the live lock owner, or prove that the recorded owner is definitively dead or PID-reused. | Serializes shields mutations, recovers definitively stale owners through inode-checked quarantine, rejects ambiguous owners, and allows token-gated takeover only through the explicit recovery contract. | | `src/lib/shields/timer-bound-lock.ts` | Runs in the host CLI and composes the transition lock with the recorded auto-restore generation. | A validated timer marker and transition owner from the host state directory. | Prevents an expired or replaced timer from authorizing a later mutation and keeps restore authority bound to one generation. | | `src/lib/shields/verify-lock.ts` | Runs in the host CLI and delegates sandbox inspection through the privileged execution adapter. | Resolved built-in agent paths and the expected locked posture recorded by the host. | Verifies modes, ownership, immutable flags, layout, and recorded content hashes before NemoClaw reports shields as locked. | diff --git a/scripts/gateway-control.sh b/scripts/gateway-control.sh index e8e2ee79480..bcfca804bfb 100755 --- a/scripts/gateway-control.sh +++ b/scripts/gateway-control.sh @@ -36,12 +36,11 @@ fail() { exit 1 } -[ "$#" -ge 2 ] || fail "SUPERVISOR_INVALID_REQUEST" +[ "$#" -eq 2 ] || fail "SUPERVISOR_INVALID_REQUEST" ACTION="$1" NONCE="$2" case "$ACTION" in - restart | recover | probe) [ "$#" -eq 2 ] || fail "SUPERVISOR_INVALID_REQUEST" ;; - launch-supervisor) [ "$#" -le 34 ] || fail "SUPERVISOR_INVALID_REQUEST" ;; + restart | recover | probe) ;; *) fail "SUPERVISOR_INVALID_ACTION" ;; esac case "$NONCE" in @@ -56,9 +55,8 @@ if [ "$PID1_ARGV0" = "/opt/openshell/bin/openshell-sandbox" ]; then [ -x "$CONTROL_MANAGED_HELPER" ] || fail "SUPERVISOR_REBUILD_REQUIRED" # Isolated mode ignores Python startup hooks, user-site packages, and # PYTHON* environment variables before the root helper imports anything. - exec python3 -I "$CONTROL_MANAGED_HELPER" "$@" + exec python3 -I "$CONTROL_MANAGED_HELPER" "$ACTION" "$NONCE" fi -[ "$ACTION" != "launch-supervisor" ] || fail "SUPERVISOR_INVALID_ACTION" case "$PID1_CMDLINE" in *nemoclaw-start*) ;; *) fail "SUPERVISOR_UNAVAILABLE" ;; diff --git a/scripts/managed-gateway-control.py b/scripts/managed-gateway-control.py index 5e9a8dc062e..7c2c1e33947 100755 --- a/scripts/managed-gateway-control.py +++ b/scripts/managed-gateway-control.py @@ -6,21 +6,14 @@ OpenShell is container PID 1 in its managed topology and starts ``nemoclaw-start`` as the unprivileged ``sandbox`` user. The shell entrypoint -still owns and reaps the gateway child, so ordinary control never launches a -second gateway and never trusts a status file writable by that user. Instead -it: +still owns and reaps the gateway child, so this helper never launches a second +gateway and never trusts a status file writable by that user. Instead it: * verifies a stable OpenShell -> nemoclaw-start -> gateway process tree; * signals the already-proven gateway through a pidfd; * waits for the entrypoint's normal respawn loop; and * independently proves the replacement process, listener, and HTTP health. -One legacy recovery action is available only when two complete process-table -scans prove that the supervisor is absent. It holds the root lifecycle lock, -refreshes the OpenShell trust bundle, and launches the fixed entrypoint under -the sandbox UID. The caller then requires the normal identity-pinned managed -health probe before host recovery succeeds. - The host enters this helper through registry-scoped ``docker exec --user root``. The installed copy is root-owned and mode 0500, which is the host request authentication boundary. No same-UID request or completion channel exists. @@ -58,7 +51,6 @@ import os import pwd import re -import secrets import select import signal import stat @@ -97,47 +89,6 @@ NONCE_RE = re.compile(r"[0-9a-f]{64}\Z") ENV_KEY_RE = re.compile(rb"[A-Za-z_][A-Za-z0-9_]*\Z") SHA256_RE = re.compile(r"[0-9a-f]{64}\Z") -SUPERVISOR_LAUNCH_ACTION = "launch-supervisor" -SUPERVISOR_LAUNCH_ENV_KEYS = frozenset( - { - "CHAT_UI_URL", - "HTTP_PROXY", - "HTTPS_PROXY", - "NEMOCLAW_AUTO_PAIR_DEADLINE_SECS", - "NEMOCLAW_AUTO_PAIR_FAST_DEADLINE_SECS", - "NEMOCLAW_AUTO_PAIR_RUN_TIMEOUT_SECS", - "NEMOCLAW_AUTO_PAIR_SLOW_INTERVAL_SECS", - "NEMOCLAW_DASHBOARD_BIND", - "NEMOCLAW_DASHBOARD_PORT", - "NEMOCLAW_HERMES_DASHBOARD", - "NEMOCLAW_HERMES_DASHBOARD_INTERNAL_PORT", - "NEMOCLAW_HERMES_DASHBOARD_PORT", - "NEMOCLAW_HERMES_DASHBOARD_TUI", - "NEMOCLAW_MINIMAL_BOOTSTRAP", - "NEMOCLAW_PROXY_HOST", - "NEMOCLAW_PROXY_PORT", - "NO_PROXY", - "OPENCLAW_HOME", - "OPENCLAW_STATE_DIR", - "OPENCLAW_WORKSPACE_DIR", - "http_proxy", - "https_proxy", - "no_proxy", - } -) -MAX_SUPERVISOR_LAUNCH_ENV_BYTES = 64 * 1024 -TRUSTED_RUNTIME_PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" -OPENSHELL_CA_BUNDLE_PATH = "/etc/openshell-tls/ca-bundle.pem" -CORPORATE_CA_BUNDLE_PATH = "/usr/local/share/nemoclaw/corporate-ca.pem" -MERGED_CA_BUNDLE_PATH = "/tmp/nemoclaw-ca-bundle.pem" -MAX_CA_BUNDLE_BYTES = 16 * 1024 * 1024 -CA_ENV_KEYS = ( - "SSL_CERT_FILE", - "CURL_CA_BUNDLE", - "REQUESTS_CA_BUNDLE", - "GIT_SSL_CAINFO", - "NODE_EXTRA_CA_CERTS", -) HERMES_MCP_STATE_RE = re.compile( r"# nemoclaw-hermes-mcp-state-v1 " r"intended=[0-9a-f]{64} applied=[0-9a-f]{64}\Z" @@ -993,43 +944,6 @@ def _supervisor_candidates( return matches, inconclusive -def _confirm_supervisor_absence( - reader: ProcReader, pid1: ProcessIdentity, sandbox_uid: int -) -> bool: - """Confirm a second zero-match scan while the exact OpenShell PID 1 remains stable.""" - - between_pid1 = reader.capture(1) - second_matches, second_inconclusive = _supervisor_candidates( - reader, pid1, sandbox_uid - ) - after_pid1 = reader.capture(1) - return bool( - between_pid1.stable_key() == pid1.stable_key() - and after_pid1.stable_key() == pid1.stable_key() - and not second_inconclusive - and len(second_matches) == 0 - ) - - -def _prove_supervisor_absent( - reader: ProcReader, -) -> tuple[ProcessIdentity, int]: - """Return the exact OpenShell identity only for stable supervisor absence.""" - - pid1 = reader.capture(1) - if not _is_openshell(pid1): - raise ControlError("SUPERVISOR_UNAVAILABLE") - sandbox_uid = _sandbox_uid() - matches, inconclusive = _supervisor_candidates(reader, pid1, sandbox_uid) - if ( - inconclusive - or len(matches) != 0 - or not _confirm_supervisor_absence(reader, pid1, sandbox_uid) - ): - raise ControlError("SUPERVISOR_UNAVAILABLE") - return pid1, sandbox_uid - - def _discover_supervisor(reader: ProcReader) -> ProcessIdentity: pid1 = reader.capture(1) if not _is_openshell(pid1): @@ -1056,12 +970,22 @@ def _discover_supervisor(reader: ProcReader) -> ProcessIdentity: raise ControlError("SUPERVISOR_UNAVAILABLE") if len(matches) == 0: # A zero-match scan is the only absence signal that may authorize the - # host to launch the managed supervisor in a legacy keepalive - # container. Re-scan the complete process table and pin PID 1 around - # both observations so ambiguity, process churn, and supervisor - # startup races remain generic unavailability rather than launch + # host to recreate a legacy Docker container with its managed startup + # command. Re-scan the complete process table and pin PID 1 around both + # observations so ambiguity, process churn, and supervisor startup + # races remain generic unavailability rather than destructive-recovery # authorization. - if _confirm_supervisor_absence(reader, pid1, sandbox_uid): + between_pid1 = reader.capture(1) + second_matches, second_inconclusive = _supervisor_candidates( + reader, pid1, sandbox_uid + ) + after_pid1 = reader.capture(1) + if ( + between_pid1.stable_key() == pid1.stable_key() + and after_pid1.stable_key() == pid1.stable_key() + and not second_inconclusive + and len(second_matches) == 0 + ): raise ControlError("SUPERVISOR_NOT_RUNNING") raise ControlError("SUPERVISOR_UNAVAILABLE") if len(matches) != 1: @@ -1589,181 +1513,6 @@ def _terminate_gateway(reader: ProcReader, identity: ProcessIdentity) -> None: os.close(pidfd) -def _supervisor_launch_environment( - runtime_environment: dict[str, str], ca_bundle: str -) -> dict[str, str]: - """Build the cold-start-compatible environment without loader hooks.""" - - environment = dict(runtime_environment) - environment.update( - { - "HOME": "/sandbox", - "LOGNAME": "sandbox", - "PATH": TRUSTED_RUNTIME_PATH, - "PYTHONNOUSERSITE": "1", - "SHELL": "/bin/bash", - "USER": "sandbox", - } - ) - environment.update({key: ca_bundle for key in CA_ENV_KEYS}) - return environment - - -def _read_trusted_ca_bundle(path: str, *, required: bool) -> bytes | None: - """Read one trusted-owner CA bundle and reject group or world writes.""" - - mapped = _system_path(path) - try: - bundle, metadata = _read_regular(mapped, MAX_CA_BUNDLE_BYTES) - except FileNotFoundError: - if required: - raise ControlError("SUPERVISOR_REBUILD_REQUIRED") - return None - except (ControlError, OSError) as exc: - raise ControlError("SUPERVISOR_REBUILD_REQUIRED") from exc - - trusted_uid, _trusted_gid = _trusted_runtime_owner() - if ( - metadata.st_uid != trusted_uid - or stat.S_IMODE(metadata.st_mode) & 0o022 - or metadata.st_size <= 0 - or metadata.st_size != len(bundle) - ): - raise ControlError("SUPERVISOR_REBUILD_REQUIRED") - return bundle - - -def _refresh_supervisor_ca_bundle() -> str: - """Refresh the root-owned merged CA before the sandbox UID relaunch.""" - - openshell_bundle = _read_trusted_ca_bundle( - OPENSHELL_CA_BUNDLE_PATH, required=True - ) - if openshell_bundle is None: - raise ControlError("SUPERVISOR_REBUILD_REQUIRED") - corporate_bundle = _read_trusted_ca_bundle( - CORPORATE_CA_BUNDLE_PATH, required=False - ) - if corporate_bundle is None: - return OPENSHELL_CA_BUNDLE_PATH - - payload = openshell_bundle.rstrip(b"\n") + b"\n" + corporate_bundle - if len(payload) > MAX_CA_BUNDLE_BYTES: - raise ControlError("SUPERVISOR_REBUILD_REQUIRED") - mapped_destination = _system_path(MERGED_CA_BUNDLE_PATH) - directory = os.path.dirname(mapped_destination) - destination_name = os.path.basename(mapped_destination) - directory_fd = _open_directory(directory) - temporary_name = f".{destination_name}.{secrets.token_hex(16)}" - temporary_fd = -1 - installed = False - try: - directory_metadata = os.fstat(directory_fd) - trusted_uid, trusted_gid = _trusted_runtime_owner() - directory_mode = stat.S_IMODE(directory_metadata.st_mode) - if ( - not stat.S_ISDIR(directory_metadata.st_mode) - or directory_metadata.st_uid != trusted_uid - or directory_metadata.st_gid != trusted_gid - or (directory_mode & 0o002 and not directory_mode & stat.S_ISVTX) - ): - raise ControlError("SUPERVISOR_REBUILD_REQUIRED") - flags = ( - os.O_WRONLY - | os.O_CREAT - | os.O_EXCL - | getattr(os, "O_NOFOLLOW", 0) - | getattr(os, "O_CLOEXEC", 0) - ) - temporary_fd = os.open(temporary_name, flags, 0o600, dir_fd=directory_fd) - offset = 0 - while offset < len(payload): - written = os.write(temporary_fd, payload[offset:]) - if written <= 0: - raise ControlError("SUPERVISOR_REBUILD_REQUIRED") - offset += written - os.fsync(temporary_fd) - os.fchmod(temporary_fd, 0o444) - installed_metadata = os.fstat(temporary_fd) - if ( - installed_metadata.st_uid != trusted_uid - or installed_metadata.st_gid != trusted_gid - or installed_metadata.st_nlink != 1 - or stat.S_IMODE(installed_metadata.st_mode) != 0o444 - or installed_metadata.st_size != len(payload) - ): - raise ControlError("SUPERVISOR_REBUILD_REQUIRED") - os.close(temporary_fd) - temporary_fd = -1 - os.replace( - temporary_name, - destination_name, - src_dir_fd=directory_fd, - dst_dir_fd=directory_fd, - ) - installed = True - return MERGED_CA_BUNDLE_PATH - except ControlError: - raise - except OSError as exc: - raise ControlError("SUPERVISOR_REBUILD_REQUIRED") from exc - finally: - if temporary_fd >= 0: - os.close(temporary_fd) - if not installed: - try: - os.unlink(temporary_name, dir_fd=directory_fd) - except OSError: - # Best-effort cleanup must not mask the original control failure. - pass - os.close(directory_fd) - - -def _spawn_managed_supervisor(environment: dict[str, str]) -> int: - """Start the fixed entrypoint as the sandbox user.""" - - try: - account = pwd.getpwnam("sandbox") - groups = os.getgrouplist(account.pw_name, account.pw_gid) - supervisor = subprocess.Popen( - [NEMOCLAW_START_PATH.decode("ascii")], - close_fds=True, - cwd="/sandbox", - env=environment, - extra_groups=groups, - group=account.pw_gid, - start_new_session=True, - stdin=subprocess.DEVNULL, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - umask=0o077, - user=account.pw_uid, - ) - except (KeyError, OSError, subprocess.SubprocessError) as exc: - raise ControlError("SUPERVISOR_UNAVAILABLE") from exc - return supervisor.pid - - -def _launch_managed_supervisor(runtime_environment: dict[str, str]) -> int: - """Serialize, re-prove absence, refresh trust, and launch one supervisor.""" - - _detect_agent() - _validate_trusted_regular(NEMOCLAW_START_PATH.decode("ascii")) - directory_fd = _open_managed_runtime_directory() - lock_fd = -1 - try: - lock_fd = _open_expected_exit_lock(directory_fd) - with ProcReader() as reader: - _prove_supervisor_absent(reader) - ca_bundle = _refresh_supervisor_ca_bundle() - environment = _supervisor_launch_environment(runtime_environment, ca_bundle) - return _spawn_managed_supervisor(environment) - finally: - if lock_fd >= 0: - os.close(lock_fd) - os.close(directory_fd) - - def _wait_for_healthy_gateway( reader: ProcReader, supervisor: ProcessIdentity, @@ -2061,48 +1810,22 @@ def _managed_failure_diagnostics() -> tuple[str, ...]: return tuple(diagnostics) -def _parse_supervisor_launch_environment(argv: list[str]) -> dict[str, str]: - environment: dict[str, str] = {} - total_bytes = 0 - for assignment in argv: - key, separator, value = assignment.partition("=") - total_bytes += len(assignment.encode("utf-8", errors="surrogateescape")) - if ( - not separator - or key not in SUPERVISOR_LAUNCH_ENV_KEYS - or key in environment - or total_bytes > MAX_SUPERVISOR_LAUNCH_ENV_BYTES - ): - raise ControlError("SUPERVISOR_INVALID_REQUEST") - environment[key] = value - return environment - - -def _validate_request(argv: list[str]) -> tuple[str, str, dict[str, str]]: - if len(argv) < 2: +def _validate_request(argv: list[str]) -> tuple[str, str]: + if len(argv) != 2: raise ControlError("SUPERVISOR_INVALID_REQUEST") - action, nonce, *arguments = argv - if action not in ("restart", "recover", "probe", SUPERVISOR_LAUNCH_ACTION): + action, nonce = argv + if action not in ("restart", "recover", "probe"): raise ControlError("SUPERVISOR_INVALID_ACTION") if not NONCE_RE.fullmatch(nonce): raise ControlError("SUPERVISOR_INVALID_NONCE") - if action == SUPERVISOR_LAUNCH_ACTION: - return action, nonce, _parse_supervisor_launch_environment(arguments) - if arguments: - raise ControlError("SUPERVISOR_INVALID_REQUEST") - return action, nonce, {} + return action, nonce def main(argv: list[str]) -> int: try: - action, nonce, runtime_environment = _validate_request(argv) + action, nonce = _validate_request(argv) _require_root() _require_installed_helper_trust() - if action == SUPERVISOR_LAUNCH_ACTION: - supervisor_pid = _launch_managed_supervisor(runtime_environment) - print(f"v1 {nonce} complete launched 0 {supervisor_pid}") - print(f"SUPERVISOR_PID={supervisor_pid}") - return 0 result, old_pid, new_pid = _control(action, nonce) print(f"v1 {nonce} complete {result} {old_pid} {new_pid}") print(f"GATEWAY_PID={new_pid}") diff --git a/src/lib/actions/sandbox/forward-recovery.ts b/src/lib/actions/sandbox/forward-recovery.ts index 23a4cfe4493..bcf4adcc366 100644 --- a/src/lib/actions/sandbox/forward-recovery.ts +++ b/src/lib/actions/sandbox/forward-recovery.ts @@ -11,7 +11,7 @@ import { } from "../../adapters/openshell/timeouts"; import * as agentRuntime from "../../agent/runtime"; import { DASHBOARD_PORT } from "../../core/ports"; -import { sleepMs, waitUntil } from "../../core/wait"; +import { waitUntil } from "../../core/wait"; import { getActiveMessagingHostForward } from "../../messaging/host-forward"; import { hydrateDerivedSandboxMessagingPlanFields } from "../../messaging/hydration"; import type { SandboxMessagingHostForwardPlan } from "../../messaging/manifest"; @@ -47,14 +47,6 @@ type SandboxForwardRecoveryOptions = { isWsl?: boolean; }; -type SandboxPortForwardRecoveryOptions = { - afterSuccess?: () => boolean; - forwardTarget?: string; - forceRestart?: boolean; - expectedBind?: string; - beforeStart?: () => boolean; -}; - type DashboardForwardStopRunner = ( args: string[], options: { ignoreError: true; stdio: "ignore"; timeout: number }, @@ -245,11 +237,16 @@ export function isSandboxPortForwardHealthy( ); } -function ensureSandboxPortForwardForPortWithRetries( +export function ensureSandboxPortForwardForPort( sandboxName: string, port: number, - options: SandboxPortForwardRecoveryOptions, - guardedDropRetries: number, + options: { + afterSuccess?: () => boolean; + forwardTarget?: string; + forceRestart?: boolean; + expectedBind?: string; + beforeStart?: () => boolean; + } = {}, ): boolean { const { afterSuccess = () => true, @@ -258,47 +255,25 @@ function ensureSandboxPortForwardForPortWithRetries( expectedBind, beforeStart = () => true, } = options; - const configuredWaitMs = Number(process.env.NEMOCLAW_FORWARD_RECOVERY_WAIT_MS ?? "3000"); - const waitMs = Number.isFinite(configuredWaitMs) ? Math.max(0, configuredWaitMs) : 3000; - const guardedSettleMs = Math.min(waitMs, 500); const acceptSuccessfulForward = () => { - let guardAccepted = false; + let accepted = false; try { - guardAccepted = afterSuccess(); + accepted = afterSuccess(); } catch { - guardAccepted = false; - } - if (!guardAccepted) { - runOpenshell(["forward", "stop", String(port), sandboxName], { - ignoreError: true, - stdio: "ignore", - }); - return false; - } - let forwardAccepted = true; - if (options.afterSuccess) { - // A newly registered port forward can report running and then drop while the - // pinned managed-health guard executes. Re-prove the authoritative owner - // and listener after that guard before status reports recovery success. - sleepMs(guardedSettleMs); - forwardAccepted = isSandboxPortForwardHealthy(sandboxName, port, expectedBind) === true; + accepted = false; } - if (forwardAccepted) return true; + if (accepted) return true; runOpenshell(["forward", "stop", String(port), sandboxName], { ignoreError: true, stdio: "ignore", }); - if (!options.afterSuccess || guardedDropRetries <= 0) return false; - return ensureSandboxPortForwardForPortWithRetries( - sandboxName, - port, - options, - guardedDropRetries - 1, - ); + return false; }; let forwardHealth = isSandboxPortForwardHealthy(sandboxName, port, expectedBind); if (forwardHealth === true && !forceRestart) return acceptSuccessfulForward(); if (forwardHealth === "occupied") return false; + const configuredWaitMs = Number(process.env.NEMOCLAW_FORWARD_RECOVERY_WAIT_MS ?? "3000"); + const waitMs = Number.isFinite(configuredWaitMs) ? Math.max(0, configuredWaitMs) : 3000; const stopResult = runOpenshell(["forward", "stop", String(port), sandboxName], { ignoreError: true, @@ -404,19 +379,6 @@ function ensureSandboxPortForwardForPortWithRetries( return settled && !occupied && acceptSuccessfulForward(); } -export function ensureSandboxPortForwardForPort( - sandboxName: string, - port: number, - options: SandboxPortForwardRecoveryOptions = {}, -): boolean { - return ensureSandboxPortForwardForPortWithRetries( - sandboxName, - port, - options, - options.afterSuccess ? 1 : 0, - ); -} - export function ensureHermesDashboardPortForwardIfEnabled(sandboxName: string): boolean | null { return ensureHermesDashboardPortForward(sandboxName, { isPortForwardHealthy: isSandboxPortForwardHealthy, diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index d9e8d202ec8..9bc28614aee 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -431,9 +431,8 @@ export async function isSandboxGatewayRunningForStatus( /** * Recover a gateway through the registered agent's managed control boundary. * Legacy custom agents retain their SSH-owned compatibility path. Built-in - * agents may relaunch a missing supervisor in the registered container, but - * the caller must still prove that exact container through the managed health - * gate. + * agents may return a transactional supervisor relaunch that the caller must + * commit or roll back after the managed health gate. */ type SandboxProcessRecovery = | { kind: "managed" | "custom" } @@ -1253,21 +1252,45 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( : null; // Wait for gateway to bind its HTTP port before declaring success. The // recovered process can be alive before the OpenAI-compatible API is ready. - const gatewayReady = waitForRecoveredSandboxGateway(sandboxName, { - quiet, - initialManagedHealthPassed: recovery.kind === "managed", - requireManagedProbe: recovery.kind === "relaunched", - timeoutSeconds: gatewayRecoveryTimeoutSeconds(recoveryAgent), - managedProbeImpl: (name) => - confirmRecoveredSandboxGatewayManaged(name, { - requestGatewaySupervisorActionImpl: requestManagedProbe, - }), - }); + let gatewayReady = false; + try { + gatewayReady = waitForRecoveredSandboxGateway(sandboxName, { + quiet, + initialManagedHealthPassed: recovery.kind === "managed", + requireManagedProbe: recovery.kind === "relaunched", + timeoutSeconds: gatewayRecoveryTimeoutSeconds(recoveryAgent), + managedProbeImpl: (name) => + confirmRecoveredSandboxGatewayManaged(name, { + requestGatewaySupervisorActionImpl: requestManagedProbe, + }), + }); + } catch (error) { + try { + relaunch?.finalize(false); + } catch { + // Preserve the original recovery error; the failure path below will + // direct the operator to inspect/rebuild the sandbox. + } + throw error; + } if (!gatewayReady) { + let rolledBack = true; + if (relaunch) { + try { + rolledBack = relaunch.finalize(false).rolledBack; + } catch { + rolledBack = false; + } + } if (!quiet) { console.error(" Gateway process started but is not responding."); printGatewayWedgeDiagnostics(sandboxName, executeSandboxExecCommand); console.error(" Check /tmp/gateway.log inside the sandbox for details."); + if (!rolledBack) { + console.error( + " Automatic rollback of the previous sandbox container failed; inspect Docker state before retrying.", + ); + } printHostManagedGatewayRecoveryHints( sandboxName, recoveryAgent, @@ -1276,6 +1299,22 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( } return { checked: true, wasRunning: false, recovered: false, forwardRecovered: false }; } + if (relaunch) { + try { + const completion = relaunch.finalize(true); + if (!completion.backupRemoved && !quiet) { + console.error( + " Warning: the recovered sandbox is healthy, but its previous container backup could not be removed.", + ); + } + } catch { + if (!quiet) { + console.error( + " Warning: the recovered sandbox is healthy, but container transaction cleanup could not be confirmed.", + ); + } + } + } const readinessFailureDetail = relaunch ? (() => { const readinessOptions: RecreatedSandboxOpenShellReadyOptions = { diff --git a/src/lib/actions/sandbox/supervisor-relaunch.test.ts b/src/lib/actions/sandbox/supervisor-relaunch.test.ts index 337f35b8427..f66776e636d 100644 --- a/src/lib/actions/sandbox/supervisor-relaunch.test.ts +++ b/src/lib/actions/sandbox/supervisor-relaunch.test.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { afterEach, describe, expect, it, vi } from "vitest"; +import type { DockerGpuPatchResult } from "../../onboard/docker-gpu-patch"; import { type ManagedSupervisorRelaunchDeps, relaunchManagedSupervisorSession, @@ -12,14 +13,20 @@ afterEach(() => { vi.unstubAllEnvs(); }); -function dockerResult(status: number) { +function patchResult(): DockerGpuPatchResult { return { - pid: 1, - output: [], - stdout: "", - stderr: "", - status, - signal: null, + applied: true, + oldContainerId: "old-container-id", + newContainerId: "new-container-id", + originalName: "openshell-alpha", + backupContainerName: "openshell-alpha-nemoclaw-backup", + mode: { + kind: "startup-command", + label: "persistent sandbox startup command", + device: "", + args: [], + }, + backupRemoved: false, }; } @@ -40,22 +47,17 @@ function baseDeps(overrides: ManagedSupervisorRelaunchDeps = {}) { }) as never, ), resolveDashboardPort: vi.fn(() => 18789), - resolveContainer: vi.fn(() => "registered-container-id"), + resolveContainer: vi.fn(() => "old-container-id"), inspectContainer: vi.fn(() => ({ Config: { Env: ["OPENSHELL_SANDBOX_COMMAND=sleep infinity"] }, })), confirmMissingSupervisor: vi.fn(() => true), - createNonce: vi.fn(() => "a".repeat(64)), - privilegedExecArgv: vi.fn(() => [ - "exec", - "--user", - "root", - "registered-container-id", - "/usr/local/bin/nemoclaw-gateway-control", - "launch-supervisor", - "a".repeat(64), - ]), - runDocker: vi.fn(() => dockerResult(0)), + recreate: vi.fn(() => patchResult()), + finalize: vi.fn(({ supervisorReady }) => + supervisorReady + ? { backupRemoved: true, rolledBack: false } + : { backupRemoved: false, rolledBack: true }, + ), ...overrides, } satisfies ManagedSupervisorRelaunchDeps; } @@ -66,7 +68,7 @@ describe("relaunchManagedSupervisorSession", () => { expect(relaunchManagedSupervisorSession("missing-box", { quiet: true, deps })).toBeNull(); expect(deps.resolveContainer).not.toHaveBeenCalled(); - expect(deps.runDocker).not.toHaveBeenCalled(); + expect(deps.recreate).not.toHaveBeenCalled(); }); it("honors the troubleshooting kill switch without mutating Docker", () => { @@ -75,7 +77,7 @@ describe("relaunchManagedSupervisorSession", () => { expect(relaunchManagedSupervisorSession("alpha", { quiet: true, deps })).toBeNull(); expect(deps.resolveContainer).not.toHaveBeenCalled(); - expect(deps.runDocker).not.toHaveBeenCalled(); + expect(deps.recreate).not.toHaveBeenCalled(); }); it("refuses a container that no longer has the legacy keepalive startup", () => { @@ -86,18 +88,18 @@ describe("relaunchManagedSupervisorSession", () => { }); expect(relaunchManagedSupervisorSession("alpha", { quiet: true, deps })).toBeNull(); - expect(deps.runDocker).not.toHaveBeenCalled(); + expect(deps.recreate).not.toHaveBeenCalled(); }); - it("refuses launch when the pinned container no longer proves supervisor absence", () => { + it("refuses recreation when the pinned container no longer proves supervisor absence", () => { const deps = baseDeps({ confirmMissingSupervisor: vi.fn(() => false) }); expect(relaunchManagedSupervisorSession("alpha", { quiet: true, deps })).toBeNull(); - expect(deps.confirmMissingSupervisor).toHaveBeenCalledWith("registered-container-id"); - expect(deps.runDocker).not.toHaveBeenCalled(); + expect(deps.confirmMissingSupervisor).toHaveBeenCalledWith("old-container-id"); + expect(deps.recreate).not.toHaveBeenCalled(); }); - it("requests a credential-free managed launch in the registered legacy keepalive container", () => { + it("pins the selected container and persists only a credential-free startup command", () => { vi.stubEnv("NEMOCLAW_EXTRA_PLACEHOLDER_KEYS", "CUSTOM_PROVIDER_CREDENTIAL"); vi.stubEnv("CUSTOM_PROVIDER_CREDENTIAL", "s3cr3t-token"); vi.stubEnv("HTTPS_PROXY", "http://proxyuser:proxypass@proxy.example:8080"); @@ -105,92 +107,55 @@ describe("relaunchManagedSupervisorSession", () => { const relaunch = relaunchManagedSupervisorSession("alpha", { quiet: true, deps }); - expect(relaunch).toEqual({ containerId: "registered-container-id" }); - expect(deps.privilegedExecArgv).toHaveBeenCalledOnce(); - const [sandboxName, launchCommand, stdin, sanitizeEnvironment, expectedContainerId] = - vi.mocked(deps.privilegedExecArgv).mock.calls[0] ?? []; - expect({ - expectedContainerId, - sandboxName, - sanitizeEnvironment, - stdin, - }).toEqual({ - expectedContainerId: "registered-container-id", + expect(relaunch).not.toBeNull(); + expect(relaunch?.containerId).toBe("new-container-id"); + expect(deps.recreate).toHaveBeenCalledOnce(); + const options = vi.mocked(deps.recreate).mock.calls[0]?.[0]; + expect(options).toMatchObject({ sandboxName: "alpha", - sanitizeEnvironment: true, - stdin: false, + expectedOldContainerId: "old-container-id", + waitForSupervisor: false, }); - expect(launchCommand?.slice(0, 3)).toEqual([ - "/usr/local/bin/nemoclaw-gateway-control", - "launch-supervisor", - "a".repeat(64), - ]); - const serialized = launchCommand?.join(" ") ?? ""; + const serialized = options?.openshellSandboxCommand.join(" ") ?? ""; expect(serialized).toContain("NEMOCLAW_DASHBOARD_PORT=18789"); + expect(serialized).toMatch(/nemoclaw-start$/); expect(serialized).not.toContain("s3cr3t-token"); expect(serialized).not.toContain("CUSTOM_PROVIDER_CREDENTIAL"); expect(serialized).not.toContain("proxypass"); - expect(deps.runDocker).toHaveBeenCalledWith( - expect.arrayContaining(["exec", "registered-container-id"]), - expect.objectContaining({ - ignoreError: true, - suppressOutput: true, - timeout: 30000, - }), - ); - }); - it("reconstructs the persisted Hermes dashboard environment", () => { - const deps = baseDeps({ - getSandbox: vi.fn(() => ({ - name: "alpha", - agent: "hermes", - dashboardPort: 18790, - hermesDashboardEnabled: true, - hermesDashboardInternalPort: 19119, - hermesDashboardPort: 18790, - hermesDashboardTui: true, - openshellDriver: "docker", - })), - getSessionAgent: vi.fn( - () => - ({ - name: "hermes", - displayName: "Hermes", - forwardPort: 18790, - }) as never, - ), - resolveDashboardPort: vi.fn(() => 18790), + expect(relaunch?.finalize(true)).toEqual({ backupRemoved: true, rolledBack: false }); + expect(deps.finalize).toHaveBeenCalledWith({ + result: expect.objectContaining({ newContainerId: "new-container-id" }), + supervisorReady: true, }); + }); - expect(relaunchManagedSupervisorSession("alpha", { quiet: true, deps })).toEqual({ - containerId: "registered-container-id", + it("rolls the container transaction back when managed readiness is not proven", () => { + const deps = baseDeps(); + const relaunch = relaunchManagedSupervisorSession("alpha", { quiet: true, deps }); + + expect(relaunch?.finalize(false)).toEqual({ backupRemoved: false, rolledBack: true }); + expect(deps.finalize).toHaveBeenCalledWith({ + result: expect.objectContaining({ backupContainerName: expect.any(String) }), + supervisorReady: false, }); - const launchCommand = vi.mocked(deps.privilegedExecArgv).mock.calls[0]?.[1] ?? []; - expect(launchCommand).toEqual( - expect.arrayContaining([ - "CHAT_UI_URL=http://127.0.0.1:18790", - "NEMOCLAW_DASHBOARD_PORT=18790", - "NEMOCLAW_HERMES_DASHBOARD=1", - "NEMOCLAW_HERMES_DASHBOARD_PORT=18790", - "NEMOCLAW_HERMES_DASHBOARD_INTERNAL_PORT=19119", - "NEMOCLAW_HERMES_DASHBOARD_TUI=1", - ]), - ); - expect(launchCommand.some((value) => value.startsWith("OPENCLAW_"))).toBe(false); }); - it("returns null when the registered container refuses managed launch", () => { - const deps = baseDeps({ runDocker: vi.fn(() => dockerResult(1)) }); + it("returns null when the pinned recreation fails", () => { + const deps = baseDeps({ + recreate: vi.fn(() => { + throw new Error("container identity changed"); + }), + }); expect(relaunchManagedSupervisorSession("alpha", { quiet: true, deps })).toBeNull(); }); - it("redacts diagnostics when trusted in-place launch fails", () => { + it("redacts diagnostics when trusted recreation fails", () => { vi.spyOn(console, "log").mockImplementation(() => undefined); const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); const deps = baseDeps({ - runDocker: vi.fn(() => { + recreate: vi.fn(() => { throw new Error( "OPENAI_API_KEY=sk-recovery-secret HTTPS_PROXY=http://proxyuser:proxypass@proxy.example:8080", ); diff --git a/src/lib/actions/sandbox/supervisor-relaunch.ts b/src/lib/actions/sandbox/supervisor-relaunch.ts index d6096972c18..729827fa866 100644 --- a/src/lib/actions/sandbox/supervisor-relaunch.ts +++ b/src/lib/actions/sandbox/supervisor-relaunch.ts @@ -1,20 +1,20 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { randomBytes } from "node:crypto"; -import { dockerCapture, dockerRun } from "../../adapters/docker"; +import { dockerCapture } from "../../adapters/docker"; import * as agentRuntime from "../../agent/runtime"; import { shouldManageDashboardForAgent } from "../../onboard/dashboard-runtime"; -import { hasZeroDockerExitStatus } from "../../onboard/docker-command-result"; import { type DockerContainerInspect, parseDockerInspectJson, } from "../../onboard/docker-gpu-patch"; -import { buildSandboxRuntimeEnvArgs } from "../../onboard/sandbox-create-launch"; import { - privilegedSandboxExecArgv, - resolveDirectSandboxContainer, -} from "../../sandbox/privileged-exec"; + type DockerGpuPatchFinalizeOutcome, + finalizeDockerGpuPatchBackup, +} from "../../onboard/docker-gpu-patch-finalize"; +import { recreateOpenShellDockerSandboxWithStartupCommand } from "../../onboard/docker-startup-command-patch"; +import { buildSandboxRuntimeEnvArgs } from "../../onboard/sandbox-create-launch"; +import { resolveDirectSandboxContainer } from "../../sandbox/privileged-exec"; import { redact, redactFull } from "../../security/redact"; import * as registry from "../../state/registry"; import { resolveSandboxDashboardPort } from "./forward-recovery"; @@ -23,23 +23,17 @@ import { resolveSandboxDashboardPort } from "./forward-recovery"; * Compatibility boundary for OpenShell 0.0.71's Docker driver: legacy * sandboxes persist `OPENSHELL_SANDBOX_COMMAND=sleep infinity` while * `scripts/nemoclaw-start.sh` owns the managed workload as a sibling process. - * Relaunch requires that inspected legacy value, the registered exact container - * identity, and the managed controller's exact pinned proof that no supervisor - * is running. The root-owned controller then rechecks absence, refreshes the - * current OpenShell CA, and launches one sandbox-UID supervisor. The caller - * accepts it only after the same pinned controller proves managed health. - * Replacing the container while its OpenShell supervisor is registered either - * publishes terminal phase Error or blocks duplicate supervisor registration. - * Regression coverage is named in `supervisor-relaunch.test.ts`, - * `process-recovery-supervisor-relaunch.test.ts`, and `sandbox-survival.test.ts`. + * Only that inspected value authorizes this migration. Regression coverage is + * named in `supervisor-relaunch.test.ts` and `gateway-guard-recovery.test.ts`. * Remove this path after supported upgrades rebuild every legacy keepalive * container with `nemoclaw-start` as its persisted startup command. */ const LEGACY_OPENSHELL_KEEPALIVE = "sleep infinity"; -const DOCKER_CONTROL_TIMEOUT_MS = 30000; +const DOCKER_INSPECT_TIMEOUT_MS = 15000; export type ManagedSupervisorRelaunch = { containerId: string; + finalize(supervisorReady: boolean): DockerGpuPatchFinalizeOutcome; }; export type ManagedSupervisorRelaunchDeps = { @@ -49,16 +43,15 @@ export type ManagedSupervisorRelaunchDeps = { resolveContainer?: typeof resolveDirectSandboxContainer; inspectContainer?: (containerId: string) => DockerContainerInspect; confirmMissingSupervisor?: (containerId: string) => boolean; - createNonce?: () => string; - privilegedExecArgv?: typeof privilegedSandboxExecArgv; - runDocker?: typeof dockerRun; + recreate?: typeof recreateOpenShellDockerSandboxWithStartupCommand; + finalize?: typeof finalizeDockerGpuPatchBackup; }; function inspectContainer(containerId: string): DockerContainerInspect { return parseDockerInspectJson( dockerCapture(["inspect", "--type", "container", containerId], { ignoreError: true, - timeout: DOCKER_CONTROL_TIMEOUT_MS, + timeout: DOCKER_INSPECT_TIMEOUT_MS, }), ); } @@ -71,7 +64,7 @@ function hasLegacyKeepaliveStartup(inspect: DockerContainerInspect): boolean { return values.length === 1 && values[0] === LEGACY_OPENSHELL_KEEPALIVE; } -function reconstructSupervisorRuntimeEnvironment( +function reconstructSupervisorLaunchCommand( sandboxName: string, entry: NonNullable>, deps: ManagedSupervisorRelaunchDeps, @@ -110,7 +103,7 @@ function reconstructSupervisorRuntimeEnvironment( env: process.env, omitCredentialEnv: true, }); - return envArgs; + return ["env", ...envArgs, "nemoclaw-start"]; } export function relaunchManagedSupervisorSession( @@ -129,40 +122,45 @@ export function relaunchManagedSupervisorSession( if (!entry) return null; const driver = entry.openshellDriver?.trim().toLowerCase() ?? null; if (driver !== null && driver !== "docker" && driver !== "vm") return null; - const runtimeEnvironment = reconstructSupervisorRuntimeEnvironment(sandboxName, entry, deps); - if (runtimeEnvironment === null) return null; + const startupCommand = reconstructSupervisorLaunchCommand(sandboxName, entry, deps); + if (startupCommand === null) return null; const resolveContainer = deps.resolveContainer ?? resolveDirectSandboxContainer; const inspect = deps.inspectContainer ?? inspectContainer; const confirmMissingSupervisor = deps.confirmMissingSupervisor; - const privilegedExecArgv = deps.privilegedExecArgv ?? privilegedSandboxExecArgv; - const runDocker = deps.runDocker ?? dockerRun; + const recreate = deps.recreate ?? recreateOpenShellDockerSandboxWithStartupCommand; + const finalize = deps.finalize ?? finalizeDockerGpuPatchBackup; try { const containerId = resolveContainer(sandboxName, driver); if (!hasLegacyKeepaliveStartup(inspect(containerId))) return null; if (!confirmMissingSupervisor?.(containerId)) return null; if (!quiet) { - console.log(" Launching the managed supervisor in the registered sandbox container..."); + console.log(" Recreating the sandbox container with its managed startup command..."); } - const nonce = (deps.createNonce ?? (() => randomBytes(32).toString("hex")))(); - const launchCommand = [ - "/usr/local/bin/nemoclaw-gateway-control", - "launch-supervisor", - nonce, - ...runtimeEnvironment, - ]; - const launchResult = runDocker( - privilegedExecArgv(sandboxName, launchCommand, false, true, containerId), - { - ignoreError: true, - suppressOutput: true, - timeout: DOCKER_CONTROL_TIMEOUT_MS, + const result = recreate({ + sandboxName, + openshellSandboxCommand: startupCommand, + expectedOldContainerId: containerId, + waitForSupervisor: false, + }); + let completed: { supervisorReady: boolean; outcome: DockerGpuPatchFinalizeOutcome } | null = + null; + return { + containerId: result.newContainerId, + finalize(supervisorReady) { + if (completed) { + if (completed.supervisorReady !== supervisorReady) { + throw new Error( + "Supervisor relaunch transaction was finalized with conflicting state.", + ); + } + return completed.outcome; + } + const outcome = finalize({ result, supervisorReady }); + completed = { supervisorReady, outcome }; + return outcome; }, - ); - if (!hasZeroDockerExitStatus(launchResult)) { - throw new Error("The registered container refused the managed supervisor launch."); - } - return { containerId }; + }; } catch (error) { if (!quiet) { const detail = error instanceof Error ? error.message : String(error); diff --git a/test/e2e/live/gateway-guard-recovery.test.ts b/test/e2e/live/gateway-guard-recovery.test.ts index ee2a4d65555..ab658d9c662 100644 --- a/test/e2e/live/gateway-guard-recovery.test.ts +++ b/test/e2e/live/gateway-guard-recovery.test.ts @@ -287,8 +287,8 @@ test("gateway recovery restores /tmp guard chain after pod-recreate wipe (#2701) // Fresh non-GPU OpenClaw containers on this OpenShell floor still carry the // legacy keepalive. Restarting the container therefore kills the initial // OpenShell workload session and deterministically leaves no managed - // supervisor. Recovery must launch the supervisor in the registered - // container and prove managed health without replacing that container. + // supervisor. Recovery must upgrade that container through the host-side + // transaction and commit only after managed control accepts the new tree. const originalContainerId = await findSandboxContainer(host, "legacy-restart-container-before"); expect( await inspectStartupCommand(host, originalContainerId, "legacy-restart-command-before"), @@ -321,13 +321,13 @@ test("gateway recovery restores /tmp guard chain after pod-recreate wipe (#2701) expect(resultText(trustedRecovery)).toContain("Probe complete: recovered OpenClaw gateway"); const recoveredContainerId = await findSandboxContainer(host, "legacy-restart-container-after"); - expect(recoveredContainerId).toBe(originalContainerId); + expect(recoveredContainerId).not.toBe(originalContainerId); const recoveredStartupCommand = await inspectStartupCommand( host, recoveredContainerId, "legacy-restart-command-after", ); - expect(recoveredStartupCommand).toBe("sleep infinity"); + expect(recoveredStartupCommand).toMatch(/(?:^| )nemoclaw-start$/); expect(recoveredStartupCommand).not.toContain("CUSTOM_PROVIDER_CREDENTIAL"); expect(recoveredStartupCommand).not.toContain(credentialCanary); diff --git a/test/gateway-supervisor-control.test.ts b/test/gateway-supervisor-control.test.ts index 0a3f292fdc1..8a9e429fd96 100644 --- a/test/gateway-supervisor-control.test.ts +++ b/test/gateway-supervisor-control.test.ts @@ -278,7 +278,6 @@ describe("root-only gateway control helper", () => { it.each([ "restart", "probe", - "launch-supervisor", ])("enters managed %s control with isolated Python before user-site startup hooks", (action) => { const root = temporaryDirectory("nemoclaw-managed-python-isolation-"); const userBase = join(root, "attacker-userbase"); @@ -340,11 +339,6 @@ describe("root-only gateway control helper", () => { ["bad action", ["replace", VALID_NONCE], "SUPERVISOR_INVALID_ACTION"], ["short nonce", ["restart", "abcd"], "SUPERVISOR_INVALID_NONCE"], ["uppercase nonce", ["recover", "B".repeat(64)], "SUPERVISOR_INVALID_NONCE"], - [ - "extra restart argument", - ["restart", VALID_NONCE, "CHAT_UI_URL=http://127.0.0.1:18789"], - "SUPERVISOR_INVALID_REQUEST", - ], ])("rejects %s before touching the control directory", (_label, args, marker) => { const result = spawnSync(CONTROL_HELPER, args, { encoding: "utf-8", diff --git a/test/managed-supervisor-launch.test.ts b/test/managed-supervisor-launch.test.ts deleted file mode 100644 index 9c2305335e8..00000000000 --- a/test/managed-supervisor-launch.test.ts +++ /dev/null @@ -1,181 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { spawnSync } from "node:child_process"; -import path from "node:path"; -import { describe, expect, it } from "vitest"; - -const HELPER = path.join(import.meta.dirname, "..", "scripts", "managed-gateway-control.py"); -const NONCE = "a".repeat(64); -const MERGED_CA = "/tmp/nemoclaw-ca-bundle.pem"; - -const SUPERVISOR_LAUNCH_ENV_HARNESS = String.raw` -import importlib.util -import json -import os -import sys - -spec = importlib.util.spec_from_file_location("managed_control_launch", sys.argv[1]) -control = importlib.util.module_from_spec(spec) -sys.modules[spec.name] = control -spec.loader.exec_module(control) - -os.environ.update({ - "LD_PRELOAD": "/attacker/preload.so", - "NODE_OPTIONS": "--require=/attacker/hook.js", - "NEMOCLAW_TEST_ESCAPE": "attacker", - "NVIDIA_INFERENCE_API_KEY": "container-owned-placeholder", -}) -action, nonce, runtime = control._validate_request([ - "launch-supervisor", - "a" * 64, - "CHAT_UI_URL=http://127.0.0.1:18789", - "NEMOCLAW_DASHBOARD_PORT=18789", - "HTTPS_PROXY=https://proxy.example/path?token=a=b", -]) -environment = control._supervisor_launch_environment( - runtime, "/tmp/nemoclaw-ca-bundle.pem" -) -print(json.dumps({ - "action": action, - "nonce": nonce, - "runtime": runtime, - "identity": { - key: environment.get(key) - for key in ("HOME", "LOGNAME", "PATH", "SHELL", "USER") - }, - "python_no_user_site": environment.get("PYTHONNOUSERSITE"), - "ca": { - key: environment.get(key) - for key in control.CA_ENV_KEYS - }, - "stripped": { - key: key in environment - for key in ("LD_PRELOAD", "NODE_OPTIONS", "NEMOCLAW_TEST_ESCAPE") - }, - "container_credential": environment.get("NVIDIA_INFERENCE_API_KEY"), -}, sort_keys=True)) -`; - -const SUPERVISOR_CA_REFRESH_HARNESS = String.raw` -import importlib.util -import json -import os -import stat -import sys -import tempfile - -spec = importlib.util.spec_from_file_location("managed_control_ca", sys.argv[1]) -control = importlib.util.module_from_spec(spec) -sys.modules[spec.name] = control -spec.loader.exec_module(control) - -with tempfile.TemporaryDirectory() as root: - system_root = os.path.join(root, "system") - openshell = os.path.join(system_root, "etc", "openshell-tls", "ca-bundle.pem") - corporate = os.path.join( - system_root, "usr", "local", "share", "nemoclaw", "corporate-ca.pem" - ) - merged = os.path.join(system_root, "tmp", "nemoclaw-ca-bundle.pem") - for directory in (os.path.dirname(openshell), os.path.dirname(corporate), os.path.dirname(merged)): - os.makedirs(directory, exist_ok=True) - for path, contents in ( - (openshell, b"CURRENT-OPENSHELL-CA\n"), - (corporate, b"CORPORATE-CA\n"), - (merged, b"STALE-OPENSHELL-CA\nCORPORATE-CA\n"), - ): - with open(path, "wb") as stream: - stream.write(contents) - os.chmod(path, 0o444) - - os.environ["NEMOCLAW_MANAGED_CONTROL_ALLOW_NONROOT_TEST"] = "1" - os.environ["NEMOCLAW_MANAGED_CONTROL_SYSTEM_ROOT"] = system_root - selected = control._refresh_supervisor_ca_bundle() - with open(merged, "rb") as stream: - contents = stream.read().decode("ascii") - metadata = os.stat(merged, follow_symlinks=False) - - print(json.dumps({ - "selected": selected, - "contents": contents, - "mode": stat.S_IMODE(metadata.st_mode), - }, sort_keys=True)) -`; - -describe("managed supervisor launch", () => { - it("allowlists launch inputs without inherited credentials or loader hooks", () => { - const result = spawnSync("python3", ["-c", SUPERVISOR_LAUNCH_ENV_HARNESS, HELPER], { - encoding: "utf-8", - timeout: 5000, - }); - - expect(result.status, result.stderr).toBe(0); - expect(JSON.parse(result.stdout)).toEqual({ - action: "launch-supervisor", - nonce: NONCE, - runtime: { - CHAT_UI_URL: "http://127.0.0.1:18789", - HTTPS_PROXY: "https://proxy.example/path?token=a=b", - NEMOCLAW_DASHBOARD_PORT: "18789", - }, - identity: { - HOME: "/sandbox", - LOGNAME: "sandbox", - PATH: "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", - SHELL: "/bin/bash", - USER: "sandbox", - }, - python_no_user_site: "1", - ca: { - CURL_CA_BUNDLE: MERGED_CA, - GIT_SSL_CAINFO: MERGED_CA, - NODE_EXTRA_CA_CERTS: MERGED_CA, - REQUESTS_CA_BUNDLE: MERGED_CA, - SSL_CERT_FILE: MERGED_CA, - }, - stripped: { - LD_PRELOAD: false, - NEMOCLAW_TEST_ESCAPE: false, - NODE_OPTIONS: false, - }, - container_credential: null, - }); - }); - - it.each([ - [["launch-supervisor", NONCE, "UNREVIEWED=value"], "SUPERVISOR_INVALID_REQUEST"], - [ - [ - "launch-supervisor", - NONCE, - "CHAT_UI_URL=http://127.0.0.1:18789", - "CHAT_UI_URL=http://127.0.0.1:18790", - ], - "SUPERVISOR_INVALID_REQUEST", - ], - [["restart", NONCE, "CHAT_UI_URL=http://127.0.0.1:18789"], "SUPERVISOR_INVALID_REQUEST"], - ])("rejects disallowed or duplicate request arguments before privilege use", (args, marker) => { - const result = spawnSync("python3", [HELPER, ...args], { - encoding: "utf-8", - timeout: 5000, - }); - - expect(result.status).toBe(1); - expect(result.stdout).toBe(""); - expect(result.stderr.trim()).toBe(marker); - }); - - it("replaces the stale merged CA with the current OpenShell and corporate roots", () => { - const result = spawnSync("python3", ["-c", SUPERVISOR_CA_REFRESH_HARNESS, HELPER], { - encoding: "utf-8", - timeout: 5000, - }); - - expect(result.status, result.stderr).toBe(0); - expect(JSON.parse(result.stdout)).toEqual({ - selected: MERGED_CA, - contents: "CURRENT-OPENSHELL-CA\nCORPORATE-CA\n", - mode: 0o444, - }); - }); -}); diff --git a/test/process-recovery-forward-failure.test.ts b/test/process-recovery-forward-failure.test.ts index 37b6476474b..1ea8f506d6e 100644 --- a/test/process-recovery-forward-failure.test.ts +++ b/test/process-recovery-forward-failure.test.ts @@ -232,51 +232,6 @@ beta 127.0.0.1 18789 12345 dead`, }); describe("ensureSandboxPortForwardForPort already-forwarded idempotency (#7085)", () => { - it("retries when a newly started forward drops during the managed health guard", () => { - vi.stubEnv("NEMOCLAW_FORWARD_RECOVERY_WAIT_MS", "0"); - let forwardLive = false; - let startCount = 0; - let guardCount = 0; - - vi.spyOn(forwardHealth, "isLocalForwardReachable").mockImplementation(() => forwardLive); - vi.spyOn(openshellRuntime, "captureOpenshell").mockImplementation(() => ({ - status: 0, - output: forwardLive - ? `SANDBOX BIND PORT PID STATUS -beta 127.0.0.1 18791 12345 running` - : "SANDBOX BIND PORT PID STATUS", - })); - const runOpenshell = vi - .spyOn(openshellRuntime, "runOpenshell") - .mockImplementation((rawArgs: unknown) => { - const args = Array.isArray(rawArgs) ? rawArgs.map(String) : []; - const forwardAction = args[0] === "forward" ? args[1] : ""; - const startsForward = Number(forwardAction === "start"); - startCount += startsForward; - forwardLive = startsForward > 0 || (forwardLive && forwardAction !== "stop"); - return { status: 0 } as never; - }); - - expect( - withFakeOpenshellBinary(() => - ensureSandboxPortForwardForPort("beta", 18791, { - afterSuccess: () => { - guardCount += 1; - forwardLive = guardCount !== 1; - return true; - }, - expectedBind: "127.0.0.1", - }), - ), - ).toBe(true); - expect(startCount).toBe(2); - expect(guardCount).toBe(2); - expect(runOpenshell).toHaveBeenCalledWith( - ["forward", "start", "--background", "18791", "beta"], - expect.objectContaining({ ignoreError: true }), - ); - }); - it("reconciles a reachable ownerless listener with a nonzero recovery wait", () => { vi.stubEnv("NEMOCLAW_FORWARD_RECOVERY_WAIT_MS", "25"); let started = false; diff --git a/test/process-recovery-supervisor-relaunch.test.ts b/test/process-recovery-supervisor-relaunch.test.ts index 1c12884348f..cb0fe7ac52e 100644 --- a/test/process-recovery-supervisor-relaunch.test.ts +++ b/test/process-recovery-supervisor-relaunch.test.ts @@ -99,15 +99,15 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { stderr: "SUPERVISOR_NOT_RUNNING", })); const resolveContainer = vi.fn(() => "old-container-id"); - const runDocker = vi.fn(() => { - throw new Error("kill switch allowed supervisor launch"); + const recreate = vi.fn(() => { + throw new Error("kill switch allowed container mutation"); }); const requestPinnedGatewaySupervisorAction = vi.fn(() => null); const relaunchManagedSupervisorSessionImpl = vi.fn( (sandboxName: string, options: Parameters[1]) => relaunchManagedSupervisorSession(sandboxName, { quiet: options.quiet, - deps: { ...options.deps, resolveContainer, runDocker }, + deps: { ...options.deps, resolveContainer, recreate }, }), ); const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); @@ -129,7 +129,7 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { ); expect(resolveContainer).not.toHaveBeenCalled(); expect(requestPinnedGatewaySupervisorAction).not.toHaveBeenCalled(); - expect(runDocker).not.toHaveBeenCalled(); + expect(recreate).not.toHaveBeenCalled(); const errorLines = errorSpy.mock.calls.map((call) => String(call[0])); expect(errorLines).toContainEqual( expect.stringContaining("Failure layer: supervisor not running"), @@ -141,11 +141,13 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { ); }); - it("keeps the registered container when in-place managed control never accepts it", () => { + it("rolls back when recreation starts but managed control never accepts it", () => { mockOpenClawSandbox("rejected-box"); setImmediateRecoveryPolling(); + const finalize = vi.fn(() => ({ backupRemoved: false, rolledBack: true })); const relaunchManagedSupervisorSessionImpl = vi.fn(() => ({ - containerId: "registered-container-id", + containerId: "replacement-container-id", + finalize, })); const requestGatewaySupervisorAction = vi.fn((_name: string, action: string) => action === "recover" ? { status: 1, stdout: "", stderr: "SUPERVISOR_NOT_RUNNING" } : null, @@ -165,15 +167,23 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { "rejected-box", "probe", 210000, - "registered-container-id", + "replacement-container-id", ); + expect(finalize).toHaveBeenCalledOnce(); + expect(finalize).toHaveBeenCalledWith(false); }); - it("accepts the in-place supervisor only after managed health passes", () => { + it("commits only after managed health accepts the recreated supervisor", () => { mockOpenClawSandbox("recovered-box"); setImmediateRecoveryPolling(); + const finalize = vi.fn((supervisorReady: boolean) => + supervisorReady + ? { backupRemoved: true, rolledBack: false } + : { backupRemoved: false, rolledBack: true }, + ); const relaunchManagedSupervisorSessionImpl = vi.fn(() => ({ - containerId: "registered-container-id", + containerId: "replacement-container-id", + finalize, })); const requestGatewaySupervisorAction = vi.fn((_name: string, action: string) => action === "recover" ? { status: 1, stdout: "", stderr: "SUPERVISOR_NOT_RUNNING" } : null, @@ -204,18 +214,26 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { "recovered-box", "probe", 210000, - "registered-container-id", + "replacement-container-id", ); + expect(finalize).toHaveBeenCalledOnce(); + expect(finalize).toHaveBeenCalledWith(true); }); - it("retries a busy pinned managed probe before starting the recovered forward", () => { + it("retries a busy pinned managed probe before starting the replacement forward", () => { mockOpenClawSandbox("busy-recovered-box"); vi.stubEnv("NEMOCLAW_GATEWAY_RECOVERY_POLL_INTERVAL_SECONDS", "0"); vi.stubEnv("NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS", "1"); vi.stubEnv("NEMOCLAW_GATEWAY_RECOVERY_SETTLE_SECONDS", "0"); vi.stubEnv("NEMOCLAW_FORWARD_RECOVERY_WAIT_MS", "0"); + const finalize = vi.fn((supervisorReady: boolean) => + supervisorReady + ? { backupRemoved: true, rolledBack: false } + : { backupRemoved: false, rolledBack: true }, + ); const relaunchManagedSupervisorSessionImpl = vi.fn(() => ({ - containerId: "registered-container-id", + containerId: "replacement-container-id", + finalize, })); const requestGatewaySupervisorAction = vi.fn((_name: string, action: string) => action === "recover" ? { status: 1, stdout: "", stderr: "SUPERVISOR_NOT_RUNNING" } : null, @@ -283,6 +301,7 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { ["sandbox", "exec", "--name", "busy-recovered-box", "--", "true"], expect.objectContaining({ ignoreError: true }), ); + expect(finalize).toHaveBeenCalledWith(true); expect(runOpenshell).toHaveBeenCalledWith( ["forward", "start", "--background", "18789", "busy-recovered-box"], expect.objectContaining({ ignoreError: true }), @@ -292,8 +311,10 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { it("uses the sandbox readiness budget after a longer gateway health wait (#7273)", () => { mockOpenClawSandbox("unready-box", 600); setImmediateRecoveryPolling(); + const finalize = vi.fn(() => ({ backupRemoved: true, rolledBack: false })); const relaunchManagedSupervisorSessionImpl = vi.fn(() => ({ - containerId: "registered-container-id", + containerId: "replacement-container-id", + finalize, })); const requestGatewaySupervisorAction = vi.fn(() => ({ status: 1, @@ -325,6 +346,8 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { forwardRecoveryFailed: true, forwardRecoveryFailureDetail: expect.stringContaining("did not become ready in OpenShell"), }); + expect(finalize).toHaveBeenCalledOnce(); + expect(finalize).toHaveBeenCalledWith(true); expect(waitForRecreatedSandboxOpenShellReadyImpl).toHaveBeenCalledWith( "unready-box", expect.objectContaining({ beforeProbe: expect.any(Function), timeoutSeconds: 180 }), @@ -335,8 +358,10 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { it("reports the last structured OpenShell error when readiness times out", () => { mockOpenClawSandbox("relay-dropped-box"); setImmediateRecoveryPolling(); + const finalize = vi.fn(() => ({ backupRemoved: true, rolledBack: false })); const relaunchManagedSupervisorSessionImpl = vi.fn(() => ({ - containerId: "registered-container-id", + containerId: "replacement-container-id", + finalize, })); const requestGatewaySupervisorAction = vi.fn(() => ({ status: 1, @@ -385,14 +410,17 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { ), }); expect(captureOpenshell).toHaveBeenCalled(); + expect(finalize).toHaveBeenCalledWith(true); expect(runOpenshell).not.toHaveBeenCalled(); }); it("reports a definitive managed health failure separately from OpenShell readiness", () => { mockOpenClawSandbox("managed-failed-box"); setImmediateRecoveryPolling(); + const finalize = vi.fn(() => ({ backupRemoved: true, rolledBack: false })); const relaunchManagedSupervisorSessionImpl = vi.fn(() => ({ - containerId: "registered-container-id", + containerId: "replacement-container-id", + finalize, })); const requestGatewaySupervisorAction = vi.fn(() => ({ status: 1, @@ -430,10 +458,11 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { forwardRecoveryFailed: true, forwardRecoveryFailureDetail: expect.stringContaining("failed the managed health guard"), }); + expect(finalize).toHaveBeenCalledWith(true); expect(captureOpenshell).not.toHaveBeenCalled(); }); - it("rejects a healthy forward when the registered container identity changes", () => { + it("rejects a healthy forward when the replacement identity changes after readiness", () => { mockOpenClawSandbox("drifted-box"); vi.mocked(agentRuntime.getSessionAgent).mockReturnValue({ name: "openclaw", @@ -443,8 +472,10 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { healthProbe: { url: "http://127.0.0.1:18789/health", port: 18789, timeout_seconds: 30 }, } as never); setImmediateRecoveryPolling(); + const finalize = vi.fn(() => ({ backupRemoved: true, rolledBack: false })); const relaunchManagedSupervisorSessionImpl = vi.fn(() => ({ - containerId: "registered-container-id", + containerId: "replacement-container-id", + finalize, })); const requestGatewaySupervisorAction = vi.fn(() => ({ status: 1, @@ -497,8 +528,9 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { "drifted-box", "probe", 15000, - "registered-container-id", + "replacement-container-id", ); + expect(finalize).toHaveBeenCalledWith(true); expect(runOpenshell).toHaveBeenCalledOnce(); expect(runOpenshell).toHaveBeenCalledWith(["forward", "stop", "18789", "drifted-box"], { ignoreError: true, From 14a03d5665689fc1abf1f99069ccb398cb5ee0a4 Mon Sep 17 00:00:00 2001 From: San Dang Date: Thu, 30 Jul 2026 20:54:12 +0530 Subject: [PATCH 17/30] fix(recovery): preserve supervisor during handoff Signed-off-by: San Dang --- .../sandbox/supervisor-relaunch.test.ts | 1 + .../actions/sandbox/supervisor-relaunch.ts | 1 + .../onboard/docker-gpu-patch-finalize.test.ts | 45 +++++++++++++++ src/lib/onboard/docker-gpu-patch-finalize.ts | 16 +++++- src/lib/onboard/docker-gpu-patch-recreate.ts | 53 +++++++++++------- src/lib/onboard/docker-gpu-patch-rollback.ts | 18 +++++- src/lib/onboard/docker-gpu-patch-types.ts | 4 ++ .../docker-startup-command-patch.test.ts | 55 +++++++++++++++++++ .../onboard/docker-startup-command-patch.ts | 1 + 9 files changed, 169 insertions(+), 25 deletions(-) diff --git a/src/lib/actions/sandbox/supervisor-relaunch.test.ts b/src/lib/actions/sandbox/supervisor-relaunch.test.ts index f66776e636d..50937047195 100644 --- a/src/lib/actions/sandbox/supervisor-relaunch.test.ts +++ b/src/lib/actions/sandbox/supervisor-relaunch.test.ts @@ -114,6 +114,7 @@ describe("relaunchManagedSupervisorSession", () => { expect(options).toMatchObject({ sandboxName: "alpha", expectedOldContainerId: "old-container-id", + keepOriginalRunningUntilFinalize: true, waitForSupervisor: false, }); const serialized = options?.openshellSandboxCommand.join(" ") ?? ""; diff --git a/src/lib/actions/sandbox/supervisor-relaunch.ts b/src/lib/actions/sandbox/supervisor-relaunch.ts index 729827fa866..fecf06465b7 100644 --- a/src/lib/actions/sandbox/supervisor-relaunch.ts +++ b/src/lib/actions/sandbox/supervisor-relaunch.ts @@ -141,6 +141,7 @@ export function relaunchManagedSupervisorSession( sandboxName, openshellSandboxCommand: startupCommand, expectedOldContainerId: containerId, + keepOriginalRunningUntilFinalize: true, waitForSupervisor: false, }); let completed: { supervisorReady: boolean; outcome: DockerGpuPatchFinalizeOutcome } | null = diff --git a/src/lib/onboard/docker-gpu-patch-finalize.test.ts b/src/lib/onboard/docker-gpu-patch-finalize.test.ts index 5a691435ce4..57a5bb5e882 100644 --- a/src/lib/onboard/docker-gpu-patch-finalize.test.ts +++ b/src/lib/onboard/docker-gpu-patch-finalize.test.ts @@ -37,6 +37,25 @@ describe("finalizeDockerGpuPatchBackup", () => { ); }); + it("force-removes the running original container after replacement health is confirmed", () => { + const dockerForceRm = vi.fn((_name: string) => ({ status: 0 })); + const dockerRm = vi.fn((_name: string) => ({ status: 0 })); + const outcome = finalizeDockerGpuPatchBackup( + { + result: { ...deferredCreateResult(), backupWasRunning: true }, + supervisorReady: true, + }, + { dockerForceRm, dockerRm }, + ); + + expect(outcome).toEqual({ backupRemoved: true, rolledBack: false }); + expect(dockerForceRm).toHaveBeenCalledWith( + "openshell-alpha-nemoclaw-gpu-backup-1780491860342", + expect.objectContaining({ ignoreError: true }), + ); + expect(dockerRm).not.toHaveBeenCalled(); + }); + it("rolls back to the backup container when supervisor reconnect failed", () => { const dockerStop = vi.fn(() => ({ status: 0 })); const dockerRm = vi.fn((_name: string) => ({ status: 0 })); @@ -65,6 +84,32 @@ describe("finalizeDockerGpuPatchBackup", () => { ).toBe(false); }); + it("rolls back to the still-running original container without restarting it", () => { + const dockerStop = vi.fn(() => ({ status: 0 })); + const dockerRm = vi.fn((_name: string) => ({ status: 0 })); + const dockerRename = vi.fn((_old: string, _next: string) => ({ status: 0 })); + const dockerStart = vi.fn(() => ({ status: 0 })); + const outcome = finalizeDockerGpuPatchBackup( + { + result: { ...deferredCreateResult(), backupWasRunning: true }, + supervisorReady: false, + }, + { dockerStop, dockerRm, dockerRename, dockerStart }, + ); + + expect(outcome).toEqual({ backupRemoved: false, rolledBack: true }); + expect(dockerStop).toHaveBeenCalledWith( + "new-container-id", + expect.objectContaining({ ignoreError: true }), + ); + expect(dockerRename).toHaveBeenCalledWith( + "openshell-alpha-nemoclaw-gpu-backup-1780491860342", + "openshell-alpha", + expect.objectContaining({ ignoreError: true }), + ); + expect(dockerStart).not.toHaveBeenCalled(); + }); + it("reports rolledBack=false when restoring the backup fails", () => { const dockerStop = vi.fn(() => ({ status: 0 })); const dockerRm = vi.fn((_name: string) => ({ status: 0 })); diff --git a/src/lib/onboard/docker-gpu-patch-finalize.ts b/src/lib/onboard/docker-gpu-patch-finalize.ts index 4eeeedf399b..e3c0dc65ff6 100644 --- a/src/lib/onboard/docker-gpu-patch-finalize.ts +++ b/src/lib/onboard/docker-gpu-patch-finalize.ts @@ -65,7 +65,9 @@ export function finalizeDockerGpuPatchBackup( // even if `docker rm` cannot delete it (e.g. concurrent admin action, // daemon timeout). Reflect the actual rm status in the outcome so // diagnostics can flag a leaked backup container. - const rmResult = resolved.dockerRm(options.result.backupContainerName, containerOpts); + const rmResult = options.result.backupWasRunning + ? resolved.dockerForceRm(options.result.backupContainerName, containerOpts) + : resolved.dockerRm(options.result.backupContainerName, containerOpts); return { backupRemoved: hasZeroDockerExitStatus(rmResult), rolledBack: false }; } const rolledBack = rollbackToBackupContainer( @@ -73,6 +75,7 @@ export function finalizeDockerGpuPatchBackup( newContainerId: options.result.newContainerId, backupContainerName: options.result.backupContainerName, originalName: options.result.originalName, + backupWasRunning: options.result.backupWasRunning, }, resolved, ); @@ -85,7 +88,12 @@ export type SupervisorReconnectOutcome = export function reconcileSupervisorReconnect( execReady: boolean, - refs: { newContainerId: string; backupContainerName: string; originalName: string }, + refs: { + newContainerId: string; + backupContainerName: string; + originalName: string; + backupWasRunning?: boolean; + }, deps: DockerGpuPatchDeps, ): SupervisorReconnectOutcome { const resolved = resolveDockerGpuPatchRollbackDeps(deps); @@ -100,7 +108,9 @@ export function reconcileSupervisorReconnect( // leaked backup container but the user-visible sandbox is healthy. // Surface the actual rm status so callers can fold it into diagnostics // alongside the deferred-finalize path in `finalizeDockerGpuPatchBackup`. - const rmResult = resolved.dockerRm(refs.backupContainerName, containerOpts); + const rmResult = refs.backupWasRunning + ? resolved.dockerForceRm(refs.backupContainerName, containerOpts) + : resolved.dockerRm(refs.backupContainerName, containerOpts); return { execReady: true, backupRemoved: hasZeroDockerExitStatus(rmResult) }; } const rolledBack = rollbackToBackupContainer(refs, resolved); diff --git a/src/lib/onboard/docker-gpu-patch-recreate.ts b/src/lib/onboard/docker-gpu-patch-recreate.ts index b205c144af8..7587437378a 100644 --- a/src/lib/onboard/docker-gpu-patch-recreate.ts +++ b/src/lib/onboard/docker-gpu-patch-recreate.ts @@ -3,6 +3,7 @@ import { dockerCapture, + dockerForceRm, dockerRename, dockerRm, dockerRun, @@ -48,6 +49,7 @@ type RecreateDeps = Required< Pick< DockerGpuPatchDeps, | "dockerCapture" + | "dockerForceRm" | "dockerRun" | "dockerRunDetached" | "dockerRename" @@ -66,6 +68,7 @@ type RecreateDeps = Required< function recreateDeps(deps: DockerGpuPatchDeps): RecreateDeps { return { dockerCapture, + dockerForceRm, dockerRun, dockerRunDetached, dockerRename, @@ -155,6 +158,7 @@ export function recreateOpenShellDockerSandboxContainer( gpuDevice?: string | null; timeoutSecs?: number; waitForSupervisor?: boolean; + keepOriginalRunningUntilFinalize?: boolean; openshellSandboxCommand?: readonly string[] | null; requiredUlimits?: readonly import("./docker-gpu-patch-types").DockerUlimit[] | null; expectedOldContainerId?: string | null; @@ -171,6 +175,11 @@ export function recreateOpenShellDockerSandboxContainer( }; try { validateRequiredDockerUlimits(options.requiredUlimits); + if (options.keepOriginalRunningUntilFinalize && options.waitForSupervisor !== false) { + throw new Error( + "Keeping the original OpenShell supervisor running requires deferred supervisor finalization.", + ); + } const containerIds = findOpenShellDockerSandboxContainerIds(options.sandboxName, deps); const oldContainerId = containerIds[0]; if (!oldContainerId) { @@ -284,21 +293,24 @@ export function recreateOpenShellDockerSandboxContainer( suppressOutput: true, timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, }; - const stopResult = d.dockerStop(oldContainerId, { - ...containerMutationOptions, - timeout: DOCKER_GPU_PATCH_STOP_TIMEOUT_MS, - }); - if (!hasZeroDockerExitStatus(stopResult)) { - context.rolledBack = hasZeroDockerExitStatus( - d.dockerStart(oldContainerId, containerMutationOptions), - ); - throw new Error( - `Could not stop original sandbox container: ${resultText(stopResult)}; ${ - context.rolledBack - ? "original sandbox container confirmed running" - : "restart failed; original sandbox container may be stopped" - }`, - ); + const backupWasRunning = options.keepOriginalRunningUntilFinalize === true; + if (!backupWasRunning) { + const stopResult = d.dockerStop(oldContainerId, { + ...containerMutationOptions, + timeout: DOCKER_GPU_PATCH_STOP_TIMEOUT_MS, + }); + if (!hasZeroDockerExitStatus(stopResult)) { + context.rolledBack = hasZeroDockerExitStatus( + d.dockerStart(oldContainerId, containerMutationOptions), + ); + throw new Error( + `Could not stop original sandbox container: ${resultText(stopResult)}; ${ + context.rolledBack + ? "original sandbox container confirmed running" + : "restart failed; original sandbox container may be stopped" + }`, + ); + } } const renameResult = d.dockerRename( oldContainerId, @@ -307,9 +319,9 @@ export function recreateOpenShellDockerSandboxContainer( ); if (!hasZeroDockerExitStatus(renameResult)) { d.dockerRename(backupContainerName, originalName, containerMutationOptions); - const restarted = hasZeroDockerExitStatus( - d.dockerStart(oldContainerId, containerMutationOptions), - ); + const restarted = + backupWasRunning || + hasZeroDockerExitStatus(d.dockerStart(oldContainerId, containerMutationOptions)); let originalNameRestored = false; try { originalNameRestored = @@ -334,7 +346,7 @@ export function recreateOpenShellDockerSandboxContainer( }); if (!hasZeroDockerExitStatus(runResult)) { context.rolledBack = restoreDockerGpuPatchBackupAfterRecreateFailure( - { newContainerId: originalName, backupContainerName, originalName }, + { newContainerId: originalName, backupContainerName, originalName, backupWasRunning }, deps, ); const containerDescription = @@ -360,7 +372,7 @@ export function recreateOpenShellDockerSandboxContainer( ); if (!newContainerId) { context.rolledBack = restoreDockerGpuPatchBackupAfterRecreateFailure( - { newContainerId: originalName, backupContainerName, originalName }, + { newContainerId: originalName, backupContainerName, originalName, backupWasRunning }, deps, ); const containerDescription = @@ -384,6 +396,7 @@ export function recreateOpenShellDockerSandboxContainer( originalName, backupContainerName, mode: selectedMode, + backupWasRunning, backupRemoved, }); if (options.waitForSupervisor === false) return result(false); diff --git a/src/lib/onboard/docker-gpu-patch-rollback.ts b/src/lib/onboard/docker-gpu-patch-rollback.ts index 81532529503..a18ae91febc 100644 --- a/src/lib/onboard/docker-gpu-patch-rollback.ts +++ b/src/lib/onboard/docker-gpu-patch-rollback.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { + dockerForceRm as defaultDockerForceRm, dockerRename as defaultDockerRename, dockerRm as defaultDockerRm, dockerStart as defaultDockerStart, @@ -26,6 +27,7 @@ type DockerRenameFn = ( ) => DockerRunResult; export type ResolvedDockerGpuPatchRollbackDeps = { + dockerForceRm: DockerContainerFn; dockerStop: DockerContainerFn; dockerRm: DockerContainerFn; dockerRename: DockerRenameFn; @@ -36,6 +38,7 @@ export function resolveDockerGpuPatchRollbackDeps( deps: DockerGpuPatchDeps, ): ResolvedDockerGpuPatchRollbackDeps { return { + dockerForceRm: deps.dockerForceRm ?? defaultDockerForceRm, dockerStop: deps.dockerStop ?? defaultDockerStop, dockerRm: deps.dockerRm ?? defaultDockerRm, dockerRename: deps.dockerRename ?? defaultDockerRename, @@ -44,7 +47,12 @@ export function resolveDockerGpuPatchRollbackDeps( } export function rollbackToBackupContainer( - refs: { newContainerId: string; backupContainerName: string; originalName: string }, + refs: { + newContainerId: string; + backupContainerName: string; + originalName: string; + backupWasRunning?: boolean; + }, deps: ResolvedDockerGpuPatchRollbackDeps, ): boolean { const containerOpts = { @@ -56,13 +64,19 @@ export function rollbackToBackupContainer( deps.dockerRm(refs.newContainerId, containerOpts); const restored = deps.dockerRename(refs.backupContainerName, refs.originalName, containerOpts); if (!hasZeroDockerExitStatus(restored)) return false; + if (refs.backupWasRunning) return true; const started = deps.dockerStart(refs.originalName, containerOpts); return hasZeroDockerExitStatus(started); } /** Restore the original sandbox after `docker run` fails during GPU recreation. */ export function restoreDockerGpuPatchBackupAfterRecreateFailure( - refs: { newContainerId: string; backupContainerName: string; originalName: string }, + refs: { + newContainerId: string; + backupContainerName: string; + originalName: string; + backupWasRunning?: boolean; + }, deps: DockerGpuPatchDeps = {}, ): boolean { return rollbackToBackupContainer(refs, resolveDockerGpuPatchRollbackDeps(deps)); diff --git a/src/lib/onboard/docker-gpu-patch-types.ts b/src/lib/onboard/docker-gpu-patch-types.ts index 32be0afe46f..4e5390134e5 100644 --- a/src/lib/onboard/docker-gpu-patch-types.ts +++ b/src/lib/onboard/docker-gpu-patch-types.ts @@ -27,6 +27,7 @@ export type DockerGpuPatchDeps = { dockerRun?: DockerRunFn; dockerRunDetached?: DockerRunFn; dockerRename?: DockerRenameFn; + dockerForceRm?: DockerContainerFn; dockerRm?: DockerContainerFn; dockerStart?: DockerContainerFn; dockerStop?: DockerContainerFn; @@ -91,6 +92,9 @@ export type DockerGpuPatchResult = { originalName: string; backupContainerName: string; mode: DockerGpuPatchMode; + // True when deferred startup-command recovery kept the original OpenShell + // supervisor running while the replacement established managed health. + backupWasRunning?: boolean; // True when the patch path also confirmed supervisor reconnect AND removed // the backup container. False when the caller deferred the reconnect wait // (via `waitForSupervisor: false`); the backup is still in place and the diff --git a/src/lib/onboard/docker-startup-command-patch.test.ts b/src/lib/onboard/docker-startup-command-patch.test.ts index 71c5c9d12c2..d9e3567c7d9 100644 --- a/src/lib/onboard/docker-startup-command-patch.test.ts +++ b/src/lib/onboard/docker-startup-command-patch.test.ts @@ -43,6 +43,61 @@ function inspectFixture(): DockerContainerInspect { } describe("Docker startup-command patch", () => { + it("keeps the registered supervisor running until deferred recovery finalizes", () => { + const dockerCapture = vi.fn((args: readonly string[]) => + args[0] === "ps" + ? "old-container-id\n" + : args[0] === "inspect" + ? JSON.stringify([inspectFixture()]) + : "", + ); + const dockerStop = vi.fn(() => ({ status: 0 })); + const dockerRename = vi.fn(() => ({ status: 0 })); + const dockerRunDetached = vi.fn(() => ({ status: 0, stdout: "new-container-id\n" })); + + const result = recreateStartupCommandForTest( + { + sandboxName: "alpha", + keepOriginalRunningUntilFinalize: true, + waitForSupervisor: false, + openshellSandboxCommand: ["env", "nemoclaw-start"], + }, + { + dockerCapture, + dockerRunDetached, + dockerRename, + dockerStop, + now: () => new Date("2026-07-10T00:00:00Z"), + }, + ); + + expect(result).toMatchObject({ + newContainerId: "new-container-id", + backupWasRunning: true, + backupRemoved: false, + }); + expect(dockerStop).not.toHaveBeenCalled(); + expect(dockerRename.mock.invocationCallOrder[0]).toBeLessThan( + dockerRunDetached.mock.invocationCallOrder[0] ?? Number.POSITIVE_INFINITY, + ); + }); + + it("requires deferred finalization when preserving the registered supervisor", () => { + const dockerCapture = vi.fn(); + + expect(() => + recreateStartupCommandForTest( + { + sandboxName: "alpha", + keepOriginalRunningUntilFinalize: true, + openshellSandboxCommand: ["env", "nemoclaw-start"], + }, + { dockerCapture }, + ), + ).toThrow(/requires deferred supervisor finalization/); + expect(dockerCapture).not.toHaveBeenCalled(); + }); + it("persists the startup command without adding GPU-only container privileges", () => { const dockerCaptureOutput: Record = { ps: "old-container-id\n", diff --git a/src/lib/onboard/docker-startup-command-patch.ts b/src/lib/onboard/docker-startup-command-patch.ts index 2b5b6f761ef..6541152e2b7 100644 --- a/src/lib/onboard/docker-startup-command-patch.ts +++ b/src/lib/onboard/docker-startup-command-patch.ts @@ -15,6 +15,7 @@ export function recreateOpenShellDockerSandboxWithStartupCommand( sandboxName: string; timeoutSecs?: number; waitForSupervisor?: boolean; + keepOriginalRunningUntilFinalize?: boolean; openshellSandboxCommand: readonly string[]; requiredUlimits?: readonly import("./docker-gpu-patch-types").DockerUlimit[] | null; expectedOldContainerId?: string | null; From 2edb3873a1c0eb84b321f10d6442234cb5730def Mon Sep 17 00:00:00 2001 From: San Dang Date: Thu, 30 Jul 2026 20:57:52 +0530 Subject: [PATCH 18/30] docs(recovery): clarify handoff state Signed-off-by: San Dang --- src/lib/onboard/docker-gpu-patch-types.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/onboard/docker-gpu-patch-types.ts b/src/lib/onboard/docker-gpu-patch-types.ts index 4e5390134e5..93ad0c20ec8 100644 --- a/src/lib/onboard/docker-gpu-patch-types.ts +++ b/src/lib/onboard/docker-gpu-patch-types.ts @@ -92,8 +92,8 @@ export type DockerGpuPatchResult = { originalName: string; backupContainerName: string; mode: DockerGpuPatchMode; - // True when deferred startup-command recovery kept the original OpenShell - // supervisor running while the replacement established managed health. + // True when recovery did not stop the original OpenShell supervisor before + // creating the replacement. The caller passes replacement health separately. backupWasRunning?: boolean; // True when the patch path also confirmed supervisor reconnect AND removed // the backup container. False when the caller deferred the reconnect wait From b06f526a4937029b6ef3afc50c7536a2b5316968 Mon Sep 17 00:00:00 2001 From: San Dang Date: Thu, 30 Jul 2026 21:16:31 +0530 Subject: [PATCH 19/30] fix(recovery): defer handoff commit until ready Signed-off-by: San Dang --- src/lib/actions/sandbox/process-recovery.ts | 45 ++++++++++++------- ...ocess-recovery-supervisor-relaunch.test.ts | 15 ++++--- 2 files changed, 37 insertions(+), 23 deletions(-) diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index 9bc28614aee..fd462b0b5c0 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -1299,22 +1299,6 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( } return { checked: true, wasRunning: false, recovered: false, forwardRecovered: false }; } - if (relaunch) { - try { - const completion = relaunch.finalize(true); - if (!completion.backupRemoved && !quiet) { - console.error( - " Warning: the recovered sandbox is healthy, but its previous container backup could not be removed.", - ); - } - } catch { - if (!quiet) { - console.error( - " Warning: the recovered sandbox is healthy, but container transaction cleanup could not be confirmed.", - ); - } - } - } const readinessFailureDetail = relaunch ? (() => { const readinessOptions: RecreatedSandboxOpenShellReadyOptions = { @@ -1336,15 +1320,42 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( })() : null; if (readinessFailureDetail) { + let rolledBack = true; + try { + rolledBack = relaunch?.finalize(false).rolledBack ?? true; + } catch { + rolledBack = false; + } + if (!rolledBack && !quiet) { + console.error( + " Automatic rollback of the previous sandbox container failed; inspect Docker state before retrying.", + ); + } return { checked: true, wasRunning: false, - recovered: true, + recovered: false, forwardRecovered: false, forwardRecoveryFailed: true, forwardRecoveryFailureDetail: readinessFailureDetail, }; } + if (relaunch) { + try { + const completion = relaunch.finalize(true); + if (!completion.backupRemoved && !quiet) { + console.error( + " Warning: the recovered sandbox is healthy, but its previous container backup could not be removed.", + ); + } + } catch { + if (!quiet) { + console.error( + " Warning: the recovered sandbox is healthy, but container transaction cleanup could not be confirmed.", + ); + } + } + } const mcpRefusal = processRecoveryMcpReconciliationRefusal(sandboxName, false); if (mcpRefusal) return mcpRefusal; const forwardRecovered = ensureSandboxPortForward(sandboxName, { diff --git a/test/process-recovery-supervisor-relaunch.test.ts b/test/process-recovery-supervisor-relaunch.test.ts index cb0fe7ac52e..6fe0db754c5 100644 --- a/test/process-recovery-supervisor-relaunch.test.ts +++ b/test/process-recovery-supervisor-relaunch.test.ts @@ -302,6 +302,9 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { expect.objectContaining({ ignoreError: true }), ); expect(finalize).toHaveBeenCalledWith(true); + expect(captureOpenshell.mock.invocationCallOrder[0]).toBeLessThan( + finalize.mock.invocationCallOrder[0] ?? Number.POSITIVE_INFINITY, + ); expect(runOpenshell).toHaveBeenCalledWith( ["forward", "start", "--background", "18789", "busy-recovered-box"], expect.objectContaining({ ignoreError: true }), @@ -341,13 +344,13 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { expect(result).toMatchObject({ checked: true, wasRunning: false, - recovered: true, + recovered: false, forwardRecovered: false, forwardRecoveryFailed: true, forwardRecoveryFailureDetail: expect.stringContaining("did not become ready in OpenShell"), }); expect(finalize).toHaveBeenCalledOnce(); - expect(finalize).toHaveBeenCalledWith(true); + expect(finalize).toHaveBeenCalledWith(false); expect(waitForRecreatedSandboxOpenShellReadyImpl).toHaveBeenCalledWith( "unready-box", expect.objectContaining({ beforeProbe: expect.any(Function), timeoutSeconds: 180 }), @@ -402,7 +405,7 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { expect(result).toMatchObject({ checked: true, wasRunning: false, - recovered: true, + recovered: false, forwardRecovered: false, forwardRecoveryFailed: true, forwardRecoveryFailureDetail: expect.stringContaining( @@ -410,7 +413,7 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { ), }); expect(captureOpenshell).toHaveBeenCalled(); - expect(finalize).toHaveBeenCalledWith(true); + expect(finalize).toHaveBeenCalledWith(false); expect(runOpenshell).not.toHaveBeenCalled(); }); @@ -453,12 +456,12 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { expect(result).toMatchObject({ checked: true, wasRunning: false, - recovered: true, + recovered: false, forwardRecovered: false, forwardRecoveryFailed: true, forwardRecoveryFailureDetail: expect.stringContaining("failed the managed health guard"), }); - expect(finalize).toHaveBeenCalledWith(true); + expect(finalize).toHaveBeenCalledWith(false); expect(captureOpenshell).not.toHaveBeenCalled(); }); From 9de564f4c3a75ad3b72e3baa7f1eb3a1440a5ad3 Mon Sep 17 00:00:00 2001 From: San Dang Date: Thu, 30 Jul 2026 22:24:53 +0530 Subject: [PATCH 20/30] fix(recovery): preserve sandbox state across restart Signed-off-by: San Dang --- .../sandbox/supervisor-relaunch.test.ts | 1 + .../actions/sandbox/supervisor-relaunch.ts | 1 + .../onboard/docker-gpu-patch-finalize.test.ts | 51 +++++++++++ src/lib/onboard/docker-gpu-patch-finalize.ts | 4 + src/lib/onboard/docker-gpu-patch-recreate.ts | 61 ++++++++++++- src/lib/onboard/docker-gpu-patch-types.ts | 6 ++ .../docker-startup-command-patch.test.ts | 87 ++++++++++++++++++- .../onboard/docker-startup-command-patch.ts | 1 + src/lib/sandbox/privileged-exec.test.ts | 32 +++++++ src/lib/sandbox/privileged-exec.ts | 20 ++++- 10 files changed, 257 insertions(+), 7 deletions(-) diff --git a/src/lib/actions/sandbox/supervisor-relaunch.test.ts b/src/lib/actions/sandbox/supervisor-relaunch.test.ts index 50937047195..9b2804a073a 100644 --- a/src/lib/actions/sandbox/supervisor-relaunch.test.ts +++ b/src/lib/actions/sandbox/supervisor-relaunch.test.ts @@ -115,6 +115,7 @@ describe("relaunchManagedSupervisorSession", () => { sandboxName: "alpha", expectedOldContainerId: "old-container-id", keepOriginalRunningUntilFinalize: true, + preserveWritableLayer: true, waitForSupervisor: false, }); const serialized = options?.openshellSandboxCommand.join(" ") ?? ""; diff --git a/src/lib/actions/sandbox/supervisor-relaunch.ts b/src/lib/actions/sandbox/supervisor-relaunch.ts index fecf06465b7..35d4df46b45 100644 --- a/src/lib/actions/sandbox/supervisor-relaunch.ts +++ b/src/lib/actions/sandbox/supervisor-relaunch.ts @@ -142,6 +142,7 @@ export function relaunchManagedSupervisorSession( openshellSandboxCommand: startupCommand, expectedOldContainerId: containerId, keepOriginalRunningUntilFinalize: true, + preserveWritableLayer: true, waitForSupervisor: false, }); let completed: { supervisorReady: boolean; outcome: DockerGpuPatchFinalizeOutcome } | null = diff --git a/src/lib/onboard/docker-gpu-patch-finalize.test.ts b/src/lib/onboard/docker-gpu-patch-finalize.test.ts index 57a5bb5e882..97d7580b3e7 100644 --- a/src/lib/onboard/docker-gpu-patch-finalize.test.ts +++ b/src/lib/onboard/docker-gpu-patch-finalize.test.ts @@ -110,6 +110,57 @@ describe("finalizeDockerGpuPatchBackup", () => { expect(dockerStart).not.toHaveBeenCalled(); }); + it("removes the writable-layer snapshot after a successful rollback", () => { + const snapshotImageId = `sha256:${"d".repeat(64)}`; + const dockerRmi = vi.fn(() => ({ status: 0 })); + const outcome = finalizeDockerGpuPatchBackup( + { + result: { + ...deferredCreateResult(), + backupWasRunning: true, + snapshotImageId, + }, + supervisorReady: false, + }, + { + dockerStop: vi.fn(() => ({ status: 0 })), + dockerRm: vi.fn(() => ({ status: 0 })), + dockerRename: vi.fn(() => ({ status: 0 })), + dockerStart: vi.fn(() => ({ status: 0 })), + dockerRmi, + }, + ); + + expect(outcome).toEqual({ backupRemoved: false, rolledBack: true }); + expect(dockerRmi).toHaveBeenCalledWith( + snapshotImageId, + expect.objectContaining({ ignoreError: true }), + ); + }); + + it("keeps the writable-layer snapshot when rollback does not restore the original", () => { + const dockerRmi = vi.fn(() => ({ status: 0 })); + const outcome = finalizeDockerGpuPatchBackup( + { + result: { + ...deferredCreateResult(), + snapshotImageId: `sha256:${"d".repeat(64)}`, + }, + supervisorReady: false, + }, + { + dockerStop: vi.fn(() => ({ status: 0 })), + dockerRm: vi.fn(() => ({ status: 0 })), + dockerRename: vi.fn(() => ({ status: 1 })), + dockerStart: vi.fn(() => ({ status: 0 })), + dockerRmi, + }, + ); + + expect(outcome).toEqual({ backupRemoved: false, rolledBack: false }); + expect(dockerRmi).not.toHaveBeenCalled(); + }); + it("reports rolledBack=false when restoring the backup fails", () => { const dockerStop = vi.fn(() => ({ status: 0 })); const dockerRm = vi.fn((_name: string) => ({ status: 0 })); diff --git a/src/lib/onboard/docker-gpu-patch-finalize.ts b/src/lib/onboard/docker-gpu-patch-finalize.ts index e3c0dc65ff6..a59ad9889ec 100644 --- a/src/lib/onboard/docker-gpu-patch-finalize.ts +++ b/src/lib/onboard/docker-gpu-patch-finalize.ts @@ -23,6 +23,7 @@ // and delete this module along with its callers in docker-gpu-patch.ts and // docker-gpu-sandbox-create.ts. +import { dockerRmi } from "../adapters/docker/image"; import { hasZeroDockerExitStatus } from "./docker-command-result"; import { DOCKER_GPU_PATCH_TIMEOUT_MS } from "./docker-gpu-patch-constants"; import { @@ -79,6 +80,9 @@ export function finalizeDockerGpuPatchBackup( }, resolved, ); + if (rolledBack && options.result.snapshotImageId) { + (deps.dockerRmi ?? dockerRmi)(options.result.snapshotImageId, containerOpts); + } return { backupRemoved: false, rolledBack }; } diff --git a/src/lib/onboard/docker-gpu-patch-recreate.ts b/src/lib/onboard/docker-gpu-patch-recreate.ts index 7587437378a..d5b51f1cbf7 100644 --- a/src/lib/onboard/docker-gpu-patch-recreate.ts +++ b/src/lib/onboard/docker-gpu-patch-recreate.ts @@ -6,6 +6,7 @@ import { dockerForceRm, dockerRename, dockerRm, + dockerRmi, dockerRun, dockerRunDetached, dockerStart, @@ -54,6 +55,7 @@ type RecreateDeps = Required< | "dockerRunDetached" | "dockerRename" | "dockerRm" + | "dockerRmi" | "dockerStart" | "dockerStop" | "sleep" @@ -73,6 +75,7 @@ function recreateDeps(deps: DockerGpuPatchDeps): RecreateDeps { dockerRunDetached, dockerRename, dockerRm, + dockerRmi, dockerStart, dockerStop, sleep: (seconds: number) => { @@ -159,6 +162,7 @@ export function recreateOpenShellDockerSandboxContainer( timeoutSecs?: number; waitForSupervisor?: boolean; keepOriginalRunningUntilFinalize?: boolean; + preserveWritableLayer?: boolean; openshellSandboxCommand?: readonly string[] | null; requiredUlimits?: readonly import("./docker-gpu-patch-types").DockerUlimit[] | null; expectedOldContainerId?: string | null; @@ -173,6 +177,8 @@ export function recreateOpenShellDockerSandboxContainer( sandboxName: options.sandboxName, modeAttempts: [], }; + let snapshotImageId: string | null = null; + let retainSnapshotImage = false; try { validateRequiredDockerUlimits(options.requiredUlimits); if (options.keepOriginalRunningUntilFinalize && options.waitForSupervisor !== false) { @@ -180,6 +186,11 @@ export function recreateOpenShellDockerSandboxContainer( "Keeping the original OpenShell supervisor running requires deferred supervisor finalization.", ); } + if (options.preserveWritableLayer && options.openshellSandboxCommand == null) { + throw new Error( + "Preserving a sandbox writable layer is supported only for startup-command recreation.", + ); + } const containerIds = findOpenShellDockerSandboxContainerIds(options.sandboxName, deps); const oldContainerId = containerIds[0]; if (!oldContainerId) { @@ -286,13 +297,45 @@ export function recreateOpenShellDockerSandboxContainer( ); } } - const cloneArgs = buildDockerGpuCloneRunArgs(inspect, selection.mode, cloneOptions); - const containerMutationOptions = { ignoreError: true, suppressOutput: true, timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, }; + if (options.preserveWritableLayer) { + // Docker commit pauses the source container by default. Do not pass the + // deprecated --pause flag: newer Docker clients print its warning into + // the captured output alongside the committed image ID. + const commitResult = d.dockerRun(["commit", oldContainerId], { + ...containerMutationOptions, + timeout: Math.max( + DOCKER_GPU_PATCH_TIMEOUT_MS, + (options.timeoutSecs ?? DOCKER_GPU_PATCH_WAIT_SECS) * 1000, + ), + }); + if (!hasZeroDockerExitStatus(commitResult)) { + throw new Error( + `Could not snapshot the sandbox writable layer before recovery: ${resultText( + commitResult, + )}`, + ); + } + const committedImages = String(commitResult.stdout || "") + .split(/\r?\n/) + .map((line) => line.trim()) + .filter((line) => /^sha256:[0-9a-f]{64}$/i.test(line)); + if (committedImages.length !== 1) { + throw new Error( + "Docker committed the sandbox writable layer but did not return a valid immutable image ID.", + ); + } + const [committedImage] = committedImages; + snapshotImageId = committedImage; + context.snapshotImageId = committedImage; + cloneOptions.image = committedImage; + } + const cloneArgs = buildDockerGpuCloneRunArgs(inspect, selection.mode, cloneOptions); + const backupWasRunning = options.keepOriginalRunningUntilFinalize === true; if (!backupWasRunning) { const stopResult = d.dockerStop(oldContainerId, { @@ -397,9 +440,13 @@ export function recreateOpenShellDockerSandboxContainer( backupContainerName, mode: selectedMode, backupWasRunning, + ...(snapshotImageId ? { snapshotImageId } : {}), backupRemoved, }); - if (options.waitForSupervisor === false) return result(false); + if (options.waitForSupervisor === false) { + retainSnapshotImage = true; + return result(false); + } const execReady = waitForOpenShellSupervisorReconnect( options.sandboxName, @@ -415,8 +462,16 @@ export function recreateOpenShellDockerSandboxContainer( context.rolledBack = reconcile.rolledBack; throw reconcile.error; } + retainSnapshotImage = true; return result(reconcile.backupRemoved); } catch (error) { + if (snapshotImageId && !retainSnapshotImage) { + d.dockerRmi(snapshotImageId, { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }); + } throw decoratePatchError(error instanceof Error ? error : new Error(String(error)), context); } } diff --git a/src/lib/onboard/docker-gpu-patch-types.ts b/src/lib/onboard/docker-gpu-patch-types.ts index 93ad0c20ec8..e46b04e55d5 100644 --- a/src/lib/onboard/docker-gpu-patch-types.ts +++ b/src/lib/onboard/docker-gpu-patch-types.ts @@ -27,6 +27,7 @@ export type DockerGpuPatchDeps = { dockerRun?: DockerRunFn; dockerRunDetached?: DockerRunFn; dockerRename?: DockerRenameFn; + dockerRmi?: DockerContainerFn; dockerForceRm?: DockerContainerFn; dockerRm?: DockerContainerFn; dockerStart?: DockerContainerFn; @@ -80,6 +81,7 @@ export type DockerGpuPatchFailureContext = { oldContainerId?: string | null; newContainerId?: string | null; backupContainerName?: string | null; + snapshotImageId?: string | null; selectedMode?: DockerGpuPatchMode | null; modeAttempts?: DockerGpuPatchModeAttempt[]; rolledBack?: boolean; @@ -95,6 +97,10 @@ export type DockerGpuPatchResult = { // True when recovery did not stop the original OpenShell supervisor before // creating the replacement. The caller passes replacement health separately. backupWasRunning?: boolean; + // Immutable image created from the original container's writable layer. + // The active replacement references it after a successful handoff; a + // rollback removes it after restoring the original container. + snapshotImageId?: string; // True when the patch path also confirmed supervisor reconnect AND removed // the backup container. False when the caller deferred the reconnect wait // (via `waitForSupervisor: false`); the backup is still in place and the diff --git a/src/lib/onboard/docker-startup-command-patch.test.ts b/src/lib/onboard/docker-startup-command-patch.test.ts index d9e3567c7d9..6ad4dfc7134 100644 --- a/src/lib/onboard/docker-startup-command-patch.test.ts +++ b/src/lib/onboard/docker-startup-command-patch.test.ts @@ -53,7 +53,10 @@ describe("Docker startup-command patch", () => { ); const dockerStop = vi.fn(() => ({ status: 0 })); const dockerRename = vi.fn(() => ({ status: 0 })); - const dockerRunDetached = vi.fn(() => ({ status: 0, stdout: "new-container-id\n" })); + const dockerRunDetached = vi.fn((_args: readonly string[]) => ({ + status: 0, + stdout: "new-container-id\n", + })); const result = recreateStartupCommandForTest( { @@ -82,6 +85,88 @@ describe("Docker startup-command patch", () => { ); }); + it("starts recovery from a paused snapshot of the sandbox writable layer", () => { + const snapshotImageId = `sha256:${"d".repeat(64)}`; + const dockerCapture = vi.fn((args: readonly string[]) => + args[0] === "ps" + ? "old-container-id\n" + : args[0] === "inspect" + ? JSON.stringify([inspectFixture()]) + : "", + ); + const dockerRun = vi.fn(() => ({ + status: 0, + stdout: `Docker informational output\n${snapshotImageId}\n`, + })); + const dockerRename = vi.fn(() => ({ status: 0 })); + const dockerRunDetached = vi.fn((_args: readonly string[]) => ({ + status: 0, + stdout: "new-container-id\n", + })); + + const result = recreateStartupCommandForTest( + { + sandboxName: "alpha", + keepOriginalRunningUntilFinalize: true, + preserveWritableLayer: true, + waitForSupervisor: false, + openshellSandboxCommand: ["env", "nemoclaw-start"], + }, + { + dockerCapture, + dockerRun, + dockerRunDetached, + dockerRename, + now: () => new Date("2026-07-10T00:00:00Z"), + }, + ); + + expect(dockerRun).toHaveBeenCalledWith( + ["commit", "old-container-id"], + expect.objectContaining({ ignoreError: true }), + ); + expect(dockerRun.mock.invocationCallOrder[0]).toBeLessThan( + dockerRename.mock.invocationCallOrder[0] ?? Number.POSITIVE_INFINITY, + ); + expect(dockerRename.mock.invocationCallOrder[0]).toBeLessThan( + dockerRunDetached.mock.invocationCallOrder[0] ?? Number.POSITIVE_INFINITY, + ); + const cloneArgs = dockerRunDetached.mock.calls[0]?.[0] ?? []; + expect(cloneArgs).toContain(snapshotImageId); + expect(cloneArgs).not.toContain(`sha256:${"c".repeat(64)}`); + expect(result.snapshotImageId).toBe(snapshotImageId); + }); + + it("does not mutate the container when its writable-layer snapshot fails", () => { + const dockerRename = vi.fn(() => ({ status: 0 })); + const dockerRunDetached = vi.fn(() => ({ status: 0, stdout: "new-container-id\n" })); + + expect(() => + recreateStartupCommandForTest( + { + sandboxName: "alpha", + preserveWritableLayer: true, + waitForSupervisor: false, + openshellSandboxCommand: ["env", "nemoclaw-start"], + }, + { + dockerCapture: vi.fn((args: readonly string[]) => + args[0] === "ps" + ? "old-container-id\n" + : args[0] === "inspect" + ? JSON.stringify([inspectFixture()]) + : "", + ), + dockerRun: vi.fn(() => ({ status: 1, stderr: "commit failed" })), + dockerRunDetached, + dockerRename, + }, + ), + ).toThrow(/Could not snapshot the sandbox writable layer.*commit failed/); + expect(dockerRename).not.toHaveBeenCalled(); + expect(dockerRunDetached).not.toHaveBeenCalled(); + }); + it("requires deferred finalization when preserving the registered supervisor", () => { const dockerCapture = vi.fn(); diff --git a/src/lib/onboard/docker-startup-command-patch.ts b/src/lib/onboard/docker-startup-command-patch.ts index 6541152e2b7..63826ce2a3e 100644 --- a/src/lib/onboard/docker-startup-command-patch.ts +++ b/src/lib/onboard/docker-startup-command-patch.ts @@ -16,6 +16,7 @@ export function recreateOpenShellDockerSandboxWithStartupCommand( timeoutSecs?: number; waitForSupervisor?: boolean; keepOriginalRunningUntilFinalize?: boolean; + preserveWritableLayer?: boolean; openshellSandboxCommand: readonly string[]; requiredUlimits?: readonly import("./docker-gpu-patch-types").DockerUlimit[] | null; expectedOldContainerId?: string | null; diff --git a/src/lib/sandbox/privileged-exec.test.ts b/src/lib/sandbox/privileged-exec.test.ts index 2598fbd6b48..1e5e14ba5bd 100644 --- a/src/lib/sandbox/privileged-exec.test.ts +++ b/src/lib/sandbox/privileged-exec.test.ts @@ -217,6 +217,38 @@ describe("privileged sandbox exec routing", () => { ); }); + it("selects the pinned container while a transaction backup is still running", () => { + withPrivilegedExecMocks( + { + getSandbox: () => ({ name: "alpha", openshellDriver: "docker" }), + listSandboxes: () => ({ sandboxes: [{ name: "alpha" }], defaultSandbox: "alpha" }), + dockerCapture: () => + [ + "replacement-container-id\topenshell-alpha", + "original-container-id\topenshell-alpha-nemoclaw-backup", + ].join("\n"), + }, + ({ privilegedSandboxExecArgv }) => { + expect( + privilegedSandboxExecArgv( + "alpha", + ["/trusted/control"], + false, + true, + "replacement-container-id", + ), + ).toEqual( + expect.arrayContaining([ + "--user", + "root", + "replacement-container-id", + "/trusted/control", + ]), + ); + }, + ); + }); + it("refuses privileged execution when the pinned container identity is empty", () => { withPrivilegedExecMocks( { diff --git a/src/lib/sandbox/privileged-exec.ts b/src/lib/sandbox/privileged-exec.ts index c45f59a60a4..5588c86881f 100644 --- a/src/lib/sandbox/privileged-exec.ts +++ b/src/lib/sandbox/privileged-exec.ts @@ -108,6 +108,7 @@ function selectDirectSandboxContainer( sandboxName: string, labeledContainerRows: string, registeredNames: readonly string[] = [sandboxName], + expectedContainerId?: string, ): string | null { const names = Array.from(new Set([...registeredNames, sandboxName])).sort( (a, b) => b.length - a.length || a.localeCompare(b), @@ -125,6 +126,16 @@ function selectDirectSandboxContainer( "refusing lifecycle execution.", ); } + if (expectedContainerId !== undefined) { + const expected = candidates.filter(({ id }) => id === expectedContainerId); + if (expected.length !== 1) { + throw new Error( + `OpenShell container identity changed for sandbox '${sandboxName}'; ` + + "refusing privileged execution against a different container.", + ); + } + return expected[0].id; + } if (candidates.length > 1) { throw new Error( `Multiple running OpenShell containers are labeled for sandbox '${sandboxName}'; ` + @@ -138,7 +149,10 @@ function expectedDirectContainerPattern(sandboxName: string): string { return `openshell-${sandboxName} or openshell-${sandboxName}-*`; } -function findDirectSandboxContainer(sandboxName: string): string | null { +function findDirectSandboxContainer( + sandboxName: string, + expectedContainerId?: string, +): string | null { const names = registeredSandboxNames(sandboxName); let output: string; try { @@ -162,7 +176,7 @@ function findDirectSandboxContainer(sandboxName: string): string | null { { cause: error }, ); } - return selectDirectSandboxContainer(sandboxName, output, names); + return selectDirectSandboxContainer(sandboxName, output, names, expectedContainerId); } function missingDirectContainerError(sandboxName: string, driver: string | null): Error { @@ -218,7 +232,7 @@ function privilegedSandboxExecArgv( // Docker/direct-container is the only supported privileged mutation path. // Try it even when older registry entries do not record a driver, then fail // clearly if no matching sandbox container is running. - const container = findDirectSandboxContainer(sandboxName); + const container = findDirectSandboxContainer(sandboxName, expectedContainerId); if (container) { if (expectedContainerId !== undefined && container !== expectedContainerId) { throw new Error( From 3ac83154e410b45b4cb3a2c37a541e3d9d1b79fd Mon Sep 17 00:00:00 2001 From: San Dang Date: Thu, 30 Jul 2026 22:32:01 +0530 Subject: [PATCH 21/30] refactor(recovery): trim handoff scaffolding Signed-off-by: San Dang --- .../sandbox/supervisor-relaunch.test.ts | 1 - .../actions/sandbox/supervisor-relaunch.ts | 1 - .../onboard/docker-gpu-patch-finalize.test.ts | 61 +++---------------- src/lib/onboard/docker-gpu-patch-finalize.ts | 6 +- src/lib/onboard/docker-gpu-patch-recreate.ts | 23 ++----- src/lib/onboard/docker-gpu-patch-types.ts | 2 - .../docker-startup-command-patch.test.ts | 53 +++------------- .../onboard/docker-startup-command-patch.ts | 1 - test/e2e/live/sandbox-survival.test.ts | 6 +- 9 files changed, 28 insertions(+), 126 deletions(-) diff --git a/src/lib/actions/sandbox/supervisor-relaunch.test.ts b/src/lib/actions/sandbox/supervisor-relaunch.test.ts index 9b2804a073a..50937047195 100644 --- a/src/lib/actions/sandbox/supervisor-relaunch.test.ts +++ b/src/lib/actions/sandbox/supervisor-relaunch.test.ts @@ -115,7 +115,6 @@ describe("relaunchManagedSupervisorSession", () => { sandboxName: "alpha", expectedOldContainerId: "old-container-id", keepOriginalRunningUntilFinalize: true, - preserveWritableLayer: true, waitForSupervisor: false, }); const serialized = options?.openshellSandboxCommand.join(" ") ?? ""; diff --git a/src/lib/actions/sandbox/supervisor-relaunch.ts b/src/lib/actions/sandbox/supervisor-relaunch.ts index 35d4df46b45..fecf06465b7 100644 --- a/src/lib/actions/sandbox/supervisor-relaunch.ts +++ b/src/lib/actions/sandbox/supervisor-relaunch.ts @@ -142,7 +142,6 @@ export function relaunchManagedSupervisorSession( openshellSandboxCommand: startupCommand, expectedOldContainerId: containerId, keepOriginalRunningUntilFinalize: true, - preserveWritableLayer: true, waitForSupervisor: false, }); let completed: { supervisorReady: boolean; outcome: DockerGpuPatchFinalizeOutcome } | null = diff --git a/src/lib/onboard/docker-gpu-patch-finalize.test.ts b/src/lib/onboard/docker-gpu-patch-finalize.test.ts index 97d7580b3e7..ee3b2eb97e3 100644 --- a/src/lib/onboard/docker-gpu-patch-finalize.test.ts +++ b/src/lib/onboard/docker-gpu-patch-finalize.test.ts @@ -85,16 +85,22 @@ describe("finalizeDockerGpuPatchBackup", () => { }); it("rolls back to the still-running original container without restarting it", () => { + const snapshotImageId = `sha256:${"d".repeat(64)}`; const dockerStop = vi.fn(() => ({ status: 0 })); const dockerRm = vi.fn((_name: string) => ({ status: 0 })); const dockerRename = vi.fn((_old: string, _next: string) => ({ status: 0 })); const dockerStart = vi.fn(() => ({ status: 0 })); + const dockerRun = vi.fn(() => ({ status: 0 })); const outcome = finalizeDockerGpuPatchBackup( { - result: { ...deferredCreateResult(), backupWasRunning: true }, + result: { + ...deferredCreateResult(), + backupWasRunning: true, + snapshotImageId, + }, supervisorReady: false, }, - { dockerStop, dockerRm, dockerRename, dockerStart }, + { dockerStop, dockerRm, dockerRename, dockerStart, dockerRun }, ); expect(outcome).toEqual({ backupRemoved: false, rolledBack: true }); @@ -108,59 +114,12 @@ describe("finalizeDockerGpuPatchBackup", () => { expect.objectContaining({ ignoreError: true }), ); expect(dockerStart).not.toHaveBeenCalled(); - }); - - it("removes the writable-layer snapshot after a successful rollback", () => { - const snapshotImageId = `sha256:${"d".repeat(64)}`; - const dockerRmi = vi.fn(() => ({ status: 0 })); - const outcome = finalizeDockerGpuPatchBackup( - { - result: { - ...deferredCreateResult(), - backupWasRunning: true, - snapshotImageId, - }, - supervisorReady: false, - }, - { - dockerStop: vi.fn(() => ({ status: 0 })), - dockerRm: vi.fn(() => ({ status: 0 })), - dockerRename: vi.fn(() => ({ status: 0 })), - dockerStart: vi.fn(() => ({ status: 0 })), - dockerRmi, - }, - ); - - expect(outcome).toEqual({ backupRemoved: false, rolledBack: true }); - expect(dockerRmi).toHaveBeenCalledWith( - snapshotImageId, + expect(dockerRun).toHaveBeenCalledWith( + ["rmi", snapshotImageId], expect.objectContaining({ ignoreError: true }), ); }); - it("keeps the writable-layer snapshot when rollback does not restore the original", () => { - const dockerRmi = vi.fn(() => ({ status: 0 })); - const outcome = finalizeDockerGpuPatchBackup( - { - result: { - ...deferredCreateResult(), - snapshotImageId: `sha256:${"d".repeat(64)}`, - }, - supervisorReady: false, - }, - { - dockerStop: vi.fn(() => ({ status: 0 })), - dockerRm: vi.fn(() => ({ status: 0 })), - dockerRename: vi.fn(() => ({ status: 1 })), - dockerStart: vi.fn(() => ({ status: 0 })), - dockerRmi, - }, - ); - - expect(outcome).toEqual({ backupRemoved: false, rolledBack: false }); - expect(dockerRmi).not.toHaveBeenCalled(); - }); - it("reports rolledBack=false when restoring the backup fails", () => { const dockerStop = vi.fn(() => ({ status: 0 })); const dockerRm = vi.fn((_name: string) => ({ status: 0 })); diff --git a/src/lib/onboard/docker-gpu-patch-finalize.ts b/src/lib/onboard/docker-gpu-patch-finalize.ts index a59ad9889ec..94757fa244a 100644 --- a/src/lib/onboard/docker-gpu-patch-finalize.ts +++ b/src/lib/onboard/docker-gpu-patch-finalize.ts @@ -23,7 +23,7 @@ // and delete this module along with its callers in docker-gpu-patch.ts and // docker-gpu-sandbox-create.ts. -import { dockerRmi } from "../adapters/docker/image"; +import { dockerRun } from "../adapters/docker/run"; import { hasZeroDockerExitStatus } from "./docker-command-result"; import { DOCKER_GPU_PATCH_TIMEOUT_MS } from "./docker-gpu-patch-constants"; import { @@ -80,8 +80,8 @@ export function finalizeDockerGpuPatchBackup( }, resolved, ); - if (rolledBack && options.result.snapshotImageId) { - (deps.dockerRmi ?? dockerRmi)(options.result.snapshotImageId, containerOpts); + if (options.result.snapshotImageId) { + (deps.dockerRun ?? dockerRun)(["rmi", options.result.snapshotImageId], containerOpts); } return { backupRemoved: false, rolledBack }; } diff --git a/src/lib/onboard/docker-gpu-patch-recreate.ts b/src/lib/onboard/docker-gpu-patch-recreate.ts index d5b51f1cbf7..f50ee238d3c 100644 --- a/src/lib/onboard/docker-gpu-patch-recreate.ts +++ b/src/lib/onboard/docker-gpu-patch-recreate.ts @@ -6,7 +6,6 @@ import { dockerForceRm, dockerRename, dockerRm, - dockerRmi, dockerRun, dockerRunDetached, dockerStart, @@ -55,7 +54,6 @@ type RecreateDeps = Required< | "dockerRunDetached" | "dockerRename" | "dockerRm" - | "dockerRmi" | "dockerStart" | "dockerStop" | "sleep" @@ -75,7 +73,6 @@ function recreateDeps(deps: DockerGpuPatchDeps): RecreateDeps { dockerRunDetached, dockerRename, dockerRm, - dockerRmi, dockerStart, dockerStop, sleep: (seconds: number) => { @@ -162,7 +159,6 @@ export function recreateOpenShellDockerSandboxContainer( timeoutSecs?: number; waitForSupervisor?: boolean; keepOriginalRunningUntilFinalize?: boolean; - preserveWritableLayer?: boolean; openshellSandboxCommand?: readonly string[] | null; requiredUlimits?: readonly import("./docker-gpu-patch-types").DockerUlimit[] | null; expectedOldContainerId?: string | null; @@ -178,7 +174,6 @@ export function recreateOpenShellDockerSandboxContainer( modeAttempts: [], }; let snapshotImageId: string | null = null; - let retainSnapshotImage = false; try { validateRequiredDockerUlimits(options.requiredUlimits); if (options.keepOriginalRunningUntilFinalize && options.waitForSupervisor !== false) { @@ -186,11 +181,6 @@ export function recreateOpenShellDockerSandboxContainer( "Keeping the original OpenShell supervisor running requires deferred supervisor finalization.", ); } - if (options.preserveWritableLayer && options.openshellSandboxCommand == null) { - throw new Error( - "Preserving a sandbox writable layer is supported only for startup-command recreation.", - ); - } const containerIds = findOpenShellDockerSandboxContainerIds(options.sandboxName, deps); const oldContainerId = containerIds[0]; if (!oldContainerId) { @@ -302,7 +292,7 @@ export function recreateOpenShellDockerSandboxContainer( suppressOutput: true, timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, }; - if (options.preserveWritableLayer) { + if (options.keepOriginalRunningUntilFinalize) { // Docker commit pauses the source container by default. Do not pass the // deprecated --pause flag: newer Docker clients print its warning into // the captured output alongside the committed image ID. @@ -331,7 +321,6 @@ export function recreateOpenShellDockerSandboxContainer( } const [committedImage] = committedImages; snapshotImageId = committedImage; - context.snapshotImageId = committedImage; cloneOptions.image = committedImage; } const cloneArgs = buildDockerGpuCloneRunArgs(inspect, selection.mode, cloneOptions); @@ -443,10 +432,7 @@ export function recreateOpenShellDockerSandboxContainer( ...(snapshotImageId ? { snapshotImageId } : {}), backupRemoved, }); - if (options.waitForSupervisor === false) { - retainSnapshotImage = true; - return result(false); - } + if (options.waitForSupervisor === false) return result(false); const execReady = waitForOpenShellSupervisorReconnect( options.sandboxName, @@ -462,11 +448,10 @@ export function recreateOpenShellDockerSandboxContainer( context.rolledBack = reconcile.rolledBack; throw reconcile.error; } - retainSnapshotImage = true; return result(reconcile.backupRemoved); } catch (error) { - if (snapshotImageId && !retainSnapshotImage) { - d.dockerRmi(snapshotImageId, { + if (snapshotImageId) { + d.dockerRun(["rmi", snapshotImageId], { ignoreError: true, suppressOutput: true, timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, diff --git a/src/lib/onboard/docker-gpu-patch-types.ts b/src/lib/onboard/docker-gpu-patch-types.ts index e46b04e55d5..bb9f0160a46 100644 --- a/src/lib/onboard/docker-gpu-patch-types.ts +++ b/src/lib/onboard/docker-gpu-patch-types.ts @@ -27,7 +27,6 @@ export type DockerGpuPatchDeps = { dockerRun?: DockerRunFn; dockerRunDetached?: DockerRunFn; dockerRename?: DockerRenameFn; - dockerRmi?: DockerContainerFn; dockerForceRm?: DockerContainerFn; dockerRm?: DockerContainerFn; dockerStart?: DockerContainerFn; @@ -81,7 +80,6 @@ export type DockerGpuPatchFailureContext = { oldContainerId?: string | null; newContainerId?: string | null; backupContainerName?: string | null; - snapshotImageId?: string | null; selectedMode?: DockerGpuPatchMode | null; modeAttempts?: DockerGpuPatchModeAttempt[]; rolledBack?: boolean; diff --git a/src/lib/onboard/docker-startup-command-patch.test.ts b/src/lib/onboard/docker-startup-command-patch.test.ts index 6ad4dfc7134..36580c7b634 100644 --- a/src/lib/onboard/docker-startup-command-patch.test.ts +++ b/src/lib/onboard/docker-startup-command-patch.test.ts @@ -43,7 +43,8 @@ function inspectFixture(): DockerContainerInspect { } describe("Docker startup-command patch", () => { - it("keeps the registered supervisor running until deferred recovery finalizes", () => { + it("preserves the registered supervisor and writable layer until recovery finalizes", () => { + const snapshotImageId = `sha256:${"d".repeat(64)}`; const dockerCapture = vi.fn((args: readonly string[]) => args[0] === "ps" ? "old-container-id\n" @@ -52,6 +53,10 @@ describe("Docker startup-command patch", () => { : "", ); const dockerStop = vi.fn(() => ({ status: 0 })); + const dockerRun = vi.fn(() => ({ + status: 0, + stdout: `Docker informational output\n${snapshotImageId}\n`, + })); const dockerRename = vi.fn(() => ({ status: 0 })); const dockerRunDetached = vi.fn((_args: readonly string[]) => ({ status: 0, @@ -67,6 +72,7 @@ describe("Docker startup-command patch", () => { }, { dockerCapture, + dockerRun, dockerRunDetached, dockerRename, dockerStop, @@ -77,50 +83,10 @@ describe("Docker startup-command patch", () => { expect(result).toMatchObject({ newContainerId: "new-container-id", backupWasRunning: true, + snapshotImageId, backupRemoved: false, }); expect(dockerStop).not.toHaveBeenCalled(); - expect(dockerRename.mock.invocationCallOrder[0]).toBeLessThan( - dockerRunDetached.mock.invocationCallOrder[0] ?? Number.POSITIVE_INFINITY, - ); - }); - - it("starts recovery from a paused snapshot of the sandbox writable layer", () => { - const snapshotImageId = `sha256:${"d".repeat(64)}`; - const dockerCapture = vi.fn((args: readonly string[]) => - args[0] === "ps" - ? "old-container-id\n" - : args[0] === "inspect" - ? JSON.stringify([inspectFixture()]) - : "", - ); - const dockerRun = vi.fn(() => ({ - status: 0, - stdout: `Docker informational output\n${snapshotImageId}\n`, - })); - const dockerRename = vi.fn(() => ({ status: 0 })); - const dockerRunDetached = vi.fn((_args: readonly string[]) => ({ - status: 0, - stdout: "new-container-id\n", - })); - - const result = recreateStartupCommandForTest( - { - sandboxName: "alpha", - keepOriginalRunningUntilFinalize: true, - preserveWritableLayer: true, - waitForSupervisor: false, - openshellSandboxCommand: ["env", "nemoclaw-start"], - }, - { - dockerCapture, - dockerRun, - dockerRunDetached, - dockerRename, - now: () => new Date("2026-07-10T00:00:00Z"), - }, - ); - expect(dockerRun).toHaveBeenCalledWith( ["commit", "old-container-id"], expect.objectContaining({ ignoreError: true }), @@ -134,7 +100,6 @@ describe("Docker startup-command patch", () => { const cloneArgs = dockerRunDetached.mock.calls[0]?.[0] ?? []; expect(cloneArgs).toContain(snapshotImageId); expect(cloneArgs).not.toContain(`sha256:${"c".repeat(64)}`); - expect(result.snapshotImageId).toBe(snapshotImageId); }); it("does not mutate the container when its writable-layer snapshot fails", () => { @@ -145,7 +110,7 @@ describe("Docker startup-command patch", () => { recreateStartupCommandForTest( { sandboxName: "alpha", - preserveWritableLayer: true, + keepOriginalRunningUntilFinalize: true, waitForSupervisor: false, openshellSandboxCommand: ["env", "nemoclaw-start"], }, diff --git a/src/lib/onboard/docker-startup-command-patch.ts b/src/lib/onboard/docker-startup-command-patch.ts index 63826ce2a3e..6541152e2b7 100644 --- a/src/lib/onboard/docker-startup-command-patch.ts +++ b/src/lib/onboard/docker-startup-command-patch.ts @@ -16,7 +16,6 @@ export function recreateOpenShellDockerSandboxWithStartupCommand( timeoutSecs?: number; waitForSupervisor?: boolean; keepOriginalRunningUntilFinalize?: boolean; - preserveWritableLayer?: boolean; openshellSandboxCommand: readonly string[]; requiredUlimits?: readonly import("./docker-gpu-patch-types").DockerUlimit[] | null; expectedOldContainerId?: string | null; diff --git a/test/e2e/live/sandbox-survival.test.ts b/test/e2e/live/sandbox-survival.test.ts index af051d51d4f..3226843f404 100644 --- a/test/e2e/live/sandbox-survival.test.ts +++ b/test/e2e/live/sandbox-survival.test.ts @@ -5,8 +5,8 @@ * * Preserves the real boundaries: install.sh/onboard, Docker, OpenShell * gateway stop/start, NemoClaw registry/list/status, sandbox SSH/exec, durable - * /sandbox/.openclaw state markers, OpenClaw gateway health after restart, and - * inference.local chat completion before and after gateway restart. + * /sandbox/.openclaw state markers, and inference.local chat completion before + * and after gateway restart. */ import fs from "node:fs"; @@ -115,7 +115,6 @@ test( "NemoClaw registry, nemoclaw list/status, and openshell sandbox list discover the sandbox", "OpenShell version supports gateway resume and state persistence", "sandbox exec/SSH-equivalent access works before and after gateway restart", - "OpenClaw gateway passes its health check after the OpenShell gateway restart", "inference.local returns a live PONG before and after gateway restart", "markers under /sandbox/.openclaw survive the gateway stop/start cycle", "final destroy removes the sandbox from NemoClaw registry/list state", @@ -368,7 +367,6 @@ test( assertions: { installCompleted: install.exitCode === 0, registryListedBeforeRestart: true, - openClawGatewayHealthyAfterRestart: true, inferenceLocalBeforeRestart: true, markersPersistedAfterRestart: true, inferenceLocalAfterRestart: true, From 691389b4d4df3b7bc881f4f63c0c1fa0afbed3dc Mon Sep 17 00:00:00 2001 From: San Dang Date: Thu, 30 Jul 2026 22:50:51 +0530 Subject: [PATCH 22/30] refactor(recovery): finish main conflict cleanup --- docs/manage-sandboxes/recover-rebuild-sandboxes.mdx | 2 +- docs/reference/commands.mdx | 2 +- docs/reference/troubleshooting.mdx | 2 +- src/lib/onboard/docker-gpu-patch-recreate.ts | 3 --- 4 files changed, 3 insertions(+), 6 deletions(-) diff --git a/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx b/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx index 58c777f81d0..8593479fee1 100644 --- a/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx +++ b/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx @@ -84,7 +84,7 @@ For a local Docker-driver sandbox whose container still uses the legacy keepaliv NemoClaw keeps the previous container available until the managed controller proves the supervisor topology, gateway health, and settle check. Before recreation, NemoClaw backs up the state directories and files declared by the agent manifest. -It commits only after the replacement container identity and state restoration pass. +It commits only after OpenShell reports the recreated sandbox as `Ready`, the replacement container identity matches, and state restoration passes. NemoClaw removes the temporary state backup after a successful restore or rollback. If state restoration and rollback both fail, it retains the backup and prints host recovery guidance. Mounted state remains available, but a committed swap does not retain other writable-layer changes. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 23e4a3e4a30..e0044a14d56 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -1166,7 +1166,7 @@ It does not use ordinary `openshell sandbox exec` or an in-sandbox manual relaun When the root-owned managed controller attests two unchanged zero-supervisor process scans with a stable PID 1 and reports `SUPERVISOR_NOT_RUNNING`, a local Docker-driver sandbox with the legacy keepalive startup can enter a transactional container recreation. The recreation uses a credential-free managed startup command, pins the registered container identity, and retains the previous container for rollback. Before recreation, NemoClaw backs up the state directories and files declared by the agent manifest. -It commits only after managed gateway health, the settle check, exact replacement identity, and state restoration pass. +It commits only after managed gateway health, the settle check, OpenShell readiness, exact replacement identity, and state restoration pass. NemoClaw removes the temporary state backup after a successful restore or rollback. If state restoration and rollback both fail, it retains the backup and prints host recovery guidance. Mounted state remains available, but a committed swap does not retain other writable-layer changes. diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index 6cad47f36ff..a2a72c18ed4 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -1019,7 +1019,7 @@ The same result is inconclusive during managed settle confirmation and can be pr NemoClaw treats `SUPERVISOR_UNAVAILABLE` as terminal because it can report unreadable or untrusted supervisor state, ambiguous discovery, or a process-identity change. It does not retry other status or output combinations. `SUPERVISOR_NOT_RUNNING` is a separate result that requires two zero-supervisor scans with a stable PID 1 and does not enter that retry loop. -On a supported local Docker-driver sandbox with the legacy keepalive startup, it can authorize a container-identity-pinned recreation that commits only after managed health and settle checks pass. +On a supported local Docker-driver sandbox with the legacy keepalive startup, it can authorize a container-identity-pinned recreation that commits only after managed health, settle, and OpenShell readiness checks pass. To bypass that trusted recreation while troubleshooting, run `NEMOCLAW_DISABLE_SUPERVISOR_RELAUNCH=1 $$nemoclaw recover`; NemoClaw leaves the container unchanged and returns rebuild or re-onboard guidance. If that bounded retry is exhausted, or if `gateway restart` reports `SUPERVISOR_BUSY`, wait for the active request to finish and retry the command. If the error mentions `SUPERVISOR_NOT_RUNNING` and trusted recreation could not proceed, `SUPERVISOR_REBUILD_REQUIRED`, a missing `nemoclaw-gateway-control` helper, or a missing managed controller, the sandbox image may predate the current lifecycle contract. diff --git a/src/lib/onboard/docker-gpu-patch-recreate.ts b/src/lib/onboard/docker-gpu-patch-recreate.ts index caeca06df9f..1ee5ec16ea4 100644 --- a/src/lib/onboard/docker-gpu-patch-recreate.ts +++ b/src/lib/onboard/docker-gpu-patch-recreate.ts @@ -3,7 +3,6 @@ import { dockerCapture, - dockerForceRm, dockerRename, dockerRm, dockerRunDetached, @@ -48,7 +47,6 @@ type RecreateDeps = Required< Pick< DockerGpuPatchDeps, | "dockerCapture" - | "dockerForceRm" | "dockerRunDetached" | "dockerRename" | "dockerRm" @@ -66,7 +64,6 @@ type RecreateDeps = Required< function recreateDeps(deps: DockerGpuPatchDeps): RecreateDeps { return { dockerCapture, - dockerForceRm, dockerRunDetached, dockerRename, dockerRm, From 8a60ab9778f4497a08eb747e6af98e3dca7021cd Mon Sep 17 00:00:00 2001 From: San Dang Date: Thu, 30 Jul 2026 23:43:48 +0530 Subject: [PATCH 23/30] fix(recovery): stop original before replacement Signed-off-by: San Dang --- .../sandbox/supervisor-relaunch.test.ts | 1 - .../actions/sandbox/supervisor-relaunch.ts | 1 - .../onboard/docker-gpu-patch-finalize.test.ts | 48 -------------- src/lib/onboard/docker-gpu-patch-finalize.ts | 16 +---- src/lib/onboard/docker-gpu-patch-recreate.ts | 57 +++++++--------- src/lib/onboard/docker-gpu-patch-rollback.ts | 18 +---- src/lib/onboard/docker-gpu-patch-types.ts | 4 -- .../docker-startup-command-patch.test.ts | 66 ++----------------- .../onboard/docker-startup-command-patch.ts | 1 - src/lib/sandbox/privileged-exec.test.ts | 32 --------- src/lib/sandbox/privileged-exec.ts | 20 +----- 11 files changed, 38 insertions(+), 226 deletions(-) diff --git a/src/lib/actions/sandbox/supervisor-relaunch.test.ts b/src/lib/actions/sandbox/supervisor-relaunch.test.ts index b7550d04e60..28f638085a0 100644 --- a/src/lib/actions/sandbox/supervisor-relaunch.test.ts +++ b/src/lib/actions/sandbox/supervisor-relaunch.test.ts @@ -135,7 +135,6 @@ describe("relaunchManagedSupervisorSession", () => { expect(options).toMatchObject({ sandboxName: "alpha", expectedOldContainerId: "old-container-id", - keepOriginalRunningUntilFinalize: true, waitForSupervisor: false, }); const serialized = options?.openshellSandboxCommand.join(" ") ?? ""; diff --git a/src/lib/actions/sandbox/supervisor-relaunch.ts b/src/lib/actions/sandbox/supervisor-relaunch.ts index aca8295a95d..ff9a2f48740 100644 --- a/src/lib/actions/sandbox/supervisor-relaunch.ts +++ b/src/lib/actions/sandbox/supervisor-relaunch.ts @@ -177,7 +177,6 @@ export function relaunchManagedSupervisorSession( sandboxName, openshellSandboxCommand: startupCommand, expectedOldContainerId: containerId, - keepOriginalRunningUntilFinalize: true, waitForSupervisor: false, }); pendingStateBackupPath = null; diff --git a/src/lib/onboard/docker-gpu-patch-finalize.test.ts b/src/lib/onboard/docker-gpu-patch-finalize.test.ts index f0947fe9cef..5a691435ce4 100644 --- a/src/lib/onboard/docker-gpu-patch-finalize.test.ts +++ b/src/lib/onboard/docker-gpu-patch-finalize.test.ts @@ -37,25 +37,6 @@ describe("finalizeDockerGpuPatchBackup", () => { ); }); - it("force-removes the running original container after replacement health is confirmed", () => { - const dockerForceRm = vi.fn((_name: string) => ({ status: 0 })); - const dockerRm = vi.fn((_name: string) => ({ status: 0 })); - const outcome = finalizeDockerGpuPatchBackup( - { - result: { ...deferredCreateResult(), backupWasRunning: true }, - supervisorReady: true, - }, - { dockerForceRm, dockerRm }, - ); - - expect(outcome).toEqual({ backupRemoved: true, rolledBack: false }); - expect(dockerForceRm).toHaveBeenCalledWith( - "openshell-alpha-nemoclaw-gpu-backup-1780491860342", - expect.objectContaining({ ignoreError: true }), - ); - expect(dockerRm).not.toHaveBeenCalled(); - }); - it("rolls back to the backup container when supervisor reconnect failed", () => { const dockerStop = vi.fn(() => ({ status: 0 })); const dockerRm = vi.fn((_name: string) => ({ status: 0 })); @@ -84,35 +65,6 @@ describe("finalizeDockerGpuPatchBackup", () => { ).toBe(false); }); - it("rolls back to the still-running original container without restarting it", () => { - const dockerStop = vi.fn(() => ({ status: 0 })); - const dockerRm = vi.fn((_name: string) => ({ status: 0 })); - const dockerRename = vi.fn((_old: string, _next: string) => ({ status: 0 })); - const dockerStart = vi.fn(() => ({ status: 0 })); - const outcome = finalizeDockerGpuPatchBackup( - { - result: { - ...deferredCreateResult(), - backupWasRunning: true, - }, - supervisorReady: false, - }, - { dockerStop, dockerRm, dockerRename, dockerStart }, - ); - - expect(outcome).toEqual({ backupRemoved: false, rolledBack: true }); - expect(dockerStop).toHaveBeenCalledWith( - "new-container-id", - expect.objectContaining({ ignoreError: true }), - ); - expect(dockerRename).toHaveBeenCalledWith( - "openshell-alpha-nemoclaw-gpu-backup-1780491860342", - "openshell-alpha", - expect.objectContaining({ ignoreError: true }), - ); - expect(dockerStart).not.toHaveBeenCalled(); - }); - it("reports rolledBack=false when restoring the backup fails", () => { const dockerStop = vi.fn(() => ({ status: 0 })); const dockerRm = vi.fn((_name: string) => ({ status: 0 })); diff --git a/src/lib/onboard/docker-gpu-patch-finalize.ts b/src/lib/onboard/docker-gpu-patch-finalize.ts index e3c0dc65ff6..4eeeedf399b 100644 --- a/src/lib/onboard/docker-gpu-patch-finalize.ts +++ b/src/lib/onboard/docker-gpu-patch-finalize.ts @@ -65,9 +65,7 @@ export function finalizeDockerGpuPatchBackup( // even if `docker rm` cannot delete it (e.g. concurrent admin action, // daemon timeout). Reflect the actual rm status in the outcome so // diagnostics can flag a leaked backup container. - const rmResult = options.result.backupWasRunning - ? resolved.dockerForceRm(options.result.backupContainerName, containerOpts) - : resolved.dockerRm(options.result.backupContainerName, containerOpts); + const rmResult = resolved.dockerRm(options.result.backupContainerName, containerOpts); return { backupRemoved: hasZeroDockerExitStatus(rmResult), rolledBack: false }; } const rolledBack = rollbackToBackupContainer( @@ -75,7 +73,6 @@ export function finalizeDockerGpuPatchBackup( newContainerId: options.result.newContainerId, backupContainerName: options.result.backupContainerName, originalName: options.result.originalName, - backupWasRunning: options.result.backupWasRunning, }, resolved, ); @@ -88,12 +85,7 @@ export type SupervisorReconnectOutcome = export function reconcileSupervisorReconnect( execReady: boolean, - refs: { - newContainerId: string; - backupContainerName: string; - originalName: string; - backupWasRunning?: boolean; - }, + refs: { newContainerId: string; backupContainerName: string; originalName: string }, deps: DockerGpuPatchDeps, ): SupervisorReconnectOutcome { const resolved = resolveDockerGpuPatchRollbackDeps(deps); @@ -108,9 +100,7 @@ export function reconcileSupervisorReconnect( // leaked backup container but the user-visible sandbox is healthy. // Surface the actual rm status so callers can fold it into diagnostics // alongside the deferred-finalize path in `finalizeDockerGpuPatchBackup`. - const rmResult = refs.backupWasRunning - ? resolved.dockerForceRm(refs.backupContainerName, containerOpts) - : resolved.dockerRm(refs.backupContainerName, containerOpts); + const rmResult = resolved.dockerRm(refs.backupContainerName, containerOpts); return { execReady: true, backupRemoved: hasZeroDockerExitStatus(rmResult) }; } const rolledBack = rollbackToBackupContainer(refs, resolved); diff --git a/src/lib/onboard/docker-gpu-patch-recreate.ts b/src/lib/onboard/docker-gpu-patch-recreate.ts index 1ee5ec16ea4..b205c144af8 100644 --- a/src/lib/onboard/docker-gpu-patch-recreate.ts +++ b/src/lib/onboard/docker-gpu-patch-recreate.ts @@ -5,6 +5,7 @@ import { dockerCapture, dockerRename, dockerRm, + dockerRun, dockerRunDetached, dockerStart, dockerStop, @@ -47,6 +48,7 @@ type RecreateDeps = Required< Pick< DockerGpuPatchDeps, | "dockerCapture" + | "dockerRun" | "dockerRunDetached" | "dockerRename" | "dockerRm" @@ -64,6 +66,7 @@ type RecreateDeps = Required< function recreateDeps(deps: DockerGpuPatchDeps): RecreateDeps { return { dockerCapture, + dockerRun, dockerRunDetached, dockerRename, dockerRm, @@ -152,7 +155,6 @@ export function recreateOpenShellDockerSandboxContainer( gpuDevice?: string | null; timeoutSecs?: number; waitForSupervisor?: boolean; - keepOriginalRunningUntilFinalize?: boolean; openshellSandboxCommand?: readonly string[] | null; requiredUlimits?: readonly import("./docker-gpu-patch-types").DockerUlimit[] | null; expectedOldContainerId?: string | null; @@ -169,11 +171,6 @@ export function recreateOpenShellDockerSandboxContainer( }; try { validateRequiredDockerUlimits(options.requiredUlimits); - if (options.keepOriginalRunningUntilFinalize && options.waitForSupervisor !== false) { - throw new Error( - "Keeping the original OpenShell supervisor running requires deferred supervisor finalization.", - ); - } const containerIds = findOpenShellDockerSandboxContainerIds(options.sandboxName, deps); const oldContainerId = containerIds[0]; if (!oldContainerId) { @@ -280,31 +277,28 @@ export function recreateOpenShellDockerSandboxContainer( ); } } + const cloneArgs = buildDockerGpuCloneRunArgs(inspect, selection.mode, cloneOptions); + const containerMutationOptions = { ignoreError: true, suppressOutput: true, timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, }; - const cloneArgs = buildDockerGpuCloneRunArgs(inspect, selection.mode, cloneOptions); - - const backupWasRunning = options.keepOriginalRunningUntilFinalize === true; - if (!backupWasRunning) { - const stopResult = d.dockerStop(oldContainerId, { - ...containerMutationOptions, - timeout: DOCKER_GPU_PATCH_STOP_TIMEOUT_MS, - }); - if (!hasZeroDockerExitStatus(stopResult)) { - context.rolledBack = hasZeroDockerExitStatus( - d.dockerStart(oldContainerId, containerMutationOptions), - ); - throw new Error( - `Could not stop original sandbox container: ${resultText(stopResult)}; ${ - context.rolledBack - ? "original sandbox container confirmed running" - : "restart failed; original sandbox container may be stopped" - }`, - ); - } + const stopResult = d.dockerStop(oldContainerId, { + ...containerMutationOptions, + timeout: DOCKER_GPU_PATCH_STOP_TIMEOUT_MS, + }); + if (!hasZeroDockerExitStatus(stopResult)) { + context.rolledBack = hasZeroDockerExitStatus( + d.dockerStart(oldContainerId, containerMutationOptions), + ); + throw new Error( + `Could not stop original sandbox container: ${resultText(stopResult)}; ${ + context.rolledBack + ? "original sandbox container confirmed running" + : "restart failed; original sandbox container may be stopped" + }`, + ); } const renameResult = d.dockerRename( oldContainerId, @@ -313,9 +307,9 @@ export function recreateOpenShellDockerSandboxContainer( ); if (!hasZeroDockerExitStatus(renameResult)) { d.dockerRename(backupContainerName, originalName, containerMutationOptions); - const restarted = - backupWasRunning || - hasZeroDockerExitStatus(d.dockerStart(oldContainerId, containerMutationOptions)); + const restarted = hasZeroDockerExitStatus( + d.dockerStart(oldContainerId, containerMutationOptions), + ); let originalNameRestored = false; try { originalNameRestored = @@ -340,7 +334,7 @@ export function recreateOpenShellDockerSandboxContainer( }); if (!hasZeroDockerExitStatus(runResult)) { context.rolledBack = restoreDockerGpuPatchBackupAfterRecreateFailure( - { newContainerId: originalName, backupContainerName, originalName, backupWasRunning }, + { newContainerId: originalName, backupContainerName, originalName }, deps, ); const containerDescription = @@ -366,7 +360,7 @@ export function recreateOpenShellDockerSandboxContainer( ); if (!newContainerId) { context.rolledBack = restoreDockerGpuPatchBackupAfterRecreateFailure( - { newContainerId: originalName, backupContainerName, originalName, backupWasRunning }, + { newContainerId: originalName, backupContainerName, originalName }, deps, ); const containerDescription = @@ -390,7 +384,6 @@ export function recreateOpenShellDockerSandboxContainer( originalName, backupContainerName, mode: selectedMode, - backupWasRunning, backupRemoved, }); if (options.waitForSupervisor === false) return result(false); diff --git a/src/lib/onboard/docker-gpu-patch-rollback.ts b/src/lib/onboard/docker-gpu-patch-rollback.ts index a18ae91febc..81532529503 100644 --- a/src/lib/onboard/docker-gpu-patch-rollback.ts +++ b/src/lib/onboard/docker-gpu-patch-rollback.ts @@ -2,7 +2,6 @@ // SPDX-License-Identifier: Apache-2.0 import { - dockerForceRm as defaultDockerForceRm, dockerRename as defaultDockerRename, dockerRm as defaultDockerRm, dockerStart as defaultDockerStart, @@ -27,7 +26,6 @@ type DockerRenameFn = ( ) => DockerRunResult; export type ResolvedDockerGpuPatchRollbackDeps = { - dockerForceRm: DockerContainerFn; dockerStop: DockerContainerFn; dockerRm: DockerContainerFn; dockerRename: DockerRenameFn; @@ -38,7 +36,6 @@ export function resolveDockerGpuPatchRollbackDeps( deps: DockerGpuPatchDeps, ): ResolvedDockerGpuPatchRollbackDeps { return { - dockerForceRm: deps.dockerForceRm ?? defaultDockerForceRm, dockerStop: deps.dockerStop ?? defaultDockerStop, dockerRm: deps.dockerRm ?? defaultDockerRm, dockerRename: deps.dockerRename ?? defaultDockerRename, @@ -47,12 +44,7 @@ export function resolveDockerGpuPatchRollbackDeps( } export function rollbackToBackupContainer( - refs: { - newContainerId: string; - backupContainerName: string; - originalName: string; - backupWasRunning?: boolean; - }, + refs: { newContainerId: string; backupContainerName: string; originalName: string }, deps: ResolvedDockerGpuPatchRollbackDeps, ): boolean { const containerOpts = { @@ -64,19 +56,13 @@ export function rollbackToBackupContainer( deps.dockerRm(refs.newContainerId, containerOpts); const restored = deps.dockerRename(refs.backupContainerName, refs.originalName, containerOpts); if (!hasZeroDockerExitStatus(restored)) return false; - if (refs.backupWasRunning) return true; const started = deps.dockerStart(refs.originalName, containerOpts); return hasZeroDockerExitStatus(started); } /** Restore the original sandbox after `docker run` fails during GPU recreation. */ export function restoreDockerGpuPatchBackupAfterRecreateFailure( - refs: { - newContainerId: string; - backupContainerName: string; - originalName: string; - backupWasRunning?: boolean; - }, + refs: { newContainerId: string; backupContainerName: string; originalName: string }, deps: DockerGpuPatchDeps = {}, ): boolean { return rollbackToBackupContainer(refs, resolveDockerGpuPatchRollbackDeps(deps)); diff --git a/src/lib/onboard/docker-gpu-patch-types.ts b/src/lib/onboard/docker-gpu-patch-types.ts index 93ad0c20ec8..32be0afe46f 100644 --- a/src/lib/onboard/docker-gpu-patch-types.ts +++ b/src/lib/onboard/docker-gpu-patch-types.ts @@ -27,7 +27,6 @@ export type DockerGpuPatchDeps = { dockerRun?: DockerRunFn; dockerRunDetached?: DockerRunFn; dockerRename?: DockerRenameFn; - dockerForceRm?: DockerContainerFn; dockerRm?: DockerContainerFn; dockerStart?: DockerContainerFn; dockerStop?: DockerContainerFn; @@ -92,9 +91,6 @@ export type DockerGpuPatchResult = { originalName: string; backupContainerName: string; mode: DockerGpuPatchMode; - // True when recovery did not stop the original OpenShell supervisor before - // creating the replacement. The caller passes replacement health separately. - backupWasRunning?: boolean; // True when the patch path also confirmed supervisor reconnect AND removed // the backup container. False when the caller deferred the reconnect wait // (via `waitForSupervisor: false`); the backup is still in place and the diff --git a/src/lib/onboard/docker-startup-command-patch.test.ts b/src/lib/onboard/docker-startup-command-patch.test.ts index f6e67353b81..9c37c6119b0 100644 --- a/src/lib/onboard/docker-startup-command-patch.test.ts +++ b/src/lib/onboard/docker-startup-command-patch.test.ts @@ -43,66 +43,6 @@ function inspectFixture(): DockerContainerInspect { } describe("Docker startup-command patch", () => { - it("preserves the registered supervisor until recovery finalizes", () => { - const dockerCapture = vi.fn((args: readonly string[]) => - args[0] === "ps" - ? "old-container-id\n" - : args[0] === "inspect" - ? JSON.stringify([inspectFixture()]) - : "", - ); - const dockerStop = vi.fn(() => ({ status: 0 })); - const dockerRename = vi.fn(() => ({ status: 0 })); - const dockerRunDetached = vi.fn((_args: readonly string[]) => ({ - status: 0, - stdout: "new-container-id\n", - })); - - const result = recreateStartupCommandForTest( - { - sandboxName: "alpha", - keepOriginalRunningUntilFinalize: true, - waitForSupervisor: false, - openshellSandboxCommand: ["env", "nemoclaw-start"], - }, - { - dockerCapture, - dockerRunDetached, - dockerRename, - dockerStop, - now: () => new Date("2026-07-10T00:00:00Z"), - }, - ); - - expect(result).toMatchObject({ - newContainerId: "new-container-id", - backupWasRunning: true, - backupRemoved: false, - }); - expect(dockerStop).not.toHaveBeenCalled(); - expect(dockerRename.mock.invocationCallOrder[0]).toBeLessThan( - dockerRunDetached.mock.invocationCallOrder[0] ?? Number.POSITIVE_INFINITY, - ); - const cloneArgs = dockerRunDetached.mock.calls[0]?.[0] ?? []; - expect(cloneArgs).toContain(`sha256:${"c".repeat(64)}`); - }); - - it("requires deferred finalization when preserving the registered supervisor", () => { - const dockerCapture = vi.fn(); - - expect(() => - recreateStartupCommandForTest( - { - sandboxName: "alpha", - keepOriginalRunningUntilFinalize: true, - openshellSandboxCommand: ["env", "nemoclaw-start"], - }, - { dockerCapture }, - ), - ).toThrow(/requires deferred supervisor finalization/); - expect(dockerCapture).not.toHaveBeenCalled(); - }); - it("persists the startup command without adding GPU-only container privileges", () => { const dockerCaptureOutput: Record = { ps: "old-container-id\n", @@ -115,6 +55,7 @@ describe("Docker startup-command patch", () => { status: 0, stdout: "new-container-id\n", })); + const dockerStop = vi.fn(() => ({ status: 0 })); const result = recreateStartupCommandForTest( { @@ -127,7 +68,7 @@ describe("Docker startup-command patch", () => { dockerCapture, dockerRunDetached, dockerRename: vi.fn(() => ({ status: 0 })), - dockerStop: vi.fn(() => ({ status: 0 })), + dockerStop, sleep: vi.fn(), now: () => new Date("2026-07-10T00:00:00Z"), }, @@ -154,6 +95,9 @@ describe("Docker startup-command patch", () => { expect.arrayContaining(["ps", "-a", "--no-trunc"]), expect.objectContaining({ ignoreError: true }), ); + expect(dockerStop.mock.invocationCallOrder[0]).toBeLessThan( + dockerRunDetached.mock.invocationCallOrder[0] ?? Number.POSITIVE_INFINITY, + ); }); it("preserves OpenShell's native CDI GPU request during restart-persistence recreation", () => { diff --git a/src/lib/onboard/docker-startup-command-patch.ts b/src/lib/onboard/docker-startup-command-patch.ts index 6541152e2b7..2b5b6f761ef 100644 --- a/src/lib/onboard/docker-startup-command-patch.ts +++ b/src/lib/onboard/docker-startup-command-patch.ts @@ -15,7 +15,6 @@ export function recreateOpenShellDockerSandboxWithStartupCommand( sandboxName: string; timeoutSecs?: number; waitForSupervisor?: boolean; - keepOriginalRunningUntilFinalize?: boolean; openshellSandboxCommand: readonly string[]; requiredUlimits?: readonly import("./docker-gpu-patch-types").DockerUlimit[] | null; expectedOldContainerId?: string | null; diff --git a/src/lib/sandbox/privileged-exec.test.ts b/src/lib/sandbox/privileged-exec.test.ts index 1e5e14ba5bd..2598fbd6b48 100644 --- a/src/lib/sandbox/privileged-exec.test.ts +++ b/src/lib/sandbox/privileged-exec.test.ts @@ -217,38 +217,6 @@ describe("privileged sandbox exec routing", () => { ); }); - it("selects the pinned container while a transaction backup is still running", () => { - withPrivilegedExecMocks( - { - getSandbox: () => ({ name: "alpha", openshellDriver: "docker" }), - listSandboxes: () => ({ sandboxes: [{ name: "alpha" }], defaultSandbox: "alpha" }), - dockerCapture: () => - [ - "replacement-container-id\topenshell-alpha", - "original-container-id\topenshell-alpha-nemoclaw-backup", - ].join("\n"), - }, - ({ privilegedSandboxExecArgv }) => { - expect( - privilegedSandboxExecArgv( - "alpha", - ["/trusted/control"], - false, - true, - "replacement-container-id", - ), - ).toEqual( - expect.arrayContaining([ - "--user", - "root", - "replacement-container-id", - "/trusted/control", - ]), - ); - }, - ); - }); - it("refuses privileged execution when the pinned container identity is empty", () => { withPrivilegedExecMocks( { diff --git a/src/lib/sandbox/privileged-exec.ts b/src/lib/sandbox/privileged-exec.ts index 5588c86881f..c45f59a60a4 100644 --- a/src/lib/sandbox/privileged-exec.ts +++ b/src/lib/sandbox/privileged-exec.ts @@ -108,7 +108,6 @@ function selectDirectSandboxContainer( sandboxName: string, labeledContainerRows: string, registeredNames: readonly string[] = [sandboxName], - expectedContainerId?: string, ): string | null { const names = Array.from(new Set([...registeredNames, sandboxName])).sort( (a, b) => b.length - a.length || a.localeCompare(b), @@ -126,16 +125,6 @@ function selectDirectSandboxContainer( "refusing lifecycle execution.", ); } - if (expectedContainerId !== undefined) { - const expected = candidates.filter(({ id }) => id === expectedContainerId); - if (expected.length !== 1) { - throw new Error( - `OpenShell container identity changed for sandbox '${sandboxName}'; ` + - "refusing privileged execution against a different container.", - ); - } - return expected[0].id; - } if (candidates.length > 1) { throw new Error( `Multiple running OpenShell containers are labeled for sandbox '${sandboxName}'; ` + @@ -149,10 +138,7 @@ function expectedDirectContainerPattern(sandboxName: string): string { return `openshell-${sandboxName} or openshell-${sandboxName}-*`; } -function findDirectSandboxContainer( - sandboxName: string, - expectedContainerId?: string, -): string | null { +function findDirectSandboxContainer(sandboxName: string): string | null { const names = registeredSandboxNames(sandboxName); let output: string; try { @@ -176,7 +162,7 @@ function findDirectSandboxContainer( { cause: error }, ); } - return selectDirectSandboxContainer(sandboxName, output, names, expectedContainerId); + return selectDirectSandboxContainer(sandboxName, output, names); } function missingDirectContainerError(sandboxName: string, driver: string | null): Error { @@ -232,7 +218,7 @@ function privilegedSandboxExecArgv( // Docker/direct-container is the only supported privileged mutation path. // Try it even when older registry entries do not record a driver, then fail // clearly if no matching sandbox container is running. - const container = findDirectSandboxContainer(sandboxName, expectedContainerId); + const container = findDirectSandboxContainer(sandboxName); if (container) { if (expectedContainerId !== undefined && container !== expectedContainerId) { throw new Error( From 8bb87f946c9b6aa25bae9d4ffbfda85d6fc7278b Mon Sep 17 00:00:00 2001 From: San Dang Date: Fri, 31 Jul 2026 00:56:07 +0530 Subject: [PATCH 24/30] fix(recovery): restart legacy supervisor in place Signed-off-by: San Dang --- .../recover-rebuild-sandboxes.mdx | 13 +- docs/reference/commands.mdx | 15 +- docs/reference/troubleshooting.mdx | 6 +- src/lib/actions/sandbox/process-recovery.ts | 92 +------ .../sandbox/supervisor-relaunch.test.ts | 243 ++---------------- .../actions/sandbox/supervisor-relaunch.ts | 201 ++++----------- .../docker-startup-command-patch.test.ts | 6 +- ...ocess-recovery-supervisor-relaunch.test.ts | 166 ++---------- 8 files changed, 115 insertions(+), 627 deletions(-) diff --git a/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx b/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx index 8593479fee1..6d5311d2ba9 100644 --- a/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx +++ b/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx @@ -80,16 +80,9 @@ Refer to [`$$nemoclaw recover`](../../reference/commands#$$nemoclaw-name- Recovery uses registry-scoped privileged direct-container control and does not fall back to ordinary `openshell sandbox exec` or a manual in-sandbox relaunch. -For a local Docker-driver sandbox whose container still uses the legacy keepalive startup, `recover` can transactionally recreate the registered container with a credential-free managed startup command. - -NemoClaw keeps the previous container available until the managed controller proves the supervisor topology, gateway health, and settle check. -Before recreation, NemoClaw backs up the state directories and files declared by the agent manifest. -It commits only after OpenShell reports the recreated sandbox as `Ready`, the replacement container identity matches, and state restoration passes. -NemoClaw removes the temporary state backup after a successful restore or rollback. -If state restoration and rollback both fail, it retains the backup and prints host recovery guidance. -Mounted state remains available, but a committed swap does not retain other writable-layer changes. -After a transactional recreation, NemoClaw uses the `NEMOCLAW_SANDBOX_READY_TIMEOUT` budget (180 seconds by default) for OpenShell to re-register the sandbox before starting the primary dashboard or API host forward. -A definitive managed-health failure still stops immediately; if re-registration does not complete within the budget, the forward stays stopped. +For a local Docker-driver sandbox whose container still uses the legacy keepalive startup, `recover` can restart the credential-free managed workload in the registered container. +NemoClaw keeps the container identity and writable state unchanged, then requires managed gateway health and OpenShell `Ready` before starting the primary dashboard or API host forward. +A definitive managed-health failure stops immediately; if readiness does not complete within the configured budget, the forward stays stopped. For the controller topology, trust boundary, and fail-closed conditions, refer to [Understand Gateway Lifecycle Control](../configure-sandboxes/understand-gateway-lifecycle-control). If recovery cannot repair a sandbox that needs credentials or a current controller contract, rebuild it. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 8dda793c6a9..000f02b50ae 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -1164,13 +1164,8 @@ The host selects the controller from the live container topology. In a direct root-entrypoint container, the request reaches the root PID 1 supervisor. In an OpenShell-managed container, the request enters the root-owned mode `0500` managed controller through a sanitized root exec while OpenShell remains PID 1. It does not use ordinary `openshell sandbox exec` or an in-sandbox manual relaunch as a fallback. -When the root-owned managed controller attests two unchanged zero-supervisor process scans with a stable PID 1 and reports `SUPERVISOR_NOT_RUNNING`, a local Docker-driver sandbox with the legacy keepalive startup can enter a transactional container recreation. -The recreation uses a credential-free managed startup command, pins the registered container identity, and retains the previous container for rollback. -Before recreation, NemoClaw backs up the state directories and files declared by the agent manifest. -It commits only after managed gateway health, the settle check, OpenShell readiness, exact replacement identity, and state restoration pass. -NemoClaw removes the temporary state backup after a successful restore or rollback. -If state restoration and rollback both fail, it retains the backup and prints host recovery guidance. -Mounted state remains available, but a committed swap does not retain other writable-layer changes. +When the root-owned managed controller attests two unchanged zero-supervisor process scans with a stable PID 1 and reports `SUPERVISOR_NOT_RUNNING`, a local Docker-driver sandbox with the legacy keepalive startup can restart its credential-free managed workload in the same registered container. +Recovery pins the container identity and requires managed gateway health, the settle check, and OpenShell readiness while leaving the writable layer unchanged. It is idempotent. When `recover` repairs a stopped built-in OpenClaw or Hermes gateway, it repeats the recovery action only for an exit status of `1` with blank stdout and a sole nonblank stderr line equal to `SUPERVISOR_BUSY`, with at most three controller attempts. The same result is inconclusive during managed settle confirmation and can be probed again only within the configured settle window. @@ -3941,7 +3936,7 @@ The following environment variables tune onboard-time wall-clock limits. `NEMOCLAW_SANDBOX_READY_TIMEOUT` also covers OpenShell command re-registration after onboarding applies policy presets. -`NEMOCLAW_SANDBOX_READY_TIMEOUT` also applies when managed recovery transactionally recreates an existing sandbox. +`NEMOCLAW_SANDBOX_READY_TIMEOUT` also applies while managed recovery waits for OpenShell readiness. Set them before running `$$nemoclaw onboard` if a slow connection or large model pull risks tripping the default. @@ -3954,7 +3949,7 @@ Set them before running `$$nemoclaw onboard` if a slow connection or large model -For managed recovery, the same timeout covers OpenShell re-registration after transactional recreation. +For managed recovery, the same timeout covers OpenShell readiness after the managed workload restarts. When the deadline expires, the primary dashboard or API host forward stays stopped. @@ -3982,7 +3977,7 @@ The following flags change defaults for commands that manage existing sandboxes. | `NEMOCLAW_CONFIRM_LEGACY_MANAGED_RECREATE` | Exact JSON array of sandbox names | Confirms to the installer that the exact listed set of pre-fingerprint OpenClaw or Hermes sandboxes used NemoClaw-managed images, allowing recovery onto the current managed image. The normalized names must exactly match the installer's printed array. Set it only after verifying every named sandbox. Recorded custom-image evidence remains blocked. | | `NEMOCLAW_DISABLE_INFERENCE_ROUTE_REPAIR` | `1` to enable | Skips the automatic DNS-proxy repair for stale `inference.local` routes during `$$nemoclaw connect` and `$$nemoclaw connect --probe-only`. Use only as a troubleshooting escape hatch. | -| `NEMOCLAW_DISABLE_SUPERVISOR_RELAUNCH` | `1` to enable | Skips the automatic trusted container recreation during `$$nemoclaw recover` when two managed scans find no supervisor while PID 1 remains stable. Use only as a troubleshooting escape hatch; recovery then falls back to the rebuild or re-onboard guidance. | +| `NEMOCLAW_DISABLE_SUPERVISOR_RELAUNCH` | `1` to enable | Skips the automatic trusted in-place workload restart during `$$nemoclaw recover` when two managed scans find no supervisor while PID 1 remains stable. Use only as a troubleshooting escape hatch; recovery then falls back to the rebuild or re-onboard guidance. | | `NEMOCLAW_SHIELDS_ACCEPT_LEGACY_BASELINE` | `1` to opt in | Allows advanced immutable-config verification to trust the current on-disk bytes for older or partial content baselines. Use only after you have rebuilt or manually inspected the sandbox state and accepted that the baseline is operator-approved. | | `NEMOCLAW_SHIELDS_SETTLE_MS` | milliseconds (default `750`, clamped to `0` to `10000`) | Settle window NemoClaw waits after re-applying a config lockdown (during shields auto-restore and `$$nemoclaw shields up` drift remediation) before re-confirming the lock still holds. Detects when an in-sandbox reconciler changes config file permissions after lockdown and re-applies the lock; if NemoClaw cannot re-confirm the lock within the retry budget, shields stay down. This narrows the window in which a reconciler can revert permissions rather than eliminating it. The best-effort `chattr +i` immutable bit remains the only fully durable lock. Raise it on hosts where the gateway settles slowly. | | `NEMOCLAW_SKIP_UNREACHABLE_SANDBOX_BACKUP` | Exactly `1` to opt in (`true`, `yes`, `0` are not accepted) | Applies to standalone `$$nemoclaw backup-all` runs. Skips running sandboxes whose in-sandbox SSH endpoint does not answer. It does not relax the installer's strict pre-upgrade backup, which still aborts if any registered sandbox is skipped or fails. Any uncommitted state since the last successful backup is not included in the skipped backup. | diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index a2a72c18ed4..badaf76a1e5 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -1019,10 +1019,10 @@ The same result is inconclusive during managed settle confirmation and can be pr NemoClaw treats `SUPERVISOR_UNAVAILABLE` as terminal because it can report unreadable or untrusted supervisor state, ambiguous discovery, or a process-identity change. It does not retry other status or output combinations. `SUPERVISOR_NOT_RUNNING` is a separate result that requires two zero-supervisor scans with a stable PID 1 and does not enter that retry loop. -On a supported local Docker-driver sandbox with the legacy keepalive startup, it can authorize a container-identity-pinned recreation that commits only after managed health, settle, and OpenShell readiness checks pass. -To bypass that trusted recreation while troubleshooting, run `NEMOCLAW_DISABLE_SUPERVISOR_RELAUNCH=1 $$nemoclaw recover`; NemoClaw leaves the container unchanged and returns rebuild or re-onboard guidance. +On a supported local Docker-driver sandbox with the legacy keepalive startup, it can authorize a container-identity-pinned workload restart that succeeds only after managed health, settle, and OpenShell readiness checks pass. +To bypass that trusted restart while troubleshooting, run `NEMOCLAW_DISABLE_SUPERVISOR_RELAUNCH=1 $$nemoclaw recover`; NemoClaw leaves the container unchanged and returns rebuild or re-onboard guidance. If that bounded retry is exhausted, or if `gateway restart` reports `SUPERVISOR_BUSY`, wait for the active request to finish and retry the command. -If the error mentions `SUPERVISOR_NOT_RUNNING` and trusted recreation could not proceed, `SUPERVISOR_REBUILD_REQUIRED`, a missing `nemoclaw-gateway-control` helper, or a missing managed controller, the sandbox image may predate the current lifecycle contract. +If the error mentions `SUPERVISOR_NOT_RUNNING` and trusted recovery could not proceed, `SUPERVISOR_REBUILD_REQUIRED`, a missing `nemoclaw-gateway-control` helper, or a missing managed controller, the sandbox image may predate the current lifecycle contract. An exact `SUPERVISOR_UNAVAILABLE` result instead means the managed controller refused the current supervisor state rather than guessing which same-UID process is the gateway. The current recovery action and any managed settle confirmation stop immediately. If `recover` reports this result, follow its host-side `gateway restart` guidance. diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index 57ea4103327..16e9b6927a0 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -431,8 +431,8 @@ export async function isSandboxGatewayRunningForStatus( /** * Recover a gateway through the registered agent's managed control boundary. * Legacy custom agents retain their SSH-owned compatibility path. Built-in - * agents may return a transactional supervisor relaunch that the caller must - * commit or roll back after the managed health gate. + * agents may return a pinned legacy supervisor relaunch that the caller must + * confirm after the managed health gate. */ type SandboxProcessRecovery = | { kind: "managed" | "custom" } @@ -690,11 +690,11 @@ function recreatedSandboxOpenShellReadinessFailureDetail( const detail = (() => { switch (failure) { case "managed-health-definitive-failure": - return "the recreated sandbox failed the managed health guard, so the primary dashboard/API host forward was not started"; + return "the recovered sandbox failed the managed health guard, so the primary dashboard/API host forward was not started"; case "managed-health-inconclusive-timeout": - return "the recreated sandbox managed health guard stayed inconclusive within the readiness deadline, so the primary dashboard/API host forward was not started"; + return "the recovered sandbox managed health guard stayed inconclusive within the readiness deadline, so the primary dashboard/API host forward was not started"; case "openshell-readiness-failure": - return "the recreated sandbox did not become ready in OpenShell, so the primary dashboard/API host forward was not started"; + return "the recovered sandbox did not become ready in OpenShell, so the primary dashboard/API host forward was not started"; } })(); return openshellError ? `${detail} Last OpenShell readiness error: ${openshellError}` : detail; @@ -713,8 +713,8 @@ const GATEWAY_RECOVERY_WAIT_DEFAULT_SECONDS = 120; /** * Wait until OpenShell has re-registered a directly recreated sandbox as * ready. This probe deliberately has no direct-Docker or SSH fallback: it is - * proving control-plane readiness, not authorizing the already completed - * replacement-container recovery. + * proving control-plane readiness, not bypassing OpenShell after the managed + * workload has restarted. */ function waitForRecreatedSandboxOpenShellReadyResult( sandboxName: string, @@ -844,7 +844,7 @@ function printHostManagedGatewayRecoveryHints( ): void { const quotedSandboxName = shellQuote(sandboxName); if (failureLayer === "supervisor not running") { - console.error(" The in-sandbox supervisor is not running, and trusted container recovery"); + console.error(" The in-sandbox supervisor is not running, and trusted supervisor recovery"); console.error(" could not restore a managed supervisor and healthy gateway."); console.error(" Recreate the sandbox runtime to restore it:"); console.error(` nemoclaw ${quotedSandboxName} rebuild --yes`); @@ -1265,32 +1265,13 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( }), }); } catch (error) { - try { - relaunch?.finalize(false); - } catch { - // Preserve the original recovery error; the failure path below will - // direct the operator to inspect/rebuild the sandbox. - } throw error; } if (!gatewayReady) { - let rolledBack = true; - if (relaunch) { - try { - rolledBack = relaunch.finalize(false).rolledBack; - } catch { - rolledBack = false; - } - } if (!quiet) { console.error(" Gateway process started but is not responding."); printGatewayWedgeDiagnostics(sandboxName, executeSandboxExecCommand); console.error(" Check /tmp/gateway.log inside the sandbox for details."); - if (!rolledBack) { - console.error( - " Automatic rollback of the previous sandbox container failed; inspect Docker state before retrying.", - ); - } printHostManagedGatewayRecoveryHints( sandboxName, recoveryAgent, @@ -1320,17 +1301,6 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( })() : null; if (readinessFailureDetail) { - let rolledBack = true; - try { - rolledBack = relaunch?.finalize(false).rolledBack ?? true; - } catch { - rolledBack = false; - } - if (!rolledBack && !quiet) { - console.error( - " Automatic rollback of the previous sandbox container failed; inspect Docker state before retrying.", - ); - } return { checked: true, wasRunning: false, @@ -1340,52 +1310,6 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( forwardRecoveryFailureDetail: readinessFailureDetail, }; } - if (relaunch) { - try { - const completion = relaunch.finalize(true); - if (completion.stateRestored === false || completion.rolledBack) { - if (!quiet) { - console.error( - completion.rolledBack - ? " Sandbox recovery did not complete; the previous container was restored." - : " Sandbox recovery failed and the previous container could not be restored automatically.", - ); - if (completion.rolledBack && completion.stateBackupRemoved === false) { - console.error(" Warning: the temporary sandbox state backup could not be removed."); - } - if (!completion.rolledBack) { - printHostManagedGatewayRecoveryHints( - sandboxName, - recoveryAgent, - managedRecoveryFailureLayer, - ); - } - } - return { - checked: true, - wasRunning: false, - recovered: false, - forwardRecovered: false, - }; - } - if (!completion.backupRemoved && !quiet) { - console.error( - " Warning: the recovered sandbox is healthy, but its previous container backup could not be removed.", - ); - } - if (completion.stateBackupRemoved === false && !quiet) { - console.error( - " Warning: the recovered sandbox is healthy, but its temporary state backup could not be removed.", - ); - } - } catch { - if (!quiet) { - console.error( - " Warning: the recovered sandbox is healthy, but container transaction cleanup could not be confirmed.", - ); - } - } - } const mcpRefusal = processRecoveryMcpReconciliationRefusal(sandboxName, false); if (mcpRefusal) return mcpRefusal; const forwardRecovered = ensureSandboxPortForward(sandboxName, { diff --git a/src/lib/actions/sandbox/supervisor-relaunch.test.ts b/src/lib/actions/sandbox/supervisor-relaunch.test.ts index 28f638085a0..36f8cd721c6 100644 --- a/src/lib/actions/sandbox/supervisor-relaunch.test.ts +++ b/src/lib/actions/sandbox/supervisor-relaunch.test.ts @@ -2,7 +2,6 @@ // SPDX-License-Identifier: Apache-2.0 import { afterEach, describe, expect, it, vi } from "vitest"; -import type { DockerGpuPatchResult } from "../../onboard/docker-gpu-patch"; import { type ManagedSupervisorRelaunchDeps, relaunchManagedSupervisorSession, @@ -13,23 +12,6 @@ afterEach(() => { vi.unstubAllEnvs(); }); -function patchResult(): DockerGpuPatchResult { - return { - applied: true, - oldContainerId: "old-container-id", - newContainerId: "new-container-id", - originalName: "openshell-alpha", - backupContainerName: "openshell-alpha-nemoclaw-backup", - mode: { - kind: "startup-command", - label: "persistent sandbox startup command", - device: "", - args: [], - }, - backupRemoved: false, - }; -} - function baseDeps(overrides: ManagedSupervisorRelaunchDeps = {}) { return { getSandbox: vi.fn(() => ({ @@ -47,38 +29,12 @@ function baseDeps(overrides: ManagedSupervisorRelaunchDeps = {}) { }) as never, ), resolveDashboardPort: vi.fn(() => 18789), - resolveContainer: vi - .fn() - .mockReturnValueOnce("old-container-id") - .mockReturnValue("new-container-id"), + resolveContainer: vi.fn(() => "original-container-id"), inspectContainer: vi.fn(() => ({ Config: { Env: ["OPENSHELL_SANDBOX_COMMAND=sleep infinity"] }, })), confirmMissingSupervisor: vi.fn(() => true), - backupState: vi.fn(() => ({ - success: true, - manifest: { - backupPath: "/tmp/rebuild-backups/alpha/recovery", - }, - backedUpDirs: ["workspace"], - failedDirs: [], - backedUpFiles: [], - failedFiles: [], - })) as never, - restoreState: vi.fn(() => ({ - success: true, - restoredDirs: ["workspace"], - failedDirs: [], - restoredFiles: [], - failedFiles: [], - })), - removeBackup: vi.fn(() => true), - recreate: vi.fn(() => patchResult()), - finalize: vi.fn(({ supervisorReady }) => - supervisorReady - ? { backupRemoved: true, rolledBack: false } - : { backupRemoved: false, rolledBack: true }, - ), + startSupervisor: vi.fn(() => ({ started: true as const })), ...overrides, } satisfies ManagedSupervisorRelaunchDeps; } @@ -89,7 +45,7 @@ describe("relaunchManagedSupervisorSession", () => { expect(relaunchManagedSupervisorSession("missing-box", { quiet: true, deps })).toBeNull(); expect(deps.resolveContainer).not.toHaveBeenCalled(); - expect(deps.recreate).not.toHaveBeenCalled(); + expect(deps.startSupervisor).not.toHaveBeenCalled(); }); it("honors the troubleshooting kill switch without mutating Docker", () => { @@ -98,7 +54,7 @@ describe("relaunchManagedSupervisorSession", () => { expect(relaunchManagedSupervisorSession("alpha", { quiet: true, deps })).toBeNull(); expect(deps.resolveContainer).not.toHaveBeenCalled(); - expect(deps.recreate).not.toHaveBeenCalled(); + expect(deps.startSupervisor).not.toHaveBeenCalled(); }); it("refuses a container that no longer has the legacy keepalive startup", () => { @@ -109,18 +65,18 @@ describe("relaunchManagedSupervisorSession", () => { }); expect(relaunchManagedSupervisorSession("alpha", { quiet: true, deps })).toBeNull(); - expect(deps.recreate).not.toHaveBeenCalled(); + expect(deps.startSupervisor).not.toHaveBeenCalled(); }); - it("refuses recreation when the pinned container no longer proves supervisor absence", () => { + it("refuses recovery when the pinned container no longer proves supervisor absence", () => { const deps = baseDeps({ confirmMissingSupervisor: vi.fn(() => false) }); expect(relaunchManagedSupervisorSession("alpha", { quiet: true, deps })).toBeNull(); - expect(deps.confirmMissingSupervisor).toHaveBeenCalledWith("old-container-id"); - expect(deps.recreate).not.toHaveBeenCalled(); + expect(deps.confirmMissingSupervisor).toHaveBeenCalledWith("original-container-id"); + expect(deps.startSupervisor).not.toHaveBeenCalled(); }); - it("pins the selected container and persists only a credential-free startup command", () => { + it("restarts the supervisor in the registered container without exposing credentials", () => { vi.stubEnv("NEMOCLAW_EXTRA_PLACEHOLDER_KEYS", "CUSTOM_PROVIDER_CREDENTIAL"); vi.stubEnv("CUSTOM_PROVIDER_CREDENTIAL", "s3cr3t-token"); vi.stubEnv("HTTPS_PROXY", "http://proxyuser:proxypass@proxy.example:8080"); @@ -128,188 +84,27 @@ describe("relaunchManagedSupervisorSession", () => { const relaunch = relaunchManagedSupervisorSession("alpha", { quiet: true, deps }); - expect(relaunch).not.toBeNull(); - expect(relaunch?.containerId).toBe("new-container-id"); - expect(deps.recreate).toHaveBeenCalledOnce(); - const options = vi.mocked(deps.recreate).mock.calls[0]?.[0]; - expect(options).toMatchObject({ - sandboxName: "alpha", - expectedOldContainerId: "old-container-id", - waitForSupervisor: false, - }); - const serialized = options?.openshellSandboxCommand.join(" ") ?? ""; + expect(relaunch?.containerId).toBe("original-container-id"); + expect(deps.startSupervisor).toHaveBeenCalledOnce(); + const [containerId, command] = vi.mocked(deps.startSupervisor).mock.calls[0] ?? []; + expect(containerId).toBe("original-container-id"); + const serialized = command?.join(" ") ?? ""; expect(serialized).toContain("NEMOCLAW_DASHBOARD_PORT=18789"); expect(serialized).toMatch(/nemoclaw-start$/); expect(serialized).not.toContain("s3cr3t-token"); expect(serialized).not.toContain("CUSTOM_PROVIDER_CREDENTIAL"); expect(serialized).not.toContain("proxypass"); - - expect(relaunch?.finalize(true)).toEqual({ - backupRemoved: true, - rolledBack: false, - stateRestored: true, - stateBackupRemoved: true, - }); - expect(deps.restoreState).toHaveBeenCalledWith("alpha", "/tmp/rebuild-backups/alpha/recovery"); - expect(deps.removeBackup).toHaveBeenCalledWith("alpha", "/tmp/rebuild-backups/alpha/recovery"); - expect(deps.finalize).toHaveBeenCalledWith({ - result: expect.objectContaining({ newContainerId: "new-container-id" }), - supervisorReady: true, - }); - }); - - it("rolls the container transaction back when managed readiness is not proven", () => { - const deps = baseDeps(); - const relaunch = relaunchManagedSupervisorSession("alpha", { quiet: true, deps }); - - expect(relaunch?.finalize(false)).toEqual({ - backupRemoved: false, - rolledBack: true, - stateRestored: false, - stateBackupRemoved: true, - }); - expect(deps.restoreState).not.toHaveBeenCalled(); - expect(deps.removeBackup).toHaveBeenCalledWith("alpha", "/tmp/rebuild-backups/alpha/recovery"); - expect(deps.finalize).toHaveBeenCalledWith({ - result: expect.objectContaining({ backupContainerName: expect.any(String) }), - supervisorReady: false, - }); - }); - - it("removes a partial state backup before it refuses recreation (#7404)", () => { - const deps = baseDeps({ - backupState: vi.fn(() => ({ - success: false, - manifest: { - backupPath: "/tmp/rebuild-backups/alpha/partial-recovery", - } as never, - backedUpDirs: [], - failedDirs: ["workspace"], - backedUpFiles: [], - failedFiles: [], - })), - }); - - expect(relaunchManagedSupervisorSession("alpha", { quiet: true, deps })).toBeNull(); - expect(deps.removeBackup).toHaveBeenCalledWith( - "alpha", - "/tmp/rebuild-backups/alpha/partial-recovery", - ); - expect(deps.recreate).not.toHaveBeenCalled(); - }); - - it("rolls back the container transaction when state restore fails", () => { - const deps = baseDeps({ - restoreState: vi.fn(() => ({ - success: false, - restoredDirs: [], - failedDirs: ["workspace"], - restoredFiles: [], - failedFiles: [], - })), - }); - const relaunch = relaunchManagedSupervisorSession("alpha", { quiet: true, deps }); - - expect(relaunch?.finalize(true)).toEqual({ - backupRemoved: false, - rolledBack: true, - stateRestored: false, - stateBackupRemoved: true, - }); - expect(deps.removeBackup).toHaveBeenCalledWith("alpha", "/tmp/rebuild-backups/alpha/recovery"); - expect(deps.finalize).toHaveBeenCalledWith({ - result: expect.objectContaining({ backupContainerName: expect.any(String) }), - supervisorReady: false, - }); - }); - - it("rolls back before restore when the replacement container identity changes", () => { - const deps = baseDeps({ - resolveContainer: vi - .fn() - .mockReturnValueOnce("old-container-id") - .mockReturnValue("different-container-id"), - }); - const relaunch = relaunchManagedSupervisorSession("alpha", { quiet: true, deps }); - - expect(relaunch?.finalize(true)).toEqual({ - backupRemoved: false, - rolledBack: true, - stateRestored: false, - stateBackupRemoved: true, - }); - expect(deps.restoreState).not.toHaveBeenCalled(); - expect(deps.removeBackup).toHaveBeenCalledWith("alpha", "/tmp/rebuild-backups/alpha/recovery"); - expect(deps.finalize).toHaveBeenCalledWith({ - result: expect.objectContaining({ backupContainerName: expect.any(String) }), - supervisorReady: false, - }); - }); - - it("retains the state backup when rollback fails", () => { - const deps = baseDeps({ - finalize: vi.fn(() => ({ backupRemoved: false, rolledBack: false })), - }); - const relaunch = relaunchManagedSupervisorSession("alpha", { quiet: true, deps }); - - expect(relaunch?.finalize(false)).toEqual({ - backupRemoved: false, - rolledBack: false, - stateRestored: false, - }); - expect(deps.removeBackup).not.toHaveBeenCalled(); - }); - - it("reports best-effort state-backup cleanup failure after a successful restore", () => { - const deps = baseDeps({ removeBackup: vi.fn(() => false) }); - const relaunch = relaunchManagedSupervisorSession("alpha", { quiet: true, deps }); - - expect(relaunch?.finalize(true)).toEqual({ - backupRemoved: true, - rolledBack: false, - stateRestored: true, - stateBackupRemoved: false, - }); }); - it("returns null when the pinned recreation fails", () => { - const deps = baseDeps({ - recreate: vi.fn(() => { - throw new Error("container identity changed"); - }), - }); - - expect(relaunchManagedSupervisorSession("alpha", { quiet: true, deps })).toBeNull(); - expect(deps.removeBackup).toHaveBeenCalledWith("alpha", "/tmp/rebuild-backups/alpha/recovery"); - }); - - it("preserves the recreation diagnostic when state-backup cleanup throws", () => { + it("returns null and redacts diagnostics when the pinned start fails", () => { vi.spyOn(console, "log").mockImplementation(() => undefined); const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); const deps = baseDeps({ - removeBackup: vi.fn(() => { - throw new Error("backup cleanup failed"); - }), - recreate: vi.fn(() => { - throw new Error("container identity changed"); - }), - }); - - expect(relaunchManagedSupervisorSession("alpha", { quiet: false, deps })).toBeNull(); - const output = errorSpy.mock.calls.flat().join("\n"); - expect(output).toContain("container identity changed"); - expect(output).not.toContain("backup cleanup failed"); - }); - - it("redacts diagnostics when trusted recreation fails", () => { - vi.spyOn(console, "log").mockImplementation(() => undefined); - const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); - const deps = baseDeps({ - recreate: vi.fn(() => { - throw new Error( + startSupervisor: vi.fn(() => ({ + started: false, + detail: "OPENAI_API_KEY=sk-recovery-secret HTTPS_PROXY=http://proxyuser:proxypass@proxy.example:8080", - ); - }), + })), }); expect(relaunchManagedSupervisorSession("alpha", { quiet: false, deps })).toBeNull(); diff --git a/src/lib/actions/sandbox/supervisor-relaunch.ts b/src/lib/actions/sandbox/supervisor-relaunch.ts index ff9a2f48740..d9afe40b6ce 100644 --- a/src/lib/actions/sandbox/supervisor-relaunch.ts +++ b/src/lib/actions/sandbox/supervisor-relaunch.ts @@ -1,46 +1,40 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { dockerCapture } from "../../adapters/docker"; +import { dockerCapture, dockerSpawnSync } from "../../adapters/docker"; import * as agentRuntime from "../../agent/runtime"; import { shouldManageDashboardForAgent } from "../../onboard/dashboard-runtime"; import { type DockerContainerInspect, parseDockerInspectJson, } from "../../onboard/docker-gpu-patch"; -import { sameContainerId } from "../../onboard/docker-gpu-patch-clone"; -import { - type DockerGpuPatchFinalizeOutcome, - finalizeDockerGpuPatchBackup, -} from "../../onboard/docker-gpu-patch-finalize"; -import { recreateOpenShellDockerSandboxWithStartupCommand } from "../../onboard/docker-startup-command-patch"; import { buildSandboxRuntimeEnvArgs } from "../../onboard/sandbox-create-launch"; -import { resolveDirectSandboxContainer } from "../../sandbox/privileged-exec"; +import { + privilegedSandboxExecArgv, + resolveDirectSandboxContainer, +} from "../../sandbox/privileged-exec"; import { redact, redactFull } from "../../security/redact"; import * as registry from "../../state/registry"; -import * as sandboxState from "../../state/sandbox"; import { resolveSandboxDashboardPort } from "./forward-recovery"; /** * Compatibility boundary for OpenShell 0.0.71's Docker driver: legacy * sandboxes persist `OPENSHELL_SANDBOX_COMMAND=sleep infinity` while * `scripts/nemoclaw-start.sh` owns the managed workload as a sibling process. - * Only that inspected value authorizes this migration. Regression coverage is - * named in `supervisor-relaunch.test.ts` and `gateway-guard-recovery.test.ts`. - * Remove this path after supported upgrades rebuild every legacy keepalive - * container with `nemoclaw-start` as its persisted startup command. + * Restart that sibling in the registered container so OpenShell identity and + * the complete writable layer survive gateway restarts. Remove this path after + * supported upgrades rebuild every legacy keepalive container with + * `nemoclaw-start` as its persisted startup command. */ const LEGACY_OPENSHELL_KEEPALIVE = "sleep infinity"; const DOCKER_INSPECT_TIMEOUT_MS = 15000; export type ManagedSupervisorRelaunch = { containerId: string; - finalize(supervisorReady: boolean): DockerGpuPatchFinalizeOutcome & { - stateRestored?: boolean; - stateBackupRemoved?: boolean; - }; }; +type SupervisorStartResult = { started: true } | { detail: string; started: false }; + export type ManagedSupervisorRelaunchDeps = { getSandbox?: typeof registry.getSandbox; getSessionAgent?: typeof agentRuntime.getSessionAgent; @@ -48,11 +42,7 @@ export type ManagedSupervisorRelaunchDeps = { resolveContainer?: typeof resolveDirectSandboxContainer; inspectContainer?: (containerId: string) => DockerContainerInspect; confirmMissingSupervisor?: (containerId: string) => boolean; - backupState?: typeof sandboxState.backupSandboxState; - restoreState?: typeof sandboxState.restoreSandboxState; - removeBackup?: typeof sandboxState.removeSandboxStateBackup; - recreate?: typeof recreateOpenShellDockerSandboxWithStartupCommand; - finalize?: typeof finalizeDockerGpuPatchBackup; + startSupervisor?: (containerId: string, command: readonly string[]) => SupervisorStartResult; }; function inspectContainer(containerId: string): DockerContainerInspect { @@ -114,6 +104,38 @@ function reconstructSupervisorLaunchCommand( return ["env", ...envArgs, "nemoclaw-start"]; } +function startSupervisorInContainer( + sandboxName: string, + containerId: string, + command: readonly string[], +): SupervisorStartResult { + try { + const [operation, ...args] = privilegedSandboxExecArgv( + sandboxName, + [...command], + false, + true, + containerId, + ); + const result = dockerSpawnSync([operation, "--detach", ...args], { + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + timeout: DOCKER_INSPECT_TIMEOUT_MS, + }); + if (!result.error && result.status === 0) return { started: true }; + const detail = + result.error?.message || + String(result.stderr || "").trim() || + `docker exec exited with status ${String(result.status ?? "unknown")}`; + return { detail, started: false }; + } catch (error) { + return { + detail: error instanceof Error ? error.message : String(error), + started: false, + }; + } +} + export function relaunchManagedSupervisorSession( sandboxName: string, { @@ -135,141 +157,30 @@ export function relaunchManagedSupervisorSession( const resolveContainer = deps.resolveContainer ?? resolveDirectSandboxContainer; const inspect = deps.inspectContainer ?? inspectContainer; - const confirmMissingSupervisor = deps.confirmMissingSupervisor; - const backupState = deps.backupState ?? sandboxState.backupSandboxState; - const restoreState = deps.restoreState ?? sandboxState.restoreSandboxState; - const removeBackup = deps.removeBackup ?? sandboxState.removeSandboxStateBackup; - const recreate = deps.recreate ?? recreateOpenShellDockerSandboxWithStartupCommand; - const finalize = deps.finalize ?? finalizeDockerGpuPatchBackup; - let pendingStateBackupPath: string | null = null; + const startSupervisor = + deps.startSupervisor ?? + ((containerId, command) => startSupervisorInContainer(sandboxName, containerId, command)); try { const containerId = resolveContainer(sandboxName, driver); if (!hasLegacyKeepaliveStartup(inspect(containerId))) return null; - if (!confirmMissingSupervisor?.(containerId)) return null; - const backup = backupState(sandboxName); - if ( - !backup.success || - !backup.manifest || - backup.failedDirs.length > 0 || - backup.failedFiles.length > 0 - ) { - if (backup.manifest) { - try { - removeBackup(sandboxName, backup.manifest.backupPath); - } catch { - // Preserve the backup failure that stopped container recreation. - } - } + if (!deps.confirmMissingSupervisor?.(containerId)) return null; + if (!quiet) { + console.log(" Restarting the managed workload in the existing sandbox container..."); + } + const start = startSupervisor(containerId, startupCommand); + if (!start.started) { if (!quiet) { console.error( - " Trusted container recovery stopped before recreation because sandbox state could not be fully backed up.", + ` Trusted supervisor recovery could not start: ${redactFull(redact(start.detail))}`, ); - console.error(" The existing sandbox container was left unchanged."); } return null; } - const backupManifest = backup.manifest; - pendingStateBackupPath = backupManifest.backupPath; - if (!quiet) { - console.log(" Recreating the sandbox container with its managed startup command..."); - } - const result = recreate({ - sandboxName, - openshellSandboxCommand: startupCommand, - expectedOldContainerId: containerId, - waitForSupervisor: false, - }); - pendingStateBackupPath = null; - let completed: { - supervisorReady: boolean; - outcome: DockerGpuPatchFinalizeOutcome & { - stateRestored?: boolean; - stateBackupRemoved?: boolean; - }; - } | null = null; - const removeSettledStateBackup = (): boolean => { - try { - return removeBackup(sandboxName, backupManifest.backupPath); - } catch { - return false; - } - }; - return { - containerId: result.newContainerId, - finalize(supervisorReady) { - if (completed) { - if (completed.supervisorReady !== supervisorReady) { - throw new Error( - "Supervisor relaunch transaction was finalized with conflicting state.", - ); - } - return completed.outcome; - } - if (!supervisorReady) { - const finalized = finalize({ result, supervisorReady: false }); - const outcome = { - ...finalized, - stateRestored: false, - ...(finalized.rolledBack ? { stateBackupRemoved: removeSettledStateBackup() } : {}), - }; - completed = { supervisorReady, outcome }; - return outcome; - } - let replacementOwned = false; - try { - replacementOwned = sameContainerId( - resolveContainer(sandboxName, driver), - result.newContainerId, - ); - } catch { - replacementOwned = false; - } - if (!replacementOwned) { - const finalized = finalize({ result, supervisorReady: false }); - const outcome = { - ...finalized, - stateRestored: false, - ...(finalized.rolledBack ? { stateBackupRemoved: removeSettledStateBackup() } : {}), - }; - completed = { supervisorReady, outcome }; - return outcome; - } - let stateRestored = false; - try { - stateRestored = restoreState(sandboxName, backupManifest.backupPath).success; - } catch { - stateRestored = false; - } - if (!stateRestored) { - const finalized = finalize({ result, supervisorReady: false }); - const outcome = { - ...finalized, - stateRestored: false, - ...(finalized.rolledBack ? { stateBackupRemoved: removeSettledStateBackup() } : {}), - }; - completed = { supervisorReady, outcome }; - return outcome; - } - const outcome = { - ...finalize({ result, supervisorReady: true }), - stateRestored: true, - stateBackupRemoved: removeSettledStateBackup(), - }; - completed = { supervisorReady, outcome }; - return outcome; - }, - }; + return { containerId }; } catch (error) { - if (pendingStateBackupPath) { - try { - removeBackup(sandboxName, pendingStateBackupPath); - } catch { - // Preserve the recreation failure that stopped container recovery. - } - } if (!quiet) { const detail = error instanceof Error ? error.message : String(error); - console.error(` Trusted container recovery could not start: ${redactFull(redact(detail))}`); + console.error(` Trusted supervisor recovery could not start: ${redactFull(redact(detail))}`); } return null; } diff --git a/src/lib/onboard/docker-startup-command-patch.test.ts b/src/lib/onboard/docker-startup-command-patch.test.ts index 9c37c6119b0..71c5c9d12c2 100644 --- a/src/lib/onboard/docker-startup-command-patch.test.ts +++ b/src/lib/onboard/docker-startup-command-patch.test.ts @@ -55,7 +55,6 @@ describe("Docker startup-command patch", () => { status: 0, stdout: "new-container-id\n", })); - const dockerStop = vi.fn(() => ({ status: 0 })); const result = recreateStartupCommandForTest( { @@ -68,7 +67,7 @@ describe("Docker startup-command patch", () => { dockerCapture, dockerRunDetached, dockerRename: vi.fn(() => ({ status: 0 })), - dockerStop, + dockerStop: vi.fn(() => ({ status: 0 })), sleep: vi.fn(), now: () => new Date("2026-07-10T00:00:00Z"), }, @@ -95,9 +94,6 @@ describe("Docker startup-command patch", () => { expect.arrayContaining(["ps", "-a", "--no-trunc"]), expect.objectContaining({ ignoreError: true }), ); - expect(dockerStop.mock.invocationCallOrder[0]).toBeLessThan( - dockerRunDetached.mock.invocationCallOrder[0] ?? Number.POSITIVE_INFINITY, - ); }); it("preserves OpenShell's native CDI GPU request during restart-persistence recreation", () => { diff --git a/test/process-recovery-supervisor-relaunch.test.ts b/test/process-recovery-supervisor-relaunch.test.ts index 33b4051ac09..17db4b165ce 100644 --- a/test/process-recovery-supervisor-relaunch.test.ts +++ b/test/process-recovery-supervisor-relaunch.test.ts @@ -99,15 +99,15 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { stderr: "SUPERVISOR_NOT_RUNNING", })); const resolveContainer = vi.fn(() => "old-container-id"); - const recreate = vi.fn(() => { - throw new Error("kill switch allowed container mutation"); + const startSupervisor = vi.fn(() => { + throw new Error("kill switch allowed supervisor mutation"); }); const requestPinnedGatewaySupervisorAction = vi.fn(() => null); const relaunchManagedSupervisorSessionImpl = vi.fn( (sandboxName: string, options: Parameters[1]) => relaunchManagedSupervisorSession(sandboxName, { quiet: options.quiet, - deps: { ...options.deps, resolveContainer, recreate }, + deps: { ...options.deps, resolveContainer, startSupervisor }, }), ); const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); @@ -129,25 +129,23 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { ); expect(resolveContainer).not.toHaveBeenCalled(); expect(requestPinnedGatewaySupervisorAction).not.toHaveBeenCalled(); - expect(recreate).not.toHaveBeenCalled(); + expect(startSupervisor).not.toHaveBeenCalled(); const errorLines = errorSpy.mock.calls.map((call) => String(call[0])); expect(errorLines).toContainEqual( expect.stringContaining("Failure layer: supervisor not running"), ); - expect(errorLines).toContainEqual(expect.stringContaining("trusted container recovery")); + expect(errorLines).toContainEqual(expect.stringContaining("trusted supervisor recovery")); expect(errorLines).toContainEqual(expect.stringContaining("rebuild --yes")); expect(errorLines).not.toContainEqual( expect.stringContaining("Retry the managed restart from the host"), ); }); - it("rolls back when recreation starts but managed control never accepts it", () => { + it("settles the in-place start when managed control never accepts it", () => { mockOpenClawSandbox("rejected-box"); setImmediateRecoveryPolling(); - const finalize = vi.fn(() => ({ backupRemoved: false, rolledBack: true })); const relaunchManagedSupervisorSessionImpl = vi.fn(() => ({ - containerId: "replacement-container-id", - finalize, + containerId: "original-container-id", })); const requestGatewaySupervisorAction = vi.fn((_name: string, action: string) => action === "recover" ? { status: 1, stdout: "", stderr: "SUPERVISOR_NOT_RUNNING" } : null, @@ -167,23 +165,15 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { "rejected-box", "probe", 210000, - "replacement-container-id", + "original-container-id", ); - expect(finalize).toHaveBeenCalledOnce(); - expect(finalize).toHaveBeenCalledWith(false); }); - it("commits only after managed health accepts the recreated supervisor", () => { + it("confirms the in-place start only after managed health accepts it", () => { mockOpenClawSandbox("recovered-box"); setImmediateRecoveryPolling(); - const finalize = vi.fn((supervisorReady: boolean) => - supervisorReady - ? { backupRemoved: true, rolledBack: false } - : { backupRemoved: false, rolledBack: true }, - ); const relaunchManagedSupervisorSessionImpl = vi.fn(() => ({ - containerId: "replacement-container-id", - finalize, + containerId: "original-container-id", })); const requestGatewaySupervisorAction = vi.fn((_name: string, action: string) => action === "recover" ? { status: 1, stdout: "", stderr: "SUPERVISOR_NOT_RUNNING" } : null, @@ -214,117 +204,18 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { "recovered-box", "probe", 210000, - "replacement-container-id", - ); - expect(finalize).toHaveBeenCalledOnce(); - expect(finalize).toHaveBeenCalledWith(true); - }); - - it("reports recovery failure when state restore rolls the replacement back", () => { - mockOpenClawSandbox("restore-failed-box"); - setImmediateRecoveryPolling(); - const finalize = vi.fn(() => ({ - backupRemoved: false, - rolledBack: true, - stateRestored: false, - })); - const relaunchManagedSupervisorSessionImpl = vi.fn(() => ({ - containerId: "replacement-container-id", - finalize, - })); - const requestGatewaySupervisorAction = vi.fn((_name: string, action: string) => - action === "recover" ? { status: 1, stdout: "", stderr: "SUPERVISOR_NOT_RUNNING" } : null, - ); - const requestPinnedGatewaySupervisorAction = vi.fn(() => ({ - status: 0, - stdout: "GATEWAY_PID=4242\n", - stderr: "", - })); - const waitForRecreatedSandboxOpenShellReadyImpl = vi.fn(() => true); - const runOpenshell = vi.spyOn(openshellRuntime, "runOpenshell"); - - const result = checkAndRecoverSandboxProcesses("restore-failed-box", { - quiet: true, - isSandboxGatewayRunningImpl: () => false, - requestGatewaySupervisorAction, - requestPinnedGatewaySupervisorAction, - relaunchManagedSupervisorSessionImpl, - waitForRecreatedSandboxOpenShellReadyImpl, - }); - - expect(result).toMatchObject({ - checked: true, - wasRunning: false, - recovered: false, - forwardRecovered: false, - }); - expect(finalize).toHaveBeenCalledOnce(); - expect(finalize).toHaveBeenCalledWith(true); - expect(waitForRecreatedSandboxOpenShellReadyImpl).toHaveBeenCalledOnce(); - expect(runOpenshell).not.toHaveBeenCalled(); - }); - - it("prints generic recovery hints when state recovery and rollback both fail", () => { - mockOpenClawSandbox("restore-and-rollback-failed-box"); - setImmediateRecoveryPolling(); - const finalize = vi.fn(() => ({ - backupRemoved: false, - rolledBack: false, - stateRestored: false, - })); - const relaunchManagedSupervisorSessionImpl = vi.fn(() => ({ - containerId: "replacement-container-id", - finalize, - })); - const requestGatewaySupervisorAction = vi.fn((_name: string, action: string) => - action === "recover" ? { status: 1, stdout: "", stderr: "SUPERVISOR_NOT_RUNNING" } : null, + "original-container-id", ); - const requestPinnedGatewaySupervisorAction = vi.fn(() => ({ - status: 0, - stdout: "GATEWAY_PID=4242\n", - stderr: "", - })); - const waitForRecreatedSandboxOpenShellReadyImpl = vi.fn(() => true); - vi.spyOn(console, "log").mockImplementation(() => undefined); - const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); - - const result = checkAndRecoverSandboxProcesses("restore-and-rollback-failed-box", { - quiet: false, - isSandboxGatewayRunningImpl: () => false, - requestGatewaySupervisorAction, - requestPinnedGatewaySupervisorAction, - relaunchManagedSupervisorSessionImpl, - waitForRecreatedSandboxOpenShellReadyImpl, - }); - - expect(result).toMatchObject({ - checked: true, - wasRunning: false, - recovered: false, - forwardRecovered: false, - }); - const output = errorSpy.mock.calls.flat().join("\n"); - expect(output).toContain( - "Sandbox recovery failed and the previous container could not be restored automatically.", - ); - expect(output).toContain("rebuild --yes"); - expect(output).not.toContain("Sandbox state restore failed"); }); - it("retries a busy pinned managed probe before starting the replacement forward", () => { + it("retries a busy pinned managed probe before starting the recovered forward", () => { mockOpenClawSandbox("busy-recovered-box"); vi.stubEnv("NEMOCLAW_GATEWAY_RECOVERY_POLL_INTERVAL_SECONDS", "0"); vi.stubEnv("NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS", "1"); vi.stubEnv("NEMOCLAW_GATEWAY_RECOVERY_SETTLE_SECONDS", "0"); vi.stubEnv("NEMOCLAW_FORWARD_RECOVERY_WAIT_MS", "0"); - const finalize = vi.fn((supervisorReady: boolean) => - supervisorReady - ? { backupRemoved: true, rolledBack: false } - : { backupRemoved: false, rolledBack: true }, - ); const relaunchManagedSupervisorSessionImpl = vi.fn(() => ({ - containerId: "replacement-container-id", - finalize, + containerId: "original-container-id", })); const requestGatewaySupervisorAction = vi.fn((_name: string, action: string) => action === "recover" ? { status: 1, stdout: "", stderr: "SUPERVISOR_NOT_RUNNING" } : null, @@ -392,10 +283,6 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { ["sandbox", "exec", "--name", "busy-recovered-box", "--", "true"], expect.objectContaining({ ignoreError: true }), ); - expect(finalize).toHaveBeenCalledWith(true); - expect(captureOpenshell.mock.invocationCallOrder[0]).toBeLessThan( - finalize.mock.invocationCallOrder[0] ?? Number.POSITIVE_INFINITY, - ); expect(runOpenshell).toHaveBeenCalledWith( ["forward", "start", "--background", "18789", "busy-recovered-box"], expect.objectContaining({ ignoreError: true }), @@ -405,10 +292,8 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { it("uses the sandbox readiness budget after a longer gateway health wait (#7273)", () => { mockOpenClawSandbox("unready-box", 600); setImmediateRecoveryPolling(); - const finalize = vi.fn(() => ({ backupRemoved: true, rolledBack: false })); const relaunchManagedSupervisorSessionImpl = vi.fn(() => ({ - containerId: "replacement-container-id", - finalize, + containerId: "original-container-id", })); const requestGatewaySupervisorAction = vi.fn(() => ({ status: 1, @@ -440,8 +325,6 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { forwardRecoveryFailed: true, forwardRecoveryFailureDetail: expect.stringContaining("did not become ready in OpenShell"), }); - expect(finalize).toHaveBeenCalledOnce(); - expect(finalize).toHaveBeenCalledWith(false); expect(waitForRecreatedSandboxOpenShellReadyImpl).toHaveBeenCalledWith( "unready-box", expect.objectContaining({ beforeProbe: expect.any(Function), timeoutSeconds: 180 }), @@ -452,10 +335,8 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { it("reports the last structured OpenShell error when readiness times out", () => { mockOpenClawSandbox("relay-dropped-box"); setImmediateRecoveryPolling(); - const finalize = vi.fn(() => ({ backupRemoved: true, rolledBack: false })); const relaunchManagedSupervisorSessionImpl = vi.fn(() => ({ - containerId: "replacement-container-id", - finalize, + containerId: "original-container-id", })); const requestGatewaySupervisorAction = vi.fn(() => ({ status: 1, @@ -504,17 +385,14 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { ), }); expect(captureOpenshell).toHaveBeenCalled(); - expect(finalize).toHaveBeenCalledWith(false); expect(runOpenshell).not.toHaveBeenCalled(); }); it("reports a definitive managed health failure separately from OpenShell readiness", () => { mockOpenClawSandbox("managed-failed-box"); setImmediateRecoveryPolling(); - const finalize = vi.fn(() => ({ backupRemoved: true, rolledBack: false })); const relaunchManagedSupervisorSessionImpl = vi.fn(() => ({ - containerId: "replacement-container-id", - finalize, + containerId: "original-container-id", })); const requestGatewaySupervisorAction = vi.fn(() => ({ status: 1, @@ -552,11 +430,10 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { forwardRecoveryFailed: true, forwardRecoveryFailureDetail: expect.stringContaining("failed the managed health guard"), }); - expect(finalize).toHaveBeenCalledWith(false); expect(captureOpenshell).not.toHaveBeenCalled(); }); - it("rejects a healthy forward when the replacement identity changes after readiness", () => { + it("rejects a healthy forward when the original identity changes after readiness", () => { mockOpenClawSandbox("drifted-box"); vi.mocked(agentRuntime.getSessionAgent).mockReturnValue({ name: "openclaw", @@ -566,10 +443,8 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { healthProbe: { url: "http://127.0.0.1:18789/health", port: 18789, timeout_seconds: 30 }, } as never); setImmediateRecoveryPolling(); - const finalize = vi.fn(() => ({ backupRemoved: true, rolledBack: false })); const relaunchManagedSupervisorSessionImpl = vi.fn(() => ({ - containerId: "replacement-container-id", - finalize, + containerId: "original-container-id", })); const requestGatewaySupervisorAction = vi.fn(() => ({ status: 1, @@ -586,7 +461,7 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { .mockReturnValueOnce(acceptedProbe) .mockReturnValueOnce(acceptedProbe) .mockImplementationOnce(() => { - throw new Error("replacement identity changed"); + throw new Error("original identity changed"); }) .mockReturnValue(acceptedProbe); const waitForRecreatedSandboxOpenShellReadyImpl = vi.fn( @@ -622,9 +497,8 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { "drifted-box", "probe", 15000, - "replacement-container-id", + "original-container-id", ); - expect(finalize).toHaveBeenCalledWith(true); expect(runOpenshell).toHaveBeenCalledOnce(); expect(runOpenshell).toHaveBeenCalledWith(["forward", "stop", "18789", "drifted-box"], { ignoreError: true, From 0fc1c1baea2ab37991a1c0bf152e66b1bf979f2a Mon Sep 17 00:00:00 2001 From: San Dang Date: Fri, 31 Jul 2026 01:00:41 +0530 Subject: [PATCH 25/30] docs(recovery): clarify writable state preservation Signed-off-by: San Dang --- docs/manage-sandboxes/recover-rebuild-sandboxes.mdx | 2 +- docs/reference/commands.mdx | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx b/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx index 6d5311d2ba9..13b7d2f25c8 100644 --- a/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx +++ b/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx @@ -81,7 +81,7 @@ Refer to [`$$nemoclaw recover`](../../reference/commands#$$nemoclaw-name- Recovery uses registry-scoped privileged direct-container control and does not fall back to ordinary `openshell sandbox exec` or a manual in-sandbox relaunch. For a local Docker-driver sandbox whose container still uses the legacy keepalive startup, `recover` can restart the credential-free managed workload in the registered container. -NemoClaw keeps the container identity and writable state unchanged, then requires managed gateway health and OpenShell `Ready` before starting the primary dashboard or API host forward. +NemoClaw keeps the container identity and preserves the existing writable-layer state, then requires managed gateway health and OpenShell `Ready` before starting the primary dashboard or API host forward. A definitive managed-health failure stops immediately; if readiness does not complete within the configured budget, the forward stays stopped. For the controller topology, trust boundary, and fail-closed conditions, refer to [Understand Gateway Lifecycle Control](../configure-sandboxes/understand-gateway-lifecycle-control). diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 000f02b50ae..4fbfe83de38 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -1165,7 +1165,13 @@ In a direct root-entrypoint container, the request reaches the root PID 1 superv In an OpenShell-managed container, the request enters the root-owned mode `0500` managed controller through a sanitized root exec while OpenShell remains PID 1. It does not use ordinary `openshell sandbox exec` or an in-sandbox manual relaunch as a fallback. When the root-owned managed controller attests two unchanged zero-supervisor process scans with a stable PID 1 and reports `SUPERVISOR_NOT_RUNNING`, a local Docker-driver sandbox with the legacy keepalive startup can restart its credential-free managed workload in the same registered container. -Recovery pins the container identity and requires managed gateway health, the settle check, and OpenShell readiness while leaving the writable layer unchanged. +Recovery pins the container identity and preserves the existing writable-layer state. +Before starting the host forward, it requires: + +- Managed gateway health. +- A successful settle check. +- OpenShell readiness. + It is idempotent. When `recover` repairs a stopped built-in OpenClaw or Hermes gateway, it repeats the recovery action only for an exit status of `1` with blank stdout and a sole nonblank stderr line equal to `SUPERVISOR_BUSY`, with at most three controller attempts. The same result is inconclusive during managed settle confirmation and can be probed again only within the configured settle window. From 0b64a9c4f8a745949f3d360eb9b4ad1e650d91a1 Mon Sep 17 00:00:00 2001 From: San Dang Date: Fri, 31 Jul 2026 07:54:41 +0530 Subject: [PATCH 26/30] test(e2e): capture sandbox survival diagnostics --- test/e2e/live/sandbox-survival.test.ts | 108 ++++++++++++++++++++++++- 1 file changed, 107 insertions(+), 1 deletion(-) diff --git a/test/e2e/live/sandbox-survival.test.ts b/test/e2e/live/sandbox-survival.test.ts index 3226843f404..a8c0fb16e9e 100644 --- a/test/e2e/live/sandbox-survival.test.ts +++ b/test/e2e/live/sandbox-survival.test.ts @@ -17,7 +17,12 @@ import { cleanupWhenCommandAvailable, cleanupWhenOpenShellAvailable, } from "../fixtures/cleanup-resources.ts"; -import { assertExitZero, resultText, sandboxAccessEnv } from "../fixtures/clients/index.ts"; +import { + assertExitZero, + type HostCliClient, + resultText, + sandboxAccessEnv, +} from "../fixtures/clients/index.ts"; import { trustedProviderEndpoint } from "../fixtures/clients/provider.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; import { requireHostedInferenceConfig } from "../fixtures/hosted-inference.ts"; @@ -29,6 +34,86 @@ const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-survival"; const MIN_OPENSHELL_VERSION = "0.0.24"; const MODEL = process.env.NEMOCLAW_MODEL ?? "nvidia/nemotron-3-super-120b-a12b"; +const SURVIVAL_DIAGNOSTICS_SCRIPT = String.raw` +set +e +sandbox_name="$1" + +printf '%s\n' '== OpenShell sandbox status ==' +openshell sandbox get "$sandbox_name" 2>&1 +printf '%s\n' '== OpenShell forwards ==' +openshell forward list 2>&1 +printf '%s\n' '== OpenShell gateway service ==' +systemctl --user status nemoclaw-openshell-gateway --no-pager -l 2>&1 +printf '%s\n' '== OpenShell gateway journal ==' +journalctl --user -u nemoclaw-openshell-gateway -n 200 --no-pager 2>&1 + +container_ids="$(docker ps -aq \ + --filter label=openshell.ai/managed-by=openshell \ + --filter "label=openshell.ai/sandbox-name=$sandbox_name")" +printf '%s\n' '== matching containers ==' +if [ -n "$container_ids" ]; then + docker ps -a --no-trunc \ + --filter label=openshell.ai/managed-by=openshell \ + --filter "label=openshell.ai/sandbox-name=$sandbox_name" \ + --format '{{.ID}} {{.Names}} {{.Status}}' +else + printf '%s\n' 'none' +fi + +for container_id in $container_ids; do + printf '%s\n' "== container $container_id inspect ==" + docker inspect "$container_id" 2>&1 | node -e ' + const fs = require("node:fs"); + const row = JSON.parse(fs.readFileSync(0, "utf8"))[0] || {}; + const prefix = "OPENSHELL_SANDBOX_COMMAND="; + const matches = (row.Config?.Env || []).filter((entry) => entry.startsWith(prefix)); + const command = matches.length === 1 ? matches[0].slice(prefix.length) : ""; + const tokens = command.trim().split(/\s+/).filter(Boolean); + process.stdout.write(JSON.stringify({ + name: row.Name || "", + configUser: row.Config?.User || "", + state: { + status: row.State?.Status || "", + running: Boolean(row.State?.Running), + restarting: Boolean(row.State?.Restarting), + pid: row.State?.Pid || 0, + exitCode: row.State?.ExitCode ?? null, + error: row.State?.Error || "", + startedAt: row.State?.StartedAt || "", + finishedAt: row.State?.FinishedAt || "", + health: row.State?.Health?.Status || "", + }, + restartPolicy: row.HostConfig?.RestartPolicy?.Name || "", + startupCommandCount: matches.length, + startupCommandIsSleepInfinity: tokens.length === 2 + && tokens[0] === "sleep" && tokens[1] === "infinity", + startupCommandEndsWithNemoclawStart: tokens.length > 0 + && ["nemoclaw-start", "/usr/local/bin/nemoclaw-start"].includes(tokens.at(-1)), + }) + "\n"); + ' + printf '%s\n' "== container $container_id host process tree ==" + docker top "$container_id" -eo pid,ppid,user,stat,comm 2>&1 + printf '%s\n' "== container $container_id runtime state ==" + docker exec "$container_id" sh -lc ' + printf "%s\n" "== pid 1 ==" + cat /proc/1/comm 2>/dev/null || true + printf "\n%s\n" "== process tree ==" + ps -eo user=,pid=,ppid=,stat=,comm= 2>&1 || true + printf "%s\n" "== direct gateway health ==" + curl -q --noproxy "*" -sS -o /dev/null -w "HTTP %{http_code}\n" \ + --connect-timeout 2 --max-time 5 http://127.0.0.1:18789/health 2>&1 || true + printf "%s\n" "== managed controller status ==" + cat /run/nemoclaw/gateway-control/status 2>&1 || true + printf "%s\n" "== startup log ==" + tail -n 300 /tmp/nemoclaw-start.log 2>&1 || true + printf "%s\n" "== gateway log ==" + tail -n 300 /tmp/gateway.log 2>&1 || true + ' 2>&1 + printf '%s\n' "== container $container_id logs ==" + docker logs --tail 300 "$container_id" 2>&1 +done +`; + function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } @@ -61,6 +146,23 @@ function installEnv(hostedEnv: NodeJS.ProcessEnv): NodeJS.ProcessEnv { }; } +async function captureSurvivalDiagnostics( + host: HostCliClient, + stage: string, + redactionValues: string[], +): Promise { + await host.command( + "sh", + ["-lc", SURVIVAL_DIAGNOSTICS_SCRIPT, "sandbox-survival-diagnostics", SANDBOX_NAME], + { + artifactName: `sandbox-survival-${stage}-diagnostics`, + env: buildAvailabilityProbeEnv(), + redactionValues, + timeoutMs: 60_000, + }, + ); +} + async function expectSandboxExecAlive( sandboxName: string, exec: ( @@ -303,19 +405,23 @@ test( await stateValidation.expectSandboxMarkers(instance, markers, "pre-restart-marker-read"); progress.phase("restart the gateway and recover the sandbox"); + await captureSurvivalDiagnostics(host, "before-gateway-restart", [apiKey]); await lifecycle.restartGatewayRuntime({ delayMs: 5_000, sandboxName: SANDBOX_NAME, }); + await captureSurvivalDiagnostics(host, "after-gateway-restart", [apiKey]); await lifecycle.waitForGatewayConnected({ attempts: 60, intervalMs: 5_000, }); + await captureSurvivalDiagnostics(host, "before-recover", [apiKey]); const recovery = await host.nemoclaw([SANDBOX_NAME, "recover"], { artifactName: "post-restart-nemoclaw-recover", env: buildAvailabilityProbeEnv(), timeoutMs: 240_000, }); + await captureSurvivalDiagnostics(host, "after-recover", [apiKey]); assertExitZero(recovery, `recover sandbox ${SANDBOX_NAME}`); progress.phase("recheck state and inference after restart"); From d1a9dd2d0de474e267e519261feb29f544ddb5de Mon Sep 17 00:00:00 2001 From: San Dang Date: Fri, 31 Jul 2026 08:27:23 +0530 Subject: [PATCH 27/30] fix(onboard): persist OpenClaw startup command Signed-off-by: San Dang --- .../recover-rebuild-sandboxes.mdx | 13 +- docs/reference/commands.mdx | 21 +- docs/reference/troubleshooting.mdx | 6 +- src/lib/actions/sandbox/process-recovery.ts | 83 +++++- .../sandbox/supervisor-relaunch.test.ts | 243 ++++++++++++++++-- .../actions/sandbox/supervisor-relaunch.ts | 201 +++++++++++---- .../onboard/docker-startup-command-agent.ts | 3 +- src/lib/onboard/sandbox-create-step.test.ts | 7 +- ...ocess-recovery-supervisor-relaunch.test.ts | 167 ++++++++++-- 9 files changed, 617 insertions(+), 127 deletions(-) diff --git a/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx b/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx index 13b7d2f25c8..58c777f81d0 100644 --- a/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx +++ b/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx @@ -80,9 +80,16 @@ Refer to [`$$nemoclaw recover`](../../reference/commands#$$nemoclaw-name- Recovery uses registry-scoped privileged direct-container control and does not fall back to ordinary `openshell sandbox exec` or a manual in-sandbox relaunch. -For a local Docker-driver sandbox whose container still uses the legacy keepalive startup, `recover` can restart the credential-free managed workload in the registered container. -NemoClaw keeps the container identity and preserves the existing writable-layer state, then requires managed gateway health and OpenShell `Ready` before starting the primary dashboard or API host forward. -A definitive managed-health failure stops immediately; if readiness does not complete within the configured budget, the forward stays stopped. +For a local Docker-driver sandbox whose container still uses the legacy keepalive startup, `recover` can transactionally recreate the registered container with a credential-free managed startup command. + +NemoClaw keeps the previous container available until the managed controller proves the supervisor topology, gateway health, and settle check. +Before recreation, NemoClaw backs up the state directories and files declared by the agent manifest. +It commits only after the replacement container identity and state restoration pass. +NemoClaw removes the temporary state backup after a successful restore or rollback. +If state restoration and rollback both fail, it retains the backup and prints host recovery guidance. +Mounted state remains available, but a committed swap does not retain other writable-layer changes. +After a transactional recreation, NemoClaw uses the `NEMOCLAW_SANDBOX_READY_TIMEOUT` budget (180 seconds by default) for OpenShell to re-register the sandbox before starting the primary dashboard or API host forward. +A definitive managed-health failure still stops immediately; if re-registration does not complete within the budget, the forward stays stopped. For the controller topology, trust boundary, and fail-closed conditions, refer to [Understand Gateway Lifecycle Control](../configure-sandboxes/understand-gateway-lifecycle-control). If recovery cannot repair a sandbox that needs credentials or a current controller contract, rebuild it. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 4fbfe83de38..dd740d586ec 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -1164,14 +1164,13 @@ The host selects the controller from the live container topology. In a direct root-entrypoint container, the request reaches the root PID 1 supervisor. In an OpenShell-managed container, the request enters the root-owned mode `0500` managed controller through a sanitized root exec while OpenShell remains PID 1. It does not use ordinary `openshell sandbox exec` or an in-sandbox manual relaunch as a fallback. -When the root-owned managed controller attests two unchanged zero-supervisor process scans with a stable PID 1 and reports `SUPERVISOR_NOT_RUNNING`, a local Docker-driver sandbox with the legacy keepalive startup can restart its credential-free managed workload in the same registered container. -Recovery pins the container identity and preserves the existing writable-layer state. -Before starting the host forward, it requires: - -- Managed gateway health. -- A successful settle check. -- OpenShell readiness. - +When the root-owned managed controller attests two unchanged zero-supervisor process scans with a stable PID 1 and reports `SUPERVISOR_NOT_RUNNING`, a local Docker-driver sandbox with the legacy keepalive startup can enter a transactional container recreation. +The recreation uses a credential-free managed startup command, pins the registered container identity, and retains the previous container for rollback. +Before recreation, NemoClaw backs up the state directories and files declared by the agent manifest. +It commits only after managed gateway health, the settle check, exact replacement identity, and state restoration pass. +NemoClaw removes the temporary state backup after a successful restore or rollback. +If state restoration and rollback both fail, it retains the backup and prints host recovery guidance. +Mounted state remains available, but a committed swap does not retain other writable-layer changes. It is idempotent. When `recover` repairs a stopped built-in OpenClaw or Hermes gateway, it repeats the recovery action only for an exit status of `1` with blank stdout and a sole nonblank stderr line equal to `SUPERVISOR_BUSY`, with at most three controller attempts. The same result is inconclusive during managed settle confirmation and can be probed again only within the configured settle window. @@ -3942,7 +3941,7 @@ The following environment variables tune onboard-time wall-clock limits. `NEMOCLAW_SANDBOX_READY_TIMEOUT` also covers OpenShell command re-registration after onboarding applies policy presets. -`NEMOCLAW_SANDBOX_READY_TIMEOUT` also applies while managed recovery waits for OpenShell readiness. +`NEMOCLAW_SANDBOX_READY_TIMEOUT` also applies when managed recovery transactionally recreates an existing sandbox. Set them before running `$$nemoclaw onboard` if a slow connection or large model pull risks tripping the default. @@ -3955,7 +3954,7 @@ Set them before running `$$nemoclaw onboard` if a slow connection or large model -For managed recovery, the same timeout covers OpenShell readiness after the managed workload restarts. +For managed recovery, the same timeout covers OpenShell re-registration after transactional recreation. When the deadline expires, the primary dashboard or API host forward stays stopped. @@ -3983,7 +3982,7 @@ The following flags change defaults for commands that manage existing sandboxes. | `NEMOCLAW_CONFIRM_LEGACY_MANAGED_RECREATE` | Exact JSON array of sandbox names | Confirms to the installer that the exact listed set of pre-fingerprint OpenClaw or Hermes sandboxes used NemoClaw-managed images, allowing recovery onto the current managed image. The normalized names must exactly match the installer's printed array. Set it only after verifying every named sandbox. Recorded custom-image evidence remains blocked. | | `NEMOCLAW_DISABLE_INFERENCE_ROUTE_REPAIR` | `1` to enable | Skips the automatic DNS-proxy repair for stale `inference.local` routes during `$$nemoclaw connect` and `$$nemoclaw connect --probe-only`. Use only as a troubleshooting escape hatch. | -| `NEMOCLAW_DISABLE_SUPERVISOR_RELAUNCH` | `1` to enable | Skips the automatic trusted in-place workload restart during `$$nemoclaw recover` when two managed scans find no supervisor while PID 1 remains stable. Use only as a troubleshooting escape hatch; recovery then falls back to the rebuild or re-onboard guidance. | +| `NEMOCLAW_DISABLE_SUPERVISOR_RELAUNCH` | `1` to enable | Skips the automatic trusted container recreation during `$$nemoclaw recover` when two managed scans find no supervisor while PID 1 remains stable. Use only as a troubleshooting escape hatch; recovery then falls back to the rebuild or re-onboard guidance. | | `NEMOCLAW_SHIELDS_ACCEPT_LEGACY_BASELINE` | `1` to opt in | Allows advanced immutable-config verification to trust the current on-disk bytes for older or partial content baselines. Use only after you have rebuilt or manually inspected the sandbox state and accepted that the baseline is operator-approved. | | `NEMOCLAW_SHIELDS_SETTLE_MS` | milliseconds (default `750`, clamped to `0` to `10000`) | Settle window NemoClaw waits after re-applying a config lockdown (during shields auto-restore and `$$nemoclaw shields up` drift remediation) before re-confirming the lock still holds. Detects when an in-sandbox reconciler changes config file permissions after lockdown and re-applies the lock; if NemoClaw cannot re-confirm the lock within the retry budget, shields stay down. This narrows the window in which a reconciler can revert permissions rather than eliminating it. The best-effort `chattr +i` immutable bit remains the only fully durable lock. Raise it on hosts where the gateway settles slowly. | | `NEMOCLAW_SKIP_UNREACHABLE_SANDBOX_BACKUP` | Exactly `1` to opt in (`true`, `yes`, `0` are not accepted) | Applies to standalone `$$nemoclaw backup-all` runs. Skips running sandboxes whose in-sandbox SSH endpoint does not answer. It does not relax the installer's strict pre-upgrade backup, which still aborts if any registered sandbox is skipped or fails. Any uncommitted state since the last successful backup is not included in the skipped backup. | diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index badaf76a1e5..6cad47f36ff 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -1019,10 +1019,10 @@ The same result is inconclusive during managed settle confirmation and can be pr NemoClaw treats `SUPERVISOR_UNAVAILABLE` as terminal because it can report unreadable or untrusted supervisor state, ambiguous discovery, or a process-identity change. It does not retry other status or output combinations. `SUPERVISOR_NOT_RUNNING` is a separate result that requires two zero-supervisor scans with a stable PID 1 and does not enter that retry loop. -On a supported local Docker-driver sandbox with the legacy keepalive startup, it can authorize a container-identity-pinned workload restart that succeeds only after managed health, settle, and OpenShell readiness checks pass. -To bypass that trusted restart while troubleshooting, run `NEMOCLAW_DISABLE_SUPERVISOR_RELAUNCH=1 $$nemoclaw recover`; NemoClaw leaves the container unchanged and returns rebuild or re-onboard guidance. +On a supported local Docker-driver sandbox with the legacy keepalive startup, it can authorize a container-identity-pinned recreation that commits only after managed health and settle checks pass. +To bypass that trusted recreation while troubleshooting, run `NEMOCLAW_DISABLE_SUPERVISOR_RELAUNCH=1 $$nemoclaw recover`; NemoClaw leaves the container unchanged and returns rebuild or re-onboard guidance. If that bounded retry is exhausted, or if `gateway restart` reports `SUPERVISOR_BUSY`, wait for the active request to finish and retry the command. -If the error mentions `SUPERVISOR_NOT_RUNNING` and trusted recovery could not proceed, `SUPERVISOR_REBUILD_REQUIRED`, a missing `nemoclaw-gateway-control` helper, or a missing managed controller, the sandbox image may predate the current lifecycle contract. +If the error mentions `SUPERVISOR_NOT_RUNNING` and trusted recreation could not proceed, `SUPERVISOR_REBUILD_REQUIRED`, a missing `nemoclaw-gateway-control` helper, or a missing managed controller, the sandbox image may predate the current lifecycle contract. An exact `SUPERVISOR_UNAVAILABLE` result instead means the managed controller refused the current supervisor state rather than guessing which same-UID process is the gateway. The current recovery action and any managed settle confirmation stop immediately. If `recover` reports this result, follow its host-side `gateway restart` guidance. diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index 16e9b6927a0..02044307a6f 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -431,8 +431,8 @@ export async function isSandboxGatewayRunningForStatus( /** * Recover a gateway through the registered agent's managed control boundary. * Legacy custom agents retain their SSH-owned compatibility path. Built-in - * agents may return a pinned legacy supervisor relaunch that the caller must - * confirm after the managed health gate. + * agents may return a transactional supervisor relaunch that the caller must + * commit or roll back after the managed health gate. */ type SandboxProcessRecovery = | { kind: "managed" | "custom" } @@ -690,11 +690,11 @@ function recreatedSandboxOpenShellReadinessFailureDetail( const detail = (() => { switch (failure) { case "managed-health-definitive-failure": - return "the recovered sandbox failed the managed health guard, so the primary dashboard/API host forward was not started"; + return "the recreated sandbox failed the managed health guard, so the primary dashboard/API host forward was not started"; case "managed-health-inconclusive-timeout": - return "the recovered sandbox managed health guard stayed inconclusive within the readiness deadline, so the primary dashboard/API host forward was not started"; + return "the recreated sandbox managed health guard stayed inconclusive within the readiness deadline, so the primary dashboard/API host forward was not started"; case "openshell-readiness-failure": - return "the recovered sandbox did not become ready in OpenShell, so the primary dashboard/API host forward was not started"; + return "the recreated sandbox did not become ready in OpenShell, so the primary dashboard/API host forward was not started"; } })(); return openshellError ? `${detail} Last OpenShell readiness error: ${openshellError}` : detail; @@ -713,8 +713,8 @@ const GATEWAY_RECOVERY_WAIT_DEFAULT_SECONDS = 120; /** * Wait until OpenShell has re-registered a directly recreated sandbox as * ready. This probe deliberately has no direct-Docker or SSH fallback: it is - * proving control-plane readiness, not bypassing OpenShell after the managed - * workload has restarted. + * proving control-plane readiness, not authorizing the already completed + * replacement-container recovery. */ function waitForRecreatedSandboxOpenShellReadyResult( sandboxName: string, @@ -844,7 +844,7 @@ function printHostManagedGatewayRecoveryHints( ): void { const quotedSandboxName = shellQuote(sandboxName); if (failureLayer === "supervisor not running") { - console.error(" The in-sandbox supervisor is not running, and trusted supervisor recovery"); + console.error(" The in-sandbox supervisor is not running, and trusted container recovery"); console.error(" could not restore a managed supervisor and healthy gateway."); console.error(" Recreate the sandbox runtime to restore it:"); console.error(` nemoclaw ${quotedSandboxName} rebuild --yes`); @@ -1265,13 +1265,32 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( }), }); } catch (error) { + try { + relaunch?.finalize(false); + } catch { + // Preserve the original recovery error; the failure path below will + // direct the operator to inspect/rebuild the sandbox. + } throw error; } if (!gatewayReady) { + let rolledBack = true; + if (relaunch) { + try { + rolledBack = relaunch.finalize(false).rolledBack; + } catch { + rolledBack = false; + } + } if (!quiet) { console.error(" Gateway process started but is not responding."); printGatewayWedgeDiagnostics(sandboxName, executeSandboxExecCommand); console.error(" Check /tmp/gateway.log inside the sandbox for details."); + if (!rolledBack) { + console.error( + " Automatic rollback of the previous sandbox container failed; inspect Docker state before retrying.", + ); + } printHostManagedGatewayRecoveryHints( sandboxName, recoveryAgent, @@ -1280,6 +1299,52 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( } return { checked: true, wasRunning: false, recovered: false, forwardRecovered: false }; } + if (relaunch) { + try { + const completion = relaunch.finalize(true); + if (completion.stateRestored === false || completion.rolledBack) { + if (!quiet) { + console.error( + completion.rolledBack + ? " Sandbox recovery did not complete; the previous container was restored." + : " Sandbox recovery failed and the previous container could not be restored automatically.", + ); + if (completion.rolledBack && completion.stateBackupRemoved === false) { + console.error(" Warning: the temporary sandbox state backup could not be removed."); + } + if (!completion.rolledBack) { + printHostManagedGatewayRecoveryHints( + sandboxName, + recoveryAgent, + managedRecoveryFailureLayer, + ); + } + } + return { + checked: true, + wasRunning: false, + recovered: false, + forwardRecovered: false, + }; + } + if (!completion.backupRemoved && !quiet) { + console.error( + " Warning: the recovered sandbox is healthy, but its previous container backup could not be removed.", + ); + } + if (completion.stateBackupRemoved === false && !quiet) { + console.error( + " Warning: the recovered sandbox is healthy, but its temporary state backup could not be removed.", + ); + } + } catch { + if (!quiet) { + console.error( + " Warning: the recovered sandbox is healthy, but container transaction cleanup could not be confirmed.", + ); + } + } + } const readinessFailureDetail = relaunch ? (() => { const readinessOptions: RecreatedSandboxOpenShellReadyOptions = { @@ -1304,7 +1369,7 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( return { checked: true, wasRunning: false, - recovered: false, + recovered: true, forwardRecovered: false, forwardRecoveryFailed: true, forwardRecoveryFailureDetail: readinessFailureDetail, diff --git a/src/lib/actions/sandbox/supervisor-relaunch.test.ts b/src/lib/actions/sandbox/supervisor-relaunch.test.ts index 36f8cd721c6..28f638085a0 100644 --- a/src/lib/actions/sandbox/supervisor-relaunch.test.ts +++ b/src/lib/actions/sandbox/supervisor-relaunch.test.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { afterEach, describe, expect, it, vi } from "vitest"; +import type { DockerGpuPatchResult } from "../../onboard/docker-gpu-patch"; import { type ManagedSupervisorRelaunchDeps, relaunchManagedSupervisorSession, @@ -12,6 +13,23 @@ afterEach(() => { vi.unstubAllEnvs(); }); +function patchResult(): DockerGpuPatchResult { + return { + applied: true, + oldContainerId: "old-container-id", + newContainerId: "new-container-id", + originalName: "openshell-alpha", + backupContainerName: "openshell-alpha-nemoclaw-backup", + mode: { + kind: "startup-command", + label: "persistent sandbox startup command", + device: "", + args: [], + }, + backupRemoved: false, + }; +} + function baseDeps(overrides: ManagedSupervisorRelaunchDeps = {}) { return { getSandbox: vi.fn(() => ({ @@ -29,12 +47,38 @@ function baseDeps(overrides: ManagedSupervisorRelaunchDeps = {}) { }) as never, ), resolveDashboardPort: vi.fn(() => 18789), - resolveContainer: vi.fn(() => "original-container-id"), + resolveContainer: vi + .fn() + .mockReturnValueOnce("old-container-id") + .mockReturnValue("new-container-id"), inspectContainer: vi.fn(() => ({ Config: { Env: ["OPENSHELL_SANDBOX_COMMAND=sleep infinity"] }, })), confirmMissingSupervisor: vi.fn(() => true), - startSupervisor: vi.fn(() => ({ started: true as const })), + backupState: vi.fn(() => ({ + success: true, + manifest: { + backupPath: "/tmp/rebuild-backups/alpha/recovery", + }, + backedUpDirs: ["workspace"], + failedDirs: [], + backedUpFiles: [], + failedFiles: [], + })) as never, + restoreState: vi.fn(() => ({ + success: true, + restoredDirs: ["workspace"], + failedDirs: [], + restoredFiles: [], + failedFiles: [], + })), + removeBackup: vi.fn(() => true), + recreate: vi.fn(() => patchResult()), + finalize: vi.fn(({ supervisorReady }) => + supervisorReady + ? { backupRemoved: true, rolledBack: false } + : { backupRemoved: false, rolledBack: true }, + ), ...overrides, } satisfies ManagedSupervisorRelaunchDeps; } @@ -45,7 +89,7 @@ describe("relaunchManagedSupervisorSession", () => { expect(relaunchManagedSupervisorSession("missing-box", { quiet: true, deps })).toBeNull(); expect(deps.resolveContainer).not.toHaveBeenCalled(); - expect(deps.startSupervisor).not.toHaveBeenCalled(); + expect(deps.recreate).not.toHaveBeenCalled(); }); it("honors the troubleshooting kill switch without mutating Docker", () => { @@ -54,7 +98,7 @@ describe("relaunchManagedSupervisorSession", () => { expect(relaunchManagedSupervisorSession("alpha", { quiet: true, deps })).toBeNull(); expect(deps.resolveContainer).not.toHaveBeenCalled(); - expect(deps.startSupervisor).not.toHaveBeenCalled(); + expect(deps.recreate).not.toHaveBeenCalled(); }); it("refuses a container that no longer has the legacy keepalive startup", () => { @@ -65,18 +109,18 @@ describe("relaunchManagedSupervisorSession", () => { }); expect(relaunchManagedSupervisorSession("alpha", { quiet: true, deps })).toBeNull(); - expect(deps.startSupervisor).not.toHaveBeenCalled(); + expect(deps.recreate).not.toHaveBeenCalled(); }); - it("refuses recovery when the pinned container no longer proves supervisor absence", () => { + it("refuses recreation when the pinned container no longer proves supervisor absence", () => { const deps = baseDeps({ confirmMissingSupervisor: vi.fn(() => false) }); expect(relaunchManagedSupervisorSession("alpha", { quiet: true, deps })).toBeNull(); - expect(deps.confirmMissingSupervisor).toHaveBeenCalledWith("original-container-id"); - expect(deps.startSupervisor).not.toHaveBeenCalled(); + expect(deps.confirmMissingSupervisor).toHaveBeenCalledWith("old-container-id"); + expect(deps.recreate).not.toHaveBeenCalled(); }); - it("restarts the supervisor in the registered container without exposing credentials", () => { + it("pins the selected container and persists only a credential-free startup command", () => { vi.stubEnv("NEMOCLAW_EXTRA_PLACEHOLDER_KEYS", "CUSTOM_PROVIDER_CREDENTIAL"); vi.stubEnv("CUSTOM_PROVIDER_CREDENTIAL", "s3cr3t-token"); vi.stubEnv("HTTPS_PROXY", "http://proxyuser:proxypass@proxy.example:8080"); @@ -84,27 +128,188 @@ describe("relaunchManagedSupervisorSession", () => { const relaunch = relaunchManagedSupervisorSession("alpha", { quiet: true, deps }); - expect(relaunch?.containerId).toBe("original-container-id"); - expect(deps.startSupervisor).toHaveBeenCalledOnce(); - const [containerId, command] = vi.mocked(deps.startSupervisor).mock.calls[0] ?? []; - expect(containerId).toBe("original-container-id"); - const serialized = command?.join(" ") ?? ""; + expect(relaunch).not.toBeNull(); + expect(relaunch?.containerId).toBe("new-container-id"); + expect(deps.recreate).toHaveBeenCalledOnce(); + const options = vi.mocked(deps.recreate).mock.calls[0]?.[0]; + expect(options).toMatchObject({ + sandboxName: "alpha", + expectedOldContainerId: "old-container-id", + waitForSupervisor: false, + }); + const serialized = options?.openshellSandboxCommand.join(" ") ?? ""; expect(serialized).toContain("NEMOCLAW_DASHBOARD_PORT=18789"); expect(serialized).toMatch(/nemoclaw-start$/); expect(serialized).not.toContain("s3cr3t-token"); expect(serialized).not.toContain("CUSTOM_PROVIDER_CREDENTIAL"); expect(serialized).not.toContain("proxypass"); + + expect(relaunch?.finalize(true)).toEqual({ + backupRemoved: true, + rolledBack: false, + stateRestored: true, + stateBackupRemoved: true, + }); + expect(deps.restoreState).toHaveBeenCalledWith("alpha", "/tmp/rebuild-backups/alpha/recovery"); + expect(deps.removeBackup).toHaveBeenCalledWith("alpha", "/tmp/rebuild-backups/alpha/recovery"); + expect(deps.finalize).toHaveBeenCalledWith({ + result: expect.objectContaining({ newContainerId: "new-container-id" }), + supervisorReady: true, + }); + }); + + it("rolls the container transaction back when managed readiness is not proven", () => { + const deps = baseDeps(); + const relaunch = relaunchManagedSupervisorSession("alpha", { quiet: true, deps }); + + expect(relaunch?.finalize(false)).toEqual({ + backupRemoved: false, + rolledBack: true, + stateRestored: false, + stateBackupRemoved: true, + }); + expect(deps.restoreState).not.toHaveBeenCalled(); + expect(deps.removeBackup).toHaveBeenCalledWith("alpha", "/tmp/rebuild-backups/alpha/recovery"); + expect(deps.finalize).toHaveBeenCalledWith({ + result: expect.objectContaining({ backupContainerName: expect.any(String) }), + supervisorReady: false, + }); + }); + + it("removes a partial state backup before it refuses recreation (#7404)", () => { + const deps = baseDeps({ + backupState: vi.fn(() => ({ + success: false, + manifest: { + backupPath: "/tmp/rebuild-backups/alpha/partial-recovery", + } as never, + backedUpDirs: [], + failedDirs: ["workspace"], + backedUpFiles: [], + failedFiles: [], + })), + }); + + expect(relaunchManagedSupervisorSession("alpha", { quiet: true, deps })).toBeNull(); + expect(deps.removeBackup).toHaveBeenCalledWith( + "alpha", + "/tmp/rebuild-backups/alpha/partial-recovery", + ); + expect(deps.recreate).not.toHaveBeenCalled(); + }); + + it("rolls back the container transaction when state restore fails", () => { + const deps = baseDeps({ + restoreState: vi.fn(() => ({ + success: false, + restoredDirs: [], + failedDirs: ["workspace"], + restoredFiles: [], + failedFiles: [], + })), + }); + const relaunch = relaunchManagedSupervisorSession("alpha", { quiet: true, deps }); + + expect(relaunch?.finalize(true)).toEqual({ + backupRemoved: false, + rolledBack: true, + stateRestored: false, + stateBackupRemoved: true, + }); + expect(deps.removeBackup).toHaveBeenCalledWith("alpha", "/tmp/rebuild-backups/alpha/recovery"); + expect(deps.finalize).toHaveBeenCalledWith({ + result: expect.objectContaining({ backupContainerName: expect.any(String) }), + supervisorReady: false, + }); + }); + + it("rolls back before restore when the replacement container identity changes", () => { + const deps = baseDeps({ + resolveContainer: vi + .fn() + .mockReturnValueOnce("old-container-id") + .mockReturnValue("different-container-id"), + }); + const relaunch = relaunchManagedSupervisorSession("alpha", { quiet: true, deps }); + + expect(relaunch?.finalize(true)).toEqual({ + backupRemoved: false, + rolledBack: true, + stateRestored: false, + stateBackupRemoved: true, + }); + expect(deps.restoreState).not.toHaveBeenCalled(); + expect(deps.removeBackup).toHaveBeenCalledWith("alpha", "/tmp/rebuild-backups/alpha/recovery"); + expect(deps.finalize).toHaveBeenCalledWith({ + result: expect.objectContaining({ backupContainerName: expect.any(String) }), + supervisorReady: false, + }); + }); + + it("retains the state backup when rollback fails", () => { + const deps = baseDeps({ + finalize: vi.fn(() => ({ backupRemoved: false, rolledBack: false })), + }); + const relaunch = relaunchManagedSupervisorSession("alpha", { quiet: true, deps }); + + expect(relaunch?.finalize(false)).toEqual({ + backupRemoved: false, + rolledBack: false, + stateRestored: false, + }); + expect(deps.removeBackup).not.toHaveBeenCalled(); + }); + + it("reports best-effort state-backup cleanup failure after a successful restore", () => { + const deps = baseDeps({ removeBackup: vi.fn(() => false) }); + const relaunch = relaunchManagedSupervisorSession("alpha", { quiet: true, deps }); + + expect(relaunch?.finalize(true)).toEqual({ + backupRemoved: true, + rolledBack: false, + stateRestored: true, + stateBackupRemoved: false, + }); }); - it("returns null and redacts diagnostics when the pinned start fails", () => { + it("returns null when the pinned recreation fails", () => { + const deps = baseDeps({ + recreate: vi.fn(() => { + throw new Error("container identity changed"); + }), + }); + + expect(relaunchManagedSupervisorSession("alpha", { quiet: true, deps })).toBeNull(); + expect(deps.removeBackup).toHaveBeenCalledWith("alpha", "/tmp/rebuild-backups/alpha/recovery"); + }); + + it("preserves the recreation diagnostic when state-backup cleanup throws", () => { vi.spyOn(console, "log").mockImplementation(() => undefined); const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); const deps = baseDeps({ - startSupervisor: vi.fn(() => ({ - started: false, - detail: + removeBackup: vi.fn(() => { + throw new Error("backup cleanup failed"); + }), + recreate: vi.fn(() => { + throw new Error("container identity changed"); + }), + }); + + expect(relaunchManagedSupervisorSession("alpha", { quiet: false, deps })).toBeNull(); + const output = errorSpy.mock.calls.flat().join("\n"); + expect(output).toContain("container identity changed"); + expect(output).not.toContain("backup cleanup failed"); + }); + + it("redacts diagnostics when trusted recreation fails", () => { + vi.spyOn(console, "log").mockImplementation(() => undefined); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); + const deps = baseDeps({ + recreate: vi.fn(() => { + throw new Error( "OPENAI_API_KEY=sk-recovery-secret HTTPS_PROXY=http://proxyuser:proxypass@proxy.example:8080", - })), + ); + }), }); expect(relaunchManagedSupervisorSession("alpha", { quiet: false, deps })).toBeNull(); diff --git a/src/lib/actions/sandbox/supervisor-relaunch.ts b/src/lib/actions/sandbox/supervisor-relaunch.ts index d9afe40b6ce..ff9a2f48740 100644 --- a/src/lib/actions/sandbox/supervisor-relaunch.ts +++ b/src/lib/actions/sandbox/supervisor-relaunch.ts @@ -1,40 +1,46 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { dockerCapture, dockerSpawnSync } from "../../adapters/docker"; +import { dockerCapture } from "../../adapters/docker"; import * as agentRuntime from "../../agent/runtime"; import { shouldManageDashboardForAgent } from "../../onboard/dashboard-runtime"; import { type DockerContainerInspect, parseDockerInspectJson, } from "../../onboard/docker-gpu-patch"; -import { buildSandboxRuntimeEnvArgs } from "../../onboard/sandbox-create-launch"; +import { sameContainerId } from "../../onboard/docker-gpu-patch-clone"; import { - privilegedSandboxExecArgv, - resolveDirectSandboxContainer, -} from "../../sandbox/privileged-exec"; + type DockerGpuPatchFinalizeOutcome, + finalizeDockerGpuPatchBackup, +} from "../../onboard/docker-gpu-patch-finalize"; +import { recreateOpenShellDockerSandboxWithStartupCommand } from "../../onboard/docker-startup-command-patch"; +import { buildSandboxRuntimeEnvArgs } from "../../onboard/sandbox-create-launch"; +import { resolveDirectSandboxContainer } from "../../sandbox/privileged-exec"; import { redact, redactFull } from "../../security/redact"; import * as registry from "../../state/registry"; +import * as sandboxState from "../../state/sandbox"; import { resolveSandboxDashboardPort } from "./forward-recovery"; /** * Compatibility boundary for OpenShell 0.0.71's Docker driver: legacy * sandboxes persist `OPENSHELL_SANDBOX_COMMAND=sleep infinity` while * `scripts/nemoclaw-start.sh` owns the managed workload as a sibling process. - * Restart that sibling in the registered container so OpenShell identity and - * the complete writable layer survive gateway restarts. Remove this path after - * supported upgrades rebuild every legacy keepalive container with - * `nemoclaw-start` as its persisted startup command. + * Only that inspected value authorizes this migration. Regression coverage is + * named in `supervisor-relaunch.test.ts` and `gateway-guard-recovery.test.ts`. + * Remove this path after supported upgrades rebuild every legacy keepalive + * container with `nemoclaw-start` as its persisted startup command. */ const LEGACY_OPENSHELL_KEEPALIVE = "sleep infinity"; const DOCKER_INSPECT_TIMEOUT_MS = 15000; export type ManagedSupervisorRelaunch = { containerId: string; + finalize(supervisorReady: boolean): DockerGpuPatchFinalizeOutcome & { + stateRestored?: boolean; + stateBackupRemoved?: boolean; + }; }; -type SupervisorStartResult = { started: true } | { detail: string; started: false }; - export type ManagedSupervisorRelaunchDeps = { getSandbox?: typeof registry.getSandbox; getSessionAgent?: typeof agentRuntime.getSessionAgent; @@ -42,7 +48,11 @@ export type ManagedSupervisorRelaunchDeps = { resolveContainer?: typeof resolveDirectSandboxContainer; inspectContainer?: (containerId: string) => DockerContainerInspect; confirmMissingSupervisor?: (containerId: string) => boolean; - startSupervisor?: (containerId: string, command: readonly string[]) => SupervisorStartResult; + backupState?: typeof sandboxState.backupSandboxState; + restoreState?: typeof sandboxState.restoreSandboxState; + removeBackup?: typeof sandboxState.removeSandboxStateBackup; + recreate?: typeof recreateOpenShellDockerSandboxWithStartupCommand; + finalize?: typeof finalizeDockerGpuPatchBackup; }; function inspectContainer(containerId: string): DockerContainerInspect { @@ -104,38 +114,6 @@ function reconstructSupervisorLaunchCommand( return ["env", ...envArgs, "nemoclaw-start"]; } -function startSupervisorInContainer( - sandboxName: string, - containerId: string, - command: readonly string[], -): SupervisorStartResult { - try { - const [operation, ...args] = privilegedSandboxExecArgv( - sandboxName, - [...command], - false, - true, - containerId, - ); - const result = dockerSpawnSync([operation, "--detach", ...args], { - encoding: "utf8", - stdio: ["ignore", "pipe", "pipe"], - timeout: DOCKER_INSPECT_TIMEOUT_MS, - }); - if (!result.error && result.status === 0) return { started: true }; - const detail = - result.error?.message || - String(result.stderr || "").trim() || - `docker exec exited with status ${String(result.status ?? "unknown")}`; - return { detail, started: false }; - } catch (error) { - return { - detail: error instanceof Error ? error.message : String(error), - started: false, - }; - } -} - export function relaunchManagedSupervisorSession( sandboxName: string, { @@ -157,30 +135,141 @@ export function relaunchManagedSupervisorSession( const resolveContainer = deps.resolveContainer ?? resolveDirectSandboxContainer; const inspect = deps.inspectContainer ?? inspectContainer; - const startSupervisor = - deps.startSupervisor ?? - ((containerId, command) => startSupervisorInContainer(sandboxName, containerId, command)); + const confirmMissingSupervisor = deps.confirmMissingSupervisor; + const backupState = deps.backupState ?? sandboxState.backupSandboxState; + const restoreState = deps.restoreState ?? sandboxState.restoreSandboxState; + const removeBackup = deps.removeBackup ?? sandboxState.removeSandboxStateBackup; + const recreate = deps.recreate ?? recreateOpenShellDockerSandboxWithStartupCommand; + const finalize = deps.finalize ?? finalizeDockerGpuPatchBackup; + let pendingStateBackupPath: string | null = null; try { const containerId = resolveContainer(sandboxName, driver); if (!hasLegacyKeepaliveStartup(inspect(containerId))) return null; - if (!deps.confirmMissingSupervisor?.(containerId)) return null; - if (!quiet) { - console.log(" Restarting the managed workload in the existing sandbox container..."); - } - const start = startSupervisor(containerId, startupCommand); - if (!start.started) { + if (!confirmMissingSupervisor?.(containerId)) return null; + const backup = backupState(sandboxName); + if ( + !backup.success || + !backup.manifest || + backup.failedDirs.length > 0 || + backup.failedFiles.length > 0 + ) { + if (backup.manifest) { + try { + removeBackup(sandboxName, backup.manifest.backupPath); + } catch { + // Preserve the backup failure that stopped container recreation. + } + } if (!quiet) { console.error( - ` Trusted supervisor recovery could not start: ${redactFull(redact(start.detail))}`, + " Trusted container recovery stopped before recreation because sandbox state could not be fully backed up.", ); + console.error(" The existing sandbox container was left unchanged."); } return null; } - return { containerId }; + const backupManifest = backup.manifest; + pendingStateBackupPath = backupManifest.backupPath; + if (!quiet) { + console.log(" Recreating the sandbox container with its managed startup command..."); + } + const result = recreate({ + sandboxName, + openshellSandboxCommand: startupCommand, + expectedOldContainerId: containerId, + waitForSupervisor: false, + }); + pendingStateBackupPath = null; + let completed: { + supervisorReady: boolean; + outcome: DockerGpuPatchFinalizeOutcome & { + stateRestored?: boolean; + stateBackupRemoved?: boolean; + }; + } | null = null; + const removeSettledStateBackup = (): boolean => { + try { + return removeBackup(sandboxName, backupManifest.backupPath); + } catch { + return false; + } + }; + return { + containerId: result.newContainerId, + finalize(supervisorReady) { + if (completed) { + if (completed.supervisorReady !== supervisorReady) { + throw new Error( + "Supervisor relaunch transaction was finalized with conflicting state.", + ); + } + return completed.outcome; + } + if (!supervisorReady) { + const finalized = finalize({ result, supervisorReady: false }); + const outcome = { + ...finalized, + stateRestored: false, + ...(finalized.rolledBack ? { stateBackupRemoved: removeSettledStateBackup() } : {}), + }; + completed = { supervisorReady, outcome }; + return outcome; + } + let replacementOwned = false; + try { + replacementOwned = sameContainerId( + resolveContainer(sandboxName, driver), + result.newContainerId, + ); + } catch { + replacementOwned = false; + } + if (!replacementOwned) { + const finalized = finalize({ result, supervisorReady: false }); + const outcome = { + ...finalized, + stateRestored: false, + ...(finalized.rolledBack ? { stateBackupRemoved: removeSettledStateBackup() } : {}), + }; + completed = { supervisorReady, outcome }; + return outcome; + } + let stateRestored = false; + try { + stateRestored = restoreState(sandboxName, backupManifest.backupPath).success; + } catch { + stateRestored = false; + } + if (!stateRestored) { + const finalized = finalize({ result, supervisorReady: false }); + const outcome = { + ...finalized, + stateRestored: false, + ...(finalized.rolledBack ? { stateBackupRemoved: removeSettledStateBackup() } : {}), + }; + completed = { supervisorReady, outcome }; + return outcome; + } + const outcome = { + ...finalize({ result, supervisorReady: true }), + stateRestored: true, + stateBackupRemoved: removeSettledStateBackup(), + }; + completed = { supervisorReady, outcome }; + return outcome; + }, + }; } catch (error) { + if (pendingStateBackupPath) { + try { + removeBackup(sandboxName, pendingStateBackupPath); + } catch { + // Preserve the recreation failure that stopped container recovery. + } + } if (!quiet) { const detail = error instanceof Error ? error.message : String(error); - console.error(` Trusted supervisor recovery could not start: ${redactFull(redact(detail))}`); + console.error(` Trusted container recovery could not start: ${redactFull(redact(detail))}`); } return null; } diff --git a/src/lib/onboard/docker-startup-command-agent.ts b/src/lib/onboard/docker-startup-command-agent.ts index 23968f21d4c..205d4be3fd6 100644 --- a/src/lib/onboard/docker-startup-command-agent.ts +++ b/src/lib/onboard/docker-startup-command-agent.ts @@ -26,7 +26,8 @@ export function resolveDockerStartupCommandPatch( } const agentName = agent?.name; return { - persistStartupCommand: agentName === "hermes" || agentName === DCODE_AGENT_NAME, + persistStartupCommand: + agentName === "openclaw" || agentName === "hermes" || agentName === DCODE_AGENT_NAME, requiredUlimits: agentName === DCODE_AGENT_NAME ? DCODE_DOCKER_ULIMITS : null, }; } diff --git a/src/lib/onboard/sandbox-create-step.test.ts b/src/lib/onboard/sandbox-create-step.test.ts index ba26917eb04..9e7a09334ff 100644 --- a/src/lib/onboard/sandbox-create-step.test.ts +++ b/src/lib/onboard/sandbox-create-step.test.ts @@ -134,7 +134,10 @@ describe("runSandboxCreateStep", () => { }); }); - it("persists the Hermes startup command for Docker-driver container restarts", async () => { + it.each([ + "openclaw", + "hermes", + ] as const)("persists the %s startup command for Docker-driver container restarts", async (agentName) => { const launch = makeLaunch({ sandboxStartupCommand: ["env", "CHAT_UI_URL=http://127.0.0.1:8642", "nemoclaw-start"], }); @@ -143,7 +146,7 @@ describe("runSandboxCreateStep", () => { await runSandboxCreateStep( makeContext({ - agent: { name: "hermes" } as SandboxCreateStepContext["agent"], + agent: { name: agentName } as SandboxCreateStepContext["agent"], prebuild: { buildCtx: "/tmp/ctx", buildId: "b1", diff --git a/test/process-recovery-supervisor-relaunch.test.ts b/test/process-recovery-supervisor-relaunch.test.ts index 17db4b165ce..b06684536e9 100644 --- a/test/process-recovery-supervisor-relaunch.test.ts +++ b/test/process-recovery-supervisor-relaunch.test.ts @@ -99,15 +99,15 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { stderr: "SUPERVISOR_NOT_RUNNING", })); const resolveContainer = vi.fn(() => "old-container-id"); - const startSupervisor = vi.fn(() => { - throw new Error("kill switch allowed supervisor mutation"); + const recreate = vi.fn(() => { + throw new Error("kill switch allowed container mutation"); }); const requestPinnedGatewaySupervisorAction = vi.fn(() => null); const relaunchManagedSupervisorSessionImpl = vi.fn( (sandboxName: string, options: Parameters[1]) => relaunchManagedSupervisorSession(sandboxName, { quiet: options.quiet, - deps: { ...options.deps, resolveContainer, startSupervisor }, + deps: { ...options.deps, resolveContainer, recreate }, }), ); const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); @@ -129,23 +129,25 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { ); expect(resolveContainer).not.toHaveBeenCalled(); expect(requestPinnedGatewaySupervisorAction).not.toHaveBeenCalled(); - expect(startSupervisor).not.toHaveBeenCalled(); + expect(recreate).not.toHaveBeenCalled(); const errorLines = errorSpy.mock.calls.map((call) => String(call[0])); expect(errorLines).toContainEqual( expect.stringContaining("Failure layer: supervisor not running"), ); - expect(errorLines).toContainEqual(expect.stringContaining("trusted supervisor recovery")); + expect(errorLines).toContainEqual(expect.stringContaining("trusted container recovery")); expect(errorLines).toContainEqual(expect.stringContaining("rebuild --yes")); expect(errorLines).not.toContainEqual( expect.stringContaining("Retry the managed restart from the host"), ); }); - it("settles the in-place start when managed control never accepts it", () => { + it("rolls back when recreation starts but managed control never accepts it", () => { mockOpenClawSandbox("rejected-box"); setImmediateRecoveryPolling(); + const finalize = vi.fn(() => ({ backupRemoved: false, rolledBack: true })); const relaunchManagedSupervisorSessionImpl = vi.fn(() => ({ - containerId: "original-container-id", + containerId: "replacement-container-id", + finalize, })); const requestGatewaySupervisorAction = vi.fn((_name: string, action: string) => action === "recover" ? { status: 1, stdout: "", stderr: "SUPERVISOR_NOT_RUNNING" } : null, @@ -165,15 +167,23 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { "rejected-box", "probe", 210000, - "original-container-id", + "replacement-container-id", ); + expect(finalize).toHaveBeenCalledOnce(); + expect(finalize).toHaveBeenCalledWith(false); }); - it("confirms the in-place start only after managed health accepts it", () => { + it("commits only after managed health accepts the recreated supervisor", () => { mockOpenClawSandbox("recovered-box"); setImmediateRecoveryPolling(); + const finalize = vi.fn((supervisorReady: boolean) => + supervisorReady + ? { backupRemoved: true, rolledBack: false } + : { backupRemoved: false, rolledBack: true }, + ); const relaunchManagedSupervisorSessionImpl = vi.fn(() => ({ - containerId: "original-container-id", + containerId: "replacement-container-id", + finalize, })); const requestGatewaySupervisorAction = vi.fn((_name: string, action: string) => action === "recover" ? { status: 1, stdout: "", stderr: "SUPERVISOR_NOT_RUNNING" } : null, @@ -204,18 +214,115 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { "recovered-box", "probe", 210000, - "original-container-id", + "replacement-container-id", + ); + expect(finalize).toHaveBeenCalledOnce(); + expect(finalize).toHaveBeenCalledWith(true); + }); + + it("reports recovery failure when state restore rolls the replacement back", () => { + mockOpenClawSandbox("restore-failed-box"); + setImmediateRecoveryPolling(); + const finalize = vi.fn(() => ({ + backupRemoved: false, + rolledBack: true, + stateRestored: false, + })); + const relaunchManagedSupervisorSessionImpl = vi.fn(() => ({ + containerId: "replacement-container-id", + finalize, + })); + const requestGatewaySupervisorAction = vi.fn((_name: string, action: string) => + action === "recover" ? { status: 1, stdout: "", stderr: "SUPERVISOR_NOT_RUNNING" } : null, + ); + const requestPinnedGatewaySupervisorAction = vi.fn(() => ({ + status: 0, + stdout: "GATEWAY_PID=4242\n", + stderr: "", + })); + const waitForRecreatedSandboxOpenShellReadyImpl = vi.fn(() => true); + const runOpenshell = vi.spyOn(openshellRuntime, "runOpenshell"); + + const result = checkAndRecoverSandboxProcesses("restore-failed-box", { + quiet: true, + isSandboxGatewayRunningImpl: () => false, + requestGatewaySupervisorAction, + requestPinnedGatewaySupervisorAction, + relaunchManagedSupervisorSessionImpl, + waitForRecreatedSandboxOpenShellReadyImpl, + }); + + expect(result).toMatchObject({ + checked: true, + wasRunning: false, + recovered: false, + forwardRecovered: false, + }); + expect(finalize).toHaveBeenCalledOnce(); + expect(finalize).toHaveBeenCalledWith(true); + expect(waitForRecreatedSandboxOpenShellReadyImpl).not.toHaveBeenCalled(); + expect(runOpenshell).not.toHaveBeenCalled(); + }); + + it("prints generic recovery hints when state recovery and rollback both fail", () => { + mockOpenClawSandbox("restore-and-rollback-failed-box"); + setImmediateRecoveryPolling(); + const finalize = vi.fn(() => ({ + backupRemoved: false, + rolledBack: false, + stateRestored: false, + })); + const relaunchManagedSupervisorSessionImpl = vi.fn(() => ({ + containerId: "replacement-container-id", + finalize, + })); + const requestGatewaySupervisorAction = vi.fn((_name: string, action: string) => + action === "recover" ? { status: 1, stdout: "", stderr: "SUPERVISOR_NOT_RUNNING" } : null, ); + const requestPinnedGatewaySupervisorAction = vi.fn(() => ({ + status: 0, + stdout: "GATEWAY_PID=4242\n", + stderr: "", + })); + vi.spyOn(console, "log").mockImplementation(() => undefined); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); + + const result = checkAndRecoverSandboxProcesses("restore-and-rollback-failed-box", { + quiet: false, + isSandboxGatewayRunningImpl: () => false, + requestGatewaySupervisorAction, + requestPinnedGatewaySupervisorAction, + relaunchManagedSupervisorSessionImpl, + }); + + expect(result).toMatchObject({ + checked: true, + wasRunning: false, + recovered: false, + forwardRecovered: false, + }); + const output = errorSpy.mock.calls.flat().join("\n"); + expect(output).toContain( + "Sandbox recovery failed and the previous container could not be restored automatically.", + ); + expect(output).toContain("rebuild --yes"); + expect(output).not.toContain("Sandbox state restore failed"); }); - it("retries a busy pinned managed probe before starting the recovered forward", () => { + it("retries a busy pinned managed probe before starting the replacement forward", () => { mockOpenClawSandbox("busy-recovered-box"); vi.stubEnv("NEMOCLAW_GATEWAY_RECOVERY_POLL_INTERVAL_SECONDS", "0"); vi.stubEnv("NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS", "1"); vi.stubEnv("NEMOCLAW_GATEWAY_RECOVERY_SETTLE_SECONDS", "0"); vi.stubEnv("NEMOCLAW_FORWARD_RECOVERY_WAIT_MS", "0"); + const finalize = vi.fn((supervisorReady: boolean) => + supervisorReady + ? { backupRemoved: true, rolledBack: false } + : { backupRemoved: false, rolledBack: true }, + ); const relaunchManagedSupervisorSessionImpl = vi.fn(() => ({ - containerId: "original-container-id", + containerId: "replacement-container-id", + finalize, })); const requestGatewaySupervisorAction = vi.fn((_name: string, action: string) => action === "recover" ? { status: 1, stdout: "", stderr: "SUPERVISOR_NOT_RUNNING" } : null, @@ -283,6 +390,7 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { ["sandbox", "exec", "--name", "busy-recovered-box", "--", "true"], expect.objectContaining({ ignoreError: true }), ); + expect(finalize).toHaveBeenCalledWith(true); expect(runOpenshell).toHaveBeenCalledWith( ["forward", "start", "--background", "18789", "busy-recovered-box"], expect.objectContaining({ ignoreError: true }), @@ -292,8 +400,10 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { it("uses the sandbox readiness budget after a longer gateway health wait (#7273)", () => { mockOpenClawSandbox("unready-box", 600); setImmediateRecoveryPolling(); + const finalize = vi.fn(() => ({ backupRemoved: true, rolledBack: false })); const relaunchManagedSupervisorSessionImpl = vi.fn(() => ({ - containerId: "original-container-id", + containerId: "replacement-container-id", + finalize, })); const requestGatewaySupervisorAction = vi.fn(() => ({ status: 1, @@ -320,11 +430,13 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { expect(result).toMatchObject({ checked: true, wasRunning: false, - recovered: false, + recovered: true, forwardRecovered: false, forwardRecoveryFailed: true, forwardRecoveryFailureDetail: expect.stringContaining("did not become ready in OpenShell"), }); + expect(finalize).toHaveBeenCalledOnce(); + expect(finalize).toHaveBeenCalledWith(true); expect(waitForRecreatedSandboxOpenShellReadyImpl).toHaveBeenCalledWith( "unready-box", expect.objectContaining({ beforeProbe: expect.any(Function), timeoutSeconds: 180 }), @@ -335,8 +447,10 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { it("reports the last structured OpenShell error when readiness times out", () => { mockOpenClawSandbox("relay-dropped-box"); setImmediateRecoveryPolling(); + const finalize = vi.fn(() => ({ backupRemoved: true, rolledBack: false })); const relaunchManagedSupervisorSessionImpl = vi.fn(() => ({ - containerId: "original-container-id", + containerId: "replacement-container-id", + finalize, })); const requestGatewaySupervisorAction = vi.fn(() => ({ status: 1, @@ -377,7 +491,7 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { expect(result).toMatchObject({ checked: true, wasRunning: false, - recovered: false, + recovered: true, forwardRecovered: false, forwardRecoveryFailed: true, forwardRecoveryFailureDetail: expect.stringContaining( @@ -385,14 +499,17 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { ), }); expect(captureOpenshell).toHaveBeenCalled(); + expect(finalize).toHaveBeenCalledWith(true); expect(runOpenshell).not.toHaveBeenCalled(); }); it("reports a definitive managed health failure separately from OpenShell readiness", () => { mockOpenClawSandbox("managed-failed-box"); setImmediateRecoveryPolling(); + const finalize = vi.fn(() => ({ backupRemoved: true, rolledBack: false })); const relaunchManagedSupervisorSessionImpl = vi.fn(() => ({ - containerId: "original-container-id", + containerId: "replacement-container-id", + finalize, })); const requestGatewaySupervisorAction = vi.fn(() => ({ status: 1, @@ -425,15 +542,16 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { expect(result).toMatchObject({ checked: true, wasRunning: false, - recovered: false, + recovered: true, forwardRecovered: false, forwardRecoveryFailed: true, forwardRecoveryFailureDetail: expect.stringContaining("failed the managed health guard"), }); + expect(finalize).toHaveBeenCalledWith(true); expect(captureOpenshell).not.toHaveBeenCalled(); }); - it("rejects a healthy forward when the original identity changes after readiness", () => { + it("rejects a healthy forward when the replacement identity changes after readiness", () => { mockOpenClawSandbox("drifted-box"); vi.mocked(agentRuntime.getSessionAgent).mockReturnValue({ name: "openclaw", @@ -443,8 +561,10 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { healthProbe: { url: "http://127.0.0.1:18789/health", port: 18789, timeout_seconds: 30 }, } as never); setImmediateRecoveryPolling(); + const finalize = vi.fn(() => ({ backupRemoved: true, rolledBack: false })); const relaunchManagedSupervisorSessionImpl = vi.fn(() => ({ - containerId: "original-container-id", + containerId: "replacement-container-id", + finalize, })); const requestGatewaySupervisorAction = vi.fn(() => ({ status: 1, @@ -461,7 +581,7 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { .mockReturnValueOnce(acceptedProbe) .mockReturnValueOnce(acceptedProbe) .mockImplementationOnce(() => { - throw new Error("original identity changed"); + throw new Error("replacement identity changed"); }) .mockReturnValue(acceptedProbe); const waitForRecreatedSandboxOpenShellReadyImpl = vi.fn( @@ -497,8 +617,9 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { "drifted-box", "probe", 15000, - "original-container-id", + "replacement-container-id", ); + expect(finalize).toHaveBeenCalledWith(true); expect(runOpenshell).toHaveBeenCalledOnce(); expect(runOpenshell).toHaveBeenCalledWith(["forward", "stop", "18789", "drifted-box"], { ignoreError: true, From ac61a67524b71b910223f93d051cd07488df903f Mon Sep 17 00:00:00 2001 From: San Dang Date: Fri, 31 Jul 2026 09:00:51 +0530 Subject: [PATCH 28/30] fix(onboard): recognize default OpenClaw startup Signed-off-by: San Dang --- src/lib/onboard/docker-startup-command-agent.ts | 2 +- src/lib/onboard/sandbox-create-step.test.ts | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/lib/onboard/docker-startup-command-agent.ts b/src/lib/onboard/docker-startup-command-agent.ts index 205d4be3fd6..de483bda8db 100644 --- a/src/lib/onboard/docker-startup-command-agent.ts +++ b/src/lib/onboard/docker-startup-command-agent.ts @@ -24,7 +24,7 @@ export function resolveDockerStartupCommandPatch( if (dockerDriverGateway !== true) { return { persistStartupCommand: false, requiredUlimits: null }; } - const agentName = agent?.name; + const agentName = agent?.name ?? "openclaw"; return { persistStartupCommand: agentName === "openclaw" || agentName === "hermes" || agentName === DCODE_AGENT_NAME, diff --git a/src/lib/onboard/sandbox-create-step.test.ts b/src/lib/onboard/sandbox-create-step.test.ts index 9e7a09334ff..88991227c4f 100644 --- a/src/lib/onboard/sandbox-create-step.test.ts +++ b/src/lib/onboard/sandbox-create-step.test.ts @@ -135,9 +135,11 @@ describe("runSandboxCreateStep", () => { }); it.each([ - "openclaw", - "hermes", - ] as const)("persists the %s startup command for Docker-driver container restarts", async (agentName) => { + { label: "OpenClaw", agent: null }, + { label: "Hermes", agent: { name: "hermes" } as SandboxCreateStepContext["agent"] }, + ])("persists the $label startup command for Docker-driver container restarts", async ({ + agent, + }) => { const launch = makeLaunch({ sandboxStartupCommand: ["env", "CHAT_UI_URL=http://127.0.0.1:8642", "nemoclaw-start"], }); @@ -146,7 +148,7 @@ describe("runSandboxCreateStep", () => { await runSandboxCreateStep( makeContext({ - agent: { name: agentName } as SandboxCreateStepContext["agent"], + agent, prebuild: { buildCtx: "/tmp/ctx", buildId: "b1", From 6ab61e8457701d0088a73ad110ed7c3f2edffd15 Mon Sep 17 00:00:00 2001 From: San Dang Date: Fri, 31 Jul 2026 09:24:08 +0530 Subject: [PATCH 29/30] test(onboard): model startup container recreation Signed-off-by: San Dang --- test/helpers/onboard-script-mocks.cjs | 37 +++++++++++++++++++++++++++ test/onboard-sandbox-build.test.ts | 9 +++++++ 2 files changed, 46 insertions(+) diff --git a/test/helpers/onboard-script-mocks.cjs b/test/helpers/onboard-script-mocks.cjs index abaf4cb4186..730057aa899 100644 --- a/test/helpers/onboard-script-mocks.cjs +++ b/test/helpers/onboard-script-mocks.cjs @@ -75,6 +75,30 @@ const OPENCLAW_SECURITY_INVENTORY_PROBE = [ `printf '%s\\n' "nemoclaw-security-inventory-ok"`, ].join("; "); +const ONBOARD_SANDBOX_OLD_CONTAINER_ID = "a".repeat(64); +const ONBOARD_SANDBOX_NEW_CONTAINER_ID = "b".repeat(64); +const ONBOARD_SANDBOX_INSPECT = { + Id: ONBOARD_SANDBOX_OLD_CONTAINER_ID, + Image: `sha256:${"c".repeat(64)}`, + Name: "/openshell-my-assistant", + Config: { + Image: "openshell/sandbox:test", + Env: ["OPENSHELL_SANDBOX_COMMAND=sleep infinity"], + Labels: { + "openshell.ai/managed-by": "openshell", + "openshell.ai/sandbox-name": "my-assistant", + }, + Entrypoint: ["/opt/openshell/bin/openshell-sandbox"], + Cmd: [], + User: "0", + WorkingDir: "/sandbox", + }, + HostConfig: { + NetworkMode: "openshell-docker", + RestartPolicy: { Name: "unless-stopped" }, + }, +}; + function isOpenClawSecurityInventoryProbe(command) { const commandArgs = Array.isArray(command) ? command.map(String) : []; const dockerArgs = commandArgs[0] === "docker" ? commandArgs.slice(1) : commandArgs; @@ -113,6 +137,19 @@ function mockSandboxExecCurl(command, options = {}) { function mockOnboardRunCapture(command, options = {}) { const normalized = normalizeCommand(command); + if ( + normalized.startsWith("docker ps -a --no-trunc ") && + normalized.includes("label=openshell.ai/sandbox-name=my-assistant") && + normalized.endsWith("--format {{.ID}}") + ) { + return `${ONBOARD_SANDBOX_OLD_CONTAINER_ID}\n${ONBOARD_SANDBOX_NEW_CONTAINER_ID}\n`; + } + if ( + normalized === + `docker inspect --type container ${ONBOARD_SANDBOX_OLD_CONTAINER_ID}` + ) { + return JSON.stringify([ONBOARD_SANDBOX_INSPECT]); + } if (isOpenClawSecurityInventoryProbe(command)) { return "nemoclaw-security-inventory-ok"; } diff --git a/test/onboard-sandbox-build.test.ts b/test/onboard-sandbox-build.test.ts index f5cb4deb67a..5d1d23ef073 100644 --- a/test/onboard-sandbox-build.test.ts +++ b/test/onboard-sandbox-build.test.ts @@ -216,6 +216,15 @@ const { createSandbox } = require(${onboardPath}); ), "expected dashboard forward (loopback or WSL 0.0.0.0)", ); + assert.ok( + payload.commands.some( + (entry: CommandEntry) => + entry.command.includes("docker run -d") && + entry.command.includes("OPENSHELL_SANDBOX_COMMAND=") && + entry.command.includes("nemoclaw-start"), + ), + "expected the default OpenClaw startup command to be persisted in the recreated container", + ); }); it("skips OpenClaw sandbox-base resolution for agent-staged Dockerfiles", async () => { From 01d1b41a82ac7347bfbdd9fee2fbcc764aa8f3a4 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Thu, 30 Jul 2026 21:16:57 -0700 Subject: [PATCH 30/30] test(e2e): prove Docker restart delivery Signed-off-by: Prekshi Vyas --- test/e2e/live/sandbox-survival.test.ts | 12 ++---------- test/pr-risk-plan.test.ts | 8 ++++++-- tools/advisors/risk-plan.mts | 13 +++++++++---- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/test/e2e/live/sandbox-survival.test.ts b/test/e2e/live/sandbox-survival.test.ts index a8c0fb16e9e..2ae366ba860 100644 --- a/test/e2e/live/sandbox-survival.test.ts +++ b/test/e2e/live/sandbox-survival.test.ts @@ -187,7 +187,7 @@ test( "install and register the OpenClaw sandbox", "prove baseline sandbox access and inference", "write persistent OpenClaw markers", - "restart the gateway and recover the sandbox", + "restart the gateway and reconnect the sandbox", "recheck state and inference after restart", "destroy the sandbox and confirm registry removal", ], @@ -404,7 +404,7 @@ test( await stateValidation.writeSandboxMarkers(instance, markers); await stateValidation.expectSandboxMarkers(instance, markers, "pre-restart-marker-read"); - progress.phase("restart the gateway and recover the sandbox"); + progress.phase("restart the gateway and reconnect the sandbox"); await captureSurvivalDiagnostics(host, "before-gateway-restart", [apiKey]); await lifecycle.restartGatewayRuntime({ delayMs: 5_000, @@ -415,14 +415,6 @@ test( attempts: 60, intervalMs: 5_000, }); - await captureSurvivalDiagnostics(host, "before-recover", [apiKey]); - const recovery = await host.nemoclaw([SANDBOX_NAME, "recover"], { - artifactName: "post-restart-nemoclaw-recover", - env: buildAvailabilityProbeEnv(), - timeoutMs: 240_000, - }); - await captureSurvivalDiagnostics(host, "after-recover", [apiKey]); - assertExitZero(recovery, `recover sandbox ${SANDBOX_NAME}`); progress.phase("recheck state and inference after restart"); await sandbox.expectListed(SANDBOX_NAME, { diff --git a/test/pr-risk-plan.test.ts b/test/pr-risk-plan.test.ts index bc92c7d0592..4c6962bcbdd 100644 --- a/test/pr-risk-plan.test.ts +++ b/test/pr-risk-plan.test.ts @@ -274,8 +274,12 @@ describe("deterministic PR risk plan", () => { expect(riskPlanRequiredTargetIds(docsAndTestsOnly)).toEqual([]); }); - it("selects post-reboot recovery for status delivery recovery changes (#7824)", () => { - const changedFile = "src/lib/actions/sandbox/status-snapshot.ts"; + it.each([ + "src/lib/actions/sandbox/status-snapshot.ts", + "src/lib/onboard/docker-driver-sandbox-recovery.ts", + "src/lib/onboard/docker-startup-command-agent.ts", + "src/lib/onboard/sandbox-create-step.ts", + ])("selects post-reboot recovery for Docker delivery changes in %s (#7824)", (changedFile) => { const result = plan(changedFile); const adjacentStatusFile = plan("src/lib/actions/sandbox/status-text.ts"); diff --git a/tools/advisors/risk-plan.mts b/tools/advisors/risk-plan.mts index 09514a3766a..6eacfb7d51a 100644 --- a/tools/advisors/risk-plan.mts +++ b/tools/advisors/risk-plan.mts @@ -14,7 +14,12 @@ const PR_E2E_TYPED_TARGET_ID_SET = new Set(PR_E2E_TYPED_TARGET_IDS); const DEEPAGENTS_HEADLESS_INFERENCE_CHECK = "test/e2e/e2e-cloud-experimental/checks/07-deepagents-code-headless-inference.sh"; const DEEPAGENTS_CODE_RUNTIME_ROOT = "agents/langchain-deepagents-code/"; -const POST_REBOOT_STATUS_RUNTIME = "src/lib/actions/sandbox/status-snapshot.ts"; +const POST_REBOOT_DELIVERY_RUNTIME_FILES = new Set([ + "src/lib/actions/sandbox/status-snapshot.ts", + "src/lib/onboard/docker-driver-sandbox-recovery.ts", + "src/lib/onboard/docker-startup-command-agent.ts", + "src/lib/onboard/sandbox-create-step.ts", +]); export type RiskTier = 0 | 1 | 2 | 3; export type RiskFamilyId = @@ -129,9 +134,9 @@ export function focusedPrE2eTargetsForChangedFiles( (file.startsWith(DEEPAGENTS_CODE_RUNTIME_ROOT) && isRuntimeRelevant(file)), ), ); - const postRebootMatchedFiles = changedFiles.includes(POST_REBOOT_STATUS_RUNTIME) - ? [POST_REBOOT_STATUS_RUNTIME] - : []; + const postRebootMatchedFiles = stableUnique( + changedFiles.filter((file) => POST_REBOOT_DELIVERY_RUNTIME_FILES.has(file)), + ); return [ ...(deepAgentsMatchedFiles.length > 0 ? [