fix(recover): preserve state across supervisor relaunch

src/lib/actions/sandbox/process-recovery.ts

@@ -1278,6 +1278,21 @@ function checkAndRecoverSandboxProcessesWithoutHostLock(
     if (relaunch) {
       try {
         const completion = relaunch.finalize(true);
+        if (completion.stateRestored === false || completion.rolledBack) {
+          if (!quiet) {
+            console.error(
+              completion.rolledBack
+                ? "  Sandbox state restore failed; the previous container was restored."
+                : "  Sandbox state restore failed and the previous container could not be restored automatically.",
+            );
+          }
+          return {
+            checked: true,
+            wasRunning: false,
+            recovered: false,
+            forwardRecovered: false,
+          };
+        }
         if (!completion.backupRemoved && !quiet) {
           console.error(
             "  Warning: the recovered sandbox is healthy, but its previous container backup could not be removed.",

src/lib/actions/sandbox/supervisor-relaunch.test.ts

@@ -47,11 +47,31 @@ function baseDeps(overrides: ManagedSupervisorRelaunchDeps = {}) {
         }) as never,
     ),
     resolveDashboardPort: vi.fn(() => 18789),
-    resolveContainer: vi.fn(() => "old-container-id"),
+    resolveContainer: vi
+      .fn()
+      .mockReturnValueOnce("old-container-id")
+      .mockReturnValue("new-container-id"),
     inspectContainer: vi.fn(() => ({
       Config: { Env: ["OPENSHELL_SANDBOX_COMMAND=sleep infinity"] },
     })),
     confirmMissingSupervisor: vi.fn(() => true),
+    backupState: vi.fn(() => ({
+      success: true,
+      manifest: {
+        backupPath: "/tmp/rebuild-backups/alpha/recovery",
+      },
+      backedUpDirs: ["workspace"],
+      failedDirs: [],
+      backedUpFiles: [],
+      failedFiles: [],
+    })) as never,
+    restoreState: vi.fn(() => ({
+      success: true,
+      restoredDirs: ["workspace"],
+      failedDirs: [],
+      restoredFiles: [],
+      failedFiles: [],
+    })),
     recreate: vi.fn(() => patchResult()),
     finalize: vi.fn(({ supervisorReady }) =>
       supervisorReady
@@ -123,7 +143,12 @@ describe("relaunchManagedSupervisorSession", () => {
     expect(serialized).not.toContain("CUSTOM_PROVIDER_CREDENTIAL");
     expect(serialized).not.toContain("proxypass");
 
-    expect(relaunch?.finalize(true)).toEqual({ backupRemoved: true, rolledBack: false });
+    expect(relaunch?.finalize(true)).toEqual({
+      backupRemoved: true,
+      rolledBack: false,
+      stateRestored: true,
+    });
+    expect(deps.restoreState).toHaveBeenCalledWith("alpha", "/tmp/rebuild-backups/alpha/recovery");
     expect(deps.finalize).toHaveBeenCalledWith({
       result: expect.objectContaining({ newContainerId: "new-container-id" }),
       supervisorReady: true,
@@ -134,7 +159,71 @@ describe("relaunchManagedSupervisorSession", () => {
     const deps = baseDeps();
     const relaunch = relaunchManagedSupervisorSession("alpha", { quiet: true, deps });
 
-    expect(relaunch?.finalize(false)).toEqual({ backupRemoved: false, rolledBack: true });
+    expect(relaunch?.finalize(false)).toEqual({
+      backupRemoved: false,
+      rolledBack: true,
+      stateRestored: false,
+    });
+    expect(deps.restoreState).not.toHaveBeenCalled();
+    expect(deps.finalize).toHaveBeenCalledWith({
+      result: expect.objectContaining({ backupContainerName: expect.any(String) }),
+      supervisorReady: false,
+    });
+  });
+
+  it("refuses recreation when sandbox state cannot be fully backed up", () => {
+    const deps = baseDeps({
+      backupState: vi.fn(() => ({
+        success: false,
+        backedUpDirs: [],
+        failedDirs: ["workspace"],
+        backedUpFiles: [],
+        failedFiles: [],
+      })),
+    });
+
+    expect(relaunchManagedSupervisorSession("alpha", { quiet: true, deps })).toBeNull();
+    expect(deps.recreate).not.toHaveBeenCalled();
+  });
+
+  it("rolls back the container transaction when state restore fails", () => {
+    const deps = baseDeps({
+      restoreState: vi.fn(() => ({
+        success: false,
+        restoredDirs: [],
+        failedDirs: ["workspace"],
+        restoredFiles: [],
+        failedFiles: [],
+      })),
+    });
+    const relaunch = relaunchManagedSupervisorSession("alpha", { quiet: true, deps });
+
+    expect(relaunch?.finalize(true)).toEqual({
+      backupRemoved: false,
+      rolledBack: true,
+      stateRestored: false,
+    });
+    expect(deps.finalize).toHaveBeenCalledWith({
+      result: expect.objectContaining({ backupContainerName: expect.any(String) }),
+      supervisorReady: false,
+    });
+  });
+
+  it("rolls back before restore when the replacement container identity changes", () => {
+    const deps = baseDeps({
+      resolveContainer: vi
+        .fn()
+        .mockReturnValueOnce("old-container-id")
+        .mockReturnValue("different-container-id"),
+    });
+    const relaunch = relaunchManagedSupervisorSession("alpha", { quiet: true, deps });
+
+    expect(relaunch?.finalize(true)).toEqual({
+      backupRemoved: false,
+      rolledBack: true,
+      stateRestored: false,
+    });
+    expect(deps.restoreState).not.toHaveBeenCalled();
     expect(deps.finalize).toHaveBeenCalledWith({
       result: expect.objectContaining({ backupContainerName: expect.any(String) }),
       supervisorReady: false,

src/lib/actions/sandbox/supervisor-relaunch.ts

@@ -8,6 +8,7 @@ import {
   type DockerContainerInspect,
   parseDockerInspectJson,
 } from "../../onboard/docker-gpu-patch";
+import { sameContainerId } from "../../onboard/docker-gpu-patch-clone";
 import {
   type DockerGpuPatchFinalizeOutcome,
   finalizeDockerGpuPatchBackup,
@@ -17,6 +18,7 @@ import { buildSandboxRuntimeEnvArgs } from "../../onboard/sandbox-create-launch"
 import { resolveDirectSandboxContainer } from "../../sandbox/privileged-exec";
 import { redact, redactFull } from "../../security/redact";
 import * as registry from "../../state/registry";
+import * as sandboxState from "../../state/sandbox";
 import { resolveSandboxDashboardPort } from "./forward-recovery";
 
 /**
@@ -33,7 +35,7 @@ const DOCKER_INSPECT_TIMEOUT_MS = 15000;
 
 export type ManagedSupervisorRelaunch = {
   containerId: string;
-  finalize(supervisorReady: boolean): DockerGpuPatchFinalizeOutcome;
+  finalize(supervisorReady: boolean): DockerGpuPatchFinalizeOutcome & { stateRestored?: boolean };
 };
 
 export type ManagedSupervisorRelaunchDeps = {
@@ -43,6 +45,8 @@ export type ManagedSupervisorRelaunchDeps = {
   resolveContainer?: typeof resolveDirectSandboxContainer;
   inspectContainer?: (containerId: string) => DockerContainerInspect;
   confirmMissingSupervisor?: (containerId: string) => boolean;
+  backupState?: typeof sandboxState.backupSandboxState;
+  restoreState?: typeof sandboxState.restoreSandboxState;
   recreate?: typeof recreateOpenShellDockerSandboxWithStartupCommand;
   finalize?: typeof finalizeDockerGpuPatchBackup;
 };
@@ -128,12 +132,30 @@ export function relaunchManagedSupervisorSession(
   const resolveContainer = deps.resolveContainer ?? resolveDirectSandboxContainer;
   const inspect = deps.inspectContainer ?? inspectContainer;
   const confirmMissingSupervisor = deps.confirmMissingSupervisor;
+  const backupState = deps.backupState ?? sandboxState.backupSandboxState;
+  const restoreState = deps.restoreState ?? sandboxState.restoreSandboxState;
   const recreate = deps.recreate ?? recreateOpenShellDockerSandboxWithStartupCommand;
   const finalize = deps.finalize ?? finalizeDockerGpuPatchBackup;
   try {
     const containerId = resolveContainer(sandboxName, driver);
     if (!hasLegacyKeepaliveStartup(inspect(containerId))) return null;
     if (!confirmMissingSupervisor?.(containerId)) return null;
+    const backup = backupState(sandboxName);
+    if (
+      !backup.success ||
+      !backup.manifest ||
+      backup.failedDirs.length > 0 ||
+      backup.failedFiles.length > 0
+    ) {
+      if (!quiet) {
+        console.error(
+          "  Trusted container recovery stopped before recreation because sandbox state could not be fully backed up.",
+        );
+        console.error("  The existing sandbox container was left unchanged.");
+      }
+      return null;
+    }
+    const backupManifest = backup.manifest;
     if (!quiet) {
       console.log("  Recreating the sandbox container with its managed startup command...");
     }
@@ -143,8 +165,10 @@ export function relaunchManagedSupervisorSession(
       expectedOldContainerId: containerId,
       waitForSupervisor: false,
     });
-    let completed: { supervisorReady: boolean; outcome: DockerGpuPatchFinalizeOutcome } | null =
-      null;
+    let completed: {
+      supervisorReady: boolean;
+      outcome: DockerGpuPatchFinalizeOutcome & { stateRestored?: boolean };
+    } | null = null;
     return {
       containerId: result.newContainerId,
       finalize(supervisorReady) {
@@ -156,7 +180,46 @@ export function relaunchManagedSupervisorSession(
           }
           return completed.outcome;
         }
-        const outcome = finalize({ result, supervisorReady });
+        if (!supervisorReady) {
+          const outcome = { ...finalize({ result, supervisorReady: false }), stateRestored: false };
+          completed = { supervisorReady, outcome };
+          return outcome;
+        }
+        let replacementOwned = false;
+        try {
+          replacementOwned = sameContainerId(
+            resolveContainer(sandboxName, driver),
+            result.newContainerId,
+          );
+        } catch {
+          replacementOwned = false;
+        }
+        if (!replacementOwned) {
+          const outcome = {
+            ...finalize({ result, supervisorReady: false }),
+            stateRestored: false,
+          };
+          completed = { supervisorReady, outcome };
+          return outcome;
+        }
+        let stateRestored = false;
+        try {
+          stateRestored = restoreState(sandboxName, backupManifest.backupPath).success;
+        } catch {
+          stateRestored = false;
+        }
+        if (!stateRestored) {
+          const outcome = {
+            ...finalize({ result, supervisorReady: false }),
+            stateRestored: false,
+          };
+          completed = { supervisorReady, outcome };
+          return outcome;
+        }
+        const outcome = {
+          ...finalize({ result, supervisorReady: true }),
+          stateRestored: true,
+        };
         completed = { supervisorReady, outcome };
         return outcome;
       },

test/process-recovery-supervisor-relaunch.test.ts

@@ -220,6 +220,50 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => {
     expect(finalize).toHaveBeenCalledWith(true);
   });
 
+  it("reports recovery failure when state restore rolls the replacement back", () => {
+    mockOpenClawSandbox("restore-failed-box");
+    setImmediateRecoveryPolling();
+    const finalize = vi.fn(() => ({
+      backupRemoved: false,
+      rolledBack: true,
+      stateRestored: false,
+    }));
+    const relaunchManagedSupervisorSessionImpl = vi.fn(() => ({
+      containerId: "replacement-container-id",
+      finalize,
+    }));
+    const requestGatewaySupervisorAction = vi.fn((_name: string, action: string) =>
+      action === "recover" ? { status: 1, stdout: "", stderr: "SUPERVISOR_NOT_RUNNING" } : null,
+    );
+    const requestPinnedGatewaySupervisorAction = vi.fn(() => ({
+      status: 0,
+      stdout: "GATEWAY_PID=4242\n",
+      stderr: "",
+    }));
+    const waitForRecreatedSandboxOpenShellReadyImpl = vi.fn(() => true);
+    const runOpenshell = vi.spyOn(openshellRuntime, "runOpenshell");
+
+    const result = checkAndRecoverSandboxProcesses("restore-failed-box", {
+      quiet: true,
+      isSandboxGatewayRunningImpl: () => false,
+      requestGatewaySupervisorAction,
+      requestPinnedGatewaySupervisorAction,
+      relaunchManagedSupervisorSessionImpl,
+      waitForRecreatedSandboxOpenShellReadyImpl,
+    });
+
+    expect(result).toMatchObject({
+      checked: true,
+      wasRunning: false,
+      recovered: false,
+      forwardRecovered: false,
+    });
+    expect(finalize).toHaveBeenCalledOnce();
+    expect(finalize).toHaveBeenCalledWith(true);
+    expect(waitForRecreatedSandboxOpenShellReadyImpl).not.toHaveBeenCalled();
+    expect(runOpenshell).not.toHaveBeenCalled();
+  });
+
   it("retries a busy pinned managed probe before starting the replacement forward", () => {
     mockOpenClawSandbox("busy-recovered-box");
     vi.stubEnv("NEMOCLAW_GATEWAY_RECOVERY_POLL_INTERVAL_SECONDS", "0");