fix(recover): preserve state across supervisor relaunch

docs/manage-sandboxes/recover-rebuild-sandboxes.mdx

@@ -82,8 +82,12 @@ Refer to [`$$nemoclaw <name> recover`](../../reference/commands#$$nemoclaw-name-
 Recovery uses registry-scoped privileged direct-container control and does not fall back to ordinary `openshell sandbox exec` or a manual in-sandbox relaunch.
 For a local Docker-driver sandbox whose container still uses the legacy keepalive startup, `recover` can transactionally recreate the registered container with a credential-free managed startup command.
 
-NemoClaw keeps the previous container available until the managed controller proves the supervisor topology, gateway health, and settle check, and attempts to restore it if that proof fails.
-The recreation preserves mounted sandbox state, but a committed swap does not retain changes stored only in the previous container's writable layer.
+NemoClaw keeps the previous container available until the managed controller proves the supervisor topology, gateway health, and settle check.
+Before recreation, NemoClaw backs up the state directories and files declared by the agent manifest.
+It commits only after the replacement container identity and state restoration pass.
+NemoClaw removes the temporary state backup after a successful restore or rollback.
+If state restoration and rollback both fail, it retains the backup and prints host recovery guidance.
+Mounted state remains available, but a committed swap does not retain other writable-layer changes.
 After a transactional recreation, NemoClaw uses the `NEMOCLAW_SANDBOX_READY_TIMEOUT` budget (180 seconds by default) for OpenShell to re-register the sandbox before starting the primary dashboard or API host forward.
 A definitive managed-health failure still stops immediately; if re-registration does not complete within the budget, the forward stays stopped.
 

docs/reference/commands.mdx

@@ -1164,8 +1164,12 @@ In a direct root-entrypoint container, the request reaches the root PID 1 superv
 In an OpenShell-managed container, the request enters the root-owned mode `0500` managed controller through a sanitized root exec while OpenShell remains PID 1.
 It does not use ordinary `openshell sandbox exec` or an in-sandbox manual relaunch as a fallback.
 When the root-owned managed controller attests two unchanged zero-supervisor process scans with a stable PID 1 and reports `SUPERVISOR_NOT_RUNNING`, a local Docker-driver sandbox with the legacy keepalive startup can enter a transactional container recreation.
-The recreation uses a credential-free managed startup command, pins the registered container identity, retains the previous container for rollback, and commits only after managed gateway health and the settle check pass.
-The recreation preserves mounted sandbox state, but a committed swap does not retain changes stored only in the previous container's writable layer.
+The recreation uses a credential-free managed startup command, pins the registered container identity, and retains the previous container for rollback.
+Before recreation, NemoClaw backs up the state directories and files declared by the agent manifest.
+It commits only after managed gateway health, the settle check, exact replacement identity, and state restoration pass.
+NemoClaw removes the temporary state backup after a successful restore or rollback.
+If state restoration and rollback both fail, it retains the backup and prints host recovery guidance.
+Mounted state remains available, but a committed swap does not retain other writable-layer changes.
 It is idempotent.
 When `recover` repairs a stopped built-in OpenClaw or Hermes gateway, it repeats the recovery action only for an exit status of `1` with blank stdout and a sole nonblank stderr line equal to `SUPERVISOR_BUSY`, with at most three controller attempts.
 The same result is inconclusive during managed settle confirmation and can be probed again only within the configured settle window.

src/lib/actions/sandbox/process-recovery.ts

@@ -1302,11 +1302,41 @@ function checkAndRecoverSandboxProcessesWithoutHostLock(
     if (relaunch) {
       try {
         const completion = relaunch.finalize(true);
+        if (completion.stateRestored === false || completion.rolledBack) {
+          if (!quiet) {
+            console.error(
+              completion.rolledBack
+                ? "  Sandbox recovery did not complete; the previous container was restored."
+                : "  Sandbox recovery failed and the previous container could not be restored automatically.",
+            );
+            if (completion.rolledBack && completion.stateBackupRemoved === false) {
+              console.error("  Warning: the temporary sandbox state backup could not be removed.");
+            }
+            if (!completion.rolledBack) {
+              printHostManagedGatewayRecoveryHints(
+                sandboxName,
+                recoveryAgent,
+                managedRecoveryFailureLayer,
+              );
+            }
+          }
+          return {
+            checked: true,
+            wasRunning: false,
+            recovered: false,
+            forwardRecovered: false,
+          };
+        }
         if (!completion.backupRemoved && !quiet) {
           console.error(
             "  Warning: the recovered sandbox is healthy, but its previous container backup could not be removed.",
           );
         }
+        if (completion.stateBackupRemoved === false && !quiet) {
+          console.error(
+            "  Warning: the recovered sandbox is healthy, but its temporary state backup could not be removed.",
+          );
+        }
       } catch {
         if (!quiet) {
           console.error(

src/lib/actions/sandbox/supervisor-relaunch.test.ts

@@ -47,11 +47,32 @@ function baseDeps(overrides: ManagedSupervisorRelaunchDeps = {}) {
         }) as never,
     ),
     resolveDashboardPort: vi.fn(() => 18789),
-    resolveContainer: vi.fn(() => "old-container-id"),
+    resolveContainer: vi
+      .fn()
+      .mockReturnValueOnce("old-container-id")
+      .mockReturnValue("new-container-id"),
     inspectContainer: vi.fn(() => ({
       Config: { Env: ["OPENSHELL_SANDBOX_COMMAND=sleep infinity"] },
     })),
     confirmMissingSupervisor: vi.fn(() => true),
+    backupState: vi.fn(() => ({
+      success: true,
+      manifest: {
+        backupPath: "/tmp/rebuild-backups/alpha/recovery",
+      },
+      backedUpDirs: ["workspace"],
+      failedDirs: [],
+      backedUpFiles: [],
+      failedFiles: [],
+    })) as never,
+    restoreState: vi.fn(() => ({
+      success: true,
+      restoredDirs: ["workspace"],
+      failedDirs: [],
+      restoredFiles: [],
+      failedFiles: [],
+    })),
+    removeBackup: vi.fn(() => true),
     recreate: vi.fn(() => patchResult()),
     finalize: vi.fn(({ supervisorReady }) =>
       supervisorReady
@@ -123,7 +144,14 @@ describe("relaunchManagedSupervisorSession", () => {
     expect(serialized).not.toContain("CUSTOM_PROVIDER_CREDENTIAL");
     expect(serialized).not.toContain("proxypass");
 
-    expect(relaunch?.finalize(true)).toEqual({ backupRemoved: true, rolledBack: false });
+    expect(relaunch?.finalize(true)).toEqual({
+      backupRemoved: true,
+      rolledBack: false,
+      stateRestored: true,
+      stateBackupRemoved: true,
+    });
+    expect(deps.restoreState).toHaveBeenCalledWith("alpha", "/tmp/rebuild-backups/alpha/recovery");
+    expect(deps.removeBackup).toHaveBeenCalledWith("alpha", "/tmp/rebuild-backups/alpha/recovery");
     expect(deps.finalize).toHaveBeenCalledWith({
       result: expect.objectContaining({ newContainerId: "new-container-id" }),
       supervisorReady: true,
@@ -134,13 +162,116 @@ describe("relaunchManagedSupervisorSession", () => {
     const deps = baseDeps();
     const relaunch = relaunchManagedSupervisorSession("alpha", { quiet: true, deps });
 
-    expect(relaunch?.finalize(false)).toEqual({ backupRemoved: false, rolledBack: true });
+    expect(relaunch?.finalize(false)).toEqual({
+      backupRemoved: false,
+      rolledBack: true,
+      stateRestored: false,
+      stateBackupRemoved: true,
+    });
+    expect(deps.restoreState).not.toHaveBeenCalled();
+    expect(deps.removeBackup).toHaveBeenCalledWith("alpha", "/tmp/rebuild-backups/alpha/recovery");
+    expect(deps.finalize).toHaveBeenCalledWith({
+      result: expect.objectContaining({ backupContainerName: expect.any(String) }),
+      supervisorReady: false,
+    });
+  });
+
+  it("removes a partial state backup before it refuses recreation (#7404)", () => {
+    const deps = baseDeps({
+      backupState: vi.fn(() => ({
+        success: false,
+        manifest: {
+          backupPath: "/tmp/rebuild-backups/alpha/partial-recovery",
+        } as never,
+        backedUpDirs: [],
+        failedDirs: ["workspace"],
+        backedUpFiles: [],
+        failedFiles: [],
+      })),
+    });
+
+    expect(relaunchManagedSupervisorSession("alpha", { quiet: true, deps })).toBeNull();
+    expect(deps.removeBackup).toHaveBeenCalledWith(
+      "alpha",
+      "/tmp/rebuild-backups/alpha/partial-recovery",
+    );
+    expect(deps.recreate).not.toHaveBeenCalled();
+  });
+
+  it("rolls back the container transaction when state restore fails", () => {
+    const deps = baseDeps({
+      restoreState: vi.fn(() => ({
+        success: false,
+        restoredDirs: [],
+        failedDirs: ["workspace"],
+        restoredFiles: [],
+        failedFiles: [],
+      })),
+    });
+    const relaunch = relaunchManagedSupervisorSession("alpha", { quiet: true, deps });
+
+    expect(relaunch?.finalize(true)).toEqual({
+      backupRemoved: false,
+      rolledBack: true,
+      stateRestored: false,
+      stateBackupRemoved: true,
+    });
+    expect(deps.removeBackup).toHaveBeenCalledWith("alpha", "/tmp/rebuild-backups/alpha/recovery");
     expect(deps.finalize).toHaveBeenCalledWith({
       result: expect.objectContaining({ backupContainerName: expect.any(String) }),
       supervisorReady: false,
     });
   });
 
+  it("rolls back before restore when the replacement container identity changes", () => {
+    const deps = baseDeps({
+      resolveContainer: vi
+        .fn()
+        .mockReturnValueOnce("old-container-id")
+        .mockReturnValue("different-container-id"),
+    });
+    const relaunch = relaunchManagedSupervisorSession("alpha", { quiet: true, deps });
+
+    expect(relaunch?.finalize(true)).toEqual({
+      backupRemoved: false,
+      rolledBack: true,
+      stateRestored: false,
+      stateBackupRemoved: true,
+    });
+    expect(deps.restoreState).not.toHaveBeenCalled();
+    expect(deps.removeBackup).toHaveBeenCalledWith("alpha", "/tmp/rebuild-backups/alpha/recovery");
+    expect(deps.finalize).toHaveBeenCalledWith({
+      result: expect.objectContaining({ backupContainerName: expect.any(String) }),
+      supervisorReady: false,
+    });
+  });
+
+  it("retains the state backup when rollback fails", () => {
+    const deps = baseDeps({
+      finalize: vi.fn(() => ({ backupRemoved: false, rolledBack: false })),
+    });
+    const relaunch = relaunchManagedSupervisorSession("alpha", { quiet: true, deps });
+
+    expect(relaunch?.finalize(false)).toEqual({
+      backupRemoved: false,
+      rolledBack: false,
+      stateRestored: false,
+    });
+    expect(deps.removeBackup).not.toHaveBeenCalled();
+  });
+
+  it("reports best-effort state-backup cleanup failure after a successful restore", () => {
+    const deps = baseDeps({ removeBackup: vi.fn(() => false) });
+    const relaunch = relaunchManagedSupervisorSession("alpha", { quiet: true, deps });
+
+    expect(relaunch?.finalize(true)).toEqual({
+      backupRemoved: true,
+      rolledBack: false,
+      stateRestored: true,
+      stateBackupRemoved: false,
+    });
+  });
+
   it("returns null when the pinned recreation fails", () => {
     const deps = baseDeps({
       recreate: vi.fn(() => {
@@ -149,6 +280,25 @@ describe("relaunchManagedSupervisorSession", () => {
     });
 
     expect(relaunchManagedSupervisorSession("alpha", { quiet: true, deps })).toBeNull();
+    expect(deps.removeBackup).toHaveBeenCalledWith("alpha", "/tmp/rebuild-backups/alpha/recovery");
+  });
+
+  it("preserves the recreation diagnostic when state-backup cleanup throws", () => {
+    vi.spyOn(console, "log").mockImplementation(() => undefined);
+    const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined);
+    const deps = baseDeps({
+      removeBackup: vi.fn(() => {
+        throw new Error("backup cleanup failed");
+      }),
+      recreate: vi.fn(() => {
+        throw new Error("container identity changed");
+      }),
+    });
+
+    expect(relaunchManagedSupervisorSession("alpha", { quiet: false, deps })).toBeNull();
+    const output = errorSpy.mock.calls.flat().join("\n");
+    expect(output).toContain("container identity changed");
+    expect(output).not.toContain("backup cleanup failed");
   });
 
   it("redacts diagnostics when trusted recreation fails", () => {