diff --git a/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx b/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx index 4d1ec536979..58c777f81d0 100644 --- a/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx +++ b/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx @@ -82,8 +82,12 @@ Refer to [`$$nemoclaw recover`](../../reference/commands#$$nemoclaw-name- Recovery uses registry-scoped privileged direct-container control and does not fall back to ordinary `openshell sandbox exec` or a manual in-sandbox relaunch. For a local Docker-driver sandbox whose container still uses the legacy keepalive startup, `recover` can transactionally recreate the registered container with a credential-free managed startup command. -NemoClaw keeps the previous container available until the managed controller proves the supervisor topology, gateway health, and settle check, and attempts to restore it if that proof fails. -The recreation preserves mounted sandbox state, but a committed swap does not retain changes stored only in the previous container's writable layer. +NemoClaw keeps the previous container available until the managed controller proves the supervisor topology, gateway health, and settle check. +Before recreation, NemoClaw backs up the state directories and files declared by the agent manifest. +It commits only after the replacement container identity and state restoration pass. +NemoClaw removes the temporary state backup after a successful restore or rollback. +If state restoration and rollback both fail, it retains the backup and prints host recovery guidance. +Mounted state remains available, but a committed swap does not retain other writable-layer changes. After a transactional recreation, NemoClaw uses the `NEMOCLAW_SANDBOX_READY_TIMEOUT` budget (180 seconds by default) for OpenShell to re-register the sandbox before starting the primary dashboard or API host forward. A definitive managed-health failure still stops immediately; if re-registration does not complete within the budget, the forward stays stopped. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index f4d83aee124..23e4a3e4a30 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -1164,8 +1164,12 @@ In a direct root-entrypoint container, the request reaches the root PID 1 superv In an OpenShell-managed container, the request enters the root-owned mode `0500` managed controller through a sanitized root exec while OpenShell remains PID 1. It does not use ordinary `openshell sandbox exec` or an in-sandbox manual relaunch as a fallback. When the root-owned managed controller attests two unchanged zero-supervisor process scans with a stable PID 1 and reports `SUPERVISOR_NOT_RUNNING`, a local Docker-driver sandbox with the legacy keepalive startup can enter a transactional container recreation. -The recreation uses a credential-free managed startup command, pins the registered container identity, retains the previous container for rollback, and commits only after managed gateway health and the settle check pass. -The recreation preserves mounted sandbox state, but a committed swap does not retain changes stored only in the previous container's writable layer. +The recreation uses a credential-free managed startup command, pins the registered container identity, and retains the previous container for rollback. +Before recreation, NemoClaw backs up the state directories and files declared by the agent manifest. +It commits only after managed gateway health, the settle check, exact replacement identity, and state restoration pass. +NemoClaw removes the temporary state backup after a successful restore or rollback. +If state restoration and rollback both fail, it retains the backup and prints host recovery guidance. +Mounted state remains available, but a committed swap does not retain other writable-layer changes. It is idempotent. When `recover` repairs a stopped built-in OpenClaw or Hermes gateway, it repeats the recovery action only for an exit status of `1` with blank stdout and a sole nonblank stderr line equal to `SUPERVISOR_BUSY`, with at most three controller attempts. The same result is inconclusive during managed settle confirmation and can be probed again only within the configured settle window. diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index 9bc28614aee..02044307a6f 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -1302,11 +1302,41 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( if (relaunch) { try { const completion = relaunch.finalize(true); + if (completion.stateRestored === false || completion.rolledBack) { + if (!quiet) { + console.error( + completion.rolledBack + ? " Sandbox recovery did not complete; the previous container was restored." + : " Sandbox recovery failed and the previous container could not be restored automatically.", + ); + if (completion.rolledBack && completion.stateBackupRemoved === false) { + console.error(" Warning: the temporary sandbox state backup could not be removed."); + } + if (!completion.rolledBack) { + printHostManagedGatewayRecoveryHints( + sandboxName, + recoveryAgent, + managedRecoveryFailureLayer, + ); + } + } + return { + checked: true, + wasRunning: false, + recovered: false, + forwardRecovered: false, + }; + } if (!completion.backupRemoved && !quiet) { console.error( " Warning: the recovered sandbox is healthy, but its previous container backup could not be removed.", ); } + if (completion.stateBackupRemoved === false && !quiet) { + console.error( + " Warning: the recovered sandbox is healthy, but its temporary state backup could not be removed.", + ); + } } catch { if (!quiet) { console.error( diff --git a/src/lib/actions/sandbox/supervisor-relaunch.test.ts b/src/lib/actions/sandbox/supervisor-relaunch.test.ts index f66776e636d..28f638085a0 100644 --- a/src/lib/actions/sandbox/supervisor-relaunch.test.ts +++ b/src/lib/actions/sandbox/supervisor-relaunch.test.ts @@ -47,11 +47,32 @@ function baseDeps(overrides: ManagedSupervisorRelaunchDeps = {}) { }) as never, ), resolveDashboardPort: vi.fn(() => 18789), - resolveContainer: vi.fn(() => "old-container-id"), + resolveContainer: vi + .fn() + .mockReturnValueOnce("old-container-id") + .mockReturnValue("new-container-id"), inspectContainer: vi.fn(() => ({ Config: { Env: ["OPENSHELL_SANDBOX_COMMAND=sleep infinity"] }, })), confirmMissingSupervisor: vi.fn(() => true), + backupState: vi.fn(() => ({ + success: true, + manifest: { + backupPath: "/tmp/rebuild-backups/alpha/recovery", + }, + backedUpDirs: ["workspace"], + failedDirs: [], + backedUpFiles: [], + failedFiles: [], + })) as never, + restoreState: vi.fn(() => ({ + success: true, + restoredDirs: ["workspace"], + failedDirs: [], + restoredFiles: [], + failedFiles: [], + })), + removeBackup: vi.fn(() => true), recreate: vi.fn(() => patchResult()), finalize: vi.fn(({ supervisorReady }) => supervisorReady @@ -123,7 +144,14 @@ describe("relaunchManagedSupervisorSession", () => { expect(serialized).not.toContain("CUSTOM_PROVIDER_CREDENTIAL"); expect(serialized).not.toContain("proxypass"); - expect(relaunch?.finalize(true)).toEqual({ backupRemoved: true, rolledBack: false }); + expect(relaunch?.finalize(true)).toEqual({ + backupRemoved: true, + rolledBack: false, + stateRestored: true, + stateBackupRemoved: true, + }); + expect(deps.restoreState).toHaveBeenCalledWith("alpha", "/tmp/rebuild-backups/alpha/recovery"); + expect(deps.removeBackup).toHaveBeenCalledWith("alpha", "/tmp/rebuild-backups/alpha/recovery"); expect(deps.finalize).toHaveBeenCalledWith({ result: expect.objectContaining({ newContainerId: "new-container-id" }), supervisorReady: true, @@ -134,13 +162,116 @@ describe("relaunchManagedSupervisorSession", () => { const deps = baseDeps(); const relaunch = relaunchManagedSupervisorSession("alpha", { quiet: true, deps }); - expect(relaunch?.finalize(false)).toEqual({ backupRemoved: false, rolledBack: true }); + expect(relaunch?.finalize(false)).toEqual({ + backupRemoved: false, + rolledBack: true, + stateRestored: false, + stateBackupRemoved: true, + }); + expect(deps.restoreState).not.toHaveBeenCalled(); + expect(deps.removeBackup).toHaveBeenCalledWith("alpha", "/tmp/rebuild-backups/alpha/recovery"); + expect(deps.finalize).toHaveBeenCalledWith({ + result: expect.objectContaining({ backupContainerName: expect.any(String) }), + supervisorReady: false, + }); + }); + + it("removes a partial state backup before it refuses recreation (#7404)", () => { + const deps = baseDeps({ + backupState: vi.fn(() => ({ + success: false, + manifest: { + backupPath: "/tmp/rebuild-backups/alpha/partial-recovery", + } as never, + backedUpDirs: [], + failedDirs: ["workspace"], + backedUpFiles: [], + failedFiles: [], + })), + }); + + expect(relaunchManagedSupervisorSession("alpha", { quiet: true, deps })).toBeNull(); + expect(deps.removeBackup).toHaveBeenCalledWith( + "alpha", + "/tmp/rebuild-backups/alpha/partial-recovery", + ); + expect(deps.recreate).not.toHaveBeenCalled(); + }); + + it("rolls back the container transaction when state restore fails", () => { + const deps = baseDeps({ + restoreState: vi.fn(() => ({ + success: false, + restoredDirs: [], + failedDirs: ["workspace"], + restoredFiles: [], + failedFiles: [], + })), + }); + const relaunch = relaunchManagedSupervisorSession("alpha", { quiet: true, deps }); + + expect(relaunch?.finalize(true)).toEqual({ + backupRemoved: false, + rolledBack: true, + stateRestored: false, + stateBackupRemoved: true, + }); + expect(deps.removeBackup).toHaveBeenCalledWith("alpha", "/tmp/rebuild-backups/alpha/recovery"); expect(deps.finalize).toHaveBeenCalledWith({ result: expect.objectContaining({ backupContainerName: expect.any(String) }), supervisorReady: false, }); }); + it("rolls back before restore when the replacement container identity changes", () => { + const deps = baseDeps({ + resolveContainer: vi + .fn() + .mockReturnValueOnce("old-container-id") + .mockReturnValue("different-container-id"), + }); + const relaunch = relaunchManagedSupervisorSession("alpha", { quiet: true, deps }); + + expect(relaunch?.finalize(true)).toEqual({ + backupRemoved: false, + rolledBack: true, + stateRestored: false, + stateBackupRemoved: true, + }); + expect(deps.restoreState).not.toHaveBeenCalled(); + expect(deps.removeBackup).toHaveBeenCalledWith("alpha", "/tmp/rebuild-backups/alpha/recovery"); + expect(deps.finalize).toHaveBeenCalledWith({ + result: expect.objectContaining({ backupContainerName: expect.any(String) }), + supervisorReady: false, + }); + }); + + it("retains the state backup when rollback fails", () => { + const deps = baseDeps({ + finalize: vi.fn(() => ({ backupRemoved: false, rolledBack: false })), + }); + const relaunch = relaunchManagedSupervisorSession("alpha", { quiet: true, deps }); + + expect(relaunch?.finalize(false)).toEqual({ + backupRemoved: false, + rolledBack: false, + stateRestored: false, + }); + expect(deps.removeBackup).not.toHaveBeenCalled(); + }); + + it("reports best-effort state-backup cleanup failure after a successful restore", () => { + const deps = baseDeps({ removeBackup: vi.fn(() => false) }); + const relaunch = relaunchManagedSupervisorSession("alpha", { quiet: true, deps }); + + expect(relaunch?.finalize(true)).toEqual({ + backupRemoved: true, + rolledBack: false, + stateRestored: true, + stateBackupRemoved: false, + }); + }); + it("returns null when the pinned recreation fails", () => { const deps = baseDeps({ recreate: vi.fn(() => { @@ -149,6 +280,25 @@ describe("relaunchManagedSupervisorSession", () => { }); expect(relaunchManagedSupervisorSession("alpha", { quiet: true, deps })).toBeNull(); + expect(deps.removeBackup).toHaveBeenCalledWith("alpha", "/tmp/rebuild-backups/alpha/recovery"); + }); + + it("preserves the recreation diagnostic when state-backup cleanup throws", () => { + vi.spyOn(console, "log").mockImplementation(() => undefined); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); + const deps = baseDeps({ + removeBackup: vi.fn(() => { + throw new Error("backup cleanup failed"); + }), + recreate: vi.fn(() => { + throw new Error("container identity changed"); + }), + }); + + expect(relaunchManagedSupervisorSession("alpha", { quiet: false, deps })).toBeNull(); + const output = errorSpy.mock.calls.flat().join("\n"); + expect(output).toContain("container identity changed"); + expect(output).not.toContain("backup cleanup failed"); }); it("redacts diagnostics when trusted recreation fails", () => { diff --git a/src/lib/actions/sandbox/supervisor-relaunch.ts b/src/lib/actions/sandbox/supervisor-relaunch.ts index 729827fa866..ff9a2f48740 100644 --- a/src/lib/actions/sandbox/supervisor-relaunch.ts +++ b/src/lib/actions/sandbox/supervisor-relaunch.ts @@ -8,6 +8,7 @@ import { type DockerContainerInspect, parseDockerInspectJson, } from "../../onboard/docker-gpu-patch"; +import { sameContainerId } from "../../onboard/docker-gpu-patch-clone"; import { type DockerGpuPatchFinalizeOutcome, finalizeDockerGpuPatchBackup, @@ -17,6 +18,7 @@ import { buildSandboxRuntimeEnvArgs } from "../../onboard/sandbox-create-launch" import { resolveDirectSandboxContainer } from "../../sandbox/privileged-exec"; import { redact, redactFull } from "../../security/redact"; import * as registry from "../../state/registry"; +import * as sandboxState from "../../state/sandbox"; import { resolveSandboxDashboardPort } from "./forward-recovery"; /** @@ -33,7 +35,10 @@ const DOCKER_INSPECT_TIMEOUT_MS = 15000; export type ManagedSupervisorRelaunch = { containerId: string; - finalize(supervisorReady: boolean): DockerGpuPatchFinalizeOutcome; + finalize(supervisorReady: boolean): DockerGpuPatchFinalizeOutcome & { + stateRestored?: boolean; + stateBackupRemoved?: boolean; + }; }; export type ManagedSupervisorRelaunchDeps = { @@ -43,6 +48,9 @@ export type ManagedSupervisorRelaunchDeps = { resolveContainer?: typeof resolveDirectSandboxContainer; inspectContainer?: (containerId: string) => DockerContainerInspect; confirmMissingSupervisor?: (containerId: string) => boolean; + backupState?: typeof sandboxState.backupSandboxState; + restoreState?: typeof sandboxState.restoreSandboxState; + removeBackup?: typeof sandboxState.removeSandboxStateBackup; recreate?: typeof recreateOpenShellDockerSandboxWithStartupCommand; finalize?: typeof finalizeDockerGpuPatchBackup; }; @@ -128,12 +136,40 @@ export function relaunchManagedSupervisorSession( const resolveContainer = deps.resolveContainer ?? resolveDirectSandboxContainer; const inspect = deps.inspectContainer ?? inspectContainer; const confirmMissingSupervisor = deps.confirmMissingSupervisor; + const backupState = deps.backupState ?? sandboxState.backupSandboxState; + const restoreState = deps.restoreState ?? sandboxState.restoreSandboxState; + const removeBackup = deps.removeBackup ?? sandboxState.removeSandboxStateBackup; const recreate = deps.recreate ?? recreateOpenShellDockerSandboxWithStartupCommand; const finalize = deps.finalize ?? finalizeDockerGpuPatchBackup; + let pendingStateBackupPath: string | null = null; try { const containerId = resolveContainer(sandboxName, driver); if (!hasLegacyKeepaliveStartup(inspect(containerId))) return null; if (!confirmMissingSupervisor?.(containerId)) return null; + const backup = backupState(sandboxName); + if ( + !backup.success || + !backup.manifest || + backup.failedDirs.length > 0 || + backup.failedFiles.length > 0 + ) { + if (backup.manifest) { + try { + removeBackup(sandboxName, backup.manifest.backupPath); + } catch { + // Preserve the backup failure that stopped container recreation. + } + } + if (!quiet) { + console.error( + " Trusted container recovery stopped before recreation because sandbox state could not be fully backed up.", + ); + console.error(" The existing sandbox container was left unchanged."); + } + return null; + } + const backupManifest = backup.manifest; + pendingStateBackupPath = backupManifest.backupPath; if (!quiet) { console.log(" Recreating the sandbox container with its managed startup command..."); } @@ -143,8 +179,21 @@ export function relaunchManagedSupervisorSession( expectedOldContainerId: containerId, waitForSupervisor: false, }); - let completed: { supervisorReady: boolean; outcome: DockerGpuPatchFinalizeOutcome } | null = - null; + pendingStateBackupPath = null; + let completed: { + supervisorReady: boolean; + outcome: DockerGpuPatchFinalizeOutcome & { + stateRestored?: boolean; + stateBackupRemoved?: boolean; + }; + } | null = null; + const removeSettledStateBackup = (): boolean => { + try { + return removeBackup(sandboxName, backupManifest.backupPath); + } catch { + return false; + } + }; return { containerId: result.newContainerId, finalize(supervisorReady) { @@ -156,12 +205,68 @@ export function relaunchManagedSupervisorSession( } return completed.outcome; } - const outcome = finalize({ result, supervisorReady }); + if (!supervisorReady) { + const finalized = finalize({ result, supervisorReady: false }); + const outcome = { + ...finalized, + stateRestored: false, + ...(finalized.rolledBack ? { stateBackupRemoved: removeSettledStateBackup() } : {}), + }; + completed = { supervisorReady, outcome }; + return outcome; + } + let replacementOwned = false; + try { + replacementOwned = sameContainerId( + resolveContainer(sandboxName, driver), + result.newContainerId, + ); + } catch { + replacementOwned = false; + } + if (!replacementOwned) { + const finalized = finalize({ result, supervisorReady: false }); + const outcome = { + ...finalized, + stateRestored: false, + ...(finalized.rolledBack ? { stateBackupRemoved: removeSettledStateBackup() } : {}), + }; + completed = { supervisorReady, outcome }; + return outcome; + } + let stateRestored = false; + try { + stateRestored = restoreState(sandboxName, backupManifest.backupPath).success; + } catch { + stateRestored = false; + } + if (!stateRestored) { + const finalized = finalize({ result, supervisorReady: false }); + const outcome = { + ...finalized, + stateRestored: false, + ...(finalized.rolledBack ? { stateBackupRemoved: removeSettledStateBackup() } : {}), + }; + completed = { supervisorReady, outcome }; + return outcome; + } + const outcome = { + ...finalize({ result, supervisorReady: true }), + stateRestored: true, + stateBackupRemoved: removeSettledStateBackup(), + }; completed = { supervisorReady, outcome }; return outcome; }, }; } catch (error) { + if (pendingStateBackupPath) { + try { + removeBackup(sandboxName, pendingStateBackupPath); + } catch { + // Preserve the recreation failure that stopped container recovery. + } + } if (!quiet) { const detail = error instanceof Error ? error.message : String(error); console.error(` Trusted container recovery could not start: ${redactFull(redact(detail))}`); diff --git a/src/lib/state/sandbox.ts b/src/lib/state/sandbox.ts index af6040273a7..55bb64c7d54 100644 --- a/src/lib/state/sandbox.ts +++ b/src/lib/state/sandbox.ts @@ -1838,6 +1838,32 @@ export type RebuildRecoveryManifestValidation = | { ok: true; manifest: RebuildManifest } | { ok: false; reason: string }; +/** + * Remove one completed rebuild backup without allowing a caller-controlled + * path to escape the sandbox's timestamped backup directory. + */ +export function removeSandboxStateBackup(sandboxName: string, backupPath: string): boolean { + const rebuildBackupsRoot = path.resolve(REBUILD_BACKUPS_DIR); + const sandboxBackupRoot = path.resolve(rebuildBackupsRoot, sandboxName); + const candidateBackupPath = path.resolve(backupPath); + + if ( + sandboxBackupRoot === rebuildBackupsRoot || + !isWithinRoot(sandboxBackupRoot, rebuildBackupsRoot) || + normalizeHostPath(path.dirname(candidateBackupPath)) !== normalizeHostPath(sandboxBackupRoot) + ) { + return false; + } + + try { + rejectSymlinksOnPath(candidateBackupPath); + rmSync(candidateBackupPath, { recursive: true, force: true }); + return !existsSync(candidateBackupPath); + } catch { + return false; + } +} + /** * Re-read and validate a prepared rebuild backup before a destructive recovery. * diff --git a/test/process-recovery-supervisor-relaunch.test.ts b/test/process-recovery-supervisor-relaunch.test.ts index cb0fe7ac52e..b06684536e9 100644 --- a/test/process-recovery-supervisor-relaunch.test.ts +++ b/test/process-recovery-supervisor-relaunch.test.ts @@ -220,6 +220,95 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { expect(finalize).toHaveBeenCalledWith(true); }); + it("reports recovery failure when state restore rolls the replacement back", () => { + mockOpenClawSandbox("restore-failed-box"); + setImmediateRecoveryPolling(); + const finalize = vi.fn(() => ({ + backupRemoved: false, + rolledBack: true, + stateRestored: false, + })); + const relaunchManagedSupervisorSessionImpl = vi.fn(() => ({ + containerId: "replacement-container-id", + finalize, + })); + const requestGatewaySupervisorAction = vi.fn((_name: string, action: string) => + action === "recover" ? { status: 1, stdout: "", stderr: "SUPERVISOR_NOT_RUNNING" } : null, + ); + const requestPinnedGatewaySupervisorAction = vi.fn(() => ({ + status: 0, + stdout: "GATEWAY_PID=4242\n", + stderr: "", + })); + const waitForRecreatedSandboxOpenShellReadyImpl = vi.fn(() => true); + const runOpenshell = vi.spyOn(openshellRuntime, "runOpenshell"); + + const result = checkAndRecoverSandboxProcesses("restore-failed-box", { + quiet: true, + isSandboxGatewayRunningImpl: () => false, + requestGatewaySupervisorAction, + requestPinnedGatewaySupervisorAction, + relaunchManagedSupervisorSessionImpl, + waitForRecreatedSandboxOpenShellReadyImpl, + }); + + expect(result).toMatchObject({ + checked: true, + wasRunning: false, + recovered: false, + forwardRecovered: false, + }); + expect(finalize).toHaveBeenCalledOnce(); + expect(finalize).toHaveBeenCalledWith(true); + expect(waitForRecreatedSandboxOpenShellReadyImpl).not.toHaveBeenCalled(); + expect(runOpenshell).not.toHaveBeenCalled(); + }); + + it("prints generic recovery hints when state recovery and rollback both fail", () => { + mockOpenClawSandbox("restore-and-rollback-failed-box"); + setImmediateRecoveryPolling(); + const finalize = vi.fn(() => ({ + backupRemoved: false, + rolledBack: false, + stateRestored: false, + })); + const relaunchManagedSupervisorSessionImpl = vi.fn(() => ({ + containerId: "replacement-container-id", + finalize, + })); + const requestGatewaySupervisorAction = vi.fn((_name: string, action: string) => + action === "recover" ? { status: 1, stdout: "", stderr: "SUPERVISOR_NOT_RUNNING" } : null, + ); + const requestPinnedGatewaySupervisorAction = vi.fn(() => ({ + status: 0, + stdout: "GATEWAY_PID=4242\n", + stderr: "", + })); + vi.spyOn(console, "log").mockImplementation(() => undefined); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); + + const result = checkAndRecoverSandboxProcesses("restore-and-rollback-failed-box", { + quiet: false, + isSandboxGatewayRunningImpl: () => false, + requestGatewaySupervisorAction, + requestPinnedGatewaySupervisorAction, + relaunchManagedSupervisorSessionImpl, + }); + + expect(result).toMatchObject({ + checked: true, + wasRunning: false, + recovered: false, + forwardRecovered: false, + }); + const output = errorSpy.mock.calls.flat().join("\n"); + expect(output).toContain( + "Sandbox recovery failed and the previous container could not be restored automatically.", + ); + expect(output).toContain("rebuild --yes"); + expect(output).not.toContain("Sandbox state restore failed"); + }); + it("retries a busy pinned managed probe before starting the replacement forward", () => { mockOpenClawSandbox("busy-recovered-box"); vi.stubEnv("NEMOCLAW_GATEWAY_RECOVERY_POLL_INTERVAL_SECONDS", "0"); diff --git a/test/snapshot-recovery-validation.test.ts b/test/snapshot-recovery-validation.test.ts index dbbb054b5b8..73d2363f8d0 100644 --- a/test/snapshot-recovery-validation.test.ts +++ b/test/snapshot-recovery-validation.test.ts @@ -53,6 +53,31 @@ beforeEach(() => { }); describe("prepared rebuild backup recovery validation (#6114)", () => { + it("removes only an exact backup child owned by the target sandbox", () => { + const manifest = writeBackup("alpha", "2026-07-01T06-50-42-043Z"); + const outsidePath = path.join(TMP_HOME, "outside-backup"); + fs.mkdirSync(outsidePath, { recursive: true }); + + expect(sandboxState.removeSandboxStateBackup("alpha", String(manifest.backupPath))).toBe(true); + expect(fs.existsSync(String(manifest.backupPath))).toBe(false); + expect(sandboxState.removeSandboxStateBackup("alpha", outsidePath)).toBe(false); + expect(fs.existsSync(outsidePath)).toBe(true); + }); + + it("refuses to remove a backup path that is a symbolic link", () => { + const sandboxBackupRoot = path.join(BACKUPS_ROOT, "alpha"); + const backupPath = path.join(sandboxBackupRoot, "2026-07-01T06-50-42-043Z"); + const outsidePath = path.join(TMP_HOME, "outside-backup"); + const outsideMarker = path.join(outsidePath, "keep.txt"); + fs.mkdirSync(sandboxBackupRoot, { recursive: true }); + fs.mkdirSync(outsidePath, { recursive: true }); + fs.writeFileSync(outsideMarker, "keep"); + fs.symlinkSync(outsidePath, backupPath, "dir"); + + expect(sandboxState.removeSandboxStateBackup("alpha", backupPath)).toBe(false); + expect(fs.readFileSync(outsideMarker, "utf8")).toBe("keep"); + }); + it("does not expose a latest backup with a missing or malformed manifest", () => { const backupPath = path.join(BACKUPS_ROOT, "alpha", "2026-07-01T06-50-41-044Z"); fs.mkdirSync(backupPath, { recursive: true });