diff --git a/docs/manage-sandboxes/backup-restore.mdx b/docs/manage-sandboxes/backup-restore.mdx index 20f5bf8711b..a4dfb300a57 100644 --- a/docs/manage-sandboxes/backup-restore.mdx +++ b/docs/manage-sandboxes/backup-restore.mdx @@ -106,6 +106,11 @@ Tag a snapshot with a human-readable label: $$nemoclaw my-assistant snapshot create --name before-upgrade ``` +When a directory or state file cannot be captured, `snapshot create` reports the failed items, removes the incomplete snapshot, and exits nonzero. +`snapshot list` shows no new entry, so a later restore cannot select a capture that never completed. +If removal fails, the command reports the listed snapshot path. +Remove that directory manually before you run `snapshot restore` because the incomplete capture remains selectable. + ## Restore a Snapshot Restore the latest snapshot: diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 966af77fcd5..9c655834698 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -3077,6 +3077,11 @@ Names must be 1 to 63 characters from `[A-Za-z0-9._-]`, start with an alphanumer $$nemoclaw my-assistant snapshot create --name before-upgrade ``` +When a directory or state file cannot be captured, the command reports the failed items, removes the incomplete snapshot, and exits nonzero. +A removed snapshot does not appear in `snapshot list` and cannot be restored, so a later restore cannot select a capture that never completed. +When removal fails, the command reports the listed snapshot path and exits nonzero. +Remove that directory manually before you run `snapshot restore` because the incomplete capture remains selectable. + ### `$$nemoclaw snapshot list` List available snapshots for a sandbox as a table of version, name, timestamp, and path. diff --git a/src/lib/actions/sandbox/snapshot-failed-create-cleanup.test.ts b/src/lib/actions/sandbox/snapshot-failed-create-cleanup.test.ts new file mode 100644 index 00000000000..ea8cfffe562 --- /dev/null +++ b/src/lib/actions/sandbox/snapshot-failed-create-cleanup.test.ts @@ -0,0 +1,161 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + backupSandboxState: vi.fn(), + captureOpenshell: vi.fn(() => ({ status: 0, output: "alpha Ready\n" })), + findBackup: vi.fn(() => ({ match: null })), + removeIncompleteSnapshot: vi.fn( + () => + ({ removed: true }) as { + removed: boolean; + error?: string; + }, + ), +})); + +vi.mock("../../adapters/openshell/runtime", () => ({ + captureOpenshell: mocks.captureOpenshell, + getOpenshellBinary: vi.fn(() => "openshell"), + runOpenshell: vi.fn(), +})); + +vi.mock("../../runtime-recovery", () => ({ + parseLiveSandboxNames: vi.fn(() => new Set(["alpha"])), +})); + +vi.mock("../../shields", () => ({ + isShieldsDown: vi.fn(() => true), +})); + +vi.mock("../../shields/timer-bound-lock", () => ({ + withTimerBoundShieldsMutationLock: vi.fn( + (_sandboxName: string, _command: string, operation: () => unknown) => operation(), + ), +})); + +vi.mock("../../state/registry", () => ({ + getBaselineExclusions: vi.fn(() => []), + getSandbox: vi.fn(() => ({ name: "alpha", agent: "openclaw" })), +})); + +vi.mock("../../state/sandbox", () => ({ + backupSandboxState: mocks.backupSandboxState, + findBackup: mocks.findBackup, + removeIncompleteSnapshot: mocks.removeIncompleteSnapshot, +})); + +vi.mock("./sandbox-gateway-routing", () => ({ + probeGatewayRunning: vi.fn(() => true), + selectSandboxGatewayIfRegistered: vi.fn(() => true), + usesGatewayMetadataProbe: vi.fn(() => false), +})); + +const INCOMPLETE_PATH = "/home/user/.nemoclaw/rebuild-backups/alpha/2026-08-04T06-53-38-310Z"; + +function failedCaptureWithPublishedSnapshot(overrides: Record = {}) { + return { + success: false, + manifest: { backupPath: INCOMPLETE_PATH }, + backedUpDirs: ["workspace"], + failedDirs: [], + backedUpFiles: [], + failedFiles: ["openclaw.json"], + ...overrides, + }; +} + +describe("snapshot create cleanup after a failed capture", () => { + beforeEach(() => { + vi.clearAllMocks(); + mocks.removeIncompleteSnapshot.mockReturnValue({ removed: true }); + mocks.findBackup.mockReturnValue({ match: null }); + vi.spyOn(console, "log").mockImplementation(() => {}); + vi.spyOn(console, "error").mockImplementation(() => {}); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + async function createSnapshot(): Promise { + const { runSandboxSnapshot } = await import("./snapshot"); + await expect(runSandboxSnapshot("alpha", { kind: "create" })).rejects.toMatchObject({ + exitCode: 1, + }); + return vi.mocked(console.error).mock.calls.flat().join("\n"); + } + + it("reports the failed directories and files", async () => { + mocks.backupSandboxState.mockReturnValue( + failedCaptureWithPublishedSnapshot({ + failedDirs: ["workspace", "skills"], + failedDirReasons: { workspace: "permission denied" }, + }), + ); + + const errors = await createSnapshot(); + + expect(errors).toContain("Snapshot failed."); + expect(errors).toContain("Failed directories: workspace (permission denied), skills"); + expect(errors).toContain("Failed files: openclaw.json"); + }); + + it("removes the snapshot so a later restore cannot select an incomplete capture (#8201)", async () => { + mocks.backupSandboxState.mockReturnValue(failedCaptureWithPublishedSnapshot()); + + const errors = await createSnapshot(); + + expect(mocks.removeIncompleteSnapshot).toHaveBeenCalledWith(INCOMPLETE_PATH); + expect(errors).toContain("Removed the incomplete snapshot."); + }); + + it("names the snapshot that is still listed when removal fails", async () => { + mocks.backupSandboxState.mockReturnValue(failedCaptureWithPublishedSnapshot()); + mocks.removeIncompleteSnapshot.mockReturnValue({ + removed: false, + error: "EACCES: permission denied", + }); + + const errors = await createSnapshot(); + + expect(errors).toContain( + `The incomplete snapshot at '${INCOMPLETE_PATH}' could not be removed: EACCES: permission denied`, + ); + expect(errors).toContain("Remove it before the next restore."); + }); + + it("does not attempt removal when the capture failed before publishing a snapshot", async () => { + mocks.backupSandboxState.mockReturnValue({ + success: false, + backedUpDirs: [], + failedDirs: [], + backedUpFiles: [], + failedFiles: [], + error: "Snapshot name 'dup' already exists.", + }); + + const errors = await createSnapshot(); + + expect(mocks.removeIncompleteSnapshot).not.toHaveBeenCalled(); + expect(errors).toContain("Snapshot name 'dup' already exists."); + }); + + it("does not touch a snapshot whose capture succeeded", async () => { + mocks.backupSandboxState.mockReturnValue({ + success: true, + manifest: { backupPath: INCOMPLETE_PATH, timestamp: "2026-08-04T06-53-38-310Z" }, + backedUpDirs: ["workspace"], + failedDirs: [], + backedUpFiles: ["openclaw.json"], + failedFiles: [], + }); + const { runSandboxSnapshot } = await import("./snapshot"); + + await runSandboxSnapshot("alpha", { kind: "create" }); + + expect(mocks.removeIncompleteSnapshot).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/actions/sandbox/snapshot.test.ts b/src/lib/actions/sandbox/snapshot.test.ts index 19cf6428de0..4d402f1d9f0 100644 --- a/src/lib/actions/sandbox/snapshot.test.ts +++ b/src/lib/actions/sandbox/snapshot.test.ts @@ -1476,25 +1476,4 @@ describe("runSandboxSnapshot", () => { "Warning: could not reconcile custom policy(ies): old-custom (remove failed)", ); }); - - it("prints failed dirs and files when snapshot creation fails without an error", async () => { - backupSandboxStateMock.mockReturnValue({ - success: false, - failedDirs: ["workspace", "skills"], - failedDirReasons: { workspace: "permission denied" }, - failedFiles: ["openclaw.json"], - }); - const consoleError = vi.spyOn(console, "error").mockImplementation(() => {}); - vi.spyOn(console, "log").mockImplementation(() => {}); - const { runSandboxSnapshot } = await import("./snapshot"); - - await expect(runSandboxSnapshot("alpha", { kind: "create" })).rejects.toMatchObject({ - exitCode: 1, - }); - - const errors = consoleError.mock.calls.flat().join("\n"); - expect(errors).toContain("Snapshot failed."); - expect(errors).toContain("Failed directories: workspace (permission denied), skills"); - expect(errors).toContain("Failed files: openclaw.json"); - }); }); diff --git a/src/lib/actions/sandbox/snapshot.ts b/src/lib/actions/sandbox/snapshot.ts index 77595a0ab18..6ec7fb850bb 100644 --- a/src/lib/actions/sandbox/snapshot.ts +++ b/src/lib/actions/sandbox/snapshot.ts @@ -659,6 +659,20 @@ function isSnapshotCreationAllowedByDcodeActivity(sandboxName: string): boolean return false; } +function removeIncompleteSnapshot(sandboxName: string, backupPath: string): void { + const removal = sandboxState.removeIncompleteSnapshot(backupPath); + if (removal.removed) { + console.error(" Removed the incomplete snapshot."); + return; + } + console.error( + ` The incomplete snapshot at '${backupPath}' could not be removed: ${removal.error}`, + ); + console.error( + ` It is listed by \`${CLI_NAME} ${sandboxName} snapshot list\`. Remove it before the next restore.`, + ); +} + function runSnapshotCreate( sandboxName: string, request: Extract, @@ -722,6 +736,10 @@ function runSnapshotCreate( console.error(` Failed files: ${result.failedFiles.join(", ")}`); } } + const incompletePath = result.manifest?.backupPath; + if (incompletePath) { + removeIncompleteSnapshot(sandboxName, incompletePath); + } snapshotExit(1); }); } diff --git a/src/lib/state/sandbox-incomplete-snapshot-removal.test.ts b/src/lib/state/sandbox-incomplete-snapshot-removal.test.ts new file mode 100644 index 00000000000..9ea4be5d45a --- /dev/null +++ b/src/lib/state/sandbox-incomplete-snapshot-removal.test.ts @@ -0,0 +1,68 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; + +import { removeIncompleteSnapshot } from "./sandbox.js"; + +const testDirectories: string[] = []; + +function createSnapshot(): string { + const backupPath = mkdtempSync(join(tmpdir(), "nemoclaw-incomplete-snapshot-")); + testDirectories.push(backupPath); + mkdirSync(join(backupPath, "workspace"), { recursive: true }); + writeFileSync(join(backupPath, "rebuild-manifest.json"), "{}"); + return backupPath; +} + +afterEach(() => { + for (const testDirectory of testDirectories.splice(0)) { + rmSync(testDirectory, { recursive: true, force: true }); + } +}); + +describe("incomplete snapshot removal", () => { + it("takes the snapshot and its captured content off disk", () => { + const backupPath = createSnapshot(); + + expect(removeIncompleteSnapshot(backupPath)).toEqual({ removed: true }); + expect(existsSync(backupPath)).toBe(false); + }); + + it("reports success for a snapshot that is already gone", () => { + const backupPath = createSnapshot(); + rmSync(backupPath, { recursive: true, force: true }); + + expect(removeIncompleteSnapshot(backupPath)).toEqual({ removed: true }); + }); + + it("reports the reason when removal throws", () => { + const backupPath = createSnapshot(); + + const result = removeIncompleteSnapshot(backupPath, { + removeBackup: () => { + throw new Error("EACCES: permission denied"); + }, + }); + + expect(result).toEqual({ removed: false, error: "EACCES: permission denied" }); + expect(existsSync(backupPath)).toBe(true); + }); + + it("reports failure when removal reports success but the snapshot remains", () => { + const backupPath = createSnapshot(); + + const result = removeIncompleteSnapshot(backupPath, { + removeBackup: () => undefined, + }); + + expect(result).toEqual({ + removed: false, + error: "the snapshot directory still exists after removal", + }); + expect(existsSync(backupPath)).toBe(true); + }); +}); diff --git a/src/lib/state/sandbox.ts b/src/lib/state/sandbox.ts index 05fed8025a5..0266b95dd86 100644 --- a/src/lib/state/sandbox.ts +++ b/src/lib/state/sandbox.ts @@ -731,6 +731,27 @@ export function sanitizeBackupDirectory( } } +export interface IncompleteSnapshotRemoval { + readonly removed: boolean; + readonly error?: string; +} + +export function removeIncompleteSnapshot( + backupPath: string, + overrides: Partial> = {}, +): IncompleteSnapshotRemoval { + const operations = { ...DEFAULT_BACKUP_SANITIZATION_OPERATIONS, ...overrides }; + try { + operations.removeBackup(backupPath); + } catch (error) { + return { removed: false, error: error instanceof Error ? error.message : String(error) }; + } + if (operations.backupExists(backupPath)) { + return { removed: false, error: "the snapshot directory still exists after removal" }; + } + return { removed: true }; +} + // ── Logging ──────────────────────────────────────────────────────── const _verbose = () => process.env.NEMOCLAW_REBUILD_VERBOSE === "1"; diff --git a/test/snapshot.test.ts b/test/snapshot.test.ts index 26f7f05ee33..c53916f28bc 100644 --- a/test/snapshot.test.ts +++ b/test/snapshot.test.ts @@ -212,6 +212,39 @@ describe("listBackups computes virtual versions", () => { expect(entry.name).toBe("before-upgrade"); expect(entry.snapshotVersion).toBe(1); }); + it("stops listing a snapshot that was removed as incomplete (#8201)", () => { + const incomplete = writeBackup("test-sandbox", "2026-04-21T14-00-00-000Z", { + name: "failtest", + }); + expect(sandboxState.listBackups("test-sandbox")).toHaveLength(1); + + expect(sandboxState.removeIncompleteSnapshot(String(incomplete.backupPath))).toEqual({ + removed: true, + }); + + expect(sandboxState.listBackups("test-sandbox")).toEqual([]); + expect(sandboxState.findBackup("test-sandbox", "failtest").match).toBeNull(); + expect(fs.existsSync(String(incomplete.backupPath))).toBe(false); + }); + it("restores the versions of surviving snapshots once an incomplete one is removed (#8201)", () => { + writeBackup("test-sandbox", "2026-04-21T14-01-00-000Z"); + const incomplete = writeBackup("test-sandbox", "2026-04-21T14-05-00-000Z"); + writeBackup("test-sandbox", "2026-04-21T14-10-00-000Z"); + expect(sandboxState.listBackups("test-sandbox").map((b) => b.timestamp)).toEqual([ + "2026-04-21T14-10-00-000Z", + "2026-04-21T14-05-00-000Z", + "2026-04-21T14-01-00-000Z", + ]); + + sandboxState.removeIncompleteSnapshot(String(incomplete.backupPath)); + + expect( + sandboxState.listBackups("test-sandbox").map((b) => [b.snapshotVersion, b.timestamp]), + ).toEqual([ + [2, "2026-04-21T14-10-00-000Z"], + [1, "2026-04-21T14-01-00-000Z"], + ]); + }); it("surfaces customPolicies (name + content + sourcePath) through the manifest round-trip", () => { const custom = [