From 6ec96315538f62aae88e635d28cacbc72ca10195 Mon Sep 17 00:00:00 2001 From: San Dang Date: Tue, 4 Aug 2026 16:29:32 +0530 Subject: [PATCH 1/2] fix(cli): improve failure diagnostics Signed-off-by: San Dang --- docs/manage-sandboxes/backup-restore.mdx | 7 +- docs/reference/troubleshooting.mdx | 14 +++ .../shared/snapshot-sanitizer-boundary.cts | 8 +- ...xport-managed-image-failure-diagnostics.ts | 1 + .../onboard/created-sandbox-failure.test.ts | 10 +++ src/lib/onboard/created-sandbox-failure.ts | 13 ++- src/lib/onboard/sandbox-create-failure.ts | 87 +++++++++++++++++-- .../onboard/sandbox-gpu-create-run-attempt.ts | 1 + src/lib/sandbox-base-image.test.ts | 10 ++- src/lib/sandbox-base-image.ts | 4 +- src/lib/security/snapshot-sanitizer.ts | 11 +++ .../state/sandbox-backup-sanitization.test.ts | 10 +++ src/lib/state/sandbox.ts | 36 +++++--- src/nemoclaw.ts | 17 +++- .../managed-image-failure-diagnostics.test.ts | 7 +- test/onboard-sandbox-create-failure.test.ts | 48 ++++++++++ .../cli/top-level-error.test.ts | 45 ++++++++++ 17 files changed, 297 insertions(+), 32 deletions(-) create mode 100644 test/package-contract/cli/top-level-error.test.ts diff --git a/docs/manage-sandboxes/backup-restore.mdx b/docs/manage-sandboxes/backup-restore.mdx index 10f2d02d2d4..e963aaf5920 100644 --- a/docs/manage-sandboxes/backup-restore.mdx +++ b/docs/manage-sandboxes/backup-restore.mdx @@ -43,7 +43,12 @@ It preserves OpenShell credential placeholders so rebuild can reattach the host- If NemoClaw cannot sanitize a copied configuration or environment file, it omits that file from the snapshot. If it cannot remove the unsafe file, snapshot creation returns an error. It deletes the incomplete backup when cleanup succeeds and reports when the backup remains. -This sanitization uses an isolated `python3` helper on POSIX hosts to keep reads, replacements, and removals anchored to opened directory descriptors. +Snapshot sanitization on POSIX hosts requires Python 3 from a trusted absolute path. +NemoClaw checks fixed installation paths and the directory that contains the Node.js executable. +It does not resolve the helper through `PATH`, which can contain user-controlled entries. +If no trusted interpreter is available, snapshot creation removes the incomplete backup when possible and tells you to install Python 3 before rerunning the command. +Install Python 3 through your operating-system package manager or Homebrew, then rerun `$$nemoclaw snapshot create`. +NemoClaw runs the helper in isolated mode to keep reads, replacements, and removals anchored to opened directory descriptors. If a copied file or parent directory changes identity during the operation, snapshot creation fails closed instead of following the changed path. diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index e9b1a592270..d1b5f00dbb6 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -528,6 +528,20 @@ Upgrade NemoClaw to a version that supports your OpenShell release, or install a For fresh installs, NemoClaw passes the blueprint range to `install-openshell.sh` and resolves a compatible published OpenShell release before downloading. If GitHub release metadata is unavailable, the script uses its bundled fallback pin and the post-install gate still enforces the configured range. +### Review Sandbox Creation Diagnostics + +When onboarding fails during `openshell sandbox create`, NemoClaw prints the redacted failure and exits with a nonzero status. +Expected CLI failures show the actionable message without internal Node.js stack frames. +The `Diagnostics saved:` line identifies the exact bundle path under the active NemoClaw state root. +The default path is `~/.nemoclaw/onboard-failures/`; a custom gateway port uses `~/.nemoclaw/gateways//onboard-failures/`. + +Open `summary.txt` in the printed directory first. +It records the retained output path under `create_output` and shows the final error lines under `failure_excerpt`. +The `sandbox-create-output.log` file retains a bounded tail of the sandbox creation output. +The bundle also retains any available OpenShell gateway or virtual machine console logs. +This keeps the failing gateway or image-build step available on Linux and macOS, even when the gateway log or console output path is unavailable. +Correct the failure reported in the bundle before you rerun `$$nemoclaw onboard --resume`. + ### Sandbox build fails during OpenClaw plugin install diff --git a/nemoclaw/src/shared/snapshot-sanitizer-boundary.cts b/nemoclaw/src/shared/snapshot-sanitizer-boundary.cts index 27d8e4c6044..9b04787ad18 100644 --- a/nemoclaw/src/shared/snapshot-sanitizer-boundary.cts +++ b/nemoclaw/src/shared/snapshot-sanitizer-boundary.cts @@ -67,7 +67,7 @@ export function setSnapshotSanitizerPythonPathForTest( snapshotSanitizerPythonPathForTest = pythonPath; } -function snapshotSanitizerPythonPath(): string | null { +export function resolveSnapshotSanitizerPythonPath(): string | null { if (process.env.VITEST === "true" && snapshotSanitizerPythonPathForTest !== undefined) { return snapshotSanitizerPythonPathForTest; } @@ -719,7 +719,7 @@ export function scanDescriptorSnapshot( targetName?: string, ): DescriptorSnapshotScan | null { const mode = targetName === undefined ? "scan-tree" : "scan-file"; - const pythonPath = snapshotSanitizerPythonPath(); + const pythonPath = resolveSnapshotSanitizerPythonPath(); if (pythonPath === null) return null; const result = spawnSync( pythonPath, @@ -751,7 +751,7 @@ export function applyDescriptorSnapshotActions( actions: readonly SnapshotSanitizationAction[], ): boolean { if (actions.length === 0) return true; - const pythonPath = snapshotSanitizerPythonPath(); + const pythonPath = resolveSnapshotSanitizerPythonPath(); if (pythonPath === null) return false; const result = spawnSync( pythonPath, @@ -774,7 +774,7 @@ export function installDescriptorSnapshotFile( content: string, ): boolean { if (!isSafeRelativePath(targetName) || targetName.includes("/")) return false; - const pythonPath = snapshotSanitizerPythonPath(); + const pythonPath = resolveSnapshotSanitizerPythonPath(); if (pythonPath === null) return false; const result = spawnSync( pythonPath, diff --git a/scripts/checks/export-managed-image-failure-diagnostics.ts b/scripts/checks/export-managed-image-failure-diagnostics.ts index d57b2faca95..ee2242be6df 100644 --- a/scripts/checks/export-managed-image-failure-diagnostics.ts +++ b/scripts/checks/export-managed-image-failure-diagnostics.ts @@ -12,6 +12,7 @@ const EXPORTED_DIAGNOSTIC_FILES = new Set([ "openshell-gateway-relevant.log", "openshell-gateway-tail.log", "rootfs-console.log", + "sandbox-create-output.log", "summary.txt", ]); diff --git a/src/lib/onboard/created-sandbox-failure.test.ts b/src/lib/onboard/created-sandbox-failure.test.ts index 69fe3edc5d2..185ae6b5234 100644 --- a/src/lib/onboard/created-sandbox-failure.test.ts +++ b/src/lib/onboard/created-sandbox-failure.test.ts @@ -72,6 +72,7 @@ describe("reportSandboxCreateFailure", () => { ).toThrow(ExitSignal); expect(deps.printCreateFailureDiagnostics).toHaveBeenCalledWith("alpha", { backupPath: "/tmp/backup", + createOutput: "boom", }); expect(deps.printRecoveryHints).toHaveBeenCalledWith("boom", { createArgs: ["sandbox", "create", "alpha"], @@ -139,6 +140,15 @@ describe("reportSandboxCreateFailure", () => { expect(hinted).not.toContain("ghp_abcdefghijklmnopqrstuvwxyz1234567890"); expect(hinted).not.toContain("sk-abcdefghijklmnopqrstuvwxyz1234567890"); expect(hinted).not.toContain("AKIAABCDEFGHIJKLMNOP"); // gitleaks:allow + const diagnosticOutput = String( + (deps.printCreateFailureDiagnostics as ReturnType).mock.calls[0]?.[1] + ?.createOutput, + ); + expect(diagnosticOutput).toContain(""); + expect(diagnosticOutput).not.toContain("secret-token"); + expect(diagnosticOutput).not.toContain("ghp_abcdefghijklmnopqrstuvwxyz1234567890"); + expect(diagnosticOutput).not.toContain("sk-abcdefghijklmnopqrstuvwxyz1234567890"); + expect(diagnosticOutput).not.toContain("AKIAABCDEFGHIJKLMNOP"); // gitleaks:allow }); it("falls back to exit code 1 when the create status is zero", () => { diff --git a/src/lib/onboard/created-sandbox-failure.ts b/src/lib/onboard/created-sandbox-failure.ts index 820c4fddbdd..56656d1d351 100644 --- a/src/lib/onboard/created-sandbox-failure.ts +++ b/src/lib/onboard/created-sandbox-failure.ts @@ -1,9 +1,14 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { redact } from "../security/redact"; +import { redact, redactFull } from "../security/redact"; import type { CreatedSandboxReadinessResult } from "./sandbox-readiness-tracing"; +/** Remove credential material before sandbox create failures reach a diagnostic sink. */ +export function redactSandboxCreateFailureOutput(output: string): string { + return redact(redactFull(output)); +} + export type SandboxCreateFailureReportOptions = { sandboxName: string; /** Non-zero exit status from the create stream. */ @@ -18,7 +23,10 @@ export type SandboxCreateFailureReportOptions = { export type SandboxCreateFailureReportDeps = { classifyCreateFailure(output: string): { kind: string }; - printCreateFailureDiagnostics(sandboxName: string, options: { backupPath: string | null }): void; + printCreateFailureDiagnostics( + sandboxName: string, + options: { backupPath: string | null; createOutput: string }, + ): void; printRecoveryHints(output: string, options: { createArgs: readonly string[] }): void; warn(message: string): void; error(message: string): void; @@ -56,6 +64,7 @@ export function reportSandboxCreateFailure( } deps.printCreateFailureDiagnostics(options.sandboxName, { backupPath: options.restoreBackupPath, + createOutput: redactSandboxCreateFailureOutput(options.createOutput), }); deps.error(" Try: openshell sandbox list # check gateway state"); deps.printRecoveryHints(redactedCreateOutput, { createArgs: options.createArgs }); diff --git a/src/lib/onboard/sandbox-create-failure.ts b/src/lib/onboard/sandbox-create-failure.ts index 87ed446fe49..ad111035525 100644 --- a/src/lib/onboard/sandbox-create-failure.ts +++ b/src/lib/onboard/sandbox-create-failure.ts @@ -8,11 +8,16 @@ import path from "node:path"; import { GATEWAY_PORT } from "../core/ports"; import { rejectSymlinksOnPath } from "../state/config-io"; import { nemoclawStateRoot } from "../state/state-root"; +import { redactSandboxCreateFailureOutput } from "./created-sandbox-failure"; const ANSI_RE = /\x1B(?:\[[0-?]*[ -/]*[@-~]|\][^\x07]*(?:\x07|\x1B\\)|[@-_])/g; const UUID_RE = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/i; const MAX_RELEVANT_LOG_LINES = 120; const MAX_GATEWAY_TAIL_LINES = 240; +const MAX_CREATE_OUTPUT_LINES = 240; +const MAX_CREATE_OUTPUT_CHARS = 32_000; +const MAX_FAILURE_EXCERPT_LINES = 8; +const TRUNCATED_OUTPUT_MARKER = "[diagnostic truncated; showing final output]"; export type SandboxCreateFailureDiagnostics = { dir: string; @@ -22,6 +27,7 @@ export type SandboxCreateFailureDiagnostics = { consoleOutput: string | null; copiedConsoleOutput: string | null; gatewayTailPath: string | null; + createOutputPath: string | null; backupPath: string | null; summaryLines: string[]; }; @@ -29,6 +35,8 @@ export type SandboxCreateFailureDiagnostics = { export type SandboxCreateFailureDiagnosticOptions = { homeDir?: string; gatewayLogPath?: string | null; + homebrewPrefix?: string | null; + createOutput?: string | null; backupPath?: string | null; now?: Date; }; @@ -45,8 +53,8 @@ function timestampForPath(now: Date): string { return now.toISOString().replace(/[:.]/g, "-"); } -function gatewayLogCandidates(homeDir: string): string[] { - return [ +function gatewayLogCandidates(homeDir: string, homebrewPrefix?: string | null): string[] { + const candidates = [ path.join( homeDir, ".local", @@ -57,17 +65,59 @@ function gatewayLogCandidates(homeDir: string): string[] { ), path.join(homeDir, ".local", "state", "openshell", "openshell-gateway.log"), ]; + const homebrewPrefixes = [ + homebrewPrefix, + process.env.HOMEBREW_PREFIX, + "/opt/homebrew", + "/usr/local", + ].filter((prefix): prefix is string => Boolean(prefix) && path.isAbsolute(prefix as string)); + for (const prefix of new Set(homebrewPrefixes)) { + candidates.push( + path.join(prefix, "var", "log", "openshell", "openshell-gateway.err.log"), + path.join(prefix, "var", "log", "openshell", "openshell-gateway.out.log"), + ); + } + return candidates; +} + +function latestGatewayLogPath(candidates: string[]): string | null { + let latest: { path: string; modified: number } | null = null; + for (const candidate of candidates) { + try { + const stat = fs.statSync(candidate); + if (!stat.isFile()) continue; + if (latest === null || stat.mtimeMs > latest.modified) { + latest = { path: candidate, modified: stat.mtimeMs }; + } + } catch { + // Continue to the next known log location. + } + } + return latest?.path ?? null; } function readLogLines(filePath: string): string[] | null { try { if (!fs.existsSync(filePath)) return null; - return stripAnsi(fs.readFileSync(filePath, "utf-8")).split(/\r?\n/); + return redactSandboxCreateFailureOutput(stripAnsi(fs.readFileSync(filePath, "utf-8"))).split( + /\r?\n/, + ); } catch { return null; } } +function createOutputTail(value: string | null | undefined): string[] { + const redacted = redactSandboxCreateFailureOutput(stripAnsi(value ?? "")).trim(); + if (!redacted) return []; + const wasTruncated = redacted.length > MAX_CREATE_OUTPUT_CHARS; + const lines = (wasTruncated ? redacted.slice(-MAX_CREATE_OUTPUT_CHARS) : redacted) + .split(/\r?\n/) + .filter((line) => line.trim()); + if (!wasTruncated) return lines.slice(-MAX_CREATE_OUTPUT_LINES); + return [TRUNCATED_OUTPUT_MARKER, ...lines.slice(-(MAX_CREATE_OUTPUT_LINES - 1))]; +} + function extractField(line: string, field: string): string | null { const match = line.match(new RegExp(`${field}=([^\\s]+)`)); return match?.[1] ?? null; @@ -113,7 +163,9 @@ function filterRelevantLines( if (!line.trim()) return false; if (line.includes(`sandbox_name=${sandboxName}`)) return true; if (sandboxId && line.includes(`sandbox_id=${sandboxId}`)) return true; - return /ERROR krun|VmCreate|ProcessExited|console_output=|state_dir=/.test(line); + return /\bERROR\b|failed to (?:build|solve)|VmCreate|ProcessExited|console_output=|state_dir=/i.test( + line, + ); }); return relevant.slice(-MAX_RELEVANT_LOG_LINES); } @@ -172,8 +224,7 @@ export function collectSandboxCreateFailureDiagnostics( const gatewayLogPath = options.gatewayLogPath ?? - gatewayLogCandidates(homeDir).find((candidate) => fs.existsSync(candidate)) ?? - null; + latestGatewayLogPath(gatewayLogCandidates(homeDir, options.homebrewPrefix)); const rawLines = gatewayLogPath ? readLogLines(gatewayLogPath) : null; const block = rawLines ? findLatestSandboxBlock(rawLines, sandboxName) : []; const sandboxId = getLatestSandboxId(block, sandboxName); @@ -192,6 +243,13 @@ export function collectSandboxCreateFailureDiagnostics( ); const stateEntries = listStateDir(stateDir); const backupPath = options.backupPath ?? null; + const createOutputLines = createOutputTail(options.createOutput); + const createOutputPath = + createOutputLines.length > 0 ? path.join(dir, "sandbox-create-output.log") : null; + + if (createOutputPath) { + fs.writeFileSync(createOutputPath, `${createOutputLines.join("\n")}\n`, { mode: 0o600 }); + } if (relevantLines.length > 0) { fs.writeFileSync( @@ -213,11 +271,23 @@ export function collectSandboxCreateFailureDiagnostics( `sandbox_id=${sandboxId ?? "unknown"}`, `gateway_log=${gatewayLogPath ?? "not-found"}`, `gateway_tail=${gatewayTailPath ?? "not-written"}`, + `create_output=${createOutputPath ?? "not-written"}`, `state_dir=${stateDir ?? "unknown"}`, `console_output=${consoleOutput ?? "unknown"}`, `copied_console_output=${copiedConsoleOutput ?? "not-copied"}`, `backup_path=${backupPath ?? "none"}`, ]; + const failureExcerpt = ( + createOutputLines.length > 0 + ? createOutputLines + : relevantLines.length > 0 + ? relevantLines + : gatewayTailLines + ).slice(-MAX_FAILURE_EXCERPT_LINES); + if (failureExcerpt.length > 0) { + summaryLines.push("failure_excerpt:"); + summaryLines.push(...failureExcerpt.map((line) => ` ${line}`)); + } if (stateEntries.length > 0) { summaryLines.push("state_dir_entries:"); summaryLines.push(...stateEntries.map((entry) => ` ${entry}`)); @@ -234,8 +304,9 @@ export function collectSandboxCreateFailureDiagnostics( consoleOutput, copiedConsoleOutput, gatewayTailPath, + createOutputPath, backupPath, - summaryLines: relevantLines.length > 0 ? relevantLines.slice(-8) : gatewayTailLines.slice(-8), + summaryLines: failureExcerpt, }; } @@ -248,7 +319,7 @@ export function printSandboxCreateFailureDiagnostics( console.error(` Diagnostics saved: ${diagnostics.dir}`); if (diagnostics.summaryLines.length > 0) { - console.error(" Recent OpenShell gateway failure:"); + console.error(" Recent sandbox creation failure:"); for (const line of diagnostics.summaryLines) { console.error(` ${line}`); } diff --git a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts index 49c3271ddd6..bcba1d2fed5 100644 --- a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts +++ b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts @@ -378,6 +378,7 @@ export function createSandboxGpuCreateAttemptRunner( await runtimePatch.rollbackManagedStartupAfterCreateFailure(); printSandboxCreateFailureDiagnostics(input.sandboxName, { backupPath: input.restoreBackupPath, + createOutput: createResult.output, }); if (compatibility) runtimePatch.printReadinessFailureIfEnabled(); else { diff --git a/src/lib/sandbox-base-image.test.ts b/src/lib/sandbox-base-image.test.ts index 07a795870ef..44f77d0b762 100644 --- a/src/lib/sandbox-base-image.test.ts +++ b/src/lib/sandbox-base-image.test.ts @@ -88,11 +88,15 @@ describe("sandbox base-image build diagnostics", () => { expect(output).toContain("****"); }); - it("bounds captured build diagnostics before returning them", () => { - const output = formatBuildFailureDiagnostics({ stderr: "x".repeat(10_000) }); + it("bounds captured build diagnostics while preserving the final failure", () => { + const output = formatBuildFailureDiagnostics({ + stderr: `initial Dockerfile output\n${"x".repeat(10_000)}\nERROR: final build step failed`, + }); expect(output.length).toBeLessThan(8_100); - expect(output.endsWith("[diagnostic truncated]")).toBe(true); + expect(output.startsWith("[diagnostic truncated; showing final output]")).toBe(true); + expect(output).toContain("ERROR: final build step failed"); + expect(output).not.toContain("initial Dockerfile output"); }); it("surfaces a redacted spawn failure cause", () => { diff --git a/src/lib/sandbox-base-image.ts b/src/lib/sandbox-base-image.ts index d628490687c..ed48c0a4649 100644 --- a/src/lib/sandbox-base-image.ts +++ b/src/lib/sandbox-base-image.ts @@ -48,7 +48,7 @@ export * from "./sandbox-base-image/source-identity"; export * from "./sandbox-base-image/types"; const BUILD_FAILURE_DIAGNOSTIC_LIMIT = 8_000; -const BUILD_FAILURE_TRUNCATED_SUFFIX = "\n[diagnostic truncated]"; +const BUILD_FAILURE_TRUNCATED_PREFIX = "[diagnostic truncated; showing final output]\n"; /** * Combine stderr + stdout from a captured `dockerBuild` failure and pass them @@ -82,7 +82,7 @@ export function formatBuildFailureDiagnostics(buildResult: { diagnostics = diagnostics.replaceAll(prefix, replacement); } return diagnostics.length > BUILD_FAILURE_DIAGNOSTIC_LIMIT - ? `${diagnostics.slice(0, BUILD_FAILURE_DIAGNOSTIC_LIMIT)}${BUILD_FAILURE_TRUNCATED_SUFFIX}` + ? `${BUILD_FAILURE_TRUNCATED_PREFIX}${diagnostics.slice(-BUILD_FAILURE_DIAGNOSTIC_LIMIT)}` : diagnostics; } diff --git a/src/lib/security/snapshot-sanitizer.ts b/src/lib/security/snapshot-sanitizer.ts index 5424ef93f34..bdea0abab5e 100644 --- a/src/lib/security/snapshot-sanitizer.ts +++ b/src/lib/security/snapshot-sanitizer.ts @@ -7,6 +7,7 @@ import { applyDescriptorSnapshotActions, decodeDescriptorSnapshotContent, inspectDescriptorSnapshotRoot, + resolveSnapshotSanitizerPythonPath, type SnapshotSanitizationAction, type SnapshotScannedFile, scanDescriptorSnapshot, @@ -21,6 +22,13 @@ import { const MAX_SANITIZATION_PASSES = 3; +export class SnapshotSanitizerPrerequisiteError extends Error { + constructor() { + super("Python 3 is required for snapshot sanitization. Install Python 3 and retry."); + this.name = "SnapshotSanitizerPrerequisiteError"; + } +} + function actionForScannedFile(file: SnapshotScannedFile): SnapshotSanitizationAction | null { const name = path.posix.basename(file.path).toLowerCase(); if (isSensitiveFile(name)) { @@ -59,6 +67,9 @@ function actionForScannedFile(file: SnapshotScannedFile): SnapshotSanitizationAc * instead of redirecting the sanitizer outside the snapshot root. */ export function sanitizeSnapshotDirectory(rootPath: string): void { + if (resolveSnapshotSanitizerPythonPath() === null) { + throw new SnapshotSanitizerPrerequisiteError(); + } for (let pass = 0; pass < MAX_SANITIZATION_PASSES; pass += 1) { const root = inspectDescriptorSnapshotRoot(rootPath); if (root === null) { diff --git a/src/lib/state/sandbox-backup-sanitization.test.ts b/src/lib/state/sandbox-backup-sanitization.test.ts index 10cc63dbd7b..203ae5cadd2 100644 --- a/src/lib/state/sandbox-backup-sanitization.test.ts +++ b/src/lib/state/sandbox-backup-sanitization.test.ts @@ -98,6 +98,16 @@ describe("rebuild backup credential sanitization", () => { expect(existsSync(backupPath)).toBe(false); }); + it("names the Python prerequisite and removes the incomplete snapshot backup (#8202)", () => { + const backupPath = createBackup(); + setSnapshotSanitizerPythonPathForTest(null); + + expect(() => sanitizeBackupDirectory(backupPath)).toThrow( + "Python 3 is required for snapshot sanitization. Install Python 3 and retry. NemoClaw removed the incomplete snapshot backup.", + ); + expect(existsSync(backupPath)).toBe(false); + }); + it("reports when cleanup leaves an incomplete backup behind", () => { const backupPath = createBackup(); const yamlPath = join(backupPath, "state", "config.yaml"); diff --git a/src/lib/state/sandbox.ts b/src/lib/state/sandbox.ts index 05fed8025a5..6ff97e750f7 100644 --- a/src/lib/state/sandbox.ts +++ b/src/lib/state/sandbox.ts @@ -47,7 +47,10 @@ import { } from "../domain/backup-failure.js"; import { shellQuote } from "../runner.js"; import { createTempSshConfig } from "../sandbox/temp-ssh-config.js"; -import { sanitizeSnapshotDirectory } from "../security/snapshot-sanitizer.js"; +import { + SnapshotSanitizerPrerequisiteError, + sanitizeSnapshotDirectory, +} from "../security/snapshot-sanitizer.js"; import { buildRestoreCleanupCommand, buildRestoreTarArgs, @@ -713,21 +716,34 @@ export function sanitizeBackupDirectory( try { operations.sanitizeDirectory(dirPath); } catch (error) { + const prerequisiteMessage = + error instanceof SnapshotSanitizerPrerequisiteError ? `${error.message} ` : ""; try { operations.removeBackup(dirPath); } catch (cleanupError) { - throw new Error("Credential sanitization failed and backup cleanup failed", { - cause: cleanupError, - }); + throw new Error( + prerequisiteMessage + ? `${prerequisiteMessage}NemoClaw could not remove the incomplete snapshot backup.` + : "Credential sanitization failed and backup cleanup failed", + { + cause: cleanupError, + }, + ); } if (operations.backupExists(dirPath)) { - throw new Error("Credential sanitization failed and the incomplete backup remains", { - cause: error, - }); + throw new Error( + prerequisiteMessage + ? `${prerequisiteMessage}The incomplete snapshot backup remains.` + : "Credential sanitization failed and the incomplete backup remains", + { cause: error }, + ); } - throw new Error("Credential sanitization failed; removed the incomplete backup", { - cause: error, - }); + throw new Error( + prerequisiteMessage + ? `${prerequisiteMessage}NemoClaw removed the incomplete snapshot backup.` + : "Credential sanitization failed; removed the incomplete backup", + { cause: error }, + ); } } diff --git a/src/nemoclaw.ts b/src/nemoclaw.ts index 685a979f1bc..34a76cb1952 100644 --- a/src/nemoclaw.ts +++ b/src/nemoclaw.ts @@ -6,11 +6,26 @@ // Keep this file intentionally small: the public grammar adapter lives in // src/lib/cli/public-dispatch.ts, while oclif owns command discovery, parsing, // help rendering, and command execution under src/commands/**. +import { log } from "./lib/cli/logger"; import { dispatchCli } from "./lib/cli/public-dispatch"; +function reportTopLevelCliError(error: unknown): void { + let message = "Command failed without an error message."; + try { + const candidate = error instanceof Error ? error.message : String(error); + if (candidate.trim()) message = candidate.trim(); + } catch { + // Keep the fallback message when an unusual thrown value cannot be stringified. + } + log.error(`Error: ${message}`); + process.exitCode = 1; +} + exports.main = dispatchCli; module.exports.dispatchCli = dispatchCli; // Compatibility for tests that require the CLI module and await completion. // Prefer calling dispatchCli(argv) directly in new in-process harnesses. exports.mainPromise = - process.env.NEMOCLAW_DISABLE_AUTO_DISPATCH === "1" ? Promise.resolve() : dispatchCli(); + process.env.NEMOCLAW_DISABLE_AUTO_DISPATCH === "1" + ? Promise.resolve() + : dispatchCli().catch(reportTopLevelCliError); diff --git a/test/managed-image-failure-diagnostics.test.ts b/test/managed-image-failure-diagnostics.test.ts index b5797dd6014..7d19cdf68e1 100644 --- a/test/managed-image-failure-diagnostics.test.ts +++ b/test/managed-image-failure-diagnostics.test.ts @@ -72,6 +72,10 @@ describe("managed-image failure diagnostic export", () => { path.join(diagnosticBundle, "rootfs-console.log"), "managed startup exited before the supervisor reconnected\n", ); + fs.writeFileSync( + path.join(diagnosticBundle, "sandbox-create-output.log"), + `sandbox build failed for ${opaqueCanary}\n`, + ); fs.writeFileSync( path.join(diagnosticBundle, "unrelated.raw"), "this raw file must never enter the artifact\n", @@ -86,7 +90,7 @@ describe("managed-image failure diagnostic export", () => { sourceRoot, }); - expect(result).toMatchObject({ bundles: 1, files: 3 }); + expect(result).toMatchObject({ bundles: 1, files: 4 }); const exported = outputText(outputRoot); expect(exported).toContain(""); expect(exported).toContain("managed startup exited before the supervisor reconnected"); @@ -153,6 +157,7 @@ describe("managed-image failure diagnostic export", () => { for (const name of [ "openshell-gateway-relevant.log", "openshell-gateway-tail.log", + "sandbox-create-output.log", "summary.txt", ]) { fs.writeFileSync(path.join(diagnosticBundle, name), largeButReadable); diff --git a/test/onboard-sandbox-create-failure.test.ts b/test/onboard-sandbox-create-failure.test.ts index 364d978d606..9472c977f96 100644 --- a/test/onboard-sandbox-create-failure.test.ts +++ b/test/onboard-sandbox-create-failure.test.ts @@ -116,4 +116,52 @@ describe("sandbox create failure diagnostics", () => { "gateway_tail=", ); }); + + it("preserves the redacted final create failure and discovers Homebrew gateway logs (#8202)", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-create-failure-macos-")); + const homeDir = path.join(tmp, "home"); + const homebrewPrefix = path.join(tmp, "homebrew"); + const logDir = path.join(homebrewPrefix, "var", "log", "openshell"); + const gatewayLogPath = path.join(logDir, "openshell-gateway.err.log"); + fs.mkdirSync(logDir, { recursive: true }); + fs.writeFileSync( + gatewayLogPath, + [ + "create_sandbox received sandbox_name=my-assistant", + "ERROR builder failed to solve: gateway-side build failed", + ].join("\n"), + ); + const secret = ["create", "output", "secret", "canary"].join("-"); + const createOutput = [ + "initial Dockerfile output", + "x".repeat(34_000), + "ERROR: process /bin/sh -c install-agent failed", + `Authorization: Bearer ${secret}`, + ].join("\n"); + + const diagnostics = collectSandboxCreateFailureDiagnostics("my-assistant", { + homeDir, + homebrewPrefix, + createOutput, + now: new Date("2026-05-12T20:35:00.000Z"), + }); + + expect(diagnostics?.gatewayLogPath).toBe(gatewayLogPath); + expect(diagnostics?.createOutputPath).toBe( + path.join(diagnostics!.dir, "sandbox-create-output.log"), + ); + const savedCreateOutput = fs.readFileSync(diagnostics!.createOutputPath!, "utf-8"); + expect(savedCreateOutput).toContain("[diagnostic truncated; showing final output]"); + expect(savedCreateOutput).toContain("ERROR: process /bin/sh -c install-agent failed"); + expect(savedCreateOutput).not.toContain("initial Dockerfile output"); + expect(savedCreateOutput).not.toContain(secret); + const relevant = fs.readFileSync( + path.join(diagnostics!.dir, "openshell-gateway-relevant.log"), + "utf-8", + ); + expect(relevant).toContain("gateway-side build failed"); + const summary = fs.readFileSync(path.join(diagnostics!.dir, "summary.txt"), "utf-8"); + expect(summary).toContain(`create_output=${diagnostics!.createOutputPath}`); + expect(summary).toContain("ERROR: process /bin/sh -c install-agent failed"); + }); }); diff --git a/test/package-contract/cli/top-level-error.test.ts b/test/package-contract/cli/top-level-error.test.ts new file mode 100644 index 00000000000..0d2a48bfde6 --- /dev/null +++ b/test/package-contract/cli/top-level-error.test.ts @@ -0,0 +1,45 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; + +const REPO_ROOT = path.join(import.meta.dirname, "../../.."); +const CLI_PATH = JSON.stringify(path.join(REPO_ROOT, "dist", "nemoclaw.js")); +const PUBLIC_DISPATCH_PATH = JSON.stringify( + path.join(REPO_ROOT, "dist", "lib", "cli", "public-dispatch.js"), +); + +describe("compiled CLI top-level errors", () => { + it("prints a concise error without an uncaught Node.js stack (#8202)", () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-top-level-error-")); + const scriptPath = path.join(tmpDir, "top-level-error.js"); + const script = String.raw` +const dispatchPath = ${PUBLIC_DISPATCH_PATH}; +require.cache[dispatchPath] = { + id: dispatchPath, + filename: dispatchPath, + loaded: true, + exports: { + dispatchCli: () => Promise.reject(new Error("Sandbox base image override was rejected.")), + }, +}; +require(${CLI_PATH}).mainPromise; +`; + fs.writeFileSync(scriptPath, script); + + const result = spawnSync(process.execPath, [scriptPath], { + cwd: REPO_ROOT, + encoding: "utf-8", + env: { ...process.env, HOME: tmpDir }, + }); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(""); + expect(result.stderr).toBe("Error: Sandbox base image override was rejected.\n"); + expect(result.stderr).not.toMatch(/\n\s+at |Node\.js v/); + }); +}); From 10918c2b0e4bcc15d71936591989e3f67309b9d9 Mon Sep 17 00:00:00 2001 From: San Dang Date: Tue, 4 Aug 2026 16:55:49 +0530 Subject: [PATCH 2/2] fix(cli): harden failure reporting Signed-off-by: San Dang --- bin/nemoclaw.js | 17 ++++++++++++++++- docs/manage-sandboxes/backup-restore.mdx | 3 ++- src/lib/onboard/created-sandbox-failure.test.ts | 6 +++++- src/lib/onboard/created-sandbox-failure.ts | 5 +++-- src/nemoclaw.ts | 17 +---------------- .../cli/top-level-error.test.ts | 10 +++++++--- 6 files changed, 34 insertions(+), 24 deletions(-) diff --git a/bin/nemoclaw.js b/bin/nemoclaw.js index c3cc215fee5..965728a282b 100755 --- a/bin/nemoclaw.js +++ b/bin/nemoclaw.js @@ -9,4 +9,19 @@ if (invokedAs === "nemo-deepagents") { process.env.NEMOCLAW_INVOKED_AS = "nemo-deepagents"; } -require("../dist/nemoclaw"); +const { log } = require("../dist/lib/cli/logger"); + +function reportTopLevelCliError(error) { + let message = "Command failed without an error message."; + try { + const candidate = error instanceof Error ? error.message : String(error); + if (candidate.trim()) message = candidate.trim(); + } catch { + // Keep the fallback message when an unusual thrown value cannot be stringified. + } + log.error(`Error: ${message}`); + process.exitCode = 1; +} + +const { mainPromise } = require("../dist/nemoclaw"); +Promise.resolve(mainPromise).catch(reportTopLevelCliError); diff --git a/docs/manage-sandboxes/backup-restore.mdx b/docs/manage-sandboxes/backup-restore.mdx index e963aaf5920..75046065e30 100644 --- a/docs/manage-sandboxes/backup-restore.mdx +++ b/docs/manage-sandboxes/backup-restore.mdx @@ -48,7 +48,8 @@ NemoClaw checks fixed installation paths and the directory that contains the Nod It does not resolve the helper through `PATH`, which can contain user-controlled entries. If no trusted interpreter is available, snapshot creation removes the incomplete backup when possible and tells you to install Python 3 before rerunning the command. Install Python 3 through your operating-system package manager or Homebrew, then rerun `$$nemoclaw snapshot create`. -NemoClaw runs the helper in isolated mode to keep reads, replacements, and removals anchored to opened directory descriptors. +NemoClaw runs the helper in isolated mode. +The helper keeps reads, replacements, and removals anchored to opened directory descriptors. If a copied file or parent directory changes identity during the operation, snapshot creation fails closed instead of following the changed path. diff --git a/src/lib/onboard/created-sandbox-failure.test.ts b/src/lib/onboard/created-sandbox-failure.test.ts index 185ae6b5234..47ea2ea74c4 100644 --- a/src/lib/onboard/created-sandbox-failure.test.ts +++ b/src/lib/onboard/created-sandbox-failure.test.ts @@ -93,7 +93,7 @@ describe("reportSandboxCreateFailure", () => { expect(withOutput.classifyCreateFailure).toHaveBeenCalledWith( "failed with Authorization: Bearer secr********", ); - expect(withOutput.error).toHaveBeenCalledWith("failed with Authorization: Bearer secr********"); + expect(withOutput.error).toHaveBeenCalledWith("failed with Authorization: Bearer "); expect(withOutput.error).not.toHaveBeenCalledWith( "failed with Authorization: Bearer secret-token", ); @@ -120,6 +120,7 @@ describe("reportSandboxCreateFailure", () => { "github ghp_abcdefghijklmnopqrstuvwxyz1234567890", "openai sk-abcdefghijklmnopqrstuvwxyz1234567890", "aws AKIAABCDEFGHIJKLMNOP", // gitleaks:allow + "telegram path /bot-full-redaction-only/", ].join("\n"); expect(() => reportSandboxCreateFailure(createFailureOptions({ createOutput }), deps)).toThrow( @@ -133,6 +134,8 @@ describe("reportSandboxCreateFailure", () => { expect(echoed).not.toContain("ghp_abcdefghijklmnopqrstuvwxyz1234567890"); expect(echoed).not.toContain("sk-abcdefghijklmnopqrstuvwxyz1234567890"); expect(echoed).not.toContain("AKIAABCDEFGHIJKLMNOP"); // gitleaks:allow + expect(echoed).not.toContain("bot-full-redaction-only"); + expect(echoed).toContain("/bot/"); const hinted = (deps.printRecoveryHints as ReturnType).mock.calls .map((call) => String(call[0])) .join("\n"); @@ -149,6 +152,7 @@ describe("reportSandboxCreateFailure", () => { expect(diagnosticOutput).not.toContain("ghp_abcdefghijklmnopqrstuvwxyz1234567890"); expect(diagnosticOutput).not.toContain("sk-abcdefghijklmnopqrstuvwxyz1234567890"); expect(diagnosticOutput).not.toContain("AKIAABCDEFGHIJKLMNOP"); // gitleaks:allow + expect(diagnosticOutput).not.toContain("bot-full-redaction-only"); }); it("falls back to exit code 1 when the create status is zero", () => { diff --git a/src/lib/onboard/created-sandbox-failure.ts b/src/lib/onboard/created-sandbox-failure.ts index 56656d1d351..358b23be225 100644 --- a/src/lib/onboard/created-sandbox-failure.ts +++ b/src/lib/onboard/created-sandbox-failure.ts @@ -44,6 +44,7 @@ export function reportSandboxCreateFailure( deps: SandboxCreateFailureReportDeps, ): void { const redactedCreateOutput = redact(options.createOutput); + const fullyRedactedCreateOutput = redactSandboxCreateFailureOutput(options.createOutput); const failure = deps.classifyCreateFailure(redactedCreateOutput); if (failure.kind === "sandbox_create_incomplete") { // The sandbox was created in the gateway but the create stream exited @@ -60,11 +61,11 @@ export function reportSandboxCreateFailure( deps.error(` Sandbox creation failed (exit ${options.createStatus}).`); if (options.createOutput) { deps.error(""); - deps.error(redactedCreateOutput); + deps.error(fullyRedactedCreateOutput); } deps.printCreateFailureDiagnostics(options.sandboxName, { backupPath: options.restoreBackupPath, - createOutput: redactSandboxCreateFailureOutput(options.createOutput), + createOutput: fullyRedactedCreateOutput, }); deps.error(" Try: openshell sandbox list # check gateway state"); deps.printRecoveryHints(redactedCreateOutput, { createArgs: options.createArgs }); diff --git a/src/nemoclaw.ts b/src/nemoclaw.ts index 34a76cb1952..685a979f1bc 100644 --- a/src/nemoclaw.ts +++ b/src/nemoclaw.ts @@ -6,26 +6,11 @@ // Keep this file intentionally small: the public grammar adapter lives in // src/lib/cli/public-dispatch.ts, while oclif owns command discovery, parsing, // help rendering, and command execution under src/commands/**. -import { log } from "./lib/cli/logger"; import { dispatchCli } from "./lib/cli/public-dispatch"; -function reportTopLevelCliError(error: unknown): void { - let message = "Command failed without an error message."; - try { - const candidate = error instanceof Error ? error.message : String(error); - if (candidate.trim()) message = candidate.trim(); - } catch { - // Keep the fallback message when an unusual thrown value cannot be stringified. - } - log.error(`Error: ${message}`); - process.exitCode = 1; -} - exports.main = dispatchCli; module.exports.dispatchCli = dispatchCli; // Compatibility for tests that require the CLI module and await completion. // Prefer calling dispatchCli(argv) directly in new in-process harnesses. exports.mainPromise = - process.env.NEMOCLAW_DISABLE_AUTO_DISPATCH === "1" - ? Promise.resolve() - : dispatchCli().catch(reportTopLevelCliError); + process.env.NEMOCLAW_DISABLE_AUTO_DISPATCH === "1" ? Promise.resolve() : dispatchCli(); diff --git a/test/package-contract/cli/top-level-error.test.ts b/test/package-contract/cli/top-level-error.test.ts index 0d2a48bfde6..c02ae669cf9 100644 --- a/test/package-contract/cli/top-level-error.test.ts +++ b/test/package-contract/cli/top-level-error.test.ts @@ -8,7 +8,7 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; const REPO_ROOT = path.join(import.meta.dirname, "../../.."); -const CLI_PATH = JSON.stringify(path.join(REPO_ROOT, "dist", "nemoclaw.js")); +const CLI_PATH = JSON.stringify(path.join(REPO_ROOT, "bin", "nemoclaw.js")); const PUBLIC_DISPATCH_PATH = JSON.stringify( path.join(REPO_ROOT, "dist", "lib", "cli", "public-dispatch.js"), ); @@ -27,14 +27,18 @@ require.cache[dispatchPath] = { dispatchCli: () => Promise.reject(new Error("Sandbox base image override was rejected.")), }, }; -require(${CLI_PATH}).mainPromise; +require(${CLI_PATH}); `; fs.writeFileSync(scriptPath, script); const result = spawnSync(process.execPath, [scriptPath], { cwd: REPO_ROOT, encoding: "utf-8", - env: { ...process.env, HOME: tmpDir }, + env: { + ...process.env, + HOME: tmpDir, + NEMOCLAW_DISABLE_AUTO_DISPATCH: "0", + }, }); expect(result.status).toBe(1);