From 6462e22bdba3a057f34952ff8429674ee8dd962a Mon Sep 17 00:00:00 2001 From: Tinson Lai Date: Thu, 30 Jul 2026 05:24:03 +0000 Subject: [PATCH 01/14] fix(state): restore Hermes cron scripts before enabling restored jobs Signed-off-by: Tinson Lai --- agents/hermes/manifest.yaml | 4 + docs/manage-sandboxes/backup-restore.mdx | 3 +- docs/reference/commands.mdx | 2 +- src/lib/state/sandbox-staged-restore.test.ts | 201 +++++++++++++++++++ src/lib/state/sandbox.ts | 47 ++++- 5 files changed, 254 insertions(+), 3 deletions(-) create mode 100644 src/lib/state/sandbox-staged-restore.test.ts diff --git a/agents/hermes/manifest.yaml b/agents/hermes/manifest.yaml index f3a49fab8f1..0a525f93b99 100644 --- a/agents/hermes/manifest.yaml +++ b/agents/hermes/manifest.yaml @@ -63,6 +63,10 @@ state_dirs: - sessions - skills - plugins + # Hermes confines the scripts that no_agent cron jobs run to + # ~/.hermes/scripts and rejects any path outside it, so cron job definitions + # are only restorable together with this directory. + - scripts - cron - logs - skins diff --git a/docs/manage-sandboxes/backup-restore.mdx b/docs/manage-sandboxes/backup-restore.mdx index 78a4d3e211b..9f5a9776933 100644 --- a/docs/manage-sandboxes/backup-restore.mdx +++ b/docs/manage-sandboxes/backup-restore.mdx @@ -41,7 +41,8 @@ Treat snapshot directories as private local data. Hermes snapshots include `SOUL.md`, the Web Dashboard profile under `.hermes/dashboard-home/`, the SQLite database behind `.hermes/state.db`, and the default kanban board in `.hermes/kanban.db`. The default-profile snapshot also includes cron execution history in `.hermes/runtime/cron-executions.db` and Discord replay state in `.hermes/gateway/discord_message_recovery.db`. -NemoClaw captures cron job definitions from `.hermes/cron` as directory state. +NemoClaw captures cron job definitions from `.hermes/cron` as directory state, together with the scripts they run from `.hermes/scripts`. +A restore moves each state directory into place as a unit and applies `.hermes/cron` last, so the running gateway does not schedule a restored job before the script it calls is available. NemoClaw uses SQLite's online backup API and restores these databases through SQLite instead of copying live raw database files. Named-profile cron and Discord databases under `.hermes/profiles//` use raw directory capture and can be inconsistent if a write overlaps the snapshot. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 7468c15e23e..84b7d52d186 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -644,7 +644,7 @@ For OpenClaw, the backed-up paths include agents, extensions, workspace, skills, -For Hermes, the backed-up paths come from `agents/hermes/manifest.yaml`, including `/sandbox/.hermes` state such as memories, sessions, skills, plugins, cron, logs, plans, workspace, messaging platform state, `runtime/state.db`, and the default kanban board in `kanban.db`. +For Hermes, the backed-up paths come from `agents/hermes/manifest.yaml`, including `/sandbox/.hermes` state such as memories, sessions, skills, plugins, scripts, cron, logs, plans, workspace, messaging platform state, `runtime/state.db`, and the default kanban board in `kanban.db`. Kanban backup does not include named boards, attachments, worker logs, scratch workspaces under `kanban/`, or external directory or worktree targets. diff --git a/src/lib/state/sandbox-staged-restore.test.ts b/src/lib/state/sandbox-staged-restore.test.ts new file mode 100644 index 00000000000..f50eb9b7902 --- /dev/null +++ b/src/lib/state/sandbox-staged-restore.test.ts @@ -0,0 +1,201 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { restoreEnvBulk } from "../../../test/helpers/env-test-helpers.js"; +import { loadAgent } from "../agent/defs.js"; +import { restoreRecreatedSandboxState } from "./sandbox.js"; + +const HERMES_DIR = "/sandbox/.hermes"; + +type MoveRecord = { target: string; scriptsPresent: boolean }; + +function writeExecutable(filePath: string, source: string): void { + fs.writeFileSync(filePath, source, { mode: 0o755 }); +} + +function runHermesRestore(options: { stateDirs: string[] }): { + moves: MoveRecord[]; + restore: ReturnType; + restoredCronJob: string | null; + restoredScript: string | null; + stagingLeftBehind: boolean; +} { + const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-staged-restore-")); + const previousOpenshellBin = process.env.NEMOCLAW_OPENSHELL_BIN; + const previousPath = process.env.PATH; + try { + const binDir = path.join(fixture, "bin"); + const shimDir = path.join(fixture, "shim"); + const hermesDir = path.join(fixture, "sandbox-root", ".hermes"); + const backupPath = path.join(fixture, "backup"); + const moveLog = path.join(fixture, "move-log.jsonl"); + fs.mkdirSync(binDir, { recursive: true }); + fs.mkdirSync(shimDir, { recursive: true }); + fs.mkdirSync(hermesDir, { recursive: true }); + + for (const stateDir of options.stateDirs) { + fs.mkdirSync(path.join(backupPath, stateDir), { recursive: true }); + } + fs.writeFileSync(path.join(backupPath, "cron", "jobs.json"), '{"jobs":[{"enabled":true}]}\n'); + if (options.stateDirs.includes("scripts")) { + fs.writeFileSync(path.join(backupPath, "scripts", "digest.sh"), "#!/bin/bash\necho ok\n"); + } + + fs.writeFileSync( + path.join(backupPath, "rebuild-manifest.json"), + JSON.stringify({ + version: 1, + sandboxName: "alpha", + timestamp: "2026-07-29T12-00-00-000Z", + agentType: "hermes", + agentVersion: null, + expectedVersion: null, + stateDirs: options.stateDirs, + backedUpDirs: options.stateDirs, + stateFiles: [], + dir: HERMES_DIR, + backupPath, + blueprintDigest: null, + }), + ); + + const openshell = path.join(binDir, "openshell"); + writeExecutable( + openshell, + `#!/usr/bin/env node +const args = process.argv.slice(2); +if (args[0] === "sandbox" && args[1] === "ssh-config") { + process.stdout.write("Host openshell-alpha\\n HostName 127.0.0.1\\n User sandbox\\n"); +} +process.exit(0); +`, + ); + + writeExecutable( + path.join(shimDir, "mv"), + `#!/usr/bin/env node +const fs = require("node:fs"); +const { spawnSync } = require("node:child_process"); +const args = process.argv.slice(2); +const target = args[args.length - 1]; +fs.appendFileSync( + ${JSON.stringify(moveLog)}, + JSON.stringify({ + target, + scriptsPresent: fs.existsSync(${JSON.stringify(path.join(hermesDir, "scripts"))}), + }) + "\\n", +); +const result = spawnSync("/bin/mv", args, { stdio: "inherit" }); +process.exit(result.status === null ? 1 : result.status); +`, + ); + + writeExecutable( + path.join(binDir, "ssh"), + `#!/usr/bin/env node +const fs = require("node:fs"); +const { spawnSync } = require("node:child_process"); +const command = (process.argv[process.argv.length - 1] || "").split(${JSON.stringify(HERMES_DIR)}).join(${JSON.stringify(hermesDir)}); +function readStdin() { + const chunks = []; + for (;;) { + const buffer = Buffer.alloc(65536); + let count = 0; + try { + count = fs.readSync(0, buffer, 0, buffer.length, null); + } catch { + break; + } + if (count === 0) break; + chunks.push(buffer.subarray(0, count)); + } + return Buffer.concat(chunks); +} +const result = spawnSync("sh", ["-c", command], { + input: readStdin(), + env: { ...process.env, PATH: ${JSON.stringify(shimDir)} + ":" + process.env.PATH }, + stdio: ["pipe", "pipe", "pipe"], +}); +process.exit(result.status === null ? 1 : result.status); +`, + ); + + process.env.NEMOCLAW_OPENSHELL_BIN = openshell; + process.env.PATH = `${binDir}${path.delimiter}${previousPath ?? ""}`; + const restore = restoreRecreatedSandboxState("alpha", backupPath, { + targetAgentType: "hermes", + }); + + const moves = fs.existsSync(moveLog) + ? fs + .readFileSync(moveLog, "utf8") + .trim() + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line) as MoveRecord) + : []; + const cronJobPath = path.join(hermesDir, "cron", "jobs.json"); + const scriptPath = path.join(hermesDir, "scripts", "digest.sh"); + return { + moves, + restore, + restoredCronJob: fs.existsSync(cronJobPath) ? fs.readFileSync(cronJobPath, "utf8") : null, + restoredScript: fs.existsSync(scriptPath) ? fs.readFileSync(scriptPath, "utf8") : null, + stagingLeftBehind: fs.existsSync(path.join(hermesDir, ".nemoclaw-restore-staging")), + }; + } finally { + restoreEnvBulk({ NEMOCLAW_OPENSHELL_BIN: previousOpenshellBin, PATH: previousPath }); + fs.rmSync(fixture, { recursive: true, force: true }); + } +} + +describe("Hermes cron state restore", () => { + it("declares the cron script directory as Hermes state", () => { + expect(loadAgent("hermes").stateDirs).toContain("scripts"); + }); + + it("restores cron scripts alongside the job definitions that call them", () => { + const result = runHermesRestore({ stateDirs: ["scripts", "cron", "workspace"] }); + + expect(result.restore.success).toBe(true); + expect(result.restore.restoredDirs).toEqual( + expect.arrayContaining(["scripts", "cron", "workspace"]), + ); + expect(result.restoredScript).toBe("#!/bin/bash\necho ok\n"); + expect(result.restoredCronJob).toBe('{"jobs":[{"enabled":true}]}\n'); + }); + + it("publishes cron job definitions only after their scripts are in place", () => { + const result = runHermesRestore({ stateDirs: ["scripts", "cron", "workspace"] }); + + const cronMove = result.moves.find((move) => move.target.endsWith("/cron")); + expect(cronMove?.scriptsPresent).toBe(true); + expect(result.moves.at(-1)?.target).toMatch(/\/cron$/); + }); + + it("applies cron last for a backup whose manifest lists it first", () => { + const result = runHermesRestore({ stateDirs: ["cron", "scripts", "workspace"] }); + + expect(result.restore.success).toBe(true); + expect(result.moves.at(-1)?.target).toMatch(/\/cron$/); + expect(result.moves.find((move) => move.target.endsWith("/cron"))?.scriptsPresent).toBe(true); + }); + + it("publishes every state directory as a unit and leaves no staging directory", () => { + const result = runHermesRestore({ stateDirs: ["scripts", "cron", "workspace"] }); + + expect(result.restore.success).toBe(true); + expect(result.moves.map((move) => path.basename(move.target)).sort()).toEqual([ + "cron", + "scripts", + "workspace", + ]); + expect(result.stagingLeftBehind).toBe(false); + }); +}); diff --git a/src/lib/state/sandbox.ts b/src/lib/state/sandbox.ts index af6040273a7..720ed9f49e2 100644 --- a/src/lib/state/sandbox.ts +++ b/src/lib/state/sandbox.ts @@ -1348,6 +1348,46 @@ export function backupSandboxState(sandboxName: string, options: BackupOptions = // ── Restore ──────────────────────────────────────────────────────── +const RESTORE_STAGING_DIR = ".nemoclaw-restore-staging"; + +// Job definitions in these directories are picked up by a gateway that keeps +// running throughout a restore, so they must land after every directory whose +// content those jobs can reference. +const SCHEDULED_WORK_STATE_DIRS = new Set(["cron"]); + +function orderStateDirsForRestore(stateDirs: readonly string[]): string[] { + return [ + ...stateDirs.filter((stateDir) => !SCHEDULED_WORK_STATE_DIRS.has(stateDir)), + ...stateDirs.filter((stateDir) => SCHEDULED_WORK_STATE_DIRS.has(stateDir)), + ]; +} + +/** + * Extract a restore archive beside the state directories, then move each + * directory into place. + * + * Extracting straight into the state directory publishes files one at a time to + * the running gateway, which can read a directory before its content is + * complete. A rename within the same directory publishes each restored + * directory whole. + */ +function buildStagedRestoreCommand(dir: string, stateDirs: readonly string[]): string { + const staging = `${dir}/${RESTORE_STAGING_DIR}`; + const quotedStaging = shellQuote(staging); + const commands = [ + `rm -rf -- ${quotedStaging}`, + `mkdir -p -- ${quotedStaging}`, + `tar --no-same-owner -xf - -C ${quotedStaging}`, + ]; + for (const stateDir of orderStateDirsForRestore(stateDirs)) { + const target = shellQuote(`${dir}/${stateDir}`); + commands.push(`rm -rf -- ${target}`); + commands.push(`mv -- ${shellQuote(`${staging}/${stateDir}`)} ${target}`); + } + commands.push(`rm -rf -- ${quotedStaging}`); + return commands.join(" && "); +} + /** * Restore state directories into a sandbox from a prior backup. */ @@ -1685,7 +1725,12 @@ function restoreSandboxStateInternal( } if (restoreTar !== undefined) { - const extractCmd = `tar --no-same-owner -xf - -C ${shellQuote(dir)}`; + // Image-managed extensions stay in place and are merged by extracting over + // the live directory, so those restores cannot use the staged swap. + const extractCmd = + pluginRestorePlan.preservedExtensionDirs.length > 0 + ? `tar --no-same-owner -xf - -C ${shellQuote(dir)}` + : buildStagedRestoreCommand(dir, localDirs); const sshResult = spawnSync("ssh", [...sshArgs(configFile, sandboxName), extractCmd], { input: restoreTar, stdio: ["pipe", "pipe", "pipe"], From 7da56a032fac16cd8e8c8ff69046b5d8ab05aa6f Mon Sep 17 00:00:00 2001 From: Tinson Lai Date: Thu, 30 Jul 2026 05:55:24 +0000 Subject: [PATCH 02/14] test(state): keep the staged restore test body linear Signed-off-by: Tinson Lai --- src/lib/state/sandbox-staged-restore.test.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/lib/state/sandbox-staged-restore.test.ts b/src/lib/state/sandbox-staged-restore.test.ts index f50eb9b7902..9dd1ceb6e03 100644 --- a/src/lib/state/sandbox-staged-restore.test.ts +++ b/src/lib/state/sandbox-staged-restore.test.ts @@ -43,9 +43,7 @@ function runHermesRestore(options: { stateDirs: string[] }): { fs.mkdirSync(path.join(backupPath, stateDir), { recursive: true }); } fs.writeFileSync(path.join(backupPath, "cron", "jobs.json"), '{"jobs":[{"enabled":true}]}\n'); - if (options.stateDirs.includes("scripts")) { - fs.writeFileSync(path.join(backupPath, "scripts", "digest.sh"), "#!/bin/bash\necho ok\n"); - } + fs.writeFileSync(path.join(backupPath, "scripts", "digest.sh"), "#!/bin/bash\necho ok\n"); fs.writeFileSync( path.join(backupPath, "rebuild-manifest.json"), From 617ea5d46781f8c6cbed0f5ea4436d88c13c20a7 Mon Sep 17 00:00:00 2001 From: Tinson Lai Date: Thu, 30 Jul 2026 06:27:09 +0000 Subject: [PATCH 03/14] fix(state): remove the restore archive copy when publishing fails Signed-off-by: Tinson Lai --- src/lib/state/sandbox-staged-restore.test.ts | 18 ++++++++++++++++-- src/lib/state/sandbox.ts | 8 +++++--- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/lib/state/sandbox-staged-restore.test.ts b/src/lib/state/sandbox-staged-restore.test.ts index 9dd1ceb6e03..08280c09854 100644 --- a/src/lib/state/sandbox-staged-restore.test.ts +++ b/src/lib/state/sandbox-staged-restore.test.ts @@ -19,7 +19,7 @@ function writeExecutable(filePath: string, source: string): void { fs.writeFileSync(filePath, source, { mode: 0o755 }); } -function runHermesRestore(options: { stateDirs: string[] }): { +function runHermesRestore(options: { stateDirs: string[]; movesFail?: boolean }): { moves: MoveRecord[]; restore: ReturnType; restoredCronJob: string | null; @@ -77,7 +77,11 @@ process.exit(0); writeExecutable( path.join(shimDir, "mv"), - `#!/usr/bin/env node + options.movesFail === true + ? `#!/usr/bin/env node +process.exit(1); +` + : `#!/usr/bin/env node const fs = require("node:fs"); const { spawnSync } = require("node:child_process"); const args = process.argv.slice(2); @@ -196,4 +200,14 @@ describe("Hermes cron state restore", () => { ]); expect(result.stagingLeftBehind).toBe(false); }); + + it("removes the archive copy when a state directory cannot be published", () => { + const result = runHermesRestore({ + stateDirs: ["scripts", "cron", "workspace"], + movesFail: true, + }); + + expect(result.restore.success).toBe(false); + expect(result.stagingLeftBehind).toBe(false); + }); }); diff --git a/src/lib/state/sandbox.ts b/src/lib/state/sandbox.ts index 720ed9f49e2..b51856485ed 100644 --- a/src/lib/state/sandbox.ts +++ b/src/lib/state/sandbox.ts @@ -1374,8 +1374,9 @@ function orderStateDirsForRestore(stateDirs: readonly string[]): string[] { function buildStagedRestoreCommand(dir: string, stateDirs: readonly string[]): string { const staging = `${dir}/${RESTORE_STAGING_DIR}`; const quotedStaging = shellQuote(staging); + const removeStaging = `rm -rf -- ${quotedStaging}`; const commands = [ - `rm -rf -- ${quotedStaging}`, + removeStaging, `mkdir -p -- ${quotedStaging}`, `tar --no-same-owner -xf - -C ${quotedStaging}`, ]; @@ -1384,8 +1385,9 @@ function buildStagedRestoreCommand(dir: string, stateDirs: readonly string[]): s commands.push(`rm -rf -- ${target}`); commands.push(`mv -- ${shellQuote(`${staging}/${stateDir}`)} ${target}`); } - commands.push(`rm -rf -- ${quotedStaging}`); - return commands.join(" && "); + // A failed extraction or move ends the chain, so the archive copy is removed + // on the way out instead of at the end of the chain. + return `trap ${shellQuote(removeStaging)} EXIT; ${commands.join(" && ")}`; } /** From 66e47a6e9f32467eb9b91cdee4a80cee72f1db77 Mon Sep 17 00:00:00 2001 From: Tinson Lai Date: Thu, 30 Jul 2026 07:40:35 +0000 Subject: [PATCH 04/14] test(state): assert the failed publication attempt in staged restore Signed-off-by: Tinson Lai --- src/lib/state/sandbox-staged-restore.test.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/lib/state/sandbox-staged-restore.test.ts b/src/lib/state/sandbox-staged-restore.test.ts index 08280c09854..5e76accb637 100644 --- a/src/lib/state/sandbox-staged-restore.test.ts +++ b/src/lib/state/sandbox-staged-restore.test.ts @@ -79,6 +79,15 @@ process.exit(0); path.join(shimDir, "mv"), options.movesFail === true ? `#!/usr/bin/env node +const fs = require("node:fs"); +const args = process.argv.slice(2); +fs.appendFileSync( + ${JSON.stringify(moveLog)}, + JSON.stringify({ + target: args[args.length - 1], + scriptsPresent: fs.existsSync(${JSON.stringify(path.join(hermesDir, "scripts"))}), + }) + "\\n", +); process.exit(1); ` : `#!/usr/bin/env node @@ -207,6 +216,8 @@ describe("Hermes cron state restore", () => { movesFail: true, }); + expect(result.moves.map((move) => path.basename(move.target))).toEqual(["scripts"]); + expect(result.restoredCronJob).toBeNull(); expect(result.restore.success).toBe(false); expect(result.stagingLeftBehind).toBe(false); }); From bef478a4ac8a79e6d60b547fada3ccbf2bb2dca9 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Thu, 30 Jul 2026 11:43:42 -0700 Subject: [PATCH 05/14] fix(shields): lock restored Hermes cron scripts Signed-off-by: Apurv Kumaria --- docs/security/best-practices.mdx | 2 +- scripts/state-dir-guard.py | 1 + src/lib/shields/state-dir-lock.ts | 1 + test/shields-up-runtime-perms.test.ts | 10 +++++++++- test/state-dir-guard.test.ts | 9 ++++++++- 5 files changed, 20 insertions(+), 3 deletions(-) diff --git a/docs/security/best-practices.mdx b/docs/security/best-practices.mdx index 2143c9a3b02..e00995d3bb5 100644 --- a/docs/security/best-practices.mdx +++ b/docs/security/best-practices.mdx @@ -290,7 +290,7 @@ Writable agent state such as plugins, skills, hooks, and workspace metadata live By default, this directory starts writable so the agent can manage its own config, install skills, and write to standard home-directory paths natively. For sensitive workloads, use a reviewed host-side immutability workflow after initial setup so the sandbox user cannot change config or high-risk state entry points. -The immutability workflow locks high-risk state directories (`skills`, `agent`, `hooks`, `cron`, `agents`, `extensions`, `plugins`, `workspace`, `memory`, `devices`, `canvas`, `telegram`, `wechat`, `whatsapp`, `platforms`, `weixin`, `profiles`, `skins`) to `root:sandbox` and removes group and world write access. +The immutability workflow locks high-risk state directories (`skills`, `agent`, `hooks`, `cron`, `agents`, `extensions`, `plugins`, `scripts`, `workspace`, `memory`, `devices`, `canvas`, `telegram`, `wechat`, `whatsapp`, `platforms`, `weixin`, `profiles`, `skins`) to `root:sandbox` and removes group and world write access. The root-only helper traverses from opened directory descriptors with no-follow semantics instead of using recursive pathname `chown` or `chmod`. Read-only preflight and unlock operations reject unsafe external symlinks, hardlinks, special files, cross-device entries, and entries that race the traversal without modifying them. After the top-level config binding is frozen, lockdown makes containment monotonic. diff --git a/scripts/state-dir-guard.py b/scripts/state-dir-guard.py index 12e7ffacc94..8055be153db 100755 --- a/scripts/state-dir-guard.py +++ b/scripts/state-dir-guard.py @@ -39,6 +39,7 @@ "agent", "hooks", "cron", + "scripts", "agents", "extensions", "plugins", diff --git a/src/lib/shields/state-dir-lock.ts b/src/lib/shields/state-dir-lock.ts index 7f11a4a329f..8eb1abb87ba 100644 --- a/src/lib/shields/state-dir-lock.ts +++ b/src/lib/shields/state-dir-lock.ts @@ -29,6 +29,7 @@ export const HIGH_RISK_STATE_DIRS = [ "agent", "hooks", "cron", + "scripts", "agents", "extensions", "plugins", diff --git a/test/shields-up-runtime-perms.test.ts b/test/shields-up-runtime-perms.test.ts index cca7ac5f0ac..86e416a5920 100644 --- a/test/shields-up-runtime-perms.test.ts +++ b/test/shields-up-runtime-perms.test.ts @@ -201,7 +201,15 @@ describe("shields-up state-dir lock preserves sandbox-group access + runtime ses it("keeps the complete protected inventory and writable sessions carve-out", () => { expect(HIGH_RISK_STATE_DIRS).toEqual( - expect.arrayContaining(["skills", "agent", "hooks", "agents", "extensions", "workspace"]), + expect.arrayContaining([ + "skills", + "agent", + "hooks", + "scripts", + "agents", + "extensions", + "workspace", + ]), ); expect(CONFIDENTIALITY_STATE_DIRS).toEqual(["credentials", "identity", "pairing"]); expect(WRITABLE_RUNTIME_SUBPATHS).toEqual(["agents/*/sessions"]); diff --git a/test/state-dir-guard.test.ts b/test/state-dir-guard.test.ts index 5dc23787d70..0411fff9843 100644 --- a/test/state-dir-guard.test.ts +++ b/test/state-dir-guard.test.ts @@ -348,17 +348,22 @@ describe("state-dir-guard", () => { expect(mode(path.join(versionDir, "plugin.js"))).toBe(0o644); }); - it("keeps the runtime ledger writable while sealing cron job definitions", () => { + it("keeps the runtime ledger writable while sealing cron jobs and scripts", () => { const { configDir } = fixture(".hermes"); const cronDir = path.join(configDir, "cron"); const cronLedger = path.join(cronDir, "executions.db"); + const scriptsDir = path.join(configDir, "scripts"); + const cronScript = path.join(scriptsDir, "digest.sh"); const runtimeDir = path.join(configDir, "runtime"); const runtimeLedger = path.join(runtimeDir, "cron-executions.db"); fs.mkdirSync(cronDir); + fs.mkdirSync(scriptsDir); fs.mkdirSync(runtimeDir); fs.chmodSync(cronDir, 0o2770); + fs.chmodSync(scriptsDir, 0o2770); fs.chmodSync(runtimeDir, 0o2770); fs.writeFileSync(cronLedger, "legacy ledger\n", { mode: 0o660 }); + fs.writeFileSync(cronScript, "#!/bin/sh\nexit 0\n", { mode: 0o770 }); fs.writeFileSync(runtimeLedger, "active ledger\n", { mode: 0o660 }); fs.chmodSync(runtimeLedger, 0o660); @@ -367,6 +372,8 @@ describe("state-dir-guard", () => { expect(locked.status, locked.stderr).toBe(0); expect(mode(cronDir)).toBe(0o755); expect(mode(cronLedger)).toBe(0o640); + expect(mode(scriptsDir)).toBe(0o755); + expect(mode(cronScript)).toBe(0o750); expect(mode(runtimeDir)).toBe(0o2770); expect(mode(runtimeLedger)).toBe(0o660); fs.appendFileSync(runtimeLedger, "still writable\n"); From cef29e342641cfa239ff9f73f8f9b6aff562c719 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Thu, 30 Jul 2026 12:43:12 -0700 Subject: [PATCH 06/14] fix(state): roll back failed staged restores Signed-off-by: Apurv Kumaria --- docs/manage-sandboxes/backup-restore.mdx | 8 +- src/lib/state/sandbox-staged-restore.test.ts | 156 ++++++++++++++++--- src/lib/state/sandbox.ts | 98 ++++++++++-- 3 files changed, 225 insertions(+), 37 deletions(-) diff --git a/docs/manage-sandboxes/backup-restore.mdx b/docs/manage-sandboxes/backup-restore.mdx index fb8f929012a..dbd461f1f60 100644 --- a/docs/manage-sandboxes/backup-restore.mdx +++ b/docs/manage-sandboxes/backup-restore.mdx @@ -42,7 +42,8 @@ Treat snapshot directories as private local data. Hermes snapshots include `SOUL.md`, the Web Dashboard profile under `.hermes/dashboard-home/`, the SQLite database behind `.hermes/state.db`, and the default kanban board in `.hermes/kanban.db`. The default-profile snapshot also includes cron execution history in `.hermes/runtime/cron-executions.db` and Discord replay state in `.hermes/gateway/discord_message_recovery.db`. NemoClaw captures cron job definitions from `.hermes/cron` as directory state, together with the scripts they run from `.hermes/scripts`. -A restore moves each state directory into place as a unit and applies `.hermes/cron` last, so the running gateway does not schedule a restored job before the script it calls is available. +A restore moves each state directory into place as a unit and applies `.hermes/cron` after `.hermes/scripts`. +This ordering does not pause or drain work that the running gateway already dispatched. NemoClaw uses SQLite's online backup API and restores these databases through SQLite instead of copying live raw database files. After it replaces a database, NemoClaw opens a write transaction against the result and fails the restore when the database cannot be written. Named-profile cron and Discord databases under `.hermes/profiles//` use raw directory capture and can be inconsistent if a write overlaps the snapshot. @@ -109,6 +110,11 @@ $$nemoclaw my-assistant snapshot restore before-upgrade $$nemoclaw my-assistant snapshot restore 2026-04-14T ``` +For restore paths that replace state directories as whole units, NemoClaw stages every replacement before it changes live state. +If publication fails before the transaction commits, NemoClaw restores the original directories. +If rollback or cleanup cannot finish, NemoClaw preserves `.nemoclaw-restore-rollback` under the agent state directory and refuses another restore so it does not overwrite the recovery copy. +Preserve the reported recovery path and verify its contents before you retry or remove it. + A running Hermes gateway keeps serving its pre-restore state databases until it reopens them. After a restore that includes Hermes state databases, the CLI prints a reminder to restart the gateway. diff --git a/src/lib/state/sandbox-staged-restore.test.ts b/src/lib/state/sandbox-staged-restore.test.ts index 5e76accb637..edaf1eba76d 100644 --- a/src/lib/state/sandbox-staged-restore.test.ts +++ b/src/lib/state/sandbox-staged-restore.test.ts @@ -19,11 +19,22 @@ function writeExecutable(filePath: string, source: string): void { fs.writeFileSync(filePath, source, { mode: 0o755 }); } -function runHermesRestore(options: { stateDirs: string[]; movesFail?: boolean }): { +function runHermesRestore(options: { + stateDirs: string[]; + movesFail?: boolean; + failPublishingDir?: string; + failRemovingRollback?: boolean; + failRollingBackDir?: string; + seedExistingState?: boolean; + seedUnrecoveredRollback?: boolean; +}): { moves: MoveRecord[]; restore: ReturnType; + rollbackScript: string | null; restoredCronJob: string | null; restoredScript: string | null; + restoredWorkspace: string | null; + rollbackLeftBehind: boolean; stagingLeftBehind: boolean; } { const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-staged-restore-")); @@ -38,6 +49,19 @@ function runHermesRestore(options: { stateDirs: string[]; movesFail?: boolean }) fs.mkdirSync(binDir, { recursive: true }); fs.mkdirSync(shimDir, { recursive: true }); fs.mkdirSync(hermesDir, { recursive: true }); + if (options.seedExistingState === true) { + fs.mkdirSync(path.join(hermesDir, "cron"), { recursive: true }); + fs.mkdirSync(path.join(hermesDir, "scripts"), { recursive: true }); + fs.mkdirSync(path.join(hermesDir, "workspace"), { recursive: true }); + fs.writeFileSync(path.join(hermesDir, "cron", "jobs.json"), "old cron\n"); + fs.writeFileSync(path.join(hermesDir, "scripts", "digest.sh"), "old script\n"); + fs.writeFileSync(path.join(hermesDir, "workspace", "notes.md"), "old workspace\n"); + } + if (options.seedUnrecoveredRollback === true) { + const rollbackScripts = path.join(hermesDir, ".nemoclaw-restore-rollback", "scripts"); + fs.mkdirSync(rollbackScripts, { recursive: true }); + fs.writeFileSync(path.join(rollbackScripts, "digest.sh"), "recoverable script\n"); + } for (const stateDir of options.stateDirs) { fs.mkdirSync(path.join(backupPath, stateDir), { recursive: true }); @@ -77,36 +101,47 @@ process.exit(0); writeExecutable( path.join(shimDir, "mv"), - options.movesFail === true - ? `#!/usr/bin/env node -const fs = require("node:fs"); -const args = process.argv.slice(2); -fs.appendFileSync( - ${JSON.stringify(moveLog)}, - JSON.stringify({ - target: args[args.length - 1], - scriptsPresent: fs.existsSync(${JSON.stringify(path.join(hermesDir, "scripts"))}), - }) + "\\n", -); -process.exit(1); -` - : `#!/usr/bin/env node + `#!/usr/bin/env node const fs = require("node:fs"); const { spawnSync } = require("node:child_process"); const args = process.argv.slice(2); +const source = args[args.length - 2] || ""; const target = args[args.length - 1]; -fs.appendFileSync( - ${JSON.stringify(moveLog)}, - JSON.stringify({ - target, - scriptsPresent: fs.existsSync(${JSON.stringify(path.join(hermesDir, "scripts"))}), - }) + "\\n", -); +const isPublish = source.includes("/.nemoclaw-restore-staging/"); +const isRollback = source.includes("/.nemoclaw-restore-rollback/"); +if (isPublish) { + fs.appendFileSync( + ${JSON.stringify(moveLog)}, + JSON.stringify({ + target, + scriptsPresent: fs.existsSync(${JSON.stringify(path.join(hermesDir, "scripts"))}), + }) + "\\n", + ); +} +if ( + ${JSON.stringify(options.movesFail === true)} || + (isPublish && target.endsWith("/" + ${JSON.stringify(options.failPublishingDir ?? "")})) || + (isRollback && target.endsWith("/" + ${JSON.stringify(options.failRollingBackDir ?? "")})) +) process.exit(1); const result = spawnSync("/bin/mv", args, { stdio: "inherit" }); process.exit(result.status === null ? 1 : result.status); `, ); + writeExecutable( + path.join(shimDir, "rm"), + `#!/usr/bin/env node +const { spawnSync } = require("node:child_process"); +const args = process.argv.slice(2); +if ( + ${JSON.stringify(options.failRemovingRollback === true)} && + args.some((arg) => arg.endsWith("/.nemoclaw-restore-rollback")) +) process.exit(1); +const result = spawnSync("/bin/rm", args, { stdio: "inherit" }); +process.exit(result.status === null ? 1 : result.status); +`, + ); + writeExecutable( path.join(binDir, "ssh"), `#!/usr/bin/env node @@ -153,11 +188,25 @@ process.exit(result.status === null ? 1 : result.status); : []; const cronJobPath = path.join(hermesDir, "cron", "jobs.json"); const scriptPath = path.join(hermesDir, "scripts", "digest.sh"); + const workspacePath = path.join(hermesDir, "workspace", "notes.md"); + const rollbackScriptPath = path.join( + hermesDir, + ".nemoclaw-restore-rollback", + "scripts", + "digest.sh", + ); return { moves, restore, + rollbackScript: fs.existsSync(rollbackScriptPath) + ? fs.readFileSync(rollbackScriptPath, "utf8") + : null, restoredCronJob: fs.existsSync(cronJobPath) ? fs.readFileSync(cronJobPath, "utf8") : null, restoredScript: fs.existsSync(scriptPath) ? fs.readFileSync(scriptPath, "utf8") : null, + restoredWorkspace: fs.existsSync(workspacePath) + ? fs.readFileSync(workspacePath, "utf8") + : null, + rollbackLeftBehind: fs.existsSync(path.join(hermesDir, ".nemoclaw-restore-rollback")), stagingLeftBehind: fs.existsSync(path.join(hermesDir, ".nemoclaw-restore-staging")), }; } finally { @@ -221,4 +270,67 @@ describe("Hermes cron state restore", () => { expect(result.restore.success).toBe(false); expect(result.stagingLeftBehind).toBe(false); }); + + it("restores the original state when a staged directory cannot be published", () => { + const result = runHermesRestore({ + stateDirs: ["scripts", "workspace", "cron"], + failPublishingDir: "workspace", + seedExistingState: true, + }); + + expect(result.moves.map((move) => path.basename(move.target))).toEqual([ + "scripts", + "workspace", + ]); + expect(result.restore.success).toBe(false); + expect(result.restoredScript).toBe("old script\n"); + expect(result.restoredWorkspace).toBe("old workspace\n"); + expect(result.restoredCronJob).toBe("old cron\n"); + expect(result.stagingLeftBehind).toBe(false); + expect(result.rollbackLeftBehind).toBe(false); + }); + + it("preserves the recovery tree when rolling the original state back fails", () => { + const result = runHermesRestore({ + stateDirs: ["scripts", "workspace", "cron"], + failPublishingDir: "workspace", + failRollingBackDir: "scripts", + seedExistingState: true, + }); + + expect(result.restore.success).toBe(false); + expect(result.restoredWorkspace).toBe("old workspace\n"); + expect(result.restoredCronJob).toBe("old cron\n"); + expect(result.rollbackScript).toBe("old script\n"); + expect(result.rollbackLeftBehind).toBe(true); + expect(result.stagingLeftBehind).toBe(false); + }); + + it("refuses a new restore while an unrecovered rollback tree exists", () => { + const result = runHermesRestore({ + stateDirs: ["scripts", "workspace", "cron"], + seedExistingState: true, + seedUnrecoveredRollback: true, + }); + + expect(result.restore.success).toBe(false); + expect(result.restoredScript).toBe("old script\n"); + expect(result.rollbackScript).toBe("recoverable script\n"); + expect(result.rollbackLeftBehind).toBe(true); + }); + + it("keeps the new state and recovery tree when post-commit cleanup fails", () => { + const result = runHermesRestore({ + stateDirs: ["scripts", "workspace", "cron"], + failRemovingRollback: true, + seedExistingState: true, + }); + + expect(result.restore.success).toBe(false); + expect(result.restoredScript).toBe("#!/bin/bash\necho ok\n"); + expect(result.restoredCronJob).toBe('{"jobs":[{"enabled":true}]}\n'); + expect(result.rollbackScript).toBe("old script\n"); + expect(result.rollbackLeftBehind).toBe(true); + expect(result.stagingLeftBehind).toBe(false); + }); }); diff --git a/src/lib/state/sandbox.ts b/src/lib/state/sandbox.ts index 4a82042f17e..b39f2762420 100644 --- a/src/lib/state/sandbox.ts +++ b/src/lib/state/sandbox.ts @@ -1350,6 +1350,7 @@ export function backupSandboxState(sandboxName: string, options: BackupOptions = // ── Restore ──────────────────────────────────────────────────────── const RESTORE_STAGING_DIR = ".nemoclaw-restore-staging"; +const RESTORE_ROLLBACK_DIR = ".nemoclaw-restore-rollback"; // Job definitions in these directories are picked up by a gateway that keeps // running throughout a restore, so they must land after every directory whose @@ -1372,23 +1373,91 @@ function orderStateDirsForRestore(stateDirs: readonly string[]): string[] { * complete. A rename within the same directory publishes each restored * directory whole. */ -function buildStagedRestoreCommand(dir: string, stateDirs: readonly string[]): string { +function buildStagedRestoreCommand( + dir: string, + stateDirs: readonly string[], + staleContentDirs: readonly string[] = [], +): string { const staging = `${dir}/${RESTORE_STAGING_DIR}`; + const rollback = `${dir}/${RESTORE_ROLLBACK_DIR}`; const quotedStaging = shellQuote(staging); + const quotedRollback = shellQuote(rollback); const removeStaging = `rm -rf -- ${quotedStaging}`; + const removeRollback = `rm -rf -- ${quotedRollback}`; + const refuseUnrecoveredRollback = + `if [ -e ${quotedRollback} ] || [ -L ${quotedRollback} ]; then ` + + `echo ${shellQuote(`Refusing restore: unrecovered NemoClaw state remains at ${rollback}`)} >&2; ` + + "exit 1; fi"; + const restoredDirSet = new Set(stateDirs); + const transitionDirs = orderStateDirsForRestore([ + ...new Set([...stateDirs, ...staleContentDirs]), + ]); + const rollbackCommands = transitionDirs + .map((stateDir, index) => ({ stateDir, index })) + .reverse() + .map(({ stateDir, index }) => { + const target = shellQuote(`${dir}/${stateDir}`); + const rollbackTarget = shellQuote(`${rollback}/${stateDir}`); + const targetParent = shellQuote(path.posix.dirname(`${dir}/${stateDir}`)); + const existingMarker = shellQuote(`${rollback}/.existing-${index}`); + const absentMarker = shellQuote(`${rollback}/.absent-${index}`); + return ( + `if [ -e ${existingMarker} ]; then ` + + `if [ -e ${rollbackTarget} ] || [ -L ${rollbackTarget} ]; then ` + + `if rm -rf -- ${target} && mkdir -p -- ${targetParent} && ` + + `mv -- ${rollbackTarget} ${target}; then :; else rollback_status=1; fi; fi; ` + + `elif [ -e ${absentMarker} ]; then ` + + `rm -rf -- ${target} || rollback_status=1; fi` + ); + }); + const cleanup = [ + "status=$?", + "rollback_status=0", + `if [ "$status" -ne 0 ] && [ "$transaction_committed" -ne 1 ]; then ` + + `${rollbackCommands.join("; ")}; fi`, + removeStaging, + `if [ "$status" -ne 0 ] && [ "$transaction_committed" -eq 1 ]; then ` + + `echo ${shellQuote( + `NemoClaw restore committed but cleanup failed; preserved recovery state at ${rollback}`, + )} >&2; ` + + `elif [ "$rollback_status" -eq 0 ]; then ${removeRollback}; ` + + `else echo ${shellQuote( + `NemoClaw restore rollback failed; preserved recovery state at ${rollback}`, + )} >&2; fi`, + 'exit "$status"', + ].join("; "); const commands = [ removeStaging, `mkdir -p -- ${quotedStaging}`, + `mkdir -p -- ${quotedRollback}`, `tar --no-same-owner -xf - -C ${quotedStaging}`, ]; - for (const stateDir of orderStateDirsForRestore(stateDirs)) { + for (const [index, stateDir] of transitionDirs.entries()) { const target = shellQuote(`${dir}/${stateDir}`); - commands.push(`rm -rf -- ${target}`); - commands.push(`mv -- ${shellQuote(`${staging}/${stateDir}`)} ${target}`); + const rollbackTarget = shellQuote(`${rollback}/${stateDir}`); + const rollbackParent = shellQuote(path.posix.dirname(`${rollback}/${stateDir}`)); + const existingMarker = shellQuote(`${rollback}/.existing-${index}`); + const absentMarker = shellQuote(`${rollback}/.absent-${index}`); + commands.push( + `if [ -e ${target} ] || [ -L ${target} ]; then ` + + `mkdir -p -- ${rollbackParent} && touch -- ${existingMarker} && ` + + `mv -- ${target} ${rollbackTarget}; else touch -- ${absentMarker}; fi`, + ); + if (restoredDirSet.has(stateDir)) { + const targetParent = shellQuote(path.posix.dirname(`${dir}/${stateDir}`)); + commands.push(`mkdir -p -- ${targetParent}`); + commands.push(`mv -- ${shellQuote(`${staging}/${stateDir}`)} ${target}`); + } } - // A failed extraction or move ends the chain, so the archive copy is removed - // on the way out instead of at the end of the chain. - return `trap ${shellQuote(removeStaging)} EXIT; ${commands.join(" && ")}`; + commands.push("transaction_committed=1"); + commands.push(removeRollback); + // A failed extraction or move ends the chain. The EXIT trap restores every + // live directory already transitioned. If rollback itself fails, keep the + // recovery tree and refuse future restores instead of deleting its only copy. + return ( + `${refuseUnrecoveredRollback}; transaction_committed=0; ` + + `trap ${shellQuote(cleanup)} EXIT; ${commands.join(" && ")}` + ); } /** @@ -1692,12 +1761,13 @@ function restoreSandboxStateInternal( restoreTar = tarResult.stdout; } - // Remove existing state dirs before extracting so stale files from later - // snapshots don't persist after restoring an earlier one. OpenClaw's - // image-managed extensions are preserved from the freshly built image and - // excluded from the restore tar; only user/non-managed extension entries - // are cleared and restored from the backup. - if (cleanupStateDirs.length > 0) { + // OpenClaw image-managed extensions must be merged into the live directory, + // so that path still cleans user-owned entries before extraction. The + // staged path below preserves live directories in its rollback tree and + // removes stale content only after every replacement is ready. + const usesStagedDirectoryRestore = + restoreTar !== undefined && pluginRestorePlan.preservedExtensionDirs.length === 0; + if (cleanupStateDirs.length > 0 && !usesStagedDirectoryRestore) { const rmCmd = buildRestoreCleanupCommand( dir, localDirs, @@ -1733,7 +1803,7 @@ function restoreSandboxStateInternal( const extractCmd = pluginRestorePlan.preservedExtensionDirs.length > 0 ? `tar --no-same-owner -xf - -C ${shellQuote(dir)}` - : buildStagedRestoreCommand(dir, localDirs); + : buildStagedRestoreCommand(dir, localDirs, staleContentDirs); const sshResult = spawnSync("ssh", [...sshArgs(configFile, sandboxName), extractCmd], { input: restoreTar, stdio: ["pipe", "pipe", "pipe"], From 447a759d9e1fc0078173f63966928102ef0cfbaf Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Thu, 30 Jul 2026 12:49:57 -0700 Subject: [PATCH 07/14] test(state): keep rollback fixtures linear Signed-off-by: Apurv Kumaria --- src/lib/state/sandbox-staged-restore.test.ts | 34 ++++++++++++-------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/src/lib/state/sandbox-staged-restore.test.ts b/src/lib/state/sandbox-staged-restore.test.ts index edaf1eba76d..b4feac013b8 100644 --- a/src/lib/state/sandbox-staged-restore.test.ts +++ b/src/lib/state/sandbox-staged-restore.test.ts @@ -19,6 +19,21 @@ function writeExecutable(filePath: string, source: string): void { fs.writeFileSync(filePath, source, { mode: 0o755 }); } +function seedExistingStateFixture(hermesDir: string): void { + fs.mkdirSync(path.join(hermesDir, "cron"), { recursive: true }); + fs.mkdirSync(path.join(hermesDir, "scripts"), { recursive: true }); + fs.mkdirSync(path.join(hermesDir, "workspace"), { recursive: true }); + fs.writeFileSync(path.join(hermesDir, "cron", "jobs.json"), "old cron\n"); + fs.writeFileSync(path.join(hermesDir, "scripts", "digest.sh"), "old script\n"); + fs.writeFileSync(path.join(hermesDir, "workspace", "notes.md"), "old workspace\n"); +} + +function seedUnrecoveredRollbackFixture(hermesDir: string): void { + const rollbackScripts = path.join(hermesDir, ".nemoclaw-restore-rollback", "scripts"); + fs.mkdirSync(rollbackScripts, { recursive: true }); + fs.writeFileSync(path.join(rollbackScripts, "digest.sh"), "recoverable script\n"); +} + function runHermesRestore(options: { stateDirs: string[]; movesFail?: boolean; @@ -49,19 +64,12 @@ function runHermesRestore(options: { fs.mkdirSync(binDir, { recursive: true }); fs.mkdirSync(shimDir, { recursive: true }); fs.mkdirSync(hermesDir, { recursive: true }); - if (options.seedExistingState === true) { - fs.mkdirSync(path.join(hermesDir, "cron"), { recursive: true }); - fs.mkdirSync(path.join(hermesDir, "scripts"), { recursive: true }); - fs.mkdirSync(path.join(hermesDir, "workspace"), { recursive: true }); - fs.writeFileSync(path.join(hermesDir, "cron", "jobs.json"), "old cron\n"); - fs.writeFileSync(path.join(hermesDir, "scripts", "digest.sh"), "old script\n"); - fs.writeFileSync(path.join(hermesDir, "workspace", "notes.md"), "old workspace\n"); - } - if (options.seedUnrecoveredRollback === true) { - const rollbackScripts = path.join(hermesDir, ".nemoclaw-restore-rollback", "scripts"); - fs.mkdirSync(rollbackScripts, { recursive: true }); - fs.writeFileSync(path.join(rollbackScripts, "digest.sh"), "recoverable script\n"); - } + const existingStateSeeder = + options.seedExistingState === true ? seedExistingStateFixture : () => undefined; + const rollbackSeeder = + options.seedUnrecoveredRollback === true ? seedUnrecoveredRollbackFixture : () => undefined; + existingStateSeeder(hermesDir); + rollbackSeeder(hermesDir); for (const stateDir of options.stateDirs) { fs.mkdirSync(path.join(backupPath, stateDir), { recursive: true }); From 25e00ce838e3c1fd078bd9999be97f68742481c5 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Fri, 31 Jul 2026 07:55:14 -0700 Subject: [PATCH 08/14] fix(state): drain Hermes scheduled work during restore Signed-off-by: Apurv Kumaria --- agents/hermes/Dockerfile | 13 +- agents/hermes/restore-cron-guard.py | 186 +++++++++++++++++++ docs/manage-sandboxes/backup-restore.mdx | 4 +- docs/reference/commands.mdx | 1 + docs/security/best-practices.mdx | 1 + src/lib/state/sandbox-staged-restore.test.ts | 170 ++++++++++++++++- src/lib/state/sandbox.ts | 59 +++++- test/helpers/vitest-watch-triggers.ts | 4 + test/hermes-final-image-layout.test.ts | 7 + test/hermes-restore-cron-guard.test.ts | 163 ++++++++++++++++ test/vitest-watch-triggers.test.ts | 4 + 11 files changed, 598 insertions(+), 14 deletions(-) create mode 100755 agents/hermes/restore-cron-guard.py create mode 100644 test/hermes-restore-cron-guard.test.ts diff --git a/agents/hermes/Dockerfile b/agents/hermes/Dockerfile index b7b965b8fa0..52ff1dd119f 100644 --- a/agents/hermes/Dockerfile +++ b/agents/hermes/Dockerfile @@ -59,6 +59,7 @@ COPY agents/hermes/runtime-config-guard.py /usr/local/lib/nemoclaw/hermes-runtim COPY agents/hermes/finalize-tirith-marker.py /usr/local/lib/nemoclaw/finalize-tirith-marker.py COPY agents/hermes/build-mcp-digest.py /usr/local/lib/nemoclaw/build-hermes-mcp-digest.py COPY agents/hermes/mcp-config-transaction.py /usr/local/lib/nemoclaw/hermes-mcp-config-transaction.py +COPY agents/hermes/restore-cron-guard.py /usr/local/lib/nemoclaw/hermes-restore-cron-guard.py COPY src/lib/actions/sandbox/openshell-child-visible-credentials.v0.0.85.json /usr/local/lib/nemoclaw/openshell-child-visible-credentials.v0.0.85.json COPY scripts/state-dir-guard.py /usr/local/lib/nemoclaw/state-dir-guard.py COPY nemoclaw-blueprint/scripts/*.js /usr/local/lib/nemoclaw/preloads/ @@ -213,9 +214,10 @@ RUN chmod -R a+rX /opt/nemoclaw-blueprint/ # minimum supported Hermes sandbox base tag guarantees those artifacts and # test/sandbox-rlimit-hooks.test.ts covers that base. RUN chmod 755 /usr/local/bin/nemoclaw-start /usr/local/lib/nemoclaw/sandbox-init.sh /usr/local/lib/nemoclaw/validate-hermes-env-secret-boundary.py /usr/local/lib/nemoclaw/patch-hermes-session-list-preview.py /usr/local/lib/nemoclaw/patch-hermes-discord-recovery-permissions.py /usr/local/lib/nemoclaw/patch-hermes-profile-policy-defaults.py /usr/local/lib/nemoclaw/seed-hermes-dashboard-config.py /usr/local/lib/nemoclaw/hermes-runtime-config-guard.py /usr/local/lib/nemoclaw/finalize-tirith-marker.py /usr/local/lib/nemoclaw/hermes-mcp-config-transaction.py \ - && chown root:root /usr/local/bin/nemoclaw-gateway-control /usr/local/lib/nemoclaw/gateway-supervisor.sh /usr/local/lib/nemoclaw/state-dir-guard.py /usr/local/lib/nemoclaw/managed-gateway-control.py /usr/local/lib/nemoclaw/build-hermes-mcp-digest.py /usr/local/lib/nemoclaw/openshell-child-visible-credentials.v0.0.85.json \ + && chown root:root /usr/local/bin/nemoclaw-gateway-control /usr/local/lib/nemoclaw/gateway-supervisor.sh /usr/local/lib/nemoclaw/state-dir-guard.py /usr/local/lib/nemoclaw/managed-gateway-control.py /usr/local/lib/nemoclaw/build-hermes-mcp-digest.py /usr/local/lib/nemoclaw/hermes-restore-cron-guard.py /usr/local/lib/nemoclaw/openshell-child-visible-credentials.v0.0.85.json \ && chmod 700 /usr/local/bin/nemoclaw-gateway-control \ && chmod 500 /usr/local/lib/nemoclaw/state-dir-guard.py /usr/local/lib/nemoclaw/managed-gateway-control.py \ + && chmod 555 /usr/local/lib/nemoclaw/hermes-restore-cron-guard.py \ && chmod 444 /usr/local/lib/nemoclaw/gateway-supervisor.sh /usr/local/lib/nemoclaw/build-hermes-mcp-digest.py \ && chmod 444 /usr/local/lib/nemoclaw/patch-hermes-langfuse-credentials.mts \ && chmod 444 /usr/local/lib/nemoclaw/openshell-child-visible-credentials.v0.0.85.json \ @@ -505,10 +507,11 @@ PY # accompanied by an updated hash below; otherwise the build fails. This blocks # silent supply-chain tampering of the build context (an attacker rewriting a # file has to also rewrite the Dockerfile-committed hash, which reviewers gate). -# Regenerate with `sha256sum agents/hermes/{hermes-wrapper.py,validate-env-secret-boundary.py,finalize-tirith-marker.py}`. +# Regenerate with `sha256sum agents/hermes/{hermes-wrapper.py,validate-env-secret-boundary.py,finalize-tirith-marker.py,restore-cron-guard.py}`. ARG NEMOCLAW_HERMES_WRAPPER_SHA256=cd851746da14162ac4701d56c274dac20024ea6a11f6ffcf2ce7fb89dff388a0 ARG NEMOCLAW_HERMES_VALIDATOR_SHA256=822c7e63d068c5d09f3291350771c1a42c9686f51bfa9bc9a1f41fbe15d163b1 ARG NEMOCLAW_HERMES_TIRITH_FINALIZER_SHA256=a1e6b1c53ab297569abb87c29d15c294d729e46005bfd022136b4c447a791819 +ARG NEMOCLAW_HERMES_RESTORE_CRON_GUARD_SHA256=c224191fac19e4c7bc1d4416e2d1b243d7bd925af02fd25fa4d2c4cf6777345d # hadolint ignore=DL4006 RUN printf '%s %s\n' \ "$NEMOCLAW_HERMES_VALIDATOR_SHA256" /usr/local/lib/nemoclaw/validate-hermes-env-secret-boundary.py \ @@ -519,6 +522,11 @@ RUN printf '%s %s\n' \ "$NEMOCLAW_HERMES_TIRITH_FINALIZER_SHA256" /usr/local/lib/nemoclaw/finalize-tirith-marker.py \ | sha256sum -c - \ || { echo "ERROR: finalize-tirith-marker.py hash mismatch (update NEMOCLAW_HERMES_TIRITH_FINALIZER_SHA256)" >&2; exit 1; } +# hadolint ignore=DL4006 +RUN printf '%s %s\n' \ + "$NEMOCLAW_HERMES_RESTORE_CRON_GUARD_SHA256" /usr/local/lib/nemoclaw/hermes-restore-cron-guard.py \ + | sha256sum -c - \ + || { echo "ERROR: hermes-restore-cron-guard.py hash mismatch (update NEMOCLAW_HERMES_RESTORE_CRON_GUARD_SHA256)" >&2; exit 1; } # Wrap the hermes CLI so the runtime env secret boundary is enforced for # `hermes gateway` no matter how it is invoked. The entrypoint guard alone left @@ -1199,6 +1207,7 @@ RUN check_metadata() { \ && check_metadata /usr/local/lib/nemoclaw/validate-hermes-env-secret-boundary.py 'root:root 755' \ && check_metadata /usr/local/lib/nemoclaw/patch-hermes-discord-recovery-permissions.py 'root:root 755' \ && check_metadata /usr/local/lib/nemoclaw/patch-hermes-profile-policy-defaults.py 'root:root 755' \ + && check_metadata /usr/local/lib/nemoclaw/hermes-restore-cron-guard.py 'root:root 555' \ && check_metadata /usr/local/bin/nemoclaw-gateway-control 'root:root 700' \ && check_metadata /usr/local/lib/nemoclaw/preloads/sandbox-safety-net.js 'root:root 444' \ && check_metadata /usr/local/lib/nemoclaw/hermes-wrapper.py 'root:root 755' \ diff --git a/agents/hermes/restore-cron-guard.py b/agents/hermes/restore-cron-guard.py new file mode 100755 index 00000000000..da67ac04d2e --- /dev/null +++ b/agents/hermes/restore-cron-guard.py @@ -0,0 +1,186 @@ +#!/opt/hermes/.venv/bin/python -I +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Quiesce Hermes cron dispatch while NemoClaw restores scheduled work state.""" + +from __future__ import annotations + +import argparse +import json +import os +import secrets +import sys +import time +from pathlib import Path +from typing import Any + +_OWNER_PREFIX = "nemoclaw-state-restore:" +_POLL_INTERVAL_SECONDS = 0.1 + + +def _configure_home(raw_home: str) -> Path: + home = Path(raw_home) + if not home.is_absolute(): + raise ValueError("Hermes restore guard requires an absolute --home path") + os.environ["HERMES_HOME"] = str(home) + return home + + +def _gateway_modules() -> tuple[Any, Any]: + from gateway import drain_control, status + + return drain_control, status + + +def _runtime_is_safely_drained(status: Any, pid: int) -> bool: + runtime = status.read_runtime_status() + return bool( + isinstance(runtime, dict) + and runtime.get("pid") == pid + and runtime.get("gateway_state") == "draining" + and status.parse_active_agents(runtime.get("active_agents")) == 0 + ) + + +def _owned_marker_present(drain_control: Any, home: Path, token: str) -> bool: + marker = drain_control.read_drain_request(home=home) + return bool(isinstance(marker, dict) and marker.get("principal") == token) + + +def _release_owned_marker(drain_control: Any, home: Path, token: str) -> None: + if not _owned_marker_present(drain_control, home, token): + return + if not drain_control.clear_drain_request(home=home): + raise RuntimeError("Hermes restore guard could not clear its drain marker") + + +def begin_drain(home: Path, timeout_seconds: float) -> str: + drain_control, status = _gateway_modules() + pid = status.get_running_pid() + if pid is None: + return "inactive" + + token = "" + if not drain_control.drain_requested(home=home): + token = f"{_OWNER_PREFIX}{secrets.token_hex(16)}" + drain_control.write_drain_request(principal=token, home=home) + + deadline = time.monotonic() + timeout_seconds + try: + while time.monotonic() < deadline: + live_pid = status.get_running_pid() + if live_pid is None or _runtime_is_safely_drained(status, live_pid): + return token or "preserved" + time.sleep(_POLL_INTERVAL_SECONDS) + except BaseException: + if token: + _release_owned_marker(drain_control, home, token) + raise + + if token: + _release_owned_marker(drain_control, home, token) + raise TimeoutError( + f"Hermes gateway did not drain active messaging, API, and cron work within {timeout_seconds:g}s" + ) + + +def assert_safely_drained(home: Path) -> None: + drain_control, status = _gateway_modules() + pid = status.get_running_pid() + if pid is None: + return + if not drain_control.drain_requested(home=home) or not _runtime_is_safely_drained( + status, pid + ): + raise RuntimeError("Hermes gateway is not safely drained for scheduled-work restore") + + +def _load_jobs(jobs_file: Path) -> list[Any]: + if not jobs_file.exists(): + return [] + data = json.loads(jobs_file.read_text(encoding="utf-8-sig")) + jobs = data.get("jobs", []) if isinstance(data, dict) else data + if not isinstance(jobs, list): + raise ValueError("Hermes cron database must contain a jobs list") + return jobs + + +def validate_enabled_scripts(home: Path) -> None: + scripts_dir = (home / "scripts").resolve() + for index, job in enumerate(_load_jobs(home / "cron" / "jobs.json")): + if not isinstance(job, dict): + raise ValueError(f"Hermes cron job at index {index} is not an object") + if not job.get("enabled", True) or job.get("state") == "paused": + continue + script = job.get("script") + if script in {None, ""}: + if job.get("no_agent"): + raise ValueError( + f"Enabled no-agent Hermes cron job at index {index} has no script" + ) + continue + if not isinstance(script, str): + raise ValueError(f"Enabled Hermes cron job at index {index} has a non-string script") + raw_path = Path(script).expanduser() + script_path = ( + raw_path.resolve() + if raw_path.is_absolute() + else (scripts_dir / raw_path).resolve() + ) + try: + script_path.relative_to(scripts_dir) + except ValueError as error: + raise ValueError( + f"Enabled Hermes cron job at index {index} resolves outside the scripts directory" + ) from error + if not script_path.is_file() or not os.access(script_path, os.R_OK): + raise ValueError( + f"Enabled Hermes cron job at index {index} references a missing or unreadable script" + ) + + +def validate_restore(home: Path) -> None: + assert_safely_drained(home) + validate_enabled_scripts(home) + + +def release_drain(home: Path, token: str) -> None: + if not token.startswith(_OWNER_PREFIX) or len(token) != len(_OWNER_PREFIX) + 32: + raise ValueError("Invalid Hermes restore drain ownership token") + drain_control, _status = _gateway_modules() + _release_owned_marker(drain_control, home, token) + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser() + parser.add_argument("action", choices=("begin", "assert-safe", "validate", "release")) + parser.add_argument("--home", required=True) + parser.add_argument("--timeout", type=float, default=60.0) + parser.add_argument("--token") + return parser + + +def main() -> int: + args = _parser().parse_args() + try: + home = _configure_home(args.home) + if args.action == "begin": + if args.timeout <= 0: + raise ValueError("Hermes restore drain timeout must be positive") + print(begin_drain(home, args.timeout)) + elif args.action == "assert-safe": + assert_safely_drained(home) + elif args.action == "validate": + validate_restore(home) + else: + if not args.token: + raise ValueError("Hermes restore drain release requires --token") + release_drain(home, args.token) + return 0 + except Exception as error: + print(f"Hermes restore guard failed: {error}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/docs/manage-sandboxes/backup-restore.mdx b/docs/manage-sandboxes/backup-restore.mdx index d46a8fabf2c..2aa34d49bd5 100644 --- a/docs/manage-sandboxes/backup-restore.mdx +++ b/docs/manage-sandboxes/backup-restore.mdx @@ -43,7 +43,9 @@ Hermes snapshots include `SOUL.md`, the Web Dashboard profile under `.hermes/das The default-profile snapshot also includes cron execution history in `.hermes/runtime/cron-executions.db` and Discord replay state in `.hermes/gateway/discord_message_recovery.db`. NemoClaw captures cron job definitions from `.hermes/cron` as directory state, together with the scripts they run from `.hermes/scripts`. A restore moves each state directory into place as a unit and applies `.hermes/cron` after `.hermes/scripts`. -This ordering does not pause or drain work that the running gateway already dispatched. +Before replacing either directory, NemoClaw asks the running Hermes gateway to drain and waits until messaging, API, and cron work reaches zero. +It validates every enabled job's referenced script before releasing a drain that NemoClaw created; a drain already owned by an operator remains in place. +If the gateway cannot drain, a referenced script is missing or unreadable, or rollback cannot recover the prior state, the restore fails closed without resuming scheduled work. NemoClaw uses SQLite's online backup API and restores these databases through SQLite instead of copying live raw database files. After it replaces a database, NemoClaw opens a write transaction against the result and fails the restore when the database cannot be written. Named-profile cron and Discord databases under `.hermes/profiles//` use raw directory capture and can be inconsistent if a write overlaps the snapshot. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 219e59d69ba..67946f5a0bd 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -648,6 +648,7 @@ For OpenClaw, the backed-up paths include agents, extensions, workspace, skills, For Hermes, the backed-up paths come from `agents/hermes/manifest.yaml`, including `/sandbox/.hermes` state such as memories, sessions, skills, plugins, scripts, cron, logs, plans, workspace, messaging platform state, `runtime/state.db`, and the default kanban board in `kanban.db`. +During restore, NemoClaw drains the Hermes gateway before replacing scripts or cron definitions, validates enabled-job script references, and then resumes only a drain that it created. Kanban backup does not include named boards, attachments, worker logs, scratch workspaces under `kanban/`, or external directory or worktree targets. diff --git a/docs/security/best-practices.mdx b/docs/security/best-practices.mdx index e00995d3bb5..5ad5aa13596 100644 --- a/docs/security/best-practices.mdx +++ b/docs/security/best-practices.mdx @@ -344,6 +344,7 @@ Direct edits to these files can be overwritten when NemoClaw regenerates the ima Hermes also stores runtime state such as `state.db`, logs, and platform sessions under the `.hermes` tree. Messaging sessions such as WhatsApp pairing can remain mutable by design so they survive rebuilds. +Hermes rebuild restore blocks new gateway dispatch and waits for active messaging, API, and cron work to drain before replacing cron scripts or job definitions. It keeps that drain in place until enabled script references validate or the prior state is rolled back. | Aspect | Detail | |---|---| diff --git a/src/lib/state/sandbox-staged-restore.test.ts b/src/lib/state/sandbox-staged-restore.test.ts index b4feac013b8..dd1a822bd0b 100644 --- a/src/lib/state/sandbox-staged-restore.test.ts +++ b/src/lib/state/sandbox-staged-restore.test.ts @@ -13,7 +13,10 @@ import { restoreRecreatedSandboxState } from "./sandbox.js"; const HERMES_DIR = "/sandbox/.hermes"; -type MoveRecord = { target: string; scriptsPresent: boolean }; +type RestoreEvent = + | { event: "guard"; action: string } + | { event: "move"; target: string; scriptsPresent: boolean; drainActive: boolean }; +type MoveRecord = Extract; function writeExecutable(filePath: string, source: string): void { fs.writeFileSync(filePath, source, { mode: 0o755 }); @@ -42,8 +45,14 @@ function runHermesRestore(options: { failRollingBackDir?: string; seedExistingState?: boolean; seedUnrecoveredRollback?: boolean; + gatewayRunning?: boolean; + preexistingDrain?: boolean; + guardValidationFails?: boolean; + missingRestoredScript?: boolean; }): { moves: MoveRecord[]; + events: RestoreEvent[]; + guardEvents: string[]; restore: ReturnType; rollbackScript: string | null; restoredCronJob: string | null; @@ -51,6 +60,7 @@ function runHermesRestore(options: { restoredWorkspace: string | null; rollbackLeftBehind: boolean; stagingLeftBehind: boolean; + drainLeftBehind: boolean; } { const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-staged-restore-")); const previousOpenshellBin = process.env.NEMOCLAW_OPENSHELL_BIN; @@ -61,6 +71,7 @@ function runHermesRestore(options: { const hermesDir = path.join(fixture, "sandbox-root", ".hermes"); const backupPath = path.join(fixture, "backup"); const moveLog = path.join(fixture, "move-log.jsonl"); + const drainMarker = path.join(fixture, "drain-active"); fs.mkdirSync(binDir, { recursive: true }); fs.mkdirSync(shimDir, { recursive: true }); fs.mkdirSync(hermesDir, { recursive: true }); @@ -70,12 +81,20 @@ function runHermesRestore(options: { options.seedUnrecoveredRollback === true ? seedUnrecoveredRollbackFixture : () => undefined; existingStateSeeder(hermesDir); rollbackSeeder(hermesDir); + if (options.preexistingDrain === true) { + fs.writeFileSync(drainMarker, "external\n"); + } for (const stateDir of options.stateDirs) { fs.mkdirSync(path.join(backupPath, stateDir), { recursive: true }); } - fs.writeFileSync(path.join(backupPath, "cron", "jobs.json"), '{"jobs":[{"enabled":true}]}\n'); - fs.writeFileSync(path.join(backupPath, "scripts", "digest.sh"), "#!/bin/bash\necho ok\n"); + fs.writeFileSync( + path.join(backupPath, "cron", "jobs.json"), + '{"jobs":[{"enabled":true,"script":"digest.sh"}]}\n', + ); + if (options.missingRestoredScript !== true) { + fs.writeFileSync(path.join(backupPath, "scripts", "digest.sh"), "#!/bin/bash\necho ok\n"); + } fs.writeFileSync( path.join(backupPath, "rebuild-manifest.json"), @@ -121,8 +140,10 @@ if (isPublish) { fs.appendFileSync( ${JSON.stringify(moveLog)}, JSON.stringify({ + event: "move", target, scriptsPresent: fs.existsSync(${JSON.stringify(path.join(hermesDir, "scripts"))}), + drainActive: fs.existsSync(${JSON.stringify(drainMarker)}), }) + "\\n", ); } @@ -136,6 +157,58 @@ process.exit(result.status === null ? 1 : result.status); `, ); + const restoreGuard = path.join(binDir, "hermes-restore-cron-guard"); + writeExecutable( + restoreGuard, + `#!/usr/bin/env node +const fs = require("node:fs"); +const path = require("node:path"); +const args = process.argv.slice(2); +const action = args[0] || ""; +fs.appendFileSync( + ${JSON.stringify(moveLog)}, + JSON.stringify({ event: "guard", action }) + "\\n", +); +if (action === "begin") { + if (!${JSON.stringify(options.gatewayRunning !== false)}) { + process.stdout.write("inactive\\n"); + } else if (fs.existsSync(${JSON.stringify(drainMarker)})) { + process.stdout.write("preserved\\n"); + } else { + const token = "nemoclaw-state-restore:0123456789abcdef0123456789abcdef"; + fs.writeFileSync(${JSON.stringify(drainMarker)}, token + "\\n"); + process.stdout.write(token + "\\n"); + } +} else if (action === "assert-safe") { + if (${JSON.stringify(options.gatewayRunning !== false)} && !fs.existsSync(${JSON.stringify(drainMarker)})) { + process.exit(1); + } +} else if (action === "validate") { + if (${JSON.stringify(options.guardValidationFails === true)}) process.exit(1); + if (${JSON.stringify(options.gatewayRunning !== false)} && !fs.existsSync(${JSON.stringify(drainMarker)})) { + process.exit(1); + } + const jobs = JSON.parse(fs.readFileSync(${JSON.stringify(path.join(hermesDir, "cron", "jobs.json"))}, "utf8")).jobs; + for (const job of jobs) { + if (!job.enabled || !job.script) continue; + try { + fs.accessSync(path.join(${JSON.stringify(path.join(hermesDir, "scripts"))}, job.script), fs.constants.R_OK); + } catch { + process.exit(1); + } + } +} else if (action === "release") { + const tokenIndex = args.indexOf("--token"); + const token = tokenIndex >= 0 ? args[tokenIndex + 1] : ""; + const owner = fs.existsSync(${JSON.stringify(drainMarker)}) + ? fs.readFileSync(${JSON.stringify(drainMarker)}, "utf8").trim() + : ""; + if (owner === token) fs.rmSync(${JSON.stringify(drainMarker)}); +} +process.exit(0); +`, + ); + writeExecutable( path.join(shimDir, "rm"), `#!/usr/bin/env node @@ -155,7 +228,9 @@ process.exit(result.status === null ? 1 : result.status); `#!/usr/bin/env node const fs = require("node:fs"); const { spawnSync } = require("node:child_process"); -const command = (process.argv[process.argv.length - 1] || "").split(${JSON.stringify(HERMES_DIR)}).join(${JSON.stringify(hermesDir)}); +const command = (process.argv[process.argv.length - 1] || "") + .split(${JSON.stringify(HERMES_DIR)}).join(${JSON.stringify(hermesDir)}) + .split("/usr/local/lib/nemoclaw/hermes-restore-cron-guard.py").join(${JSON.stringify(restoreGuard)}); function readStdin() { const chunks = []; for (;;) { @@ -186,14 +261,15 @@ process.exit(result.status === null ? 1 : result.status); targetAgentType: "hermes", }); - const moves = fs.existsSync(moveLog) + const events = fs.existsSync(moveLog) ? fs .readFileSync(moveLog, "utf8") .trim() .split("\n") .filter(Boolean) - .map((line) => JSON.parse(line) as MoveRecord) + .map((line) => JSON.parse(line) as RestoreEvent) : []; + const moves = events.filter((event): event is MoveRecord => event.event === "move"); const cronJobPath = path.join(hermesDir, "cron", "jobs.json"); const scriptPath = path.join(hermesDir, "scripts", "digest.sh"); const workspacePath = path.join(hermesDir, "workspace", "notes.md"); @@ -205,6 +281,12 @@ process.exit(result.status === null ? 1 : result.status); ); return { moves, + events, + guardEvents: events + .filter( + (event): event is Extract => event.event === "guard", + ) + .map((event) => event.action), restore, rollbackScript: fs.existsSync(rollbackScriptPath) ? fs.readFileSync(rollbackScriptPath, "utf8") @@ -216,6 +298,7 @@ process.exit(result.status === null ? 1 : result.status); : null, rollbackLeftBehind: fs.existsSync(path.join(hermesDir, ".nemoclaw-restore-rollback")), stagingLeftBehind: fs.existsSync(path.join(hermesDir, ".nemoclaw-restore-staging")), + drainLeftBehind: fs.existsSync(drainMarker), }; } finally { restoreEnvBulk({ NEMOCLAW_OPENSHELL_BIN: previousOpenshellBin, PATH: previousPath }); @@ -236,7 +319,46 @@ describe("Hermes cron state restore", () => { expect.arrayContaining(["scripts", "cron", "workspace"]), ); expect(result.restoredScript).toBe("#!/bin/bash\necho ok\n"); - expect(result.restoredCronJob).toBe('{"jobs":[{"enabled":true}]}\n'); + expect(result.restoredCronJob).toBe('{"jobs":[{"enabled":true,"script":"digest.sh"}]}\n'); + }); + + it("drains the gateway before moving live scheduled-work state and resumes after validation", () => { + const result = runHermesRestore({ stateDirs: ["scripts", "cron", "workspace"] }); + + expect(result.restore.success).toBe(true); + expect(result.guardEvents).toEqual(["begin", "assert-safe", "validate", "release"]); + expect(result.moves.every((move) => move.drainActive)).toBe(true); + expect( + result.events.findIndex((event) => event.event === "guard" && event.action === "begin"), + ).toBeLessThan(result.events.findIndex((event) => event.event === "move")); + expect( + result.events.findIndex((event) => event.event === "guard" && event.action === "validate"), + ).toBeGreaterThan(result.events.map((event) => event.event).lastIndexOf("move")); + expect(result.drainLeftBehind).toBe(false); + }); + + it("preserves a drain that another operator already owned", () => { + const result = runHermesRestore({ + stateDirs: ["scripts", "cron", "workspace"], + preexistingDrain: true, + }); + + expect(result.restore.success).toBe(true); + expect(result.guardEvents).toEqual(["begin", "assert-safe", "validate"]); + expect(result.moves.every((move) => move.drainActive)).toBe(true); + expect(result.drainLeftBehind).toBe(true); + }); + + it("does not create a drain marker when the gateway is inactive", () => { + const result = runHermesRestore({ + stateDirs: ["scripts", "cron", "workspace"], + gatewayRunning: false, + }); + + expect(result.restore.success).toBe(true); + expect(result.guardEvents).toEqual(["begin", "assert-safe", "validate"]); + expect(result.moves.every((move) => !move.drainActive)).toBe(true); + expect(result.drainLeftBehind).toBe(false); }); it("publishes cron job definitions only after their scripts are in place", () => { @@ -296,6 +418,36 @@ describe("Hermes cron state restore", () => { expect(result.restoredCronJob).toBe("old cron\n"); expect(result.stagingLeftBehind).toBe(false); expect(result.rollbackLeftBehind).toBe(false); + expect(result.guardEvents.at(-1)).toBe("release"); + expect(result.drainLeftBehind).toBe(false); + }); + + it("rolls back instead of resuming when restored enabled-job scripts fail validation", () => { + const result = runHermesRestore({ + stateDirs: ["scripts", "workspace", "cron"], + guardValidationFails: true, + seedExistingState: true, + }); + + expect(result.restore.success).toBe(false); + expect(result.guardEvents).toEqual(["begin", "assert-safe", "validate", "release"]); + expect(result.restoredScript).toBe("old script\n"); + expect(result.restoredCronJob).toBe("old cron\n"); + expect(result.rollbackLeftBehind).toBe(false); + expect(result.drainLeftBehind).toBe(false); + }); + + it("rejects an enabled restored job whose script is absent", () => { + const result = runHermesRestore({ + stateDirs: ["scripts", "workspace", "cron"], + missingRestoredScript: true, + seedExistingState: true, + }); + + expect(result.restore.success).toBe(false); + expect(result.restoredScript).toBe("old script\n"); + expect(result.restoredCronJob).toBe("old cron\n"); + expect(result.drainLeftBehind).toBe(false); }); it("preserves the recovery tree when rolling the original state back fails", () => { @@ -312,6 +464,8 @@ describe("Hermes cron state restore", () => { expect(result.rollbackScript).toBe("old script\n"); expect(result.rollbackLeftBehind).toBe(true); expect(result.stagingLeftBehind).toBe(false); + expect(result.guardEvents).not.toContain("release"); + expect(result.drainLeftBehind).toBe(true); }); it("refuses a new restore while an unrecovered rollback tree exists", () => { @@ -336,7 +490,7 @@ describe("Hermes cron state restore", () => { expect(result.restore.success).toBe(false); expect(result.restoredScript).toBe("#!/bin/bash\necho ok\n"); - expect(result.restoredCronJob).toBe('{"jobs":[{"enabled":true}]}\n'); + expect(result.restoredCronJob).toBe('{"jobs":[{"enabled":true,"script":"digest.sh"}]}\n'); expect(result.rollbackScript).toBe("old script\n"); expect(result.rollbackLeftBehind).toBe(true); expect(result.stagingLeftBehind).toBe(false); diff --git a/src/lib/state/sandbox.ts b/src/lib/state/sandbox.ts index b39f2762420..78b63992a7e 100644 --- a/src/lib/state/sandbox.ts +++ b/src/lib/state/sandbox.ts @@ -1351,6 +1351,9 @@ export function backupSandboxState(sandboxName: string, options: BackupOptions = const RESTORE_STAGING_DIR = ".nemoclaw-restore-staging"; const RESTORE_ROLLBACK_DIR = ".nemoclaw-restore-rollback"; +const HERMES_RESTORE_CRON_GUARD = "/usr/local/lib/nemoclaw/hermes-restore-cron-guard.py"; +const HERMES_RESTORE_DRAIN_TIMEOUT_SECONDS = 60; +const HERMES_SCHEDULED_WORK_STATE_DIRS = new Set(["cron", "scripts"]); // Job definitions in these directories are picked up by a gateway that keeps // running throughout a restore, so they must land after every directory whose @@ -1364,6 +1367,26 @@ function orderStateDirsForRestore(stateDirs: readonly string[]): string[] { ]; } +function buildHermesRestoreGuardCommand( + action: "begin" | "assert-safe" | "validate" | "release", + dir: string, +): string { + const command = [shellQuote(HERMES_RESTORE_CRON_GUARD), action, "--home", shellQuote(dir)]; + if (action === "begin") { + command.push("--timeout", String(HERMES_RESTORE_DRAIN_TIMEOUT_SECONDS)); + } else if (action === "release") { + command.push("--token", '"$drain_token"'); + } + return command.join(" "); +} + +function buildHermesDrainReleaseCommand(dir: string): string { + return ( + 'case "$drain_token" in nemoclaw-state-restore:*) ' + + `${buildHermesRestoreGuardCommand("release", dir)} && drain_token= ;; *) : ;; esac` + ); +} + /** * Extract a restore archive beside the state directories, then move each * directory into place. @@ -1377,6 +1400,7 @@ function buildStagedRestoreCommand( dir: string, stateDirs: readonly string[], staleContentDirs: readonly string[] = [], + options: { quiesceHermesScheduledWork?: boolean } = {}, ): string { const staging = `${dir}/${RESTORE_STAGING_DIR}`; const rollback = `${dir}/${RESTORE_ROLLBACK_DIR}`; @@ -1392,6 +1416,10 @@ function buildStagedRestoreCommand( const transitionDirs = orderStateDirsForRestore([ ...new Set([...stateDirs, ...staleContentDirs]), ]); + const quiesceHermesScheduledWork = + options.quiesceHermesScheduledWork === true && + transitionDirs.some((stateDir) => HERMES_SCHEDULED_WORK_STATE_DIRS.has(stateDir)); + const releaseHermesDrain = buildHermesDrainReleaseCommand(dir); const rollbackCommands = transitionDirs .map((stateDir, index) => ({ stateDir, index })) .reverse() @@ -1415,6 +1443,15 @@ function buildStagedRestoreCommand( "rollback_status=0", `if [ "$status" -ne 0 ] && [ "$transaction_committed" -ne 1 ]; then ` + `${rollbackCommands.join("; ")}; fi`, + quiesceHermesScheduledWork + ? `if [ "$rollback_status" -eq 0 ]; then ${releaseHermesDrain}; ` + + 'if [ "$?" -ne 0 ]; then status=1; fi; ' + + 'elif [ -n "$drain_token" ] && [ "$drain_token" != inactive ] && ' + + '[ "$drain_token" != preserved ]; then ' + + `echo ${shellQuote( + "Hermes restore rollback failed; preserving the scheduler drain", + )} >&2; fi` + : ":", removeStaging, `if [ "$status" -ne 0 ] && [ "$transaction_committed" -eq 1 ]; then ` + `echo ${shellQuote( @@ -1432,6 +1469,14 @@ function buildStagedRestoreCommand( `mkdir -p -- ${quotedRollback}`, `tar --no-same-owner -xf - -C ${quotedStaging}`, ]; + if (quiesceHermesScheduledWork) { + commands.push( + `drain_token="$(${buildHermesRestoreGuardCommand("begin", dir)})"`, + 'case "$drain_token" in inactive|preserved|nemoclaw-state-restore:*) : ;; ' + + `*) echo ${shellQuote("Hermes restore guard returned an invalid drain token")} >&2; false ;; esac`, + buildHermesRestoreGuardCommand("assert-safe", dir), + ); + } for (const [index, stateDir] of transitionDirs.entries()) { const target = shellQuote(`${dir}/${stateDir}`); const rollbackTarget = shellQuote(`${rollback}/${stateDir}`); @@ -1449,13 +1494,19 @@ function buildStagedRestoreCommand( commands.push(`mv -- ${shellQuote(`${staging}/${stateDir}`)} ${target}`); } } + if (quiesceHermesScheduledWork) { + commands.push(buildHermesRestoreGuardCommand("validate", dir)); + } commands.push("transaction_committed=1"); + if (quiesceHermesScheduledWork) { + commands.push(releaseHermesDrain); + } commands.push(removeRollback); // A failed extraction or move ends the chain. The EXIT trap restores every // live directory already transitioned. If rollback itself fails, keep the // recovery tree and refuse future restores instead of deleting its only copy. return ( - `${refuseUnrecoveredRollback}; transaction_committed=0; ` + + `${refuseUnrecoveredRollback}; transaction_committed=0; drain_token=; ` + `trap ${shellQuote(cleanup)} EXIT; ${commands.join(" && ")}` ); } @@ -1803,11 +1854,13 @@ function restoreSandboxStateInternal( const extractCmd = pluginRestorePlan.preservedExtensionDirs.length > 0 ? `tar --no-same-owner -xf - -C ${shellQuote(dir)}` - : buildStagedRestoreCommand(dir, localDirs, staleContentDirs); + : buildStagedRestoreCommand(dir, localDirs, staleContentDirs, { + quiesceHermesScheduledWork: manifest.agentType === "hermes", + }); const sshResult = spawnSync("ssh", [...sshArgs(configFile, sandboxName), extractCmd], { input: restoreTar, stdio: ["pipe", "pipe", "pipe"], - timeout: 120000, + timeout: manifest.agentType === "hermes" ? 180000 : 120000, }); if (sshResult.status === 0) { diff --git a/test/helpers/vitest-watch-triggers.ts b/test/helpers/vitest-watch-triggers.ts index 86390a961a1..4aed3ce496d 100644 --- a/test/helpers/vitest-watch-triggers.ts +++ b/test/helpers/vitest-watch-triggers.ts @@ -75,6 +75,10 @@ export const vitestWatchTriggerPatterns: VitestWatchTriggerPattern[] = [ pattern: /(?:^|\/)agents\/hermes\/(?:mcp-config-transaction|runtime-config-guard)\.py$/, testsToRun: runTests("src/lib/actions/sandbox/gateway-restart-hermes-drift.test.ts"), }, + { + pattern: /(?:^|\/)agents\/hermes\/restore-cron-guard\.py$/, + testsToRun: runTests("test/hermes-restore-cron-guard.test.ts"), + }, { pattern: /(?:^|\/)test\/e2e\/lib\/ci-compatible-inference\.sh$/, testsToRun: runTests("test/e2e/support/hosted-inference.test.ts"), diff --git a/test/hermes-final-image-layout.test.ts b/test/hermes-final-image-layout.test.ts index 80c1af4bb04..8a9b2b442af 100644 --- a/test/hermes-final-image-layout.test.ts +++ b/test/hermes-final-image-layout.test.ts @@ -26,6 +26,11 @@ const HERMES_INTEGRITY_FILES = [ source: "agents/hermes/finalize-tirith-marker.py", target: "/usr/local/lib/nemoclaw/finalize-tirith-marker.py", }, + { + arg: "NEMOCLAW_HERMES_RESTORE_CRON_GUARD_SHA256", + source: "agents/hermes/restore-cron-guard.py", + target: "/usr/local/lib/nemoclaw/hermes-restore-cron-guard.py", + }, { arg: "NEMOCLAW_HERMES_LANGFUSE_PATCHER_SHA256", source: "agents/hermes/patch-langfuse-credentials.mts", @@ -236,6 +241,7 @@ describe("Hermes final image layout", () => { "COPY agents/hermes/finalize-tirith-marker.py /usr/local/lib/nemoclaw/finalize-tirith-marker.py", "COPY agents/hermes/build-mcp-digest.py /usr/local/lib/nemoclaw/build-hermes-mcp-digest.py", "COPY agents/hermes/mcp-config-transaction.py /usr/local/lib/nemoclaw/hermes-mcp-config-transaction.py", + "COPY agents/hermes/restore-cron-guard.py /usr/local/lib/nemoclaw/hermes-restore-cron-guard.py", "COPY src/lib/actions/sandbox/openshell-child-visible-credentials.v0.0.85.json /usr/local/lib/nemoclaw/openshell-child-visible-credentials.v0.0.85.json", "COPY scripts/state-dir-guard.py /usr/local/lib/nemoclaw/state-dir-guard.py", "COPY nemoclaw-blueprint/scripts/*.js /usr/local/lib/nemoclaw/preloads/", @@ -328,6 +334,7 @@ describe("Hermes final image layout", () => { "/usr/local/lib/nemoclaw/validate-hermes-env-secret-boundary.py 'root:root 755'", "/usr/local/lib/nemoclaw/patch-hermes-discord-recovery-permissions.py 'root:root 755'", "/usr/local/lib/nemoclaw/patch-hermes-profile-policy-defaults.py 'root:root 755'", + "/usr/local/lib/nemoclaw/hermes-restore-cron-guard.py 'root:root 555'", "/usr/local/bin/nemoclaw-gateway-control 'root:root 700'", "/usr/local/lib/nemoclaw/preloads/sandbox-safety-net.js 'root:root 444'", "/usr/local/lib/nemoclaw/hermes-wrapper.py 'root:root 755'", diff --git a/test/hermes-restore-cron-guard.test.ts b/test/hermes-restore-cron-guard.test.ts new file mode 100644 index 00000000000..05c307cbcba --- /dev/null +++ b/test/hermes-restore-cron-guard.test.ts @@ -0,0 +1,163 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +const GUARD = path.resolve(import.meta.dirname, "..", "agents/hermes/restore-cron-guard.py"); + +function runGuardModule(source: string, args: string[] = []) { + return spawnSync("python3", ["-c", source, GUARD, ...args], { + encoding: "utf8", + timeout: 10_000, + }); +} + +const LOAD_GUARD = ` +import importlib.util +import pathlib +import sys +spec = importlib.util.spec_from_file_location("nemoclaw_restore_cron_guard", sys.argv[1]) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) +`; + +describe("Hermes restore cron guard", () => { + it("waits for the gateway drain acknowledgement that includes all active work", () => { + const result = runGuardModule(`${LOAD_GUARD} +class Drain: + marker = None + def drain_requested(self, *, home): return self.marker is not None + def write_drain_request(self, *, principal, home): self.marker = {"principal": principal} + def read_drain_request(self, *, home): return self.marker + def clear_drain_request(self, *, home): self.marker = None; return True +class Status: + states = [("running", 2), ("draining", 1), ("draining", 0)] + def get_running_pid(self): return 42 + def read_runtime_status(self): + state, active = self.states.pop(0) if len(self.states) > 1 else self.states[0] + return {"pid": 42, "gateway_state": state, "active_agents": active} + def parse_active_agents(self, value): return max(0, int(value)) +drain = Drain() +status = Status() +module._gateway_modules = lambda: (drain, status) +module.secrets.token_hex = lambda _size: "a" * 32 +module.time.sleep = lambda _seconds: None +token = module.begin_drain(pathlib.Path("/sandbox/.hermes"), 1) +print(token) +print(drain.marker["principal"]) +`); + + expect(result.status).toBe(0); + expect(result.stdout.trim().split("\n")).toEqual([ + "nemoclaw-state-restore:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "nemoclaw-state-restore:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + ]); + }); + + it("preserves an operator-owned drain and releases only its own marker", () => { + const result = runGuardModule(`${LOAD_GUARD} +class Drain: + marker = {"principal": "operator"} + cleared = 0 + def drain_requested(self, *, home): return True + def write_drain_request(self, *, principal, home): raise AssertionError("must not overwrite") + def read_drain_request(self, *, home): return self.marker + def clear_drain_request(self, *, home): self.cleared += 1; self.marker = None; return True +class Status: + def get_running_pid(self): return 42 + def read_runtime_status(self): return {"pid": 42, "gateway_state": "draining", "active_agents": 0} + def parse_active_agents(self, value): return int(value) +drain = Drain() +status = Status() +module._gateway_modules = lambda: (drain, status) +print(module.begin_drain(pathlib.Path("/sandbox/.hermes"), 1)) +module.release_drain(pathlib.Path("/sandbox/.hermes"), "nemoclaw-state-restore:" + "b" * 32) +print(drain.marker["principal"], drain.cleared) +drain.marker = {"principal": "nemoclaw-state-restore:" + "b" * 32} +module.release_drain(pathlib.Path("/sandbox/.hermes"), "nemoclaw-state-restore:" + "b" * 32) +print(drain.marker, drain.cleared) +`); + + expect(result.status).toBe(0); + expect(result.stdout.trim().split("\n")).toEqual(["preserved", "operator 0", "None 1"]); + }); + + it("accepts only enabled jobs whose referenced scripts resolve to readable files", () => { + const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cron-guard-")); + try { + const scripts = path.join(fixture, "scripts"); + const cron = path.join(fixture, "cron"); + fs.mkdirSync(scripts); + fs.mkdirSync(cron); + fs.writeFileSync(path.join(scripts, "digest.sh"), "echo ok\n", { mode: 0o600 }); + fs.writeFileSync( + path.join(cron, "jobs.json"), + JSON.stringify({ + jobs: [ + { enabled: true, script: "digest.sh" }, + { enabled: false, script: "missing-disabled.sh" }, + ], + }), + ); + + const result = runGuardModule( + `${LOAD_GUARD}\nmodule.validate_enabled_scripts(pathlib.Path(sys.argv[2]))`, + [fixture], + ); + expect(result.status).toBe(0); + } finally { + fs.rmSync(fixture, { recursive: true, force: true }); + } + }); + + it.each([ + ["missing", "missing.sh"], + ["path escape", "../outside.sh"], + ])("rejects an enabled job with a %s script", (_case, script) => { + const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cron-guard-")); + try { + fs.mkdirSync(path.join(fixture, "scripts")); + fs.mkdirSync(path.join(fixture, "cron")); + fs.writeFileSync(path.join(fixture, "outside.sh"), "echo outside\n"); + fs.writeFileSync( + path.join(fixture, "cron", "jobs.json"), + JSON.stringify({ jobs: [{ enabled: true, script }] }), + ); + + const result = runGuardModule( + `${LOAD_GUARD}\nmodule.validate_enabled_scripts(pathlib.Path(sys.argv[2]))`, + [fixture], + ); + expect(result.status).not.toBe(0); + expect(result.stderr).toMatch(/missing or unreadable|outside the scripts directory/u); + } finally { + fs.rmSync(fixture, { recursive: true, force: true }); + } + }); + + it("rejects an enabled no-agent job without a script", () => { + const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cron-guard-")); + try { + fs.mkdirSync(path.join(fixture, "scripts")); + fs.mkdirSync(path.join(fixture, "cron")); + fs.writeFileSync( + path.join(fixture, "cron", "jobs.json"), + JSON.stringify({ jobs: [{ enabled: true, no_agent: true }] }), + ); + + const result = runGuardModule( + `${LOAD_GUARD}\nmodule.validate_enabled_scripts(pathlib.Path(sys.argv[2]))`, + [fixture], + ); + expect(result.status).not.toBe(0); + expect(result.stderr).toContain("has no script"); + } finally { + fs.rmSync(fixture, { recursive: true, force: true }); + } + }); +}); diff --git a/test/vitest-watch-triggers.test.ts b/test/vitest-watch-triggers.test.ts index 599426fe744..d265facc771 100644 --- a/test/vitest-watch-triggers.test.ts +++ b/test/vitest-watch-triggers.test.ts @@ -57,6 +57,7 @@ const OPAQUE_INPUTS = [ "nemoclaw-blueprint/policies/presets/claude-code.yaml", "agents/hermes/runtime-config-guard.py", "agents/hermes/mcp-config-transaction.py", + "agents/hermes/restore-cron-guard.py", "test/e2e/lib/ci-compatible-inference.sh", "scripts/setup-jetson.sh", "scripts/e2e/sanitize-trace-timing.py", @@ -112,6 +113,9 @@ describe("Vitest opaque-input watch triggers", () => { expect(triggeredBy("agents/hermes/mcp-config-transaction.py")).toEqual([ "src/lib/actions/sandbox/gateway-restart-hermes-drift.test.ts", ]); + expect(triggeredBy("agents/hermes/restore-cron-guard.py")).toEqual([ + "test/hermes-restore-cron-guard.test.ts", + ]); expect(triggeredBy("test/e2e/lib/ci-compatible-inference.sh")).toEqual([ "test/e2e/support/hosted-inference.test.ts", ]); From 9ec10b38a78e6d2c345b86a95d02d488a7848519 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Fri, 31 Jul 2026 08:00:30 -0700 Subject: [PATCH 09/14] test(state): linearize Hermes restore fixtures Signed-off-by: Apurv Kumaria --- src/lib/state/sandbox-staged-restore.test.ts | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/lib/state/sandbox-staged-restore.test.ts b/src/lib/state/sandbox-staged-restore.test.ts index dd1a822bd0b..5922cb60627 100644 --- a/src/lib/state/sandbox-staged-restore.test.ts +++ b/src/lib/state/sandbox-staged-restore.test.ts @@ -79,11 +79,21 @@ function runHermesRestore(options: { options.seedExistingState === true ? seedExistingStateFixture : () => undefined; const rollbackSeeder = options.seedUnrecoveredRollback === true ? seedUnrecoveredRollbackFixture : () => undefined; + const drainSeeder = + options.preexistingDrain === true + ? () => fs.writeFileSync(drainMarker, "external\n") + : () => undefined; + const restoredScriptSeeder = + options.missingRestoredScript === true + ? () => undefined + : () => + fs.writeFileSync( + path.join(backupPath, "scripts", "digest.sh"), + "#!/bin/bash\necho ok\n", + ); existingStateSeeder(hermesDir); rollbackSeeder(hermesDir); - if (options.preexistingDrain === true) { - fs.writeFileSync(drainMarker, "external\n"); - } + drainSeeder(); for (const stateDir of options.stateDirs) { fs.mkdirSync(path.join(backupPath, stateDir), { recursive: true }); @@ -92,9 +102,7 @@ function runHermesRestore(options: { path.join(backupPath, "cron", "jobs.json"), '{"jobs":[{"enabled":true,"script":"digest.sh"}]}\n', ); - if (options.missingRestoredScript !== true) { - fs.writeFileSync(path.join(backupPath, "scripts", "digest.sh"), "#!/bin/bash\necho ok\n"); - } + restoredScriptSeeder(); fs.writeFileSync( path.join(backupPath, "rebuild-manifest.json"), From e1986543bca6c7c6c3289ebcb9aedf3fb8ab4286 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Fri, 31 Jul 2026 08:43:02 -0700 Subject: [PATCH 10/14] test(hermes): cover restore guard image contracts Signed-off-by: Apurv Kumaria --- src/lib/onboard/managed-startup/profile.ts | 1 + test/hermes-doctor-config-hash.test.ts | 7 +++++-- test/sandbox-provisioning.test.ts | 5 ++++- test/sandbox-rlimit-hooks.test.ts | 4 ++++ 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/lib/onboard/managed-startup/profile.ts b/src/lib/onboard/managed-startup/profile.ts index d57722afd4f..13d61575318 100644 --- a/src/lib/onboard/managed-startup/profile.ts +++ b/src/lib/onboard/managed-startup/profile.ts @@ -748,6 +748,7 @@ export const MANAGED_STARTUP_PROFILE_EXCLUDED_DOCKER_INPUTS = { { input: "NEMOCLAW_HERMES_BACKUP_SOURCE_SHA256", reason: "integrity-pin" }, { input: "NEMOCLAW_HERMES_DISCORD_RECOVERY_PATCHER_SHA256", reason: "integrity-pin" }, { input: "NEMOCLAW_HERMES_LANGFUSE_PATCHER_SHA256", reason: "integrity-pin" }, + { input: "NEMOCLAW_HERMES_RESTORE_CRON_GUARD_SHA256", reason: "integrity-pin" }, { input: "NEMOCLAW_HERMES_WRAPPER_SHA256", reason: "integrity-pin" }, { input: "NEMOCLAW_HERMES_VALIDATOR_SHA256", reason: "integrity-pin" }, { input: "NEMOCLAW_HERMES_TIRITH_FINALIZER_SHA256", reason: "integrity-pin" }, diff --git a/test/hermes-doctor-config-hash.test.ts b/test/hermes-doctor-config-hash.test.ts index ff32a19fabf..716217670cc 100644 --- a/test/hermes-doctor-config-hash.test.ts +++ b/test/hermes-doctor-config-hash.test.ts @@ -66,6 +66,7 @@ describe("Hermes doctor and config hash boundary", () => { const libDir = path.join(tmp, "usr-local-lib-nemoclaw"); const preloadsDir = path.join(libDir, "preloads"); const buildMcpDigestPath = path.join(libDir, "build-hermes-mcp-digest.py"); + const restoreCronGuardPath = path.join(libDir, "hermes-restore-cron-guard.py"); const mcpConfigTransactionPath = path.join(libDir, "hermes-mcp-config-transaction.py"); const langfuseCredentialPatcherPath = path.join( libDir, @@ -104,6 +105,7 @@ describe("Hermes doctor and config hash boundary", () => { path.join(libDir, "hermes-runtime-config-guard.py"), path.join(libDir, "finalize-tirith-marker.py"), buildMcpDigestPath, + restoreCronGuardPath, mcpConfigTransactionPath, mcpCredentialBoundaryPath, path.join(libDir, "state-dir-guard.py"), @@ -142,7 +144,7 @@ describe("Hermes doctor and config hash boundary", () => { expect(result.stderr).toBe(""); expect(fs.readFileSync(chownLogPath, "utf-8")).toBe( [ - `root:root ${path.join(binDir, "nemoclaw-gateway-control")} ${path.join(libDir, "gateway-supervisor.sh")} ${path.join(libDir, "state-dir-guard.py")} ${path.join(libDir, "managed-gateway-control.py")} ${buildMcpDigestPath} ${mcpCredentialBoundaryPath}`, + `root:root ${path.join(binDir, "nemoclaw-gateway-control")} ${path.join(libDir, "gateway-supervisor.sh")} ${path.join(libDir, "state-dir-guard.py")} ${path.join(libDir, "managed-gateway-control.py")} ${buildMcpDigestPath} ${restoreCronGuardPath} ${mcpCredentialBoundaryPath}`, `-R 0:0 ${preloadsDir}`, "", ].join("\n"), @@ -155,6 +157,7 @@ describe("Hermes doctor and config hash boundary", () => { expect(mode(langfuseCredentialPatcherPath)).toBe("444"); expect(mode(mcpCredentialBoundaryPath)).toBe("444"); expect(mode(buildMcpDigestPath)).toBe("444"); + expect(mode(restoreCronGuardPath)).toBe("555"); expect(mode(path.join(libDir, "gateway-supervisor.sh"))).toBe("444"); expect(mode(path.join(libDir, "state-dir-guard.py"))).toBe("500"); expect(mode(path.join(libDir, "managed-gateway-control.py"))).toBe("500"); @@ -255,7 +258,7 @@ describe("Hermes doctor and config hash boundary", () => { expect([mode(configPath), mode(envPath)]).toEqual(["640", "640"]); const hash = runDockerShell(hashCommand, sandboxRoot); - expect(hash.result.status).toBe(0); + expect(hash.result.status, hash.result.stderr).toBe(0); expect(hash.result.stderr).toBe(""); expect(mode(path.join(etcDir, "hermes.config-hash"))).toBe("444"); const verifyHash = spawnSync("sha256sum", ["-c", path.join(etcDir, "hermes.config-hash")], { diff --git a/test/sandbox-provisioning.test.ts b/test/sandbox-provisioning.test.ts index 1b1769b0b93..324e6463370 100644 --- a/test/sandbox-provisioning.test.ts +++ b/test/sandbox-provisioning.test.ts @@ -1107,6 +1107,7 @@ describe("Hermes sandbox provisioning", () => { const gatewayControlPath = path.join(localBin, "nemoclaw-gateway-control"); const gatewaySupervisorPath = path.join(localLib, "gateway-supervisor.sh"); const buildMcpDigestPath = path.join(localLib, "build-hermes-mcp-digest.py"); + const restoreCronGuardPath = path.join(localLib, "hermes-restore-cron-guard.py"); const mcpConfigTransactionPath = path.join(localLib, "hermes-mcp-config-transaction.py"); const langfuseCredentialPatcherPath = path.join( localLib, @@ -1128,6 +1129,7 @@ describe("Hermes sandbox provisioning", () => { path.join(localLib, "hermes-runtime-config-guard.py"), path.join(localLib, "finalize-tirith-marker.py"), buildMcpDigestPath, + restoreCronGuardPath, mcpConfigTransactionPath, mcpManifest, gatewaySupervisorPath, @@ -1156,13 +1158,14 @@ describe("Hermes sandbox provisioning", () => { expect(result.status, result.stderr).toBe(0); expect(calls).toContain( - `chown root:root ${gatewayControlPath} ${gatewaySupervisorPath} ${stateDirGuardPath} ${managedGatewayControlPath} ${buildMcpDigestPath} ${mcpManifest}`, + `chown root:root ${gatewayControlPath} ${gatewaySupervisorPath} ${stateDirGuardPath} ${managedGatewayControlPath} ${buildMcpDigestPath} ${restoreCronGuardPath} ${mcpManifest}`, ); expect((fs.statSync(gatewayControlPath).mode & 0o777).toString(8)).toBe("700"); expect((fs.statSync(mcpConfigTransactionPath).mode & 0o777).toString(8)).toBe("755"); expect((fs.statSync(langfuseCredentialPatcherPath).mode & 0o777).toString(8)).toBe("444"); expect((fs.statSync(mcpManifest).mode & 0o777).toString(8)).toBe("444"); expect((fs.statSync(buildMcpDigestPath).mode & 0o777).toString(8)).toBe("444"); + expect((fs.statSync(restoreCronGuardPath).mode & 0o777).toString(8)).toBe("555"); expect((fs.statSync(gatewaySupervisorPath).mode & 0o777).toString(8)).toBe("444"); expect((fs.statSync(stateDirGuardPath).mode & 0o777).toString(8)).toBe("500"); expect((fs.statSync(managedGatewayControlPath).mode & 0o777).toString(8)).toBe("500"); diff --git a/test/sandbox-rlimit-hooks.test.ts b/test/sandbox-rlimit-hooks.test.ts index 0b8fbb74e40..411644ee286 100644 --- a/test/sandbox-rlimit-hooks.test.ts +++ b/test/sandbox-rlimit-hooks.test.ts @@ -564,6 +564,7 @@ describe("sandbox rlimit system hooks (#2173)", () => { const runtimeGuard = path.join(localLib, "hermes-runtime-config-guard.py"); const tirithMarkerFinalizer = path.join(localLib, "finalize-tirith-marker.py"); const buildMcpDigest = path.join(localLib, "build-hermes-mcp-digest.py"); + const restoreCronGuard = path.join(localLib, "hermes-restore-cron-guard.py"); const mcpTransaction = path.join(localLib, "hermes-mcp-config-transaction.py"); const mcpCredentialBoundary = path.join( localLib, @@ -594,6 +595,7 @@ describe("sandbox rlimit system hooks (#2173)", () => { fs.writeFileSync(runtimeGuard, "# runtime guard fixture\n"); fs.writeFileSync(tirithMarkerFinalizer, "# Tirith marker finalizer fixture\n"); fs.writeFileSync(buildMcpDigest, "# build MCP digest fixture\n"); + fs.writeFileSync(restoreCronGuard, "# restore cron guard fixture\n"); fs.writeFileSync(mcpTransaction, "# MCP transaction fixture\n"); fs.writeFileSync(mcpCredentialBoundary, "{}\n"); fs.mkdirSync(preloadDir, { mode: 0o777 }); @@ -639,6 +641,7 @@ describe("sandbox rlimit system hooks (#2173)", () => { .replaceAll("/usr/local/lib/nemoclaw/hermes-runtime-config-guard.py", runtimeGuard) .replaceAll("/usr/local/lib/nemoclaw/finalize-tirith-marker.py", tirithMarkerFinalizer) .replaceAll("/usr/local/lib/nemoclaw/build-hermes-mcp-digest.py", buildMcpDigest) + .replaceAll("/usr/local/lib/nemoclaw/hermes-restore-cron-guard.py", restoreCronGuard) .replaceAll("/usr/local/lib/nemoclaw/hermes-mcp-config-transaction.py", mcpTransaction) .replaceAll( "/usr/local/lib/nemoclaw/openshell-child-visible-credentials.v0.0.85.json", @@ -675,6 +678,7 @@ describe("sandbox rlimit system hooks (#2173)", () => { expect(fs.statSync(langfuseCredentialPatcher).mode & 0o777).toBe(0o444); expect(fs.statSync(mcpCredentialBoundary).mode & 0o777).toBe(0o444); expect(fs.statSync(buildMcpDigest).mode & 0o777).toBe(0o444); + expect(fs.statSync(restoreCronGuard).mode & 0o777).toBe(0o555); expect(hardenedDir.uid).toBe(fixtureOwner.uid); expect(hardenedDir.gid).toBe(fixtureOwner.gid); expect(hardenedSafetyNet.uid).toBe(fixtureOwner.uid); From 6949d3eb37da3cd24ffe64c2a30f02deace8bdf6 Mon Sep 17 00:00:00 2001 From: Tinson Lai Date: Tue, 4 Aug 2026 06:24:40 +0000 Subject: [PATCH 11/14] fix(state): close the staged restore command builder Signed-off-by: Tinson Lai --- src/lib/state/sandbox.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib/state/sandbox.ts b/src/lib/state/sandbox.ts index 71060774f8c..9a622be39a3 100644 --- a/src/lib/state/sandbox.ts +++ b/src/lib/state/sandbox.ts @@ -1787,6 +1787,7 @@ function buildStagedRestoreCommand( `${refuseUnrecoveredRollback}; transaction_committed=0; drain_token=; ` + `trap ${shellQuote(cleanup)} EXIT; ${commands.join(" && ")}` ); +} function snapshotManifestAuthority(manifest: RebuildManifest): RebuildManifest { const normalized = { From 9affedad29724fb8c87faffc9a41ade979601f96 Mon Sep 17 00:00:00 2001 From: Tinson Lai Date: Tue, 4 Aug 2026 06:54:56 +0000 Subject: [PATCH 12/14] fix(state): hold Hermes restore drain ownership atomically Claim an exclusive restore drain marker before the gateway PID check so a gateway that starts during publication still sees the drain, release only a marker the restore owns, and validate enabled cron scripts against the gateway account rather than the restore process. Signed-off-by: Tinson Lai --- agents/hermes/restore-cron-guard.py | 92 ++++++++++++-- docs/security/best-practices.mdx | 3 +- test/hermes-restore-cron-guard.test.ts | 158 ++++++++++++++++++++++--- 3 files changed, 224 insertions(+), 29 deletions(-) diff --git a/agents/hermes/restore-cron-guard.py b/agents/hermes/restore-cron-guard.py index da67ac04d2e..525cc611d6b 100755 --- a/agents/hermes/restore-cron-guard.py +++ b/agents/hermes/restore-cron-guard.py @@ -6,9 +6,12 @@ from __future__ import annotations import argparse +import grp import json import os +import pwd import secrets +import stat import sys import time from pathlib import Path @@ -16,6 +19,8 @@ _OWNER_PREFIX = "nemoclaw-state-restore:" _POLL_INTERVAL_SECONDS = 0.1 +_OWNERSHIP_FILE = ".nemoclaw-restore-drain" +_GATEWAY_USER = "gateway" def _configure_home(raw_home: str) -> Path: @@ -54,31 +59,64 @@ def _release_owned_marker(drain_control: Any, home: Path, token: str) -> None: raise RuntimeError("Hermes restore guard could not clear its drain marker") +def _ownership_path(home: Path) -> Path: + return home / _OWNERSHIP_FILE + + +def _claim_ownership(home: Path, token: str) -> bool: + try: + descriptor = os.open( + _ownership_path(home), os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600 + ) + except FileExistsError: + return False + with os.fdopen(descriptor, "w", encoding="utf-8") as handle: + handle.write(token) + return True + + +def _release_ownership(home: Path, token: str) -> None: + path = _ownership_path(home) + try: + recorded = path.read_text(encoding="utf-8") + except FileNotFoundError: + return + if recorded == token: + path.unlink(missing_ok=True) + + def begin_drain(home: Path, timeout_seconds: float) -> str: drain_control, status = _gateway_modules() - pid = status.get_running_pid() - if pid is None: - return "inactive" + token = f"{_OWNER_PREFIX}{secrets.token_hex(16)}" + if not _claim_ownership(home, token): + raise RuntimeError("Another NemoClaw restore already owns the Hermes drain") - token = "" - if not drain_control.drain_requested(home=home): - token = f"{_OWNER_PREFIX}{secrets.token_hex(16)}" - drain_control.write_drain_request(principal=token, home=home) - - deadline = time.monotonic() + timeout_seconds + owned = False try: + if drain_control.drain_requested(home=home): + result = "preserved" + else: + drain_control.write_drain_request(principal=token, home=home) + owned = True + result = token + + deadline = time.monotonic() + timeout_seconds while time.monotonic() < deadline: live_pid = status.get_running_pid() if live_pid is None or _runtime_is_safely_drained(status, live_pid): - return token or "preserved" + if not owned: + _release_ownership(home, token) + return result time.sleep(_POLL_INTERVAL_SECONDS) except BaseException: - if token: + if owned: _release_owned_marker(drain_control, home, token) + _release_ownership(home, token) raise - if token: + if owned: _release_owned_marker(drain_control, home, token) + _release_ownership(home, token) raise TimeoutError( f"Hermes gateway did not drain active messaging, API, and cron work within {timeout_seconds:g}s" ) @@ -95,6 +133,33 @@ def assert_safely_drained(home: Path) -> None: raise RuntimeError("Hermes gateway is not safely drained for scheduled-work restore") +def _gateway_identity() -> tuple[int, set[int]] | None: + try: + entry = pwd.getpwnam(_GATEWAY_USER) + except KeyError: + return None + memberships = { + group.gr_gid for group in grp.getgrall() if _GATEWAY_USER in group.gr_mem + } + memberships.add(entry.pw_gid) + return entry.pw_uid, memberships + + +def _readable_by_gateway(script_path: Path) -> bool: + identity = _gateway_identity() + if identity is None: + return os.access(script_path, os.R_OK) + uid, gids = identity + if uid == os.geteuid(): + return os.access(script_path, os.R_OK) + info = script_path.stat() + if info.st_uid == uid: + return bool(info.st_mode & stat.S_IRUSR) + if info.st_gid in gids: + return bool(info.st_mode & stat.S_IRGRP) + return bool(info.st_mode & stat.S_IROTH) + + def _load_jobs(jobs_file: Path) -> list[Any]: if not jobs_file.exists(): return [] @@ -133,7 +198,7 @@ def validate_enabled_scripts(home: Path) -> None: raise ValueError( f"Enabled Hermes cron job at index {index} resolves outside the scripts directory" ) from error - if not script_path.is_file() or not os.access(script_path, os.R_OK): + if not script_path.is_file() or not _readable_by_gateway(script_path): raise ValueError( f"Enabled Hermes cron job at index {index} references a missing or unreadable script" ) @@ -149,6 +214,7 @@ def release_drain(home: Path, token: str) -> None: raise ValueError("Invalid Hermes restore drain ownership token") drain_control, _status = _gateway_modules() _release_owned_marker(drain_control, home, token) + _release_ownership(home, token) def _parser() -> argparse.ArgumentParser: diff --git a/docs/security/best-practices.mdx b/docs/security/best-practices.mdx index 5ad5aa13596..37cd47d8918 100644 --- a/docs/security/best-practices.mdx +++ b/docs/security/best-practices.mdx @@ -344,7 +344,8 @@ Direct edits to these files can be overwritten when NemoClaw regenerates the ima Hermes also stores runtime state such as `state.db`, logs, and platform sessions under the `.hermes` tree. Messaging sessions such as WhatsApp pairing can remain mutable by design so they survive rebuilds. -Hermes rebuild restore blocks new gateway dispatch and waits for active messaging, API, and cron work to drain before replacing cron scripts or job definitions. It keeps that drain in place until enabled script references validate or the prior state is rolled back. +Hermes rebuild restore blocks new gateway dispatch and waits for active messaging, API, and cron work to drain before replacing cron scripts or job definitions. +It keeps that drain in place until enabled script references validate or the prior state is rolled back. | Aspect | Detail | |---|---| diff --git a/test/hermes-restore-cron-guard.test.ts b/test/hermes-restore-cron-guard.test.ts index 05c307cbcba..e0ec03e3969 100644 --- a/test/hermes-restore-cron-guard.test.ts +++ b/test/hermes-restore-cron-guard.test.ts @@ -28,7 +28,10 @@ spec.loader.exec_module(module) describe("Hermes restore cron guard", () => { it("waits for the gateway drain acknowledgement that includes all active work", () => { - const result = runGuardModule(`${LOAD_GUARD} + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cron-guard-")); + try { + const result = runGuardModule( + `${LOAD_GUARD} class Drain: marker = None def drain_requested(self, *, home): return self.marker is not None @@ -47,20 +50,107 @@ status = Status() module._gateway_modules = lambda: (drain, status) module.secrets.token_hex = lambda _size: "a" * 32 module.time.sleep = lambda _seconds: None -token = module.begin_drain(pathlib.Path("/sandbox/.hermes"), 1) +token = module.begin_drain(pathlib.Path(sys.argv[2]), 1) print(token) print(drain.marker["principal"]) -`); +`, + [home], + ); - expect(result.status).toBe(0); - expect(result.stdout.trim().split("\n")).toEqual([ - "nemoclaw-state-restore:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "nemoclaw-state-restore:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - ]); + expect(result.status).toBe(0); + expect(result.stdout.trim().split("\n")).toEqual([ + "nemoclaw-state-restore:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "nemoclaw-state-restore:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + ]); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + + it("holds a drain marker while the gateway is not running", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cron-guard-")); + try { + const result = runGuardModule( + `${LOAD_GUARD} +class Drain: + marker = None + def drain_requested(self, *, home): return self.marker is not None + def write_drain_request(self, *, principal, home): self.marker = {"principal": principal} + def read_drain_request(self, *, home): return self.marker + def clear_drain_request(self, *, home): self.marker = None; return True +class Status: + def get_running_pid(self): return None + def read_runtime_status(self): return None + def parse_active_agents(self, value): return 0 +drain = Drain() +status = Status() +module._gateway_modules = lambda: (drain, status) +module.secrets.token_hex = lambda _size: "c" * 32 +home = pathlib.Path(sys.argv[2]) +token = module.begin_drain(home, 1) +print(token) +print(drain.marker["principal"]) +module.release_drain(home, token) +print(drain.marker, module._ownership_path(home).exists()) +`, + [home], + ); + + expect(result.status).toBe(0); + expect(result.stdout.trim().split("\n")).toEqual([ + "nemoclaw-state-restore:cccccccccccccccccccccccccccccccc", + "nemoclaw-state-restore:cccccccccccccccccccccccccccccccc", + "None False", + ]); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + + it("refuses a second restore while another restore owns the drain", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cron-guard-")); + try { + const result = runGuardModule( + `${LOAD_GUARD} +class Drain: + marker = None + def drain_requested(self, *, home): return self.marker is not None + def write_drain_request(self, *, principal, home): self.marker = {"principal": principal} + def read_drain_request(self, *, home): return self.marker + def clear_drain_request(self, *, home): self.marker = None; return True +class Status: + def get_running_pid(self): return None + def read_runtime_status(self): return None + def parse_active_agents(self, value): return 0 +drain = Drain() +status = Status() +module._gateway_modules = lambda: (drain, status) +home = pathlib.Path(sys.argv[2]) +first = module.begin_drain(home, 1) +try: + module.begin_drain(home, 1) +except RuntimeError as error: + print(error) +print(drain.marker["principal"] == first) +`, + [home], + ); + + expect(result.status).toBe(0); + expect(result.stdout.trim().split("\n")).toEqual([ + "Another NemoClaw restore already owns the Hermes drain", + "True", + ]); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } }); it("preserves an operator-owned drain and releases only its own marker", () => { - const result = runGuardModule(`${LOAD_GUARD} + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cron-guard-")); + try { + const result = runGuardModule( + `${LOAD_GUARD} class Drain: marker = {"principal": "operator"} cleared = 0 @@ -75,16 +165,28 @@ class Status: drain = Drain() status = Status() module._gateway_modules = lambda: (drain, status) -print(module.begin_drain(pathlib.Path("/sandbox/.hermes"), 1)) -module.release_drain(pathlib.Path("/sandbox/.hermes"), "nemoclaw-state-restore:" + "b" * 32) +home = pathlib.Path(sys.argv[2]) +print(module.begin_drain(home, 1)) +print(module._ownership_path(home).exists()) +module.release_drain(home, "nemoclaw-state-restore:" + "b" * 32) print(drain.marker["principal"], drain.cleared) drain.marker = {"principal": "nemoclaw-state-restore:" + "b" * 32} -module.release_drain(pathlib.Path("/sandbox/.hermes"), "nemoclaw-state-restore:" + "b" * 32) +module.release_drain(home, "nemoclaw-state-restore:" + "b" * 32) print(drain.marker, drain.cleared) -`); +`, + [home], + ); - expect(result.status).toBe(0); - expect(result.stdout.trim().split("\n")).toEqual(["preserved", "operator 0", "None 1"]); + expect(result.status).toBe(0); + expect(result.stdout.trim().split("\n")).toEqual([ + "preserved", + "False", + "operator 0", + "None 1", + ]); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } }); it("accepts only enabled jobs whose referenced scripts resolve to readable files", () => { @@ -140,6 +242,32 @@ print(drain.marker, drain.cleared) } }); + it("rejects an enabled job whose script the gateway account cannot read", () => { + const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cron-guard-")); + try { + fs.mkdirSync(path.join(fixture, "scripts")); + fs.mkdirSync(path.join(fixture, "cron")); + fs.writeFileSync(path.join(fixture, "scripts", "digest.sh"), "echo ok\n", { mode: 0o600 }); + fs.writeFileSync( + path.join(fixture, "cron", "jobs.json"), + JSON.stringify({ jobs: [{ enabled: true, script: "digest.sh" }] }), + ); + + const result = runGuardModule( + `${LOAD_GUARD} +import os +module._gateway_identity = lambda: (os.geteuid() + 1, set()) +module.validate_enabled_scripts(pathlib.Path(sys.argv[2])) +`, + [fixture], + ); + expect(result.status).not.toBe(0); + expect(result.stderr).toContain("missing or unreadable"); + } finally { + fs.rmSync(fixture, { recursive: true, force: true }); + } + }); + it("rejects an enabled no-agent job without a script", () => { const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cron-guard-")); try { From 6a74ed6dff1e3fde267bd7534c8febcc14762d45 Mon Sep 17 00:00:00 2001 From: Tinson Lai Date: Wed, 5 Aug 2026 03:07:54 +0000 Subject: [PATCH 13/14] fix(hermes): pin the current restore cron guard digest The image build and the final-image layout contract both compare agents/hermes/restore-cron-guard.py against the committed digest, and the pinned value still described an earlier revision of the script, so buildx stopped at the sha256sum guard before startup. Signed-off-by: Tinson Lai --- agents/hermes/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agents/hermes/Dockerfile b/agents/hermes/Dockerfile index 2b2d2958c78..23314bd9c24 100644 --- a/agents/hermes/Dockerfile +++ b/agents/hermes/Dockerfile @@ -519,7 +519,7 @@ ARG NEMOCLAW_HERMES_CLI_ADAPTER_SHA256=989edf54a8c09c6efb348600a8aa2f264c0b71408 ARG NEMOCLAW_HERMES_CLI_ADAPTER_VALIDATOR_SHA256=db4046e79e513eab67b069a8eda20167b8b65529cf26842531d2ad673c670330 ARG NEMOCLAW_HERMES_VALIDATOR_SHA256=822c7e63d068c5d09f3291350771c1a42c9686f51bfa9bc9a1f41fbe15d163b1 ARG NEMOCLAW_HERMES_TIRITH_FINALIZER_SHA256=a1e6b1c53ab297569abb87c29d15c294d729e46005bfd022136b4c447a791819 -ARG NEMOCLAW_HERMES_RESTORE_CRON_GUARD_SHA256=c224191fac19e4c7bc1d4416e2d1b243d7bd925af02fd25fa4d2c4cf6777345d +ARG NEMOCLAW_HERMES_RESTORE_CRON_GUARD_SHA256=acc9488881acd0c1753e8e3a77843022b409c7fb93cc6e4945400a2d7a694aa3 # hadolint ignore=DL4006 RUN printf '%s %s\n' \ "$NEMOCLAW_HERMES_VALIDATOR_SHA256" /usr/local/lib/nemoclaw/validate-hermes-env-secret-boundary.py \ From a7544ee88430f6700da922a36ae2216a71d8073e Mon Sep 17 00:00:00 2001 From: Tinson Lai Date: Wed, 5 Aug 2026 06:18:33 +0000 Subject: [PATCH 14/14] fix(hermes): reject cron scripts the gateway cannot reach The restore guard only checked the script's own permission bits, so a script inside a directory the gateway cannot search passed validation and then failed when the scheduler opened it. Validation now walks every directory from the scripts root to the script's parent, and the pinned image digest follows the changed script. Signed-off-by: Tinson Lai --- agents/hermes/Dockerfile | 2 +- agents/hermes/restore-cron-guard.py | 41 ++++++++++--- src/lib/state/sandbox-staged-restore.test.ts | 1 + test/hermes-restore-cron-guard.test.ts | 62 +++++++++++++++++++- 4 files changed, 96 insertions(+), 10 deletions(-) diff --git a/agents/hermes/Dockerfile b/agents/hermes/Dockerfile index 23314bd9c24..2c23be55f89 100644 --- a/agents/hermes/Dockerfile +++ b/agents/hermes/Dockerfile @@ -519,7 +519,7 @@ ARG NEMOCLAW_HERMES_CLI_ADAPTER_SHA256=989edf54a8c09c6efb348600a8aa2f264c0b71408 ARG NEMOCLAW_HERMES_CLI_ADAPTER_VALIDATOR_SHA256=db4046e79e513eab67b069a8eda20167b8b65529cf26842531d2ad673c670330 ARG NEMOCLAW_HERMES_VALIDATOR_SHA256=822c7e63d068c5d09f3291350771c1a42c9686f51bfa9bc9a1f41fbe15d163b1 ARG NEMOCLAW_HERMES_TIRITH_FINALIZER_SHA256=a1e6b1c53ab297569abb87c29d15c294d729e46005bfd022136b4c447a791819 -ARG NEMOCLAW_HERMES_RESTORE_CRON_GUARD_SHA256=acc9488881acd0c1753e8e3a77843022b409c7fb93cc6e4945400a2d7a694aa3 +ARG NEMOCLAW_HERMES_RESTORE_CRON_GUARD_SHA256=c469183e61a95dd558c115c14f084050a072dcc8ea0361f5e467937c61701caa # hadolint ignore=DL4006 RUN printf '%s %s\n' \ "$NEMOCLAW_HERMES_VALIDATOR_SHA256" /usr/local/lib/nemoclaw/validate-hermes-env-secret-boundary.py \ diff --git a/agents/hermes/restore-cron-guard.py b/agents/hermes/restore-cron-guard.py index 525cc611d6b..b9277721967 100755 --- a/agents/hermes/restore-cron-guard.py +++ b/agents/hermes/restore-cron-guard.py @@ -145,19 +145,40 @@ def _gateway_identity() -> tuple[int, set[int]] | None: return entry.pw_uid, memberships -def _readable_by_gateway(script_path: Path) -> bool: +def _accessible_by_gateway(path: Path, access_mode: int, bits: tuple[int, int, int]) -> bool: identity = _gateway_identity() if identity is None: - return os.access(script_path, os.R_OK) + return os.access(path, access_mode) uid, gids = identity if uid == os.geteuid(): - return os.access(script_path, os.R_OK) - info = script_path.stat() + return os.access(path, access_mode) + owner_bit, group_bit, other_bit = bits + info = path.stat() if info.st_uid == uid: - return bool(info.st_mode & stat.S_IRUSR) + return bool(info.st_mode & owner_bit) if info.st_gid in gids: - return bool(info.st_mode & stat.S_IRGRP) - return bool(info.st_mode & stat.S_IROTH) + return bool(info.st_mode & group_bit) + return bool(info.st_mode & other_bit) + + +def _readable_by_gateway(script_path: Path) -> bool: + return _accessible_by_gateway( + script_path, os.R_OK, (stat.S_IRUSR, stat.S_IRGRP, stat.S_IROTH) + ) + + +def _searchable_by_gateway(directory: Path) -> bool: + return _accessible_by_gateway( + directory, os.X_OK, (stat.S_IXUSR, stat.S_IXGRP, stat.S_IXOTH) + ) + + +def _enclosing_directories(scripts_dir: Path, script_parent: Path) -> list[Path]: + directories = [script_parent] + while directories[-1] != scripts_dir: + directories.append(directories[-1].parent) + directories.reverse() + return directories def _load_jobs(jobs_file: Path) -> list[Any]: @@ -202,6 +223,12 @@ def validate_enabled_scripts(home: Path) -> None: raise ValueError( f"Enabled Hermes cron job at index {index} references a missing or unreadable script" ) + for directory in _enclosing_directories(scripts_dir, script_path.parent): + if not _searchable_by_gateway(directory): + raise ValueError( + f"Enabled Hermes cron job at index {index} references a script the Hermes " + "gateway cannot reach through its directories" + ) def validate_restore(home: Path) -> None: diff --git a/src/lib/state/sandbox-staged-restore.test.ts b/src/lib/state/sandbox-staged-restore.test.ts index 5922cb60627..05e6d624fd0 100644 --- a/src/lib/state/sandbox-staged-restore.test.ts +++ b/src/lib/state/sandbox-staged-restore.test.ts @@ -453,6 +453,7 @@ describe("Hermes cron state restore", () => { }); expect(result.restore.success).toBe(false); + expect(result.guardEvents).toEqual(["begin", "assert-safe", "validate", "release"]); expect(result.restoredScript).toBe("old script\n"); expect(result.restoredCronJob).toBe("old cron\n"); expect(result.drainLeftBehind).toBe(false); diff --git a/test/hermes-restore-cron-guard.test.ts b/test/hermes-restore-cron-guard.test.ts index e0ec03e3969..769c0f22fe8 100644 --- a/test/hermes-restore-cron-guard.test.ts +++ b/test/hermes-restore-cron-guard.test.ts @@ -26,7 +26,7 @@ module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) `; -describe("Hermes restore cron guard", () => { +describe("Hermes restore cron guard (#7806)", () => { it("waits for the gateway drain acknowledgement that includes all active work", () => { const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cron-guard-")); try { @@ -208,7 +208,10 @@ print(drain.marker, drain.cleared) ); const result = runGuardModule( - `${LOAD_GUARD}\nmodule.validate_enabled_scripts(pathlib.Path(sys.argv[2]))`, + `${LOAD_GUARD} +module._gateway_identity = lambda: None +module.validate_enabled_scripts(pathlib.Path(sys.argv[2])) +`, [fixture], ); expect(result.status).toBe(0); @@ -268,6 +271,61 @@ module.validate_enabled_scripts(pathlib.Path(sys.argv[2])) } }); + it("rejects an enabled job whose script sits behind a directory the gateway cannot search", () => { + const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cron-guard-")); + const nested = path.join(fixture, "scripts", "private"); + try { + fs.mkdirSync(path.join(fixture, "scripts")); + fs.mkdirSync(path.join(fixture, "cron")); + fs.mkdirSync(nested, { mode: 0o700 }); + fs.writeFileSync(path.join(nested, "digest.sh"), "echo ok\n", { mode: 0o644 }); + fs.writeFileSync( + path.join(fixture, "cron", "jobs.json"), + JSON.stringify({ jobs: [{ enabled: true, script: "private/digest.sh" }] }), + ); + + const result = runGuardModule( + `${LOAD_GUARD} +import os +module._gateway_identity = lambda: (os.geteuid() + 1, set()) +module.validate_enabled_scripts(pathlib.Path(sys.argv[2])) +`, + [fixture], + ); + expect(result.status).not.toBe(0); + expect(result.stderr).toContain("cannot reach through its directories"); + } finally { + fs.rmSync(fixture, { recursive: true, force: true }); + } + }); + + it("accepts an enabled job whose script directories the gateway can search", () => { + const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cron-guard-")); + const nested = path.join(fixture, "scripts", "shared"); + try { + fs.mkdirSync(path.join(fixture, "scripts"), { mode: 0o755 }); + fs.mkdirSync(path.join(fixture, "cron")); + fs.mkdirSync(nested, { mode: 0o755 }); + fs.writeFileSync(path.join(nested, "digest.sh"), "echo ok\n", { mode: 0o644 }); + fs.writeFileSync( + path.join(fixture, "cron", "jobs.json"), + JSON.stringify({ jobs: [{ enabled: true, script: "shared/digest.sh" }] }), + ); + + const result = runGuardModule( + `${LOAD_GUARD} +import os +module._gateway_identity = lambda: (os.geteuid() + 1, set()) +module.validate_enabled_scripts(pathlib.Path(sys.argv[2])) +`, + [fixture], + ); + expect(result.status).toBe(0); + } finally { + fs.rmSync(fixture, { recursive: true, force: true }); + } + }); + it("rejects an enabled no-agent job without a script", () => { const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cron-guard-")); try {