fix(state): restore Hermes cron scripts before enabling restored jobs

agents/hermes/manifest.yaml

@@ -63,6 +63,10 @@ state_dirs:
   - sessions
   - skills
   - plugins
+  # Hermes confines the scripts that no_agent cron jobs run to
+  # ~/.hermes/scripts and rejects any path outside it, so cron job definitions
+  # are only restorable together with this directory.
+  - scripts
   - cron
   - logs
   - skins

docs/manage-sandboxes/backup-restore.mdx

@@ -41,7 +41,8 @@ Treat snapshot directories as private local data.
 <AgentOnly variant="hermes">
 Hermes snapshots include `SOUL.md`, the Web Dashboard profile under `.hermes/dashboard-home/`, the SQLite database behind `.hermes/state.db`, and the default kanban board in `.hermes/kanban.db`.
 The default-profile snapshot also includes cron execution history in `.hermes/runtime/cron-executions.db` and Discord replay state in `.hermes/gateway/discord_message_recovery.db`.
-NemoClaw captures cron job definitions from `.hermes/cron` as directory state.
+NemoClaw captures cron job definitions from `.hermes/cron` as directory state, together with the scripts they run from `.hermes/scripts`.
+A restore moves each state directory into place as a unit and applies `.hermes/cron` last, so the running gateway does not schedule a restored job before the script it calls is available.
 NemoClaw uses SQLite's online backup API and restores these databases through SQLite instead of copying live raw database files.
 After it replaces a database, NemoClaw opens a write transaction against the result and fails the restore when the database cannot be written.
 Named-profile cron and Discord databases under `.hermes/profiles/<name>/` use raw directory capture and can be inconsistent if a write overlaps the snapshot.

docs/reference/commands.mdx

@@ -647,7 +647,7 @@ For OpenClaw, the backed-up paths include agents, extensions, workspace, skills,
 </AgentOnly>
 <AgentOnly variant="hermes">
 
-For Hermes, the backed-up paths come from `agents/hermes/manifest.yaml`, including `/sandbox/.hermes` state such as memories, sessions, skills, plugins, cron, logs, plans, workspace, messaging platform state, `runtime/state.db`, and the default kanban board in `kanban.db`.
+For Hermes, the backed-up paths come from `agents/hermes/manifest.yaml`, including `/sandbox/.hermes` state such as memories, sessions, skills, plugins, scripts, cron, logs, plans, workspace, messaging platform state, `runtime/state.db`, and the default kanban board in `kanban.db`.
 Kanban backup does not include named boards, attachments, worker logs, scratch workspaces under `kanban/`, or external directory or worktree targets.
 
 </AgentOnly>

src/lib/state/sandbox-staged-restore.test.ts

@@ -0,0 +1,224 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+import { describe, expect, it } from "vitest";
+
+import { restoreEnvBulk } from "../../../test/helpers/env-test-helpers.js";
+import { loadAgent } from "../agent/defs.js";
+import { restoreRecreatedSandboxState } from "./sandbox.js";
+
+const HERMES_DIR = "/sandbox/.hermes";
+
+type MoveRecord = { target: string; scriptsPresent: boolean };
+
+function writeExecutable(filePath: string, source: string): void {
+  fs.writeFileSync(filePath, source, { mode: 0o755 });
+}
+
+function runHermesRestore(options: { stateDirs: string[]; movesFail?: boolean }): {
+  moves: MoveRecord[];
+  restore: ReturnType<typeof restoreRecreatedSandboxState>;
+  restoredCronJob: string | null;
+  restoredScript: string | null;
+  stagingLeftBehind: boolean;
+} {
+  const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-staged-restore-"));
+  const previousOpenshellBin = process.env.NEMOCLAW_OPENSHELL_BIN;
+  const previousPath = process.env.PATH;
+  try {
+    const binDir = path.join(fixture, "bin");
+    const shimDir = path.join(fixture, "shim");
+    const hermesDir = path.join(fixture, "sandbox-root", ".hermes");
+    const backupPath = path.join(fixture, "backup");
+    const moveLog = path.join(fixture, "move-log.jsonl");
+    fs.mkdirSync(binDir, { recursive: true });
+    fs.mkdirSync(shimDir, { recursive: true });
+    fs.mkdirSync(hermesDir, { recursive: true });
+
+    for (const stateDir of options.stateDirs) {
+      fs.mkdirSync(path.join(backupPath, stateDir), { recursive: true });
+    }
+    fs.writeFileSync(path.join(backupPath, "cron", "jobs.json"), '{"jobs":[{"enabled":true}]}\n');
+    fs.writeFileSync(path.join(backupPath, "scripts", "digest.sh"), "#!/bin/bash\necho ok\n");
+
+    fs.writeFileSync(
+      path.join(backupPath, "rebuild-manifest.json"),
+      JSON.stringify({
+        version: 1,
+        sandboxName: "alpha",
+        timestamp: "2026-07-29T12-00-00-000Z",
+        agentType: "hermes",
+        agentVersion: null,
+        expectedVersion: null,
+        stateDirs: options.stateDirs,
+        backedUpDirs: options.stateDirs,
+        stateFiles: [],
+        dir: HERMES_DIR,
+        backupPath,
+        blueprintDigest: null,
+      }),
+    );
+
+    const openshell = path.join(binDir, "openshell");
+    writeExecutable(
+      openshell,
+      `#!/usr/bin/env node
+const args = process.argv.slice(2);
+if (args[0] === "sandbox" && args[1] === "ssh-config") {
+  process.stdout.write("Host openshell-alpha\\n  HostName 127.0.0.1\\n  User sandbox\\n");
+}
+process.exit(0);
+`,
+    );
+
+    writeExecutable(
+      path.join(shimDir, "mv"),
+      options.movesFail === true
+        ? `#!/usr/bin/env node
+const fs = require("node:fs");
+const args = process.argv.slice(2);
+fs.appendFileSync(
+  ${JSON.stringify(moveLog)},
+  JSON.stringify({
+    target: args[args.length - 1],
+    scriptsPresent: fs.existsSync(${JSON.stringify(path.join(hermesDir, "scripts"))}),
+  }) + "\\n",
+);
+process.exit(1);
+`
+        : `#!/usr/bin/env node
+const fs = require("node:fs");
+const { spawnSync } = require("node:child_process");
+const args = process.argv.slice(2);
+const target = args[args.length - 1];
+fs.appendFileSync(
+  ${JSON.stringify(moveLog)},
+  JSON.stringify({
+    target,
+    scriptsPresent: fs.existsSync(${JSON.stringify(path.join(hermesDir, "scripts"))}),
+  }) + "\\n",
+);
+const result = spawnSync("/bin/mv", args, { stdio: "inherit" });
+process.exit(result.status === null ? 1 : result.status);
+`,
+    );
+
+    writeExecutable(
+      path.join(binDir, "ssh"),
+      `#!/usr/bin/env node
+const fs = require("node:fs");
+const { spawnSync } = require("node:child_process");
+const command = (process.argv[process.argv.length - 1] || "").split(${JSON.stringify(HERMES_DIR)}).join(${JSON.stringify(hermesDir)});
+function readStdin() {
+  const chunks = [];
+  for (;;) {
+    const buffer = Buffer.alloc(65536);
+    let count = 0;
+    try {
+      count = fs.readSync(0, buffer, 0, buffer.length, null);
+    } catch {
+      break;
+    }
+    if (count === 0) break;
+    chunks.push(buffer.subarray(0, count));
+  }
+  return Buffer.concat(chunks);
+}
+const result = spawnSync("sh", ["-c", command], {
+  input: readStdin(),
+  env: { ...process.env, PATH: ${JSON.stringify(shimDir)} + ":" + process.env.PATH },
+  stdio: ["pipe", "pipe", "pipe"],
+});
+process.exit(result.status === null ? 1 : result.status);
+`,
+    );
+
+    process.env.NEMOCLAW_OPENSHELL_BIN = openshell;
+    process.env.PATH = `${binDir}${path.delimiter}${previousPath ?? ""}`;
+    const restore = restoreRecreatedSandboxState("alpha", backupPath, {
+      targetAgentType: "hermes",
+    });
+
+    const moves = fs.existsSync(moveLog)
+      ? fs
+          .readFileSync(moveLog, "utf8")
+          .trim()
+          .split("\n")
+          .filter(Boolean)
+          .map((line) => JSON.parse(line) as MoveRecord)
+      : [];
+    const cronJobPath = path.join(hermesDir, "cron", "jobs.json");
+    const scriptPath = path.join(hermesDir, "scripts", "digest.sh");
+    return {
+      moves,
+      restore,
+      restoredCronJob: fs.existsSync(cronJobPath) ? fs.readFileSync(cronJobPath, "utf8") : null,
+      restoredScript: fs.existsSync(scriptPath) ? fs.readFileSync(scriptPath, "utf8") : null,
+      stagingLeftBehind: fs.existsSync(path.join(hermesDir, ".nemoclaw-restore-staging")),
+    };
+  } finally {
+    restoreEnvBulk({ NEMOCLAW_OPENSHELL_BIN: previousOpenshellBin, PATH: previousPath });
+    fs.rmSync(fixture, { recursive: true, force: true });
+  }
+}
+
+describe("Hermes cron state restore", () => {
+  it("declares the cron script directory as Hermes state", () => {
+    expect(loadAgent("hermes").stateDirs).toContain("scripts");
+  });
+
+  it("restores cron scripts alongside the job definitions that call them", () => {
+    const result = runHermesRestore({ stateDirs: ["scripts", "cron", "workspace"] });
+
+    expect(result.restore.success).toBe(true);
+    expect(result.restore.restoredDirs).toEqual(
+      expect.arrayContaining(["scripts", "cron", "workspace"]),
+    );
+    expect(result.restoredScript).toBe("#!/bin/bash\necho ok\n");
+    expect(result.restoredCronJob).toBe('{"jobs":[{"enabled":true}]}\n');
+  });
+
+  it("publishes cron job definitions only after their scripts are in place", () => {
+    const result = runHermesRestore({ stateDirs: ["scripts", "cron", "workspace"] });
+
+    const cronMove = result.moves.find((move) => move.target.endsWith("/cron"));
+    expect(cronMove?.scriptsPresent).toBe(true);
+    expect(result.moves.at(-1)?.target).toMatch(/\/cron$/);
+  });
+
+  it("applies cron last for a backup whose manifest lists it first", () => {
+    const result = runHermesRestore({ stateDirs: ["cron", "scripts", "workspace"] });
+
+    expect(result.restore.success).toBe(true);
+    expect(result.moves.at(-1)?.target).toMatch(/\/cron$/);
+    expect(result.moves.find((move) => move.target.endsWith("/cron"))?.scriptsPresent).toBe(true);
+  });
+
+  it("publishes every state directory as a unit and leaves no staging directory", () => {
+    const result = runHermesRestore({ stateDirs: ["scripts", "cron", "workspace"] });
+
+    expect(result.restore.success).toBe(true);
+    expect(result.moves.map((move) => path.basename(move.target)).sort()).toEqual([
+      "cron",
+      "scripts",
+      "workspace",
+    ]);
+    expect(result.stagingLeftBehind).toBe(false);
+  });
+
+  it("removes the archive copy when a state directory cannot be published", () => {
+    const result = runHermesRestore({
+      stateDirs: ["scripts", "cron", "workspace"],
+      movesFail: true,
+    });
+
+    expect(result.moves.map((move) => path.basename(move.target))).toEqual(["scripts"]);
+    expect(result.restoredCronJob).toBeNull();
+    expect(result.restore.success).toBe(false);
+    expect(result.stagingLeftBehind).toBe(false);
+  });
+});

src/lib/state/sandbox.ts

@@ -1349,6 +1349,48 @@ export function backupSandboxState(sandboxName: string, options: BackupOptions =
 
 // ── Restore ────────────────────────────────────────────────────────
 
+const RESTORE_STAGING_DIR = ".nemoclaw-restore-staging";
+
+// Job definitions in these directories are picked up by a gateway that keeps
+// running throughout a restore, so they must land after every directory whose
+// content those jobs can reference.
+const SCHEDULED_WORK_STATE_DIRS = new Set(["cron"]);
+
+function orderStateDirsForRestore(stateDirs: readonly string[]): string[] {
+  return [
+    ...stateDirs.filter((stateDir) => !SCHEDULED_WORK_STATE_DIRS.has(stateDir)),
+    ...stateDirs.filter((stateDir) => SCHEDULED_WORK_STATE_DIRS.has(stateDir)),
+  ];
+}
+
+/**
+ * Extract a restore archive beside the state directories, then move each
+ * directory into place.
+ *
+ * Extracting straight into the state directory publishes files one at a time to
+ * the running gateway, which can read a directory before its content is
+ * complete. A rename within the same directory publishes each restored
+ * directory whole.
+ */
+function buildStagedRestoreCommand(dir: string, stateDirs: readonly string[]): string {
+  const staging = `${dir}/${RESTORE_STAGING_DIR}`;
+  const quotedStaging = shellQuote(staging);
+  const removeStaging = `rm -rf -- ${quotedStaging}`;
+  const commands = [
+    removeStaging,
+    `mkdir -p -- ${quotedStaging}`,
+    `tar --no-same-owner -xf - -C ${quotedStaging}`,
+  ];
+  for (const stateDir of orderStateDirsForRestore(stateDirs)) {
+    const target = shellQuote(`${dir}/${stateDir}`);
+    commands.push(`rm -rf -- ${target}`);
+    commands.push(`mv -- ${shellQuote(`${staging}/${stateDir}`)} ${target}`);
+  }
+  // A failed extraction or move ends the chain, so the archive copy is removed
+  // on the way out instead of at the end of the chain.
+  return `trap ${shellQuote(removeStaging)} EXIT; ${commands.join(" && ")}`;
+}
+
 /**
  * Restore state directories into a sandbox from a prior backup.
  */
@@ -1686,7 +1728,12 @@ function restoreSandboxStateInternal(
     }
 
     if (restoreTar !== undefined) {
-      const extractCmd = `tar --no-same-owner -xf - -C ${shellQuote(dir)}`;
+      // Image-managed extensions stay in place and are merged by extracting over
+      // the live directory, so those restores cannot use the staged swap.
+      const extractCmd =
+        pluginRestorePlan.preservedExtensionDirs.length > 0
+          ? `tar --no-same-owner -xf - -C ${shellQuote(dir)}`
+          : buildStagedRestoreCommand(dir, localDirs);
       const sshResult = spawnSync("ssh", [...sshArgs(configFile, sandboxName), extractCmd], {
         input: restoreTar,
         stdio: ["pipe", "pipe", "pipe"],