fix(shields): wait for Hermes inference convergence

src/lib/shields/index.ts

@@ -70,7 +70,10 @@ const {
 }: typeof import("./permissive-runtime") = require("./permissive-runtime");
 const { cleanupTempDir } = require("../onboard/temp-files");
 const { verifyShieldsLockState }: typeof import("./verify-lock") = require("./verify-lock");
-const { relockAndReconfirm }: typeof import("./relock-reconfirm") = require("./relock-reconfirm");
+const {
+  relockAndReconfirm,
+  waitForHermesInferenceRouteConvergence,
+}: typeof import("./relock-reconfirm") = require("./relock-reconfirm");
 const {
   inspectAnyShieldsTransitionLockOwner,
   isShieldsTransitionLockUnavailable,
@@ -3840,6 +3843,7 @@ function shieldsDownWithoutHostLock(sandboxName: string, opts: ShieldsDownOpts =
   //     OpenClaw uses sandbox:sandbox 0660/2770 here so the gateway UID, which
   //     is a member of the sandbox group, can mutate runtime config.
   console.log(`  Unlocking ${target.agentName} config (${target.configPath})...`);
+  let inferenceRouteConvergenceFailed = false;
   try {
     unlockAgentConfig(
       sandboxName,
@@ -3848,6 +3852,18 @@ function shieldsDownWithoutHostLock(sandboxName: string, opts: ShieldsDownOpts =
       opts.allowLegacyHermesProtocol === true,
       protocol,
     );
+    if (target.agentName === "hermes") {
+      console.log("  Confirming Hermes inference route after policy transition...");
+      const convergence = waitForHermesInferenceRouteConvergence(sandboxName, { run });
+      if (!convergence.ok) {
+        inferenceRouteConvergenceFailed = true;
+        const status =
+          convergence.httpStatus > 0 ? `HTTP ${convergence.httpStatus}` : "unavailable";
+        throw new Error(
+          `Hermes inference route did not converge after policy transition (${status}; ${convergence.attempts} attempts)`,
+        );
+      }
+    }
   } catch (err) {
     const message = err instanceof Error ? err.message : String(err);
     const rollback = rollbackShieldsDown(
@@ -3878,9 +3894,15 @@ function shieldsDownWithoutHostLock(sandboxName: string, opts: ShieldsDownOpts =
         `  Config rollback is incomplete.${timerAuthority} Manual intervention is required.`,
       );
     }
-    console.error(
-      `  Re-run \`nemoclaw ${sandboxName} shields down\` after correcting file ownership.`,
-    );
+    if (inferenceRouteConvergenceFailed) {
+      console.error(
+        `  Recover the Hermes inference route, then re-run \`nemoclaw ${sandboxName} shields down\`.`,
+      );
+    } else {
+      console.error(
+        `  Re-run \`nemoclaw ${sandboxName} shields down\` after correcting file ownership.`,
+      );
+    }
     return failShieldsCommand(message, opts.throwOnError);
   }
 

src/lib/shields/inference-convergence.test.ts

@@ -0,0 +1,118 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { describe, expect, it, vi } from "vitest";
+
+import { waitForHermesInferenceRouteConvergence } from "./inference-convergence";
+
+function probe(status: number, output: string) {
+  return { status, stdout: output, stderr: "" };
+}
+
+describe("Hermes inference convergence after a Shields policy transition", () => {
+  it("returns on the first healthy route probe", () => {
+    const run = vi.fn((_command: readonly string[], _options: object) => probe(0, "OK 200"));
+    const sleep = vi.fn();
+
+    const result = waitForHermesInferenceRouteConvergence("hermes-box", { run, sleep });
+
+    expect(result).toEqual({ ok: true, attempts: 1, httpStatus: 200 });
+    expect(sleep).not.toHaveBeenCalled();
+    expect(run.mock.calls[0]?.[0]).toEqual([
+      "sandbox",
+      "exec",
+      "--name",
+      "hermes-box",
+      "--",
+      "sh",
+      "-c",
+      expect.stringContaining("https://inference.local/v1/models"),
+    ]);
+  });
+
+  it("waits for a transient HTTP 503 to converge", () => {
+    const run = vi
+      .fn((_command: readonly string[], _options: object) => probe(0, "OK 200"))
+      .mockReturnValueOnce(probe(0, "BROKEN 503"))
+      .mockReturnValueOnce(probe(0, "OK 200"));
+    const sleep = vi.fn();
+
+    const result = waitForHermesInferenceRouteConvergence("hermes-box", {
+      retryDelayMs: 750,
+      run,
+      sleep,
+    });
+
+    expect(result).toEqual({ ok: true, attempts: 2, httpStatus: 200 });
+    expect(sleep).toHaveBeenCalledOnce();
+    expect(sleep).toHaveBeenCalledWith(750);
+  });
+
+  it("fails after the bounded probe budget instead of reporting Shields down ready", () => {
+    const run = vi.fn((_command: readonly string[], _options: object) => probe(0, "BROKEN 503"));
+    const sleep = vi.fn();
+
+    const result = waitForHermesInferenceRouteConvergence("hermes-box", {
+      maxAttempts: 3,
+      run,
+      sleep,
+    });
+
+    expect(result).toEqual({ ok: false, attempts: 3, httpStatus: 503 });
+    expect(run).toHaveBeenCalledTimes(3);
+    expect(sleep).toHaveBeenCalledTimes(2);
+  });
+
+  it("does not accept preambled probe output as convergence even when the command succeeds", () => {
+    const run = vi.fn((_command: readonly string[], _options: object) => ({
+      status: 0,
+      stdout: "attacker preamble\nOK 200",
+      stderr: "",
+    }));
+
+    const result = waitForHermesInferenceRouteConvergence("hermes-box", {
+      maxAttempts: 1,
+      run,
+    });
+
+    expect(result).toEqual({ ok: false, attempts: 1, httpStatus: 0 });
+  });
+
+  it.each([
+    Number.NaN,
+    Number.POSITIVE_INFINITY,
+  ])("uses the bounded default attempt budget for non-finite maxAttempts (%s)", (maxAttempts) => {
+    const run = vi.fn((_command: readonly string[], _options: object) => probe(0, "BROKEN 503"));
+    const sleep = vi.fn();
+
+    const result = waitForHermesInferenceRouteConvergence("hermes-box", {
+      maxAttempts,
+      run,
+      sleep,
+    });
+
+    expect(result).toEqual({ ok: false, attempts: 4, httpStatus: 503 });
+    expect(run).toHaveBeenCalledTimes(4);
+    expect(sleep).toHaveBeenCalledTimes(3);
+  });
+
+  it.each([
+    Number.NaN,
+    Number.POSITIVE_INFINITY,
+  ])("uses the default delay for non-finite retryDelayMs (%s)", (retryDelayMs) => {
+    const run = vi
+      .fn((_command: readonly string[], _options: object) => probe(0, "OK 200"))
+      .mockReturnValueOnce(probe(0, "BROKEN 503"));
+    const sleep = vi.fn();
+
+    const result = waitForHermesInferenceRouteConvergence("hermes-box", {
+      retryDelayMs,
+      run,
+      sleep,
+    });
+
+    expect(result).toEqual({ ok: true, attempts: 2, httpStatus: 200 });
+    expect(sleep).toHaveBeenCalledOnce();
+    expect(sleep).toHaveBeenCalledWith(500);
+  });
+});

src/lib/shields/inference-convergence.ts

@@ -0,0 +1,84 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import {
+  buildSandboxInferenceRouteProbeArgs,
+  parseSandboxInferenceRouteProbeResult,
+} from "../actions/sandbox/connect-inference-route-probe";
+
+const DEFAULT_MAX_ATTEMPTS = 4;
+const DEFAULT_RETRY_DELAY_MS = 500;
+const INFERENCE_ROUTE_PROBE_TIMEOUT_MS = 10_000;
+
+interface InferenceRouteCommandResult {
+  status: number | null;
+  stdout?: string | Buffer | null;
+  stderr?: string | Buffer | null;
+}
+
+type RunInferenceRoute = (
+  command: readonly string[],
+  options: { ignoreError: true; suppressOutput: true; timeout: number },
+) => InferenceRouteCommandResult;
+
+function sleepMs(milliseconds: number): void {
+  if (milliseconds <= 0 || !Number.isFinite(milliseconds)) return;
+  Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, milliseconds);
+}
+
+export interface InferenceRouteConvergenceResult {
+  ok: boolean;
+  attempts: number;
+  httpStatus: number;
+}
+
+export interface InferenceRouteConvergenceOptions {
+  maxAttempts?: number;
+  retryDelayMs?: number;
+  run: RunInferenceRoute;
+  sleep?: (milliseconds: number) => void;
+}
+
+/**
+ * Require the Hermes inference route to be usable after a live policy
+ * replacement. OpenShell's `policy set --wait` confirms the policy version is
+ * active, but the inference proxy can briefly continue returning HTTP 503
+ * after that acknowledgement. Shields down must not report completion during
+ * that gap because callers immediately resume agent work.
+ */
+export function waitForHermesInferenceRouteConvergence(
+  sandboxName: string,
+  options: InferenceRouteConvergenceOptions,
+): InferenceRouteConvergenceResult {
+  const configuredMaxAttempts = options.maxAttempts ?? DEFAULT_MAX_ATTEMPTS;
+  const configuredRetryDelayMs = options.retryDelayMs ?? DEFAULT_RETRY_DELAY_MS;
+  const maxAttempts = Number.isFinite(configuredMaxAttempts)
+    ? Math.max(1, Math.trunc(configuredMaxAttempts))
+    : DEFAULT_MAX_ATTEMPTS;
+  const retryDelayMs = Number.isFinite(configuredRetryDelayMs)
+    ? Math.max(0, Math.trunc(configuredRetryDelayMs))
+    : DEFAULT_RETRY_DELAY_MS;
+  const sleep = options.sleep ?? sleepMs;
+  let httpStatus = 0;
+
+  for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
+    const probe = options.run(
+      buildSandboxInferenceRouteProbeArgs(sandboxName, { name: "hermes" }),
+      {
+        ignoreError: true,
+        suppressOutput: true,
+        timeout: INFERENCE_ROUTE_PROBE_TIMEOUT_MS,
+      },
+    );
+    const parsed = parseSandboxInferenceRouteProbeResult({
+      status: probe.status,
+      output: String(probe.stdout ?? ""),
+      stderr: String(probe.stderr ?? ""),
+    });
+    httpStatus = parsed.httpStatus;
+    if (parsed.healthy) return { ok: true, attempts: attempt, httpStatus };
+    if (attempt < maxAttempts) sleep(retryDelayMs);
+  }
+
+  return { ok: false, attempts: maxAttempts, httpStatus };
+}