test(e2e): stabilize managed startup recovery

test/e2e/fixtures/phases/lifecycle.ts

@@ -36,6 +36,7 @@ const STATUS_TIMEOUT_MS = 5 * 60_000;
 const REBUILD_TIMEOUT_MS = 20 * 60_000;
 const SANDBOX_READY_ATTEMPTS = 30;
 const SANDBOX_READY_DELAY_MS = 5_000;
+const STOPPED_SANDBOX_STATUS = "Failure layer: sandbox_container_stopped";
 const USER_SERVICE_UNAVAILABLE_EXIT = 75;
 const NEMOCLAW_OPENSHELL_GATEWAY_USER_SERVICE_MARKER_LINE =
   "# NEMOCLAW_MANAGED_OPENSHELL_GATEWAY=1";
@@ -442,13 +443,10 @@ export class LifecyclePhaseFixture {
     // We invoke status through the host CLI client so artifacts are
     // captured and the command goes through the same
     // shellProbe/redaction layer the rest of the fixture code uses.
-    // Status must exit zero to prove the restored delivery path is ready;
-    // state-validation still verifies registry and container preservation.
-    const statusResult = await this.host.expectStatus(instance.sandboxName, {
-      artifactName: `lifecycle-post-reboot-nemoclaw-status-${instance.sandboxName}`,
-      env: buildAvailabilityProbeEnv(),
-      timeoutMs: STATUS_TIMEOUT_MS,
-    });
+    // OpenShell can report the gateway as connected before it starts the
+    // preserved sandbox container. Retry only that observed transition.
+    // Other status failures remain terminal.
+    const statusResult = await this.waitForPostRebootSandboxStatus(instance.sandboxName);
     steps.push({
       id: `nemoclaw-status:${instance.sandboxName}`,
       results: [statusResult],
@@ -457,6 +455,33 @@ export class LifecyclePhaseFixture {
     return { profile: "post-reboot-recovery", steps };
   }
 
+  private async waitForPostRebootSandboxStatus(sandboxName: string): Promise<ShellProbeResult> {
+    let last: ShellProbeResult | undefined;
+    const deadlineMs = Date.now() + STATUS_TIMEOUT_MS;
+    for (let attempt = 1; attempt <= SANDBOX_READY_ATTEMPTS; attempt += 1) {
+      const remainingMs = deadlineMs - Date.now();
+      if (remainingMs <= 0) break;
+      last = await this.host.nemoclaw([sandboxName, "status"], {
+        artifactName:
+          `lifecycle-post-reboot-nemoclaw-status-${sandboxName}-` +
+          `attempt-${String(attempt).padStart(2, "0")}`,
+        env: buildAvailabilityProbeEnv(),
+        timeoutMs: Math.min(60_000, remainingMs),
+      });
+      if (last.exitCode === 0) return last;
+      const detail = `${last.stdout}\n${last.stderr}`;
+      if (!detail.includes(STOPPED_SANDBOX_STATUS)) {
+        assertExitZero(last, `nemoclaw ${sandboxName} status`);
+      }
+      if (attempt < SANDBOX_READY_ATTEMPTS && Date.now() < deadlineMs) {
+        await sleep(Math.min(SANDBOX_READY_DELAY_MS, deadlineMs - Date.now()));
+      }
+    }
+    if (!last) throw new Error(`nemoclaw ${sandboxName} status did not run before its deadline`);
+    assertExitZero(last, `nemoclaw ${sandboxName} status`);
+    return last;
+  }
+
   private async ensureOpenShellGatewayUserService(): Promise<UserServiceStageResult> {
     const result = await this.host.command(
       "bash",

test/e2e/live/gateway-guard-recovery.test.ts

@@ -33,8 +33,8 @@
  *   - Deliberately out of scope for this merge gate: physical DGX Spark /
  *     GB10 / aarch64 hardware, provider breadth beyond `cloud-openclaw`, and
  *     destructive host reboot / OOM / manual `kubectl delete pod` triggers.
- *     The Docker-driver branch below does restart the registered sandbox
- *     container, then proves the legacy keepalive migration restores the
+ *     The Docker-driver branch below restarts the registered sandbox
+ *     container, then proves that its persisted startup command restores the
  *     managed supervisor topology without relying on ordinary sandbox exec.
  *     Kubernetes triggers still need a dedicated platform-runtime job.
  *
@@ -147,7 +147,7 @@ test("gateway recovery restores /tmp guard chain after pod-recreate wipe (#2701)
       "wipe guard chain and gateway tree",
       "recover gateway through connect probe",
       "validate recovered guard and stable PID",
-      "restart legacy Docker sandbox",
+      "restart Docker sandbox with persisted startup command",
       "recover managed supervisor and inference",
     ],
   },
@@ -167,14 +167,14 @@ test("gateway recovery restores /tmp guard chain after pod-recreate wipe (#2701)
   await artifacts.target.declare({
     id: "gateway-guard-recovery",
     boundary: "sandbox-lifecycle",
-    issues: ["#2701", "#2478", "#6635"],
+    issues: ["#2701", "#2478"],
     acceptanceCoverage: {
       covered: [
         "production connect --probe-only recovery route",
         "authenticated PID 1 OpenClaw recovery supervisor",
         "pod-recreate-equivalent empty /tmp guard chain plus missing gateway process",
-        "Docker container restart with a legacy keepalive startup",
-        "container-identity-pinned supervisor recreation with managed health proof",
+        "Docker container restart with a persisted managed startup command",
+        "container identity preservation with managed supervisor health proof",
         "no rebuild required for the recovered runtime state",
       ],
       intentionallyOutOfScope: [
@@ -282,32 +282,32 @@ test("gateway recovery restores /tmp guard chain after pod-recreate wipe (#2701)
 
   expect(stablePid).toBeGreaterThan(0);
 
-  progress.phase("restart legacy Docker sandbox");
-  // ── Assert #6635 legacy Docker restart recovery ────────────────
-  // Fresh non-GPU OpenClaw containers on this OpenShell floor still carry the
-  // legacy keepalive. Restarting the container therefore kills the initial
-  // OpenShell workload session and deterministically leaves no managed
-  // supervisor. Recovery must upgrade that container through the host-side
-  // transaction and commit only after managed control accepts the new tree.
-  const originalContainerId = await findSandboxContainer(host, "legacy-restart-container-before");
-  expect(
-    await inspectStartupCommand(host, originalContainerId, "legacy-restart-command-before"),
-  ).toBe("sleep infinity");
+  progress.phase("restart Docker sandbox with persisted startup command");
+  // A Docker restart must reuse the container and its credential-free managed
+  // startup command. The command must restore the supervisor without a
+  // container recreation transaction.
+  const originalContainerId = await findSandboxContainer(host, "restart-container-before");
+  const originalStartupCommand = await inspectStartupCommand(
+    host,
+    originalContainerId,
+    "restart-command-before",
+  );
+  expect(originalStartupCommand).toMatch(/(?:^| )nemoclaw-start$/);
   await host.cleanupForward(18789, {
-    artifactName: "legacy-restart-stop-dashboard-forward",
+    artifactName: "restart-stop-dashboard-forward",
     env: buildAvailabilityProbeEnv(),
   });
   const restart = await host.command("docker", ["restart", originalContainerId], {
-    artifactName: "legacy-restart-docker-restart",
+    artifactName: "restart-docker-restart",
     env: buildAvailabilityProbeEnv(),
     timeoutMs: 120_000,
   });
   expect(restart.exitCode, resultText(restart)).toBe(0);
 
   progress.phase("recover managed supervisor and inference");
-  const credentialCanary = "nemoclaw-e2e-recovery-secret-6635";
+  const credentialCanary = "nemoclaw-e2e-recovery-secret-restart";
   const trustedRecovery = await host.nemoclaw([instance.sandboxName, "recover"], {
-    artifactName: "legacy-restart-trusted-recover",
+    artifactName: "restart-trusted-recover",
     env: {
       ...buildAvailabilityProbeEnv(),
       NEMOCLAW_EXTRA_PLACEHOLDER_KEYS: "CUSTOM_PROVIDER_CREDENTIAL",
@@ -318,14 +318,16 @@ test("gateway recovery restores /tmp guard chain after pod-recreate wipe (#2701)
   });
   expect(trustedRecovery.timedOut, resultText(trustedRecovery)).toBe(false);
   expect(trustedRecovery.exitCode, resultText(trustedRecovery)).toBe(0);
-  expect(resultText(trustedRecovery)).toContain("Probe complete: recovered OpenClaw gateway");
+  expect(resultText(trustedRecovery)).toMatch(
+    /Probe complete: (?:recovered OpenClaw gateway|OpenClaw gateway is running)/,
+  );
 
-  const recoveredContainerId = await findSandboxContainer(host, "legacy-restart-container-after");
-  expect(recoveredContainerId).not.toBe(originalContainerId);
+  const recoveredContainerId = await findSandboxContainer(host, "restart-container-after");
+  expect(recoveredContainerId).toBe(originalContainerId);
   const recoveredStartupCommand = await inspectStartupCommand(
     host,
     recoveredContainerId,
-    "legacy-restart-command-after",
+    "restart-command-after",
   );
   expect(recoveredStartupCommand).toMatch(/(?:^| )nemoclaw-start$/);
   expect(recoveredStartupCommand).not.toContain("CUSTOM_PROVIDER_CREDENTIAL");
@@ -335,7 +337,7 @@ test("gateway recovery restores /tmp guard chain after pod-recreate wipe (#2701)
     instance.sandboxName,
     ["python3", "-c", SUPERVISOR_TOPOLOGY_SCRIPT],
     {
-      artifactName: "legacy-restart-managed-supervisor-topology",
+      artifactName: "restart-managed-supervisor-topology",
       env: buildAvailabilityProbeEnv(),
     },
   );
@@ -346,7 +348,7 @@ test("gateway recovery restores /tmp guard chain after pod-recreate wipe (#2701)
     "curl",
     ["-sS", "-o", "/dev/null", "-w", "%{http_code}", "http://127.0.0.1:18789/health"],
     {
-      artifactName: "legacy-restart-forwarded-health",
+      artifactName: "restart-forwarded-health",
       env: buildAvailabilityProbeEnv(),
       timeoutMs: 30_000,
     },
@@ -362,12 +364,12 @@ test("gateway recovery restores /tmp guard chain after pod-recreate wipe (#2701)
       "main",
       "--json",
       "--session-id",
-      `e2e-6635-${Date.now()}-${process.pid}`,
+      `e2e-restart-${Date.now()}-${process.pid}`,
       "-m",
       "What is 6 multiplied by 7? Reply with only the integer, no extra words.",
     ],
     {
-      artifactName: "legacy-restart-agent-inference",
+      artifactName: "restart-agent-inference",
       env: buildAvailabilityProbeEnv(),
       timeoutMs: 120_000,
     },

test/e2e/support/e2e-phase-lifecycle.test.ts

@@ -6,7 +6,7 @@ import fs from "node:fs";
 import os from "node:os";
 import path from "node:path";
 
-import { describe, expect, expectTypeOf, it } from "vitest";
+import { afterEach, describe, expect, expectTypeOf, it, vi } from "vitest";
 
 import {
   type CommandRunner,
@@ -126,6 +126,10 @@ function restoreEnv(name: string, value: string | undefined): void {
   Object.assign(process.env, value === undefined ? {} : { [name]: value });
 }
 
+afterEach(() => {
+  vi.useRealTimers();
+});
+
 describe("LifecyclePhaseFixture.preparePostReboot", () => {
   it("installs OpenShell and stages the gateway user service when openshell-gateway is unavailable", async () => {
     const runner = new FakeRunner();
@@ -217,6 +221,41 @@ describe("LifecyclePhaseFixture.simulate post-reboot-recovery (stop-original)",
     );
   });
 
+  it("retries the stopped-container status until the sandbox starts", async () => {
+    vi.useFakeTimers();
+    const runner = new FakeRunner();
+    const cleanup = new FakeCleanup();
+    const prepared = await preparedPostRebootFixture(runner, cleanup);
+    runner.enqueue(shellResult(0, "container-1\n")); // discover
+    runner.enqueue(shellResult(0)); // docker stop
+    runner.enqueue(shellResult(0)); // forward stop
+    runner.enqueue(shellResult(0)); // gateway stop
+    runner.enqueue(shellResult(0)); // pid stop
+    runner.enqueue(shellResult(0)); // container stop
+    runner.enqueue(shellResult(0)); // user service restart
+    runner.enqueue(shellResult(0, "Connected to nemoclaw\n")); // openshell status
+    runner.enqueue(
+      shellResult(
+        1,
+        "Failure layer: sandbox_container_stopped — sandbox container exists but is not running.",
+      ),
+    );
+    runner.enqueue(shellResult(0)); // status after OpenShell starts the container
+
+    const simulation = prepared.simulate("post-reboot-recovery", instance());
+    await vi.advanceTimersByTimeAsync(5_000);
+    const result = await simulation;
+
+    expect(result.steps.at(-1)?.results[0]?.exitCode).toBe(0);
+    expect(
+      runner.calls.filter(
+        (call) =>
+          call.command === "nemoclaw" &&
+          call.args.join(" ") === "e2e-ubuntu-repo-cloud-openclaw status",
+      ),
+    ).toHaveLength(2);
+  });
+
   it("fails when no Docker container carries the OpenShell sandbox-name label", async () => {
     const runner = new FakeRunner();
     const cleanup = new FakeCleanup();