diff --git a/test/e2e/fixtures/phases/lifecycle.ts b/test/e2e/fixtures/phases/lifecycle.ts index b63c30d0507..e6e1dc14980 100644 --- a/test/e2e/fixtures/phases/lifecycle.ts +++ b/test/e2e/fixtures/phases/lifecycle.ts @@ -36,6 +36,7 @@ const STATUS_TIMEOUT_MS = 5 * 60_000; const REBUILD_TIMEOUT_MS = 20 * 60_000; const SANDBOX_READY_ATTEMPTS = 30; const SANDBOX_READY_DELAY_MS = 5_000; +const STOPPED_SANDBOX_STATUS = "Failure layer: sandbox_container_stopped"; const USER_SERVICE_UNAVAILABLE_EXIT = 75; const NEMOCLAW_OPENSHELL_GATEWAY_USER_SERVICE_MARKER_LINE = "# NEMOCLAW_MANAGED_OPENSHELL_GATEWAY=1"; @@ -442,13 +443,10 @@ export class LifecyclePhaseFixture { // We invoke status through the host CLI client so artifacts are // captured and the command goes through the same // shellProbe/redaction layer the rest of the fixture code uses. - // Status must exit zero to prove the restored delivery path is ready; - // state-validation still verifies registry and container preservation. - const statusResult = await this.host.expectStatus(instance.sandboxName, { - artifactName: `lifecycle-post-reboot-nemoclaw-status-${instance.sandboxName}`, - env: buildAvailabilityProbeEnv(), - timeoutMs: STATUS_TIMEOUT_MS, - }); + // OpenShell can report the gateway as connected before it starts the + // preserved sandbox container. Retry only that observed transition. + // Other status failures remain terminal. + const statusResult = await this.waitForPostRebootSandboxStatus(instance.sandboxName); steps.push({ id: `nemoclaw-status:${instance.sandboxName}`, results: [statusResult], @@ -457,6 +455,33 @@ export class LifecyclePhaseFixture { return { profile: "post-reboot-recovery", steps }; } + private async waitForPostRebootSandboxStatus(sandboxName: string): Promise { + let last: ShellProbeResult | undefined; + const deadlineMs = Date.now() + STATUS_TIMEOUT_MS; + for (let attempt = 1; attempt <= SANDBOX_READY_ATTEMPTS; attempt += 1) { + const remainingMs = deadlineMs - Date.now(); + if (remainingMs <= 0) break; + last = await this.host.nemoclaw([sandboxName, "status"], { + artifactName: + `lifecycle-post-reboot-nemoclaw-status-${sandboxName}-` + + `attempt-${String(attempt).padStart(2, "0")}`, + env: buildAvailabilityProbeEnv(), + timeoutMs: Math.min(60_000, remainingMs), + }); + if (last.exitCode === 0) return last; + const detail = `${last.stdout}\n${last.stderr}`; + if (!detail.includes(STOPPED_SANDBOX_STATUS)) { + assertExitZero(last, `nemoclaw ${sandboxName} status`); + } + if (attempt < SANDBOX_READY_ATTEMPTS && Date.now() < deadlineMs) { + await sleep(Math.min(SANDBOX_READY_DELAY_MS, deadlineMs - Date.now())); + } + } + if (!last) throw new Error(`nemoclaw ${sandboxName} status did not run before its deadline`); + assertExitZero(last, `nemoclaw ${sandboxName} status`); + return last; + } + private async ensureOpenShellGatewayUserService(): Promise { const result = await this.host.command( "bash", diff --git a/test/e2e/live/gateway-guard-recovery.test.ts b/test/e2e/live/gateway-guard-recovery.test.ts index ab658d9c662..23757768bcb 100644 --- a/test/e2e/live/gateway-guard-recovery.test.ts +++ b/test/e2e/live/gateway-guard-recovery.test.ts @@ -33,8 +33,8 @@ * - Deliberately out of scope for this merge gate: physical DGX Spark / * GB10 / aarch64 hardware, provider breadth beyond `cloud-openclaw`, and * destructive host reboot / OOM / manual `kubectl delete pod` triggers. - * The Docker-driver branch below does restart the registered sandbox - * container, then proves the legacy keepalive migration restores the + * The Docker-driver branch below restarts the registered sandbox + * container, then proves that its persisted startup command restores the * managed supervisor topology without relying on ordinary sandbox exec. * Kubernetes triggers still need a dedicated platform-runtime job. * @@ -147,7 +147,7 @@ test("gateway recovery restores /tmp guard chain after pod-recreate wipe (#2701) "wipe guard chain and gateway tree", "recover gateway through connect probe", "validate recovered guard and stable PID", - "restart legacy Docker sandbox", + "restart Docker sandbox with persisted startup command", "recover managed supervisor and inference", ], }, @@ -167,14 +167,14 @@ test("gateway recovery restores /tmp guard chain after pod-recreate wipe (#2701) await artifacts.target.declare({ id: "gateway-guard-recovery", boundary: "sandbox-lifecycle", - issues: ["#2701", "#2478", "#6635"], + issues: ["#2701", "#2478"], acceptanceCoverage: { covered: [ "production connect --probe-only recovery route", "authenticated PID 1 OpenClaw recovery supervisor", "pod-recreate-equivalent empty /tmp guard chain plus missing gateway process", - "Docker container restart with a legacy keepalive startup", - "container-identity-pinned supervisor recreation with managed health proof", + "Docker container restart with a persisted managed startup command", + "container identity preservation with managed supervisor health proof", "no rebuild required for the recovered runtime state", ], intentionallyOutOfScope: [ @@ -282,32 +282,32 @@ test("gateway recovery restores /tmp guard chain after pod-recreate wipe (#2701) expect(stablePid).toBeGreaterThan(0); - progress.phase("restart legacy Docker sandbox"); - // ── Assert #6635 legacy Docker restart recovery ──────────────── - // Fresh non-GPU OpenClaw containers on this OpenShell floor still carry the - // legacy keepalive. Restarting the container therefore kills the initial - // OpenShell workload session and deterministically leaves no managed - // supervisor. Recovery must upgrade that container through the host-side - // transaction and commit only after managed control accepts the new tree. - const originalContainerId = await findSandboxContainer(host, "legacy-restart-container-before"); - expect( - await inspectStartupCommand(host, originalContainerId, "legacy-restart-command-before"), - ).toBe("sleep infinity"); + progress.phase("restart Docker sandbox with persisted startup command"); + // A Docker restart must reuse the container and its credential-free managed + // startup command. The command must restore the supervisor without a + // container recreation transaction. + const originalContainerId = await findSandboxContainer(host, "restart-container-before"); + const originalStartupCommand = await inspectStartupCommand( + host, + originalContainerId, + "restart-command-before", + ); + expect(originalStartupCommand).toMatch(/(?:^| )nemoclaw-start$/); await host.cleanupForward(18789, { - artifactName: "legacy-restart-stop-dashboard-forward", + artifactName: "restart-stop-dashboard-forward", env: buildAvailabilityProbeEnv(), }); const restart = await host.command("docker", ["restart", originalContainerId], { - artifactName: "legacy-restart-docker-restart", + artifactName: "restart-docker-restart", env: buildAvailabilityProbeEnv(), timeoutMs: 120_000, }); expect(restart.exitCode, resultText(restart)).toBe(0); progress.phase("recover managed supervisor and inference"); - const credentialCanary = "nemoclaw-e2e-recovery-secret-6635"; + const credentialCanary = "nemoclaw-e2e-recovery-secret-restart"; const trustedRecovery = await host.nemoclaw([instance.sandboxName, "recover"], { - artifactName: "legacy-restart-trusted-recover", + artifactName: "restart-trusted-recover", env: { ...buildAvailabilityProbeEnv(), NEMOCLAW_EXTRA_PLACEHOLDER_KEYS: "CUSTOM_PROVIDER_CREDENTIAL", @@ -318,14 +318,16 @@ test("gateway recovery restores /tmp guard chain after pod-recreate wipe (#2701) }); expect(trustedRecovery.timedOut, resultText(trustedRecovery)).toBe(false); expect(trustedRecovery.exitCode, resultText(trustedRecovery)).toBe(0); - expect(resultText(trustedRecovery)).toContain("Probe complete: recovered OpenClaw gateway"); + expect(resultText(trustedRecovery)).toMatch( + /Probe complete: (?:recovered OpenClaw gateway|OpenClaw gateway is running)/, + ); - const recoveredContainerId = await findSandboxContainer(host, "legacy-restart-container-after"); - expect(recoveredContainerId).not.toBe(originalContainerId); + const recoveredContainerId = await findSandboxContainer(host, "restart-container-after"); + expect(recoveredContainerId).toBe(originalContainerId); const recoveredStartupCommand = await inspectStartupCommand( host, recoveredContainerId, - "legacy-restart-command-after", + "restart-command-after", ); expect(recoveredStartupCommand).toMatch(/(?:^| )nemoclaw-start$/); expect(recoveredStartupCommand).not.toContain("CUSTOM_PROVIDER_CREDENTIAL"); @@ -335,7 +337,7 @@ test("gateway recovery restores /tmp guard chain after pod-recreate wipe (#2701) instance.sandboxName, ["python3", "-c", SUPERVISOR_TOPOLOGY_SCRIPT], { - artifactName: "legacy-restart-managed-supervisor-topology", + artifactName: "restart-managed-supervisor-topology", env: buildAvailabilityProbeEnv(), }, ); @@ -346,7 +348,7 @@ test("gateway recovery restores /tmp guard chain after pod-recreate wipe (#2701) "curl", ["-sS", "-o", "/dev/null", "-w", "%{http_code}", "http://127.0.0.1:18789/health"], { - artifactName: "legacy-restart-forwarded-health", + artifactName: "restart-forwarded-health", env: buildAvailabilityProbeEnv(), timeoutMs: 30_000, }, @@ -362,12 +364,12 @@ test("gateway recovery restores /tmp guard chain after pod-recreate wipe (#2701) "main", "--json", "--session-id", - `e2e-6635-${Date.now()}-${process.pid}`, + `e2e-restart-${Date.now()}-${process.pid}`, "-m", "What is 6 multiplied by 7? Reply with only the integer, no extra words.", ], { - artifactName: "legacy-restart-agent-inference", + artifactName: "restart-agent-inference", env: buildAvailabilityProbeEnv(), timeoutMs: 120_000, }, diff --git a/test/e2e/support/e2e-phase-lifecycle.test.ts b/test/e2e/support/e2e-phase-lifecycle.test.ts index bbe39eb0efa..d8ac9644bc1 100644 --- a/test/e2e/support/e2e-phase-lifecycle.test.ts +++ b/test/e2e/support/e2e-phase-lifecycle.test.ts @@ -6,7 +6,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { describe, expect, expectTypeOf, it } from "vitest"; +import { afterEach, describe, expect, expectTypeOf, it, vi } from "vitest"; import { type CommandRunner, @@ -126,6 +126,10 @@ function restoreEnv(name: string, value: string | undefined): void { Object.assign(process.env, value === undefined ? {} : { [name]: value }); } +afterEach(() => { + vi.useRealTimers(); +}); + describe("LifecyclePhaseFixture.preparePostReboot", () => { it("installs OpenShell and stages the gateway user service when openshell-gateway is unavailable", async () => { const runner = new FakeRunner(); @@ -217,6 +221,41 @@ describe("LifecyclePhaseFixture.simulate post-reboot-recovery (stop-original)", ); }); + it("retries the stopped-container status until the sandbox starts", async () => { + vi.useFakeTimers(); + const runner = new FakeRunner(); + const cleanup = new FakeCleanup(); + const prepared = await preparedPostRebootFixture(runner, cleanup); + runner.enqueue(shellResult(0, "container-1\n")); // discover + runner.enqueue(shellResult(0)); // docker stop + runner.enqueue(shellResult(0)); // forward stop + runner.enqueue(shellResult(0)); // gateway stop + runner.enqueue(shellResult(0)); // pid stop + runner.enqueue(shellResult(0)); // container stop + runner.enqueue(shellResult(0)); // user service restart + runner.enqueue(shellResult(0, "Connected to nemoclaw\n")); // openshell status + runner.enqueue( + shellResult( + 1, + "Failure layer: sandbox_container_stopped — sandbox container exists but is not running.", + ), + ); + runner.enqueue(shellResult(0)); // status after OpenShell starts the container + + const simulation = prepared.simulate("post-reboot-recovery", instance()); + await vi.advanceTimersByTimeAsync(5_000); + const result = await simulation; + + expect(result.steps.at(-1)?.results[0]?.exitCode).toBe(0); + expect( + runner.calls.filter( + (call) => + call.command === "nemoclaw" && + call.args.join(" ") === "e2e-ubuntu-repo-cloud-openclaw status", + ), + ).toHaveLength(2); + }); + it("fails when no Docker container carries the OpenShell sandbox-name label", async () => { const runner = new FakeRunner(); const cleanup = new FakeCleanup();