fix(e2e): recover current lifecycle regressions

src/lib/actions/sandbox/process-recovery.test.ts

@@ -37,6 +37,8 @@ const OPENSHELL_RELAY_TARGET_NOT_FOUND_STDERR = `Error:   × code: 'The service
 const OPENSHELL_RELAY_TARGET_REFUSED_STDERR = `Error:   × code: 'The service is currently unavailable', message: "Connection
   │ refused (os error 111)"
 `;
+const OPENSHELL_TRANSIENT_ERROR_PHASE_STDERR =
+  "Error: sandbox 'recreated-box' is not ready (phase: Error); wait for it to reach Ready state.\n";
 
 describe("recreated sandbox OpenShell readiness", () => {
   afterEach(() => {
@@ -80,6 +82,33 @@ describe("recreated sandbox OpenShell readiness", () => {
     expect(sleeps).toEqual([3, 3]);
   });
 
+  it("retries the same-sandbox Error phase until OpenShell accepts the sandbox", () => {
+    const captureOpenshellImpl = vi
+      .fn()
+      .mockReturnValueOnce({
+        status: 1,
+        output: OPENSHELL_TRANSIENT_ERROR_PHASE_STDERR.trim(),
+        stdout: "",
+        stderr: OPENSHELL_TRANSIENT_ERROR_PHASE_STDERR,
+      })
+      .mockReturnValueOnce({ status: 0, output: "", stdout: "", stderr: "" });
+    const beforeProbe = vi.fn(() => true);
+    const sleeps: number[] = [];
+
+    expect(
+      waitForRecreatedSandboxOpenShellReady("recreated-box", {
+        beforeProbe,
+        captureOpenshellImpl,
+        intervalSeconds: 3,
+        sleepImpl: (seconds) => sleeps.push(seconds),
+        timeoutSeconds: 30,
+      }),
+    ).toBe(true);
+    expect(beforeProbe).toHaveBeenCalledTimes(2);
+    expect(captureOpenshellImpl).toHaveBeenCalledTimes(2);
+    expect(sleeps).toEqual([3]);
+  });
+
   it("retries the exact supervisor reconnect states exposed during direct recreation", () => {
     const reconnecting = [
       OPENSHELL_SUPERVISOR_NOT_CONNECTED_STDERR,
@@ -205,6 +234,8 @@ describe("recreated sandbox OpenShell readiness", () => {
   │ relay failed: status: DeadlineExceeded, message: \\"relay requester timed
   │ out\\", details: [], metadata: MetadataMap { headers: {} }"`,
     `Error:   × code: 'The service is currently unavailable', message: "permission denied"`,
+    "Error: sandbox 'other-box' is not ready (phase: Error); wait for it to reach Ready state.",
+    "Error: sandbox 'recreated-box' is not ready (phase: Failed); wait for it to reach Ready state.",
   ])("does not retry an unrelated OpenShell error", (stderr) => {
     const captureOpenshellImpl = vi.fn(() => ({
       status: 1,

src/lib/actions/sandbox/process-recovery.ts

@@ -610,10 +610,19 @@ function hasRetryableOpenshellResultShape(result: ReturnType<typeof captureOpens
 
 function isRetryableOpenshellReRegistrationState(
   result: ReturnType<typeof captureOpenshell>,
+  sandboxName: string,
 ): boolean {
   if (!hasRetryableOpenshellResultShape(result)) return false;
   const error = normalizeOpenshellStructuredError(String(result.stderr));
   if (error === OPENSHELL_SANDBOX_NOT_READY) return true;
+  // OpenShell can publish Ready before replacement registration settles.
+  // Retry only if the readiness probe reports phase Error for this sandbox.
+  if (
+    error ===
+    `Error: sandbox '${sandboxName}' is not ready (phase: Error); wait for it to reach Ready state.`
+  ) {
+    return true;
+  }
 
   // OpenShell 0.0.85 can keep the recreated sandbox's cached phase at Ready
   // while its replacement supervisor session is still registering. The exec
@@ -763,7 +772,10 @@ function waitForRecreatedSandboxOpenShellReadyResult(
     // mutation outcome to reconcile. Treat that exact timeout as inconclusive
     // and retry behind the pinned managed-health guard on the next iteration.
     // All other unexpected OpenShell failures remain definitive.
-    if (!isRetryableOpenshellReRegistrationState(result) && !isCommandTimeout(result)) {
+    if (
+      !isRetryableOpenshellReRegistrationState(result, sandboxName) &&
+      !isCommandTimeout(result)
+    ) {
       return {
         failure: "openshell-readiness-failure",
         openshellError: lastOpenshellError,

test/e2e/fixtures/phases/lifecycle.ts

@@ -85,12 +85,14 @@ export function buildOpenShellGatewayUserServiceStageScript(): string {
     "systemctl --user daemon-reload",
     "if systemctl --user cat openshell-gateway >/dev/null 2>&1; then",
     `  printf '%s%s\\n' "$result_prefix" upstream`,
+    "  trap - EXIT",
     "  exit 0",
     "fi",
     `if [ ! -f "$unit" ] || ! grep -Fxq "$marker" "$unit"; then exit ${USER_SERVICE_UNAVAILABLE_EXIT}; fi`,
     'if [ "$had_marked_unit" -eq 0 ]; then outcome=staged; else outcome=existing; fi',
     "systemctl --user enable nemoclaw-openshell-gateway >/dev/null",
     `printf '%s%s\\n' "$result_prefix" "$outcome"`,
+    "trap - EXIT",
   ].join("\n");
 }
 

test/e2e/live/openclaw-plugin-runtime-exdev-env.ts

@@ -0,0 +1,18 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+export const RELEASE_BASELINE_TEST_SELECTOR = "release-baseline";
+export const CURRENT_LIFECYCLE_TEST_SELECTOR = "current-lifecycle";
+export const RELEASE_SANDBOX_BASE_IMAGE_REF = "ghcr.io/nvidia/nemoclaw/sandbox-base:v0.0.71";
+
+export type OpenClawPluginRuntimeExdevSelector =
+  | typeof RELEASE_BASELINE_TEST_SELECTOR
+  | typeof CURRENT_LIFECYCLE_TEST_SELECTOR;
+
+export function buildOpenClawPluginRuntimeExdevBaseImageEnv(
+  selector: OpenClawPluginRuntimeExdevSelector,
+): NodeJS.ProcessEnv {
+  return selector === RELEASE_BASELINE_TEST_SELECTOR
+    ? { NEMOCLAW_SANDBOX_BASE_IMAGE_REF: RELEASE_SANDBOX_BASE_IMAGE_REF }
+    : {};
+}

test/e2e/live/openclaw-plugin-runtime-exdev.test.ts

@@ -28,6 +28,13 @@ import { startFakeOpenAiCompatibleServer } from "../fixtures/fake-openai-compati
 import { CLI_ENTRYPOINT, REPO_ROOT } from "../fixtures/paths.ts";
 import type { TestProgress } from "../fixtures/progress.ts";
 import { parseJsonFromText } from "./json-envelope.ts";
+import {
+  buildOpenClawPluginRuntimeExdevBaseImageEnv,
+  CURRENT_LIFECYCLE_TEST_SELECTOR,
+  type OpenClawPluginRuntimeExdevSelector,
+  RELEASE_BASELINE_TEST_SELECTOR,
+  RELEASE_SANDBOX_BASE_IMAGE_REF,
+} from "./openclaw-plugin-runtime-exdev-env.ts";
 import {
   createOpenShellDriverConfigTestWrapper,
   type OpenShellComponents,
@@ -72,7 +79,6 @@ const NEMOCLAW_RELEASE_COMMIT = "e4b9111f5f0535c2fc3d6fbe8dc8dca101a6fdce";
 const NEMOCLAW_RELEASE_OPENSHELL_VERSION = "0.0.71";
 const CURRENT_OPENSHELL_VERSION = "0.0.85";
 const NEMOCLAW_SOURCE_REPOSITORY = "https://github.com/NVIDIA/NemoClaw.git";
-const SANDBOX_BASE_IMAGE_REF = "ghcr.io/nvidia/nemoclaw/sandbox-base:v0.0.71";
 const RELEASE_BUILDER_IMAGE_REF =
   "node:22-trixie-slim@sha256:2d9f5c76c8f4dd36e8f253bee5d828a83a6c09f36188f0b0414325232e0b175d";
 const CURRENT_BUILDER_IMAGE_REF =
@@ -117,9 +123,6 @@ const EXDEV_PATTERNS = [
 ];
 type WeatherFixtureVersion = "v1" | "v2" | "v3";
 
-const RELEASE_BASELINE_TEST_SELECTOR = "release-baseline";
-const CURRENT_LIFECYCLE_TEST_SELECTOR = "current-lifecycle";
-
 const GATEWAY_CATALOG_CALL_SOURCE = String.raw`
 import { Buffer } from "node:buffer";
 import { accessSync, constants, realpathSync } from "node:fs";
@@ -488,7 +491,7 @@ function createCustomPluginDockerfile(
   ).toBe(WEATHER_OPENCLAW_VERSION);
 
   const runtime = source
-    .replace(baseImageAnchor, `ARG BASE_IMAGE=${SANDBOX_BASE_IMAGE_REF}\n`)
+    .replace(baseImageAnchor, `ARG BASE_IMAGE=${RELEASE_SANDBOX_BASE_IMAGE_REF}\n`)
     .replace(builderImageAnchor, `FROM ${builderImageRef} AS builder\n`)
     .replace(runtimeAnchor, "FROM ${BASE_IMAGE} AS nemoclaw-runtime\n");
   const pluginDirName = path.basename(context.pluginDirPath);
@@ -925,6 +928,7 @@ async function startDeploymentFixture(
   artifacts: ArtifactSink,
   cleanup: CleanupRegistry,
   progress: TestProgress,
+  selector: OpenClawPluginRuntimeExdevSelector,
 ): Promise<NodeJS.ProcessEnv> {
   const fake = await startFakeOpenAiCompatibleServer({
     apiKey: "nemoclaw-exdev-dummy-key",
@@ -944,12 +948,12 @@ async function startDeploymentFixture(
   });
 
   return liveEnv({
+    ...buildOpenClawPluginRuntimeExdevBaseImageEnv(selector),
     COMPATIBLE_API_KEY: "nemoclaw-exdev-dummy-key",
     NEMOCLAW_ENDPOINT_URL: fake.baseUrl,
     NEMOCLAW_MODEL: "nemoclaw-exdev-probe",
     NEMOCLAW_PROVIDER_KEY: "nemoclaw-exdev-dummy-key",
     NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME,
-    NEMOCLAW_SANDBOX_BASE_IMAGE_REF: SANDBOX_BASE_IMAGE_REF,
     NEMOCLAW_POLICY_MODE: "skip",
     NEMOCLAW_PREFERRED_API: "openai-completions",
     NEMOCLAW_PROVIDER: "custom",
@@ -999,7 +1003,7 @@ test("the release-baseline custom plugin loads with its exact NemoClaw and OpenS
     nemoclawSourceRelease: NEMOCLAW_RELEASE_TAG,
     nemoclawSourceCommit: NEMOCLAW_RELEASE_COMMIT,
     taggedOpenshellVersion: NEMOCLAW_RELEASE_OPENSHELL_VERSION,
-    sandboxBaseImageRef: SANDBOX_BASE_IMAGE_REF,
+    sandboxBaseImageRef: RELEASE_SANDBOX_BASE_IMAGE_REF,
     openclawVersion: WEATHER_OPENCLAW_VERSION,
   });
 
@@ -1047,7 +1051,12 @@ test("the release-baseline custom plugin loads with its exact NemoClaw and OpenS
   );
   const taggedOpenShellWrapper = createOpenShellTmpfsWrapper(taggedPinnedOpenshell.cli);
   cleanup.add("remove v0.0.71 EXDEV OpenShell PATH wrapper", taggedOpenShellWrapper.remove);
-  const deploymentEnv = await startDeploymentFixture(artifacts, cleanup, progress);
+  const deploymentEnv = await startDeploymentFixture(
+    artifacts,
+    cleanup,
+    progress,
+    RELEASE_BASELINE_TEST_SELECTOR,
+  );
   const taggedSandboxEnv = withOpenShellWrapperEnv(
     deploymentEnv,
     taggedOpenShellWrapper,
@@ -1154,6 +1163,7 @@ test("the current-lifecycle custom plugin survives restart, recreation, and rebu
     regressionTargets: ["#6108", "#3513", "#3127"],
     contract: [
       "the current CLI uses OpenShell 0.0.85 for current lifecycle coverage",
+      "the current CLI selects and validates its compatible sandbox base image",
       "gateway log, runtime inspection, tools.catalog, and tools.invoke prove weather/get_weather",
       "custom-plugin v1 survives restart, recreation installs v2, and rebuild installs v3",
       "workspace state survives both onboarding recreation and rebuild",
@@ -1167,7 +1177,7 @@ test("the current-lifecycle custom plugin survives restart, recreation, and rebu
     nemoclawSourceRelease: NEMOCLAW_RELEASE_TAG,
     nemoclawSourceCommit: NEMOCLAW_RELEASE_COMMIT,
     currentOpenshellVersion: CURRENT_OPENSHELL_VERSION,
-    sandboxBaseImageRef: SANDBOX_BASE_IMAGE_REF,
+    sandboxBaseImageResolution: "current-cli",
     openclawVersion: WEATHER_OPENCLAW_VERSION,
   });
 
@@ -1218,7 +1228,12 @@ test("the current-lifecycle custom plugin survives restart, recreation, and rebu
     cleanup,
     CURRENT_BUILDER_IMAGE_REF,
   );
-  const deploymentEnv = await startDeploymentFixture(artifacts, cleanup, progress);
+  const deploymentEnv = await startDeploymentFixture(
+    artifacts,
+    cleanup,
+    progress,
+    CURRENT_LIFECYCLE_TEST_SELECTOR,
+  );
   progress.phase("install current OpenShell and onboard plugin v1");
   await stopOpenShellGatewayBeforeVersionSwitch(host, "existing");
   const pinnedOpenshell = await installAndResolvePinnedOpenShell(

test/e2e/support/lifecycle-user-service.test.ts

@@ -19,15 +19,19 @@ import {
 const installer = fileURLToPath(new URL("../../../scripts/install.sh", import.meta.url));
 
 describe("reboot lifecycle OpenShell gateway user-service fixture", () => {
-  it("stages, enables, and removes the repository service template", () => {
-    const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-lifecycle-stage-service-"));
+  it("stages, enables, and removes the repository service without installer cleanup", () => {
+    const root = fs.mkdtempSync(
+      path.join(os.tmpdir(), "nemoclaw-installer-lifecycle-stage-service-"),
+    );
     const home = path.join(root, "home");
     const configHome = path.join(root, "config");
     const bin = path.join(home, ".local", "bin");
     const log = path.join(root, "systemctl.log");
     const unit = path.join(configHome, "systemd", "user", "nemoclaw-openshell-gateway.service");
+    const installerCleanupSentinel = path.join(root, "installer-cleanup-sentinel");
 
     fs.mkdirSync(bin, { recursive: true });
+    fs.writeFileSync(installerCleanupSentinel, "fixture-owned\n");
     fs.writeFileSync(path.join(bin, "openshell-gateway"), "#!/bin/sh\n", { mode: 0o755 });
     fs.writeFileSync(
       path.join(bin, "systemctl"),
@@ -46,13 +50,15 @@ describe("reboot lifecycle OpenShell gateway user-service fixture", () => {
         PATH: `${bin}:${path.dirname(process.execPath)}:/usr/bin:/bin`,
         XDG_CONFIG_HOME: configHome,
       });
+      env.NEMOCLAW_INSTALLER_STAGED = installerCleanupSentinel;
       const staged = execFileSync(
         "bash",
         ["-lc", buildOpenShellGatewayUserServiceStageScript(), "stage-service", installer],
         { encoding: "utf8", env, killSignal: "SIGKILL", timeout: 30_000 },
       );
 
       expect(staged).toContain("NEMOCLAW_E2E_GATEWAY_USER_SERVICE=staged");
+      expect(fs.existsSync(installerCleanupSentinel)).toBe(true);
       expect(fs.readFileSync(unit, "utf8")).toContain(`ExecStart=${bin}/openshell-gateway`);
       expect(fs.statSync(unit).mode & 0o777).toBe(0o600);
 

test/e2e/support/openclaw-plugin-runtime-exdev-env.test.ts

@@ -0,0 +1,25 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { describe, expect, it } from "vitest";
+
+import {
+  buildOpenClawPluginRuntimeExdevBaseImageEnv,
+  CURRENT_LIFECYCLE_TEST_SELECTOR,
+  RELEASE_BASELINE_TEST_SELECTOR,
+  RELEASE_SANDBOX_BASE_IMAGE_REF,
+} from "../live/openclaw-plugin-runtime-exdev-env.ts";
+
+describe("OpenClaw plugin runtime EXDEV base image selection", () => {
+  it("pins the release baseline to its matching sandbox base image", () => {
+    expect(buildOpenClawPluginRuntimeExdevBaseImageEnv(RELEASE_BASELINE_TEST_SELECTOR)).toEqual({
+      NEMOCLAW_SANDBOX_BASE_IMAGE_REF: RELEASE_SANDBOX_BASE_IMAGE_REF,
+    });
+  });
+
+  it("does not override base-image resolution for the current-lifecycle test", () => {
+    expect(buildOpenClawPluginRuntimeExdevBaseImageEnv(CURRENT_LIFECYCLE_TEST_SELECTOR)).toEqual(
+      {},
+    );
+  });
+});