diff --git a/src/lib/actions/sandbox/process-recovery.test.ts b/src/lib/actions/sandbox/process-recovery.test.ts index 4c5ca21b8b7..b814b0b0b94 100644 --- a/src/lib/actions/sandbox/process-recovery.test.ts +++ b/src/lib/actions/sandbox/process-recovery.test.ts @@ -37,6 +37,8 @@ const OPENSHELL_RELAY_TARGET_NOT_FOUND_STDERR = `Error: × code: 'The service const OPENSHELL_RELAY_TARGET_REFUSED_STDERR = `Error: × code: 'The service is currently unavailable', message: "Connection │ refused (os error 111)" `; +const OPENSHELL_TRANSIENT_ERROR_PHASE_STDERR = + "Error: sandbox 'recreated-box' is not ready (phase: Error); wait for it to reach Ready state.\n"; describe("recreated sandbox OpenShell readiness", () => { afterEach(() => { @@ -80,6 +82,33 @@ describe("recreated sandbox OpenShell readiness", () => { expect(sleeps).toEqual([3, 3]); }); + it("retries the same-sandbox Error phase until OpenShell accepts the sandbox", () => { + const captureOpenshellImpl = vi + .fn() + .mockReturnValueOnce({ + status: 1, + output: OPENSHELL_TRANSIENT_ERROR_PHASE_STDERR.trim(), + stdout: "", + stderr: OPENSHELL_TRANSIENT_ERROR_PHASE_STDERR, + }) + .mockReturnValueOnce({ status: 0, output: "", stdout: "", stderr: "" }); + const beforeProbe = vi.fn(() => true); + const sleeps: number[] = []; + + expect( + waitForRecreatedSandboxOpenShellReady("recreated-box", { + beforeProbe, + captureOpenshellImpl, + intervalSeconds: 3, + sleepImpl: (seconds) => sleeps.push(seconds), + timeoutSeconds: 30, + }), + ).toBe(true); + expect(beforeProbe).toHaveBeenCalledTimes(2); + expect(captureOpenshellImpl).toHaveBeenCalledTimes(2); + expect(sleeps).toEqual([3]); + }); + it("retries the exact supervisor reconnect states exposed during direct recreation", () => { const reconnecting = [ OPENSHELL_SUPERVISOR_NOT_CONNECTED_STDERR, @@ -205,6 +234,8 @@ describe("recreated sandbox OpenShell readiness", () => { │ relay failed: status: DeadlineExceeded, message: \\"relay requester timed │ out\\", details: [], metadata: MetadataMap { headers: {} }"`, `Error: × code: 'The service is currently unavailable', message: "permission denied"`, + "Error: sandbox 'other-box' is not ready (phase: Error); wait for it to reach Ready state.", + "Error: sandbox 'recreated-box' is not ready (phase: Failed); wait for it to reach Ready state.", ])("does not retry an unrelated OpenShell error", (stderr) => { const captureOpenshellImpl = vi.fn(() => ({ status: 1, diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index 013cfa5adeb..c3f0d58372c 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -610,10 +610,19 @@ function hasRetryableOpenshellResultShape(result: ReturnType, + sandboxName: string, ): boolean { if (!hasRetryableOpenshellResultShape(result)) return false; const error = normalizeOpenshellStructuredError(String(result.stderr)); if (error === OPENSHELL_SANDBOX_NOT_READY) return true; + // OpenShell can publish Ready before replacement registration settles. + // Retry only if the readiness probe reports phase Error for this sandbox. + if ( + error === + `Error: sandbox '${sandboxName}' is not ready (phase: Error); wait for it to reach Ready state.` + ) { + return true; + } // OpenShell 0.0.85 can keep the recreated sandbox's cached phase at Ready // while its replacement supervisor session is still registering. The exec @@ -763,7 +772,10 @@ function waitForRecreatedSandboxOpenShellReadyResult( // mutation outcome to reconcile. Treat that exact timeout as inconclusive // and retry behind the pinned managed-health guard on the next iteration. // All other unexpected OpenShell failures remain definitive. - if (!isRetryableOpenshellReRegistrationState(result) && !isCommandTimeout(result)) { + if ( + !isRetryableOpenshellReRegistrationState(result, sandboxName) && + !isCommandTimeout(result) + ) { return { failure: "openshell-readiness-failure", openshellError: lastOpenshellError, diff --git a/test/e2e/fixtures/phases/lifecycle.ts b/test/e2e/fixtures/phases/lifecycle.ts index 17a3b301e96..50d6363e5e7 100644 --- a/test/e2e/fixtures/phases/lifecycle.ts +++ b/test/e2e/fixtures/phases/lifecycle.ts @@ -85,12 +85,14 @@ export function buildOpenShellGatewayUserServiceStageScript(): string { "systemctl --user daemon-reload", "if systemctl --user cat openshell-gateway >/dev/null 2>&1; then", ` printf '%s%s\\n' "$result_prefix" upstream`, + " trap - EXIT", " exit 0", "fi", `if [ ! -f "$unit" ] || ! grep -Fxq "$marker" "$unit"; then exit ${USER_SERVICE_UNAVAILABLE_EXIT}; fi`, 'if [ "$had_marked_unit" -eq 0 ]; then outcome=staged; else outcome=existing; fi', "systemctl --user enable nemoclaw-openshell-gateway >/dev/null", `printf '%s%s\\n' "$result_prefix" "$outcome"`, + "trap - EXIT", ].join("\n"); } diff --git a/test/e2e/live/openclaw-plugin-runtime-exdev-env.ts b/test/e2e/live/openclaw-plugin-runtime-exdev-env.ts new file mode 100644 index 00000000000..4e8c65a7971 --- /dev/null +++ b/test/e2e/live/openclaw-plugin-runtime-exdev-env.ts @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export const RELEASE_BASELINE_TEST_SELECTOR = "release-baseline"; +export const CURRENT_LIFECYCLE_TEST_SELECTOR = "current-lifecycle"; +export const RELEASE_SANDBOX_BASE_IMAGE_REF = "ghcr.io/nvidia/nemoclaw/sandbox-base:v0.0.71"; + +export type OpenClawPluginRuntimeExdevSelector = + | typeof RELEASE_BASELINE_TEST_SELECTOR + | typeof CURRENT_LIFECYCLE_TEST_SELECTOR; + +export function buildOpenClawPluginRuntimeExdevBaseImageEnv( + selector: OpenClawPluginRuntimeExdevSelector, +): NodeJS.ProcessEnv { + return selector === RELEASE_BASELINE_TEST_SELECTOR + ? { NEMOCLAW_SANDBOX_BASE_IMAGE_REF: RELEASE_SANDBOX_BASE_IMAGE_REF } + : {}; +} diff --git a/test/e2e/live/openclaw-plugin-runtime-exdev.test.ts b/test/e2e/live/openclaw-plugin-runtime-exdev.test.ts index 6a6e7c1a0d4..9e37facc121 100644 --- a/test/e2e/live/openclaw-plugin-runtime-exdev.test.ts +++ b/test/e2e/live/openclaw-plugin-runtime-exdev.test.ts @@ -28,6 +28,13 @@ import { startFakeOpenAiCompatibleServer } from "../fixtures/fake-openai-compati import { CLI_ENTRYPOINT, REPO_ROOT } from "../fixtures/paths.ts"; import type { TestProgress } from "../fixtures/progress.ts"; import { parseJsonFromText } from "./json-envelope.ts"; +import { + buildOpenClawPluginRuntimeExdevBaseImageEnv, + CURRENT_LIFECYCLE_TEST_SELECTOR, + type OpenClawPluginRuntimeExdevSelector, + RELEASE_BASELINE_TEST_SELECTOR, + RELEASE_SANDBOX_BASE_IMAGE_REF, +} from "./openclaw-plugin-runtime-exdev-env.ts"; import { createOpenShellDriverConfigTestWrapper, type OpenShellComponents, @@ -72,7 +79,6 @@ const NEMOCLAW_RELEASE_COMMIT = "e4b9111f5f0535c2fc3d6fbe8dc8dca101a6fdce"; const NEMOCLAW_RELEASE_OPENSHELL_VERSION = "0.0.71"; const CURRENT_OPENSHELL_VERSION = "0.0.85"; const NEMOCLAW_SOURCE_REPOSITORY = "https://github.com/NVIDIA/NemoClaw.git"; -const SANDBOX_BASE_IMAGE_REF = "ghcr.io/nvidia/nemoclaw/sandbox-base:v0.0.71"; const RELEASE_BUILDER_IMAGE_REF = "node:22-trixie-slim@sha256:2d9f5c76c8f4dd36e8f253bee5d828a83a6c09f36188f0b0414325232e0b175d"; const CURRENT_BUILDER_IMAGE_REF = @@ -117,9 +123,6 @@ const EXDEV_PATTERNS = [ ]; type WeatherFixtureVersion = "v1" | "v2" | "v3"; -const RELEASE_BASELINE_TEST_SELECTOR = "release-baseline"; -const CURRENT_LIFECYCLE_TEST_SELECTOR = "current-lifecycle"; - const GATEWAY_CATALOG_CALL_SOURCE = String.raw` import { Buffer } from "node:buffer"; import { accessSync, constants, realpathSync } from "node:fs"; @@ -488,7 +491,7 @@ function createCustomPluginDockerfile( ).toBe(WEATHER_OPENCLAW_VERSION); const runtime = source - .replace(baseImageAnchor, `ARG BASE_IMAGE=${SANDBOX_BASE_IMAGE_REF}\n`) + .replace(baseImageAnchor, `ARG BASE_IMAGE=${RELEASE_SANDBOX_BASE_IMAGE_REF}\n`) .replace(builderImageAnchor, `FROM ${builderImageRef} AS builder\n`) .replace(runtimeAnchor, "FROM ${BASE_IMAGE} AS nemoclaw-runtime\n"); const pluginDirName = path.basename(context.pluginDirPath); @@ -925,6 +928,7 @@ async function startDeploymentFixture( artifacts: ArtifactSink, cleanup: CleanupRegistry, progress: TestProgress, + selector: OpenClawPluginRuntimeExdevSelector, ): Promise { const fake = await startFakeOpenAiCompatibleServer({ apiKey: "nemoclaw-exdev-dummy-key", @@ -944,12 +948,12 @@ async function startDeploymentFixture( }); return liveEnv({ + ...buildOpenClawPluginRuntimeExdevBaseImageEnv(selector), COMPATIBLE_API_KEY: "nemoclaw-exdev-dummy-key", NEMOCLAW_ENDPOINT_URL: fake.baseUrl, NEMOCLAW_MODEL: "nemoclaw-exdev-probe", NEMOCLAW_PROVIDER_KEY: "nemoclaw-exdev-dummy-key", NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, - NEMOCLAW_SANDBOX_BASE_IMAGE_REF: SANDBOX_BASE_IMAGE_REF, NEMOCLAW_POLICY_MODE: "skip", NEMOCLAW_PREFERRED_API: "openai-completions", NEMOCLAW_PROVIDER: "custom", @@ -999,7 +1003,7 @@ test("the release-baseline custom plugin loads with its exact NemoClaw and OpenS nemoclawSourceRelease: NEMOCLAW_RELEASE_TAG, nemoclawSourceCommit: NEMOCLAW_RELEASE_COMMIT, taggedOpenshellVersion: NEMOCLAW_RELEASE_OPENSHELL_VERSION, - sandboxBaseImageRef: SANDBOX_BASE_IMAGE_REF, + sandboxBaseImageRef: RELEASE_SANDBOX_BASE_IMAGE_REF, openclawVersion: WEATHER_OPENCLAW_VERSION, }); @@ -1047,7 +1051,12 @@ test("the release-baseline custom plugin loads with its exact NemoClaw and OpenS ); const taggedOpenShellWrapper = createOpenShellTmpfsWrapper(taggedPinnedOpenshell.cli); cleanup.add("remove v0.0.71 EXDEV OpenShell PATH wrapper", taggedOpenShellWrapper.remove); - const deploymentEnv = await startDeploymentFixture(artifacts, cleanup, progress); + const deploymentEnv = await startDeploymentFixture( + artifacts, + cleanup, + progress, + RELEASE_BASELINE_TEST_SELECTOR, + ); const taggedSandboxEnv = withOpenShellWrapperEnv( deploymentEnv, taggedOpenShellWrapper, @@ -1154,6 +1163,7 @@ test("the current-lifecycle custom plugin survives restart, recreation, and rebu regressionTargets: ["#6108", "#3513", "#3127"], contract: [ "the current CLI uses OpenShell 0.0.85 for current lifecycle coverage", + "the current CLI selects and validates its compatible sandbox base image", "gateway log, runtime inspection, tools.catalog, and tools.invoke prove weather/get_weather", "custom-plugin v1 survives restart, recreation installs v2, and rebuild installs v3", "workspace state survives both onboarding recreation and rebuild", @@ -1167,7 +1177,7 @@ test("the current-lifecycle custom plugin survives restart, recreation, and rebu nemoclawSourceRelease: NEMOCLAW_RELEASE_TAG, nemoclawSourceCommit: NEMOCLAW_RELEASE_COMMIT, currentOpenshellVersion: CURRENT_OPENSHELL_VERSION, - sandboxBaseImageRef: SANDBOX_BASE_IMAGE_REF, + sandboxBaseImageResolution: "current-cli", openclawVersion: WEATHER_OPENCLAW_VERSION, }); @@ -1218,7 +1228,12 @@ test("the current-lifecycle custom plugin survives restart, recreation, and rebu cleanup, CURRENT_BUILDER_IMAGE_REF, ); - const deploymentEnv = await startDeploymentFixture(artifacts, cleanup, progress); + const deploymentEnv = await startDeploymentFixture( + artifacts, + cleanup, + progress, + CURRENT_LIFECYCLE_TEST_SELECTOR, + ); progress.phase("install current OpenShell and onboard plugin v1"); await stopOpenShellGatewayBeforeVersionSwitch(host, "existing"); const pinnedOpenshell = await installAndResolvePinnedOpenShell( diff --git a/test/e2e/support/lifecycle-user-service.test.ts b/test/e2e/support/lifecycle-user-service.test.ts index 0a50d374842..17b4d2a6581 100644 --- a/test/e2e/support/lifecycle-user-service.test.ts +++ b/test/e2e/support/lifecycle-user-service.test.ts @@ -19,15 +19,19 @@ import { const installer = fileURLToPath(new URL("../../../scripts/install.sh", import.meta.url)); describe("reboot lifecycle OpenShell gateway user-service fixture", () => { - it("stages, enables, and removes the repository service template", () => { - const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-lifecycle-stage-service-")); + it("stages, enables, and removes the repository service without installer cleanup", () => { + const root = fs.mkdtempSync( + path.join(os.tmpdir(), "nemoclaw-installer-lifecycle-stage-service-"), + ); const home = path.join(root, "home"); const configHome = path.join(root, "config"); const bin = path.join(home, ".local", "bin"); const log = path.join(root, "systemctl.log"); const unit = path.join(configHome, "systemd", "user", "nemoclaw-openshell-gateway.service"); + const installerCleanupSentinel = path.join(root, "installer-cleanup-sentinel"); fs.mkdirSync(bin, { recursive: true }); + fs.writeFileSync(installerCleanupSentinel, "fixture-owned\n"); fs.writeFileSync(path.join(bin, "openshell-gateway"), "#!/bin/sh\n", { mode: 0o755 }); fs.writeFileSync( path.join(bin, "systemctl"), @@ -46,6 +50,7 @@ describe("reboot lifecycle OpenShell gateway user-service fixture", () => { PATH: `${bin}:${path.dirname(process.execPath)}:/usr/bin:/bin`, XDG_CONFIG_HOME: configHome, }); + env.NEMOCLAW_INSTALLER_STAGED = installerCleanupSentinel; const staged = execFileSync( "bash", ["-lc", buildOpenShellGatewayUserServiceStageScript(), "stage-service", installer], @@ -53,6 +58,7 @@ describe("reboot lifecycle OpenShell gateway user-service fixture", () => { ); expect(staged).toContain("NEMOCLAW_E2E_GATEWAY_USER_SERVICE=staged"); + expect(fs.existsSync(installerCleanupSentinel)).toBe(true); expect(fs.readFileSync(unit, "utf8")).toContain(`ExecStart=${bin}/openshell-gateway`); expect(fs.statSync(unit).mode & 0o777).toBe(0o600); diff --git a/test/e2e/support/openclaw-plugin-runtime-exdev-env.test.ts b/test/e2e/support/openclaw-plugin-runtime-exdev-env.test.ts new file mode 100644 index 00000000000..c18f2a3b1d8 --- /dev/null +++ b/test/e2e/support/openclaw-plugin-runtime-exdev-env.test.ts @@ -0,0 +1,25 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + buildOpenClawPluginRuntimeExdevBaseImageEnv, + CURRENT_LIFECYCLE_TEST_SELECTOR, + RELEASE_BASELINE_TEST_SELECTOR, + RELEASE_SANDBOX_BASE_IMAGE_REF, +} from "../live/openclaw-plugin-runtime-exdev-env.ts"; + +describe("OpenClaw plugin runtime EXDEV base image selection", () => { + it("pins the release baseline to its matching sandbox base image", () => { + expect(buildOpenClawPluginRuntimeExdevBaseImageEnv(RELEASE_BASELINE_TEST_SELECTOR)).toEqual({ + NEMOCLAW_SANDBOX_BASE_IMAGE_REF: RELEASE_SANDBOX_BASE_IMAGE_REF, + }); + }); + + it("does not override base-image resolution for the current-lifecycle test", () => { + expect(buildOpenClawPluginRuntimeExdevBaseImageEnv(CURRENT_LIFECYCLE_TEST_SELECTOR)).toEqual( + {}, + ); + }); +});