Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
82 changes: 68 additions & 14 deletions test/e2e/fixtures/phases/lifecycle.ts
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,9 @@ const NEMOCLAW_OPENSHELL_GATEWAY_USER_SERVICE_MARKER_LINE =
const NEMOCLAW_INSTALLER = fileURLToPath(
new URL("../../../../scripts/install.sh", import.meta.url),
);
const NEMOCLAW_OPENSHELL_INSTALLER = fileURLToPath(
new URL("../../../../scripts/install-openshell.sh", import.meta.url),
);
const USER_SERVICE_STAGE_RESULT_PREFIX = "NEMOCLAW_E2E_GATEWAY_USER_SERVICE=";

type UserServiceStageResult = "upstream" | "existing" | "staged";
Expand Down Expand Up @@ -143,6 +146,23 @@ export function buildOpenShellGatewayUserServiceRestartScript(): string {
].join("\n");
}

export function buildOpenShellGatewayUserServiceDiagnosticsScript(): string {
return [
"set +e",
"service=openshell-gateway",
'if ! systemctl --user cat "$service" >/dev/null 2>&1; then',
" service=nemoclaw-openshell-gateway",
"fi",
'printf "OpenShell gateway user service: %s\\n" "$service"',
'systemctl --user show "$service" --no-pager --property=ActiveState --property=SubState --property=Result --property=ExecMainCode --property=ExecMainStatus',
'systemctl --user status "$service" --no-pager --full',
"if command -v journalctl >/dev/null 2>&1; then",
' journalctl --user --unit "$service" --no-pager --lines=200',
"fi",
"exit 0",
].join("\n");
}

export type LifecycleProfile = "post-reboot-recovery" | "dcode-rebuild-invalid-credential";

export interface LifecycleCleanup {
Expand Down Expand Up @@ -205,13 +225,42 @@ function instanceName(instance: NemoClawInstance | string): string {
}

export class LifecyclePhaseFixture {
private postRebootUserServiceStage: UserServiceStageResult | undefined;

constructor(
private readonly host: HostCliClient,
private readonly sandbox: SandboxClient,
private readonly cleanup: LifecycleCleanup,
private readonly gateway?: GatewayClient,
) {}

/**
* Ensure OpenShell is installed and stage the OpenShell gateway user service
* before onboarding. Onboarding must see the service so it writes the
* Docker-driver environment that the unit needs after a user-manager restart.
*/
async preparePostReboot(): Promise<UserServiceStageResult> {
if (this.postRebootUserServiceStage) return this.postRebootUserServiceStage;

if (!(await this.host.isCommandAvailable("openshell-gateway"))) {
const install = await this.host.command("bash", [NEMOCLAW_OPENSHELL_INSTALLER], {
artifactName: "lifecycle-prereq-install-openshell",
env: buildAvailabilityProbeEnv(),
timeoutMs: 10 * 60_000,
});
assertExitZero(install, "install OpenShell before reboot lifecycle onboarding");
}

const stage = await this.ensureOpenShellGatewayUserService();
this.postRebootUserServiceStage = stage;
if (stage === "staged") {
this.cleanup.add("lifecycle.remove-staged-gateway-user-service", async () => {
await this.removeStagedOpenShellGatewayUserService();
});
}
return stage;
}

async rebuildSandbox(
instance: NemoClawInstance | string,
options: RebuildSandboxOptions = {},
Expand Down Expand Up @@ -319,12 +368,17 @@ export class LifecyclePhaseFixture {
* - `docker start` the labeled container so the sandbox returns
* to a usable state for any teardown that expects it live;
* - remove a user service staged only for this source-checkout
* fixture, then restore the original gateway runtime shape.
* fixture after the sandbox cleanup has used it.
*/
async simulatePostReboot(
instance: NemoClawInstance,
options: PostRebootOptions = {},
): Promise<LifecycleResult> {
if (!this.postRebootUserServiceStage) {
throw new Error(
"OpenShell gateway user service must be prepared before post-reboot onboarding.",
);
}
const mode: PostRebootMode = options.mode ?? "stop-original";
const steps: LifecycleResult["steps"] = [];

Expand All @@ -337,17 +391,6 @@ export class LifecyclePhaseFixture {
);
}
const originalName = containerNames[0];
const userServiceStage = await this.ensureOpenShellGatewayUserService();
let previousRuntime: HostGatewayRuntime | null = null;
if (userServiceStage === "staged") {
this.cleanup.add("lifecycle.remove-staged-gateway-user-service", async () => {
await this.removeStagedOpenShellGatewayUserService();
await this.startGatewayRuntime(previousRuntime, {
sandboxName: instance.sandboxName,
});
await this.waitForGatewayConnected();
});
}

const stop = await this.host.command("docker", ["stop", originalName], {
artifactName: `lifecycle-post-reboot-docker-stop-${originalName}`,
Expand Down Expand Up @@ -385,7 +428,7 @@ export class LifecyclePhaseFixture {
});
}

previousRuntime = await this.restartGatewayRuntime({
const previousRuntime = await this.restartGatewayRuntime({
delayMs: 0,
requireUserService: true,
sandboxName: instance.sandboxName,
Expand Down Expand Up @@ -623,8 +666,19 @@ export class LifecyclePhaseFixture {
}
if (attempt < attempts) await new Promise((resolve) => setTimeout(resolve, intervalMs));
}
const diagnostics = await this.host.command(
"sh",
["-lc", buildOpenShellGatewayUserServiceDiagnosticsScript()],
{
artifactName: "lifecycle-gateway-user-service-diagnostics",
env: buildAvailabilityProbeEnv(),
timeoutMs: 30_000,
},
);
throw new Error(
`gateway did not become healthy after restart: ${lastError instanceof Error ? lastError.message : String(lastError ?? "unknown")}`,
`gateway did not become healthy after restart: ${
lastError instanceof Error ? lastError.message : String(lastError ?? "unknown")
}; service diagnostics: ${diagnostics.artifacts.result}`,
);
}

Expand Down
18 changes: 0 additions & 18 deletions test/e2e/live/openclaw-plugin-runtime-exdev-env.ts

This file was deleted.

35 changes: 35 additions & 0 deletions test/e2e/live/openclaw-plugin-runtime-exdev-fixture.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

export const RELEASE_BASELINE_TEST_SELECTOR = "release-baseline";
export const CURRENT_LIFECYCLE_TEST_SELECTOR = "current-lifecycle";
export const RELEASE_SANDBOX_BASE_IMAGE_REF = "ghcr.io/nvidia/nemoclaw/sandbox-base:v0.0.71";

export type OpenClawPluginRuntimeExdevSelector =
| typeof RELEASE_BASELINE_TEST_SELECTOR
| typeof CURRENT_LIFECYCLE_TEST_SELECTOR;

export type OpenClawPluginRuntimeExdevFixture = {
selector: OpenClawPluginRuntimeExdevSelector;
source: "release" | "current";
baseImageEnv: NodeJS.ProcessEnv;
};

export function resolveOpenClawPluginRuntimeExdevFixture(
selector: OpenClawPluginRuntimeExdevSelector,
): OpenClawPluginRuntimeExdevFixture {
if (selector === RELEASE_BASELINE_TEST_SELECTOR) {
return {
selector,
source: "release",
baseImageEnv: {
NEMOCLAW_SANDBOX_BASE_IMAGE_REF: RELEASE_SANDBOX_BASE_IMAGE_REF,
},
};
}
return {
selector,
source: "current",
baseImageEnv: {},
};
}
Loading
Loading