Skip to content
257 changes: 69 additions & 188 deletions .github/workflows/e2e.yaml

Large diffs are not rendered by default.

9 changes: 7 additions & 2 deletions test/e2e/support/e2e-operations-workflow-boundary.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -106,8 +106,13 @@ describe("E2E operations workflow boundary", () => {
const workflow = readE2eOperationsWorkflow();
const job = workflow.jobs["cloud-onboard"];
job.env!.E2E_TARGET_ID = "different-job";
const run = job.steps!.find((step) => String(step.run ?? "").includes("npx vitest"))!;
run.run = run.run!.replace("test/e2e/risk-signal-reporter.ts", "default");
const run = job.steps!.find((step) =>
String(step.run ?? "").includes("tools/e2e/live-vitest-invocation.mts run --test-path"),
)!;
run.run = run.run!.replace(
"tools/e2e/live-vitest-invocation.mts run --test-path",
"tools/e2e/live-vitest-invocation.mts runx --test-path",
);
const upload = job.steps!.find((step) =>
step.uses?.startsWith("NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@"),
)!;
Expand Down
214 changes: 214 additions & 0 deletions test/e2e/support/live-vitest-invocation.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,214 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { spawnSync } from "node:child_process";

import { describe, expect, it } from "vitest";

import {
buildLiveVitestArgs,
LIVE_VITEST_PROJECT,
type LiveVitestSpawner,
RISK_SIGNAL_REPORTER,
runLiveVitestCommand,
validateLiveProject,
validateLiveSelector,
validateLiveTestPath,
} from "../../../tools/e2e/live-vitest-invocation.mts";

describe("validateLiveProject (#6961)", () => {
it("accepts the live project and defaults to it", () => {
expect(validateLiveProject("e2e-live")).toBe(LIVE_VITEST_PROJECT);
expect(validateLiveProject(undefined)).toBe(LIVE_VITEST_PROJECT);
});

it("rejects any other project", () => {
for (const project of ["cli", "e2e-support", "e2e-live-extra", "integration"]) {
expect(() => validateLiveProject(project)).toThrow(/unsupported vitest project/);
}
});
});

describe("validateLiveTestPath (#6961)", () => {
it("accepts a real live test path", () => {
expect(validateLiveTestPath("test/e2e/live/registry-targets.test.ts")).toBe(
"test/e2e/live/registry-targets.test.ts",
);
});

it("rejects paths outside the live test root", () => {
expect(() => validateLiveTestPath("test/e2e/support/thing.test.ts")).toThrow(
/must be under test\/e2e\/live/,
);
expect(() => validateLiveTestPath("src/lib/onboard.ts")).toThrow(/must be under/);
});

it("rejects '..' traversal", () => {
expect(() => validateLiveTestPath("test/e2e/live/../support/x.test.ts")).toThrow(/traverse/);
});

it("rejects absolute paths", () => {
expect(() => validateLiveTestPath("/etc/passwd")).toThrow(/unsupported character|absolute/);
});

it("rejects shell metacharacters", () => {
for (const bad of [
"test/e2e/live/x.test.ts; rm -rf /",
"test/e2e/live/$(whoami).test.ts",
"test/e2e/live/x.test.ts && curl evil",
"test/e2e/live/`id`.test.ts",
"test/e2e/live/x.test.ts|cat",
]) {
expect(() => validateLiveTestPath(bad)).toThrow(/unsupported character/);
}
});

it("requires a .test.ts file", () => {
expect(() => validateLiveTestPath("test/e2e/live/fixtures")).toThrow(/\.test\.ts/);
});

it("requires a non-empty path", () => {
expect(() => validateLiveTestPath("")).toThrow(/required/);
expect(() => validateLiveTestPath(undefined)).toThrow(/required/);
});
});

describe("validateLiveSelector (#6961)", () => {
it("accepts anchored title patterns", () => {
expect(validateLiveSelector("^ubuntu-repo-cloud-openclaw$")).toBe(
"^ubuntu-repo-cloud-openclaw$",
);
expect(validateLiveSelector("^skill-agent$")).toBe("^skill-agent$");
});

it("treats an absent or empty selector as no selector", () => {
expect(validateLiveSelector(undefined)).toBeUndefined();
expect(validateLiveSelector("")).toBeUndefined();
expect(validateLiveSelector(" ")).toBeUndefined();
});

it("rejects shell metacharacters in the expanded selector", () => {
for (const bad of [
"^$(touch pwned)$",
"^x$; rm -rf /",
"^x$ && evil",
"^`id`$",
"^x|y$",
"^x>out$",
]) {
expect(() => validateLiveSelector(bad)).toThrow(/unsupported character/);
}
});
});

describe("buildLiveVitestArgs (#6961)", () => {
it("builds the standard invocation with a selector", () => {
expect(
buildLiveVitestArgs({
testPath: "test/e2e/live/registry-targets.test.ts",
selector: "^ubuntu-repo-cloud-openclaw$",
}),
).toEqual([
"vitest",
"run",
"--project",
"e2e-live",
"test/e2e/live/registry-targets.test.ts",
"-t",
"^ubuntu-repo-cloud-openclaw$",
"--silent=false",
"--reporter=default",
`--reporter=${RISK_SIGNAL_REPORTER}`,
]);
});

it("omits the selector arguments for a single-file target", () => {
expect(
buildLiveVitestArgs({
testPath: "test/e2e/live/diagnostics.test.ts",
}),
).toEqual([
"vitest",
"run",
"--project",
"e2e-live",
"test/e2e/live/diagnostics.test.ts",
"--silent=false",
"--reporter=default",
`--reporter=${RISK_SIGNAL_REPORTER}`,
]);
});

it("fails closed on an invalid input before producing any argv", () => {
expect(() =>
buildLiveVitestArgs({
testPath: "test/e2e/live/x.test.ts",
selector: "^x$; rm -rf /",
}),
).toThrow(/unsupported character/);
expect(() =>
buildLiveVitestArgs({
testPath: "test/e2e/support/x.test.ts",
selector: "^x$",
project: "e2e-live",
}),
).toThrow(/must be under/);
});
});

describe("runLiveVitestCommand (#6961)", () => {
const validArgs = ["run", "--test-path", "test/e2e/live/diagnostics.test.ts"];

it.each([
["child status", { status: 7, signal: null }, 7],
["child signal", { status: null, signal: "SIGTERM" as NodeJS.Signals }, 143],
["missing status and signal", { status: null, signal: null }, 1],
])("preserves %s", (_label, result, expected) => {
let spawned: Parameters<LiveVitestSpawner> | undefined;
const spawn: LiveVitestSpawner = (...args) => {
spawned = args;
return result;
};

expect(runLiveVitestCommand(validArgs, spawn)).toBe(expected);
expect(spawned).toEqual([
"npx",
[
"vitest",
"run",
"--project",
"e2e-live",
"test/e2e/live/diagnostics.test.ts",
"--silent=false",
"--reporter=default",
`--reporter=${RISK_SIGNAL_REPORTER}`,
],
{ stdio: "inherit" },
]);
});

it("surfaces child launch failures", () => {
const launchError = new Error("spawn npx ENOENT");
const spawn: LiveVitestSpawner = () => ({
status: null,
signal: null,
error: launchError,
});

expect(() => runLiveVitestCommand(validArgs, spawn)).toThrow(launchError);
});

it.each([
["missing", []],
["unsupported", ["runx"]],
])("fails the direct CLI for a %s subcommand", (_label, args) => {
const result = spawnSync(
process.execPath,
["--experimental-strip-types", "tools/e2e/live-vitest-invocation.mts", ...args],
{ encoding: "utf8" },
);

expect(result.status).toBe(1);
expect(result.stderr).toContain('expected "run"');
});
});
8 changes: 4 additions & 4 deletions test/e2e/support/mcp-workflow-boundary.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,10 @@ describe("MCP workflow artifact boundary", () => {
(step) => step.name === "Run MCP OpenShell provider live test",
);
requireFixture(run?.run, `${jobName} MCP live-test fixture is missing`);
const reporter = "--reporter=test/e2e/risk-signal-reporter.ts";
requireFixture(run.run.includes(reporter), `${jobName} reporter fixture is missing`);
const updatedRun = run.run.replace(` ${reporter}`, "");
requireFixture(updatedRun !== run.run, `${jobName} reporter could not be removed`);
const helper = "tools/e2e/live-vitest-invocation.mts run --test-path";
requireFixture(run.run.includes(helper), `${jobName} live-vitest helper fixture is missing`);
const updatedRun = run.run.replace(helper, "vitest run");
requireFixture(updatedRun !== run.run, `${jobName} live-vitest helper could not be removed`);
run.run = updatedRun;
fs.writeFileSync(workflowPath, YAML.stringify(workflow));

Expand Down
2 changes: 1 addition & 1 deletion tools/e2e/hermes-dashboard-workflow-boundary.mts
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ export function validateHermesDashboardWorkflow(workflow: HermesDashboardWorkflo
}

const run = findStep(job, "Run Hermes dashboard live Vitest test");
if (!run.run?.includes("npx vitest run --project e2e-live")) {
if (!run.run?.includes("tools/e2e/live-vitest-invocation.mts run --test-path")) {
errors.push(`${JOB_NAME} must run the live Vitest project`);
}
if (!run.run?.includes("test/e2e/live/hermes-e2e.test.ts")) {
Expand Down
2 changes: 1 addition & 1 deletion tools/e2e/hermes-gpu-startup-workflow-boundary.mts
Original file line number Diff line number Diff line change
Expand Up @@ -295,7 +295,7 @@ removalCondition:`,
true,
) ||
/\b(?:install\s+-m|chmod)\s+0?644\b/u.test(run) ||
!run.includes("npx vitest run --project e2e-live") ||
!run.includes("tools/e2e/live-vitest-invocation.mts run --test-path") ||
!run.includes("test/e2e/live/hermes-gpu-startup.test.ts")
) {
errors.push(`${JOB_NAME} trusted runtime boundary failed`);
Expand Down
129 changes: 129 additions & 0 deletions tools/e2e/live-vitest-invocation.mts
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { spawnSync } from "node:child_process";
import { pathToFileURL } from "node:url";

import { spawnExitCode } from "../../src/lib/core/process-exit.ts";
import { parseArgs } from "../advisors/io.mts";

export const LIVE_VITEST_PROJECT = "e2e-live";
export const LIVE_TEST_ROOT = "test/e2e/live/";
export const RISK_SIGNAL_REPORTER = "test/e2e/risk-signal-reporter.ts";

const SHELL_METACHARACTER = /[^A-Za-z0-9_./^$=:@+-]/u;
const TEST_PATH_PATTERN = /^[A-Za-z0-9_./-]+$/u;

export interface LiveVitestInvocation {
testPath: string | undefined;
selector?: string | undefined;
project?: string | undefined;
}

export interface LiveVitestSpawnResult {
status: number | null;
signal?: NodeJS.Signals | null;
error?: Error | undefined;
}

export type LiveVitestSpawner = (
command: string,
args: string[],
options: { stdio: "inherit" },
) => LiveVitestSpawnResult;

function assertNoShellMetacharacters(value: string, field: string): void {
const match = SHELL_METACHARACTER.exec(value);
if (match) {
throw new Error(`${field} contains an unsupported character ${JSON.stringify(match[0])}`);
}
}

export function validateLiveProject(project: string | undefined): string {
const resolved = (project ?? LIVE_VITEST_PROJECT).trim();
if (resolved !== LIVE_VITEST_PROJECT) {
throw new Error(
`unsupported vitest project ${JSON.stringify(resolved)}; this helper only runs ${LIVE_VITEST_PROJECT}`,
);
}
return resolved;
}

export function validateLiveTestPath(testPath: string | undefined): string {
const value = (testPath ?? "").trim();
if (!value) {
throw new Error("test path is required");
}
if (!TEST_PATH_PATTERN.test(value)) {
assertNoShellMetacharacters(value, "test path");
throw new Error(`test path ${JSON.stringify(value)} has an unsupported character`);
}
if (value.startsWith("/")) {
throw new Error("test path must be repository-relative, not absolute");
}
if (value.split("/").includes("..")) {
throw new Error("test path must not traverse with '..'");
}
if (!value.startsWith(LIVE_TEST_ROOT)) {
throw new Error(`test path must be under ${LIVE_TEST_ROOT}, got ${JSON.stringify(value)}`);
}
if (!value.endsWith(".test.ts")) {
throw new Error("test path must name a .test.ts file");
}
return value;
}

export function validateLiveSelector(selector: string | undefined): string | undefined {
const value = (selector ?? "").trim();
if (!value) {
return undefined;
}
assertNoShellMetacharacters(value, "selector");
return value;
}

export function buildLiveVitestArgs(invocation: LiveVitestInvocation): string[] {
const project = validateLiveProject(invocation.project);
const testPath = validateLiveTestPath(invocation.testPath);
const selector = validateLiveSelector(invocation.selector);
const selectorArgs = selector ? ["-t", selector] : [];
return [
"vitest",
"run",
"--project",
project,
testPath,
...selectorArgs,
"--silent=false",
"--reporter=default",
`--reporter=${RISK_SIGNAL_REPORTER}`,
];
}

export function runLiveVitestCli(cliArgs: string[], spawn: LiveVitestSpawner = spawnSync): number {
const args = parseArgs(cliArgs);
const argv = buildLiveVitestArgs({
testPath: args.testPath,
selector: args.selector,
project: args.project,
});
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Outdated
const result = spawn("npx", argv, { stdio: "inherit" });
if (result.error) {
throw result.error;
}
return spawnExitCode(result);
}

export function runLiveVitestCommand(argv: string[], spawn: LiveVitestSpawner = spawnSync): number {
const [command, ...cliArgs] = argv;
if (command !== "run") {
throw new Error(
`unsupported live Vitest command ${JSON.stringify(command ?? "")}; expected "run"`,
);
}
return runLiveVitestCli(cliArgs, spawn);
}

if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
process.exit(runLiveVitestCommand(process.argv.slice(2)));
}
Loading
Loading