Skip to content
257 changes: 69 additions & 188 deletions .github/workflows/e2e.yaml

Large diffs are not rendered by default.

9 changes: 7 additions & 2 deletions test/e2e/support/e2e-operations-workflow-boundary.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -106,8 +106,13 @@ describe("E2E operations workflow boundary", () => {
const workflow = readE2eOperationsWorkflow();
const job = workflow.jobs["cloud-onboard"];
job.env!.E2E_TARGET_ID = "different-job";
const run = job.steps!.find((step) => String(step.run ?? "").includes("npx vitest"))!;
run.run = run.run!.replace("test/e2e/risk-signal-reporter.ts", "default");
const run = job.steps!.find((step) =>
String(step.run ?? "").includes("tools/e2e/live-vitest-invocation.mts run"),
)!;
run.run = run.run!.replace(
"npx tsx tools/e2e/live-vitest-invocation.mts run",
"npx vitest run --project e2e-live",
);
const upload = job.steps!.find((step) =>
step.uses?.startsWith("NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@"),
)!;
Expand Down
153 changes: 153 additions & 0 deletions test/e2e/support/live-vitest-invocation.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,153 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { describe, expect, it } from "vitest";

import {
buildLiveVitestArgs,
LIVE_VITEST_PROJECT,
RISK_SIGNAL_REPORTER,
validateLiveProject,
validateLiveSelector,
validateLiveTestPath,
} from "../../../tools/e2e/live-vitest-invocation.mts";

describe("validateLiveProject (#6961)", () => {
it("accepts the live project and defaults to it", () => {
expect(validateLiveProject("e2e-live")).toBe(LIVE_VITEST_PROJECT);
expect(validateLiveProject(undefined)).toBe(LIVE_VITEST_PROJECT);
});

it("rejects any other project", () => {
for (const project of ["cli", "e2e-support", "e2e-live-extra", "integration"]) {
expect(() => validateLiveProject(project)).toThrow(/unsupported vitest project/);
}
});
});

describe("validateLiveTestPath (#6961)", () => {
it("accepts a real live test path", () => {
expect(validateLiveTestPath("test/e2e/live/registry-targets.test.ts")).toBe(
"test/e2e/live/registry-targets.test.ts",
);
});

it("rejects paths outside the live test root", () => {
expect(() => validateLiveTestPath("test/e2e/support/thing.test.ts")).toThrow(
/must be under test\/e2e\/live/,
);
expect(() => validateLiveTestPath("src/lib/onboard.ts")).toThrow(/must be under/);
});

it("rejects '..' traversal", () => {
expect(() => validateLiveTestPath("test/e2e/live/../support/x.test.ts")).toThrow(/traverse/);
});

it("rejects absolute paths", () => {
expect(() => validateLiveTestPath("/etc/passwd")).toThrow(/unsupported character|absolute/);
});

it("rejects shell metacharacters", () => {
for (const bad of [
"test/e2e/live/x.test.ts; rm -rf /",
"test/e2e/live/$(whoami).test.ts",
"test/e2e/live/x.test.ts && curl evil",
"test/e2e/live/`id`.test.ts",
"test/e2e/live/x.test.ts|cat",
]) {
expect(() => validateLiveTestPath(bad)).toThrow(/unsupported character/);
}
});

it("requires a .test.ts file", () => {
expect(() => validateLiveTestPath("test/e2e/live/fixtures")).toThrow(/\.test\.ts/);
});

it("requires a non-empty path", () => {
expect(() => validateLiveTestPath("")).toThrow(/required/);
expect(() => validateLiveTestPath(undefined)).toThrow(/required/);
});
});

describe("validateLiveSelector (#6961)", () => {
it("accepts anchored title patterns", () => {
expect(validateLiveSelector("^ubuntu-repo-cloud-openclaw$")).toBe(
"^ubuntu-repo-cloud-openclaw$",
);
expect(validateLiveSelector("^skill-agent$")).toBe("^skill-agent$");
});

it("treats an absent or empty selector as no selector", () => {
expect(validateLiveSelector(undefined)).toBeUndefined();
expect(validateLiveSelector("")).toBeUndefined();
expect(validateLiveSelector(" ")).toBeUndefined();
});

it("rejects shell metacharacters in the expanded selector", () => {
for (const bad of [
"^$(touch pwned)$",
"^x$; rm -rf /",
"^x$ && evil",
"^`id`$",
"^x|y$",
"^x>out$",
]) {
expect(() => validateLiveSelector(bad)).toThrow(/unsupported character/);
}
});
});

describe("buildLiveVitestArgs (#6961)", () => {
it("builds the standard invocation with a selector", () => {
expect(
buildLiveVitestArgs({
testPath: "test/e2e/live/registry-targets.test.ts",
selector: "^ubuntu-repo-cloud-openclaw$",
}),
).toEqual([
"vitest",
"run",
"--project",
"e2e-live",
"test/e2e/live/registry-targets.test.ts",
"-t",
"^ubuntu-repo-cloud-openclaw$",
"--silent=false",
"--reporter=default",
`--reporter=${RISK_SIGNAL_REPORTER}`,
]);
});

it("omits the selector arguments for a single-file target", () => {
expect(
buildLiveVitestArgs({
testPath: "test/e2e/live/diagnostics.test.ts",
}),
).toEqual([
"vitest",
"run",
"--project",
"e2e-live",
"test/e2e/live/diagnostics.test.ts",
"--silent=false",
"--reporter=default",
`--reporter=${RISK_SIGNAL_REPORTER}`,
]);
});

it("fails closed on an invalid input before producing any argv", () => {
expect(() =>
buildLiveVitestArgs({
testPath: "test/e2e/live/x.test.ts",
selector: "^x$; rm -rf /",
}),
).toThrow(/unsupported character/);
expect(() =>
buildLiveVitestArgs({
testPath: "test/e2e/support/x.test.ts",
selector: "^x$",
project: "e2e-live",
}),
).toThrow(/must be under/);
});
});
8 changes: 4 additions & 4 deletions test/e2e/support/mcp-workflow-boundary.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,10 @@ describe("MCP workflow artifact boundary", () => {
(step) => step.name === "Run MCP OpenShell provider live test",
);
requireFixture(run?.run, `${jobName} MCP live-test fixture is missing`);
const reporter = "--reporter=test/e2e/risk-signal-reporter.ts";
requireFixture(run.run.includes(reporter), `${jobName} reporter fixture is missing`);
const updatedRun = run.run.replace(` ${reporter}`, "");
requireFixture(updatedRun !== run.run, `${jobName} reporter could not be removed`);
const helper = "tools/e2e/live-vitest-invocation.mts run";
requireFixture(run.run.includes(helper), `${jobName} live-vitest helper fixture is missing`);
const updatedRun = run.run.replace(helper, "vitest run");
requireFixture(updatedRun !== run.run, `${jobName} live-vitest helper could not be removed`);
run.run = updatedRun;
fs.writeFileSync(workflowPath, YAML.stringify(workflow));

Expand Down
2 changes: 1 addition & 1 deletion tools/e2e/hermes-dashboard-workflow-boundary.mts
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ export function validateHermesDashboardWorkflow(workflow: HermesDashboardWorkflo
}

const run = findStep(job, "Run Hermes dashboard live Vitest test");
if (!run.run?.includes("npx vitest run --project e2e-live")) {
if (!run.run?.includes("tools/e2e/live-vitest-invocation.mts run")) {
errors.push(`${JOB_NAME} must run the live Vitest project`);
}
if (!run.run?.includes("test/e2e/live/hermes-e2e.test.ts")) {
Expand Down
2 changes: 1 addition & 1 deletion tools/e2e/hermes-gpu-startup-workflow-boundary.mts
Original file line number Diff line number Diff line change
Expand Up @@ -295,7 +295,7 @@ removalCondition:`,
true,
) ||
/\b(?:install\s+-m|chmod)\s+0?644\b/u.test(run) ||
!run.includes("npx vitest run --project e2e-live") ||
!run.includes("tools/e2e/live-vitest-invocation.mts run") ||
!run.includes("test/e2e/live/hermes-gpu-startup.test.ts")
) {
errors.push(`${JOB_NAME} trusted runtime boundary failed`);
Expand Down
107 changes: 107 additions & 0 deletions tools/e2e/live-vitest-invocation.mts
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { spawnSync } from "node:child_process";
import { pathToFileURL } from "node:url";

import { parseArgs } from "../advisors/io.mts";

export const LIVE_VITEST_PROJECT = "e2e-live";
export const LIVE_TEST_ROOT = "test/e2e/live/";
export const RISK_SIGNAL_REPORTER = "test/e2e/risk-signal-reporter.ts";

const SHELL_METACHARACTER = /[^A-Za-z0-9_./^$=:@+-]/u;
const TEST_PATH_PATTERN = /^[A-Za-z0-9_./-]+$/u;

export interface LiveVitestInvocation {
testPath: string | undefined;
selector?: string | undefined;
project?: string | undefined;
}

function assertNoShellMetacharacters(value: string, field: string): void {
const match = SHELL_METACHARACTER.exec(value);
if (match) {
throw new Error(`${field} contains an unsupported character ${JSON.stringify(match[0])}`);
}
}

export function validateLiveProject(project: string | undefined): string {
const resolved = (project ?? LIVE_VITEST_PROJECT).trim();
if (resolved !== LIVE_VITEST_PROJECT) {
throw new Error(
`unsupported vitest project ${JSON.stringify(resolved)}; this helper only runs ${LIVE_VITEST_PROJECT}`,
);
}
return resolved;
}

export function validateLiveTestPath(testPath: string | undefined): string {
const value = (testPath ?? "").trim();
if (!value) {
throw new Error("test path is required");
}
if (!TEST_PATH_PATTERN.test(value)) {
assertNoShellMetacharacters(value, "test path");
throw new Error(`test path ${JSON.stringify(value)} has an unsupported character`);
}
if (value.startsWith("/")) {
throw new Error("test path must be repository-relative, not absolute");
}
if (value.split("/").includes("..")) {
throw new Error("test path must not traverse with '..'");
}
if (!value.startsWith(LIVE_TEST_ROOT)) {
throw new Error(`test path must be under ${LIVE_TEST_ROOT}, got ${JSON.stringify(value)}`);
}
if (!value.endsWith(".test.ts")) {
throw new Error("test path must name a .test.ts file");
}
return value;
}

export function validateLiveSelector(selector: string | undefined): string | undefined {
const value = (selector ?? "").trim();
if (!value) {
return undefined;
}
assertNoShellMetacharacters(value, "selector");
return value;
}

export function buildLiveVitestArgs(invocation: LiveVitestInvocation): string[] {
const project = validateLiveProject(invocation.project);
const testPath = validateLiveTestPath(invocation.testPath);
const selector = validateLiveSelector(invocation.selector);
const selectorArgs = selector ? ["-t", selector] : [];
return [
"vitest",
"run",
"--project",
project,
testPath,
...selectorArgs,
"--silent=false",
"--reporter=default",
`--reporter=${RISK_SIGNAL_REPORTER}`,
];
}

function runCli(): void {
const args = parseArgs(process.argv.slice(3));
const argv = buildLiveVitestArgs({
testPath: args.testPath,
selector: args.selector,
project: args.project,
});
const result = spawnSync("npx", argv, { stdio: "inherit" });
process.exit(typeof result.status === "number" ? result.status : 1);
Comment thread
laitingsheng marked this conversation as resolved.
Outdated
}

if (
process.argv[1] &&
import.meta.url === pathToFileURL(process.argv[1]).href &&
process.argv[2] === "run"
) {
runCli();
}
7 changes: 5 additions & 2 deletions tools/e2e/mcp-workflow-boundary.mts
Original file line number Diff line number Diff line change
Expand Up @@ -392,13 +392,16 @@ function validateJobExecution(
errors.push("mcp-bridge stable lane must run its full MCP lifecycle unconditionally");
}
}
for (const required of ["--project e2e-live", "test/e2e/live/mcp-bridge.test.ts"]) {
for (const required of [
"tools/e2e/live-vitest-invocation.mts run",
"test/e2e/live/mcp-bridge.test.ts",
]) {
requireContains(errors, run.run, required, `${jobName} must run the unified MCP live test`);
}
requireContains(
errors,
run.run,
"--reporter=test/e2e/risk-signal-reporter.ts",
"tools/e2e/live-vitest-invocation.mts run",
`${jobName} must publish canonical risk-signal evidence`,
);
requireEqual(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,7 @@ export function validateOpenClawPluginRuntimeExdevWorkflow(
'test -z "${DOCKERHUB_USERNAME:-}"',
'test -z "${DOCKERHUB_TOKEN:-}"',
"env -u DOCKER_CONFIG -u DOCKERHUB_USERNAME -u DOCKERHUB_TOKEN",
"npx vitest run --project e2e-live",
"tools/e2e/live-vitest-invocation.mts run",
"test/e2e/live/openclaw-plugin-runtime-exdev.test.ts",
]) {
requireRunContains(errors, run, fragment, runName);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,7 @@ export function validateOpenShellGatewayAuthContractWorkflow(

const runName = "Run OpenShell gateway auth contract live test";
const run = findStep(job, runName);
requireRunContains(errors, run, "npx vitest run --project e2e-live");
requireRunContains(errors, run, "tools/e2e/live-vitest-invocation.mts run");
requireRunContains(errors, run, "test/e2e/live/openshell-gateway-auth-source-contract.test.ts");
if (Object.keys(run.env ?? {}).length > 0 || JSON.stringify(run).includes("secrets.")) {
errors.push(`${JOB_NAME} live test must not receive workflow credentials`);
Expand Down
13 changes: 9 additions & 4 deletions tools/e2e/operations-workflow-boundary.mts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ const FULL_SHA_ACTION = /^[^\s@]+@[0-9a-f]{40}$/u;
const GITHUB_SCRIPT_NODE24_ACTION =
"actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3";
const PR_GATE_REPORTER = "test/e2e/risk-signal-reporter.ts";
const LIVE_VITEST_HELPER = "tools/e2e/live-vitest-invocation.mts run";
const E2E_ARTIFACT_ACTION = "NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@";
const ISSUE_API_REFERENCE = /\bgithub\.rest\.issues\b/u;
const ISSUE_MUTATION_BEYOND_COMMENT =
Expand Down Expand Up @@ -231,12 +232,16 @@ function validatePrGateEvidenceProducers(errors: string[], workflow: OperationsW
if (typeof job.env?.E2E_ARTIFACT_DIR !== "string" || !job.env.E2E_ARTIFACT_DIR) {
errors.push(`${jobId} must expose an evidence artifact directory`);
}
const vitestSteps = (job.steps ?? []).filter((step) =>
String(step.run ?? "").includes("npx vitest"),
);
const vitestSteps = (job.steps ?? []).filter((step) => {
const run = String(step.run ?? "");
return run.includes("npx vitest") || run.includes(LIVE_VITEST_HELPER);
});
if (
vitestSteps.length === 0 ||
vitestSteps.some((step) => !String(step.run).includes(PR_GATE_REPORTER))
vitestSteps.some((step) => {
const run = String(step.run);
return !run.includes(LIVE_VITEST_HELPER) && !run.includes(PR_GATE_REPORTER);
})
) {
errors.push(`${jobId} must attach the risk-signal reporter to every Vitest invocation`);
}
Expand Down
2 changes: 1 addition & 1 deletion tools/e2e/sandbox-operations-workflow-boundary.mts
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,7 @@ export function validateSandboxOperationsWorkflow(workflow: {
errors.push(`${JOB_NAME} exposes the inference key outside the live test step`);
}
}
requireRunContains(errors, run, "npx vitest run --project e2e-live");
requireRunContains(errors, run, "tools/e2e/live-vitest-invocation.mts run");
requireRunContains(errors, run, "test/e2e/live/sandbox-operations.test.ts");

const cleanup = findStep(job, "Clean up Docker auth");
Expand Down
5 changes: 4 additions & 1 deletion tools/e2e/upload-e2e-artifacts-workflow-boundary.mts
Original file line number Diff line number Diff line change
Expand Up @@ -280,7 +280,10 @@ export function validateUploadE2eArtifactsInvocations(workflow: WorkflowRecord):
env.NEMOCLAW_RUN_LIVE_E2E === "1" ||
SHARED_E2E_JOBS.has(jobName) ||
jobSteps.some(
(step) => typeof step.run === "string" && step.run.includes("--project e2e-live"),
(step) =>
typeof step.run === "string" &&
(step.run.includes("--project e2e-live") ||
step.run.includes("tools/e2e/live-vitest-invocation.mts run")),
)
);
})
Expand Down
Loading
Loading