diff --git a/src/lib/onboard/docker-startup-command-agent.ts b/src/lib/onboard/docker-startup-command-agent.ts index 23968f21d4c..de483bda8db 100644 --- a/src/lib/onboard/docker-startup-command-agent.ts +++ b/src/lib/onboard/docker-startup-command-agent.ts @@ -24,9 +24,10 @@ export function resolveDockerStartupCommandPatch( if (dockerDriverGateway !== true) { return { persistStartupCommand: false, requiredUlimits: null }; } - const agentName = agent?.name; + const agentName = agent?.name ?? "openclaw"; return { - persistStartupCommand: agentName === "hermes" || agentName === DCODE_AGENT_NAME, + persistStartupCommand: + agentName === "openclaw" || agentName === "hermes" || agentName === DCODE_AGENT_NAME, requiredUlimits: agentName === DCODE_AGENT_NAME ? DCODE_DOCKER_ULIMITS : null, }; } diff --git a/src/lib/onboard/sandbox-create-step.test.ts b/src/lib/onboard/sandbox-create-step.test.ts index ba26917eb04..88991227c4f 100644 --- a/src/lib/onboard/sandbox-create-step.test.ts +++ b/src/lib/onboard/sandbox-create-step.test.ts @@ -134,7 +134,12 @@ describe("runSandboxCreateStep", () => { }); }); - it("persists the Hermes startup command for Docker-driver container restarts", async () => { + it.each([ + { label: "OpenClaw", agent: null }, + { label: "Hermes", agent: { name: "hermes" } as SandboxCreateStepContext["agent"] }, + ])("persists the $label startup command for Docker-driver container restarts", async ({ + agent, + }) => { const launch = makeLaunch({ sandboxStartupCommand: ["env", "CHAT_UI_URL=http://127.0.0.1:8642", "nemoclaw-start"], }); @@ -143,7 +148,7 @@ describe("runSandboxCreateStep", () => { await runSandboxCreateStep( makeContext({ - agent: { name: "hermes" } as SandboxCreateStepContext["agent"], + agent, prebuild: { buildCtx: "/tmp/ctx", buildId: "b1", diff --git a/test/e2e/live/sandbox-survival.test.ts b/test/e2e/live/sandbox-survival.test.ts index 6b2d6f753cf..2ae366ba860 100644 --- a/test/e2e/live/sandbox-survival.test.ts +++ b/test/e2e/live/sandbox-survival.test.ts @@ -17,7 +17,12 @@ import { cleanupWhenCommandAvailable, cleanupWhenOpenShellAvailable, } from "../fixtures/cleanup-resources.ts"; -import { assertExitZero, resultText, sandboxAccessEnv } from "../fixtures/clients/index.ts"; +import { + assertExitZero, + type HostCliClient, + resultText, + sandboxAccessEnv, +} from "../fixtures/clients/index.ts"; import { trustedProviderEndpoint } from "../fixtures/clients/provider.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; import { requireHostedInferenceConfig } from "../fixtures/hosted-inference.ts"; @@ -29,6 +34,86 @@ const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-survival"; const MIN_OPENSHELL_VERSION = "0.0.24"; const MODEL = process.env.NEMOCLAW_MODEL ?? "nvidia/nemotron-3-super-120b-a12b"; +const SURVIVAL_DIAGNOSTICS_SCRIPT = String.raw` +set +e +sandbox_name="$1" + +printf '%s\n' '== OpenShell sandbox status ==' +openshell sandbox get "$sandbox_name" 2>&1 +printf '%s\n' '== OpenShell forwards ==' +openshell forward list 2>&1 +printf '%s\n' '== OpenShell gateway service ==' +systemctl --user status nemoclaw-openshell-gateway --no-pager -l 2>&1 +printf '%s\n' '== OpenShell gateway journal ==' +journalctl --user -u nemoclaw-openshell-gateway -n 200 --no-pager 2>&1 + +container_ids="$(docker ps -aq \ + --filter label=openshell.ai/managed-by=openshell \ + --filter "label=openshell.ai/sandbox-name=$sandbox_name")" +printf '%s\n' '== matching containers ==' +if [ -n "$container_ids" ]; then + docker ps -a --no-trunc \ + --filter label=openshell.ai/managed-by=openshell \ + --filter "label=openshell.ai/sandbox-name=$sandbox_name" \ + --format '{{.ID}} {{.Names}} {{.Status}}' +else + printf '%s\n' 'none' +fi + +for container_id in $container_ids; do + printf '%s\n' "== container $container_id inspect ==" + docker inspect "$container_id" 2>&1 | node -e ' + const fs = require("node:fs"); + const row = JSON.parse(fs.readFileSync(0, "utf8"))[0] || {}; + const prefix = "OPENSHELL_SANDBOX_COMMAND="; + const matches = (row.Config?.Env || []).filter((entry) => entry.startsWith(prefix)); + const command = matches.length === 1 ? matches[0].slice(prefix.length) : ""; + const tokens = command.trim().split(/\s+/).filter(Boolean); + process.stdout.write(JSON.stringify({ + name: row.Name || "", + configUser: row.Config?.User || "", + state: { + status: row.State?.Status || "", + running: Boolean(row.State?.Running), + restarting: Boolean(row.State?.Restarting), + pid: row.State?.Pid || 0, + exitCode: row.State?.ExitCode ?? null, + error: row.State?.Error || "", + startedAt: row.State?.StartedAt || "", + finishedAt: row.State?.FinishedAt || "", + health: row.State?.Health?.Status || "", + }, + restartPolicy: row.HostConfig?.RestartPolicy?.Name || "", + startupCommandCount: matches.length, + startupCommandIsSleepInfinity: tokens.length === 2 + && tokens[0] === "sleep" && tokens[1] === "infinity", + startupCommandEndsWithNemoclawStart: tokens.length > 0 + && ["nemoclaw-start", "/usr/local/bin/nemoclaw-start"].includes(tokens.at(-1)), + }) + "\n"); + ' + printf '%s\n' "== container $container_id host process tree ==" + docker top "$container_id" -eo pid,ppid,user,stat,comm 2>&1 + printf '%s\n' "== container $container_id runtime state ==" + docker exec "$container_id" sh -lc ' + printf "%s\n" "== pid 1 ==" + cat /proc/1/comm 2>/dev/null || true + printf "\n%s\n" "== process tree ==" + ps -eo user=,pid=,ppid=,stat=,comm= 2>&1 || true + printf "%s\n" "== direct gateway health ==" + curl -q --noproxy "*" -sS -o /dev/null -w "HTTP %{http_code}\n" \ + --connect-timeout 2 --max-time 5 http://127.0.0.1:18789/health 2>&1 || true + printf "%s\n" "== managed controller status ==" + cat /run/nemoclaw/gateway-control/status 2>&1 || true + printf "%s\n" "== startup log ==" + tail -n 300 /tmp/nemoclaw-start.log 2>&1 || true + printf "%s\n" "== gateway log ==" + tail -n 300 /tmp/gateway.log 2>&1 || true + ' 2>&1 + printf '%s\n' "== container $container_id logs ==" + docker logs --tail 300 "$container_id" 2>&1 +done +`; + function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } @@ -61,6 +146,23 @@ function installEnv(hostedEnv: NodeJS.ProcessEnv): NodeJS.ProcessEnv { }; } +async function captureSurvivalDiagnostics( + host: HostCliClient, + stage: string, + redactionValues: string[], +): Promise { + await host.command( + "sh", + ["-lc", SURVIVAL_DIAGNOSTICS_SCRIPT, "sandbox-survival-diagnostics", SANDBOX_NAME], + { + artifactName: `sandbox-survival-${stage}-diagnostics`, + env: buildAvailabilityProbeEnv(), + redactionValues, + timeoutMs: 60_000, + }, + ); +} + async function expectSandboxExecAlive( sandboxName: string, exec: ( @@ -303,10 +405,12 @@ test( await stateValidation.expectSandboxMarkers(instance, markers, "pre-restart-marker-read"); progress.phase("restart the gateway and reconnect the sandbox"); + await captureSurvivalDiagnostics(host, "before-gateway-restart", [apiKey]); await lifecycle.restartGatewayRuntime({ delayMs: 5_000, sandboxName: SANDBOX_NAME, }); + await captureSurvivalDiagnostics(host, "after-gateway-restart", [apiKey]); await lifecycle.waitForGatewayConnected({ attempts: 60, intervalMs: 5_000, @@ -324,6 +428,7 @@ test( artifactName: "post-restart-nemoclaw-status", timeoutMs: 120_000, }); + await stateValidation.from("cloud-openclaw-ready", instance); await expectSandboxExecAlive(SANDBOX_NAME, execShell, "post-restart-sandbox-exec-alive"); await stateValidation.expectSandboxMarkers(instance, markers, "post-restart-marker-read"); await stateValidation.expectSandboxDirectoryPopulated( diff --git a/test/helpers/onboard-script-mocks.cjs b/test/helpers/onboard-script-mocks.cjs index abaf4cb4186..730057aa899 100644 --- a/test/helpers/onboard-script-mocks.cjs +++ b/test/helpers/onboard-script-mocks.cjs @@ -75,6 +75,30 @@ const OPENCLAW_SECURITY_INVENTORY_PROBE = [ `printf '%s\\n' "nemoclaw-security-inventory-ok"`, ].join("; "); +const ONBOARD_SANDBOX_OLD_CONTAINER_ID = "a".repeat(64); +const ONBOARD_SANDBOX_NEW_CONTAINER_ID = "b".repeat(64); +const ONBOARD_SANDBOX_INSPECT = { + Id: ONBOARD_SANDBOX_OLD_CONTAINER_ID, + Image: `sha256:${"c".repeat(64)}`, + Name: "/openshell-my-assistant", + Config: { + Image: "openshell/sandbox:test", + Env: ["OPENSHELL_SANDBOX_COMMAND=sleep infinity"], + Labels: { + "openshell.ai/managed-by": "openshell", + "openshell.ai/sandbox-name": "my-assistant", + }, + Entrypoint: ["/opt/openshell/bin/openshell-sandbox"], + Cmd: [], + User: "0", + WorkingDir: "/sandbox", + }, + HostConfig: { + NetworkMode: "openshell-docker", + RestartPolicy: { Name: "unless-stopped" }, + }, +}; + function isOpenClawSecurityInventoryProbe(command) { const commandArgs = Array.isArray(command) ? command.map(String) : []; const dockerArgs = commandArgs[0] === "docker" ? commandArgs.slice(1) : commandArgs; @@ -113,6 +137,19 @@ function mockSandboxExecCurl(command, options = {}) { function mockOnboardRunCapture(command, options = {}) { const normalized = normalizeCommand(command); + if ( + normalized.startsWith("docker ps -a --no-trunc ") && + normalized.includes("label=openshell.ai/sandbox-name=my-assistant") && + normalized.endsWith("--format {{.ID}}") + ) { + return `${ONBOARD_SANDBOX_OLD_CONTAINER_ID}\n${ONBOARD_SANDBOX_NEW_CONTAINER_ID}\n`; + } + if ( + normalized === + `docker inspect --type container ${ONBOARD_SANDBOX_OLD_CONTAINER_ID}` + ) { + return JSON.stringify([ONBOARD_SANDBOX_INSPECT]); + } if (isOpenClawSecurityInventoryProbe(command)) { return "nemoclaw-security-inventory-ok"; } diff --git a/test/onboard-sandbox-build.test.ts b/test/onboard-sandbox-build.test.ts index f5cb4deb67a..5d1d23ef073 100644 --- a/test/onboard-sandbox-build.test.ts +++ b/test/onboard-sandbox-build.test.ts @@ -216,6 +216,15 @@ const { createSandbox } = require(${onboardPath}); ), "expected dashboard forward (loopback or WSL 0.0.0.0)", ); + assert.ok( + payload.commands.some( + (entry: CommandEntry) => + entry.command.includes("docker run -d") && + entry.command.includes("OPENSHELL_SANDBOX_COMMAND=") && + entry.command.includes("nemoclaw-start"), + ), + "expected the default OpenClaw startup command to be persisted in the recreated container", + ); }); it("skips OpenClaw sandbox-base resolution for agent-staged Dockerfiles", async () => { diff --git a/test/pr-risk-plan.test.ts b/test/pr-risk-plan.test.ts index bc92c7d0592..4c6962bcbdd 100644 --- a/test/pr-risk-plan.test.ts +++ b/test/pr-risk-plan.test.ts @@ -274,8 +274,12 @@ describe("deterministic PR risk plan", () => { expect(riskPlanRequiredTargetIds(docsAndTestsOnly)).toEqual([]); }); - it("selects post-reboot recovery for status delivery recovery changes (#7824)", () => { - const changedFile = "src/lib/actions/sandbox/status-snapshot.ts"; + it.each([ + "src/lib/actions/sandbox/status-snapshot.ts", + "src/lib/onboard/docker-driver-sandbox-recovery.ts", + "src/lib/onboard/docker-startup-command-agent.ts", + "src/lib/onboard/sandbox-create-step.ts", + ])("selects post-reboot recovery for Docker delivery changes in %s (#7824)", (changedFile) => { const result = plan(changedFile); const adjacentStatusFile = plan("src/lib/actions/sandbox/status-text.ts"); diff --git a/tools/advisors/risk-plan.mts b/tools/advisors/risk-plan.mts index 09514a3766a..6eacfb7d51a 100644 --- a/tools/advisors/risk-plan.mts +++ b/tools/advisors/risk-plan.mts @@ -14,7 +14,12 @@ const PR_E2E_TYPED_TARGET_ID_SET = new Set(PR_E2E_TYPED_TARGET_IDS); const DEEPAGENTS_HEADLESS_INFERENCE_CHECK = "test/e2e/e2e-cloud-experimental/checks/07-deepagents-code-headless-inference.sh"; const DEEPAGENTS_CODE_RUNTIME_ROOT = "agents/langchain-deepagents-code/"; -const POST_REBOOT_STATUS_RUNTIME = "src/lib/actions/sandbox/status-snapshot.ts"; +const POST_REBOOT_DELIVERY_RUNTIME_FILES = new Set([ + "src/lib/actions/sandbox/status-snapshot.ts", + "src/lib/onboard/docker-driver-sandbox-recovery.ts", + "src/lib/onboard/docker-startup-command-agent.ts", + "src/lib/onboard/sandbox-create-step.ts", +]); export type RiskTier = 0 | 1 | 2 | 3; export type RiskFamilyId = @@ -129,9 +134,9 @@ export function focusedPrE2eTargetsForChangedFiles( (file.startsWith(DEEPAGENTS_CODE_RUNTIME_ROOT) && isRuntimeRelevant(file)), ), ); - const postRebootMatchedFiles = changedFiles.includes(POST_REBOOT_STATUS_RUNTIME) - ? [POST_REBOOT_STATUS_RUNTIME] - : []; + const postRebootMatchedFiles = stableUnique( + changedFiles.filter((file) => POST_REBOOT_DELIVERY_RUNTIME_FILES.has(file)), + ); return [ ...(deepAgentsMatchedFiles.length > 0 ? [