diff --git a/.agents/skills/_shared/controlled-words.md b/.agents/skills/_shared/controlled-words.md index 866123359ae..eeac64fe12c 100644 --- a/.agents/skills/_shared/controlled-words.md +++ b/.agents/skills/_shared/controlled-words.md @@ -184,7 +184,7 @@ other literal identifiers. | `custom endpoint` | Technical noun | A user-supplied inference endpoint that is not one of NemoClaw's named provider choices. | compatible endpoint before compatibility is validated | | `hosted inference` | Technical noun | Inference served by a remote provider-operated service. | local inference, cloud model | | `inference` | Technical noun | Model execution that produces a response from an input. | AI, generation when model execution is intended | -| `inference health` | Technical noun | The complete classification from the named `inference.local` `/v1/models` status probe: `reachable` for HTTP `200` through `499`, `unhealthy` for HTTP `500` through `599`, or `unreachable` when no qualifying HTTP response arrives. | model health, successful inference, validation request | +| `inference health` | Technical noun | The classification reported for a sandbox's inference route: the `inference.local` `/v1/models` probe result, and, when that route is reachable and the sandbox records a provider and a model, the result of one inference request sent over the same route. Values are `healthy`, `unauthorized`, `reachable`, `unhealthy`, `unreachable`, and `not probed`. | model health, successful inference | | `inference profile` | Technical noun | A blueprint selection that defines an inference provider type, provider name, endpoint, model, credential input, and route settings. | provider profile, model profile | | `inference request` | Technical noun | One request sent through an inference route to a model. | API call when the inference purpose matters | | `inference route reachability` | Technical noun | The `reachable` inference-health result produced when `https://inference.local/v1/models` returns HTTP `200` through `499`. It does not establish valid credentials, successful model invocation, readiness, compatibility, or support. | inference health, successful inference, validation request | @@ -391,7 +391,7 @@ A result can support more than one claim only when its evidence meets each defin | Class | Claim | Establishes | Does not establish | |---|---|---|---| | Operational | `inference route reachability` | The named `/v1/models` route returned HTTP `200` through `499`. | Valid credentials, successful model invocation, readiness, compatibility, or support. | -| Operational | `inference health` | The named `/v1/models` probe produced a `reachable`, `unhealthy`, or `unreachable` classification. | Valid credentials, successful model invocation, readiness, compatibility, or support. | +| Operational | `inference health` | The named `/v1/models` probe classification, plus the result of one inference request over the same route when NemoClaw sent one. | Broader API conformance, other requests or models, readiness, compatibility, or support. | | Operational | `readiness check` | A service or resource meets named criteria to begin its intended work. | Broader reliability, compatibility, or support. | | Operational | `validation request` | One authenticated request succeeded for the named endpoint, API family, model, and request shape. | Broader API conformance, other requests or models, reliability, or support. | | Evidence | `verification` | Evidence confirms the stated result for the named revision and environment. | Compatibility or support unless the evidence and decision establish them. | diff --git a/ci/source-architecture-budget.json b/ci/source-architecture-budget.json index 51bc8d20f1b..be263d10474 100644 --- a/ci/source-architecture-budget.json +++ b/ci/source-architecture-budget.json @@ -40,7 +40,7 @@ "src/lib/actions/sandbox/connect.ts": 38, "src/lib/actions/sandbox/destroy.ts": 29, "src/lib/actions/sandbox/doctor.ts": 30, - "src/lib/actions/sandbox/status-snapshot.ts": 21, + "src/lib/actions/sandbox/status-snapshot.ts": 20, "src/lib/actions/sandbox/policy-channel.ts": 29, "src/lib/actions/sandbox/process-recovery.ts": 21, "src/lib/actions/sandbox/rebuild-pipeline.ts": 28, diff --git a/docs/deployment/deploy-to-headless-server.mdx b/docs/deployment/deploy-to-headless-server.mdx index f6a229e1ea2..8ab30001168 100644 --- a/docs/deployment/deploy-to-headless-server.mdx +++ b/docs/deployment/deploy-to-headless-server.mdx @@ -235,8 +235,8 @@ $$nemoclaw headless-agent connect --probe-only ``` `$$nemoclaw headless-agent status` exits nonzero when the sandbox, gateway, local container, or authoritative inference route is not verified. -Its main `Inference` line probes `https://inference.local/v1/models` from inside the sandbox. -HTTP `200` through `499` reports `reachable`, while HTTP `500` through `599` reports `unhealthy`. +Its main `Inference` line probes `https://inference.local/v1/models` from inside the sandbox, then sends one inference request over the same route when that probe reports the route reachable. +The line reports `healthy` when the route served the request, `unauthorized` when the route rejected it with HTTP `401` or `403`, and `unhealthy` when the route returned HTTP `500` through `599`. During an SSH session, `status` points to `$$nemoclaw headless-agent dashboard-url` only when the agent gateway is running and loopback dashboard access needs a port forward. @@ -251,7 +251,7 @@ It does not restart or replace the shared host OpenShell gateway. Readiness requires all of these results: - The exact OpenShell sandbox row is `Ready` or `Running`. -- `$$nemoclaw headless-agent status` exits with status `0` and reports the inference route as `reachable`. +- `$$nemoclaw headless-agent status` exits with status `0` and reports the inference route as `healthy`. - `$$nemoclaw headless-agent connect --probe-only` exits with status `0`. ## Access the Dashboard and API diff --git a/docs/inference/choose-local-inference-server.mdx b/docs/inference/choose-local-inference-server.mdx index 359ae3e1e61..8d0fe656f0f 100644 --- a/docs/inference/choose-local-inference-server.mdx +++ b/docs/inference/choose-local-inference-server.mdx @@ -127,8 +127,8 @@ $$nemoclaw my-assistant status $$nemoclaw my-assistant doctor ``` -Accept the result when `status` reports the inference route as `reachable` and `doctor` exits with status `0`. -Route reachability does not by itself establish successful model invocation. +Accept the result when `status` reports the inference route as `healthy` and `doctor` exits with status `0`. +The `healthy` state means the route served one inference request; it does not establish results for other requests or models. Verify the bounded host publication: diff --git a/docs/inference/set-up-llama-cpp.mdx b/docs/inference/set-up-llama-cpp.mdx index f47315995fe..4daefc2059b 100644 --- a/docs/inference/set-up-llama-cpp.mdx +++ b/docs/inference/set-up-llama-cpp.mdx @@ -158,7 +158,7 @@ $$nemoclaw my-assistant doctor docker inspect --format '{{json .HostConfig.PortBindings}}' nemoclaw-llama-cpp ``` -Accept the result when `status` reports `Managed llama.cpp: running`, the inference route is `reachable`, and `doctor` exits with status `0`. +Accept the result when `status` reports `Managed llama.cpp: running`, the inference route is `healthy`, and `doctor` exits with status `0`. The Docker inspection output must be `{}`, confirming that the container has no Docker-published port. The host-owned private bridge still listens on loopback. These checks do not establish agent and model qualification. diff --git a/docs/inference/set-up-ollama.mdx b/docs/inference/set-up-ollama.mdx index de52d9f7760..c079a73cde8 100644 --- a/docs/inference/set-up-ollama.mdx +++ b/docs/inference/set-up-ollama.mdx @@ -178,7 +178,8 @@ The wizard manages the proxy lifecycle: All proxy endpoints require the token, including `GET /api/tags`. The host-side proxy liveness check treats any HTTP response, including `401`, as evidence that the proxy answered. -The authoritative `status` and `doctor` route probe runs inside the sandbox and reports HTTP `200` through `499` as `healthy`, HTTP `500` through `599` as `unhealthy`, and transport failures or unavailable probes as `unreachable` or `not probed`. +The `status` and `doctor` route probe runs inside the sandbox and reports HTTP `200` through `499` as reachable, HTTP `500` through `599` as `unhealthy`, and transport failures or unavailable probes as `unreachable` or `not probed`. +`status` then sends one inference request over the reachable route and reports `healthy` when the route served it, or `unauthorized` when the proxy token is no longer accepted. If Ollama is already bound to a non-loopback address, onboarding restarts it on `127.0.0.1:11434` so the proxy becomes the only network path. diff --git a/docs/inference/set-up-vllm-on-two-dgx-stations.mdx b/docs/inference/set-up-vllm-on-two-dgx-stations.mdx index 5280e41ebe8..11086b51602 100644 --- a/docs/inference/set-up-vllm-on-two-dgx-stations.mdx +++ b/docs/inference/set-up-vllm-on-two-dgx-stations.mdx @@ -82,8 +82,8 @@ $$nemoclaw my-assistant status ``` Confirm that `inference get` reports the `vllm-local` provider and the `nemotron-ultra` model. -Continue only when the `Inference` row in the status output reports `reachable`. -This result confirms that the sandbox route returned HTTP `200` through `499`; it does not prove a successful model invocation. +Continue only when the `Inference` row in the status output reports `healthy`. +This result confirms that the sandbox route served one inference request; it does not establish results for other requests or models. ## Review Reboot and Resume Behavior diff --git a/docs/monitoring/monitor-sandbox-activity.mdx b/docs/monitoring/monitor-sandbox-activity.mdx index 1f6dfcdbe2d..95caecebdf8 100644 --- a/docs/monitoring/monitor-sandbox-activity.mdx +++ b/docs/monitoring/monitor-sandbox-activity.mdx @@ -122,7 +122,7 @@ If the request fails, check these items. 1. Run `$$nemoclaw status` to confirm the active provider and endpoint. Check the main `Inference` line first. - If it shows `unhealthy`, `unreachable`, or `not probed`, inspect the labeled upstream, local-backend, and auth-proxy diagnostics to identify the failing hop. + If it shows `unauthorized`, `unhealthy`, `unreachable`, or `not probed`, inspect the labeled upstream, local-backend, and auth-proxy diagnostics to identify the failing hop. Restart a local backend only when its own diagnostic fails. 2. Run `$$nemoclaw logs --follow` to view error messages from the blueprint runner. 3. Verify that the host can reach the inference endpoint. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 0d5e11f465b..4cf7fbcc41b 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -1617,6 +1617,13 @@ Starting an already-running sandbox skips the container start and still runs the A paused container is unpaused. If the container was removed entirely, `start` fails and points you to `$$nemoclaw rebuild`. +After the gateway and forward checks pass, `start` sends one inference request through `https://inference.local` using the sandbox's recorded provider and model. +A gateway that answers the `/v1/models` probe can still reject an inference request, so the command exits non-zero when that request fails. +It prints the probe result, including the HTTP status when the route returned one, and points you to the sandbox doctor command. +Each run sends one 16-token request through the stored provider credential, so `start` waits up to 30 seconds for it and consumes provider tokens on a hosted route. +When the sandbox records no provider or no model, `start` skips the request and exits `0`. +`doctor` still classifies an HTTP `401` or `403` route response as reachable, so correct the provider credential when `start` reports one of those statuses. + ### `$$nemoclaw status` Show sandbox-scoped status, health, and inference configuration for one registered sandbox. @@ -1643,6 +1650,19 @@ Refer to [Use Shared Gateway Routes](../inference/manage-inference/use-shared-ga `openshellDriver` and `openshellVersion` are always strings (falling back to `"unknown"` when the registry has no value), so consumers can rely on `typeof` checks. `agent` is always a string and reports `openclaw` when the registry records no agent for the sandbox. `failureLayer` is `null` when no preflight failure was detected and otherwise one of `docker_unreachable`, `sandbox_container_stopped`, or `sandbox_dashboard_port_conflict`; when set, `inferenceHealth` is suppressed to `null` so automation does not see a stale remote-provider healthy status during a local outage. +`inferenceHealth.ok` reports whether the inference route served one inference request sent from inside the sandbox, not whether that route answered an HTTP request. +The route probe treats any final HTTP status from `200` through `499` as reachable, so a route with an invalidated provider credential answers HTTP `401` while the route is up. +The request uses the live gateway route's provider and model, and falls back to the recorded values when the live route is unreadable. +Each run sends one 16-token request through the stored provider credential, so `status` waits up to 30 seconds for it and consumes provider tokens on a hosted route. +When NemoClaw sends an inference request, `inferenceHealth.subprobes` reports the route probe result as the `route reachability` hop, so a failing verdict still shows that the route itself answered. +`inferenceHealth.failureLabel` reports why the inference request failed: + +- `unauthorized` when the route rejected it with HTTP `401` or `403`. +- `unhealthy` when the route returned another failing HTTP status. +- `unreachable` when the request returned no HTTP status, including a probe that could not run. + +A host-side upstream probe under `inferenceHealth.subprobes` stays a diagnostic and does not change `inferenceHealth.ok`, because the sandbox route is the one the agent uses. +When the route probe failed, or the sandbox records no provider or no model, NemoClaw skips the inference request and `inferenceHealth` reports the route probe result alone. `dockerPaused` is `true` when NemoClaw detects that the Docker-driver sandbox container is paused. In that case, text output keeps OpenShell's authoritative phase but prints a `docker unpause ` recovery hint instead of sending you directly to rebuild. For terminal runtime sandboxes, the command also checks cgroup OOM kill counters. @@ -1686,19 +1706,22 @@ $$nemoclaw my-assistant status --json $$nemoclaw sandbox status my-assistant --json ``` -The command probes `https://inference.local/v1/models` from inside the sandbox as the authoritative inference health check. -This check exercises the same route that agent traffic uses. +The command probes `https://inference.local/v1/models` from inside the sandbox, and when that probe reports the route reachable it sends one inference request over the same route. +That inference request is the authoritative inference health check, and both checks exercise the route that agent traffic uses. The main `Inference` line reports one of these states: | State | Meaning | |-------|---------| -| `reachable` | The route returned an HTTP status from `200` through `499`. Authentication responses such as `401` and `403` confirm route reachability. | -| `unhealthy` | The route returned an HTTP status from `500` through `599`. | -| `unreachable` | The route had a transport failure, returned no final HTTP status (`000` or an interim `100` through `199`), or returned an invalid status outside `100` through `599`. | +| `healthy` | The route served the inference request. | +| `unauthorized` | The route rejected the inference request with HTTP `401` or `403`. | +| `reachable` | The route returned an HTTP status from `200` through `499` and NemoClaw did not send an inference request. | +| `unhealthy` | The route returned an HTTP status from `500` through `599`, or it rejected the inference request with another failing status. | +| `unreachable` | The route had a transport failure, returned no final HTTP status (`000` or an interim `100` through `199`), returned an invalid status outside `100` through `599`, or the inference request returned no HTTP status. | | `not probed` | NemoClaw could not run the authoritative route probe from a reachable sandbox. | | `not verified` | NemoClaw could not verify the sandbox or gateway state, so it skips inference probing. | -An authentication response confirms that the route is reachable, not that provider credentials are valid. +An authentication response on the route probe alone confirms that the route is reachable, not that provider credentials are valid. +`$$nemoclaw doctor` sends no inference request, so it reports an HTTP `401` or `403` route response as reachable and exits `0` where `status` reports `unauthorized`. The command can also print direct host-side provider checks such as `Inference (upstream)` and provider-specific subprobes. For supported remote providers, this diagnostic sends an authenticated request to the configured model and accepts only a recognized Chat Completions, streaming Chat Completions, or Anthropic Messages response. It uses a 3-second connection timeout, a 5-second total timeout, and an 8-token output limit. diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index 46614931f5b..7a6d6ec2152 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -1883,8 +1883,9 @@ Check the active provider and endpoint: $$nemoclaw status ``` -The main `Inference` line probes `https://inference.local/v1/models` from inside the sandbox, so it reflects the route the agent actually uses. -If that line shows `unhealthy`, `unreachable`, or `not probed`, inspect the labeled diagnostic lines to identify the failing hop. +The main `Inference` line probes `https://inference.local/v1/models` from inside the sandbox and then sends one inference request over the same route, so it reflects the route the agent uses. +If that line shows `unauthorized`, `unhealthy`, `unreachable`, or `not probed`, inspect the labeled diagnostic lines to identify the failing hop. +An `unauthorized` line means the route answered but rejected the request, so refresh the provider credential rather than the route. For local Ollama and local vLLM, `Inference (ollama backend)` or the corresponding local-backend line reports the host-side service separately. For Local Ollama, current releases can also print `Inference (auth proxy)` when a proxy token is available. If a local backend or auth-proxy diagnostic fails, start the backend or re-run onboarding so NemoClaw can recreate the proxy token, restart the proxy, and refresh the route. diff --git a/docs/security/credential-rotation.mdx b/docs/security/credential-rotation.mdx index 842f183e475..d1a436f18c2 100644 --- a/docs/security/credential-rotation.mdx +++ b/docs/security/credential-rotation.mdx @@ -197,7 +197,8 @@ Plan for rebuild downtime when automating messaging rotation and recreation down ## Verify the Replacement `$$nemoclaw credentials list` confirms that a provider exists, but it does not reveal or validate the stored value. -`$$nemoclaw status` performs reachability checks without sending cloud API keys, so even an HTTP `401` or `403` can count as reachable. +`$$nemoclaw status` performs host-side reachability checks without sending cloud API keys, so even an HTTP `401` or `403` can count as reachable there. +`$$nemoclaw status` also sends one inference request through the stored provider credential and reports `unauthorized` when the route rejects it with HTTP `401` or `403`. `$$nemoclaw inference get` reports the active route and does not authenticate a model request. Complete a real request through the rotated integration before declaring the rotation successful. diff --git a/src/lib/actions/sandbox/rebuild-inference-preflight.test.ts b/src/lib/actions/sandbox/inference-invocation-probe.test.ts similarity index 78% rename from src/lib/actions/sandbox/rebuild-inference-preflight.test.ts rename to src/lib/actions/sandbox/inference-invocation-probe.test.ts index 3f267c24f9e..8131c905983 100644 --- a/src/lib/actions/sandbox/rebuild-inference-preflight.test.ts +++ b/src/lib/actions/sandbox/inference-invocation-probe.test.ts @@ -4,9 +4,9 @@ import { describe, expect, it, vi } from "vitest"; import { - buildRebuildInferenceProbeCommand, - preflightRebuildInferenceRoute, -} from "./rebuild-inference-preflight"; + buildSandboxInferenceInvocationCommand, + probeSandboxInferenceInvocation, +} from "./inference-invocation-probe"; const input = { sandboxName: "dcode-workspace", @@ -15,9 +15,9 @@ const input = { preferredInferenceApi: "openai-completions", }; -describe("atomic rebuild inference preflight", () => { +describe("sandbox inference invocation probe", () => { it("probes the recorded model through inference.local without embedding a credential (#6195)", () => { - const command = buildRebuildInferenceProbeCommand(input); + const command = buildSandboxInferenceInvocationCommand(input); expect(command).toContain("https://inference.local/v1/chat/completions"); expect(command).toContain('"model":"nvidia/nemotron"'); @@ -33,11 +33,12 @@ describe("atomic rebuild inference preflight", () => { stderr: "upstream authentication failed for sk-secret-value-that-is-long-enough", })); - const result = preflightRebuildInferenceRoute(input, { execute }); + const result = probeSandboxInferenceInvocation(input, { execute }); expect(result).toEqual({ ok: false, - detail: "existing sandbox inference probe returned HTTP 401", + detail: "sandbox inference invocation probe returned HTTP 401", + httpStatus: 401, }); expect(JSON.stringify(result)).not.toContain("sk-secret-value-that-is-long-enough"); }); @@ -49,11 +50,12 @@ describe("atomic rebuild inference preflight", () => { stderr: "upstream echoed canary-replay-marker", })); - const result = preflightRebuildInferenceRoute(input, { execute }); + const result = probeSandboxInferenceInvocation(input, { execute }); expect(result).toEqual({ ok: false, - detail: "existing sandbox inference probe returned HTTP 500", + detail: "sandbox inference invocation probe returned HTTP 500", + httpStatus: 500, }); expect(JSON.stringify(result)).not.toContain("canary-replay-marker"); }); @@ -61,11 +63,11 @@ describe("atomic rebuild inference preflight", () => { it("accepts a successful completion through the stored gateway route (#6195)", () => { const execute = vi.fn(() => ({ status: 0, stdout: "200\n{}", stderr: "" })); - expect(preflightRebuildInferenceRoute(input, { execute })).toEqual({ ok: true }); + expect(probeSandboxInferenceInvocation(input, { execute })).toEqual({ ok: true }); }); it("sends max_completion_tokens for a GPT-5 model on the chat completions route", () => { - const command = buildRebuildInferenceProbeCommand({ ...input, model: "gpt-5.4" }); + const command = buildSandboxInferenceInvocationCommand({ ...input, model: "gpt-5.4" }); expect(command).toContain("https://inference.local/v1/chat/completions"); expect(command).toContain('"max_completion_tokens":16'); @@ -73,21 +75,21 @@ describe("atomic rebuild inference preflight", () => { }); it("sends max_completion_tokens for an o-series model on the chat completions route", () => { - const command = buildRebuildInferenceProbeCommand({ ...input, model: "o3-mini" }); + const command = buildSandboxInferenceInvocationCommand({ ...input, model: "o3-mini" }); expect(command).toContain('"max_completion_tokens":16'); expect(command).not.toContain('"max_tokens"'); }); it("keeps max_tokens for a model that supports the legacy chat completions field", () => { - const command = buildRebuildInferenceProbeCommand({ ...input, model: "nvidia/nemotron" }); + const command = buildSandboxInferenceInvocationCommand({ ...input, model: "nvidia/nemotron" }); expect(command).toContain('"max_tokens":16'); expect(command).not.toContain('"max_completion_tokens"'); }); it("sends max_output_tokens on the responses route", () => { - const command = buildRebuildInferenceProbeCommand({ + const command = buildSandboxInferenceInvocationCommand({ ...input, preferredInferenceApi: "openai-responses", }); @@ -106,7 +108,7 @@ describe("atomic rebuild inference preflight", () => { ["anthropic messages", "claude-sonnet-4-6", "anthropic-messages", "max_tokens"], ])("requests a reply budget the endpoint accepts on the %s route (#7939)", (_route, model, preferredInferenceApi, field) => { const endpointMinimumReplyTokens = 16; - const command = buildRebuildInferenceProbeCommand({ + const command = buildSandboxInferenceInvocationCommand({ ...input, model, preferredInferenceApi, diff --git a/src/lib/actions/sandbox/rebuild-inference-preflight.ts b/src/lib/actions/sandbox/inference-invocation-probe.ts similarity index 61% rename from src/lib/actions/sandbox/rebuild-inference-preflight.ts rename to src/lib/actions/sandbox/inference-invocation-probe.ts index 739fb82d13a..aa0c16b6a70 100644 --- a/src/lib/actions/sandbox/rebuild-inference-preflight.ts +++ b/src/lib/actions/sandbox/inference-invocation-probe.ts @@ -6,20 +6,29 @@ import { MIN_PROBE_REPLY_TOKENS, resolveMaxTokensField } from "../../inference/m import { shellQuote } from "../../runner"; import { executeSandboxExecCommand, type SandboxCommandResult } from "./process-recovery"; -export type RebuildInferencePreflightInput = { +export type SandboxInferenceInvocationInput = { sandboxName: string; provider: string; model: string; preferredInferenceApi: string | null; }; -export type RebuildInferencePreflightResult = { ok: true } | { ok: false; detail: string }; +export type SandboxInferenceInvocationResult = + | { ok: true } + | { ok: false; detail: string; httpStatus: number | null }; -export type RebuildInferencePreflightDeps = { +export type SandboxInferenceInvocationDeps = { execute?: (sandboxName: string, command: string, timeout?: number) => SandboxCommandResult | null; }; -function buildProbeRequest(input: RebuildInferencePreflightInput): { +/** + * Rebuild preflight recreates the sandbox and tolerates a slow first token. + * Status and start run in an interactive wait and use the shorter timeout. + */ +export const REBUILD_INFERENCE_INVOCATION_TIMEOUT_MS = 100_000; +export const READINESS_INFERENCE_INVOCATION_TIMEOUT_MS = 30_000; + +function buildProbeRequest(input: SandboxInferenceInvocationInput): { endpoint: string; headers: string[]; payload: Record; @@ -63,7 +72,9 @@ function buildProbeRequest(input: RebuildInferencePreflightInput): { }; } -export function buildRebuildInferenceProbeCommand(input: RebuildInferencePreflightInput): string { +export function buildSandboxInferenceInvocationCommand( + input: SandboxInferenceInvocationInput, +): string { const request = buildProbeRequest(input); const headerArgs = ["Content-Type: application/json", ...request.headers] .map((header) => `-H ${shellQuote(header)}`) @@ -78,23 +89,36 @@ export function buildRebuildInferenceProbeCommand(input: RebuildInferencePreflig } /** - * Exercise the configured gateway route from the still-running sandbox. The - * request uses OpenShell's stored provider credential through inference.local; - * no host credential is placed in the command or its output. + * Send one minimal agent request over the configured gateway route from the + * still-running sandbox. The request uses OpenShell's stored provider + * credential through inference.local; no host credential is placed in the + * command or its output. */ -export function preflightRebuildInferenceRoute( - input: RebuildInferencePreflightInput, - deps: RebuildInferencePreflightDeps = {}, -): RebuildInferencePreflightResult { +export function probeSandboxInferenceInvocation( + input: SandboxInferenceInvocationInput, + deps: SandboxInferenceInvocationDeps = {}, + timeoutMs: number = REBUILD_INFERENCE_INVOCATION_TIMEOUT_MS, +): SandboxInferenceInvocationResult { const execute = deps.execute ?? executeSandboxExecCommand; - const result = execute(input.sandboxName, buildRebuildInferenceProbeCommand(input), 100_000); + const result = execute( + input.sandboxName, + buildSandboxInferenceInvocationCommand(input), + timeoutMs, + ); if (result?.status === 0) return { ok: true }; - if (!result) return { ok: false, detail: "existing sandbox inference probe was unavailable" }; + if (!result) { + return { + ok: false, + detail: "sandbox inference invocation probe was unavailable", + httpStatus: null, + }; + } const httpStatus = result.stdout.match(/(?:^|\n)([1-5]\d\d)(?:\n|$)/)?.[1]; return { ok: false, detail: httpStatus - ? `existing sandbox inference probe returned HTTP ${httpStatus}` - : `existing sandbox inference probe exited with status ${result.status}`, + ? `sandbox inference invocation probe returned HTTP ${httpStatus}` + : `sandbox inference invocation probe exited with status ${result.status}`, + httpStatus: httpStatus ? Number.parseInt(httpStatus, 10) : null, }; } diff --git a/src/lib/actions/sandbox/inference-route-health.ts b/src/lib/actions/sandbox/inference-route-health.ts index 8a3559bd719..f99d5275577 100644 --- a/src/lib/actions/sandbox/inference-route-health.ts +++ b/src/lib/actions/sandbox/inference-route-health.ts @@ -4,12 +4,22 @@ import { captureOpenshellForStatus, isCommandTimeout } from "../../adapters/openshell/runtime"; import { OPENSHELL_INFERENCE_ROUTE_PROBE_TIMEOUT_MS } from "../../adapters/openshell/timeouts"; import * as agentRuntime from "../../agent/runtime"; +import type { ProviderHealthStatus } from "../../inference/health"; import { buildSandboxInferenceRouteProbeArgs, classifyInferenceRouteFailureLabel, isDcodeManagedExecMissingDetail, parseSandboxInferenceRouteProbeResult, } from "./connect-inference-route-probe"; +import { + probeSandboxInferenceInvocation, + READINESS_INFERENCE_INVOCATION_TIMEOUT_MS, + type SandboxInferenceInvocationInput, + type SandboxInferenceInvocationResult, +} from "./inference-invocation-probe"; + +export type { SandboxInferenceInvocationResult } from "./inference-invocation-probe"; +export type ProbeSandboxInferenceInvocation = typeof probeSandboxInferenceInvocation; export type SandboxInferenceRouteHealth = { ok: boolean; @@ -90,3 +100,124 @@ export async function probeSandboxInferenceGatewayHealth( `check the in-sandbox proxy and gateway.`, }; } + +function providerHealthDiagnostics( + providerHealth: ProviderHealthStatus | null, +): ProviderHealthStatus[] { + if (!providerHealth) return []; + const { subprobes = [], ...primary } = providerHealth; + const labeledPrimary = primary.probeLabel ? primary : { ...primary, probeLabel: "upstream" }; + return [labeledPrimary, ...subprobes]; +} + +function classifyInferenceInvocationFailureLabel( + httpStatus: number | null, +): NonNullable { + if (httpStatus === null) return "unreachable"; + if (httpStatus === 401 || httpStatus === 403) return "unauthorized"; + return "unhealthy"; +} + +/** + * Report the reachable route as its own hop so an operator can tell a broken + * route from a reachable route that will not serve an inference request. + */ +function reachableRouteSubprobe( + gateway: SandboxInferenceRouteHealth, + endpoint: string, +): ProviderHealthStatus { + return { + ok: true, + probed: true, + providerLabel: "Inference route", + probeLabel: "route reachability", + endpoint, + detail: gateway.detail, + okLabel: "reachable", + }; +} + +/** + * The route probe reads any final HTTP 200-499 as reachable, so a route with + * an invalidated provider credential answers 401 and still passes it. Health + * therefore reports the result of one inference request, and keeps the route + * probe as a subprobe so a failure shows that the route itself answered. + */ +function buildInvokedRouteHealth( + gateway: SandboxInferenceRouteHealth, + endpoint: string, + invocation: SandboxInferenceInvocationResult, +): ProviderHealthStatus { + if (invocation.ok) { + return { + ok: true, + probed: true, + providerLabel: "Inference route", + endpoint, + detail: "Inference gateway served an inference request on https://inference.local.", + subprobes: [reachableRouteSubprobe(gateway, endpoint)], + }; + } + return { + ok: false, + probed: true, + providerLabel: "Inference route", + endpoint, + detail: `Inference gateway did not serve an inference request: ${invocation.detail}.`, + failureLabel: classifyInferenceInvocationFailureLabel(invocation.httpStatus), + subprobes: [reachableRouteSubprobe(gateway, endpoint)], + }; +} + +export function buildSandboxInferenceRouteHealth( + gateway: SandboxInferenceRouteHealth | null, + providerHealth: ProviderHealthStatus | null, + invocation: SandboxInferenceInvocationResult | null, +): ProviderHealthStatus { + const endpoint = gateway?.endpoint ?? "https://inference.local/v1/models"; + const diagnostics = providerHealthDiagnostics(providerHealth); + let routeHealth: ProviderHealthStatus; + if (gateway?.ok && invocation) { + routeHealth = buildInvokedRouteHealth(gateway, endpoint, invocation); + } else if (gateway) { + routeHealth = { + ok: gateway.ok, + probed: true, + providerLabel: "Inference route", + endpoint, + detail: gateway.detail, + ...(gateway.ok + ? { okLabel: "reachable" } + : { + failureLabel: classifyInferenceRouteFailureLabel(gateway.httpStatus), + }), + }; + } else { + routeHealth = { + ok: false, + probed: false, + providerLabel: "Inference route", + endpoint, + detail: `Could not probe ${endpoint} from inside the sandbox.`, + }; + } + const subprobes = [...(routeHealth.subprobes ?? []), ...diagnostics]; + return subprobes.length > 0 ? { ...routeHealth, subprobes } : routeHealth; +} + +export function runSandboxInferenceInvocationProbe( + input: SandboxInferenceInvocationInput, + probe: ProbeSandboxInferenceInvocation = probeSandboxInferenceInvocation, + onProbeError: (error: unknown) => void = () => {}, +): SandboxInferenceInvocationResult { + try { + return probe(input, {}, READINESS_INFERENCE_INVOCATION_TIMEOUT_MS); + } catch (error) { + onProbeError(error); + return { + ok: false, + detail: "sandbox inference invocation probe could not run", + httpStatus: null, + }; + } +} diff --git a/src/lib/actions/sandbox/rebuild-dcode-preflight.ts b/src/lib/actions/sandbox/rebuild-dcode-preflight.ts index 67387f43ec5..cfdedaf41ab 100644 --- a/src/lib/actions/sandbox/rebuild-dcode-preflight.ts +++ b/src/lib/actions/sandbox/rebuild-dcode-preflight.ts @@ -27,13 +27,13 @@ import * as onboardSession from "../../state/onboard-session"; import * as registry from "../../state/registry"; import * as sandboxState from "../../state/sandbox"; import type { ToolDisclosure } from "../../tool-disclosure"; +import { probeSandboxInferenceInvocation } from "./inference-invocation-probe"; import { DCODE_AGENT_NAME, type ResolvedDcodeRebuildTarget, resolveDcodeRebuildTarget, } from "./rebuild-dcode-target"; import type { RebuildSandboxEntry } from "./rebuild-flow-helpers"; -import { preflightRebuildInferenceRoute } from "./rebuild-inference-preflight"; import { disposePreparedDcodeRebuildImage, type PreparedDcodeRebuildImage, @@ -212,7 +212,7 @@ function requireInferenceRoute( target: ResolvedDcodeRebuildTarget, bail: DcodeRebuildPreflightBail, ): void { - const result = preflightRebuildInferenceRoute({ sandboxName, ...target }); + const result = probeSandboxInferenceInvocation({ sandboxName, ...target }); if (!result.ok) { fail( `recorded inference credentials or route were rejected: ${result.detail}`, diff --git a/src/lib/actions/sandbox/start.test.ts b/src/lib/actions/sandbox/start.test.ts index 33f6e4a39b1..87ab9636da5 100644 --- a/src/lib/actions/sandbox/start.test.ts +++ b/src/lib/actions/sandbox/start.test.ts @@ -519,4 +519,65 @@ describe("startSandbox", () => { await expect(startSandbox("my-sandbox", h.deps)).rejects.toThrow("probe exploded"); }); + + it("reports the recorded route as ready only after it serves an agent request", async () => { + const probeInferenceInvocation = vi.fn(() => ({ ok: true }) as const); + const h = harness({ probeInferenceInvocation }); + h.getSandbox.mockReturnValue( + sandbox({ + provider: "ollama-local", + model: "nemotron-3-nano:30b", + preferredInferenceApi: "openai-completions", + }), + ); + + const result = await startSandbox("my-sandbox", h.deps); + + expect(result.exitCode).toBe(0); + expect(probeInferenceInvocation).toHaveBeenCalledWith( + { + sandboxName: "my-sandbox", + provider: "ollama-local", + model: "nemotron-3-nano:30b", + preferredInferenceApi: "openai-completions", + }, + {}, + 30_000, + ); + expect(probeInferenceInvocation.mock.invocationCallOrder[0]).toBeGreaterThan( + h.verifyGateway.mock.invocationCallOrder[0], + ); + }); + + it("exits nonzero when the started gateway will not serve an agent request", async () => { + const probeInferenceInvocation = vi.fn( + () => + ({ + ok: false, + detail: "sandbox inference invocation probe returned HTTP 401", + httpStatus: 401, + }) as const, + ); + const h = harness({ probeInferenceInvocation }); + h.getSandbox.mockReturnValue( + sandbox({ provider: "ollama-local", model: "nemotron-3-nano:30b" }), + ); + + const result = await startSandbox("my-sandbox", h.deps); + + expect(result.exitCode).toBe(1); + const output = h.log.mock.calls.map(([line]) => line).join("\n"); + expect(output).toContain("HTTP 401"); + expect(output).toContain("doctor"); + }); + + it("stays unattested instead of failing when the sandbox records no route", async () => { + const probeInferenceInvocation = vi.fn(() => ({ ok: true }) as const); + const h = harness({ probeInferenceInvocation }); + + const result = await startSandbox("my-sandbox", h.deps); + + expect(result.exitCode).toBe(0); + expect(probeInferenceInvocation).not.toHaveBeenCalled(); + }); }); diff --git a/src/lib/actions/sandbox/start.ts b/src/lib/actions/sandbox/start.ts index 1a197771925..a1ad6c85382 100644 --- a/src/lib/actions/sandbox/start.ts +++ b/src/lib/actions/sandbox/start.ts @@ -8,6 +8,11 @@ import { } from "../../onboard/runtime-provider/access"; import type { SandboxEntry } from "../../state/registry"; import * as registry from "../../state/registry"; +import { + probeSandboxInferenceInvocation, + READINESS_INFERENCE_INVOCATION_TIMEOUT_MS, + type SandboxInferenceInvocationResult, +} from "./inference-invocation-probe"; import { resolveSandboxLifecycleProvider, type SandboxLifecycleResult, @@ -56,6 +61,7 @@ export interface SandboxStartDeps { restoreStartupState?: (sandboxName: string) => SandboxStartupRecoveryResult; waitForManagedGatewaySupervisor?: (sandboxName: string) => boolean; verifyGateway?: (sandboxName: string) => Promise; + probeInferenceInvocation?: typeof probeSandboxInferenceInvocation; log?: (message: string) => void; } @@ -100,6 +106,35 @@ function preservedSandboxRecoveryError(sandboxName: string, detail: unknown): Er ); } +/** + * A started gateway that answers the /v1/models probe can still reject an + * inference request, so start sends one inference request with the recorded + * provider and model before it reports success. A registry entry with no + * provider or no model has nothing to request, so start skips the request + * instead of failing. + */ +function checkStartedSandboxInference( + sandboxName: string, + sandbox: SandboxEntry, + deps: SandboxStartDeps, + log: (message: string) => void, +): SandboxInferenceInvocationResult | null { + const model = (sandbox.model ?? "").trim(); + const provider = (sandbox.provider ?? "").trim(); + if (!model || !provider) return null; + log(" Checking that the sandbox serves an agent request…"); + return (deps.probeInferenceInvocation ?? probeSandboxInferenceInvocation)( + { + sandboxName, + provider, + model, + preferredInferenceApi: sandbox.preferredInferenceApi ?? null, + }, + {}, + READINESS_INFERENCE_INVOCATION_TIMEOUT_MS, + ); +} + /** * Restart a stopped sandbox through the lifecycle facet bound to its durable * provider identity, then restore startup state before verifying readiness and @@ -130,6 +165,7 @@ export async function startSandbox( const result = resolved.lifecycle.start(input); if (result.exitCode !== 0) return result; + const readiness: { inference: SandboxInferenceInvocationResult | null } = { inference: null }; await resolved.lifecycle.verifyStarted(input, async (name) => { log(" Restoring sandbox startup state…"); const restoreStartupState = @@ -168,6 +204,12 @@ export async function startSandbox( if (failure) throw preservedSandboxRecoveryError(name, failure); log(" Checking gateway health and host forwards…"); await (deps.verifyGateway ?? verifyGateway)(name); + readiness.inference = checkStartedSandboxInference(name, resolved.sandbox, deps, log); }); + if (readiness.inference && !readiness.inference.ok) { + log(` The sandbox started but inference is not usable: ${readiness.inference.detail}.`); + log(` Run the sandbox doctor command for '${sandboxName}' to identify the failing hop.`); + return { exitCode: 1 }; + } return { exitCode: 0 }; } diff --git a/src/lib/actions/sandbox/status-inference.test.ts b/src/lib/actions/sandbox/status-inference.test.ts index a53a1985726..3e850ab53fe 100644 --- a/src/lib/actions/sandbox/status-inference.test.ts +++ b/src/lib/actions/sandbox/status-inference.test.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { describe, expect, it, vi } from "vitest"; +import { buildSandboxInferenceInvocationCommand } from "./inference-invocation-probe"; import { collectSandboxStatusSnapshot, getSandboxStatusInferenceHealth, @@ -15,6 +16,7 @@ describe("sandbox status inference.local route health (#6192)", () => { provider?: string; liveProvider?: string; liveModel?: string; + preferredInferenceApi?: string; providerHealth?: ReturnType; providerProbeThrows?: boolean; routeHealth: { @@ -32,6 +34,7 @@ describe("sandbox status inference.local route health (#6192)", () => { agent: options.agent ?? "openclaw", model: "nvidia/nemotron", provider, + preferredInferenceApi: options.preferredInferenceApi, }; return { getSandbox: () => sandbox, @@ -57,6 +60,10 @@ describe("sandbox status inference.local route health (#6192)", () => { ? async () => Promise.reject(new Error("openshell unavailable TOKEN=super-secret")) : async () => options.routeHealth, ), + probeSandboxInferenceInvocationImpl: vi.fn( + (_input: Parameters[0]) => + ({ ok: true }) as const, + ), probeTerminalRuntimeHealth: vi.fn(() => ({ kind: "ok" as const, oomKillCount: 0 as const })), reportInferenceProbeError, }; @@ -173,9 +180,9 @@ describe("sandbox status inference.local route health (#6192)", () => { const snapshot = await collectSandboxStatusSnapshot("alpha", { deps }); expect(snapshot.inferenceHealth).toMatchObject({ ok: true, probed: true }); - expect(snapshot.inferenceHealth?.subprobes).toEqual([ + expect(snapshot.inferenceHealth?.subprobes).toContainEqual( expect.objectContaining({ ok: false, probeLabel: "upstream" }), - ]); + ); }); it("probes the live route while status displays the sandbox's recorded route (#6315)", async () => { @@ -205,6 +212,41 @@ describe("sandbox status inference.local route health (#6192)", () => { }); }); + it("does not apply the recorded API family to a different live route", async () => { + const deps = snapshotDeps({ + provider: "compatible-endpoint", + preferredInferenceApi: "openai-responses", + liveProvider: "openai-api", + liveModel: "gpt-5.2", + routeHealth: { + ok: true, + endpoint: "https://inference.local/v1/models", + httpStatus: 200, + detail: "route reachable", + }, + }); + deps.probeSandboxInferenceInvocationImpl.mockImplementation((input) => { + const command = buildSandboxInferenceInvocationCommand(input); + expect(command).toContain("https://inference.local/v1/chat/completions"); + expect(command).not.toContain("https://inference.local/v1/responses"); + return { ok: true }; + }); + + const snapshot = await collectSandboxStatusSnapshot("alpha", { deps }); + + expect(snapshot.inferenceHealth).toMatchObject({ ok: true, probed: true }); + expect(deps.probeSandboxInferenceInvocationImpl).toHaveBeenCalledWith( + { + sandboxName: "alpha", + provider: "openai-api", + model: "gpt-5.2", + preferredInferenceApi: null, + }, + {}, + 30_000, + ); + }); + it("keeps inference.local authoritative when the upstream diagnostic throws (#6192)", async () => { const deps = snapshotDeps({ providerProbeThrows: true, @@ -219,14 +261,14 @@ describe("sandbox status inference.local route health (#6192)", () => { const snapshot = await collectSandboxStatusSnapshot("alpha", { deps }); expect(snapshot.inferenceHealth).toMatchObject({ ok: true, probed: true }); - expect(snapshot.inferenceHealth?.subprobes).toEqual([ + expect(snapshot.inferenceHealth?.subprobes).toContainEqual( expect.objectContaining({ ok: false, probed: false, probeLabel: "upstream", detail: "Direct provider health probe could not run.", }), - ]); + ); }); it("preserves local backend and auth-proxy diagnostics beneath the route result", async () => { @@ -261,6 +303,7 @@ describe("sandbox status inference.local route health (#6192)", () => { const snapshot = await collectSandboxStatusSnapshot("alpha", { deps }); expect(snapshot.inferenceHealth?.subprobes?.map((probe) => probe.probeLabel)).toEqual([ + "route reachability", "ollama backend", "auth proxy", ]); diff --git a/src/lib/actions/sandbox/status-snapshot-inference-health.test.ts b/src/lib/actions/sandbox/status-snapshot-inference-health.test.ts index de7a3eee9c8..6b63206439b 100644 --- a/src/lib/actions/sandbox/status-snapshot-inference-health.test.ts +++ b/src/lib/actions/sandbox/status-snapshot-inference-health.test.ts @@ -5,6 +5,7 @@ import { describe, expect, it, vi } from "vitest"; import type { ProviderHealthStatus } from "../../inference/health"; import type { SandboxEntry } from "../../state/registry"; +import type { SandboxInferenceInvocationResult } from "./inference-invocation-probe"; import type { SandboxInferenceRouteHealth } from "./inference-route-health"; import { collectSandboxStatusSnapshot } from "./status-snapshot"; @@ -16,6 +17,7 @@ import { collectSandboxStatusSnapshot } from "./status-snapshot"; function snapshotDeps( gateway: SandboxInferenceRouteHealth | null, providerHealth: ProviderHealthStatus | null = null, + invocation: SandboxInferenceInvocationResult = { ok: true }, ) { const sandbox: SandboxEntry = { name: "alpha", @@ -38,6 +40,7 @@ function snapshotDeps( }, probeProviderHealthImpl: () => providerHealth, probeSandboxInferenceGatewayHealthImpl: async () => gateway, + probeSandboxInferenceInvocationImpl: () => invocation, }, }; } @@ -129,7 +132,7 @@ describe("collectSandboxStatusSnapshot inference route health", () => { expect(probeSandboxInferenceGatewayHealthImpl).toHaveBeenCalledTimes(2); expect(delayInferenceRecoveryProbe).toHaveBeenCalledOnce(); expect(delayInferenceRecoveryProbe).toHaveBeenCalledWith(2_000); - expect(snapshot.inferenceHealth).toMatchObject({ ok: true, okLabel: "reachable" }); + expect(snapshot.inferenceHealth).toMatchObject({ ok: true, probed: true }); }); it("reports the inference route as unreachable after all post-recovery probes", async () => { @@ -193,7 +196,7 @@ describe("collectSandboxStatusSnapshot inference route health", () => { expect(delayInferenceRecoveryProbe).not.toHaveBeenCalled(); }); - it("labels a reachable route okLabel: reachable, not a bare healthy claim (#6846)", async () => { + it("reports a served agent request as healthy and keeps reachability as its own hop (#6846)", async () => { const gateway: SandboxInferenceRouteHealth = { ok: true, endpoint: "https://inference.local/v1/models", @@ -209,9 +212,20 @@ describe("collectSandboxStatusSnapshot inference route health", () => { probed: true, providerLabel: "Inference route", endpoint: "https://inference.local/v1/models", - okLabel: "reachable", }); expect(snapshot.inferenceHealth?.failureLabel).toBeUndefined(); + expect(snapshot.inferenceHealth?.okLabel).toBeUndefined(); + expect(snapshot.inferenceHealth?.subprobes).toEqual([ + { + ok: true, + probed: true, + providerLabel: "Inference route", + probeLabel: "route reachability", + endpoint: "https://inference.local/v1/models", + detail: gateway.detail, + okLabel: "reachable", + }, + ]); }); it("does not set okLabel for a 5xx route failure, and classifies it unhealthy (#6846)", async () => { @@ -262,7 +276,7 @@ describe("collectSandboxStatusSnapshot inference route health", () => { expect(snapshot.inferenceHealth?.failureLabel).toBeUndefined(); }); - it("keeps a failed model-invocation subprobe distinct from the reachable route label (#6846)", async () => { + it("keeps an upstream subprobe failure out of the served-route verdict (#6846)", async () => { const gateway: SandboxInferenceRouteHealth = { ok: true, endpoint: "https://inference.local/v1/models", @@ -284,9 +298,60 @@ describe("collectSandboxStatusSnapshot inference route health", () => { snapshotDeps(gateway, providerHealth), ); - expect(snapshot.inferenceHealth).toMatchObject({ ok: true, okLabel: "reachable" }); - expect(snapshot.inferenceHealth?.subprobes).toEqual([ - { ...providerHealth, probeLabel: "upstream" }, - ]); + expect(snapshot.inferenceHealth).toMatchObject({ ok: true }); + expect(snapshot.inferenceHealth?.subprobes).toContainEqual({ + ...providerHealth, + probeLabel: "upstream", + }); + }); + + it("reports an unauthorized verdict when the reachable route rejects an agent request", async () => { + const gateway: SandboxInferenceRouteHealth = { + ok: true, + endpoint: "https://inference.local/v1/models", + httpStatus: 401, + detail: + "Inference gateway responded HTTP 401 on https://inference.local/v1/models (full chain reachable).", + }; + + const snapshot = await collectSandboxStatusSnapshot( + "alpha", + snapshotDeps(gateway, null, { + ok: false, + detail: "sandbox inference invocation probe returned HTTP 401", + httpStatus: 401, + }), + ); + + expect(snapshot.inferenceHealth).toMatchObject({ + ok: false, + probed: true, + failureLabel: "unauthorized", + }); + expect(snapshot.inferenceHealth?.okLabel).toBeUndefined(); + expect(snapshot.inferenceHealth?.detail).toContain("HTTP 401"); + expect(snapshot.inferenceHealth?.subprobes).toContainEqual( + expect.objectContaining({ probeLabel: "route reachability", ok: true }), + ); + }); + + it("does not send an agent request when the route probe already failed", async () => { + const gateway: SandboxInferenceRouteHealth = { + ok: false, + endpoint: "https://inference.local/v1/models", + httpStatus: 0, + detail: + "Inference gateway unreachable on https://inference.local/v1/models from inside the sandbox.", + }; + const options = snapshotDeps(gateway); + const probeSandboxInferenceInvocationImpl = vi.fn(() => ({ ok: true }) as const); + + const snapshot = await collectSandboxStatusSnapshot("alpha", { + ...options, + deps: { ...options.deps, probeSandboxInferenceInvocationImpl }, + }); + + expect(probeSandboxInferenceInvocationImpl).not.toHaveBeenCalled(); + expect(snapshot.inferenceHealth).toMatchObject({ ok: false, failureLabel: "unreachable" }); }); }); diff --git a/src/lib/actions/sandbox/status-snapshot-recovery.test.ts b/src/lib/actions/sandbox/status-snapshot-recovery.test.ts index 47008ef8e7f..48a1f768c66 100644 --- a/src/lib/actions/sandbox/status-snapshot-recovery.test.ts +++ b/src/lib/actions/sandbox/status-snapshot-recovery.test.ts @@ -73,6 +73,7 @@ function snapshotDeps(recoveryResult: unknown) { }, probeProviderHealthImpl, probeSandboxInferenceGatewayHealthImpl, + probeSandboxInferenceInvocationImpl: vi.fn(() => ({ ok: true }) as const), recoverSandboxProcesses: vi.fn(() => recoveryResult) as never, }; } diff --git a/src/lib/actions/sandbox/status-snapshot.ts b/src/lib/actions/sandbox/status-snapshot.ts index e184eb5d844..3bf055e3a88 100644 --- a/src/lib/actions/sandbox/status-snapshot.ts +++ b/src/lib/actions/sandbox/status-snapshot.ts @@ -42,11 +42,15 @@ import { buildGatewayInferenceGetArgs, canSandboxGatewayRouteRealign, } from "./connect-inference-gateway"; -import { classifyInferenceRouteFailureLabel } from "./connect-inference-route-probe"; import { getSandboxDockerRuntime } from "./docker-health"; import type { SandboxGatewayState } from "./gateway-state"; import { getReconciledSandboxGatewayState, getSandboxGatewayStateForStatus } from "./gateway-state"; -import { probeSandboxInferenceGatewayHealth } from "./inference-route-health"; +import { + buildSandboxInferenceRouteHealth, + type ProbeSandboxInferenceInvocation, + probeSandboxInferenceGatewayHealth, + runSandboxInferenceInvocationProbe, +} from "./inference-route-health"; import { getSandboxStatusPreflight, type SandboxStatusFailureLayer, @@ -110,15 +114,6 @@ export function maybeGetSandboxStatusInferenceHealth( ); } -function providerHealthDiagnostics( - providerHealth: ProviderHealthStatus | null, -): ProviderHealthStatus[] { - if (!providerHealth) return []; - const { subprobes = [], ...primary } = providerHealth; - const labeledPrimary = primary.probeLabel ? primary : { ...primary, probeLabel: "upstream" }; - return [labeledPrimary, ...subprobes]; -} - /** True when the authoritative inference route must make status exit nonzero. */ export function isInferenceHealthFailing(inferenceHealth: ProviderHealthStatus | null): boolean { return Boolean(inferenceHealth && (!inferenceHealth.probed || !inferenceHealth.ok)); @@ -155,35 +150,6 @@ export function normalizeSandboxStatusHostMounts(value: unknown): registry.Sandb }); } -function buildSandboxInferenceRouteHealth( - gateway: Awaited>, - providerHealth: ProviderHealthStatus | null, -): ProviderHealthStatus { - const endpoint = gateway?.endpoint ?? "https://inference.local/v1/models"; - const diagnostics = providerHealthDiagnostics(providerHealth); - const routeHealth: ProviderHealthStatus = gateway - ? { - ok: gateway.ok, - probed: true, - providerLabel: "Inference route", - endpoint, - detail: gateway.detail, - ...(gateway.ok - ? { okLabel: "reachable" } - : { - failureLabel: classifyInferenceRouteFailureLabel(gateway.httpStatus), - }), - } - : { - ok: false, - probed: false, - providerLabel: "Inference route", - endpoint, - detail: `Could not probe ${endpoint} from inside the sandbox.`, - }; - return diagnostics.length > 0 ? { ...routeHealth, subprobes: diagnostics } : routeHealth; -} - export interface SandboxStatusReport { schemaVersion: 1; name: string; @@ -338,6 +304,7 @@ interface CollectSandboxStatusSnapshotDeps { captureOpenshellForStatusImpl?: typeof captureOpenshellForStatus; probeProviderHealthImpl?: ProbeProviderHealth; probeSandboxInferenceGatewayHealthImpl?: ProbeSandboxInferenceGatewayHealth; + probeSandboxInferenceInvocationImpl?: ProbeSandboxInferenceInvocation; delayInferenceRecoveryProbe?: DelayInferenceRecoveryProbe; reportInferenceProbeError?: (message: string) => void; probeTerminalRuntimeHealth?: ProbeTerminalRuntimeHealth; @@ -626,7 +593,44 @@ export async function collectSandboxStatusSnapshot( reportInferenceProbeError(error, opts.deps?.reportInferenceProbeError ?? console.error); gatewayChain = null; } - inferenceHealth = buildSandboxInferenceRouteHealth(gatewayChain, providerHealth); + // Take the provider and model as one pair. Falling back per field can pair + // a live model with a recorded provider and request a route neither one + // describes. + const invocationRoute = + live?.provider && live.model + ? { + provider: live.provider, + model: live.model, + // The live gateway RPC does not expose a stored API override. Do + // not combine its provider/model with the recorded sandbox's API + // family; resolve the live route's compatible default instead. + preferredInferenceApi: null, + } + : { + provider: currentProvider, + model: currentModel, + preferredInferenceApi: sb?.preferredInferenceApi ?? null, + }; + const invocationModel = (invocationRoute.model || "").trim(); + const invocationProvider = (invocationRoute.provider || "").trim(); + const invocation = + gatewayChain?.ok && invocationModel && invocationProvider + ? runSandboxInferenceInvocationProbe( + { + sandboxName, + provider: invocationProvider, + model: invocationModel, + preferredInferenceApi: invocationRoute.preferredInferenceApi, + }, + opts.deps?.probeSandboxInferenceInvocationImpl, + (error) => + reportInferenceProbeError( + error, + opts.deps?.reportInferenceProbeError ?? console.error, + ), + ) + : null; + inferenceHealth = buildSandboxInferenceRouteHealth(gatewayChain, providerHealth, invocation); } const statusAgent = resolveSandboxStatusAgent(sb?.agent || "openclaw"); const terminalRuntimeHealth = diff --git a/test/cli/helpers.ts b/test/cli/helpers.ts index 7ae4ac1a531..45168a067c3 100644 --- a/test/cli/helpers.ts +++ b/test/cli/helpers.ts @@ -8,6 +8,7 @@ import os from "node:os"; import path from "node:path"; import { parse as parseYaml } from "yaml"; +import { SANDBOX_EXEC_STARTED_MARKER } from "../../src/lib/actions/sandbox/sandbox-exec-output"; import type { OwnedTestResources } from "../helpers/owned-test-resources"; import { execTimeout, testTimeout, testTimeoutOptions } from "../helpers/timeouts"; @@ -313,9 +314,27 @@ export function writeHealthyDockerStub(localBin: string): void { ); } +/** + * Answer the agent-request readiness probe inside an `openshell sandbox exec` + * stub. The probe posts a completion over the same transport as the route + * probe, and that transport trusts stdout only after the exec marker. + */ +export function inferenceInvocationStubLines(httpStatus = "200", exitCode = 0): string[] { + return [ + ' case "$*" in', + " *chat/completions*|*/v1/responses*|*/v1/messages*)", + ` printf '%s\\n' '${SANDBOX_EXEC_STARTED_MARKER}'`, + ` printf '%s\\n' ${JSON.stringify(httpStatus)}`, + ` exit ${String(exitCode)}`, + " ;;", + " esac", + ]; +} + export function healthyInferenceRouteStubLines(): string[] { return [ 'if [ "$1" = "sandbox" ] && [ "$2" = "exec" ]; then', + ...inferenceInvocationStubLines(), " echo 'OK 200'", " exit 0", "fi", diff --git a/test/cli/sandbox-status-json.test.ts b/test/cli/sandbox-status-json.test.ts index 5a2bd94dbf1..435621e497c 100644 --- a/test/cli/sandbox-status-json.test.ts +++ b/test/cli/sandbox-status-json.test.ts @@ -8,6 +8,7 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; import { + inferenceInvocationStubLines, runWithEnv, testTimeoutOptions, writeHealthyDockerStub, @@ -20,6 +21,8 @@ function createInferenceRouteStatusSetup(options: { routeExit?: number; upstreamHttpStatus?: string; upstreamExit?: number; + invocationHttpStatus?: string; + invocationExit?: number; }) { const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-status-route-")); const localBin = path.join(home, "bin"); @@ -70,6 +73,7 @@ function createInferenceRouteStatusSetup(options: { " exit 0", "fi", 'if [ "$1" = "sandbox" ] && [ "$2" = "exec" ]; then', + ...inferenceInvocationStubLines(options.invocationHttpStatus, options.invocationExit), ...(options.executeRouteCommand ? [ ' while [ "$#" -gt 0 ] && [ "$1" != "--" ]; do shift; done', @@ -156,6 +160,7 @@ describe("CLI sandbox status JSON output", testTimeoutOptions(20_000), () => { " exit 0", "fi", 'if [ "$1" = "sandbox" ] && [ "$2" = "exec" ]; then', + ...inferenceInvocationStubLines(), " echo 'OK 200'", " exit 0", "fi", @@ -272,14 +277,42 @@ describe("CLI sandbox status JSON output", testTimeoutOptions(20_000), () => { probed: true, endpoint: "https://inference.local/v1/models", }); - expect(parsed.inferenceHealth.subprobes).toEqual([ + expect(parsed.inferenceHealth.subprobes).toContainEqual( expect.objectContaining({ ok: true, probed: false, probeLabel: "upstream" }), - ]); + ); + }); + + it.each([ + 401, 403, + ])("sandbox status --json fails an inference.local HTTP %s that rejects an agent request", (httpStatus) => { + const { home, localBin, sandboxName } = createInferenceRouteStatusSetup({ + routeOutput: `OK ${httpStatus}`, + invocationHttpStatus: String(httpStatus), + invocationExit: 1, + }); + + const result = runWithEnv(`${sandboxName} status --json`, { + HOME: home, + PATH: `${localBin}:${process.env.PATH || ""}`, + }); + + expect(result.code).toBe(1); + const parsed = JSON.parse(result.out); + expect(parsed.inferenceHealth).toMatchObject({ + ok: false, + probed: true, + failureLabel: "unauthorized", + endpoint: "https://inference.local/v1/models", + }); + expect(parsed.inferenceHealth.detail).toContain(String(httpStatus)); + expect(parsed.inferenceHealth.subprobes).toContainEqual( + expect.objectContaining({ ok: true, probeLabel: "route reachability" }), + ); }); it.each([ 401, 403, - ])("sandbox status --json treats an inference.local HTTP %s as healthy (#6192)", (httpStatus) => { + ])("sandbox status --json keeps an inference.local HTTP %s reachable when it still serves an agent request (#6192)", (httpStatus) => { const { home, localBin, sandboxName } = createInferenceRouteStatusSetup({ routeOutput: `OK ${httpStatus}`, }); @@ -334,6 +367,7 @@ describe("CLI sandbox status JSON output", testTimeoutOptions(20_000), () => { [ "#!/usr/bin/env bash", 'if [ "$1" = "sandbox" ] && [ "$2" = "exec" ]; then', + ...inferenceInvocationStubLines(), " echo 'OK 200'", " exit 0", "fi", @@ -649,6 +683,7 @@ describe("CLI sandbox status JSON output", testTimeoutOptions(20_000), () => { " exit 0", "fi", 'if [ "$1" = "sandbox" ] && [ "$2" = "exec" ]; then', + ...inferenceInvocationStubLines(), " echo 'OK 200'", " exit 0", "fi", diff --git a/test/cli/sandbox-status-text.test.ts b/test/cli/sandbox-status-text.test.ts index 78eb4e24cbc..961aaf5fef9 100644 --- a/test/cli/sandbox-status-text.test.ts +++ b/test/cli/sandbox-status-text.test.ts @@ -9,6 +9,7 @@ import { describe, expect, it } from "vitest"; import { healthyInferenceRouteStubLines, + inferenceInvocationStubLines, runWithEnv, testTimeoutOptions, writeHealthyDockerStub, @@ -222,6 +223,7 @@ describe("CLI sandbox status text output", () => { " exit 0", "fi", 'if [ "$1" = "sandbox" ] && [ "$2" = "exec" ]; then', + ...inferenceInvocationStubLines(), " echo 'OK 200'", " exit 0", "fi", @@ -260,7 +262,7 @@ describe("CLI sandbox status text output", () => { expect(r.out).toContain("Run `nemoclaw alpha rebuild` to restore."); }); - it("sandbox status reports reachable inference and an unprobed upstream when openshellDriver is not docker", () => { + it("sandbox status reports served inference, its reachability hop, and an unprobed upstream when openshellDriver is not docker", () => { const home = fs.mkdtempSync( path.join(os.tmpdir(), "nemoclaw-cli-sandbox-status-non-docker-driver-"), ); @@ -310,7 +312,10 @@ describe("CLI sandbox status text output", () => { expect(r.out).toContain("Sandbox: alpha"); expect(r.out).toContain("Provider: openai-api"); expect(r.out).toContain("Model: gpt-4o-mini"); - expect(r.out).toContain("Inference: reachable (https://inference.local/v1/models)"); + expect(r.out).toContain("Inference: healthy (https://inference.local/v1/models)"); + expect(r.out).toContain( + "Inference (route reachability): reachable (https://inference.local/v1/models)", + ); expect(r.out).toContain("Inference (upstream): not probed"); }); diff --git a/test/helpers/rebuild-flow-dcode-harness.ts b/test/helpers/rebuild-flow-dcode-harness.ts index c5f0aefba55..dd32e501f68 100644 --- a/test/helpers/rebuild-flow-dcode-harness.ts +++ b/test/helpers/rebuild-flow-dcode-harness.ts @@ -413,7 +413,7 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): vi.spyOn(nim, "detectGpu").mockReturnValue(null); const routeResults = [...(overrides.dcodeRouteResults ?? [{ ok: true }])]; const preflightDcodeRouteSpy = vi - .spyOn(rebuildInference, "preflightRebuildInferenceRoute") + .spyOn(rebuildInference, "probeSandboxInferenceInvocation") .mockImplementation(() => routeResults.shift() ?? { ok: true }); const preparedDcodeBuildContext = { buildCtx: "/tmp/dcode-rebuild-context", diff --git a/test/helpers/rebuild-flow-harness.ts b/test/helpers/rebuild-flow-harness.ts index 177383f6842..e7f84a49a8e 100644 --- a/test/helpers/rebuild-flow-harness.ts +++ b/test/helpers/rebuild-flow-harness.ts @@ -50,7 +50,7 @@ export const processRecovery = requireDist("./process-recovery.js"); export const { rebuildOnboardDependencies } = requireDist("./rebuild-onboard-dependencies.js"); export const rebuildCustomImagePreflight = requireDist("./rebuild-custom-image-preflight.js"); export const rebuildFlowHelpers = requireDist("./rebuild-flow-helpers.js"); -export const rebuildInference = requireDist("./rebuild-inference-preflight.js"); +export const rebuildInference = requireDist("./inference-invocation-probe.js"); export const rebuildManagedImage = requireDist("./rebuild-managed-image-preflight.js"); export const rebuildMessagingConflict = requireDist("./rebuild-messaging-conflict-preflight.js"); export const rebuildPreparedImageContext = requireDist("./rebuild-prepared-image-context.js"); diff --git a/test/nemoclaw-cli-recovery.test.ts b/test/nemoclaw-cli-recovery.test.ts index e0bccfb09ed..139ea8c00c9 100644 --- a/test/nemoclaw-cli-recovery.test.ts +++ b/test/nemoclaw-cli-recovery.test.ts @@ -7,6 +7,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { describe, it } from "vitest"; +import { SANDBOX_EXEC_STARTED_MARKER } from "../src/lib/actions/sandbox/sandbox-exec-output"; import { testTimeoutOptions } from "./helpers/timeouts"; describe("nemoclaw CLI runtime recovery", () => { @@ -84,6 +85,11 @@ if (args[0] === "sandbox" && args[1] === "get" && (args[2] === "my-assistant" || } if (args[0] === "sandbox" && args[1] === "exec") { + const command = args.join(" "); + if (command.includes("chat/completions") || command.includes("/v1/responses") || command.includes("/v1/messages")) { + process.stdout.write(${JSON.stringify(`${SANDBOX_EXEC_STARTED_MARKER}\n200\n`)}); + process.exit(0); + } process.stdout.write("OK 200\\n"); process.exit(0); }