fix(dcode): classify managed non-interactive failures (#8121)

agents/langchain-deepagents-code/patch-managed-deepagents-code.py

@@ -786,21 +786,118 @@ def _run_single_hook(command, event, payload_bytes) -> None:
 import json as _nemoclaw_json
 import logging as _nemoclaw_logging
 import os as _nemoclaw_os
-import re as _nemoclaw_re
-import sqlite3 as _nemoclaw_sqlite3
+import ssl as _nemoclaw_ssl
 import sys as _nemoclaw_sys
 import threading as _nemoclaw_threading
 import time as _nemoclaw_time
 
+import httpx as _nemoclaw_httpx
+from deepagents_code import model_config as _nemoclaw_model_config
+from langgraph_sdk import errors as _nemoclaw_langgraph_errors
+
 # NemoClaw-managed Deep Agents Code hardening v2.
-_NEMOCLAW_PROVIDER_CAPACITY_ERROR = _nemoclaw_re.compile(
-    r"""\bAPIError\(["']ResourceExhausted:\s*
-        Worker\s+local\s+total\s+request\s+limit\s+reached\s*
-        \(\d+/\d+\)""",
-    _nemoclaw_re.IGNORECASE | _nemoclaw_re.VERBOSE,
-)
+# Classify only imported exception class objects from the active client error
+# chain. Pinned LangGraph stores `BaseException` checkpoints as
+# `repr(exception)`, which application code can replace through `__repr__`.
+# Checkpoint text is therefore not a trustworthy error-type source.
+_NEMOCLAW_EXCEPTION_CLASSIFIERS = {
+    _nemoclaw_langgraph_errors.RateLimitError: (
+        "RateLimited",
+        "rate_limited",
+        "true",
+    ),
+    _nemoclaw_langgraph_errors.AuthenticationError: (
+        "Unauthorized",
+        "authorization_rejected",
+        "false",
+    ),
+    _nemoclaw_langgraph_errors.PermissionDeniedError: (
+        "Unauthorized",
+        "authorization_rejected",
+        "false",
+    ),
+    _nemoclaw_langgraph_errors.NotFoundError: (
+        "NotFound",
+        "model_or_route_not_found",
+        "false",
+    ),
+    _nemoclaw_langgraph_errors.APITimeoutError: (
+        "Timeout",
+        "request_timeout",
+        "true",
+    ),
+    _nemoclaw_langgraph_errors.APIConnectionError: (
+        "Unavailable",
+        "route_unreachable",
+        "true",
+    ),
+    _nemoclaw_langgraph_errors.InternalServerError: (
+        "InternalServerError",
+        "remote_server_error",
+        "true",
+    ),
+    _nemoclaw_langgraph_errors.APIStatusError: (
+        "APIError",
+        "agent_remote_failure",
+        "false",
+    ),
+    _nemoclaw_langgraph_errors.APIError: (
+        "APIError",
+        "agent_remote_failure",
+        "false",
+    ),
+    _nemoclaw_httpx.ConnectTimeout: ("Timeout", "request_timeout", "true"),
+    _nemoclaw_httpx.ReadTimeout: ("Timeout", "request_timeout", "true"),
+    _nemoclaw_httpx.WriteTimeout: ("Timeout", "request_timeout", "true"),
+    _nemoclaw_httpx.PoolTimeout: ("Timeout", "request_timeout", "true"),
+    _nemoclaw_httpx.ConnectError: ("Unavailable", "route_unreachable", "true"),
+    _nemoclaw_httpx.ReadError: ("Unavailable", "route_unreachable", "true"),
+    _nemoclaw_httpx.WriteError: ("Unavailable", "route_unreachable", "true"),
+    _nemoclaw_httpx.CloseError: ("Unavailable", "route_unreachable", "true"),
+    _nemoclaw_httpx.ProxyError: ("Unavailable", "route_unreachable", "true"),
+    _nemoclaw_ssl.SSLCertVerificationError: (
+        "Unavailable",
+        "route_unreachable",
+        "true",
+    ),
+    _nemoclaw_ssl.SSLError: ("Unavailable", "route_unreachable", "true"),
+    TimeoutError: ("Timeout", "request_timeout", "true"),
+    ConnectionError: ("Unavailable", "route_unreachable", "true"),
+    ConnectionAbortedError: ("Unavailable", "route_unreachable", "true"),
+    ConnectionRefusedError: ("Unavailable", "route_unreachable", "true"),
+    ConnectionResetError: ("Unavailable", "route_unreachable", "true"),
+    BrokenPipeError: ("Unavailable", "route_unreachable", "true"),
+    _nemoclaw_model_config.ModelConfigError: (
+        "ModelConfigError",
+        "model_configuration",
+        "false",
+    ),
+    _nemoclaw_model_config.NoCredentialsConfiguredError: (
+        "ModelConfigError",
+        "model_configuration",
+        "false",
+    ),
+    _nemoclaw_model_config.UnknownProviderError: (
+        "ModelConfigError",
+        "model_configuration",
+        "false",
+    ),
+    _nemoclaw_model_config.MissingCredentialsError: (
+        "ModelConfigError",
+        "model_configuration",
+        "false",
+    ),
+    _nemoclaw_model_config.MissingProviderPackageError: (
+        "ModelConfigError",
+        "model_configuration",
+        "false",
+    ),
+}
+
+# Bound the __cause__/__context__ walk so a self-referential or deeply chained
+# exception cannot turn diagnostics into an unbounded loop.
+_NEMOCLAW_EXCEPTION_CHAIN_LIMIT = 8
 
-_NEMOCLAW_MANAGED_STATE_DB = "/sandbox/.deepagents/.state/sessions.db"
 _NEMOCLAW_JSON_SCHEMA_VERSION = 1
 _NEMOCLAW_JSON_MAX_BYTES = 1_048_576
 _NEMOCLAW_JSON_ENVELOPE_RESERVE_BYTES = 4_096
@@ -1089,45 +1186,26 @@ async def _nemoclaw_run_json_non_interactive(timeout_seconds, *args, **kwargs):
     return _nemoclaw_write_json_envelope(run, status, exit_code)
 
 
-def _nemoclaw_classify_persisted_error(thread_id):
-    """Classify the observed provider-capacity error for one managed thread."""
-    if (
-        not isinstance(thread_id, str)
-        or not thread_id
-        or not _nemoclaw_os.path.isfile(_NEMOCLAW_MANAGED_STATE_DB)
-    ):
-        return None
-    try:
-        conn = _nemoclaw_sqlite3.connect(_NEMOCLAW_MANAGED_STATE_DB, timeout=2)
-        conn.execute("PRAGMA query_only = ON")
-        try:
-            cursor = conn.execute(
-                "SELECT substr(value, 1, 4096) FROM writes "
-                "WHERE thread_id = ? AND channel = '__error__' "
-                "ORDER BY rowid DESC LIMIT 5",
-                (thread_id,),
-            )
-            for (value,) in cursor:
-                if not isinstance(value, (str, bytes)):
-                    continue
-                text = (
-                    value
-                    if isinstance(value, str)
-                    else value.decode("utf-8", errors="replace")
-                )
-                if _NEMOCLAW_PROVIDER_CAPACITY_ERROR.search(text):
-                    return ("ResourceExhausted", "upstream_provider_capacity", "true")
-        finally:
-            conn.close()
-    except Exception:
-        # Diagnostics must not replace the original non-interactive exit result.
-        pass
+def _nemoclaw_classify_active_exception():
+    """Classify the in-flight exception by imported class identity (#8121)."""
+    error = _nemoclaw_sys.exc_info()[1]
+    seen = set()
+    depth = 0
+    while error is not None and depth < _NEMOCLAW_EXCEPTION_CHAIN_LIMIT:
+        if id(error) in seen:
+            return None
+        seen.add(id(error))
+        classification = _NEMOCLAW_EXCEPTION_CLASSIFIERS.get(type(error))
+        if classification:
+            return classification
+        error = error.__cause__ or error.__context__
+        depth += 1
     return None
 
 
 def _nemoclaw_report_non_interactive_error(thread_id, console):
-    """Emit bounded diagnostics without logging the exception or checkpoint row."""
-    classified = _nemoclaw_classify_persisted_error(thread_id)
+    """Emit bounded diagnostics without logging exception content."""
+    classified = _nemoclaw_classify_active_exception()
     logger = _nemoclaw_logging.getLogger("nemoclaw.managed.non_interactive")
     if classified:
         error_class, category, retryable = classified
@@ -1139,9 +1217,12 @@ def _nemoclaw_report_non_interactive_error(thread_id, console):
             retryable,
             thread_id,
         )
+        # The values come from a fixed table keyed by imported exception class
+        # objects, so the console line stays a closed vocabulary.
         console.print(
             f"\n[red]Model request failed: {error_class} "
-            f"(correlation_id={thread_id})[/red]"
+            f"(category={category} retryable={retryable} "
+            f"correlation_id={thread_id})[/red]"
         )
         return
     logger.warning(

docs/manage-sandboxes/run-deep-agents-code.mdx

@@ -83,6 +83,17 @@ The process exit code matches `data.exit_code`.
 The terminal statuses are `success`, `agent_failure`, `process_failure`, `timeout`, `turn_limit`, `cancelled`, and `output_limit`.
 A timeout exits `124`, cancellation exits `130`, and other failures exit nonzero.
 
+When a headless run fails, the managed runtime classifies the active client exception chain when it contains a pinned exception class.
+Known classifications cover LangGraph SDK request failures, transport and TLS failures, and managed model-configuration failures.
+The stderr diagnostic includes `error_class`, `category`, `retryable`, and `correlation_id`.
+Use `correlation_id` to match the failure to logs.
+The `retryable` field is diagnostic information and does not cause an automatic retry.
+
+The runtime does not classify persisted checkpoint exception text.
+If no pinned exception class matches, stderr reports `error_class=unknown category=unknown retryable=false`.
+Text mode also prints `Unexpected error` with only the correlation ID.
+The runtime emits only fixed classification labels and does not copy the exception class name, message, or checkpoint content.
+
 The complete serialized envelope is limited to 1 MiB.
 If the response cannot fit, the runtime discards it and emits a bounded `output_limit` failure envelope.
 If it cannot write the envelope, the command exits nonzero and reports the write failure on stderr.

test/e2e/e2e-cloud-experimental/checks/04-deepagents-code-fresh-reonboard.sh

@@ -62,6 +62,33 @@ assert_identity() {
   [ "$endpoint" = "https://inference.local/v1" ] || fail "$phase identity endpoint is '${endpoint:-missing}'"
 }
 
+is_positive_integer() {
+  [[ "$1" =~ ^[1-9][0-9]*$ ]]
+}
+
+wait_for_status_after_reonboard() {
+  local attempt attempts delay_seconds status_json status
+  attempts="${NEMOCLAW_E2E_DCODE_STATUS_ATTEMPTS:-5}"
+  delay_seconds="${NEMOCLAW_E2E_DCODE_STATUS_DELAY_SECONDS:-5}"
+  is_positive_integer "$attempts" || fail "status attempts must be a positive integer"
+  [[ "$delay_seconds" =~ ^[0-9]+$ ]] || fail "status retry delay must be a non-negative integer"
+
+  for ((attempt = 1; attempt <= attempts; attempt++)); do
+    if status_json="$("$CLI" "$SANDBOX_NAME" status --json)"; then
+      printf '%s\n' "$status_json"
+      return 0
+    else
+      status=$?
+    fi
+    if [ "$attempt" -lt "$attempts" ]; then
+      sleep "$delay_seconds"
+    fi
+  done
+
+  printf '%s\n' "$status_json"
+  return "$status"
+}
+
 seed_config_source() {
   cat <<'PY'
 import os
@@ -137,6 +164,10 @@ print("NEMOCLAW_DCODE_FRESH_CONFIG_VERIFIED")
 PY
 }
 
+if [[ "${BASH_SOURCE[0]}" != "$0" ]]; then
+  return 0
+fi
+
 [ -n "$SANDBOX_NAME" ] || fail "sandbox name is required"
 
 # The generic cloud-onboard target runs every shared check against its OpenClaw
@@ -257,7 +288,28 @@ if (JSON.parse(process.env.CONFIG_MODEL_JSON) !== "openai:" + process.env.MODEL_
 ' || fail "keyed config get did not return model B after re-onboard"
 pass "keyed config get reports model B after re-onboard"
 
-status_json="$("$CLI" "$SANDBOX_NAME" status --json)" || fail "nemoclaw status failed after re-onboard: ${status_json:-<no stdout>}"
+# Invalid state: OpenShell publishes the recreated sandbox as Ready before its
+#   in-sandbox inference route accepts health probes, so the first status call
+#   after a fresh re-onboard can report failureLabel=unreachable for a sandbox
+#   that becomes healthy moments later.
+# Source boundary: readiness is published by OpenShell's sandbox lifecycle and
+#   only consumed here (the Ready assertion above reads it from `list`). The
+#   probe is NemoClaw's own probeSandboxInferenceGatewayHealth in
+#   src/lib/actions/sandbox/inference-route-health.ts, which reports the route
+#   state at the instant it runs and documents that it must not wait.
+# Source-fix constraint: NemoClaw cannot make OpenShell delay Ready until the
+#   route serves, and making `status` retry internally would turn a
+#   point-in-time report into a wait, hiding real outages from every other
+#   caller. The retry therefore belongs to this check, the only consumer that
+#   knows a re-onboard just happened.
+# Regression: test/e2e/support/platform-parity-cloud-experimental.test.ts covers
+#   eventual status success and retry exhaustion.
+# Removal condition: delete this retry once OpenShell publishes Ready only after
+#   the in-sandbox inference route serves, or once NemoClaw exposes an explicit
+#   readiness-wait command this check can call instead.
+# Keep this bounded so persistent route failures still stop the target before
+# the remaining runtime checks.
+status_json="$(wait_for_status_after_reonboard)" || fail "nemoclaw status failed after bounded post-re-onboard readiness checks: ${status_json:-<no stdout>}"
 STATUS_JSON="$status_json" SANDBOX_NAME="$SANDBOX_NAME" MODEL_B="$model_b" node -e '
 const status = JSON.parse(process.env.STATUS_JSON);
 if (status.name !== process.env.SANDBOX_NAME ||

test/e2e/support/platform-parity-cloud-experimental.test.ts

@@ -431,11 +431,69 @@
     expect(result.status, result.stderr).toBe(0);
   });
 
+  it.each([
+    ["accepts a later passing status", "unhealthy-then-ready", 0, 2],
+    ["fails after three unsuccessful status attempts", "unhealthy-always", 1, 3],
+  ] as const)("%s for a fresh DCode re-onboard", (_label, mode, expectedStatus, expectedAttempts) => {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-dcode-status-readiness-"));
+    const mockCli = path.join(tempDir, "nemoclaw");
+    const counterFile = path.join(tempDir, "attempts");
+    try {
+      fs.writeFileSync(
+        mockCli,
+        [
+          "#!/bin/bash",
+          "set -euo pipefail",
+          "count=0",
+          'if [ -f "$MOCK_STATUS_COUNTER_FILE" ]; then',
+          '  read -r count <"$MOCK_STATUS_COUNTER_FILE"',
+          "fi",
+          "count=$((count + 1))",
+          `printf '%s\\n' "$count" >"$MOCK_STATUS_COUNTER_FILE"`,
+          'if [ "$MOCK_STATUS_MODE" = "unhealthy-always" ] || [ "$count" -eq 1 ]; then',
+          `  printf '%s\\n' '{"inferenceHealth":{"ok":false,"failureLabel":"unreachable"}}'`,
+          "  exit 1",
+          "fi",
+          `printf '%s\\n' '{"inferenceHealth":{"ok":true}}'`,
+          "",
+        ].join("\n"),
+        { mode: 0o755 },
+      );
+
+      const result = spawnSync(
+        "/bin/bash",
+        [
+          "-c",
+          'source "$1"; CLI="$2"; SANDBOX_NAME="deepagents-sandbox"; NEMOCLAW_E2E_DCODE_STATUS_ATTEMPTS=3; NEMOCLAW_E2E_DCODE_STATUS_DELAY_SECONDS=0; wait_for_status_after_reonboard',
+          "bash",
+          dcodeFreshReonboardCheck,
+          mockCli,
+        ],
+        {
+          encoding: "utf8",
+          env: {
+            ...process.env,
+            MOCK_STATUS_COUNTER_FILE: counterFile,
+            MOCK_STATUS_MODE: mode,
+          },
+        },
+      );
+
+      expect(result.status, result.stdout + "\n" + result.stderr).toBe(expectedStatus);
+      expect(Number(fs.readFileSync(counterFile, "utf8").trim())).toBe(expectedAttempts);
+      expect(result.stdout).toContain(
+        mode === "unhealthy-always" ? '"failureLabel":"unreachable"' : '"ok":true',
+      );
+    } finally {
+      fs.rmSync(tempDir, { force: true, recursive: true });
+    }
+  });
+
   it.each([
     ["retries one fail-closed inference timeout", "timeout-then-success", 0, 2, 1],
     ["fails after the bounded inference timeout retry", "timeout-always", 1, 2, 1],
     ["does not retry a non-timeout inference failure", "http-401", 1, 1, 0],
   ] as const)("%s during a named DCode rebuild", (_label, mode, expectedStatus, expectedAttempts, expectedRetryMessages) => {
     const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-dcode-rebuild-retry-"));
     const mockCli = path.join(tempDir, "nemoclaw");
     const counterFile = path.join(tempDir, "attempts");