diff --git a/.github/actions/resolve-hermes-base-image/action.yaml b/.github/actions/resolve-hermes-base-image/action.yaml index fd967e27ad5..6763c5b0844 100644 --- a/.github/actions/resolve-hermes-base-image/action.yaml +++ b/.github/actions/resolve-hermes-base-image/action.yaml @@ -26,6 +26,16 @@ runs: [[ -n "$have" ]] && [[ "$(printf '%s\n%s\n' "$min_glibc" "$have" | sort -V | head -n 1)" == "$min_glibc" ]] } + # Build-time package/import guard only. Authenticated HTTPS execution is + # validated by test/e2e/live/mcp-bridge.test.ts against a + # final Hermes sandbox image and OpenShell policy. + mcp_client_imports_ok() { + local ref="$1" + docker run --rm --entrypoint /opt/hermes/.venv/bin/python "$ref" -c \ + 'import mcp; from tools import mcp_tool; assert getattr(mcp_tool, "_MCP_AVAILABLE", False); assert getattr(mcp_tool, "_MCP_HTTP_AVAILABLE", False)' \ + >/dev/null 2>&1 + } + layout_ok() { local ref="$1" docker run --rm --entrypoint sh "$ref" -lc ' @@ -44,18 +54,22 @@ runs: if ! docker pull "$ref" >/dev/null 2>&1; then return 1 fi - version="$(glibc_version "$ref" || true)" + digest_ref="$(docker image inspect "$ref" --format '{{range .RepoDigests}}{{println .}}{{end}}' | grep -F -m 1 "${image}@sha256:" || true)" + if [[ -z "$digest_ref" ]]; then + echo "::warning::Hermes sandbox base image ${ref} did not expose an immutable GHCR repo digest (may be a fresh tag); building locally" + return 1 + fi + version="$(glibc_version "$digest_ref" || true)" if ! glibc_ok "$version"; then echo "::warning::Hermes sandbox base image ${ref} has glibc ${version:-unknown}; need >= ${min_glibc}" return 1 fi - if ! layout_ok "$ref"; then + if ! layout_ok "$digest_ref"; then echo "::warning::Hermes sandbox base image ${ref} contains retired sandbox state; trying another candidate" return 1 fi - digest_ref="$(docker image inspect "$ref" --format '{{range .RepoDigests}}{{println .}}{{end}}' | grep -F -m 1 "${image}@sha256:" || true)" - if [[ -z "$digest_ref" ]]; then - echo "::warning::Hermes sandbox base image ${ref} did not expose an immutable GHCR repo digest (may be a fresh tag); building locally" + if ! mcp_client_imports_ok "$digest_ref"; then + echo "::warning::Hermes sandbox base image ${ref} lacks the packaged MCP Streamable HTTP client imports" return 1 fi echo "HERMES_BASE_IMAGE=${digest_ref}" >> "$GITHUB_ENV" @@ -85,4 +99,8 @@ runs: echo "::error::Local Hermes sandbox base image contains retired sandbox state" exit 1 fi + if ! mcp_client_imports_ok nemoclaw-hermes-base-local; then + echo "::error::Local Hermes sandbox base image lacks the packaged MCP Streamable HTTP client imports" + exit 1 + fi echo "HERMES_BASE_IMAGE=nemoclaw-hermes-base-local" >> "$GITHUB_ENV" diff --git a/.github/workflows/brev-nightly-e2e.yaml b/.github/workflows/brev-nightly-e2e.yaml index af04aad7ce5..8c7298e80ee 100644 --- a/.github/workflows/brev-nightly-e2e.yaml +++ b/.github/workflows/brev-nightly-e2e.yaml @@ -9,6 +9,7 @@ name: E2E / Brev Nightly # Suites: # all credential-sanitization + telegram-injection # messaging-providers Telegram + Discord provider/L7 proxy validation +# messaging-compatible-endpoint local compatible-endpoint Telegram validation # full install/onboard/inference/CLI path on: @@ -16,10 +17,6 @@ on: - cron: "0 6 * * *" workflow_dispatch: inputs: - branch: - description: "Branch to test (default: ref used for this dispatch; schedule always tests main)" - required: false - default: "" keep_alive: description: "Keep Brev instances alive after tests (for SSH debugging)" required: false @@ -28,6 +25,12 @@ on: permissions: contents: read + # GitHub validates a reusable workflow's complete permission ceiling before + # evaluating skipped jobs. The called workflow explicitly downgrades its + # secret-bearing validation job to read-only; only its no-checkout reporter + # may use these write grants when a different caller supplies pr_number. + checks: write + pull-requests: write concurrency: group: brev-nightly-e2e-${{ github.event_name }}-${{ github.event_name == 'workflow_dispatch' && github.ref || 'schedule' }} @@ -39,10 +42,12 @@ jobs: strategy: fail-fast: false matrix: - test_suite: [all, messaging-providers, full] + test_suite: [all, messaging-providers, messaging-compatible-endpoint, full] uses: ./.github/workflows/e2e-branch-validation.yaml with: - branch: ${{ github.event_name == 'schedule' && 'main' || inputs.branch || github.ref_name }} + # Bind tested code to the ref that supplied this reviewed workflow. Do + # not let a write-scoped trusted caller select a second arbitrary branch. + branch: ${{ github.ref_name }} test_suite: ${{ matrix.test_suite }} use_launchable: true keep_alive: ${{ github.event_name == 'workflow_dispatch' && inputs.keep_alive || false }} diff --git a/.github/workflows/cloudflared-update-check.yaml b/.github/workflows/cloudflared-update-check.yaml new file mode 100644 index 00000000000..f843a27cc0b --- /dev/null +++ b/.github/workflows/cloudflared-update-check.yaml @@ -0,0 +1,30 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: Dependencies / cloudflared Update Check + +on: + schedule: + - cron: "23 13 * * 1" + workflow_dispatch: {} + +permissions: + contents: read + +concurrency: + group: cloudflared-update-check + cancel-in-progress: false + +jobs: + check-cloudflared: + name: Check cloudflared release pin + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Checkout + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Compare reviewed pin with the latest upstream release + run: bash scripts/checks/check-cloudflared-update.sh diff --git a/.github/workflows/e2e-branch-validation.yaml b/.github/workflows/e2e-branch-validation.yaml index 7280dee4586..d80d1ee8771 100644 --- a/.github/workflows/e2e-branch-validation.yaml +++ b/.github/workflows/e2e-branch-validation.yaml @@ -42,11 +42,14 @@ name: E2E / Branch Validation # isolation, openclaw.json config patching, network reachability, # and L7 proxy token rewriting for Telegram + Discord. Creates # its own sandbox (e2e-msg-provider). (~15 min) +# messaging-compatible-endpoint — Telegram-enabled OpenClaw through a local +# OpenAI-compatible endpoint. Creates its own sandbox +# (e2e-msg-compat) on a separate fresh instance. # dashboard-remote-bind — Verifies opt-in remote dashboard forwards bind 0.0.0.0. # gpu — Provisions a Brev GPU VM and runs the Ollama GPU E2E # sandbox proof suite from source. (~45 min) -# all — Runs credential-sanitization + telegram-injection (NOT full, -# which destroys the sandbox the security tests need). +# all — Runs credential-sanitization + telegram-injection, each with +# its own sandbox lifecycle (NOT the independent full journey). # # Required secrets: BREV_API_KEY + BREV_ORG_ID (or legacy BREV_API_TOKEN), NVIDIA_INFERENCE_API_KEY # Instance cost: Brev CPU credits (~$0.10/run for 4x16 instance) @@ -68,6 +71,7 @@ on: - credential-sanitization - telegram-injection - messaging-providers + - messaging-compatible-endpoint - dashboard-remote-bind - gpu - all @@ -81,10 +85,6 @@ on: required: false type: boolean default: false - brev_token: - description: "Brev refresh token (overrides BREV_API_TOKEN secret if provided)" - required: false - default: "" brev_provider: description: "Brev provider filter for provisioning (blank = Brev default/any)" required: false @@ -122,14 +122,10 @@ on: required: false type: boolean default: true - setup_script_url: - required: false - type: string - default: "" keep_alive: required: false type: boolean - default: true + default: false brev_provider: required: false type: string @@ -166,46 +162,64 @@ permissions: pull-requests: write concurrency: - group: e2e-branch-validation-${{ inputs.pr_number || github.run_id }} + # A caller may fan this reusable workflow out as a suite matrix. Include the + # suite so sibling jobs do not cancel each other while repeat runs of the + # same PR/suite still replace stale work. + group: e2e-branch-validation-${{ inputs.pr_number || github.run_id }}-${{ inputs.test_suite }} cancel-in-progress: true jobs: e2e-branch-validation: # if: github.repository == 'NVIDIA/NemoClaw' # Disabled for fork testing — re-enable before merge runs-on: ubuntu-latest - timeout-minutes: 90 + timeout-minutes: 130 + # Target-branch code receives Brev/inference secrets, so this job must not + # inherit the caller's reporting write grants. + permissions: + contents: read + pull-requests: read env: - BREV_E2E_INSTANCE_NAME: e2e-${{ inputs.pr_number || 'run' }}-${{ github.run_id }}-${{ github.run_attempt }} + BREV_E2E_INSTANCE_NAME: e2e-${{ inputs.pr_number || 'run' }}-${{ inputs.test_suite }}-${{ github.run_id }}-${{ github.run_attempt }} + outputs: + tested_sha: ${{ steps.tested-ref.outputs.sha }} steps: + - name: Validate test suite + env: + TEST_SUITE: ${{ inputs.test_suite }} + run: | + case "$TEST_SUITE" in + full|credential-sanitization|telegram-injection|messaging-providers|messaging-compatible-endpoint|dashboard-remote-bind|gpu|all) ;; + *) + echo "::error::test_suite is not one of the supported Brev E2E suites" + exit 1 + ;; + esac + - name: Resolve branch from PR number if: inputs.pr_number != '' env: GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ inputs.pr_number }} run: | - BRANCH=$(gh pr view ${{ inputs.pr_number }} --repo ${{ github.repository }} --json headRefName -q .headRefName) - echo "Resolved PR #${{ inputs.pr_number }} → branch: $BRANCH" + set -euo pipefail + if [[ ! "$PR_NUMBER" =~ ^[1-9][0-9]*$ ]]; then + echo "::error::pr_number must be a positive integer" + exit 1 + fi + BRANCH=$(gh pr view "$PR_NUMBER" --repo "$GITHUB_REPOSITORY" --json headRefName -q .headRefName) + echo "Resolved PR #$PR_NUMBER → branch: $BRANCH" echo "RESOLVED_BRANCH=$BRANCH" >> "$GITHUB_ENV" - name: Checkout target branch uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: ref: ${{ env.RESOLVED_BRANCH || inputs.branch || 'main' }} + persist-credentials: false - - name: Create check run (pending) - if: inputs.pr_number != '' - env: - GH_TOKEN: ${{ github.token }} + - id: tested-ref + name: Record exact tested revision run: | - PR_SHA=$(gh pr view ${{ inputs.pr_number }} --json headRefOid -q .headRefOid) - CHECK_RUN_ID=$(gh api repos/${{ github.repository }}/check-runs \ - -f name="Brev E2E (${{ inputs.test_suite }})" \ - -f head_sha="$PR_SHA" \ - -f status="in_progress" \ - -f "output[title]=Running on ephemeral Brev instance" \ - -f "output[summary]=Tests in progress..." \ - --jq '.id') - echo "CHECK_RUN_ID=$CHECK_RUN_ID" >> "$GITHUB_ENV" - echo "PR_SHA=$PR_SHA" >> "$GITHUB_ENV" + echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" - name: Setup Node.js uses: actions/setup-node@v6 @@ -215,13 +229,17 @@ jobs: - name: Install Brev CLI env: - BREV_API_TOKEN: ${{ inputs.brev_token || secrets.BREV_API_TOKEN }} + BREV_API_TOKEN: ${{ secrets.BREV_API_TOKEN }} BREV_API_KEY: ${{ secrets.BREV_API_KEY }} BREV_ORG_ID: ${{ secrets.BREV_ORG_ID }} + BREV_CLI_VERSION: "0.6.324" + BREV_CLI_SHA256: "c7056c17d4810134e3fe7194c233619b1b888a640df1929ea7c6f69c0425e58c" run: | + set -euo pipefail # Brev CLI v0.6.324+ — CPU instances use `brev search cpu | brev create` # Startup scripts use `brev create --startup-script @file` (not brev start --cpu) - curl -fsSL -o /tmp/brev.tar.gz "https://github.com/brevdev/brev-cli/releases/download/v0.6.324/brev-cli_0.6.324_linux_amd64.tar.gz" + curl -fsSL -o /tmp/brev.tar.gz "https://github.com/brevdev/brev-cli/releases/download/v${BREV_CLI_VERSION}/brev-cli_${BREV_CLI_VERSION}_linux_amd64.tar.gz" + printf '%s %s\n' "$BREV_CLI_SHA256" /tmp/brev.tar.gz | sha256sum -c - tar -xzf /tmp/brev.tar.gz -C /usr/local/bin brev chmod +x /usr/local/bin/brev @@ -252,13 +270,13 @@ jobs: - name: Run ephemeral Brev E2E env: NEMOCLAW_RUN_BRANCH_VALIDATION_E2E: "1" - BREV_API_TOKEN: ${{ inputs.brev_token || secrets.BREV_API_TOKEN }} + NEMOCLAW_BREV_WORKFLOW_OWNS_INSTANCE: "1" + BREV_API_TOKEN: ${{ secrets.BREV_API_TOKEN }} NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }} GITHUB_TOKEN: ${{ github.token }} INSTANCE_NAME: ${{ env.BREV_E2E_INSTANCE_NAME }} TEST_SUITE: ${{ inputs.test_suite }} USE_LAUNCHABLE: ${{ inputs.use_launchable && '1' || '0' }} - LAUNCHABLE_SETUP_SCRIPT: ${{ inputs.setup_script_url || '' }} BREV_PROVIDER: ${{ inputs.brev_provider || vars.BREV_PROVIDER || '' }} BREV_GPU_TYPE: ${{ inputs.brev_gpu_type || vars.BREV_GPU_TYPE || '' }} BREV_GPU_NAME: ${{ inputs.brev_gpu_name || vars.BREV_GPU_NAME || '' }} @@ -268,45 +286,6 @@ jobs: KEEP_ALIVE: ${{ inputs.keep_alive }} run: npx vitest run --project e2e-branch-validation --silent=false --reporter=default - - name: Update check run (completed) - if: always() && inputs.pr_number != '' && env.CHECK_RUN_ID != '' - env: - GH_TOKEN: ${{ github.token }} - run: | - CONCLUSION=${{ job.status == 'success' && 'success' || 'failure' }} - gh api repos/${{ github.repository }}/check-runs/${{ env.CHECK_RUN_ID }} \ - -X PATCH \ - -f status="completed" \ - -f conclusion="$CONCLUSION" \ - -f "output[title]=Brev E2E (${{ inputs.test_suite }}): ${CONCLUSION}" \ - -f "output[summary]=See workflow run for details." - - - name: Post PR comment - if: always() && inputs.pr_number != '' - env: - GH_TOKEN: ${{ github.token }} - run: | - if [ "${{ job.status }}" = "success" ]; then - EMOJI="✅" - STATUS="PASSED" - else - EMOJI="❌" - STATUS="FAILED" - fi - INSTANCE="${{ env.BREV_E2E_INSTANCE_NAME }}" - BRANCH="${RESOLVED_BRANCH:-${{ inputs.branch || 'main' }}}" - BODY="${EMOJI} **Brev E2E** (${{ inputs.test_suite }}): **${STATUS}** on branch \`${BRANCH}\` — [See logs](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})" - if [ "${{ inputs.keep_alive }}" = "true" ]; then - BODY="${BODY} - - > **Instance \`${INSTANCE}\` is still running.** To SSH in: - > \`\`\` - > brev refresh && ssh ${INSTANCE} - > \`\`\` - > When done, delete it: \`brev delete ${INSTANCE}\`" - fi - gh pr comment ${{ inputs.pr_number }} --repo ${{ github.repository }} --body "$BODY" - # Collect debugging artifacts from the Brev VM on failure before the # instance gets torn down. Captures the onboard log, sandbox list, # docker state, and gateway status so downstream onboard failures are @@ -337,7 +316,7 @@ jobs: if: failure() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: brev-debug-bundle + name: brev-debug-bundle-${{ inputs.test_suite }}-${{ github.run_attempt }} path: brev-debug-bundle/ if-no-files-found: ignore @@ -345,6 +324,136 @@ jobs: if: failure() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: e2e-branch-validation-logs + name: e2e-branch-validation-logs-${{ inputs.test_suite }}-${{ github.run_attempt }} path: /tmp/brev-e2e-*.log if-no-files-found: ignore + + - name: Delete Brev instance + if: always() && !inputs.keep_alive + env: + INSTANCE: ${{ env.BREV_E2E_INSTANCE_NAME }} + run: | + set -euo pipefail + + if ! command -v brev >/dev/null 2>&1; then + echo "Brev CLI is unavailable; the validation step could not have created ${INSTANCE}." + exit 0 + fi + + for attempt in 1 2 3; do + if output="$(timeout 30s brev delete "$INSTANCE" 2>&1)"; then + printf '%s\n' "$output" + echo "Brev deletion requested for ${INSTANCE}." + exit 0 + else + status=$? + fi + + if list_json="$(timeout 30s brev ls --json 2>/dev/null)" && \ + jq -e ' + (type == "array" and all(.[]; type == "object")) or + (type == "object" and + ((.workspaces? // null) | type == "array") and + all(.workspaces[]; type == "object")) + ' \ + <<<"$list_json" >/dev/null; then + if ! jq -e --arg name "$INSTANCE" ' + (if type == "array" then . else .workspaces end) + | any(.[]; + ((.name // .workspaceName // .instanceName // .Name // "") | tostring) + == $name) + ' <<<"$list_json" >/dev/null; then + echo "Brev instance ${INSTANCE} is already absent." + exit 0 + fi + fi + + if [ "$attempt" -eq 3 ]; then + printf '%s\n' "$output" >&2 + echo "::error::Failed to delete Brev instance ${INSTANCE} after ${attempt} attempts." + exit "$status" + fi + + echo "::warning::Brev delete attempt ${attempt} failed; refreshing before retry." + timeout 30s brev refresh >/dev/null 2>&1 || true + sleep "$((attempt * 5))" + done + + report-pr: + name: Report Brev E2E result + needs: e2e-branch-validation + if: always() && inputs.pr_number != '' + runs-on: ubuntu-latest + # This job receives no Brev/inference secret and checks out no target code, + # keeping its write token outside the validation data plane. + permissions: + contents: read + checks: write + pull-requests: write + steps: + - name: Publish completed check and PR comment + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ inputs.pr_number }} + TEST_SUITE: ${{ inputs.test_suite }} + VALIDATION_RESULT: ${{ needs.e2e-branch-validation.result }} + TESTED_SHA: ${{ needs.e2e-branch-validation.outputs.tested_sha }} + KEEP_ALIVE: ${{ inputs.keep_alive }} + INSTANCE_NAME: e2e-${{ inputs.pr_number || 'run' }}-${{ inputs.test_suite }}-${{ github.run_id }}-${{ github.run_attempt }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + set -euo pipefail + if [[ ! "$PR_NUMBER" =~ ^[1-9][0-9]*$ ]]; then + echo "::error::pr_number must be a positive integer" + exit 1 + fi + pr_json="$(gh pr view "$PR_NUMBER" --repo "$GITHUB_REPOSITORY" --json headRefName,headRefOid)" + branch="$(jq -r '.headRefName' <<<"$pr_json")" + current_sha="$(jq -r '.headRefOid' <<<"$pr_json")" + if [[ "$current_sha" != "$TESTED_SHA" ]]; then + echo "::error::PR head moved after Brev validation; refusing to report stale evidence" + exit 1 + fi + case "$VALIDATION_RESULT" in + success) + conclusion="success" + status="PASSED" + emoji="✅" + ;; + cancelled) + conclusion="cancelled" + status="CANCELLED" + emoji="⚪" + ;; + skipped) + conclusion="skipped" + status="SKIPPED" + emoji="⚪" + ;; + *) + conclusion="failure" + status="FAILED" + emoji="❌" + ;; + esac + gh api "repos/$GITHUB_REPOSITORY/check-runs" \ + -f "name=Brev E2E ($TEST_SUITE)" \ + -f "head_sha=$TESTED_SHA" \ + -f status="completed" \ + -f "conclusion=$conclusion" \ + -f "output[title]=Brev E2E ($TEST_SUITE): $conclusion" \ + -f "output[summary]=See workflow run for details." + body_file="$(mktemp)" + printf "%s **Brev E2E** (%s): **%s** on branch \`%s\` — [See logs](%s)\n" \ + "$emoji" "$TEST_SUITE" "$status" "$branch" "$RUN_URL" >"$body_file" + if [[ "$KEEP_ALIVE" == "true" ]]; then + cat >>"$body_file" < **Instance \`$INSTANCE_NAME\` is still running.** To SSH in: + > \`\`\` + > brev refresh && ssh $INSTANCE_NAME + > \`\`\` + > When done, delete it: \`brev delete $INSTANCE_NAME\` + EOF + fi + gh pr comment "$PR_NUMBER" --repo "$GITHUB_REPOSITORY" --body-file "$body_file" diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 9f23f78cedc..346b6701b35 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -14,7 +14,7 @@ on: default: "" type: string jobs: - description: "Optional comma-separated free-standing live E2E job ids. Empty runs default-enabled jobs only when targets is also empty; explicit-only jobs hermes-gpu-startup, openshell-gateway-auth-contract, jetson-nvmap-gpu, and sandbox-rlimits-connect are skipped unless selected." + description: "Optional comma-separated free-standing live E2E job ids. Empty runs default-enabled jobs only when targets is also empty; explicit-only jobs openshell-gateway-auth-contract, mcp-bridge-dev, hermes-gpu-startup, sandbox-rlimits-connect, and jetson-nvmap-gpu are skipped unless selected." required: false default: "" type: string @@ -524,6 +524,205 @@ jobs: shell: bash run: bash .github/scripts/docker-auth-cleanup.sh + mcp-bridge: + needs: generate-matrix + if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',mcp-bridge,') || contains(format(',{0},', inputs.targets), ',mcp-bridge,') }} + runs-on: ubuntu-latest + permissions: + contents: read + # Three destructive agent scenarios each have a 45-minute Vitest budget, + # plus pinned OpenShell installation and cold image setup. + timeout-minutes: 180 + env: + E2E_JOB: "1" + E2E_TARGET_ID: "mcp-bridge" + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/mcp-bridge + NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js + NEMOCLAW_MCP_BRIDGE_AGENT_MATRIX: "1" + NEMOCLAW_OPENSHELL_CHANNEL: stable + NEMOCLAW_RUN_LIVE_E2E: "1" + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - *dockerhub-auth + + - name: Prepare E2E workspace + uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@50281ee84c4a6fc759da95ea28fc0b7d9c378a28 + + - name: Install and verify cloudflared prerequisite + # Update posture: maintainers review upstream cloudflared releases and + # update the version and reviewed SHA256 together in both explicit MCP + # lanes; mutable package repositories and unreviewed latest releases + # are intentionally rejected by the workflow-contract tests. + env: + CLOUDFLARED_VERSION: "2026.6.1" + CLOUDFLARED_DEB_SHA256: "ccd02ec216c62bfa573395d8f72cb2e91e95cbdf8726a8acc06b3e2d9aa31526" + run: | + set -euo pipefail + cloudflared_deb="${RUNNER_TEMP}/cloudflared-${CLOUDFLARED_VERSION}-linux-amd64.deb" + curl -fL "https://github.com/cloudflare/cloudflared/releases/download/${CLOUDFLARED_VERSION}/cloudflared-linux-amd64.deb" -o "${cloudflared_deb}" + printf '%s %s\n' "${CLOUDFLARED_DEB_SHA256}" "${cloudflared_deb}" | sha256sum -c - + package="$(dpkg-deb -f "${cloudflared_deb}" Package)" + version="$(dpkg-deb -f "${cloudflared_deb}" Version)" + architecture="$(dpkg-deb -f "${cloudflared_deb}" Architecture)" + if [[ "${package}" != "cloudflared" || "${version}" != "${CLOUDFLARED_VERSION}" || "${architecture}" != "amd64" ]]; then + printf 'Unexpected cloudflared package metadata: package=%s version=%s architecture=%s\n' "${package}" "${version}" "${architecture}" >&2 + exit 1 + fi + sudo dpkg -i "${cloudflared_deb}" + cloudflared --version | grep -F "cloudflared version ${CLOUDFLARED_VERSION}" + + - name: Generate MCP test TLS + run: bash test/e2e/setup-mcp-test-tls.sh + + - name: Install OpenShell CLI + env: + NEMOCLAW_OPENSHELL_FORCE_INSTALL: "1" + run: | + set -euo pipefail + bash scripts/install-openshell.sh + + - name: Run MCP OpenShell provider live test + run: | + set -euo pipefail + export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH" + if command -v openshell >/dev/null 2>&1; then + OPENSHELL_BIN="$(command -v openshell)" + elif [ -x "$HOME/.local/bin/openshell" ]; then + OPENSHELL_BIN="$HOME/.local/bin/openshell" + else + echo "::error::OpenShell CLI not found after install" + exit 1 + fi + export OPENSHELL_BIN + "$OPENSHELL_BIN" --version + npx vitest run --project e2e-live \ + test/e2e/live/mcp-bridge.test.ts \ + --silent=false --reporter=default + + - id: mcp_artifact_secret_scan + name: Scan MCP artifacts for fixture credentials + if: always() + run: >- + npx tsx tools/e2e/assert-mcp-artifact-secrets-absent.mts + e2e-artifacts/live/mcp-bridge + + - name: Upload MCP server artifacts + if: ${{ always() && steps.mcp_artifact_secret_scan.outcome == 'success' }} + uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 + with: + name: e2e-mcp-bridge + path: e2e-artifacts/live/mcp-bridge/ + + - name: Clean up Docker auth + if: always() + shell: bash + run: bash .github/scripts/docker-auth-cleanup.sh + + mcp-bridge-dev: + needs: generate-matrix + # Moving OpenShell dev artifacts are compatibility evidence only and must + # never enter scheduled or default manual runs without explicit selection. + if: ${{ contains(format(',{0},', inputs.jobs), ',mcp-bridge-dev,') || contains(format(',{0},', inputs.targets), ',mcp-bridge-dev,') }} + runs-on: ubuntu-latest + permissions: + contents: read + timeout-minutes: 180 + env: + E2E_JOB: "1" + E2E_DEFAULT_ENABLED: "0" + E2E_TARGET_ID: "mcp-bridge-dev" + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/mcp-bridge-dev + NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js + NEMOCLAW_MCP_BRIDGE_AGENT_MATRIX: "1" + NEMOCLAW_OPENSHELL_CHANNEL: dev + NEMOCLAW_RUN_LIVE_E2E: "1" + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - *dockerhub-auth + + - name: Prepare E2E workspace + uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@50281ee84c4a6fc759da95ea28fc0b7d9c378a28 + + - name: Install and verify cloudflared prerequisite + # Update posture: keep this dev compatibility lane on the same reviewed + # version/SHA256 pair as the stable lane; workflow-contract tests fail + # if the pins diverge or installation becomes mutable. + env: + CLOUDFLARED_VERSION: "2026.6.1" + CLOUDFLARED_DEB_SHA256: "ccd02ec216c62bfa573395d8f72cb2e91e95cbdf8726a8acc06b3e2d9aa31526" + run: | + set -euo pipefail + cloudflared_deb="${RUNNER_TEMP}/cloudflared-${CLOUDFLARED_VERSION}-linux-amd64.deb" + curl -fL "https://github.com/cloudflare/cloudflared/releases/download/${CLOUDFLARED_VERSION}/cloudflared-linux-amd64.deb" -o "${cloudflared_deb}" + printf '%s %s\n' "${CLOUDFLARED_DEB_SHA256}" "${cloudflared_deb}" | sha256sum -c - + package="$(dpkg-deb -f "${cloudflared_deb}" Package)" + version="$(dpkg-deb -f "${cloudflared_deb}" Version)" + architecture="$(dpkg-deb -f "${cloudflared_deb}" Architecture)" + if [[ "${package}" != "cloudflared" || "${version}" != "${CLOUDFLARED_VERSION}" || "${architecture}" != "amd64" ]]; then + printf 'Unexpected cloudflared package metadata: package=%s version=%s architecture=%s\n' "${package}" "${version}" "${architecture}" >&2 + exit 1 + fi + sudo dpkg -i "${cloudflared_deb}" + cloudflared --version | grep -F "cloudflared version ${CLOUDFLARED_VERSION}" + + - name: Generate MCP test TLS + run: bash test/e2e/setup-mcp-test-tls.sh + + - name: Revoke Docker auth before unverified dev tooling + shell: bash + run: bash .github/scripts/docker-auth-cleanup.sh + + - name: Install OpenShell CLI + env: + NEMOCLAW_ACCEPT_DEV_UNVERIFIED_INSTALL: "1" + NEMOCLAW_OPENSHELL_FORCE_INSTALL: "1" + run: | + set -euo pipefail + bash scripts/install-openshell.sh + + - name: Run MCP OpenShell provider live test + run: | + set -euo pipefail + export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH" + if command -v openshell >/dev/null 2>&1; then + OPENSHELL_BIN="$(command -v openshell)" + elif [ -x "$HOME/.local/bin/openshell" ]; then + OPENSHELL_BIN="$HOME/.local/bin/openshell" + else + echo "::error::OpenShell CLI not found after install" + exit 1 + fi + export OPENSHELL_BIN + "$OPENSHELL_BIN" --version + npx vitest run --project e2e-live \ + test/e2e/live/mcp-bridge.test.ts \ + --silent=false --reporter=default + + - id: mcp_artifact_secret_scan + name: Scan MCP artifacts for fixture credentials + if: always() + run: >- + npx tsx tools/e2e/assert-mcp-artifact-secrets-absent.mts + e2e-artifacts/live/mcp-bridge-dev + + - name: Upload MCP server artifacts + if: ${{ always() && steps.mcp_artifact_secret_scan.outcome == 'success' }} + uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 + with: + name: e2e-mcp-bridge-dev + path: e2e-artifacts/live/mcp-bridge-dev/ + + - name: Clean up Docker auth + if: always() + shell: bash + run: bash .github/scripts/docker-auth-cleanup.sh + onboard-negative-paths: needs: generate-matrix if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',onboard-negative-paths,') || contains(format(',{0},', inputs.targets), ',onboard-negative-paths,') }} @@ -4376,6 +4575,8 @@ jobs: live, openshell-version-pin, openshell-gateway-auth-contract, + mcp-bridge, + mcp-bridge-dev, onboard-negative-paths, skill-agent, openclaw-skill-cli, @@ -4475,6 +4676,11 @@ jobs: target: 'openshell-gateway-auth-contract', reason: 'default dispatch excludes the resource-heavy OpenShell auth-contract probe unless selected', }, + 'mcp-bridge-dev': { + job: 'mcp-bridge-dev', + target: 'mcp-bridge-dev', + reason: 'default dispatch excludes moving OpenShell dev artifacts unless explicitly selected', + }, 'jetson-nvmap-gpu': { job: 'jetson-nvmap-gpu', target: 'jetson-nvmap-gpu', @@ -4605,7 +4811,7 @@ jobs: ? '**Requested jobs:** _(selector rejected by workflow validation)_' : requestedJobs ? `**Requested jobs:** \`${requestedJobs}\`` - : '**Requested jobs:** _(default — all default-enabled free-standing jobs; explicit-only jobs `hermes-gpu-startup`, `openshell-gateway-auth-contract`, `jetson-nvmap-gpu`, and `sandbox-rlimits-connect` are skipped unless selected)_', + : '**Requested jobs:** _(default — all default-enabled free-standing jobs; explicit-only jobs `openshell-gateway-auth-contract`, `mcp-bridge-dev`, `hermes-gpu-startup`, `sandbox-rlimits-connect`, and `jetson-nvmap-gpu` are skipped unless selected)_', `**Summary:** ${passed.length} passed, ${failed.length} failed, ${cancelled.length} cancelled, ${skipped.length} skipped`, '', '| Job | Result |', diff --git a/.github/workflows/regression-e2e.yaml b/.github/workflows/regression-e2e.yaml index 15ee54ccbff..5472018f379 100644 --- a/.github/workflows/regression-e2e.yaml +++ b/.github/workflows/regression-e2e.yaml @@ -177,8 +177,8 @@ jobs: if-no-files-found: ignore # ── OpenShell version-pin E2E ────────────────────────────── - # Coverage guard for #3474. If a host has sticky OpenShell 0.0.45 on PATH - # but this NemoClaw release supports only <=0.0.44, install-openshell.sh + # Coverage guard for #3474. If a host has sticky OpenShell above the pinned + # supported version on PATH, install-openshell.sh # must replace it with the pinned compatible release instead of hard-failing. openshell-version-pin-e2e: needs: select_regression_jobs diff --git a/Dockerfile b/Dockerfile index 82b853e8d65..a9cdc7c956f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -43,6 +43,12 @@ RUN ln -s /opt/nemoclaw/node_modules /opt/nemoclaw-root/node_modules \ FROM ${BASE_IMAGE} ARG OPENCLAW_VERSION=2026.5.27 ARG OPENCLAW_2026_5_27_INTEGRITY=sha512-2N93zhdAo88KAbHt6T7KvYXf4s7XIkYXBgv1npYpn7e1Y9FvrtgtpsA38my9rtFW+70uXEojRPX5/OqnuDqJPw== +# Keep the version, integrity, runtime lock, license, and advisory baseline +# synchronized with agents/openclaw/dependency-review.md. +ARG MCPORTER_VERSION=0.7.3 +ARG MCPORTER_0_7_3_INTEGRITY=sha512-egoPVYqTnWb3NjRIxo+xc8OrAI0dlPrJm9pAiZx0pImuNIV5rKhGtTnIfH/Y1ldGPVu74ibj3KR5c9U/QSdQFA== +COPY agents/openclaw/mcporter-runtime/package.json /usr/local/lib/nemoclaw/mcporter-runtime/package.json +COPY agents/openclaw/mcporter-runtime/package-lock.json /usr/local/lib/nemoclaw/mcporter-runtime/package-lock.json # OpenShell blocks the link-local EC2 Instance Metadata Service. Keep AWS SDK # credential chains from attempting an impossible metadata discovery path. @@ -166,6 +172,26 @@ RUN set -eu; \ rm -rf /usr/local/lib/node_modules/openclaw /usr/local/bin/openclaw; \ npm install -g --no-audit --no-fund --no-progress "openclaw@${OPENCLAW_VERSION}"; \ fi; \ + MCPORTER_EXPECTED_INTEGRITY=""; \ + if [ "$MCPORTER_VERSION" = "0.7.3" ]; then MCPORTER_EXPECTED_INTEGRITY="$MCPORTER_0_7_3_INTEGRITY"; fi; \ + if [ -n "$MCPORTER_EXPECTED_INTEGRITY" ]; then \ + MCPORTER_REGISTRY_INTEGRITY=$(npm view "mcporter@${MCPORTER_VERSION}" dist.integrity); \ + if [ "$MCPORTER_REGISTRY_INTEGRITY" != "$MCPORTER_EXPECTED_INTEGRITY" ]; then \ + echo "ERROR: mcporter ${MCPORTER_VERSION} npm integrity mismatch" >&2; \ + echo "Expected: ${MCPORTER_EXPECTED_INTEGRITY}" >&2; \ + echo "Actual: ${MCPORTER_REGISTRY_INTEGRITY}" >&2; exit 1; \ + fi; \ + fi; \ + # Always reinstall from the committed lock. Matching top-level versions can + # otherwise hide drift in mcporter's ranged transitive dependencies. + echo "INFO: Installing locked mcporter $MCPORTER_VERSION dependency graph"; \ + rm -rf /usr/local/lib/node_modules/mcporter /usr/local/bin/mcporter; \ + npm --prefix /usr/local/lib/nemoclaw/mcporter-runtime ci \ + --ignore-scripts --omit=dev --no-audit --no-fund --no-progress; \ + ln -s /usr/local/lib/nemoclaw/mcporter-runtime/node_modules/.bin/mcporter /usr/local/bin/mcporter; \ + test "$(mcporter --version)" = "$MCPORTER_VERSION"; \ + npm --prefix /usr/local/lib/nemoclaw/mcporter-runtime audit --omit=dev --audit-level=low; \ + npm --prefix /usr/local/lib/nemoclaw/mcporter-runtime audit signatures; \ # Pre-install the codex-acp package so the embedded ACPx runtime can # call the local binary instead of `npx @zed-industries/codex-acp`. # The sandbox's L7 proxy denies @zed-industries/* package URLs @@ -787,7 +813,6 @@ ENV NPM_CONFIG_OFFLINE=true \ # OCI image imported by k3s. # hadolint ignore=DL3059,DL4006 RUN openclaw plugins install /opt/nemoclaw \ - && openclaw plugins enable nemoclaw \ && openclaw plugins inspect nemoclaw --json > /dev/null \ && if [ -d /sandbox/.openclaw/plugin-runtime-deps ]; then \ find /sandbox/.openclaw/plugin-runtime-deps -type f \( \ diff --git a/Dockerfile.base b/Dockerfile.base index f3176f5187d..46c8cdcb9dc 100644 --- a/Dockerfile.base +++ b/Dockerfile.base @@ -200,6 +200,12 @@ RUN chmod 444 /usr/local/lib/nemoclaw/sandbox-rlimits.sh \ # with the openclaw_version input for a one-off build without editing this file. ARG OPENCLAW_VERSION=2026.5.27 ARG OPENCLAW_2026_5_27_INTEGRITY=sha512-2N93zhdAo88KAbHt6T7KvYXf4s7XIkYXBgv1npYpn7e1Y9FvrtgtpsA38my9rtFW+70uXEojRPX5/OqnuDqJPw== +# Keep the version, integrity, runtime lock, license, and advisory baseline +# synchronized with agents/openclaw/dependency-review.md. +ARG MCPORTER_VERSION=0.7.3 +ARG MCPORTER_0_7_3_INTEGRITY=sha512-egoPVYqTnWb3NjRIxo+xc8OrAI0dlPrJm9pAiZx0pImuNIV5rKhGtTnIfH/Y1ldGPVu74ibj3KR5c9U/QSdQFA== +COPY agents/openclaw/mcporter-runtime/package.json /usr/local/lib/nemoclaw/mcporter-runtime/package.json +COPY agents/openclaw/mcporter-runtime/package-lock.json /usr/local/lib/nemoclaw/mcporter-runtime/package-lock.json # Keep OpenClaw's jiti-generated source cache out of /tmp so provider marker # names do not persist in runtime snapshots or leak-scan inputs. @@ -234,7 +240,24 @@ RUN --mount=type=bind,source=nemoclaw-blueprint/blueprint.yaml,target=/tmp/bluep echo "Actual: ${REGISTRY_INTEGRITY}"; exit 1; \ fi; \ fi; \ - npm install -g "openclaw@${OPENCLAW_VERSION}" \ + MCPORTER_EXPECTED_INTEGRITY=""; \ + if [ "$MCPORTER_VERSION" = "0.7.3" ]; then MCPORTER_EXPECTED_INTEGRITY="$MCPORTER_0_7_3_INTEGRITY"; fi; \ + if [ -n "$MCPORTER_EXPECTED_INTEGRITY" ]; then \ + MCPORTER_REGISTRY_INTEGRITY=$(npm view "mcporter@${MCPORTER_VERSION}" dist.integrity); \ + if [ "$MCPORTER_REGISTRY_INTEGRITY" != "$MCPORTER_EXPECTED_INTEGRITY" ]; then \ + echo "Error: mcporter ${MCPORTER_VERSION} npm integrity mismatch"; \ + echo "Expected: ${MCPORTER_EXPECTED_INTEGRITY}"; \ + echo "Actual: ${MCPORTER_REGISTRY_INTEGRITY}"; exit 1; \ + fi; \ + fi; \ + npm install -g --no-audit --no-fund --no-progress "openclaw@${OPENCLAW_VERSION}" \ + && rm -rf /usr/local/lib/node_modules/mcporter /usr/local/bin/mcporter \ + && npm --prefix /usr/local/lib/nemoclaw/mcporter-runtime ci \ + --ignore-scripts --omit=dev --no-audit --no-fund --no-progress \ + && ln -s /usr/local/lib/nemoclaw/mcporter-runtime/node_modules/.bin/mcporter /usr/local/bin/mcporter \ + && test "$(mcporter --version)" = "$MCPORTER_VERSION" \ + && npm --prefix /usr/local/lib/nemoclaw/mcporter-runtime audit --omit=dev --audit-level=low \ + && npm --prefix /usr/local/lib/nemoclaw/mcporter-runtime audit signatures \ && pip3 install --no-cache-dir --break-system-packages "pyyaml==6.0.3" diff --git a/agents/hermes/Dockerfile b/agents/hermes/Dockerfile index 7f2ee2bc254..ccdb66174ed 100644 --- a/agents/hermes/Dockerfile +++ b/agents/hermes/Dockerfile @@ -22,6 +22,14 @@ RUN set -eu; \ test -x /usr/local/bin/hermes; \ /usr/local/bin/hermes --version +# Managed MCP requires the packaged Hermes client surface. A published base can +# carry the expected Hermes version while still having been built without the +# optional `mcp` dependency group, in which case Hermes silently disables both +# MCP discovery and Streamable HTTP support. This is a build-time import guard; +# the live MCP E2E proves authenticated HTTPS execution through OpenShell. +RUN /opt/hermes/.venv/bin/python -c \ + 'import mcp; from tools import mcp_tool; assert getattr(mcp_tool, "_MCP_AVAILABLE", False), "Hermes MCP client runtime is unavailable"; assert getattr(mcp_tool, "_MCP_HTTP_AVAILABLE", False), "Hermes MCP Streamable HTTP runtime is unavailable"' + # Published base images can lag Dockerfile.base while local feature branches # still layer this final image on top. Invalid state: the selected base has # Hermes source under /opt/hermes but lacks hermes_cli/web_dist. Prebuild the @@ -118,6 +126,8 @@ COPY scripts/managed-gateway-control.py /usr/local/lib/nemoclaw/managed-gateway- COPY agents/hermes/validate-env-secret-boundary.py /usr/local/lib/nemoclaw/validate-hermes-env-secret-boundary.py COPY agents/hermes/seed-dashboard-config.py /usr/local/lib/nemoclaw/seed-hermes-dashboard-config.py COPY agents/hermes/runtime-config-guard.py /usr/local/lib/nemoclaw/hermes-runtime-config-guard.py +COPY agents/hermes/mcp-config-transaction.py /usr/local/lib/nemoclaw/hermes-mcp-config-transaction.py +COPY src/lib/actions/sandbox/openshell-child-visible-credentials.v0.0.72.json /usr/local/lib/nemoclaw/openshell-child-visible-credentials.v0.0.72.json COPY scripts/state-dir-guard.py /usr/local/lib/nemoclaw/state-dir-guard.py COPY nemoclaw-blueprint/scripts/*.js /usr/local/lib/nemoclaw/preloads/ # Dockerfile.base is the source of truth for rlimit hooks. This Hermes replay @@ -125,11 +135,12 @@ COPY nemoclaw-blueprint/scripts/*.js /usr/local/lib/nemoclaw/preloads/ # profile hook, bashrc hook, or root-owned helper mode. Remove it once the # minimum supported Hermes sandbox base tag guarantees those artifacts and # test/sandbox-rlimit-hooks.test.ts covers that base. -RUN chmod 755 /usr/local/bin/nemoclaw-start /usr/local/lib/nemoclaw/sandbox-init.sh /usr/local/lib/nemoclaw/validate-hermes-env-secret-boundary.py /usr/local/lib/nemoclaw/seed-hermes-dashboard-config.py /usr/local/lib/nemoclaw/hermes-runtime-config-guard.py \ - && chown root:root /usr/local/bin/nemoclaw-gateway-control /usr/local/lib/nemoclaw/gateway-supervisor.sh /usr/local/lib/nemoclaw/state-dir-guard.py /usr/local/lib/nemoclaw/managed-gateway-control.py \ +RUN chmod 755 /usr/local/bin/nemoclaw-start /usr/local/lib/nemoclaw/sandbox-init.sh /usr/local/lib/nemoclaw/validate-hermes-env-secret-boundary.py /usr/local/lib/nemoclaw/seed-hermes-dashboard-config.py /usr/local/lib/nemoclaw/hermes-runtime-config-guard.py /usr/local/lib/nemoclaw/hermes-mcp-config-transaction.py \ + && chown root:root /usr/local/bin/nemoclaw-gateway-control /usr/local/lib/nemoclaw/gateway-supervisor.sh /usr/local/lib/nemoclaw/state-dir-guard.py /usr/local/lib/nemoclaw/managed-gateway-control.py /usr/local/lib/nemoclaw/openshell-child-visible-credentials.v0.0.72.json \ && chmod 700 /usr/local/bin/nemoclaw-gateway-control \ && chmod 500 /usr/local/lib/nemoclaw/state-dir-guard.py /usr/local/lib/nemoclaw/managed-gateway-control.py \ && chmod 444 /usr/local/lib/nemoclaw/gateway-supervisor.sh \ + && chmod 444 /usr/local/lib/nemoclaw/openshell-child-visible-credentials.v0.0.72.json \ && if [ -d /usr/local/lib/nemoclaw/preloads ]; then \ chown -R 0:0 /usr/local/lib/nemoclaw/preloads \ && find /usr/local/lib/nemoclaw/preloads -type f -exec chmod 444 {} + \ diff --git a/agents/hermes/Dockerfile.base b/agents/hermes/Dockerfile.base index 273f4c41f2c..b191bde74de 100644 --- a/agents/hermes/Dockerfile.base +++ b/agents/hermes/Dockerfile.base @@ -33,7 +33,7 @@ ARG HERMES_VERSION=v2026.6.19 ARG HERMES_SEMVER=0.17.0 ARG HERMES_TARBALL_SHA256=69b805ec0a7a7be880068ba8a3b17479d7ba29f0cac0a2e9c6692c02f346ba91 ARG HERMES_NPM_INTEGRITY=sha512-PzSJiYqmwpTudmakYs2oCJ57OW3VwEJYf8buTuKvuRvcYEUf/KOTu2dD6pLf2XYgDKErpvcDaoSAJ1nGCyvzAA== -ARG HERMES_UV_EXTRAS="anthropic messaging web pty" +ARG HERMES_UV_EXTRAS="anthropic messaging web pty mcp" ARG UV_VERSION=0.11.8 # build-essential: hermes-agent >= 0.16.0 ships npm dependencies that need a @@ -172,9 +172,10 @@ RUN printf '%s\n' \ # The image prebakes only the extras mapped to NemoClaw-supported onboarding # integrations: anthropic (native Anthropic Messages routing), messaging # (Telegram, Discord, Slack, WeChat, WhatsApp), web (API health/UI runtime), -# and pty (optional browser TUI bridge). These extras are resolved from the -# selected Hermes release's uv.lock via `uv sync --frozen`, so dependency -# changes remain tied to HERMES_VERSION/HERMES_TARBALL_SHA256 review. +# pty (optional browser TUI bridge), and mcp (managed MCP bridge consumer). +# These extras are resolved from the selected Hermes release's uv.lock via +# `uv sync --frozen`, so dependency changes remain tied to +# HERMES_VERSION/HERMES_TARBALL_SHA256 review. # Microsoft Teams adapter dependencies are installed by the manifest-driven # final image when selected. # New Hermes integrations should be installed by the agent workflow when they @@ -271,13 +272,9 @@ RUN set -eu; \ # route that uses File/Form, so without python-multipart the plugin's API routes # fail to mount ("Form data requires python-multipart to be installed"). # -# It is NOT a dependency of hermes-agent core or any extra we enable; upstream -# only pulls it transitively via the mcp/daytona/all extras. Rather than drag in -# the MCP client (a new agent capability) or the Daytona cloud-provider SDK (an -# unused, egress/telemetry-capable SDK) just to obtain a dependency-free form -# parser, vendor python-multipart directly — pinned and hash-verified to the -# exact version resolved in the checksum-pinned release's uv.lock, so it stays -# tied to HERMES_VERSION/HERMES_TARBALL_SHA256 review like every other dep. +# It is resolved by the pinned Hermes web/mcp extras today. Keep this +# hash-verified backstop tied to the selected release's uv.lock so older base +# cache layers and future extra reshuffles cannot silently drop the parser. # Re-review the version and both hashes on every Hermes version bump. # hadolint ignore=DL3059 RUN printf '%s\n' \ @@ -293,4 +290,6 @@ RUN printf '%s\n' \ ENV PATH="/usr/local/bin:/opt/hermes/.venv/bin:${PATH}" \ HERMES_TUI_DIR="/opt/hermes/ui-tui" \ HERMES_WEB_DIST="/opt/hermes/hermes_cli/web_dist" -RUN /usr/local/bin/hermes --version +RUN /usr/local/bin/hermes --version \ + && /opt/hermes/.venv/bin/python -c \ + 'import mcp; from tools import mcp_tool; assert getattr(mcp_tool, "_MCP_AVAILABLE", False), "Hermes MCP client runtime is unavailable"; assert getattr(mcp_tool, "_MCP_HTTP_AVAILABLE", False), "Hermes MCP Streamable HTTP runtime is unavailable"' diff --git a/agents/hermes/manifest.yaml b/agents/hermes/manifest.yaml index 90aa5492e35..e9e6e01bee0 100644 --- a/agents/hermes/manifest.yaml +++ b/agents/hermes/manifest.yaml @@ -120,6 +120,11 @@ inference: provider_options: - hermesProvider +# ── MCP server support ─────────────────────────────────────────── +mcp: + support: bridge + adapter: hermes-config + # ── Phone-home hosts ─────────────────────────────────────────── # Agent-specific egress endpoints needed for updates, auth, etc. phone_home_hosts: diff --git a/agents/hermes/mcp-config-transaction.py b/agents/hermes/mcp-config-transaction.py new file mode 100755 index 00000000000..b877567bbc0 --- /dev/null +++ b/agents/hermes/mcp-config-transaction.py @@ -0,0 +1,969 @@ +#!/opt/hermes/.venv/bin/python +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Transactional Hermes MCP config mutation and gateway reload control. + +This helper never proxies MCP traffic and never handles raw service +credentials. NemoClaw invokes it as a one-shot ordinary OpenShell sandbox exec +command in the Hermes sandbox namespaces. No persistent control listener or +host-side MCP data-plane process is exposed. + +Pinned Hermes exposes interactive ``hermes mcp add/remove/list`` commands, but +they prompt, write service credentials into Hermes-owned environment state, and +do not provide NemoClaw's noninteractive ownership/hash transaction with an +acknowledged managed gateway reload/restart (https://github.com/NousResearch/hermes-agent/issues/690 +and https://github.com/NousResearch/hermes-agent/issues/52417). Direct config +edits would therefore expose a partial-write/reload race and violate the +OpenShell provider boundary. This helper owns the atomic write, ownership +checks, and reload acknowledgement instead; hermes-mcp-config-transaction.test.ts +locks that contract. Remove it when the minimum supported Hermes capability +provides equivalent noninteractive mutation, credential isolation, ownership, +and acknowledged reload guarantees. +""" + +from __future__ import annotations + +import argparse +import http.client +import importlib.util +import ipaddress +import json +import os +import grp +import pwd +import re +import signal +import stat +import sys +import time +import unicodedata +from pathlib import Path +from types import ModuleType +from urllib.parse import urlsplit + +import yaml + + +CONFIG_PATH = "/sandbox/.hermes/config.yaml" +HERMES_DIR = "/sandbox/.hermes" +GATEWAY_PID_PATH = f"{HERMES_DIR}/gateway.pid" +STRICT_HASH_PATH = "/etc/nemoclaw/hermes.config-hash" +GUARD_PATH = "/usr/local/lib/nemoclaw/hermes-runtime-config-guard.py" +ROOT_LIFECYCLE_MARKER = "/run/nemoclaw/hermes-root-lifecycle" +SERVICE_MANAGER_PATH = b"/usr/local/bin/nemoclaw-start" +RELOAD_TIMEOUT_SECONDS = 300 +SERVER_NAME_RE = re.compile(r"^[A-Za-z][A-Za-z0-9_-]{0,63}$") +ENV_PLACEHOLDER_RE = re.compile( + r"^Bearer openshell:resolve:env:([A-Za-z_][A-Za-z0-9_]{0,127})$" +) +BOUNDARY_MANIFEST_NAME = "openshell-child-visible-credentials.v0.0.72.json" +ANSI_ESCAPE_RE = re.compile( + r"\x1b(?:\[[0-?]*[ -/]*[@-~]|\][^\x07]*(?:\x07|\x1b\\)|[@-_])" +) +AUTHORIZATION_FIELD_RE = re.compile( + r"(?i)((?:[\"']?authorization[\"']?)\s*[:=]\s*)[^;}\]]+" +) +BEARER_VALUE_RE = re.compile(r"(?i)(\bBearer\s+)[^;}\]]+") +SENSITIVE_ASSIGNMENT_RE = re.compile( + r"(?i)((?:[\"']?(?:api[_-]?key|token|secret|password|credential)[\"']?)" + r"\s*[:=]\s*)[^;}\]]+" +) +URL_USERINFO_RE = re.compile(r"(?i)(https?://)[^/@\s]+@") +SENSITIVE_QUERY_RE = re.compile( + r"(?i)([?&](?:api[_-]?key|token|secret|password|credential|auth)\s*=)[^&#\s]+" +) +SENSITIVE_PAYLOAD_KEY_RE = re.compile( + r"(?i)(?:authorization|bearer|api[_-]?key|token|secret|password|credential)" +) +MAX_ERROR_MESSAGE_LENGTH = 512 +MAX_GATEWAY_PID_RECORD_BYTES = 4096 +GATEWAY_INTERNAL_PORT = 18642 +GATEWAY_PUBLIC_PORT = 8642 +BLOCKED_IPV4_NETWORKS = tuple( + ipaddress.ip_network(cidr) + for cidr in ( + "0.0.0.0/8", + "10.0.0.0/8", + "100.64.0.0/10", + "127.0.0.0/8", + "169.254.0.0/16", + "172.16.0.0/12", + "192.0.0.0/24", + "192.0.2.0/24", + "192.31.196.0/24", + "192.52.193.0/24", + "192.88.99.0/24", + "192.168.0.0/16", + "192.175.48.0/24", + "198.18.0.0/15", + "198.51.100.0/24", + "203.0.113.0/24", + "224.0.0.0/4", + "240.0.0.0/4", + ) +) +TRUSTED_HERMES_GATEWAY_LAUNCHERS = { + b"/usr/local/bin/hermes.real", + b"/usr/local/lib/nemoclaw/hermes", + b"/opt/hermes/.venv/bin/hermes", +} + + +def _load_credential_boundary_manifest() -> dict[str, object]: + # invalidState: the transaction accepts a credential name against a missing, + # corrupt, or wrong-version OpenShell boundary manifest. + # sourceBoundary: NemoClaw owns one reviewed manifest installed beside this + # helper in images; the second path is the deterministic source-checkout layout. + # whyNotSourceFix: OpenShell v0.0.72 has no machine-readable child-env contract. + # regressionTest: hermes-mcp-config-transaction and image packaging tests cover + # both layouts, strict parsing, version alignment, and reserved-name parity. + # removalCondition: use an upstream capability manifest once the minimum + # supported OpenShell release provides one. + candidates = ( + Path(__file__).with_name(BOUNDARY_MANIFEST_NAME), + Path(__file__).resolve().parents[2] + / "src" + / "lib" + / "actions" + / "sandbox" + / BOUNDARY_MANIFEST_NAME, + ) + manifest_path = next((path for path in candidates if path.is_file()), None) + if manifest_path is None: + raise RuntimeError("Hermes MCP credential boundary manifest is missing") + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + if not isinstance(manifest, dict) or manifest.get("openshellVersion") != "0.0.72": + raise RuntimeError("Hermes MCP credential boundary manifest is invalid") + return manifest + + +def _manifest_strings(manifest: dict[str, object], key: str) -> frozenset[str]: + values = manifest.get(key) + if ( + not isinstance(values, list) + or not values + or not all(isinstance(value, str) and value for value in values) + ): + raise RuntimeError(f"Hermes MCP credential boundary manifest has invalid {key}") + return frozenset(values) + + +_CREDENTIAL_BOUNDARY_MANIFEST = _load_credential_boundary_manifest() +_RAW_CHILD_VALUE_KEYS = _manifest_strings( + _CREDENTIAL_BOUNDARY_MANIFEST, "rawChildValueKeys" +) +_REWRITTEN_CHILD_VALUE_KEYS = _manifest_strings( + _CREDENTIAL_BOUNDARY_MANIFEST, "rewrittenChildValueKeys" +) +_RUNTIME_CONTROL_KEYS = _manifest_strings( + _CREDENTIAL_BOUNDARY_MANIFEST, "runtimeControlKeys" +) +_RUNTIME_CONTROL_PREFIXES = _manifest_strings( + _CREDENTIAL_BOUNDARY_MANIFEST, "runtimeControlPrefixes" +) + + +def _credential_name_is_reserved(name: str) -> bool: + return ( + name in _RAW_CHILD_VALUE_KEYS + or name in _REWRITTEN_CHILD_VALUE_KEYS + or name in _RUNTIME_CONTROL_KEYS + or any(name.startswith(prefix) for prefix in _RUNTIME_CONTROL_PREFIXES) + ) + + +def _load_guard() -> ModuleType: + spec = importlib.util.spec_from_file_location( + "nemoclaw_hermes_runtime_guard", GUARD_PATH + ) + if spec is None or spec.loader is None: + raise RuntimeError("Hermes runtime config guard could not be loaded") + module = importlib.util.module_from_spec(spec) + # dataclasses resolves the defining module through sys.modules while the + # guard is executing, so register it before exec_module(). + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def _assert_mutable_snapshot(snapshot: object) -> None: + mode = int(getattr(snapshot, "mode")) + uid = int(getattr(snapshot, "uid")) + gid = int(getattr(snapshot, "gid")) + if os.geteuid() == 0: + expected_uid = pwd.getpwnam("sandbox").pw_uid + expected_gid = grp.getgrnam("sandbox").gr_gid + owner_matches = uid == expected_uid and gid == expected_gid + else: + owner_matches = uid == os.geteuid() + if not owner_matches or not (mode & stat.S_IWUSR): + raise RuntimeError( + "Hermes config is locked or is not owned by the sandbox identity. " + "Lower shields before changing managed MCP servers." + ) + + +def _parse_payload(raw: str) -> dict[str, object]: + payload = json.loads(raw) + if not isinstance(payload, dict): + raise ValueError("MCP mutation payload must be an object") + return payload + + +def _display_safe_text(value: object) -> str: + """Collapse terminal controls so one error cannot forge extra log lines.""" + text = ANSI_ESCAPE_RE.sub("", str(value)) + text = "".join( + character + for character in text + if unicodedata.category(character) not in {"Cc", "Cf", "Cs"} + ) + return " ".join(text.split()) + + +def _sensitive_payload_values(payload: object) -> tuple[str, ...]: + values: list[str] = [] + + def visit(value: object, sensitive: bool = False) -> None: + if isinstance(value, dict): + for key, child in value.items(): + key_is_sensitive = isinstance(key, str) and bool( + SENSITIVE_PAYLOAD_KEY_RE.search(key) + ) + visit(child, sensitive or key_is_sensitive) + elif isinstance(value, list): + for child in value: + visit(child, sensitive) + elif sensitive and isinstance(value, str) and value: + values.append(_display_safe_text(value)) + + visit(payload) + return tuple(sorted(set(values), key=len, reverse=True)) + + +def _sanitize_error_message(error: Exception, payload: object = None) -> str: + """Return a bounded, single-line diagnostic without credential material.""" + if isinstance(error, yaml.YAMLError): + return "Invalid Hermes config: YAML parsing failed" + if isinstance(error, (json.JSONDecodeError, UnicodeError)): + return "Hermes MCP mutation payload could not be decoded" + + message = _display_safe_text(error) + for value in _sensitive_payload_values(payload): + if value: + message = message.replace(value, "") + message = AUTHORIZATION_FIELD_RE.sub(r"\1", message) + message = BEARER_VALUE_RE.sub(r"\1", message) + message = SENSITIVE_ASSIGNMENT_RE.sub(r"\1", message) + message = URL_USERINFO_RE.sub(r"\1@", message) + message = SENSITIVE_QUERY_RE.sub(r"\1", message) + if not message: + message = "Hermes MCP transaction failed" + return message[:MAX_ERROR_MESSAGE_LENGTH] + + +def _validate_payload(action: str, payload: dict[str, object]) -> None: + if action not in {"add", "remove"}: + raise ValueError("Unsupported MCP config action") + allowed = {"server", "url", "headers"} + allowed.add("replace_existing" if action == "add" else "force") + unexpected = sorted(set(payload) - allowed) + if unexpected: + raise ValueError( + f"MCP mutation payload contains unsupported fields: {', '.join(unexpected)}" + ) + server = payload.get("server") + if not isinstance(server, str) or not SERVER_NAME_RE.fullmatch(server): + raise ValueError("MCP mutation payload has an invalid server name") + flag_name = "replace_existing" if action == "add" else "force" + if not isinstance(payload.get(flag_name), bool): + raise ValueError(f"MCP mutation payload {flag_name} must be boolean") + # Forced cleanup is server-name scoped: _mutate removes only this exact + # mapping key and deliberately skips ownership matching. Do not strand a + # legacy entry merely because its persisted URL or header shape is no + # longer accepted for add/non-force mutation. + if action == "remove" and payload["force"] is True: + return + raw_url = payload.get("url") + if not isinstance(raw_url, str) or len(raw_url) > 2048: + raise ValueError("MCP mutation payload has an invalid URL") + parsed = urlsplit(raw_url) + if parsed.scheme != "https" or not parsed.hostname: + raise ValueError("MCP mutation payload URL must use HTTPS") + if parsed.username or parsed.password or parsed.query or parsed.fragment: + raise ValueError("MCP mutation payload URL contains forbidden components") + hostname = parsed.hostname.lower().rstrip(".") + # Fail closed on every IPv6 literal, including globally routable addresses, + # before the IPv4-only classification below. DNS names are resolved and + # validated by the host boundary, then pinned into OpenShell allowed_ips; + # this in-sandbox transaction never establishes the network connection. + if ":" in hostname: + raise ValueError("IPv6-literal MCP URLs are not supported") + if not hostname.isascii() or any(char in hostname for char in "*?[]{};"): + raise ValueError("MCP mutation payload URL has a non-literal hostname") + try: + port = parsed.port + except ValueError as error: + raise ValueError("MCP mutation payload URL has an invalid port") from error + if port == 0: + raise ValueError("MCP mutation payload URL port must be nonzero") + host_aliases = { + "host.openshell.internal", + "host.docker.internal", + "host.containers.internal", + } + if action == "add" and hostname in host_aliases: + raise ValueError( + "Authenticated MCP OpenShell host aliases are unavailable with OpenShell v0.0.72" + ) + if not (action == "remove" and hostname in host_aliases) and ( + hostname in {"localhost", "local", "internal", "metadata"} + or any( + hostname.endswith(f".{suffix}") + for suffix in ("localhost", "local", "internal", "metadata") + ) + ): + raise ValueError("MCP mutation payload URL uses a reserved hostname") + try: + address = ipaddress.ip_address(hostname) + except ValueError: + address = None + if address is None and re.fullmatch( + r"(?:0x[0-9a-f]+|[0-9]+)(?:\.(?:0x[0-9a-f]+|[0-9]+))*", + hostname, + ): + raise ValueError("MCP mutation payload URL uses an ambiguous numeric host") + if address is not None and ( + not address.is_global + or any(address in network for network in BLOCKED_IPV4_NETWORKS) + ): + raise ValueError("MCP mutation payload URL uses a non-global address") + path = parsed.path or "/" + path_segments = path.split("/") + if ( + not path.startswith("/") + or "" in path_segments[1:-1] + or any(segment in {".", ".."} for segment in path_segments) + or any(char in path for char in ("%", "\\", ";", "*", "?", "[", "]", "{", "}")) + ): + raise ValueError("MCP mutation payload URL path must be literal and canonical") + default_port = 443 + authority = hostname if port in {None, default_port} else f"{hostname}:{port}" + canonical = f"{parsed.scheme}://{authority}{path}" + if raw_url != canonical: + raise ValueError("MCP mutation payload URL must be canonical") + headers = payload.get("headers") + if not isinstance(headers, dict) or set(headers) != {"Authorization"}: + raise ValueError("MCP mutation payload must contain one Authorization header") + authorization = headers.get("Authorization") + authorization_match = ( + ENV_PLACEHOLDER_RE.fullmatch(authorization) + if isinstance(authorization, str) + else None + ) + if authorization_match is None: + raise ValueError( + "Hermes MCP Authorization must contain an OpenShell environment placeholder" + ) + if action == "add" and _credential_name_is_reserved(authorization_match.group(1)): + raise ValueError( + "Hermes MCP Authorization uses a reserved credential environment name" + ) + + +def _managed_candidate(payload: dict[str, object]) -> dict[str, object]: + headers = payload.get("headers") + if not isinstance(headers, dict): + raise ValueError("MCP mutation payload headers must be an object") + candidate: dict[str, object] = { + "url": payload.get("url"), + "enabled": True, + "timeout": 120, + "connect_timeout": 60, + "tools": {"resources": True, "prompts": True}, + } + if headers: + candidate["headers"] = headers + return candidate + + +def _mutate(data: object, action: str, payload: dict[str, object]) -> tuple[dict, bool]: + if not isinstance(data, dict): + raise ValueError("Invalid Hermes config: expected a YAML object") + server_name = payload.get("server") + if not isinstance(server_name, str) or not server_name: + raise ValueError("MCP mutation payload has no server name") + + servers = data.get("mcp_servers") + if servers is None: + servers = {} + data["mcp_servers"] = servers + if not isinstance(servers, dict): + raise ValueError("Invalid Hermes config: mcp_servers must be an object") + + if action == "add": + replace = payload.get("replace_existing") is True + if server_name in servers and not replace: + raise ValueError( + f"MCP server '{server_name}' already exists in Hermes config and is not managed by NemoClaw." + ) + candidate = _managed_candidate(payload) + if servers.get(server_name) == candidate: + return data, False + servers[server_name] = candidate + return data, True + + if action != "remove": + raise ValueError(f"Unsupported MCP config action '{action}'") + if server_name not in servers: + return data, False + if payload.get("force") is not True: + current = servers.get(server_name) + if current != _managed_candidate(payload): + raise ValueError( + f"Refusing to remove modified Hermes MCP server '{server_name}'. Use --force to remove it." + ) + servers.pop(server_name, None) + if not servers: + data.pop("mcp_servers", None) + return data, True + + +def _managed_hash_paths(privileged: bool) -> tuple[str, ...]: + compatibility = os.path.join(HERMES_DIR, ".config-hash") + return (STRICT_HASH_PATH, compatibility) if privileged else (compatibility,) + + +def _refresh_and_verify_hashes(guard: ModuleType, privileged: bool) -> None: + if privileged: + guard.refresh_hashes(HERMES_DIR, STRICT_HASH_PATH, "strict") + guard.refresh_hashes(HERMES_DIR, STRICT_HASH_PATH, "compat") + compat_text, _ = guard._read_text(os.path.join(HERMES_DIR, ".config-hash")) + expected_text, _, _ = guard._hash_text( + os.path.join(HERMES_DIR, "config.yaml"), + os.path.join(HERMES_DIR, ".env"), + ) + if compat_text != expected_text: + raise RuntimeError("Hermes compatibility config hash is stale") + if privileged: + strict_text, _ = guard._read_text(STRICT_HASH_PATH) + else: + strict_text = compat_text + if strict_text != compat_text: + raise RuntimeError("Hermes strict and compatibility config hashes differ") + + +def _restore_hash_snapshots( + guard: ModuleType, originals: dict[str, tuple[str, object]] +) -> None: + for path, (original_text, original_snapshot) in originals.items(): + _, current_snapshot = guard._read_text(path) + guard._write_existing( + path, + original_text, + current_snapshot, + mode=int(getattr(original_snapshot, "mode")), + ) + restored_text, _ = guard._read_text(path) + if restored_text != original_text: + raise RuntimeError(f"Failed to restore Hermes hash file {path}") + + +def apply_transaction(action: str, payload: dict[str, object]) -> bool: + _validate_payload(action, payload) + privileged = os.geteuid() == 0 + guard = _load_guard() + original_text, original_snapshot = guard._read_text(CONFIG_PATH) + _assert_mutable_snapshot(original_snapshot) + hash_originals = { + path: guard._read_text(path) for path in _managed_hash_paths(privileged) + } + parsed = yaml.safe_load(original_text) + if parsed is None: + parsed = {} + updated, changed = _mutate(parsed, action, payload) + if not changed: + try: + _refresh_and_verify_hashes(guard, privileged) + except Exception as hash_error: + try: + _restore_hash_snapshots(guard, hash_originals) + except Exception as rollback_error: + raise RuntimeError( + f"Hermes MCP hash refresh failed ({hash_error}); " + f"hash rollback also failed ({rollback_error})" + ) from rollback_error + raise + return False + + updated_text = yaml.safe_dump(updated, sort_keys=False) + replacement_snapshot = None + try: + guard._write_existing( + CONFIG_PATH, + updated_text, + original_snapshot, + mode=original_snapshot.mode, + ) + _, replacement_snapshot = guard._read_text(CONFIG_PATH) + _refresh_and_verify_hashes(guard, privileged) + except Exception as mutation_error: + if replacement_snapshot is None: + raise + try: + guard._write_existing( + CONFIG_PATH, + original_text, + replacement_snapshot, + mode=original_snapshot.mode, + ) + _refresh_and_verify_hashes(guard, privileged) + except Exception as rollback_error: + raise RuntimeError( + f"Hermes MCP config update failed ({mutation_error}); rollback also failed ({rollback_error})" + ) from rollback_error + raise + return True + + +def apply_transaction_and_reload( + action: str, payload: dict[str, object] +) -> dict[str, object]: + """Commit config+hashes and runtime reload as one recoverable operation.""" + _validate_payload(action, payload) + privileged = os.geteuid() == 0 + guard = _load_guard() + original_text, original_snapshot = guard._read_text(CONFIG_PATH) + hash_originals = { + path: guard._read_text(path) for path in _managed_hash_paths(privileged) + } + parsed = yaml.safe_load(original_text) + if parsed is None: + parsed = {} + expected_data, expected_changed = _mutate(parsed, action, payload) + expected_text = ( + yaml.safe_dump(expected_data, sort_keys=False) + if expected_changed + else original_text + ) + + changed = apply_transaction(action, payload) + try: + reloaded = reload_gateway() + except Exception as reload_error: + if not changed: + raise RuntimeError( + f"Hermes MCP runtime reload failed with unchanged config ({reload_error})" + ) from reload_error + rollback_errors: list[str] = [] + try: + current_text, current_snapshot = guard._read_text(CONFIG_PATH) + if current_text != expected_text: + raise RuntimeError( + "Hermes config changed concurrently after MCP mutation; refusing rollback" + ) + guard._write_existing( + CONFIG_PATH, + original_text, + current_snapshot, + mode=int(getattr(original_snapshot, "mode")), + ) + try: + _refresh_and_verify_hashes(guard, privileged) + except Exception: + _restore_hash_snapshots(guard, hash_originals) + raise + except Exception as rollback_error: + rollback_errors.append(f"config/hash rollback failed: {rollback_error}") + else: + try: + rollback_reloaded = reload_gateway() + if not rollback_reloaded: + rollback_errors.append( + "old-config runtime reload was not verified because the gateway stopped" + ) + except Exception as rollback_reload_error: + rollback_errors.append( + f"old-config runtime reload failed: {rollback_reload_error}" + ) + detail = "; ".join(rollback_errors) or "config and hashes were restored" + raise RuntimeError( + f"Hermes MCP runtime reload failed ({reload_error}); {detail}" + ) from reload_error + return {"ok": True, "changed": changed, "reloaded": reloaded} + + +def _process_arguments(pid: int) -> list[bytes]: + try: + with open(f"/proc/{pid}/cmdline", "rb") as command_line: + return [ + argument + for argument in command_line.read(16 * 1024).split(b"\0") + if argument + ] + except FileNotFoundError: + return [] + + +def _is_trusted_gateway_process(pid: int) -> bool: + arguments = _process_arguments(pid) + return any( + arguments[index] in TRUSTED_HERMES_GATEWAY_LAUNCHERS + and arguments[index + 1 : index + 3] == [b"gateway", b"run"] + for index in range(max(0, len(arguments) - 2)) + ) + + +def _process_parent_pid(pid: int) -> int | None: + try: + with open(f"/proc/{pid}/status", encoding="utf-8") as status_file: + for line in status_file: + if line.startswith("PPid:"): + return int(line.split()[1]) + except (FileNotFoundError, ValueError, IndexError): + return None + return None + + +def _is_service_manager_process(pid: int) -> bool: + arguments = _process_arguments(pid) + if not arguments: + return False + if arguments == [SERVICE_MANAGER_PATH]: + return True + return ( + os.path.basename(arguments[0]) in {b"bash", b"sh"} + and len(arguments) == 2 + and arguments[1] == SERVICE_MANAGER_PATH + ) + + +def _gateway_has_managed_parent(pid: int) -> bool: + parent_pid = _process_parent_pid(pid) + return parent_pid is not None and _is_service_manager_process(parent_pid) + + +def _gateway_pid_record_candidate(expected_uid: int) -> tuple[int, int | None] | None: + """Read Hermes runtime metadata as an untrusted PID candidate. + + Pinned Hermes rejects NemoClaw's root-owned ``hermes.real`` wrapper target + before returning the otherwise valid PID/lock record. The candidate is + never authority by itself: ``_gateway_identity`` still requires the live + same-UID process, exact trusted launcher argv, managed parent, and a stable + process start identity before mutation or reload. + """ + + no_follow = getattr(os, "O_NOFOLLOW", 0) + non_blocking = getattr(os, "O_NONBLOCK", 0) + if not no_follow or not non_blocking: + raise PermissionError("Hermes gateway PID record cannot be opened safely") + flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | no_follow | non_blocking + try: + descriptor = os.open(GATEWAY_PID_PATH, flags) + except FileNotFoundError: + return None + except OSError as error: + raise PermissionError( + "Hermes gateway PID record cannot be opened safely" + ) from error + + try: + before = os.fstat(descriptor) + if ( + not stat.S_ISREG(before.st_mode) + or before.st_uid != expected_uid + or before.st_nlink != 1 + or before.st_size <= 0 + or before.st_size > MAX_GATEWAY_PID_RECORD_BYTES + ): + raise PermissionError("Hermes gateway PID record is unsafe") + raw = os.read(descriptor, MAX_GATEWAY_PID_RECORD_BYTES + 1) + after = os.fstat(descriptor) + if ( + len(raw) != before.st_size + or len(raw) > MAX_GATEWAY_PID_RECORD_BYTES + or ( + before.st_dev, + before.st_ino, + before.st_mode, + before.st_uid, + before.st_nlink, + before.st_size, + before.st_mtime_ns, + before.st_ctime_ns, + ) + != ( + after.st_dev, + after.st_ino, + after.st_mode, + after.st_uid, + after.st_nlink, + after.st_size, + after.st_mtime_ns, + after.st_ctime_ns, + ) + ): + raise PermissionError("Hermes gateway PID record changed while reading") + finally: + os.close(descriptor) + + try: + decoded = raw.decode("utf-8").strip() + except UnicodeDecodeError as error: + raise PermissionError("Hermes gateway PID record is malformed") from error + try: + record: object = json.loads(decoded) + except json.JSONDecodeError: + try: + record = {"pid": int(decoded)} + except ValueError as error: + raise PermissionError("Hermes gateway PID record is malformed") from error + if isinstance(record, int) and not isinstance(record, bool): + record = {"pid": record} + if not isinstance(record, dict): + raise PermissionError("Hermes gateway PID record is malformed") + + pid = record.get("pid") + recorded_start = record.get("start_time") + if isinstance(pid, bool) or not isinstance(pid, int) or pid <= 1: + raise PermissionError("Hermes gateway PID record is malformed") + if recorded_start is not None and ( + isinstance(recorded_start, bool) + or not isinstance(recorded_start, int) + or recorded_start <= 0 + ): + raise PermissionError("Hermes gateway PID record is malformed") + return pid, recorded_start + + +def _gateway_identity() -> tuple[int, object] | None: + os.environ["HERMES_HOME"] = HERMES_DIR + from gateway.status import get_process_start_time, get_running_pid + + expected_uid = pwd.getpwnam("gateway").pw_uid if os.geteuid() == 0 else os.geteuid() + pid = get_running_pid(cleanup_stale=False) + if not pid: + from gateway.status import is_gateway_runtime_lock_active + + if not is_gateway_runtime_lock_active(): + return None + candidate = _gateway_pid_record_candidate(expected_uid) + if candidate is None: + return None + numeric_pid, recorded_start = candidate + else: + numeric_pid = int(pid) + recorded_start = None + try: + owner_uid = os.stat(f"/proc/{numeric_pid}").st_uid + except FileNotFoundError: + return None + if owner_uid != expected_uid: + expected_identity = "gateway" if os.geteuid() == 0 else "sandbox" + raise PermissionError( + f"Hermes gateway is not owned by the expected {expected_identity} identity" + ) + if not _is_trusted_gateway_process(numeric_pid): + raise PermissionError( + "Hermes gateway PID does not identify the trusted launcher" + ) + start_time = get_process_start_time(numeric_pid) + if start_time is None: + raise PermissionError("Hermes gateway process start identity is unavailable") + if recorded_start is not None and recorded_start != start_time: + return None + if get_process_start_time(numeric_pid) != start_time: + return None + return numeric_pid, start_time + + +def _gateway_health_endpoint_ready(port: int, timeout_seconds: float = 2) -> bool: + connection = http.client.HTTPConnection("127.0.0.1", port, timeout=timeout_seconds) + try: + connection.request("GET", "/health") + response = connection.getresponse() + response.read() + return response.status in {200, 401} + except OSError: + return False + finally: + connection.close() + + +def _gateway_health_phase(deadline: float | None = None) -> tuple[bool, str]: + # Hermes can bind its internal API before the managed service loop repairs + # the public socat relay after a SIGUSR1 reload. A successful MCP command + # must not return during that gap: callers use the documented public port. + def probe_timeout() -> float: + if deadline is None: + return 2 + return max(0, min(2, deadline - time.monotonic())) + + internal_timeout = probe_timeout() + if internal_timeout <= 0 or not _gateway_health_endpoint_ready( + GATEWAY_INTERNAL_PORT, internal_timeout + ): + return False, "waiting-for-internal-health-on-18642" + public_timeout = probe_timeout() + if public_timeout <= 0 or not _gateway_health_endpoint_ready( + GATEWAY_PUBLIC_PORT, public_timeout + ): + return False, "waiting-for-public-relay-health-on-8642" + return True, "waiting-for-stable-replacement-identity" + + +def _gateway_healthy() -> bool: + return _gateway_health_phase()[0] + + +def reload_gateway() -> bool: + previous = _gateway_identity() + if previous is None: + return False + try: + os.kill(previous[0], signal.SIGUSR1) + except ProcessLookupError: + if _gateway_identity() is None: + return False + raise + + started_at = time.monotonic() + deadline = started_at + RELOAD_TIMEOUT_SECONDS + re_kick_not_before = started_at + (RELOAD_TIMEOUT_SECONDS / 2) + re_kick_attempted = False + re_kick_sent = False + phase_order = { + "waiting-for-replacement-identity": 0, + "waiting-for-internal-health-on-18642": 1, + "waiting-for-public-relay-health-on-8642": 2, + "waiting-for-stable-replacement-identity": 3, + } + last_safe_phase = "waiting-for-replacement-identity" + while True: + now = time.monotonic() + if now >= deadline: + break + current = _gateway_identity() + if current is not None and current != previous: + healthy, observed_phase = _gateway_health_phase(deadline) + if phase_order[observed_phase] > phase_order[last_safe_phase]: + last_safe_phase = observed_phase + if healthy: + confirmed = _gateway_identity() + if confirmed == current and time.monotonic() < deadline: + return True + + # A pinned Hermes gateway can remain alive without converging after the + # first SIGUSR1. Give it half of the existing total deadline, then + # permit one additional desired-config signal. Re-read the complete + # trusted identity immediately before signaling and require its managed + # parent so known stale or unmanaged identities are refused. + now = time.monotonic() + if ( + not re_kick_attempted + and now >= re_kick_not_before + and now < deadline + and current is not None + and _gateway_has_managed_parent(current[0]) + and _gateway_identity() == current + and time.monotonic() < deadline + ): + re_kick_attempted = True + try: + os.kill(current[0], signal.SIGUSR1) + except ProcessLookupError: + pass + else: + re_kick_sent = True + remaining = deadline - time.monotonic() + if remaining <= 0: + break + time.sleep(min(1, remaining)) + raise TimeoutError( + "Hermes gateway did not complete its managed MCP reload " + f"(last safe phase: {last_safe_phase}; " + f"re-kick attempted: {'yes' if re_kick_attempted else 'no'}; " + f"re-kick sent: {'yes' if re_kick_sent else 'no'})" + ) + + +def _assert_non_root_lifecycle_identity() -> None: + """Allow only an active same-uid Hermes workload topology. + + Root-started sandboxes stamp a root-owned read-only runtime marker and run + Hermes as the dedicated gateway uid. OpenShell current main starts the + workload and gateway as the sandbox uid. Direct sandbox execution cannot + cross from the former topology into the latter. + """ + # invalidState: an ordinary sandbox process claims same-UID mutation + # authority while Hermes actually runs in the legacy root-separated + # topology. + # sourceBoundary: OpenShell owns workload topology; NemoClaw owns the + # immutable root-lifecycle marker and validates it before mutation. + # whyNotSourceFix: OpenShell 0.0.72 supports both topologies but exposes no + # attested same-UID capability that this packaged helper can query. + # regressionTest: hermes-mcp-config-transaction.test.ts rejects both probe + # and add when the root-lifecycle marker identifies the legacy topology. + # removalCondition: remove this marker check when OpenShell unifies the + # topology or exposes an attested execution-identity capability. + try: + root_marker = os.lstat(ROOT_LIFECYCLE_MARKER) + except FileNotFoundError: + root_marker = None + if root_marker is not None: + if not stat.S_ISREG(root_marker.st_mode) or root_marker.st_uid != 0: + raise PermissionError("Hermes root lifecycle marker is unsafe") + raise PermissionError( + "Hermes MCP mutation requires a same-uid OpenShell sandbox runtime" + ) + identity = _gateway_identity() + if identity is None: + raise RuntimeError("Hermes gateway is not running for managed MCP reload") + if not _gateway_has_managed_parent(identity[0]): + raise RuntimeError( + "Hermes gateway is not running under the managed service lifecycle" + ) + if _gateway_identity() != identity: + raise RuntimeError("Hermes gateway is not running for managed MCP reload") + + +def probe() -> dict[str, object]: + """Prove the packaged helper is available without mutating config.""" + if os.geteuid() != 0: + _assert_non_root_lifecycle_identity() + return {"ok": True} + + +def execute(action: str, payload: dict[str, object]) -> dict[str, object]: + _validate_payload(action, payload) + if os.geteuid() != 0: + _assert_non_root_lifecycle_identity() + return apply_transaction_and_reload(action, payload) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("action", choices=("add", "remove", "probe")) + parser.add_argument("--payload") + args = parser.parse_args() + payload: dict[str, object] | None = None + try: + if args.action == "probe": + if args.payload is not None: + raise ValueError("Hermes MCP lifecycle probe does not accept --payload") + result = probe() + elif args.payload is None: + raise ValueError("Hermes MCP mutation requires --payload") + else: + payload = _parse_payload(args.payload) + result = execute(args.action, payload) + except Exception as error: + print(_sanitize_error_message(error, payload), file=sys.stderr) + return 2 + print(json.dumps(result, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/agents/hermes/start.sh b/agents/hermes/start.sh index 3a245cc1f68..8955d266ff2 100755 --- a/agents/hermes/start.sh +++ b/agents/hermes/start.sh @@ -2080,6 +2080,23 @@ hermes_socat_bridge_healthy() { gateway_control_pid_owns_tcp_listener "$pid" "$port" } +hermes_api_socat_bridge_healthy() { + local pid="$1" + local port="$2" + local code + hermes_socat_bridge_healthy api-socat "$pid" "$port" || return 1 + # A listener-owning socat parent can survive a gateway SIGUSR1 replacement + # while its relay path no longer reaches the replacement. Validate the same + # public HTTP path clients use so the managed supervisor repairs that stale + # bridge instead of treating its listener as sufficient proof of health. + code="$(curl -so /dev/null -w '%{http_code}' --max-time 2 \ + "http://127.0.0.1:${port}/health" 2>/dev/null || echo 000)" + case "$code" in + 200 | 401) hermes_socat_bridge_healthy api-socat "$pid" "$port" ;; + *) return 1 ;; + esac +} + hermes_dashboard_healthy() { local pid="$1" local code @@ -2096,7 +2113,7 @@ hermes_dashboard_healthy() { } hermes_auxiliaries_need_recovery() { - hermes_socat_bridge_healthy api-socat "${SOCAT_PID:-}" "$PUBLIC_PORT" || return 0 + hermes_api_socat_bridge_healthy "${SOCAT_PID:-}" "$PUBLIC_PORT" || return 0 hermes_dashboard_healthy "${DASHBOARD_PID:-}" || return 0 hermes_socat_bridge_healthy dashboard-socat "${DASHBOARD_SOCAT_PID:-}" "$DASHBOARD_PUBLIC_PORT" || return 0 return 1 @@ -2137,11 +2154,12 @@ ensure_hermes_supervised_auxiliaries() { dashboard_user=sandbox fi - if ! hermes_socat_bridge_healthy api-socat "${SOCAT_PID:-}" "$PUBLIC_PORT"; then + if ! hermes_api_socat_bridge_healthy "${SOCAT_PID:-}" "$PUBLIC_PORT"; then hermes_stop_tracked_role api-socat "${SOCAT_PID:-0}" current "$PUBLIC_PORT" || return 1 SOCAT_PID="" start_socat_forwarder \ "$PUBLIC_PORT" "$INTERNAL_PORT" "API" SOCAT_PID "$GATEWAY_PID" "$gateway_user" || return 1 + hermes_api_socat_bridge_healthy "$SOCAT_PID" "$PUBLIC_PORT" || return 1 fi if ! hermes_dashboard_healthy "${DASHBOARD_PID:-}"; then # A live PID is not sufficient: it may be reused, alive without the exact @@ -2765,6 +2783,24 @@ if [ ${#NEMOCLAW_CMD[@]} -gt 0 ]; then exec "${STEP_DOWN_PREFIX_SANDBOX[@]}" "${NEMOCLAW_CMD[@]}" fi +# Same-uid MCP transaction commands are valid only in OpenShell's non-root +# workload topology. Stamp the legacy root-separated path before its gateway +# can start so ordinary sandbox exec fails closed there. +# invalidState: an ordinary sandbox process claims same-UID mutation authority +# while Hermes actually runs in the legacy root-separated topology. +# sourceBoundary: OpenShell owns workload topology; NemoClaw owns the immutable +# root-lifecycle marker and stamps it before starting the root-separated gateway. +# whyNotSourceFix: OpenShell 0.0.72 supports both topologies but exposes no +# attested same-UID capability that this packaged entrypoint can query. +# regressionTest: hermes-mcp-config-transaction.test.ts rejects both probe and +# add when the root-lifecycle marker identifies the legacy topology. +# removalCondition: remove this marker stamp when OpenShell unifies the topology +# or exposes an attested execution-identity capability. +install -d -m 0755 -o root -g root /run/nemoclaw +printf '%s\n' 'root-separated' >/run/nemoclaw/hermes-root-lifecycle +chown root:root /run/nemoclaw/hermes-root-lifecycle +chmod 0444 /run/nemoclaw/hermes-root-lifecycle + # SECURITY: Protect gateway log from sandbox user tampering prepare_restricted_log /tmp/gateway.log gateway:gateway 600 diff --git a/agents/langchain-deepagents-code/dcode-wrapper.sh b/agents/langchain-deepagents-code/dcode-wrapper.sh index a5f7aa1defe..5471fe0b0fd 100755 --- a/agents/langchain-deepagents-code/dcode-wrapper.sh +++ b/agents/langchain-deepagents-code/dcode-wrapper.sh @@ -28,6 +28,7 @@ export OPENAI_BASE_URL="${OPENAI_BASE_URL:-https://inference.local/v1}" unset PYTHONHOME PYTHONPATH readonly DEEPAGENTS_ENV_FILE="/sandbox/.deepagents/.env" +readonly OPENSHELL_ENV_PLACEHOLDER_PREFIX="openshell:resolve:env:" readonly DEEPAGENTS_CONFIG_FILE="/sandbox/.deepagents/config.toml" readonly OPENSHELL_TLS_KEY_PATH="/etc/openshell/tls/client/tls.key" readonly DEEPAGENTS_AUTH_FILE="/sandbox/.deepagents/.state/auth.json" @@ -70,6 +71,9 @@ run_dcode() { # dcode may resolve those to credentials the raw scan cannot see. # * Runtime env iteration uses `env -0` so names that are not valid Bash # identifiers (e.g. with hyphens) are still classified. +# * OpenShell credential placeholders are allowed only when the complete +# value names the same valid env key, either canonically or with an +# OpenShell `v_` revision prefix. Any other occurrence is refused. # - Regression: the parity tests in # test/langchain-deepagents-code-image.test.ts pin the canonical # TOKEN_PREFIX_PATTERNS, CONTEXT_PATTERNS, and SECRET_BLOCK_PATTERNS @@ -295,6 +299,41 @@ is_dynamic_dotenv_value() { return 1 } +is_openshell_env_placeholder_for_name() { + local name="$1" + local value="$2" + local canonical revision_prefix revision_suffix versioned revision + + # OPENSHELL_TLS_KEY is supervisor infrastructure, not a provider credential. + # Only its exact mounted path is accepted from the runtime environment below; + # never let a provider placeholder bypass that name/value allowlist. + [ "$name" != "OPENSHELL_TLS_KEY" ] || return 1 + + # Keep this identifier contract aligned with OpenShell provider env keys. + if [ -z "$name" ] || [ "${#name}" -gt 128 ]; then + return 1 + fi + case "$name" in + [0123456789]* | *[!ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_]*) return 1 ;; + esac + + canonical="${OPENSHELL_ENV_PLACEHOLDER_PREFIX}${name}" + [ "$value" = "$canonical" ] && return 0 + + revision_prefix="${OPENSHELL_ENV_PLACEHOLDER_PREFIX}v" + revision_suffix="_${name}" + versioned="${value#"$revision_prefix"}" + [ "$versioned" != "$value" ] || return 1 + revision="${versioned%"$revision_suffix"}" + [ "$revision" != "$versioned" ] || return 1 + [ "$versioned" = "$revision$revision_suffix" ] || return 1 + [ "${#revision}" -le 20 ] || return 1 + case "$revision" in + "" | *[!0-9]*) return 1 ;; + *) return 0 ;; + esac +} + refuse_secret_env() { local source="$1" local name="$2" @@ -311,6 +350,14 @@ refuse_dynamic_env() { exit 2 } +refuse_invalid_openshell_placeholder() { + local source="$1" + local name="$2" + printf 'dcode: refusing to start — %s contains an invalid OpenShell credential placeholder in %s.\n' "$source" "$name" >&2 + printf ' Use only the exact placeholder for that same environment variable.\n' >&2 + exit 2 +} + refuse_auth_store_credentials() { local source="$1" printf 'dcode: refusing to start — %s contains stored Deep Agents Code credentials.\n' "$source" >&2 @@ -324,6 +371,12 @@ assert_no_secret_runtime_env() { name="${pair%%=*}" [ "$name" != "$pair" ] || continue value="${pair#*=}" + if [[ "$value" == *"$OPENSHELL_ENV_PLACEHOLDER_PREFIX"* ]]; then + if is_openshell_env_placeholder_for_name "$name" "$value"; then + continue + fi + refuse_invalid_openshell_placeholder "runtime environment variable" "$name" + fi if is_managed_token_value_for_name "$name" "$value"; then continue fi @@ -381,6 +434,12 @@ assert_no_secret_env_file() { if is_dynamic_dotenv_value "$value"; then refuse_dynamic_env "$env_file" "$key" fi + if [[ "$value" == *"$OPENSHELL_ENV_PLACEHOLDER_PREFIX"* ]]; then + if is_openshell_env_placeholder_for_name "$key" "$value"; then + continue + fi + refuse_invalid_openshell_placeholder "$env_file" "$key" + fi if is_managed_token_value_for_name "$key" "$value"; then continue fi @@ -444,6 +503,11 @@ assert_no_secret_env_file assert_no_auth_store_credentials assert_no_codex_auth_credentials +if [ "${1:-}" = "--nemoclaw-mcp-capability" ] && [ "$#" -eq 1 ]; then + printf '%s\n' 'NEMOCLAW_DEEPAGENTS_MCP_CAPABILITY=1' + exit 0 +fi + # SECURITY: managed identity/status display boundary. # - Invalid state: config.toml and runtime environment values are mutable inside # the sandbox and can contain terminal controls, credentials, unsafe endpoint @@ -678,7 +742,7 @@ for arg in "$@"; do --sandbox-setup | --sandbox-setup=*) reject_managed_override "sandbox isolation" "$arg" ;; - --mcp-config | --mcp-config=* | --trust-project-mcp | --no-mcp=*) + --mcp-config | --mcp-config=* | --trust-project-mcp | --no-mcp | --no-mcp=*) reject_managed_override "MCP posture" "$arg" ;; --shell-allow-list | --shell-allow-list=* | -S | -S?*) @@ -766,6 +830,17 @@ while [ "$arg_index" -lt "${#dcode_args[@]}" ]; do arg_index=$((arg_index + 1)) done -extra_args=(--sandbox none --no-mcp) +extra_args=(--sandbox none) +# The root-owned package helper validates the complete sandbox-user-owned file +# as strict HTTPS-only NemoClaw config before any upstream parser sees it. +managed_mcp_config="$( + /opt/venv/bin/python3 -I -c \ + 'from deepagents_code._nemoclaw_managed import managed_mcp_config_path; print(managed_mcp_config_path() or "")' +)" +if [ -n "$managed_mcp_config" ]; then + extra_args+=(--mcp-config "$managed_mcp_config") +else + extra_args+=(--no-mcp) +fi run_dcode "${extra_args[@]}" "$@" diff --git a/agents/langchain-deepagents-code/manifest.yaml b/agents/langchain-deepagents-code/manifest.yaml index a2973e0c267..aa0ac8232a5 100644 --- a/agents/langchain-deepagents-code/manifest.yaml +++ b/agents/langchain-deepagents-code/manifest.yaml @@ -47,13 +47,15 @@ state_dirs: # ── Top-level durable state files ─────────────────────────────── # config.toml is non-secret NemoClaw-generated provider/model configuration. -# .env and .mcp.json are intentionally omitted because they may contain -# user-added service credentials; this managed harness disables MCP at runtime. +# .env and user-authored .deepagents/.mcp.json content are intentionally omitted +# because they may contain service credentials. NemoClaw writes only direct-HTTP +# bridge endpoint config and OpenShell placeholders to the user-level MCP file, +# then restores its managed entries from the registry after rebuild. state_files: - path: config.toml user_managed_files: - - .env - - .mcp.json + - .deepagents/.env + - .deepagents/.mcp.json device_pairing: false @@ -66,6 +68,11 @@ inference: model_config_key: "models.default" proxy_support: implicit +# ── MCP server support ─────────────────────────────────────────── +mcp: + support: bridge + adapter: deepagents-config + package_registry: hosts: - pypi.org diff --git a/agents/langchain-deepagents-code/patch-managed-deepagents-code.py b/agents/langchain-deepagents-code/patch-managed-deepagents-code.py index fcf62b3d484..6986baee31a 100644 --- a/agents/langchain-deepagents-code/patch-managed-deepagents-code.py +++ b/agents/langchain-deepagents-code/patch-managed-deepagents-code.py @@ -97,10 +97,19 @@ args.sandbox_snapshot_name = None if hasattr(args, "sandbox_setup"): args.sandbox_setup = None + from deepagents_code._nemoclaw_managed import ( + assert_safe_runtime as _nemoclaw_assert_safe_runtime, + managed_mcp_config_path as _nemoclaw_managed_mcp_config_path, + ) + + # The pinned release treats this as its trusted user-level config; + # /sandbox/.mcp.json is project-level and remains untrusted. + managed_mcp_config = _nemoclaw_managed_mcp_config_path() + has_managed_mcp = managed_mcp_config is not None if hasattr(args, "mcp_config"): - args.mcp_config = None + args.mcp_config = managed_mcp_config if has_managed_mcp else None if hasattr(args, "no_mcp"): - args.no_mcp = True + args.no_mcp = not has_managed_mcp if hasattr(args, "trust_project_mcp"): args.trust_project_mcp = False if hasattr(args, "shell_allow_list"): @@ -118,8 +127,6 @@ if hasattr(args, "startup_cmd"): args.startup_cmd = None - from deepagents_code._nemoclaw_managed import assert_safe_runtime as _nemoclaw_assert_safe_runtime - _nemoclaw_assert_safe_runtime() ''' @@ -423,8 +430,12 @@ async def run_non_interactive(*args, **kwargs): kwargs["model_params"] = None kwargs["profile_override"] = None kwargs["sandbox_type"] = "none" - kwargs["mcp_config_path"] = None - kwargs["no_mcp"] = True + from deepagents_code._nemoclaw_managed import managed_mcp_config_path + + managed_mcp_config = managed_mcp_config_path() + has_managed_mcp = managed_mcp_config is not None + kwargs["mcp_config_path"] = managed_mcp_config if has_managed_mcp else None + kwargs["no_mcp"] = not has_managed_mcp kwargs["trust_project_mcp"] = False kwargs["enable_interpreter"] = False kwargs["interpreter_ptc"] = None @@ -627,6 +638,7 @@ def _nemoclaw_select_with_auth_check(self, model_spec: str, provider: str) -> No from __future__ import annotations import json +import ipaddress import os import re import stat @@ -636,6 +648,7 @@ def _nemoclaw_select_with_auth_check(self, model_spec: str, provider: str) -> No _MANAGED_STATE_DIR = Path("/sandbox/.deepagents/.state") _AUTH_FILE = _MANAGED_STATE_DIR / "auth.json" _CODEX_AUTH_FILE = _MANAGED_STATE_DIR / "chatgpt-auth.json" +_MCP_CONFIG_FILE = Path("/sandbox/.deepagents/.mcp.json") _INFERENCE_BASE_URL_FILE = Path( "/usr/local/share/nemoclaw/dcode-inference-base-url" ) @@ -652,6 +665,13 @@ def _nemoclaw_select_with_auth_check(self, model_spec: str, provider: str) -> No "OTEL_EXPORTER_OTLP_HEADERS", "OTEL_EXPORTER_OTLP_TRACES_HEADERS", } +_OPENSHELL_ENV_PLACEHOLDER_PREFIX = "openshell:resolve:env:" +_MCP_SERVER_NAME = re.compile(r"[A-Za-z][A-Za-z0-9_-]{0,63}") +_MCP_ENV_NAME = re.compile(r"[A-Za-z_][A-Za-z0-9_]{0,127}") +_MCP_DNS_NAME = re.compile( + r"(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)*" + r"[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?" +) _SECRET_PATTERNS = tuple( (platform, re.compile(pattern, flags)) for platform, pattern, flags in ( @@ -684,6 +704,17 @@ def _contains_other_platform_secret(value: str, platform: str) -> bool: ) +def _is_openshell_placeholder_for_name(name: str, value: str) -> bool: + if name == "OPENSHELL_TLS_KEY" or not _MCP_ENV_NAME.fullmatch(name): + return False + canonical = f"{_OPENSHELL_ENV_PLACEHOLDER_PREFIX}{name}" + versioned = re.fullmatch( + rf"{re.escape(_OPENSHELL_ENV_PLACEHOLDER_PREFIX)}v[0-9]{{1,20}}_{re.escape(name)}", + value, + ) + return value == canonical or versioned is not None + + def _is_managed_value(name: str, value: str) -> bool: if name == "DEEPAGENTS_CODE_OPENAI_API_KEY": return value == "nemoclaw-managed-inference" @@ -704,6 +735,13 @@ def _is_managed_value(name: str, value: str) -> bool: def _assert_safe_environment() -> None: for name, value in os.environ.items(): + if _OPENSHELL_ENV_PLACEHOLDER_PREFIX in value: + if _is_openshell_placeholder_for_name(name, value): + continue + raise RuntimeError( + f"runtime environment variable {name} contains an invalid " + "OpenShell credential placeholder" + ) if _is_managed_value(name, value): continue if _contains_secret_shape(value) or ( @@ -739,6 +777,108 @@ def _assert_safe_auth_state() -> None: ) +def _validate_managed_mcp_url(value: object) -> None: + if not isinstance(value, str) or not value or len(value) > 2048: + raise RuntimeError("managed MCP server URL is invalid") + if value != value.strip() or any(ord(character) < 32 for character in value): + raise RuntimeError("managed MCP server URL is invalid") + if any(character in value for character in ("%", "\\", "*", "[", "]", "{", "}", ";")): + raise RuntimeError("managed MCP server URL is not canonical") + parsed = urlparse(value) + if ( + parsed.scheme != "https" + or not parsed.netloc + or not parsed.hostname + or parsed.username is not None + or parsed.password is not None + or parsed.params + or parsed.query + or parsed.fragment + or not parsed.path.startswith("/") + or "//" in parsed.path + ): + raise RuntimeError("managed MCP server URL is invalid") + try: + port = parsed.port + except ValueError as exc: + raise RuntimeError("managed MCP server URL port is invalid") from exc + if port is not None and not 1 <= port <= 65535: + raise RuntimeError("managed MCP server URL port is invalid") + hostname = parsed.hostname + expected_netloc = hostname if port is None else f"{hostname}:{port}" + if parsed.netloc != expected_netloc: + raise RuntimeError("managed MCP server URL hostname is not canonical") + try: + address = ipaddress.ip_address(hostname) + except ValueError: + if ( + hostname != hostname.lower() + or hostname.endswith(".") + or not _MCP_DNS_NAME.fullmatch(hostname) + or hostname == "localhost" + or hostname.endswith((".localhost", ".local", ".internal")) + ): + raise RuntimeError("managed MCP server URL hostname is invalid") + else: + if address.version != 4 or not address.is_global: + raise RuntimeError("managed MCP server URL address is not public IPv4") + if _contains_secret_shape(parsed.path): + raise RuntimeError("managed MCP server URL path contains credential-shaped data") + + +def _validate_managed_mcp_entry(server: object, entry: object) -> None: + if not isinstance(server, str) or not _MCP_SERVER_NAME.fullmatch(server): + raise RuntimeError("managed MCP config contains an invalid server name") + if not isinstance(entry, dict) or set(entry) != {"type", "url", "headers"}: + raise RuntimeError(f"managed MCP server {server} has an invalid shape") + if entry["type"] != "http": + raise RuntimeError(f"managed MCP server {server} must use HTTP transport") + _validate_managed_mcp_url(entry["url"]) + headers = entry["headers"] + if not isinstance(headers, dict) or set(headers) != {"Authorization"}: + raise RuntimeError(f"managed MCP server {server} has invalid headers") + authorization = headers["Authorization"] + if not isinstance(authorization, str) or not authorization.startswith("Bearer "): + raise RuntimeError(f"managed MCP server {server} has invalid authorization") + placeholder = authorization.removeprefix("Bearer ") + if not placeholder.startswith(_OPENSHELL_ENV_PLACEHOLDER_PREFIX): + raise RuntimeError(f"managed MCP server {server} must use an OpenShell placeholder") + suffix = placeholder.removeprefix(_OPENSHELL_ENV_PLACEHOLDER_PREFIX) + match = re.fullmatch(r"(?:v[0-9]{1,20}_)?([A-Za-z_][A-Za-z0-9_]{0,127})", suffix) + if match is None or not _is_openshell_placeholder_for_name(match.group(1), placeholder): + raise RuntimeError(f"managed MCP server {server} has an invalid OpenShell placeholder") + + +def managed_mcp_config_path() -> str | None: + """Return only a complete, strict, HTTP-only NemoClaw MCP config.""" + path = _MCP_CONFIG_FILE + if not path.exists() and not path.is_symlink(): + return None + if not path.is_file() or path.is_symlink(): + raise RuntimeError("managed MCP config is missing or unsafe") + try: + metadata = path.stat() + raw = path.read_text(encoding="utf-8") + except OSError as exc: + raise RuntimeError("managed MCP config is unreadable") from exc + if metadata.st_uid != os.getuid() or stat.S_IMODE(metadata.st_mode) != 0o600: + raise RuntimeError("managed MCP config has unsafe ownership or mode") + if not raw or len(raw.encode("utf-8")) > 262144: + raise RuntimeError("managed MCP config has invalid size") + try: + data = json.loads(raw) + except Exception as exc: + raise RuntimeError("managed MCP config is malformed") from exc + if not isinstance(data, dict) or set(data) != {"mcpServers"}: + raise RuntimeError("managed MCP config must contain only mcpServers") + servers = data["mcpServers"] + if not isinstance(servers, dict) or not servers or len(servers) > 64: + raise RuntimeError("managed MCP config has an invalid server map") + for server, entry in servers.items(): + _validate_managed_mcp_entry(server, entry) + return str(path) + + def managed_inference_base_url() -> str: """Read and validate the root-owned inference route baked into the image.""" path = _INFERENCE_BASE_URL_FILE diff --git a/agents/openclaw/dependency-review.md b/agents/openclaw/dependency-review.md new file mode 100644 index 00000000000..fa7fc58b4fa --- /dev/null +++ b/agents/openclaw/dependency-review.md @@ -0,0 +1,32 @@ + + + +# OpenClaw MCP Runtime Dependency Review + +This file records the reviewed `mcporter` baseline installed in the OpenClaw sandbox image. +Update it and `agents/openclaw/mcporter-runtime/package*.json` together whenever `MCPORTER_VERSION` or its integrity value changes in `Dockerfile.base` or `Dockerfile`. + +- Package: `mcporter@0.7.3` +- Purpose: in-sandbox OpenClaw MCP configuration and client adapter; it is not a host bridge, proxy, relay, or listener. +- Registry source: `https://registry.npmjs.org/mcporter/-/mcporter-0.7.3.tgz` +- Repository: `https://github.com/steipete/mcporter` +- License: `MIT`, from the npm registry package metadata. +- npm integrity: `sha512-egoPVYqTnWb3NjRIxo+xc8OrAI0dlPrJm9pAiZx0pImuNIV5rKhGtTnIfH/Y1ldGPVu74ibj3KR5c9U/QSdQFA==` +- Registry metadata independently queried from npm: 2026-06-30. +- Locked graph: `agents/openclaw/mcporter-runtime/package-lock.json` (npm lockfile version 3). +- Lock regeneration command: `npm --prefix agents/openclaw/mcporter-runtime install --package-lock-only --ignore-scripts --omit=dev` +- Advisory command: `npm --prefix agents/openclaw/mcporter-runtime ci --ignore-scripts --omit=dev && npm --prefix agents/openclaw/mcporter-runtime audit --omit=dev && npm --prefix agents/openclaw/mcporter-runtime audit signatures` +- Advisory review date: 2026-06-30. +- Advisory result: `0` known vulnerabilities across the resolved production dependency graph; npm verified registry signatures for all `120` resolved packages and attestations for `12` packages. + +Both image paths install the committed graph with `npm ci --ignore-scripts --omit=dev` because the published package declares no install-time lifecycle script and NemoClaw needs only its already-built CLI. +Disabling scripts also prevents transitive packages from executing lifecycle code during the trusted image build. +The lock records the exact version, registry URL, and integrity for every transitive package; the top-level registry integrity check remains an independent control. + +## Source-of-Truth Boundary + +- `invalidState`: the image installs a package graph, tarball, license, or advisory state that differs from the independently queried npm registry records for `mcporter@0.7.3`. +- `sourceBoundary`: npm owns registry metadata, tarball integrity, provenance signatures, and advisory responses; NemoClaw owns the exact lock, script-disabled install, Docker integrity assertion, and review record. +- `whyNotSourceFix`: a repository note cannot make external registry state trustworthy, so image builds execute `npm audit` and `npm audit signatures` against the locked production graph and reviewers compare the lock with the registry response. +- `regressionTest`: `test/mcporter-supply-chain.test.ts` keeps the version, integrity, lock metadata, Docker install flags, audit commands, and this review synchronized. +- `removalCondition`: remove this runtime dependency and review when OpenClaw provides the required authenticated Streamable HTTP client lifecycle without mcporter, or repeat the independent review for a newly pinned version. diff --git a/agents/openclaw/manifest.yaml b/agents/openclaw/manifest.yaml index 7bedc6e2577..01b09565f7e 100644 --- a/agents/openclaw/manifest.yaml +++ b/agents/openclaw/manifest.yaml @@ -81,6 +81,11 @@ inference: provider_type: gateway_managed proxy_support: explicit # configured in openclaw.json providers block +# ── MCP server support ─────────────────────────────────────────── +mcp: + support: bridge + adapter: mcporter + # ── Phone-home hosts ─────────────────────────────────────────── phone_home_hosts: - openclaw.ai diff --git a/agents/openclaw/mcporter-runtime/package-lock.json b/agents/openclaw/mcporter-runtime/package-lock.json new file mode 100644 index 00000000000..5d189f3e231 --- /dev/null +++ b/agents/openclaw/mcporter-runtime/package-lock.json @@ -0,0 +1,1801 @@ +{ + "name": "nemoclaw-mcporter-runtime", + "version": "0.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "nemoclaw-mcporter-runtime", + "version": "0.0.0", + "license": "Apache-2.0", + "dependencies": { + "mcporter": "0.7.3" + }, + "engines": { + "node": ">=22.16.0" + } + }, + "node_modules/@emnapi/core": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.11.1.tgz", + "integrity": "sha512-RSvbQmHzdKzNsLYa/wHrbc3KN4sYLKAdPZxqiM2HATqv/SBk2/ENSHpvXGaLOMcsAyz0poEGqkmmKYG3OWiJEQ==", + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "@emnapi/wasi-threads": "1.2.2", + "tslib": "^2.4.0" + } + }, + "node_modules/@emnapi/runtime": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.1.tgz", + "integrity": "sha512-vgj7R3y3Wgx24IQaGPA/R6YFXLHVMOZ0uVEyIQPaWs+rd1AzfEMXlAC22FYwO1XkKR6NPsq7mUandH8oIRdZFw==", + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@emnapi/wasi-threads": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.2.tgz", + "integrity": "sha512-c95qOXkHdydNKhscBTebqEC1CVAZpyqOfVfBzQ1qgzyl3gfeldUjIggDbIZgDKsHLgnsM+igH7TJ/eAasaVuMA==", + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@hono/node-server": { + "version": "1.19.14", + "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-1.19.14.tgz", + "integrity": "sha512-GwtvgtXxnWsucXvbQXkRgqksiH2Qed37H9xHZocE5sA3N8O8O8/8FA3uclQXxXVzc9XBZuEOMK7+r02FmSpHtw==", + "license": "MIT", + "engines": { + "node": ">=18.14.1" + }, + "peerDependencies": { + "hono": "^4" + } + }, + "node_modules/@iarna/toml": { + "version": "2.2.5", + "resolved": "https://registry.npmjs.org/@iarna/toml/-/toml-2.2.5.tgz", + "integrity": "sha512-trnsAYxU3xnS1gPHPyU961coFyLkh4gAD/0zQ5mymY4yOZ+CYvsPqUbOFSw0aDM4y0tV7tiFxL/1XfXPNC6IPg==", + "license": "ISC" + }, + "node_modules/@modelcontextprotocol/sdk": { + "version": "1.29.0", + "resolved": "https://registry.npmjs.org/@modelcontextprotocol/sdk/-/sdk-1.29.0.tgz", + "integrity": "sha512-zo37mZA9hJWpULgkRpowewez1y6ML5GsXJPY8FI0tBBCd77HEvza4jDqRKOXgHNn867PVGCyTdzqpz0izu5ZjQ==", + "license": "MIT", + "dependencies": { + "@hono/node-server": "^1.19.9", + "ajv": "^8.17.1", + "ajv-formats": "^3.0.1", + "content-type": "^1.0.5", + "cors": "^2.8.5", + "cross-spawn": "^7.0.5", + "eventsource": "^3.0.2", + "eventsource-parser": "^3.0.0", + "express": "^5.2.1", + "express-rate-limit": "^8.2.1", + "hono": "^4.11.4", + "jose": "^6.1.3", + "json-schema-typed": "^8.0.2", + "pkce-challenge": "^5.0.0", + "raw-body": "^3.0.0", + "zod": "^3.25 || ^4.0", + "zod-to-json-schema": "^3.25.1" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@cfworker/json-schema": "^4.1.1", + "zod": "^3.25 || ^4.0" + }, + "peerDependenciesMeta": { + "@cfworker/json-schema": { + "optional": true + }, + "zod": { + "optional": false + } + } + }, + "node_modules/@napi-rs/wasm-runtime": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.6.tgz", + "integrity": "sha512-ZLv/JdUfkvOy9eCnnBaGfiO+XimbjebAeO+MRQqD/B+FR1tnRN0tpKSJHRbE8sFfS6aqsXZ67TQjfwfsxULVbg==", + "license": "MIT", + "optional": true, + "dependencies": { + "@tybys/wasm-util": "^0.10.3" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + }, + "peerDependencies": { + "@emnapi/core": "^1.7.1", + "@emnapi/runtime": "^1.7.1" + } + }, + "node_modules/@oxc-project/types": { + "version": "0.103.0", + "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.103.0.tgz", + "integrity": "sha512-bkiYX5kaXWwUessFRSoXFkGIQTmc6dLGdxuRTrC+h8PSnIdZyuXHHlLAeTmOue5Br/a0/a7dHH0Gca6eXn9MKg==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/Boshen" + } + }, + "node_modules/@rolldown/binding-android-arm64": { + "version": "1.0.0-beta.57", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.0.0-beta.57.tgz", + "integrity": "sha512-GoOVDy8bjw9z1K30Oo803nSzXJS/vWhFijFsW3kzvZCO8IZwFnNa6pGctmbbJstKl3Fv6UBwyjJQN6msejW0IQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-arm64": { + "version": "1.0.0-beta.57", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.0.0-beta.57.tgz", + "integrity": "sha512-9c4FOhRGpl+PX7zBK5p17c5efpF9aSpTPgyigv57hXf5NjQUaJOOiejPLAtFiKNBIfm5Uu6yFkvLKzOafNvlTw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-x64": { + "version": "1.0.0-beta.57", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.0.0-beta.57.tgz", + "integrity": "sha512-6RsB8Qy4LnGqNGJJC/8uWeLWGOvbRL/KG5aJ8XXpSEupg/KQtlBEiFaYU/Ma5Usj1s+bt3ItkqZYAI50kSplBA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-freebsd-x64": { + "version": "1.0.0-beta.57", + "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.0.0-beta.57.tgz", + "integrity": "sha512-uA9kG7+MYkHTbqwv67Tx+5GV5YcKd33HCJIi0311iYBd25yuwyIqvJfBdt1VVB8tdOlyTb9cPAgfCki8nhwTQg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm-gnueabihf": { + "version": "1.0.0-beta.57", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.0.0-beta.57.tgz", + "integrity": "sha512-3KkS0cHsllT2T+Te+VZMKHNw6FPQihYsQh+8J4jkzwgvAQpbsbXmrqhkw3YU/QGRrD8qgcOvBr6z5y6Jid+rmw==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-gnu": { + "version": "1.0.0-beta.57", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.0.0-beta.57.tgz", + "integrity": "sha512-A3/wu1RgsHhqP3rVH2+sM81bpk+Qd2XaHTl8LtX5/1LNR7QVBFBCpAoiXwjTdGnI5cMdBVi7Z1pi52euW760Fw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-musl": { + "version": "1.0.0-beta.57", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.0.0-beta.57.tgz", + "integrity": "sha512-d0kIVezTQtazpyWjiJIn5to8JlwfKITDqwsFv0Xc6s31N16CD2PC/Pl2OtKgS7n8WLOJbfqgIp5ixYzTAxCqMg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-gnu": { + "version": "1.0.0-beta.57", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.0.0-beta.57.tgz", + "integrity": "sha512-E199LPijo98yrLjPCmETx8EF43sZf9t3guSrLee/ej1rCCc3zDVTR4xFfN9BRAapGVl7/8hYqbbiQPTkv73kUg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-musl": { + "version": "1.0.0-beta.57", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.0.0-beta.57.tgz", + "integrity": "sha512-++EQDpk/UJ33kY/BNsh7A7/P1sr/jbMuQ8cE554ZIy+tCUWCivo9zfyjDUoiMdnxqX6HLJEqqGnbGQOvzm2OMQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-openharmony-arm64": { + "version": "1.0.0-beta.57", + "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.0.0-beta.57.tgz", + "integrity": "sha512-voDEBcNqxbUv/GeXKFtxXVWA+H45P/8Dec4Ii/SbyJyGvCqV1j+nNHfnFUIiRQ2Q40DwPe/djvgYBs9PpETiMA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-wasm32-wasi": { + "version": "1.0.0-beta.57", + "resolved": "https://registry.npmjs.org/@rolldown/binding-wasm32-wasi/-/binding-wasm32-wasi-1.0.0-beta.57.tgz", + "integrity": "sha512-bRhcF7NLlCnpkzLVlVhrDEd0KH22VbTPkPTbMjlYvqhSmarxNIq5vtlQS8qmV7LkPKHrNLWyJW/V/sOyFba26Q==", + "cpu": [ + "wasm32" + ], + "license": "MIT", + "optional": true, + "dependencies": { + "@napi-rs/wasm-runtime": "^1.1.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@rolldown/binding-win32-arm64-msvc": { + "version": "1.0.0-beta.57", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.0.0-beta.57.tgz", + "integrity": "sha512-rnDVGRks2FQ2hgJ2g15pHtfxqkGFGjJQUDWzYznEkE8Ra2+Vag9OffxdbJMZqBWXHVM0iS4dv8qSiEn7bO+n1Q==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-win32-x64-msvc": { + "version": "1.0.0-beta.57", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.0.0-beta.57.tgz", + "integrity": "sha512-OqIUyNid1M4xTj6VRXp/Lht/qIP8fo25QyAZlCP+p6D2ATCEhyW4ZIFLnC9zAGN/HMbXoCzvwfa8Jjg/8J4YEg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/pluginutils": { + "version": "1.0.0-beta.57", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-beta.57.tgz", + "integrity": "sha512-aQNelgx14tGA+n2tNSa9x6/jeoCL9fkDeCei7nOKnHx0fEFRRMu5ReiITo+zZD5TzWDGGRjbSYCs93IfRIyTuQ==", + "license": "MIT" + }, + "node_modules/@tybys/wasm-util": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.3.tgz", + "integrity": "sha512-F3fo1MYrRJYL3zER0OUOmkutjr1Vp23m7OsSgp7nq4SP6OqX6C/56XFIPAl5bt3zaBRjmW7SGz3u/6LwFpYcOg==", + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/accepts": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", + "integrity": "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==", + "license": "MIT", + "dependencies": { + "mime-types": "^3.0.0", + "negotiator": "^1.0.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/acorn": { + "version": "8.17.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.17.0.tgz", + "integrity": "sha512-xRQbDb9BnwDafYNn6Vwl839DYVjqXYb1XVGtWAZ1kcDc6iwAL4hg3B1dZlRiuENFeO2H53gFG3in621AdERVAg==", + "license": "MIT", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/ajv": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, + "node_modules/ansi-regex": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz", + "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/body-parser": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.3.0.tgz", + "integrity": "sha512-2cGmJupaNgg+QUwVLAucDuWuoMZ6EX9iHDRswZ5lsNYEmwPaRknMPCLZz07yTzVq/83p4o/wzbDZbBrTvGGTIw==", + "license": "MIT", + "dependencies": { + "bytes": "^3.1.2", + "content-type": "^2.0.0", + "debug": "^4.4.3", + "http-errors": "^2.0.1", + "iconv-lite": "^0.7.2", + "on-finished": "^2.4.1", + "qs": "^6.15.2", + "raw-body": "^3.0.2", + "type-is": "^2.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/body-parser/node_modules/content-type": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.0.0.tgz", + "integrity": "sha512-j/O/d7GcZCyNl7/hwZAb606rzqkyvaDctLmckbxLzHvFBzTJHuGEdodATcP3yIRoDrLHkIATJuvzbFlp/ki2cQ==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/chalk": { + "version": "5.6.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.6.2.tgz", + "integrity": "sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA==", + "license": "MIT", + "engines": { + "node": "^12.17.0 || ^14.13 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/cli-cursor": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-5.0.0.tgz", + "integrity": "sha512-aCj4O5wKyszjMmDT4tZj93kxyydN/K5zPWSCe6/0AV/AA1pqe5ZBIw0a2ZfPQV7lL5/yb5HsUreJ6UFAF1tEQw==", + "license": "MIT", + "dependencies": { + "restore-cursor": "^5.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/cli-spinners": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/cli-spinners/-/cli-spinners-3.4.0.tgz", + "integrity": "sha512-bXfOC4QcT1tKXGorxL3wbJm6XJPDqEnij2gQ2m7ESQuE+/z9YFIWnl/5RpTiKWbMq3EVKR4fRLJGn6DVfu0mpw==", + "license": "MIT", + "engines": { + "node": ">=18.20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/commander": { + "version": "14.0.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-14.0.3.tgz", + "integrity": "sha512-H+y0Jo/T1RZ9qPP4Eh1pkcQcLRglraJaSLoyOtHxu6AapkjWVCy2Sit1QQ4x3Dng8qDlSsZEet7g5Pq06MvTgw==", + "license": "MIT", + "engines": { + "node": ">=20" + } + }, + "node_modules/content-disposition": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.1.0.tgz", + "integrity": "sha512-5jRCH9Z/+DRP7rkvY83B+yGIGX96OYdJmzngqnw2SBSxqCFPd0w2km3s5iawpGX8krnwSGmF0FW5Nhr0Hfai3g==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", + "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz", + "integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==", + "license": "MIT", + "engines": { + "node": ">=6.6.0" + } + }, + "node_modules/cors": { + "version": "2.8.6", + "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.6.tgz", + "integrity": "sha512-tJtZBBHA6vjIAaF6EnIaq6laBBP9aq/Y3ouVJjEfoHbRBcHBAHYcMh/w8LDrk2PvIMMq8gmopa5D4V8RmbrxGw==", + "license": "MIT", + "dependencies": { + "object-assign": "^4", + "vary": "^1" + }, + "engines": { + "node": ">= 0.10" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", + "license": "MIT" + }, + "node_modules/encodeurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.2.tgz", + "integrity": "sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-toolkit": { + "version": "1.49.0", + "resolved": "https://registry.npmjs.org/es-toolkit/-/es-toolkit-1.49.0.tgz", + "integrity": "sha512-G5iZ6Pc/FNRY/soKZHC+TxGDD83rHUDXxzaWhGCX44vAv/tMs56WMusnm/KMNK+luUPsgA9U28cGr4RDlSzL2g==", + "license": "MIT", + "workspaces": [ + "docs", + "benchmarks" + ] + }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", + "license": "MIT" + }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/eventsource": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/eventsource/-/eventsource-3.0.7.tgz", + "integrity": "sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA==", + "license": "MIT", + "dependencies": { + "eventsource-parser": "^3.0.1" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/eventsource-parser": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.1.0.tgz", + "integrity": "sha512-kJezFj9YFAMLeORyi7aCLxLbD5/qWMQnoMVlVPyHIll7lgRJCc3JVln9Vgl9nwQi0YkMnhdGTMNn7CkRRAptMg==", + "license": "MIT", + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/express": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", + "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", + "license": "MIT", + "dependencies": { + "accepts": "^2.0.0", + "body-parser": "^2.2.1", + "content-disposition": "^1.0.0", + "content-type": "^1.0.5", + "cookie": "^0.7.1", + "cookie-signature": "^1.2.1", + "debug": "^4.4.0", + "depd": "^2.0.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "finalhandler": "^2.1.0", + "fresh": "^2.0.0", + "http-errors": "^2.0.0", + "merge-descriptors": "^2.0.0", + "mime-types": "^3.0.0", + "on-finished": "^2.4.1", + "once": "^1.4.0", + "parseurl": "^1.3.3", + "proxy-addr": "^2.0.7", + "qs": "^6.14.0", + "range-parser": "^1.2.1", + "router": "^2.2.0", + "send": "^1.1.0", + "serve-static": "^2.2.0", + "statuses": "^2.0.1", + "type-is": "^2.0.1", + "vary": "^1.1.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/express-rate-limit": { + "version": "8.5.2", + "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-8.5.2.tgz", + "integrity": "sha512-5Kb34ipNX694DH48vN9irak1Qx30nb0PLYHXfJgw4YEjiC3ZEmZJhwOp+VfiCYwFzvFTdB9QkArYS5kXa2cx2A==", + "license": "MIT", + "dependencies": { + "ip-address": "^10.2.0" + }, + "engines": { + "node": ">= 16" + }, + "funding": { + "url": "https://github.com/sponsors/express-rate-limit" + }, + "peerDependencies": { + "express": ">= 4.11" + } + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "license": "MIT" + }, + "node_modules/fast-uri": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.3.tgz", + "integrity": "sha512-i70LwGWUduXqzicKXWshooq+sWL1K3WUU5rKZNG/0i3a1OSoX3HqhH5WbWwTmqWfor4urUakGPiRQcleRZTwOg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/finalhandler": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz", + "integrity": "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "on-finished": "^2.4.1", + "parseurl": "^1.3.3", + "statuses": "^2.0.1" + }, + "engines": { + "node": ">= 18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fresh": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz", + "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-east-asian-width": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/get-east-asian-width/-/get-east-asian-width-1.6.0.tgz", + "integrity": "sha512-QRbvDIbx6YklUe6RxeTeleMR0yv3cYH6PsPZHcnVn7xv7zO1BHN8r0XETu8n6Ye3Q+ahtSarc3WgtNWmehIBfA==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", + "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/hono": { + "version": "4.12.27", + "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.27.tgz", + "integrity": "sha512-1yrb/+w6HWQJrUCLkJ2IF5jNIPvvFkblV5RNOYl6bV+OA6p9GLcMpHFFGTosSvHvcAUibuUukRqhlYI4z32C7Q==", + "license": "MIT", + "engines": { + "node": ">=16.9.0" + } + }, + "node_modules/http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "license": "MIT", + "dependencies": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + }, + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/iconv-lite": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.2.tgz", + "integrity": "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/ip-address": { + "version": "10.2.0", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.2.0.tgz", + "integrity": "sha512-/+S6j4E9AHvW9SWMSEY9Xfy66O5PWvVEJ08O0y5JGyEKQpojb0K0GKpz/v5HJ/G0vi3D2sjGK78119oXZeE0qA==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/is-interactive": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-interactive/-/is-interactive-2.0.0.tgz", + "integrity": "sha512-qP1vozQRI+BMOPcjFzrjXuQvdak2pHNUMZoeG2eRbiSqyvbEf/wQtEOTOX1guk6E3t36RkaqiSt8A/6YElNxLQ==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-promise": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", + "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", + "license": "MIT" + }, + "node_modules/is-unicode-supported": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-2.1.0.tgz", + "integrity": "sha512-mE00Gnza5EEB3Ds0HfMyllZzbBrmLOX3vfWoj9A9PEnTfratQ/BcaJOuMhnkhjXvb2+FkY3VuHqtAGpTPmglFQ==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "license": "ISC" + }, + "node_modules/jose": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/jose/-/jose-6.2.3.tgz", + "integrity": "sha512-YYVDInQKFJfR/xa3ojUTl8c2KoTwiL1R5Wg9YCydwH0x0B9grbzlg5HC7mMjCtUJjbQ/YnGEZIhI5tCgfTb4Hw==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/panva" + } + }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "license": "MIT" + }, + "node_modules/json-schema-typed": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/json-schema-typed/-/json-schema-typed-8.0.2.tgz", + "integrity": "sha512-fQhoXdcvc3V28x7C7BMs4P5+kNlgUURe2jmUT1T//oBRMDrqy1QPelJimwZGo7Hg9VPV3EQV5Bnq4hbFy2vetA==", + "license": "BSD-2-Clause" + }, + "node_modules/jsonc-parser": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-3.3.1.tgz", + "integrity": "sha512-HUgH65KyejrUFPvHFPbqOY0rsFip3Bo5wb4ngvdi1EpCYWUQDC5V+Y7mZws+DLkr4M//zQJoanu1SP+87Dv1oQ==", + "license": "MIT" + }, + "node_modules/log-symbols": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-7.0.1.tgz", + "integrity": "sha512-ja1E3yCr9i/0hmBVaM0bfwDjnGy8I/s6PP4DFp+yP+a+mrHO4Rm7DtmnqROTUkHIkqffC84YY7AeqX6oFk0WFg==", + "license": "MIT", + "dependencies": { + "is-unicode-supported": "^2.0.0", + "yoctocolors": "^2.1.1" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/mcporter": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/mcporter/-/mcporter-0.7.3.tgz", + "integrity": "sha512-egoPVYqTnWb3NjRIxo+xc8OrAI0dlPrJm9pAiZx0pImuNIV5rKhGtTnIfH/Y1ldGPVu74ibj3KR5c9U/QSdQFA==", + "license": "MIT", + "dependencies": { + "@iarna/toml": "^2.2.5", + "@modelcontextprotocol/sdk": "^1.25.1", + "acorn": "^8.15.0", + "commander": "^14.0.2", + "es-toolkit": "^1.43.0", + "jsonc-parser": "^3.3.1", + "ora": "^9.0.0", + "rolldown": "1.0.0-beta.57", + "zod": "^4.2.1" + }, + "bin": { + "mcporter": "dist/cli.js" + }, + "engines": { + "node": ">=20.11.0" + } + }, + "node_modules/media-typer": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.0.tgz", + "integrity": "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/merge-descriptors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz", + "integrity": "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/mimic-function": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/mimic-function/-/mimic-function-5.0.1.tgz", + "integrity": "sha512-VP79XUPxV2CigYP3jWwAUFSku2aKqBH7uTAapFWCBqutsbmDo96KY5o8uh6U+/YSIn5OxJnXp73beVkpqMIGhA==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/negotiator": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.0.0.tgz", + "integrity": "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-inspect": { + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", + "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "license": "MIT", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/onetime": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-7.0.0.tgz", + "integrity": "sha512-VXJjc87FScF88uafS3JllDgvAm+c/Slfz06lorj2uAY34rlUu0Nt+v8wreiImcrgAjjIHp1rXpTDlLOGw29WwQ==", + "license": "MIT", + "dependencies": { + "mimic-function": "^5.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ora": { + "version": "9.4.1", + "resolved": "https://registry.npmjs.org/ora/-/ora-9.4.1.tgz", + "integrity": "sha512-6VlU9MLXbjVQD04AZCMX28hVtA5bUoadvUqO76MUCVA0ilwJbMiHsITRPfyVm6p/BC0Av/BXMujx39WCe1LEqw==", + "license": "MIT", + "dependencies": { + "chalk": "^5.6.2", + "cli-cursor": "^5.0.0", + "cli-spinners": "^3.2.0", + "is-interactive": "^2.0.0", + "is-unicode-supported": "^2.1.0", + "log-symbols": "^7.0.1", + "stdin-discarder": "^0.3.2", + "string-width": "^8.1.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-to-regexp": { + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.4.2.tgz", + "integrity": "sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/pkce-challenge": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/pkce-challenge/-/pkce-challenge-5.0.1.tgz", + "integrity": "sha512-wQ0b/W4Fr01qtpHlqSqspcj3EhBvimsdh0KlHhH8HRZnMsEa0ea2fTULOXOS9ccQr3om+GcGRk4e+isrZWV8qQ==", + "license": "MIT", + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "license": "MIT", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/qs": { + "version": "6.15.3", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.3.tgz", + "integrity": "sha512-O9gl3zCl5h5blw1KGUzQKhA5oUXSl8rwUIM5o0S3nCXMliSvy5Dzx7/DJcI+SwgICv+IneSZwhBh1oSyEHA71A==", + "license": "BSD-3-Clause", + "dependencies": { + "es-define-property": "^1.0.1", + "side-channel": "^1.1.1" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/range-parser": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.3.0.tgz", + "integrity": "sha512-hek2mFQpPuI4E1BBKrSto+BU3e3x4xuarsbiwr3+lf7p44juvFMV0XFWQAP3xUyqXA4RrXLIoaSUGbSt056ZMw==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/raw-body": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz", + "integrity": "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==", + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.7.0", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/restore-cursor": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-5.1.0.tgz", + "integrity": "sha512-oMA2dcrw6u0YfxJQXm342bFKX/E4sG9rbTzO9ptUcR/e8A33cHuvStiYOwH7fszkZlZ1z/ta9AAoPk2F4qIOHA==", + "license": "MIT", + "dependencies": { + "onetime": "^7.0.0", + "signal-exit": "^4.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/rolldown": { + "version": "1.0.0-beta.57", + "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.0.0-beta.57.tgz", + "integrity": "sha512-lMMxcNN71GMsSko8RyeTaFoATHkCh4IWU7pYF73ziMYjhHZWfVesC6GQ+iaJCvZmVjvgSks9Ks1aaqEkBd8udg==", + "license": "MIT", + "dependencies": { + "@oxc-project/types": "=0.103.0", + "@rolldown/pluginutils": "1.0.0-beta.57" + }, + "bin": { + "rolldown": "bin/cli.mjs" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "optionalDependencies": { + "@rolldown/binding-android-arm64": "1.0.0-beta.57", + "@rolldown/binding-darwin-arm64": "1.0.0-beta.57", + "@rolldown/binding-darwin-x64": "1.0.0-beta.57", + "@rolldown/binding-freebsd-x64": "1.0.0-beta.57", + "@rolldown/binding-linux-arm-gnueabihf": "1.0.0-beta.57", + "@rolldown/binding-linux-arm64-gnu": "1.0.0-beta.57", + "@rolldown/binding-linux-arm64-musl": "1.0.0-beta.57", + "@rolldown/binding-linux-x64-gnu": "1.0.0-beta.57", + "@rolldown/binding-linux-x64-musl": "1.0.0-beta.57", + "@rolldown/binding-openharmony-arm64": "1.0.0-beta.57", + "@rolldown/binding-wasm32-wasi": "1.0.0-beta.57", + "@rolldown/binding-win32-arm64-msvc": "1.0.0-beta.57", + "@rolldown/binding-win32-x64-msvc": "1.0.0-beta.57" + } + }, + "node_modules/router": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/router/-/router-2.2.0.tgz", + "integrity": "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "depd": "^2.0.0", + "is-promise": "^4.0.0", + "parseurl": "^1.3.3", + "path-to-regexp": "^8.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, + "node_modules/send": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/send/-/send-1.2.1.tgz", + "integrity": "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.3", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "fresh": "^2.0.0", + "http-errors": "^2.0.1", + "mime-types": "^3.0.2", + "ms": "^2.1.3", + "on-finished": "^2.4.1", + "range-parser": "^1.2.1", + "statuses": "^2.0.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/serve-static": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.1.tgz", + "integrity": "sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==", + "license": "MIT", + "dependencies": { + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "parseurl": "^1.3.3", + "send": "^1.2.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", + "license": "ISC" + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/side-channel": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.1.tgz", + "integrity": "sha512-6x6dK6zJdpTzF4sQeNYxwtvBzf6Eg4GtlesS94HOvTudUeyK2WXAaIfmDgsyslYrRBeFIlsi54AYsFGUuhmvrQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4", + "side-channel-list": "^1.0.1", + "side-channel-map": "^1.0.1", + "side-channel-weakmap": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-list": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.1.tgz", + "integrity": "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-map": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", + "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-weakmap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", + "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3", + "side-channel-map": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "license": "ISC", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/stdin-discarder": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/stdin-discarder/-/stdin-discarder-0.3.2.tgz", + "integrity": "sha512-eCPu1qRxPVkl5605OTWF8Wz40b4Mf45NY5LQmVPQ599knfs5QhASUm9GbJ5BDMDOXgrnh0wyEdvzmL//YMlw0A==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/string-width": { + "version": "8.2.1", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-8.2.1.tgz", + "integrity": "sha512-IIaP0g3iy9Cyy18w3M9YcaDudujEAVHKt3a3QJg1+sr/oX96TbaGUubG0hJyCjCBThFH+tFpcIyoUHUn1ogaLA==", + "license": "MIT", + "dependencies": { + "get-east-asian-width": "^1.5.0", + "strip-ansi": "^7.1.2" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/strip-ansi": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.2.0.tgz", + "integrity": "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^6.2.2" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "license": "MIT", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD", + "optional": true + }, + "node_modules/type-is": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.1.0.tgz", + "integrity": "sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA==", + "license": "MIT", + "dependencies": { + "content-type": "^2.0.0", + "media-typer": "^1.1.0", + "mime-types": "^3.0.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/type-is/node_modules/content-type": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.0.0.tgz", + "integrity": "sha512-j/O/d7GcZCyNl7/hwZAb606rzqkyvaDctLmckbxLzHvFBzTJHuGEdodATcP3yIRoDrLHkIATJuvzbFlp/ki2cQ==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" + }, + "node_modules/yoctocolors": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/yoctocolors/-/yoctocolors-2.1.2.tgz", + "integrity": "sha512-CzhO+pFNo8ajLM2d2IW/R93ipy99LWjtwblvC1RsoSUMZgyLbYFr221TnSNT7GjGdYui6P459mw9JH/g/zW2ug==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/zod": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.4.3.tgz", + "integrity": "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/zod-to-json-schema": { + "version": "3.25.2", + "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.25.2.tgz", + "integrity": "sha512-O/PgfnpT1xKSDeQYSCfRI5Gy3hPf91mKVDuYLUHZJMiDFptvP41MSnWofm8dnCm0256ZNfZIM7DSzuSMAFnjHA==", + "license": "ISC", + "peerDependencies": { + "zod": "^3.25.28 || ^4" + } + } + } +} diff --git a/agents/openclaw/mcporter-runtime/package.json b/agents/openclaw/mcporter-runtime/package.json new file mode 100644 index 00000000000..b24e8fb7ba2 --- /dev/null +++ b/agents/openclaw/mcporter-runtime/package.json @@ -0,0 +1,14 @@ +{ + "$comment": "SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\nSPDX-License-Identifier: Apache-2.0", + "name": "nemoclaw-mcporter-runtime", + "version": "0.0.0", + "private": true, + "description": "Locked production dependency graph for the in-sandbox mcporter runtime", + "license": "Apache-2.0", + "dependencies": { + "mcporter": "0.7.3" + }, + "engines": { + "node": ">=22.16.0" + } +} diff --git a/ci/platform-matrix.json b/ci/platform-matrix.json index a3ab579293f..67f47445ee6 100644 --- a/ci/platform-matrix.json +++ b/ci/platform-matrix.json @@ -31,7 +31,7 @@ "status": "tested", "prd_priority": "P0", "ci_tested": true, - "notes": "Primary tested path. Ubuntu 24.04 is the validated distro in production source (`DEFAULT_COMPAT_IMAGE` in `src/lib/onboard/docker-driver-gateway-launch.ts:10` and the preflight tests pin 24.04 only); the installer's package-manager probes assume apt-get. Other distros (Ubuntu 22.04, Fedora, Rocky, Alma, NixOS, Arch) may work but are not validated." + "notes": "Primary tested path. Ubuntu 24.04 is the validated distro in production source (`DEFAULT_COMPAT_IMAGE` in `src/lib/onboard/docker-driver-gateway-compat.ts:11` and the preflight tests pin 24.04 only); the installer's package-manager probes assume apt-get. Other distros (Ubuntu 22.04, Fedora, Rocky, Alma, NixOS, Arch) may work but are not validated." }, { "name": "macOS (Apple Silicon)", @@ -93,7 +93,7 @@ "name": "Other OpenAI-compatible endpoint", "status": "caveated", "endpoint_type": "Custom OpenAI-compatible", - "notes": "Adapter path validated against OpenRouter as the `compatible-endpoint` provider with `openrouter/auto` (see `src/lib/inference/config.test.ts:119`); the onboarding prompt that surfaces OpenRouter as the worked example is at `src/lib/onboard.ts:3661`. Behavior on other OpenAI-compatible proxies, gateways, and self-hosted implementations may vary; this row claims the adapter, not the universe of compatible endpoints." + "notes": "Adapter path validated against OpenRouter as the `compatible-endpoint` provider with `openrouter/auto` (see `src/lib/inference/config.test.ts:119`); the onboarding prompt that surfaces OpenRouter as the worked example is at `src/lib/onboard.ts:3585`. Behavior on other OpenAI-compatible proxies, gateways, and self-hosted implementations may vary; this row claims the adapter, not the universe of compatible endpoints." }, { "name": "Anthropic", @@ -129,7 +129,7 @@ "name": "Local NVIDIA NIM", "status": "experimental", "endpoint_type": "Local OpenAI-compatible", - "notes": "Requires `NEMOCLAW_EXPERIMENTAL=1` and a NIM-capable NVIDIA GPU. Host must have the NVIDIA Container Toolkit installed and a CDI spec present (`onboard` asserts CDI presence with `assertCdiNvidiaGpuSpecPresent`, `src/lib/onboard.ts:1632`). NIM images pull from `nvcr.io` and require NGC registry login. NemoClaw gates this path behind the experimental flag because it does not auto-select a NIM image for the host today. You must explicitly pick from the validated image list. Managed vLLM has host-specific default models and is not gated on the same boxes. Validated images referenced in `src/lib/inference/config.ts` and `nemoclaw/src/index.ts`: `nvidia/nemotron-3-super-120b-a12b` (default cloud model), `nvidia/nemotron-3-nano-30b-a3b`, `nvidia/llama-3.3-nemotron-super-49b-v1.5`." + "notes": "Requires `NEMOCLAW_EXPERIMENTAL=1` and a NIM-capable NVIDIA GPU. Host must have the NVIDIA Container Toolkit installed and a CDI spec present (`onboard` asserts CDI presence with `assertCdiNvidiaGpuSpecPresent`, `src/lib/onboard/fatal-runtime-preflight.ts:78`). NIM images pull from `nvcr.io` and require NGC registry login. NemoClaw gates this path behind the experimental flag because it does not auto-select a NIM image for the host today. You must explicitly pick from the validated image list. Managed vLLM has host-specific default models and is not gated on the same boxes. Validated images referenced in `src/lib/inference/config.ts` and `nemoclaw/src/index.ts`: `nvidia/nemotron-3-super-120b-a12b` (default cloud model), `nvidia/nemotron-3-nano-30b-a3b`, `nvidia/llama-3.3-nemotron-super-49b-v1.5`." }, { "name": "Local vLLM (already running)", @@ -218,12 +218,12 @@ { "name": "Podman / other container runtimes", "status": "unsupported", - "notes": "Onboard surfaces an explicit unsupported-runtime error for Podman (`src/lib/onboard.ts:1599` prints the rejection; `src/lib/onboard/preflight.ts:586` flags the unsupported runtime upstream). Only Docker Engine, Docker Desktop, and Colima are supported. See issue #420 (closed)." + "notes": "Onboard surfaces an explicit unsupported-runtime error for Podman (`src/lib/onboard/fatal-runtime-preflight.ts:50` prints the rejection; `src/lib/onboard/preflight.ts:676` flags the unsupported runtime upstream). Only Docker Engine, Docker Desktop, and Colima are supported. See issue #420 (closed)." }, { "name": "Intel Mac (macOS x86_64)", "status": "unsupported", - "notes": "OpenShell does not publish macOS x86_64 standalone gateway assets. Install hard-fails on x86_64 macOS (`scripts/install-openshell.sh:315`). See issue #954 (closed)." + "notes": "OpenShell does not publish macOS x86_64 standalone gateway assets. Install hard-fails on x86_64 macOS (`scripts/install-openshell.sh:654`). See issue #954 (closed)." }, { "name": "Non-Ubuntu/Debian Linux distros", @@ -248,7 +248,7 @@ { "name": "Non-NVIDIA GPUs (AMD/ROCm, Intel Arc, Apple Metal)", "status": "unsupported", - "notes": "Local vLLM and NIM paths assert NVIDIA CDI presence with `assertCdiNvidiaGpuSpecPresent` (`src/lib/onboard.ts:1632`). NemoClaw does not install non-NVIDIA accelerator drivers." + "notes": "Local vLLM and NIM paths assert NVIDIA CDI presence with `assertCdiNvidiaGpuSpecPresent` (`src/lib/onboard/fatal-runtime-preflight.ts:78`). NemoClaw does not install non-NVIDIA accelerator drivers." }, { "name": "Other LangChain, AutoGen, CrewAI, or non-listed agent harnesses", diff --git a/ci/test-file-size-budget.json b/ci/test-file-size-budget.json index bb7e003f9fb..8b923c9a9e9 100644 --- a/ci/test-file-size-budget.json +++ b/ci/test-file-size-budget.json @@ -6,8 +6,8 @@ "src/lib/inference/nim.test.ts": 2068, "src/lib/onboard/preflight.test.ts": 1904, "test/channels-add-preset.test.ts": 1871, - "test/generate-openclaw-config.test.ts": 1982, - "test/install-preflight.test.ts": 4005, + "test/generate-openclaw-config.test.ts": 1972, + "test/install-preflight.test.ts": 3934, "test/nemoclaw-start.test.ts": 4841, "test/onboard-messaging.test.ts": 2062, "test/onboard-selection.test.ts": 6867, diff --git a/docs/about/release-notes.mdx b/docs/about/release-notes.mdx index 470f339c968..f590ef2bdb8 100644 --- a/docs/about/release-notes.mdx +++ b/docs/about/release-notes.mdx @@ -23,6 +23,8 @@ NemoClaw v0.0.74 advances to OpenShell `0.0.72` and adopts its safe policy round - Stable installs pin OpenShell `0.0.72` release artifacts and supervisor image, adding MCP Streamable HTTP and JSON-RPC request-policy enforcement. - Policy mutations now read the round-trippable base policy instead of the effective policy, preventing provider-composed `_provider_*` entries from being sent back through `policy set` while preserving existing MCP rules. For more information, refer to [OpenShell 0.0.72 Compatibility Review](../security/openshell-0.0.72-compatibility-review) and [Customize the Network Policy](../network-policy/customize-network-policy). +- Managed MCP commands now add, list, inspect, rotate, restart, and remove authenticated HTTPS Streamable HTTP servers for OpenClaw, Hermes, and LangChain Deep Agents Code through native OpenShell policy enforcement and provider-backed credential replacement. + For more information, refer to [Set Up MCP Servers](../manage-sandboxes/set-up-mcp-servers) and the [accepted architecture decision](https://github.com/NVIDIA/NemoClaw/issues/566#issuecomment-4847534784). ## v0.0.73 diff --git a/docs/deployment/set-up-mcp-bridge.mdx b/docs/deployment/set-up-mcp-bridge.mdx new file mode 100644 index 00000000000..21e6db37d2e --- /dev/null +++ b/docs/deployment/set-up-mcp-bridge.mdx @@ -0,0 +1,289 @@ +--- +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +title: "Set Up MCP Servers" +sidebar-title: "Set Up MCP Servers" +description: "Connect sandboxed agents to authenticated Streamable HTTP MCP servers through OpenShell policy enforcement and credential replacement." +description-agent: "Explains how to add, inspect, rotate, recover, rebuild, and remove authenticated Streamable HTTP MCP servers through native OpenShell policy and credential replacement with no host-side MCP data-plane bridge, proxy, relay, or listener. Use when configuring MCP for OpenClaw, Hermes, or LangChain Deep Agents Code." +keywords: ["nemoclaw mcp", "authenticated mcp", "openshell credential replacement", "streamable http mcp"] +content: + type: "how_to" +skill: + priority: 30 +--- + +NemoClaw lets a sandboxed agent use MCP Streamable HTTP servers without copying external service credentials into the sandbox. + +The integration has three parts: + +- An OpenShell provider stores credentials outside the sandbox. +- A generated OpenShell network policy grants the MCP endpoint through `protocol: mcp` and applies explicit JSON-RPC MCP method rules. +- An agent adapter writes the MCP endpoint into OpenClaw, Hermes, or LangChain Deep Agents Code config. + +This integration depends on the OpenShell MCP/JSON-RPC L7 policy support from [NVIDIA/OpenShell#1865](https://github.com/NVIDIA/OpenShell/pull/1865). +NemoClaw v0.0.74 defaults to the pinned stable OpenShell `0.0.72` release, which exposes native `protocol: mcp` policy handling and provider-backed credential replacement. +The optional OpenShell development channel is compatibility evidence only and is not a shipping target. + +NemoClaw accepts Streamable HTTP MCP endpoints only. +NemoClaw does not launch an MCP server, stdio adapter, bridge, credential proxy, data-plane relay, or listener on the host. +The sandbox agent connects directly to the configured endpoint, and OpenShell enforces policy and replaces credentials in its existing sandbox egress path. +No NemoClaw host process remains running after an `mcp` lifecycle command returns. + +## Architecture Decision + +**Status:** [Accepted on June 30, 2026](https://github.com/NVIDIA/NemoClaw/issues/566#issuecomment-4847534784) as the normative design for NemoClaw v0.0.74 and the implementation that supersedes the original acceptance text in NVIDIA/NemoClaw#566. + +This native OpenShell design supersedes the original host-side stdio-to-HTTP proxy proposed in NVIDIA/NemoClaw#566. +NemoClaw does not accept an inline secret-and-command tuple, persist the raw bearer value supplied through `--env`, or operate a host-side MCP data-plane process. +The only supported managed path is authenticated Streamable HTTP through an OpenShell `protocol: mcp` policy, with the raw credential held by the OpenShell provider and replaced only on an authorized outbound request. +Host-side MCP data-plane bridges, proxies, relays, listeners, and stdio translation are explicitly out of scope for this feature. +The original issue's Claude-style `-e KEY=VALUE -- `, plaintext `http://host.docker.internal:`, stored environment value, and `127.0.0.1` proxy clauses are rejected by this decision rather than deferred implementation work. + +| Decision boundary | Accepted native OpenShell design | Superseded host proxy design | +| --- | --- | --- | +| Credential boundary | OpenShell stores the raw value and resolves a sandbox placeholder only on an authorized request. | A NemoClaw host process would receive and retain the raw value while proxying traffic. | +| Policy enforcement | OpenShell evaluates the destination, path, adapter identity, pinned addresses, and MCP methods before credential replacement. | The proxy would become a second authorization implementation outside OpenShell policy. | +| Data-plane exposure | The sandbox connects through OpenShell's existing egress path; NemoClaw leaves no host listener or MCP traffic process. | A host listener and stdio-to-HTTP relay would expand the data plane and local attack surface. | +| Failure behavior | Provider, policy, and adapter mutations fail closed and preserve retryable registry state when ownership or readiness cannot be proven. | Proxy failure could strand a listener, subprocess, or partially persisted secret-bearing launch state. | +| Crash recovery | Randomized provider ownership records and per-sandbox lifecycle locks let `restart`, `rebuild`, `remove`, and `destroy` reconcile durable state. | Recovery would also have to discover and terminate orphan host processes and reconstruct their secret-bearing invocation state. | + +The accepted design record is tracked in [NVIDIA/NemoClaw#566](https://github.com/NVIDIA/NemoClaw/issues/566), and [NVIDIA/NemoClaw#5876](https://github.com/NVIDIA/NemoClaw/pull/5876) is its implementation. + +## Add an MCP Server + +Use the same workflow for OpenClaw, Hermes, and LangChain Deep Agents Code sandboxes. +NemoClaw selects the agent-specific adapter from the sandbox registry. +Rebuild sandboxes created before this release onto a current image before the first managed MCP change. +Hermes and Deep Agents probe their managed MCP runtime before an active add or restart changes a live provider or policy; OpenClaw verifies the pinned `mcporter` during adapter registration and rolls back an incomplete add. +When recovery finds that a provider was already deleted, NemoClaw may first remove only that dangling sandbox-spec reference because OpenShell cannot start the capability-probe child while a missing provider name remains attached. +That prerequisite does not delete or replace a live provider, credential, or policy, and the durable bridge manifest remains retryable if the later capability probe fails. + +```bash +export GITHUB_MCP_TOKEN=ghp_... +$$nemoclaw my-sandbox mcp add github --url https://api.githubcopilot.com/mcp/ --env GITHUB_MCP_TOKEN +unset GITHUB_MCP_TOKEN +``` + +The assignment above is illustrative. +Load real values from an approved secret manager or a masked prompt so the credential is not recorded in shell history. + +`--env KEY` reads the value from the host process environment and stores it in OpenShell's provider store. +NemoClaw persists only the variable name, writes `openshell:resolve:env:KEY` into the sandbox-side MCP config, and relies on OpenShell to resolve the placeholder at egress. + +Do not reuse OpenShell's Google Cloud compatibility names as MCP bearer keys. +NemoClaw rejects `GCP_PROJECT_ID`, `GOOGLE_CLOUD_PROJECT`, `CLOUD_ML_REGION`, `GCP_LOCATION`, `GCP_SERVICE_ACCOUNT_EMAIL`, `GOOSE_PROVIDER`, `ANTHROPIC_VERTEX_PROJECT_ID`, and `VERTEX_LOCATION` because OpenShell exposes those non-secret configuration names as child-process values. +NemoClaw also rejects `GCE_METADATA_HOST`, `GCE_METADATA_IP`, and `METADATA_SERVER_DETECTION`, which OpenShell rewrites for its metadata emulator. +The child-visible compatibility list is pinned to OpenShell `v0.0.72` commit `8cb16de9eae4c44d7d31e1493747d8c10abb5963` and must be reviewed with every OpenShell version change. +Choose a dedicated name such as `MY_SERVICE_MCP_TOKEN`. +NemoClaw also rejects host subprocess control names such as `PATH`, proxy/TLS variables, and `OPENSHELL_*`, `GRPC_*`, `LC_*`, or `XDG_*` keys so the selected credential cannot be inherited by unrelated OpenShell commands. +Loader, shell, language, and agent runtime controls such as `LD_PRELOAD`, `BASH_ENV`, `NODE_OPTIONS`, `PYTHONHOME`, `NEMOCLAW_*`, and `OPENCLAW_*` are rejected as well because OpenShell attaches provider keys to fresh sandbox execs; use a dedicated service name such as `MY_SERVICE_MCP_TOKEN`. + +NemoClaw requires exactly one `--env` bearer credential per server. +Every endpoint must use HTTPS. +The full URL, including its path, is persisted and displayed, so never put a credential in the URL path. +NemoClaw rejects userinfo, query strings, fragments, and known secret-shaped path material; put the bearer value in `--env KEY`. +Use a distinct environment variable name for each managed MCP server in the same sandbox. +OpenShell static credential keys are sandbox-wide and cannot be attached twice. +Endpoint paths must be literal and canonical, so NemoClaw rejects percent escapes, backslashes, semicolons, OpenShell glob metacharacters, and explicit port zero. +NemoClaw resolves public hostnames before registration, rejects private, local, and special-use targets, and pins the resolved addresses in the generated policy. +OpenShell re-resolves the hostname for each new connection, requires every current answer to match those pinned `allowed_ips`, and connects to the same validated socket addresses. +The pinned implementation is [`NVIDIA/OpenShell@8cb16de9eae4c44d7d31e1493747d8c10abb5963`](https://github.com/NVIDIA/OpenShell/tree/8cb16de9eae4c44d7d31e1493747d8c10abb5963). +In that implementation, [`crates/openshell-supervisor-network/src/proxy.rs:2476-2502`](https://github.com/NVIDIA/OpenShell/blob/8cb16de9eae4c44d7d31e1493747d8c10abb5963/crates/openshell-supervisor-network/src/proxy.rs#L2476-L2502) resolves the socket-address list, [`crates/openshell-supervisor-network/src/proxy.rs:2527-2567`](https://github.com/NVIDIA/OpenShell/blob/8cb16de9eae4c44d7d31e1493747d8c10abb5963/crates/openshell-supervisor-network/src/proxy.rs#L2527-L2567) validates that list, and [`crates/openshell-supervisor-network/src/proxy.rs:2622-2630`](https://github.com/NVIDIA/OpenShell/blob/8cb16de9eae4c44d7d31e1493747d8c10abb5963/crates/openshell-supervisor-network/src/proxy.rs#L2622-L2630) returns it unchanged. +The CONNECT path passes the returned list directly to `TcpStream::connect` at [`crates/openshell-supervisor-network/src/proxy.rs:822-832`](https://github.com/NVIDIA/OpenShell/blob/8cb16de9eae4c44d7d31e1493747d8c10abb5963/crates/openshell-supervisor-network/src/proxy.rs#L822-L832). +The explicit HTTP-forward path carries the same returned list from [`crates/openshell-supervisor-network/src/proxy.rs:3885-3893`](https://github.com/NVIDIA/OpenShell/blob/8cb16de9eae4c44d7d31e1493747d8c10abb5963/crates/openshell-supervisor-network/src/proxy.rs#L3885-L3893) to [`crates/openshell-supervisor-network/src/proxy.rs:4123-4125`](https://github.com/NVIDIA/OpenShell/blob/8cb16de9eae4c44d7d31e1493747d8c10abb5963/crates/openshell-supervisor-network/src/proxy.rs#L4123-L4125). +A DNS change to a private, special-use, or otherwise unpinned address therefore fails closed instead of widening the route. +The stable and development `mcp-bridge` live lanes verify this OpenShell contract before any NemoClaw MCP mutation and independently of all three agent adapters: the OpenClaw scenario applies a raw `protocol: mcp` policy with `allowed_ips: [1.1.1.1]`, remaps the hostname to a reachable private runner address, sends a raw MCP `tools/list` request, requires an exact HTTP 403, verifies that the upstream server recorded zero requests, and restores the exact base policy in `finally`. +`restart` resolves the hostname again before updating that policy. +Authenticated MCP rejects `host.openshell.internal`, `host.docker.internal`, and `host.containers.internal` on stable OpenShell `v0.0.72`. +That release has a trusted-gateway branch for one narrow link-local topology, but it does not expose an attested driver gateway address that NemoClaw can pin; non-link-local driver aliases otherwise fall back to mutable exact-host resolution. +Host-alias support is deferred until OpenShell exposes attested gateway state for exact policy pinning. +Use a normal HTTPS DNS endpoint with public address records in the meantime. + +## Authenticated MCP Security Boundary + +Authenticated MCP is the intended configuration. +The agent stores only the `openshell:resolve:env:KEY` placeholder. +OpenShell keeps the raw credential in its provider store and combines credential replacement with the generated MCP policy at egress. + +For the normal MCP client path, OpenShell evaluates the effective network policy for the destination host and port, adapter binary, literal endpoint path, and MCP method before it replaces placeholders in the allowed HTTP request headers. +The generated MCP policy grants only the configured destination, path, adapter binaries, pinned addresses, explicit MCP method profile, and a 131,072-byte maximum request body. +NemoClaw accepts only canonical HTTPS MCP URLs and writes the credential placeholder only into the `Authorization` header. + +### Stable OpenShell 0.0.72 Limitations + +OpenShell v0.0.72 attributes network policy with `/proc//exe` and process ancestors, so the script-based adapters require Node or Python interpreter grants rather than immutable package-entrypoint identities. +NemoClaw compensates with an exact HTTPS destination, path, MCP method profile, and DNS pins, plus a unique least-privilege credential for each server. +Remove the interpreter grants when OpenShell exposes stable script or package entrypoint attribution. + +OpenShell v0.0.72 attaches static provider credentials at sandbox scope rather than reserving a credential key exclusively for one endpoint. +It also does not expose an immutable provider binding on an attachment, provide a `tls: require` policy mode, bind the HTTP `Host` header to the policy destination, or include query parameters in MCP path matching. +NemoClaw rejects credential-key reuse between managed MCP servers, creates a dedicated provider for each definition, reports the residual risk in `status`, and requires a unique least-privilege token and environment key. +Operators must avoid granting a broader inspected-HTTP route to the same adapter runtime because such a route could resolve the sandbox-scoped placeholder. +The generated agent configuration uses the canonical HTTPS URL, but malicious code running as an allowed interpreter can deliberately change the scheme, `Host` header, or query string within the supported OpenShell policy contract. + +OpenShell v0.0.72 updates, attaches, detaches, and deletes providers by mutable name rather than an atomic immutable identity. +NemoClaw compensates with randomized provider names, the per-sandbox lifecycle lock, and immediate ownership checks against the recorded provider ID, type, and credential-key metadata before mutations. +NemoClaw fails closed and preserves retryable state when those checks do not match, but the checks do not provide compare-and-swap behavior against another OpenShell client. +Do not concurrently replace or mutate a managed provider through another OpenShell client while an MCP lifecycle command is running. + +Use an MCP service you trust with the credential it receives. +MCP response bodies and SSE streams return through OpenShell's existing sandbox egress path. +As with any authenticated API, a server that possesses a credential can deliberately return that value in its response. +This does not expose the raw credential to the sandbox before the request is authorized and sent to that server. + +## Agent Adapters + +OpenClaw uses `mcporter config add` in the sandbox. + +Hermes writes this managed HTTP entry under `/sandbox/.hermes/config.yaml`: + +```yaml +mcp_servers: + github: + url: https://api.githubcopilot.com/mcp/ + enabled: true + timeout: 120 + connect_timeout: 60 + tools: + resources: true + prompts: true + headers: + Authorization: Bearer openshell:resolve:env:GITHUB_MCP_TOKEN +``` + +Hermes config changes and gateway reloads stay inside the sandbox. +When Hermes shields are up, run `nemohermes shields down --timeout 15m --reason "MCP maintenance"` before `mcp add`, `mcp restart`, `mcp remove`, or destroy, then run `nemohermes shields up` after the change if the sandbox still exists. +Choose a window long enough for the command to finish, allowing at least 15 minutes per configured server for an all-server restart or destroy; rebuild opens and owns its separate 30-minute crash-recoverable window automatically. +NemoClaw checks the host shields posture before MCP provider, policy, attachment, or adapter mutation, and the in-sandbox helper checks the config file again at commit time. +If shields are raised concurrently between those checks, the command fails instead of writing locked config, but an earlier policy or provider stage may already have completed; lower shields again and retry the durable MCP transaction to converge it. +`mcp list` and `mcp status` are read-only and do not require lowering shields. +NemoClaw invokes the validated transaction helper as a one-shot ordinary `openshell sandbox exec --no-tty` command with a fixed executable path and argument shape. +The helper runs as the normal sandbox identity, rejects the legacy root-separated runtime topology, validates the gateway PID and launcher before signaling it, updates the managed compatibility hash, verifies loopback health after reload, and rolls back the config and hashes if reload fails. +Within the existing five-minute reload deadline, if the first signal has not converged after half the budget, the helper may send one additional `SIGUSR1` only after revalidating the current gateway identity and its managed parent. +Success still requires a replacement gateway identity, healthy loopback endpoints on internal port `18642` and public port `8642`, and a stable final identity; timeout diagnostics preserve the furthest safely observed phase, and rollback behavior is unchanged. +There is no host listener, persistent control socket, MCP relay, or service for this operation. +The command carries no MCP traffic or raw service credential, and its payload contains only the endpoint definition and OpenShell placeholder. + +LangChain Deep Agents Code writes an HTTP entry under its user-level discovery path, `/sandbox/.deepagents/.mcp.json`. +Deep Agents Code `0.1.12` treats the sandbox-root `.mcp.json` as project configuration and gates it on project trust, so NemoClaw does not use that path for managed MCP definitions. + +```json +{ + "mcpServers": { + "github": { + "type": "http", + "url": "https://api.githubcopilot.com/mcp/", + "headers": { + "Authorization": "Bearer openshell:resolve:env:GITHUB_MCP_TOKEN" + } + } + } +} +``` + +External service credential values such as the value of `GITHUB_MCP_TOKEN` remain in OpenShell provider state, not in sandbox files or NemoClaw's sandbox registry. + +## Operate MCP Servers + +```bash +$$nemoclaw my-sandbox mcp list +$$nemoclaw my-sandbox mcp status github --json +$$nemoclaw my-sandbox mcp restart github +$$nemoclaw my-sandbox mcp remove github +``` + +`list --json` and `status --json` never include environment values. +They report provider presence, provider attachment, whether the live generated policy content still matches the registered policy, environment readiness, and adapter registration state. +The per-server `warnings` array reports the current sandbox-scoped provider risk while the managed provider is attached and states the OpenShell enforcement capabilities required to remove that warning. +The `env.missing` field is an array of recorded host variable names that are currently unset; an empty array means every recorded name is exported. +An existing valid provider can remain ready when that host variable is unset because OpenShell retains the credential. +The JSON value `support.mode: "bridge"` identifies the agent's config-adapter capability, not a host-side traffic bridge. + +### Rotate a Credential + +Export the replacement value under the same host environment name used by `mcp add`, then restart that managed server: + +```bash +export GITHUB_MCP_TOKEN='replacement-value' +$$nemoclaw my-sandbox mcp restart github +unset GITHUB_MCP_TOKEN +``` + +`restart` requires a successful OpenShell provider update, waits until the sandbox has received a new opaque provider revision, reapplies the generated policy, and refreshes the agent adapter. +An ambiguous or failed update is never treated as successful merely because another writer advanced the provider revision. +The raw value is passed only to the OpenShell provider command through its process environment, and it is not added to argv, NemoClaw state, or sandbox configuration. +Revoke the old credential upstream after the command succeeds. +If the provider was deleted, the same command recreates it from the exported value. + +When the host variable is not exported, `restart` reuses an existing provider whose current ID, type, and credential-key metadata match the registry without reading its credential. +If the provider is missing, export the recorded variable before retrying. +Running `restart` without a server name refreshes every managed MCP server. +Export only the variables whose credentials you intend to replace. + +### Rebuild and Destroy + +`rebuild` preserves each provider that matches the recorded ID, type, and credential-key metadata at inspection time. +It removes the agent adapter entry and detaches the provider before replacing the sandbox. +It then reattaches the provider, waits for credential readiness, reapplies the generated policy, and restores the adapter. +Removing the old adapter entry does not require the current Deep Agents launcher marker, so an MCP entry created by a compatible older image cannot block its own removal or upgrade. +The replacement image must expose the exact managed launcher marker before NemoClaw reattaches any provider or reapplies policy and reports rebuild restoration as successful. +If sandbox replacement fails, NemoClaw attempts to restore the previous attachment and adapter state. +A rollback targets the same old image and restores its previously compatible entry without imposing the new-image marker. +A later `mcp restart` can retry an incomplete post-rebuild restore. + +`destroy` removes the adapter entry and detaches providers that match the recorded metadata before asking OpenShell to delete the sandbox. +Like remove and rebuild teardown, this scrub does not require the new Deep Agents launcher marker from an older image. +If deletion is refused, NemoClaw attempts to restore the previous MCP state, reports any rollback failure, and preserves recovery state. +Provider deletion and registry cleanup happen only after OpenShell confirms that the sandbox is gone. +NemoClaw prechecks the recorded provider ID and credential-key shape before mutation and uses a random per-add provider-name suffix to avoid accidental name reuse. +The stable OpenShell limitations section describes why these ownership checks do not form an atomic identity binding. + +`remove --force` may remove a modified same-name agent adapter entry so an operator can clear local config. +Provider deletion still requires the exact recorded provider ID, type, and credential key, and policy deletion still requires exact owned content; force never claims an unowned or drifted provider or same-key live policy. +If any cleanup step leaves a residual, the command exits nonzero and preserves the managed MCP registry entry so cleanup can be retried. +It never detaches the provider from other sandboxes, so a residual provider may require manual cleanup. + +Removing a server blocks new requests and reconnects, but it does not terminate an MCP response or SSE stream that was already open. +For immediate revocation, revoke the upstream credential first, then run `$$nemoclaw rebuild --yes` or destroy the sandbox to terminate an already-open response or SSE stream. + +## Troubleshooting + +If `restart` reports a missing provider and the original credential is not registered in OpenShell, export the same variable name used during `add` and retry. + +If `status` reports an incomplete add transaction, rerun the original `mcp add` command with the same URL and environment-variable name. +Re-export the value if the provider still needs to be created. +To abandon the transaction, run `mcp remove --force`. +NemoClaw cleans only resources whose ownership it can prove and keeps the registry entry when residual cleanup remains. + +If MCP add or restart reports that `mcporter`, the Hermes transaction helper, or the managed Deep Agents MCP-aware launcher is unavailable, rebuild the sandbox onto a current image before retrying. +An existing Deep Agents MCP entry remains removable, destroyable, and eligible for rebuild teardown on an older image; the rebuilt image must pass the launcher probe before its MCP runtime is restored. + +If the generated policy or provider has drifted, `restart` fails closed instead of overwriting same-name state. +Resolve the reported OpenShell ownership or content mismatch, then retry. +`remove --force` can continue cleaning other independently owned resources, but it does not claim or delete the drifted resource. + +Registry entries created by an earlier preview branch with an OpenShell host-alias URL or a credential name that is now reserved remain visible so they can be removed safely, but `status` reports the unsupported boundary and `restart` and `rebuild` fail closed. +Remove the legacy entry before rebuilding or destroying the sandbox, then add a normal public HTTPS DNS endpoint with a dedicated service credential name. + +If NemoClaw reports that MCP policy capability is unavailable, install the required OpenShell build and rerun onboarding. +NemoClaw checks inspectable installed OpenShell artifacts for the `protocol: mcp` capability and does not enable managed MCP from a version number alone. +For image-backed or compressed supervisors without an inspectable host runtime artifact, that onboarding check is provisional. +Before any credential or provider side effect, the MCP command loads the exact generated policy with `policy set --wait` and exact-matches the effective state; a runtime that rejects `protocol: mcp` therefore fails closed. + +If a Hermes sandbox is alive but its gateway is not running after a supervisor or container restart, run `$$nemoclaw recover` before retrying the MCP command. +Recovery re-establishes the managed Hermes service lifecycle, API forwarding, and the exit-75 reload loop used for transactional MCP configuration changes. + +If a mutating MCP command times out waiting for the per-sandbox lifecycle lock, first confirm that no `mcp add`, `mcp restart`, `mcp remove`, `rebuild`, or `destroy` command for that sandbox is still running, then retry the original command. +Every mutating command automatically recovers a lock whose local process is provably dead or whose PID now has a different process-start identity, including locks left while stale-lock cleanup was in progress. +NemoClaw deliberately does not expose a force-unlock flag: a live owner, a different host or PID namespace, or an incomplete legacy owner record fails closed because removing it could overlap a provider, policy, or adapter mutation. +For a state directory shared across hosts or PID namespaces, resolve the owner on that host or stop sharing the state directory before retrying; do not delete the lock file while ownership is ambiguous. + +Stdio-only MCP servers are not supported. +NemoClaw does not start, wrap, or translate them, so configure a native Streamable HTTP MCP endpoint. + +The generated policy permits this explicit MCP client-to-server profile: `initialize`, `notifications/initialized`, `ping`, `tools/list`, `tools/call`, `resources/list`, `resources/read`, `resources/templates/list`, `resources/subscribe`, `resources/unsubscribe`, `prompts/list`, `prompts/get`, `tasks/list`, `tasks/get`, `tasks/update`, `tasks/result`, `tasks/cancel`, `completion/complete`, `logging/setLevel`, `server/discover`, `messages/listen`, `notifications/cancelled`, `notifications/progress`, `notifications/roots/list_changed`, and `notifications/elicitation/complete`. +Those methods remain bounded to the configured endpoint path, selected agent adapter binaries, pinned addresses, and a 131,072-byte request body. +`tools/call` currently permits every tool exposed by that server. +`strict_tool_names` validates tool name syntax and is not a tool authorization allowlist. +OpenShell also handles the protocol-required empty receive-stream `GET` and client response frames for server-originated MCP requests. +Those frames are transport behavior rather than additional client-initiated method grants. diff --git a/docs/get-started/prerequisites.mdx b/docs/get-started/prerequisites.mdx index 9ed00429729..a6b7408d38f 100644 --- a/docs/get-started/prerequisites.mdx +++ b/docs/get-started/prerequisites.mdx @@ -87,7 +87,7 @@ The table comes from [`ci/platform-matrix.json`](https://github.com/NVIDIA/NemoC {/* platform-matrix:begin */} | OS | Container runtime | Status | Notes | |----|-------------------|--------|-------| -| Linux | Docker | Tested | Primary tested path. Ubuntu 24.04 is the validated distro in production source (`DEFAULT_COMPAT_IMAGE` in `src/lib/onboard/docker-driver-gateway-launch.ts:10` and the preflight tests pin 24.04 only); the installer's package-manager probes assume apt-get. Other distros (Ubuntu 22.04, Fedora, Rocky, Alma, NixOS, Arch) may work but are not validated. | +| Linux | Docker | Tested | Primary tested path. Ubuntu 24.04 is the validated distro in production source (`DEFAULT_COMPAT_IMAGE` in `src/lib/onboard/docker-driver-gateway-compat.ts:11` and the preflight tests pin 24.04 only); the installer's package-manager probes assume apt-get. Other distros (Ubuntu 22.04, Fedora, Rocky, Alma, NixOS, Arch) may work but are not validated. | | macOS (Apple Silicon) | Colima, Docker Desktop | Tested with limitations | Start the container runtime (Colima or Docker Desktop) before running the installer. Homebrew Colima users must install both Colima and the Docker CLI (`brew install colima docker`) before `docker info` can work. Xcode Command Line Tools (`xcode-select --install`) are typically required for Node native modules during install. NemoClaw recommends them but does not enforce them during preflight. | | DGX Spark | Docker | Tested | Use the standard installer and `$$nemoclaw onboard`. For an end-to-end walkthrough with local inference, see the [NVIDIA Spark playbook](https://build.nvidia.com/spark/nemoclaw). | | Windows WSL2 | Docker Desktop (WSL backend) | Tested with limitations | Requires WSL2 with Docker Desktop backend. | diff --git a/docs/get-started/quickstart-langchain-deepagents-code.mdx b/docs/get-started/quickstart-langchain-deepagents-code.mdx index 85fcd9d4bfc..f9b2ac0f460 100644 --- a/docs/get-started/quickstart-langchain-deepagents-code.mdx +++ b/docs/get-started/quickstart-langchain-deepagents-code.mdx @@ -64,18 +64,21 @@ dcode -n "Summarize this repository" ``` The managed `dcode`, `dcode.real`, and `deepagents-code` launchers use `/opt/venv/bin/python3 -I` to run the pinned package with an isolated import path and `HOME=/sandbox`. -They disable Deep Agents Code package update checks and the LangGraph server version check, block CLI and TUI update/install commands, and disable nested remote sandbox providers, remote async subagents, MCP commands and auto-loading, startup commands, executable hooks, ACP mode, interpreter tool calling, shell allow-list overrides, LangSmith tracing, and OpenTelemetry export. +They disable Deep Agents Code package update checks and the LangGraph server version check, block CLI and TUI update/install commands, and disable nested remote sandbox providers, remote async subagents, MCP commands and project auto-loading, startup commands, executable hooks, ACP mode, interpreter tool calling, shell allow-list overrides, LangSmith tracing, and OpenTelemetry export. The managed model constructor accepts only Deep Agents Code's `openai` provider path and reads its endpoint from a root-owned image file. It supplies the non-secret gateway placeholder key and ignores mutable provider classes, credentials, endpoints, and constructor parameters in Deep Agents Code config. CLI and TUI model parameter overrides and custom rubric models are blocked. Project and user-defined subagents remain available, but they inherit the managed chat model instead of accepting their own model override. +MCP servers registered through `nemoclaw mcp add` remain available through the single managed user-level config and OpenShell egress policy; arbitrary project and user MCP configuration remains blocked. +Before launch, NemoClaw validates the complete managed file as HTTPS-only definitions with exact OpenShell credential placeholders; stdio commands, extra headers, raw credentials, and unrelated top-level configuration fail closed. +For authenticated MCP setup and credential rotation, see [Set Up MCP Servers](../manage-sandboxes/set-up-mcp-servers). This isolated-mode guarantee applies to those managed launchers, not arbitrary Python commands in the sandbox. Interactive shell execution and other destructive tools remain behind human-in-the-loop approval prompts. Thread-wide auto-approval and shell allow-list auto-approval are disabled. Headless `dcode -n` is an explicit automation boundary. It has no approval UI and automatically approves non-shell tool requests, including file writes and edits. -The managed headless path still disables shell execution, startup commands, interpreter tool calling, executable hooks, MCP, nested remote sandboxes, remote async subagents, and alternate model routes. +The managed headless path still disables shell execution, startup commands, interpreter tool calling, executable hooks, unmanaged MCP configuration, nested remote sandboxes, remote async subagents, and alternate model routes. Use the interactive TUI when you need to inspect each destructive tool request before it runs. To confirm which sandbox a session is in, run the identity command: @@ -102,7 +105,8 @@ Deep Agents Code state lives under `/sandbox/.deepagents`. NemoClaw snapshot and rebuild flows preserve the app state directory, skills, and generated config when those files exist. Run `nemoclaw snapshot create` after active `dcode` tasks finish. For `langchain-deepagents-code` sandboxes, NemoClaw refuses backup when it detects an active `dcode` task or cannot verify that the state tree is idle. -NemoClaw intentionally does not preserve `.env` or `.mcp.json` because users may put Tavily, LangSmith, MCP service, or provider credentials there, and this managed harness disables both Deep Agents Code dotenv loading and MCP at runtime. +NemoClaw intentionally does not back up `.deepagents/.env` or user-authored portions of `.deepagents/.mcp.json` because users may put Tavily, LangSmith, MCP service, or provider credentials there. +NemoClaw restores its managed MCP definitions separately from the credential-free registry; service credentials remain in OpenShell provider state. It also does not preserve `hooks.json`; executable Deep Agents Code hooks are disabled in the managed harness. If `.deepagents/.state/auth.json` contains upstream credentials, or `.deepagents/.state/chatgpt-auth.json` exists, the managed Deep Agents Code launch paths refuse to start until that credential state is removed. Before a managed Deep Agents Code rebuild changes the sandbox, NemoClaw selects its recorded OpenShell gateway, tests the recorded inference route through `https://inference.local`, successfully builds the replacement from a pinned base and fingerprinted context, and revalidates the target and route. @@ -120,7 +124,8 @@ Use NemoClaw-managed credential paths when support is available instead of stori NemoClaw does not enable Tavily or LangSmith by default for this harness. The sandbox policy denies `api.tavily.com` and `api.smith.langchain.com` until you opt in. -To enable Tavily, apply the maintained `tavily` policy preset so the sandbox may reach the Tavily API, register the credential with the OpenShell gateway, then rebuild the sandbox so the new provider attaches. +To enable Tavily for the target sandbox, apply the maintained `tavily` policy preset, register the credential with the OpenShell gateway, then rebuild the sandbox so the new provider attaches. +The policy preset is a per-sandbox managed-Python opt-in, but provider registration is gateway-wide: `tavily-search` attaches to every sandbox that you build or rebuild afterward. ```bash # Preview the endpoints the preset opens: @@ -131,21 +136,28 @@ nemo-deepagents policy-add tavily --yes export TAVILY_API_KEY=tvly-... # Register the provider with the gateway: nemo-deepagents credentials add tavily-search --type tavily --credential TAVILY_API_KEY +# Remove the raw key from the host shell after the gateway stores it: +unset TAVILY_API_KEY # Attach the new provider to the sandbox: nemo-deepagents rebuild ``` The shared `tavily` preset only opens `POST /search` and `POST /extract` egress to `api.tavily.com:443`. -Keep `TAVILY_API_KEY` in the host shell only; the gateway injects it at egress, and the sandbox never sees the raw value. +Attaching the credential provider alone does not authorize the managed Python interpreter; the explicit policy preset is the interpreter-level opt-in. +Export `TAVILY_API_KEY` only for registration, then remove it from the host shell; the gateway injects the stored value at egress, and the sandbox never sees the raw value. NemoClaw does not bake `TAVILY_API_KEY` into the managed config or image, and the managed wrapper rejects direct service-key injection into `dcode`. Because OpenShell attributes the harness's calls to the sandbox `python3` interpreter, this egress is process-wide for sandbox Python rather than a `dcode`-only boundary. -Remove the access again when it is no longer needed. +Remove the target sandbox's managed-Python opt-in when it is no longer needed. ```bash nemo-deepagents policy-remove tavily --yes ``` +This does not unregister the gateway-wide `tavily-search` provider; its credential and Node/curl routes remain available to sandboxes that attach it. +When no sandbox needs the provider, destroy those sandboxes or detach it from each one with `openshell sandbox provider detach tavily-search`, then remove it globally with `nemo-deepagents credentials reset tavily-search --yes`. +OpenShell rejects provider deletion while any sandbox still has it attached. + ### Tracing (LangSmith and OpenTelemetry) NemoClaw does not support LangSmith or OpenTelemetry tracing for this managed harness. diff --git a/docs/get-started/quickstart.mdx b/docs/get-started/quickstart.mdx index a94215ee392..39c83e1d62c 100644 --- a/docs/get-started/quickstart.mdx +++ b/docs/get-started/quickstart.mdx @@ -320,14 +320,7 @@ Treat the authenticated URL like a password. ### Chat with the Agent from the Terminal -Use a two-terminal workflow for prompts that may need network access. -In one terminal, connect to the sandbox and use the OpenClaw CLI. -Open a second host terminal and run `openshell term` to watch for blocked network egress requests and approve or deny them while the agent runs. -For remote sandboxes and detailed approval controls, refer to [Approve or Deny Agent Network Requests](../network-policy/approve-network-requests). - -```bash -openshell term -``` +Connect to the sandbox and use the OpenClaw CLI. ```bash nemoclaw my-gpt-claw connect diff --git a/docs/index.yml b/docs/index.yml index f633b841c59..8eea7f133f0 100644 --- a/docs/index.yml +++ b/docs/index.yml @@ -88,6 +88,9 @@ navigation: - page: "Set Up Messaging Channels" path: _build/agent-variants/manage-sandboxes/messaging-channels.openclaw.generated.mdx slug: messaging-channels + - page: "Set Up MCP Servers" + path: _build/agent-variants/deployment/set-up-mcp-bridge.openclaw.generated.mdx + slug: set-up-mcp-servers - page: "Workspace Files" path: _build/agent-variants/manage-sandboxes/workspace-files.openclaw.generated.mdx slug: workspace-files @@ -140,12 +143,12 @@ navigation: - page: "Credential Storage" path: _build/agent-variants/security/credential-storage.openclaw.generated.mdx slug: credential-storage - - page: "OpenShell 0.0.72 Compatibility Review" - path: _build/agent-variants/security/openshell-0.0.72-compatibility-review.openclaw.generated.mdx - slug: openshell-0.0.72-compatibility-review - page: "Trusted Computing Base" path: _build/agent-variants/security/tcb-boundary.openclaw.generated.mdx slug: trusted-computing-base + - page: "OpenShell 0.0.72 Compatibility Review" + path: _build/agent-variants/security/openshell-0.0.72-compatibility-review.openclaw.generated.mdx + slug: openshell-0.0.72-compatibility-review - page: "OpenShell 0.0.71 Review" path: _build/agent-variants/security/openshell-0.0.71-gateway-auth-review.openclaw.generated.mdx slug: openshell-0.0.71-gateway-auth-review @@ -262,6 +265,9 @@ navigation: - page: "Set Up Messaging Channels" path: _build/agent-variants/manage-sandboxes/messaging-channels.hermes.generated.mdx slug: messaging-channels + - page: "Set Up MCP Servers" + path: _build/agent-variants/deployment/set-up-mcp-bridge.hermes.generated.mdx + slug: set-up-mcp-servers - page: "Workspace Files" path: _build/agent-variants/manage-sandboxes/workspace-files.hermes.generated.mdx slug: workspace-files @@ -301,12 +307,12 @@ navigation: - page: "Credential Storage" path: _build/agent-variants/security/credential-storage.hermes.generated.mdx slug: credential-storage - - page: "OpenShell 0.0.72 Compatibility Review" - path: _build/agent-variants/security/openshell-0.0.72-compatibility-review.hermes.generated.mdx - slug: openshell-0.0.72-compatibility-review - page: "Trusted Computing Base" path: _build/agent-variants/security/tcb-boundary.hermes.generated.mdx slug: trusted-computing-base + - page: "OpenShell 0.0.72 Compatibility Review" + path: _build/agent-variants/security/openshell-0.0.72-compatibility-review.hermes.generated.mdx + slug: openshell-0.0.72-compatibility-review - page: "OpenShell 0.0.71 Review" path: _build/agent-variants/security/openshell-0.0.71-gateway-auth-review.hermes.generated.mdx slug: openshell-0.0.71-gateway-auth-review diff --git a/docs/inference/inference-options.mdx b/docs/inference/inference-options.mdx index 00e2b814449..6a66873931b 100644 --- a/docs/inference/inference-options.mdx +++ b/docs/inference/inference-options.mdx @@ -43,13 +43,13 @@ NemoClaw uses provider-specific local tokens for those routes, and rebuilds of l |----------|--------|---------------|-------| | NVIDIA Endpoints | Tested | OpenAI-compatible | Hosted models on integrate.api.nvidia.com | | OpenAI | Tested | Native OpenAI-compatible | Uses OpenAI model IDs | -| Other OpenAI-compatible endpoint | Tested with limitations | Custom OpenAI-compatible | Adapter path validated against OpenRouter as the `compatible-endpoint` provider with `openrouter/auto` (see `src/lib/inference/config.test.ts:119`); the onboarding prompt that surfaces OpenRouter as the worked example is at `src/lib/onboard.ts:3661`. Behavior on other OpenAI-compatible proxies, gateways, and self-hosted implementations may vary; this row claims the adapter, not the universe of compatible endpoints. | +| Other OpenAI-compatible endpoint | Tested with limitations | Custom OpenAI-compatible | Adapter path validated against OpenRouter as the `compatible-endpoint` provider with `openrouter/auto` (see `src/lib/inference/config.test.ts:119`); the onboarding prompt that surfaces OpenRouter as the worked example is at `src/lib/onboard.ts:3585`. Behavior on other OpenAI-compatible proxies, gateways, and self-hosted implementations may vary; this row claims the adapter, not the universe of compatible endpoints. | | Anthropic | Tested | Native Anthropic | Uses anthropic-messages | | Other Anthropic-compatible endpoint | Tested with limitations | Custom Anthropic-compatible | Adapter path validated with AWS Bedrock (`src/lib/onboard/bedrock-runtime.ts`). Behavior on other Anthropic-compatible proxies and gateways may vary; this row claims the adapter, not the universe of compatible endpoints. | | Google Gemini | Tested | OpenAI-compatible | Uses Google's OpenAI-compatible endpoint | | Hermes Provider | Hermes only | OpenAI-compatible route | Available when onboarding Hermes Agent through `nemohermes` | | Local Ollama | Tested with limitations | Local Ollama API | Available when Ollama is installed or running on the host. Validated default models: `qwen3.6:35b` (high VRAM), `nemotron-3-nano:30b` (medium VRAM), `qwen3.5:9b` (low VRAM fallback). | -| Local NVIDIA NIM | Experimental | Local OpenAI-compatible | Requires `NEMOCLAW_EXPERIMENTAL=1` and a NIM-capable NVIDIA GPU. Host must have the NVIDIA Container Toolkit installed and a CDI spec present (`onboard` asserts CDI presence with `assertCdiNvidiaGpuSpecPresent`, `src/lib/onboard.ts:1632`). NIM images pull from `nvcr.io` and require NGC registry login. NemoClaw gates this path behind the experimental flag because it does not auto-select a NIM image for the host today. You must explicitly pick from the validated image list. Managed vLLM has host-specific default models and is not gated on the same boxes. Validated images referenced in `src/lib/inference/config.ts` and `nemoclaw/src/index.ts`: `nvidia/nemotron-3-super-120b-a12b` (default cloud model), `nvidia/nemotron-3-nano-30b-a3b`, `nvidia/llama-3.3-nemotron-super-49b-v1.5`. | +| Local NVIDIA NIM | Experimental | Local OpenAI-compatible | Requires `NEMOCLAW_EXPERIMENTAL=1` and a NIM-capable NVIDIA GPU. Host must have the NVIDIA Container Toolkit installed and a CDI spec present (`onboard` asserts CDI presence with `assertCdiNvidiaGpuSpecPresent`, `src/lib/onboard/fatal-runtime-preflight.ts:78`). NIM images pull from `nvcr.io` and require NGC registry login. NemoClaw gates this path behind the experimental flag because it does not auto-select a NIM image for the host today. You must explicitly pick from the validated image list. Managed vLLM has host-specific default models and is not gated on the same boxes. Validated images referenced in `src/lib/inference/config.ts` and `nemoclaw/src/index.ts`: `nvidia/nemotron-3-super-120b-a12b` (default cloud model), `nvidia/nemotron-3-nano-30b-a3b`, `nvidia/llama-3.3-nemotron-super-49b-v1.5`. | | Local vLLM (already running) | Tested with limitations | Local OpenAI-compatible | Appears in the onboarding menu when NemoClaw detects a server already on `localhost:8000`. No flag required. Model is whatever the existing server serves. | | Local vLLM (managed install/start) | Tested with limitations | Local OpenAI-compatible | Appears by default on DGX Spark and DGX Station. Generic Linux NVIDIA GPU hosts require `NEMOCLAW_EXPERIMENTAL=1` or `NEMOCLAW_PROVIDER=install-vllm`. Host must have the NVIDIA Container Toolkit installed and a CDI spec present (`onboard` asserts CDI presence). NemoClaw pulls or starts the stable NGC vLLM container for each host profile. See `src/lib/inference/vllm.ts:55,177` for the pins. DGX Spark and DGX Station use `nvcr.io/nvidia/vllm:26.05.post1-py3`; generic Linux NVIDIA GPU hosts use `nvcr.io/nvidia/vllm:26.03.post1-py3`. Validated defaults are listed in `src/lib/inference/vllm-models.ts`: DGX Spark uses `nvidia/Qwen3.6-35B-A3B-NVFP4`, DGX Station uses `deepseek-ai/DeepSeek-V4-Flash`, and Linux NVIDIA GPU uses `nvidia/NVIDIA-Nemotron-3-Nano-4B-FP8`. Image pulls require NGC registry login (`docker login nvcr.io`); onboard prompts for the NGC API key when authentication is missing. | {/* provider-status:end */} diff --git a/docs/reference/commands-nemohermes.mdx b/docs/reference/commands-nemohermes.mdx index 52509bd055e..c7c9d81e7bc 100644 --- a/docs/reference/commands-nemohermes.mdx +++ b/docs/reference/commands-nemohermes.mdx @@ -813,6 +813,11 @@ If you want to upgrade the sandbox while preserving state, use `nemohermes shields down --timeout 15m --reason "MCP maintenance"`, allowing at least 15 minutes per configured server. +If sandbox deletion is refused after destroy has restored lockdown, NemoClaw opens an owner-bound timed rollback window, restores the preserved MCP state, and re-locks shields; the timer retains auto-restore authority if the host process exits. + If a shields auto-restore timer is active, `destroy` holds the same per-sandbox transition through state wipe and deletion. It restores and verifies lockdown and revokes the active timer before deletion. It clears the remaining local shields state only after deletion succeeds. @@ -1135,6 +1140,100 @@ nemohermes my-assistant channels status --channel whatsapp The probe is bounded by an in-sandbox `openshell sandbox exec` with a hard timeout, captures only short matched bridge log signals (e.g. `connection.open`, `401 unauthorized`, `qr expired`), and never forwards message bodies to the host diagnostic output. +### `nemohermes mcp list` + +List MCP servers configured for a sandbox. +The command reports the selected agent's MCP support status and, for each configured server, whether the generated OpenShell provider, policy, and agent adapter are present. + +```bash +nemohermes my-assistant mcp list [--json] +``` + +| Flag | Description | +|------|-------------| +| `--json` | Emit sandbox, support, and MCP server state as JSON without credential values | + +### `nemohermes mcp add` + +Add an MCP Streamable HTTP server to a sandbox. +Pass `--url` for the MCP endpoint and the required single `--env KEY` bearer credential for the sandbox-side MCP client. +NemoClaw registers that credential in an OpenShell provider, installs a generated OpenShell `protocol: mcp` policy for the target endpoint, attaches the provider to the running sandbox, and writes only an `openshell:resolve:env:KEY` placeholder into the agent config. +Inline `--env KEY=VALUE` is rejected because it would expose the value in NemoClaw process arguments. +Load the variable from a secret manager or masked prompt, export it without recording the value in shell history, and pass only `--env KEY`. +All endpoints must use HTTPS. +The full URL and path are persisted and displayed, so URLs cannot contain userinfo, query strings, fragments, known secret-shaped path material, percent-escaped or glob-style paths, or port zero. +NemoClaw generates a narrow `protocol: mcp` policy for the destination, literal path, adapter binaries, pinned addresses, explicit MCP methods, and a 131,072-byte request-body limit. +OpenShell `0.0.72` evaluates that policy before replacing the attached provider placeholder in the allowed request header. +Static provider placeholders are sandbox-scoped rather than endpoint-exclusive, so do not grant broader inspected-HTTP routes to the same adapter runtime and credential key. +The sandbox client connects directly through OpenShell's existing egress path, and NemoClaw does not run a host-side MCP data-plane bridge, proxy, relay, or listener. +For full setup details, see [Set Up MCP Servers](../manage-sandboxes/set-up-mcp-servers). + +Hermes MCP add, restart, and remove mutate managed config and are refused while shields are up. +Run `nemohermes shields down --timeout 15m --reason "MCP maintenance"` before the mutation, then run `nemohermes shields up` after it; list and status remain read-only. +Allow at least 15 minutes per configured server for an all-server restart or destroy; rebuild opens its own crash-recoverable maintenance window. +Keep shields down until the command returns; a concurrent relock refuses the config commit and can leave an earlier policy/provider stage for the next retry to converge. + +```bash +export GITHUB_MCP_TOKEN=ghp_... +nemohermes my-assistant mcp add github --url https://api.githubcopilot.com/mcp/ --env GITHUB_MCP_TOKEN +unset GITHUB_MCP_TOKEN +``` + +### `nemohermes mcp status` + +Inspect MCP server state for one server or for all configured servers. +Status includes OpenShell provider presence and credential-key shape, provider attachment, generated policy content match, adapter registration, current host-variable availability, and the selected agent's MCP support mode. +While a managed provider is attached, text and JSON status warn that its credential is sandbox-scoped until OpenShell supports endpoint-exclusive binding plus Host, scheme, and query enforcement. + +```bash +nemohermes my-assistant mcp status [server] [--json] +``` + +| Flag | Description | +|------|-------------| +| `--json` | Emit status as JSON without credential values | + +### `nemohermes mcp restart` + +Refresh one MCP server registration, or every server on the sandbox when no server is supplied. +Restart reapplies the generated policy, reattaches the OpenShell provider when needed, and refreshes the sandbox agent adapter registration. +If the recorded host variable is exported, restart replaces the provider credential and waits for its new opaque revision. +Otherwise, restart reuses an existing provider whose current metadata match the registry. +A missing provider requires the variable to be exported before retrying. +When that provider is already absent but its name still blocks sandbox exec, +restart first detaches only the dangling sandbox-spec reference, then runs the +agent capability probe before changing a live provider or policy. + +Hermes shields must be down for this config mutation. +Keep them down until the command returns; a concurrent relock refuses the config commit and can leave an earlier policy/provider stage for the next retry to converge. + +```bash +nemohermes my-assistant mcp restart [server] +``` + +### `nemohermes mcp remove` + +Remove an MCP server from a sandbox. + +Hermes shields must be down for this config mutation. +Keep them down until the command returns; a concurrent relock refuses the config commit and can leave an earlier policy/provider stage for the next retry to converge. + +NemoClaw unregisters the sandbox agent adapter, prechecks and removes the recorded OpenShell provider, removes the owned generated policy, and clears the sandbox registry entry. +Deep Agents teardown does not require the managed launcher marker from the old image, so an existing compatible MCP entry cannot block `mcp remove`, sandbox rebuild, or destroy merely because the image predates that probe. +A replacement image must pass the marker probe before post-rebuild MCP providers or policy are restored. +The command fails closed on observed drift. +`--force` may remove a modified same-name agent adapter entry, but provider deletion still requires the exact recorded ID, type, and credential key and policy deletion still requires exact owned content. +Residuals preserve registry state. +OpenShell `0.0.72` mutates providers by name, so do not concurrently replace a managed provider through another OpenShell client during this command. + +```bash +nemohermes my-assistant mcp remove github [--force] +``` + +| Flag | Description | +|------|-------------| +| `--force` | Remove same-name adapter config and continue exact-ownership provider/policy cleanup; preserve registry state when residuals remain | + ### `nemohermes skill install ` Deploy a skill directory to a running sandbox. @@ -2140,6 +2239,7 @@ Defaults are sized for typical hardware; override only if you see false-positive | Variable | Default | Effect | |----------|---------|--------| +| `NEMOCLAW_MCP_PROVIDER_SYNC_TIMEOUT_SECONDS` | `30` | Maximum time to wait for an OpenShell MCP provider credential revision to become active or fully revoked inside the sandbox. Integer seconds; raise only when provider synchronization is unusually slow. | | `NEMOCLAW_SANDBOX_EXEC_TIMEOUT_MS` | per call site (typically `15000`) | Overrides the default timeout for `openshell sandbox exec` calls issued by recovery and lifecycle helpers. Integer milliseconds; non-positive or non-numeric values fall back to the per-call-site default. | | `NEMOCLAW_STATUS_PROBE_TIMEOUT_MS` | built-in default | Overrides the timeout for the OpenShell status probe used by `nemohermes status`. Integer milliseconds; non-positive or non-numeric values fall back to the default. | diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 3e4d78aa7a3..4432234ac24 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -1123,6 +1123,15 @@ If you want to upgrade the sandbox while preserving state, use `$$nemoclaw + +If the Hermes sandbox has managed MCP entries, shields must be down before destroy can scrub their adapter configuration. +Use `$$nemoclaw shields down --timeout 15m --reason "MCP maintenance"`, allowing at least 15 minutes per configured server. +If sandbox deletion is refused after destroy has restored lockdown, NemoClaw opens an owner-bound timed rollback window, restores the preserved MCP state, and re-locks shields; the timer retains auto-restore authority if the host process exits. + + + If a shields auto-restore timer is active, `destroy` holds the same per-sandbox transition through state wipe and deletion. It restores and verifies lockdown and revokes the active timer before deletion. It clears the remaining local shields state only after deletion succeeds. @@ -1445,6 +1454,112 @@ $$nemoclaw my-assistant channels status --channel whatsapp The probe is bounded by an in-sandbox `openshell sandbox exec` with a hard timeout, captures only short matched bridge log signals (e.g. `connection.open`, `401 unauthorized`, `qr expired`), and never forwards message bodies to the host diagnostic output. +### `$$nemoclaw mcp list` + +List MCP servers configured for a sandbox. +The command reports the selected agent's MCP support status and, for each configured server, whether the generated OpenShell provider, policy, and agent adapter are present. + +```bash +$$nemoclaw my-assistant mcp list [--json] +``` + +| Flag | Description | +|------|-------------| +| `--json` | Emit sandbox, support, and MCP server state as JSON without credential values | + +### `$$nemoclaw mcp add` + +Add an MCP Streamable HTTP server to a sandbox. +Pass `--url` for the MCP endpoint and the required single `--env KEY` bearer credential for the sandbox-side MCP client. +NemoClaw registers that credential in an OpenShell provider, installs a generated OpenShell `protocol: mcp` policy for the target endpoint, attaches the provider to the running sandbox, and writes only an `openshell:resolve:env:KEY` placeholder into the agent config. +Inline `--env KEY=VALUE` is rejected because it would expose the value in NemoClaw process arguments. +Load the variable from a secret manager or masked prompt, export it without recording the value in shell history, and pass only `--env KEY`. +All endpoints must use HTTPS. +The full URL and path are persisted and displayed, so URLs cannot contain userinfo, query strings, fragments, known secret-shaped path material, percent-escaped or glob-style paths, or port zero. +NemoClaw generates a narrow `protocol: mcp` policy for the destination, literal path, adapter binaries, pinned addresses, explicit MCP methods, and a 131,072-byte request-body limit. +OpenShell `0.0.72` evaluates that policy before replacing the attached provider placeholder in the allowed request header. +Static provider placeholders are sandbox-scoped rather than endpoint-exclusive, so do not grant broader inspected-HTTP routes to the same adapter runtime and credential key. +The sandbox client connects directly through OpenShell's existing egress path, and NemoClaw does not run a host-side MCP data-plane bridge, proxy, relay, or listener. +For full setup details, see [Set Up MCP Servers](../manage-sandboxes/set-up-mcp-servers). + + + +Hermes MCP add, restart, and remove mutate managed config and are refused while shields are up. +Run `$$nemoclaw shields down --timeout 15m --reason "MCP maintenance"` before the mutation, then run `$$nemoclaw shields up` after it; list and status remain read-only. +Allow at least 15 minutes per configured server for an all-server restart or destroy; rebuild opens its own crash-recoverable maintenance window. +Keep shields down until the command returns; a concurrent relock refuses the config commit and can leave an earlier policy/provider stage for the next retry to converge. + + + +```bash +export GITHUB_MCP_TOKEN=ghp_... +$$nemoclaw my-assistant mcp add github --url https://api.githubcopilot.com/mcp/ --env GITHUB_MCP_TOKEN +unset GITHUB_MCP_TOKEN +``` + +### `$$nemoclaw mcp status` + +Inspect MCP server state for one server or for all configured servers. +Status includes OpenShell provider presence and credential-key shape, provider attachment, generated policy content match, adapter registration, current host-variable availability, and the selected agent's MCP support mode. +While a managed provider is attached, text and JSON status warn that its credential is sandbox-scoped until OpenShell supports endpoint-exclusive binding plus Host, scheme, and query enforcement. + +```bash +$$nemoclaw my-assistant mcp status [server] [--json] +``` + +| Flag | Description | +|------|-------------| +| `--json` | Emit status as JSON without credential values | + +### `$$nemoclaw mcp restart` + +Refresh one MCP server registration, or every server on the sandbox when no server is supplied. +Restart reapplies the generated policy, reattaches the OpenShell provider when needed, and refreshes the sandbox agent adapter registration. +If the recorded host variable is exported, restart replaces the provider credential and waits for its new opaque revision. +Otherwise, restart reuses an existing provider whose current metadata match the registry. +A missing provider requires the variable to be exported before retrying. +When that provider is already absent but its name still blocks sandbox exec, +restart first detaches only the dangling sandbox-spec reference, then runs the +agent capability probe before changing a live provider or policy. + + + +Hermes shields must be down for this config mutation. +Keep them down until the command returns; a concurrent relock refuses the config commit and can leave an earlier policy/provider stage for the next retry to converge. + + + +```bash +$$nemoclaw my-assistant mcp restart [server] +``` + +### `$$nemoclaw mcp remove` + +Remove an MCP server from a sandbox. + + + +Hermes shields must be down for this config mutation. +Keep them down until the command returns; a concurrent relock refuses the config commit and can leave an earlier policy/provider stage for the next retry to converge. + + + +NemoClaw unregisters the sandbox agent adapter, prechecks and removes the recorded OpenShell provider, removes the owned generated policy, and clears the sandbox registry entry. +Deep Agents teardown does not require the managed launcher marker from the old image, so an existing compatible MCP entry cannot block `mcp remove`, sandbox rebuild, or destroy merely because the image predates that probe. +A replacement image must pass the marker probe before post-rebuild MCP providers or policy are restored. +The command fails closed on observed drift. +`--force` may remove a modified same-name agent adapter entry, but provider deletion still requires the exact recorded ID, type, and credential key and policy deletion still requires exact owned content. +Residuals preserve registry state. +OpenShell `0.0.72` mutates providers by name, so do not concurrently replace a managed provider through another OpenShell client during this command. + +```bash +$$nemoclaw my-assistant mcp remove github [--force] +``` + +| Flag | Description | +|------|-------------| +| `--force` | Remove same-name adapter config and continue exact-ownership provider/policy cleanup; preserve registry state when residuals remain | + ### `$$nemoclaw skill install ` Deploy a skill directory to a running sandbox. @@ -2645,6 +2760,7 @@ Defaults are sized for typical hardware; override only if you see false-positive | Variable | Default | Effect | |----------|---------|--------| +| `NEMOCLAW_MCP_PROVIDER_SYNC_TIMEOUT_SECONDS` | `30` | Maximum time to wait for an OpenShell MCP provider credential revision to become active or fully revoked inside the sandbox. Integer seconds; raise only when provider synchronization is unusually slow. | | `NEMOCLAW_SANDBOX_EXEC_TIMEOUT_MS` | per call site (typically `15000`) | Overrides the default timeout for `openshell sandbox exec` calls issued by recovery and lifecycle helpers. Integer milliseconds; non-positive or non-numeric values fall back to the per-call-site default. | | `NEMOCLAW_STATUS_PROBE_TIMEOUT_MS` | built-in default | Overrides the timeout for the OpenShell status probe used by `$$nemoclaw status`. Integer milliseconds; non-positive or non-numeric values fall back to the default. | diff --git a/docs/reference/platform-support.mdx b/docs/reference/platform-support.mdx index c2fe16ae1d9..f4bc00f77fa 100644 --- a/docs/reference/platform-support.mdx +++ b/docs/reference/platform-support.mdx @@ -78,7 +78,7 @@ For the onboarding-time supported set without deferred rows, refer to [Prerequis {/* platform-matrix-full:begin */} | OS | Container runtime | Status | PRD priority | CI | Notes | |----|-------------------|--------|--------------|----|-------| -| Linux | Docker | Tested | P0 | Yes | Primary tested path. Ubuntu 24.04 is the validated distro in production source (`DEFAULT_COMPAT_IMAGE` in `src/lib/onboard/docker-driver-gateway-launch.ts:10` and the preflight tests pin 24.04 only); the installer's package-manager probes assume apt-get. Other distros (Ubuntu 22.04, Fedora, Rocky, Alma, NixOS, Arch) may work but are not validated. | +| Linux | Docker | Tested | P0 | Yes | Primary tested path. Ubuntu 24.04 is the validated distro in production source (`DEFAULT_COMPAT_IMAGE` in `src/lib/onboard/docker-driver-gateway-compat.ts:11` and the preflight tests pin 24.04 only); the installer's package-manager probes assume apt-get. Other distros (Ubuntu 22.04, Fedora, Rocky, Alma, NixOS, Arch) may work but are not validated. | | macOS (Apple Silicon) | Colima, Docker Desktop | Tested with limitations | P0 | Yes | Start the container runtime (Colima or Docker Desktop) before running the installer. Homebrew Colima users must install both Colima and the Docker CLI (`brew install colima docker`) before `docker info` can work. Xcode Command Line Tools (`xcode-select --install`) are typically required for Node native modules during install. NemoClaw recommends them but does not enforce them during preflight. | | DGX Spark | Docker | Tested | P1 | Yes | Use the standard installer and `$$nemoclaw onboard`. For an end-to-end walkthrough with local inference, see the [NVIDIA Spark playbook](https://build.nvidia.com/spark/nemoclaw). | | Windows WSL2 | Docker Desktop (WSL backend) | Tested with limitations | P1 | No | Requires WSL2 with Docker Desktop backend. | @@ -95,13 +95,13 @@ NemoClaw routes inference through the OpenShell gateway. Each row below is a pro |----------|--------|---------------|-------| | NVIDIA Endpoints | Tested | OpenAI-compatible | Hosted models on integrate.api.nvidia.com | | OpenAI | Tested | Native OpenAI-compatible | Uses OpenAI model IDs | -| Other OpenAI-compatible endpoint | Tested with limitations | Custom OpenAI-compatible | Adapter path validated against OpenRouter as the `compatible-endpoint` provider with `openrouter/auto` (see `src/lib/inference/config.test.ts:119`); the onboarding prompt that surfaces OpenRouter as the worked example is at `src/lib/onboard.ts:3661`. Behavior on other OpenAI-compatible proxies, gateways, and self-hosted implementations may vary; this row claims the adapter, not the universe of compatible endpoints. | +| Other OpenAI-compatible endpoint | Tested with limitations | Custom OpenAI-compatible | Adapter path validated against OpenRouter as the `compatible-endpoint` provider with `openrouter/auto` (see `src/lib/inference/config.test.ts:119`); the onboarding prompt that surfaces OpenRouter as the worked example is at `src/lib/onboard.ts:3585`. Behavior on other OpenAI-compatible proxies, gateways, and self-hosted implementations may vary; this row claims the adapter, not the universe of compatible endpoints. | | Anthropic | Tested | Native Anthropic | Uses anthropic-messages | | Other Anthropic-compatible endpoint | Tested with limitations | Custom Anthropic-compatible | Adapter path validated with AWS Bedrock (`src/lib/onboard/bedrock-runtime.ts`). Behavior on other Anthropic-compatible proxies and gateways may vary; this row claims the adapter, not the universe of compatible endpoints. | | Google Gemini | Tested | OpenAI-compatible | Uses Google's OpenAI-compatible endpoint | | Hermes Provider | Hermes only | OpenAI-compatible route | Available when onboarding Hermes Agent through `nemohermes` | | Local Ollama | Tested with limitations | Local Ollama API | Available when Ollama is installed or running on the host. Validated default models: `qwen3.6:35b` (high VRAM), `nemotron-3-nano:30b` (medium VRAM), `qwen3.5:9b` (low VRAM fallback). | -| Local NVIDIA NIM | Experimental | Local OpenAI-compatible | Requires `NEMOCLAW_EXPERIMENTAL=1` and a NIM-capable NVIDIA GPU. Host must have the NVIDIA Container Toolkit installed and a CDI spec present (`onboard` asserts CDI presence with `assertCdiNvidiaGpuSpecPresent`, `src/lib/onboard.ts:1632`). NIM images pull from `nvcr.io` and require NGC registry login. NemoClaw gates this path behind the experimental flag because it does not auto-select a NIM image for the host today. You must explicitly pick from the validated image list. Managed vLLM has host-specific default models and is not gated on the same boxes. Validated images referenced in `src/lib/inference/config.ts` and `nemoclaw/src/index.ts`: `nvidia/nemotron-3-super-120b-a12b` (default cloud model), `nvidia/nemotron-3-nano-30b-a3b`, `nvidia/llama-3.3-nemotron-super-49b-v1.5`. | +| Local NVIDIA NIM | Experimental | Local OpenAI-compatible | Requires `NEMOCLAW_EXPERIMENTAL=1` and a NIM-capable NVIDIA GPU. Host must have the NVIDIA Container Toolkit installed and a CDI spec present (`onboard` asserts CDI presence with `assertCdiNvidiaGpuSpecPresent`, `src/lib/onboard/fatal-runtime-preflight.ts:78`). NIM images pull from `nvcr.io` and require NGC registry login. NemoClaw gates this path behind the experimental flag because it does not auto-select a NIM image for the host today. You must explicitly pick from the validated image list. Managed vLLM has host-specific default models and is not gated on the same boxes. Validated images referenced in `src/lib/inference/config.ts` and `nemoclaw/src/index.ts`: `nvidia/nemotron-3-super-120b-a12b` (default cloud model), `nvidia/nemotron-3-nano-30b-a3b`, `nvidia/llama-3.3-nemotron-super-49b-v1.5`. | | Local vLLM (already running) | Tested with limitations | Local OpenAI-compatible | Appears in the onboarding menu when NemoClaw detects a server already on `localhost:8000`. No flag required. Model is whatever the existing server serves. | | Local vLLM (managed install/start) | Tested with limitations | Local OpenAI-compatible | Appears by default on DGX Spark and DGX Station. Generic Linux NVIDIA GPU hosts require `NEMOCLAW_EXPERIMENTAL=1` or `NEMOCLAW_PROVIDER=install-vllm`. Host must have the NVIDIA Container Toolkit installed and a CDI spec present (`onboard` asserts CDI presence). NemoClaw pulls or starts the stable NGC vLLM container for each host profile. See `src/lib/inference/vllm.ts:55,177` for the pins. DGX Spark and DGX Station use `nvcr.io/nvidia/vllm:26.05.post1-py3`; generic Linux NVIDIA GPU hosts use `nvcr.io/nvidia/vllm:26.03.post1-py3`. Validated defaults are listed in `src/lib/inference/vllm-models.ts`: DGX Spark uses `nvidia/Qwen3.6-35B-A3B-NVFP4`, DGX Station uses `deepseek-ai/DeepSeek-V4-Flash`, and Linux NVIDIA GPU uses `nvidia/NVIDIA-Nemotron-3-Nano-4B-FP8`. Image pulls require NGC registry login (`docker login nvcr.io`); onboard prompts for the NGC API key when authentication is missing. | {/* provider-status-full:end */} @@ -160,13 +160,13 @@ They are listed here so launch material, sales conversations, and support triage {/* out-of-scope:begin */} | Item | Status | Why | |------|--------|-----| -| Podman / other container runtimes | Unsupported | Onboard surfaces an explicit unsupported-runtime error for Podman (`src/lib/onboard.ts:1599` prints the rejection; `src/lib/onboard/preflight.ts:586` flags the unsupported runtime upstream). Only Docker Engine, Docker Desktop, and Colima are supported. See issue #420 (closed). | -| Intel Mac (macOS x86_64) | Unsupported | OpenShell does not publish macOS x86_64 standalone gateway assets. Install hard-fails on x86_64 macOS (`scripts/install-openshell.sh:315`). See issue #954 (closed). | +| Podman / other container runtimes | Unsupported | Onboard surfaces an explicit unsupported-runtime error for Podman (`src/lib/onboard/fatal-runtime-preflight.ts:50` prints the rejection; `src/lib/onboard/preflight.ts:676` flags the unsupported runtime upstream). Only Docker Engine, Docker Desktop, and Colima are supported. See issue #420 (closed). | +| Intel Mac (macOS x86_64) | Unsupported | OpenShell does not publish macOS x86_64 standalone gateway assets. Install hard-fails on x86_64 macOS (`scripts/install-openshell.sh:654`). See issue #954 (closed). | | Non-Ubuntu/Debian Linux distros | Unsupported | Installer assumes `apt-get`. Fedora/Rocky/Alma/Arch/NixOS are not validated and the installer's package-manager probes do not cover them. See open issue #899 (Fedora hang). | | Native Kubernetes or OpenShift deployments | Unsupported | NemoClaw runs the sandbox as a Docker container, not a Kubernetes pod. The default Docker-driver topology does not embed k3s. Operator-managed K8s/OpenShift deployments are out of scope; see issue #407 (community OpenShift through agent-sandbox CRD). | | Air-gapped / offline installs | Unsupported | Onboard assumes network reachability for package fetches, container pulls, and provider validation. See open issues #4872 and #2218 (production-deployment epic covering air-gapped support, China network guidance, multi-host topology). | | Windows-on-ARM GPU passthrough | Unsupported | Windows-on-ARM CPU paths run under WSL2 'tested with limitations', but GPU passthrough on WOA is denylisted (`src/lib/onboard/wsl-docker-desktop-gpu.ts:188`, `src/lib/inference/gpu-trust.test.ts:70`). See closed issue #4565. | -| Non-NVIDIA GPUs (AMD/ROCm, Intel Arc, Apple Metal) | Unsupported | Local vLLM and NIM paths assert NVIDIA CDI presence with `assertCdiNvidiaGpuSpecPresent` (`src/lib/onboard.ts:1632`). NemoClaw does not install non-NVIDIA accelerator drivers. | +| Non-NVIDIA GPUs (AMD/ROCm, Intel Arc, Apple Metal) | Unsupported | Local vLLM and NIM paths assert NVIDIA CDI presence with `assertCdiNvidiaGpuSpecPresent` (`src/lib/onboard/fatal-runtime-preflight.ts:78`). NemoClaw does not install non-NVIDIA accelerator drivers. | | Other LangChain, AutoGen, CrewAI, or non-listed agent harnesses | Unsupported | LangChain Deep Agents Code is the only integrated LangChain-family harness (see the Agents section above; status `Experimental`). Other LangChain harnesses, AutoGen, CrewAI, and any agent runtime not listed in the Agents table are not integrated. Bringing more harnesses is tracked as a research epic (see open issue #4861) but is not on the current roadmap. | | Multi-user host sharing | Unsupported | Sandboxes are scoped to a single host user. NemoClaw treats multi-user hosts as a risk and warns at onboard; see `docs/security/openclaw-controls.mdx` Multi-user detection. | | Hosted SaaS / managed NemoClaw | Unsupported | There is no managed offering. Supported deployment paths are Local CLI onboard, Remote GPU with Brev CLI, and Brev web UI. | diff --git a/docs/security/credential-storage.mdx b/docs/security/credential-storage.mdx index 1c7677dd58d..6450aac4493 100644 --- a/docs/security/credential-storage.mdx +++ b/docs/security/credential-storage.mdx @@ -70,7 +70,10 @@ Use this precedence to: - Prefix any command with the credential to override the gateway-stored value: `NVIDIA_INFERENCE_API_KEY=nvapi-... $$nemoclaw onboard`. - Use short-lived or rotated credentials in CI by exporting them once per pipeline run. -- Avoid registering credentials in the gateway entirely if your environment supplies them. +- Avoid registering credentials in the gateway entirely if the specific command supports environment-only use. + +Managed MCP is an exception: `$$nemoclaw mcp add` always creates and attaches an OpenShell provider, and `--env KEY` supplies only the transient input value. +For that credential boundary, refer to [Set Up MCP Servers](../manage-sandboxes/set-up-mcp-servers). When the host environment is empty, day-two operations such as `$$nemoclaw rebuild` and remote-provider updates can reuse the credential already registered with the OpenShell gateway. Export the credential only when you want to create, replace, or rotate the stored provider value. @@ -149,4 +152,4 @@ On the next run NemoClaw prompts again unless the credential is supplied through ## Related Files -For the broader sandbox security model and operational trade-offs, refer to [Security Best Practices](best-practices) and [Architecture](../reference/architecture). +For the broader sandbox security model and operational trade-offs, refer to [Security Best Practices](best-practices), [Architecture](../reference/architecture), and [Set Up MCP Servers](../manage-sandboxes/set-up-mcp-servers). diff --git a/docs/security/openshell-0.0.72-compatibility-review.mdx b/docs/security/openshell-0.0.72-compatibility-review.mdx index 16225d38263..62c84018638 100644 --- a/docs/security/openshell-0.0.72-compatibility-review.mdx +++ b/docs/security/openshell-0.0.72-compatibility-review.mdx @@ -11,14 +11,15 @@ content: --- This review covers NemoClaw's stable OpenShell `0.0.72` pin, Docker-driver gateway authentication, policy mutation, and MCP and JSON-RPC policy compatibility. -The review was completed on June 29, 2026. +The dependency compatibility review was completed on June 29, 2026; the MCP integration and DNS source/runtime supplement were reviewed on June 30, 2026. ## Release Identity - The stable tag is `NVIDIA/OpenShell@v0.0.72` at commit `8cb16de9eae4c44d7d31e1493747d8c10abb5963`. - The upstream [v0.0.72 release workflow](https://github.com/NVIDIA/OpenShell/actions/runs/28382086068) completed all 54 jobs at that commit, including the MCP conformance lane, package smoke tests, release publication, and GHCR tags. - NemoClaw pins the eight consumed CLI, gateway, and sandbox assets to the digests published by the [GitHub release API](https://api.github.com/repos/NVIDIA/OpenShell/releases/tags/v0.0.72). -- The stable Docker-driver default pins the multi-architecture supervisor manifest as `ghcr.io/nvidia/openshell/supervisor@sha256:80ed9cda5bf672fefdb9dcd4604b40a8b09c0891b6eb9d03e10227c7e3dfb49d`. Explicit operator overrides and the opt-in development channel remain separate trust decisions. +- The stable Docker-driver default pins the multi-architecture supervisor manifest as `ghcr.io/nvidia/openshell/supervisor@sha256:80ed9cda5bf672fefdb9dcd4604b40a8b09c0891b6eb9d03e10227c7e3dfb49d`. + Explicit operator overrides and the opt-in development channel remain separate trust decisions. ## Source-of-Truth Boundaries @@ -55,7 +56,8 @@ These version-specific pins are removed only when NemoClaw drops `0.0.72` suppor - `regressionTest`: `test/install-openshell-version-check.test.ts` proves the development channel fails without `NEMOCLAW_ACCEPT_DEV_UNVERIFIED_INSTALL=1` and succeeds with it. - `removalCondition`: Remove the opt-in when NemoClaw no longer tests unreleased OpenShell builds or the development channel publishes artifacts through an independently verified immutable pipeline. -The development channel is compatibility evidence only. Use it in trusted test environments, never as the stable shipping configuration. +The development channel is compatibility evidence only. +Use it in trusted test environments, never as the stable shipping configuration. ## Round-Trippable Policy Boundary @@ -78,10 +80,25 @@ OpenShell `0.0.72` adds `protocol: mcp` for MCP Streamable HTTP and `protocol: j MCP rules can match methods and `tools/call` tool names, support allow and deny rules, and fail closed for malformed or ambiguous request frames. The upstream MCP conformance lane passed `initialize`, `tools_call`, and `elicitation-sep1034-client-defaults` with no expected failures. -This dependency PR preserves the new MCP and JSON-RPC YAML fields when NemoClaw merges existing policies. -It does not widen NemoClaw's strict blueprint-addition schema to author new MCP endpoints because that is a separate product and API change. +NemoClaw preserves the new MCP and JSON-RPC YAML fields when it merges existing policies. +The strict blueprint-addition schema does not author MCP endpoints; `nemoclaw mcp add` is the supported managed product path described in [Set Up MCP Servers](../manage-sandboxes/set-up-mcp-servers) and the [accepted architecture decision](https://github.com/NVIDIA/NemoClaw/issues/566#issuecomment-4847534784). OpenShell enforcement covers sandbox-to-server Streamable HTTP requests, not stdio MCP or generic inbound traffic. +## DNS Pinning Source and Runtime Contract + +The MCP integration pins the OpenShell DNS enforcement contract to [`NVIDIA/OpenShell@8cb16de9eae4c44d7d31e1493747d8c10abb5963`](https://github.com/NVIDIA/OpenShell/tree/8cb16de9eae4c44d7d31e1493747d8c10abb5963). +In that implementation, [`crates/openshell-supervisor-network/src/proxy.rs:2476-2502`](https://github.com/NVIDIA/OpenShell/blob/8cb16de9eae4c44d7d31e1493747d8c10abb5963/crates/openshell-supervisor-network/src/proxy.rs#L2476-L2502) produces one socket-address list, [`crates/openshell-supervisor-network/src/proxy.rs:2527-2567`](https://github.com/NVIDIA/OpenShell/blob/8cb16de9eae4c44d7d31e1493747d8c10abb5963/crates/openshell-supervisor-network/src/proxy.rs#L2527-L2567) validates every address in that list, and [`crates/openshell-supervisor-network/src/proxy.rs:2622-2630`](https://github.com/NVIDIA/OpenShell/blob/8cb16de9eae4c44d7d31e1493747d8c10abb5963/crates/openshell-supervisor-network/src/proxy.rs#L2622-L2630) returns the validated list unchanged. +The CONNECT path passes that returned list directly to `TcpStream::connect` at [`crates/openshell-supervisor-network/src/proxy.rs:822-832`](https://github.com/NVIDIA/OpenShell/blob/8cb16de9eae4c44d7d31e1493747d8c10abb5963/crates/openshell-supervisor-network/src/proxy.rs#L822-L832). +The explicit HTTP-forward path carries the same returned list from [`crates/openshell-supervisor-network/src/proxy.rs:3885-3893`](https://github.com/NVIDIA/OpenShell/blob/8cb16de9eae4c44d7d31e1493747d8c10abb5963/crates/openshell-supervisor-network/src/proxy.rs#L3885-L3893) to [`crates/openshell-supervisor-network/src/proxy.rs:4123-4125`](https://github.com/NVIDIA/OpenShell/blob/8cb16de9eae4c44d7d31e1493747d8c10abb5963/crates/openshell-supervisor-network/src/proxy.rs#L4123-L4125). +There is no second hostname resolution between validation and connection in either path. + +The stable and development `mcp-bridge` live lanes isolate that upstream contract from NemoClaw's MCP implementation before the OpenClaw scenario performs any managed MCP mutation. +They apply a raw OpenShell `protocol: mcp` policy with `allowed_ips: [1.1.1.1]`, remap the hostname to a reachable private runner address, send a raw MCP `tools/list` request, require an exact HTTP 403, verify zero upstream requests without calling `nemoclaw mcp` or any agent adapter, and restore the exact base policy in `finally`. + +The live MCP scenario registers a hostname while it resolves to a pinned public address, remaps it to a reachable unpinned runner address, and sends an MCP `tools/list` request beneath each adapter runtime identity. +OpenClaw uses the managed Node identity, Hermes uses its managed Python identity, and LangChain Deep Agents Code uses its managed virtual-environment Python identity. +The scenario requires an OpenShell HTTP 403 or CONNECT 403 for every adapter and verifies that the upstream MCP server recorded zero requests. + ## Local Contract Coverage - Installer and runner tests pin all eight published release digests. @@ -89,3 +106,5 @@ OpenShell enforcement covers sandbox-to-server Streamable HTTP requests, not std - Policy tests cover `--base` command construction and MCP and JSON-RPC field preservation. - Blueprint tests prove the merged policy excludes reserved provider entries. - The live gateway authentication and gateway-upgrade scenarios run against `0.0.72`. +- The stable and development MCP live lanes independently prove raw OpenShell `allowed_ips` rebinding denial with an exact HTTP 403 and zero upstream requests, then restore the base policy. +- The live MCP matrix proves DNS rebinding denial with zero upstream requests for OpenClaw, Hermes, and LangChain Deep Agents Code. diff --git a/nemoclaw-blueprint/blueprint.yaml b/nemoclaw-blueprint/blueprint.yaml index 05851b7253e..87b5c3ecfdc 100644 --- a/nemoclaw-blueprint/blueprint.yaml +++ b/nemoclaw-blueprint/blueprint.yaml @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 version: "0.1.0" +# Requires OpenShell MCP/JSON-RPC L7 policy support from NVIDIA/OpenShell#1865. min_openshell_version: "0.0.72" max_openshell_version: "0.0.72" min_openclaw_version: "2026.3.11" diff --git a/nemoclaw/src/lib/subprocess-env.ts b/nemoclaw/src/lib/subprocess-env.ts index 90927a186c7..a73102d98eb 100644 --- a/nemoclaw/src/lib/subprocess-env.ts +++ b/nemoclaw/src/lib/subprocess-env.ts @@ -40,14 +40,34 @@ const TLS = [ const TOOLCHAIN = ["DOCKER_HOST", "KUBECONFIG", "SSH_AUTH_SOCK", "RUST_LOG", "RUST_BACKTRACE"]; -const ALLOWED_ENV_NAMES = new Set([...SYSTEM, ...TEMP, ...LOCALE, ...PROXY, ...TLS, ...TOOLCHAIN]); +export const SUBPROCESS_ENV_ALLOWED_NAMES: readonly string[] = Object.freeze([ + ...SYSTEM, + ...TEMP, + ...LOCALE, + ...PROXY, + ...TLS, + ...TOOLCHAIN, +]); +const ALLOWED_ENV_NAMES = new Set(SUBPROCESS_ENV_ALLOWED_NAMES); // ── Allowed prefixes ─────────────────────────────────────────── -const ALLOWED_ENV_PREFIXES = ["LC_", "XDG_", "OPENSHELL_", "GRPC_"]; +export const SUBPROCESS_ENV_ALLOWED_PREFIXES: readonly string[] = Object.freeze([ + "LC_", + "XDG_", + "OPENSHELL_", + "GRPC_", +]); // ── Public API ───────────────────────────────────────────────── +export function isSubprocessEnvNameAllowed(name: string): boolean { + return ( + ALLOWED_ENV_NAMES.has(name) || + SUBPROCESS_ENV_ALLOWED_PREFIXES.some((prefix) => name.startsWith(prefix)) + ); +} + /** * When any HTTP proxy is forwarded, augment NO_PROXY so the host proxy is * never asked to forward traffic destined for the host loopback, the @@ -102,7 +122,7 @@ export function buildSubprocessEnv(extra?: Record): Record = {}; for (const [key, value] of Object.entries(process.env)) { if (value === undefined) continue; - if (ALLOWED_ENV_NAMES.has(key) || ALLOWED_ENV_PREFIXES.some((p) => key.startsWith(p))) { + if (isSubprocessEnvNameAllowed(key)) { env[key] = value; } } diff --git a/package-lock.json b/package-lock.json index df4b45987b4..8989292e110 100644 --- a/package-lock.json +++ b/package-lock.json @@ -35,6 +35,7 @@ "@types/node": "^25.5.2", "@vitest/coverage-v8": "^4.1.0", "ajv": "^8.17.0", + "fast-check": "^4.8.0", "tsx": "^4.21.0", "typescript": "^6.0.2", "vitest": "^4.1.9" @@ -3962,6 +3963,29 @@ "@types/yauzl": "^2.9.1" } }, + "node_modules/fast-check": { + "version": "4.8.0", + "resolved": "https://registry.npmjs.org/fast-check/-/fast-check-4.8.0.tgz", + "integrity": "sha512-GOJ158CUMnN6cSahsv4+ExARvIDuzzinFjkp0E9WtiBa5zcVeLozVkWaE4IzFcc+Y48Wp1EDlUZsXRyAztQcSg==", + "dev": true, + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/dubzzz" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fast-check" + } + ], + "license": "MIT", + "dependencies": { + "pure-rand": "^8.0.0" + }, + "engines": { + "node": ">=12.17.0" + } + }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -5961,6 +5985,23 @@ "once": "^1.3.1" } }, + "node_modules/pure-rand": { + "version": "8.4.1", + "resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-8.4.1.tgz", + "integrity": "sha512-c58R2+SPFcSIPXoU834QN/KPDDOSd8sXcSrqf6e83Me6Rrp1EYkxukkjXMVrKvKaADs1SOyNkWdfvLf6zY8qLQ==", + "dev": true, + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/dubzzz" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fast-check" + } + ], + "license": "MIT" + }, "node_modules/qrcode-terminal": { "version": "0.12.0", "resolved": "https://registry.npmjs.org/qrcode-terminal/-/qrcode-terminal-0.12.0.tgz", diff --git a/package.json b/package.json index a23a43c1a8e..2f40073d94c 100644 --- a/package.json +++ b/package.json @@ -108,6 +108,7 @@ "@types/node": "^25.5.2", "@vitest/coverage-v8": "^4.1.0", "ajv": "^8.17.0", + "fast-check": "^4.8.0", "tsx": "^4.21.0", "typescript": "^6.0.2", "vitest": "^4.1.9" diff --git a/schemas/policy-preset.schema.json b/schemas/policy-preset.schema.json index cd30ce52444..b97c0c7f618 100644 --- a/schemas/policy-preset.schema.json +++ b/schemas/policy-preset.schema.json @@ -48,10 +48,12 @@ "properties": { "host": { "type": "string" }, "port": { "type": "integer", "minimum": 1, "maximum": 65535 }, - "protocol": { "type": "string", "enum": ["rest", "websocket"] }, + "protocol": { "type": "string", "enum": ["rest", "websocket", "json-rpc", "mcp"] }, "enforcement": { "type": "string", "enum": ["enforce", "audit"] }, "tls": { "type": "string", "enum": ["terminate", "passthrough", "skip"] }, "access": { "type": "string", "enum": ["full"] }, + "json_rpc": { "$ref": "#/$defs/jsonRpcOptions" }, + "mcp": { "$ref": "#/$defs/mcpOptions" }, "websocket_credential_rewrite": { "type": "boolean" }, "request_body_credential_rewrite": { "type": "boolean" }, "allowed_ips": { @@ -63,43 +65,125 @@ "type": "array", "items": { "$ref": "#/$defs/rule" }, "minItems": 1 + }, + "deny_rules": { + "type": "array", + "items": { "$ref": "#/$defs/denyRule" }, + "minItems": 1 } }, - "if": { - "properties": { "protocol": { "enum": ["rest", "websocket"] } }, - "required": ["protocol"] - }, - "then": { "required": ["rules"] } + "allOf": [ + { + "if": { + "properties": { "protocol": { "enum": ["rest", "websocket"] } }, + "required": ["protocol"] + }, + "then": { + "anyOf": [ + { "required": ["rules"] }, + { "required": ["access"] } + ] + } + }, + { + "if": { + "properties": { "protocol": { "const": "json-rpc" } }, + "required": ["protocol"] + }, + "then": { + "required": ["rules"], + "not": { "required": ["access"] } + } + }, + { + "if": { + "properties": { "protocol": { "const": "mcp" } }, + "required": ["protocol"] + }, + "then": { + "not": { "required": ["access"] }, + "anyOf": [ + { "required": ["rules"] }, + { + "required": ["mcp"], + "properties": { + "mcp": { + "required": ["allow_all_known_mcp_methods"], + "properties": { + "allow_all_known_mcp_methods": { "const": true } + } + } + } + } + ] + } + } + ] }, "rule": { "type": "object", "required": ["allow"], "additionalProperties": false, "properties": { - "allow": { + "allow": { "$ref": "#/$defs/l7Matcher" } + } + }, + "denyRule": { + "$ref": "#/$defs/l7Matcher" + }, + "l7Matcher": { + "type": "object", + "additionalProperties": false, + "properties": { + "method": { "type": "string", "minLength": 1 }, + "path": { "type": "string", "pattern": "^/" }, + "tool": { "$ref": "#/$defs/matcher" }, + "params": { + "type": "object", + "additionalProperties": { "$ref": "#/$defs/paramMatcher" } + } + } + }, + "matcher": { + "oneOf": [ + { "type": "string", "minLength": 1 }, + { "type": "object", - "required": ["method", "path"], "additionalProperties": false, "properties": { - "method": { - "type": "string", - "enum": [ - "GET", - "POST", - "PUT", - "PATCH", - "DELETE", - "HEAD", - "OPTIONS", - "WEBSOCKET_TEXT" - ] - }, - "path": { - "type": "string", - "pattern": "^/" + "any": { + "type": "array", + "items": { "type": "string", "minLength": 1 }, + "minItems": 1 } - } + }, + "required": ["any"] } + ] + }, + "paramMatcher": { + "oneOf": [ + { "$ref": "#/$defs/matcher" }, + { + "type": "object", + "additionalProperties": { "$ref": "#/$defs/paramMatcher" } + } + ] + }, + "jsonRpcOptions": { + "type": "object", + "additionalProperties": false, + "properties": { + "max_body_bytes": { "type": "integer", "minimum": 1, "maximum": 1048576 } + } + }, + "mcpOptions": { + "type": "object", + "additionalProperties": false, + "properties": { + "max_body_bytes": { "type": "integer", "minimum": 1, "maximum": 1048576 }, + "strict_tool_names": { "type": "boolean" }, + "allow_all_known_mcp_methods": { "type": "boolean" } } }, "binary": { diff --git a/schemas/sandbox-policy.schema.json b/schemas/sandbox-policy.schema.json index dcd79ea8169..4bf75276eab 100644 --- a/schemas/sandbox-policy.schema.json +++ b/schemas/sandbox-policy.schema.json @@ -73,10 +73,12 @@ "properties": { "host": { "type": "string" }, "port": { "type": "integer", "minimum": 1, "maximum": 65535 }, - "protocol": { "type": "string", "enum": ["rest", "websocket"] }, + "protocol": { "type": "string", "enum": ["rest", "websocket", "json-rpc", "mcp"] }, "enforcement": { "type": "string", "enum": ["enforce", "audit"] }, "tls": { "type": "string", "enum": ["terminate", "passthrough", "skip"] }, "access": { "type": "string", "enum": ["full"] }, + "json_rpc": { "$ref": "#/$defs/jsonRpcOptions" }, + "mcp": { "$ref": "#/$defs/mcpOptions" }, "websocket_credential_rewrite": { "type": "boolean" }, "request_body_credential_rewrite": { "type": "boolean" }, "allowed_ips": { @@ -88,48 +90,125 @@ "type": "array", "items": { "$ref": "#/$defs/rule" }, "minItems": 1 + }, + "deny_rules": { + "type": "array", + "items": { "$ref": "#/$defs/denyRule" }, + "minItems": 1 } }, - "if": { - "properties": { "protocol": { "enum": ["rest", "websocket"] } }, - "required": ["protocol"] - }, - "then": { - "anyOf": [ - { "required": ["rules"] }, - { "required": ["access"] } - ] - } + "allOf": [ + { + "if": { + "properties": { "protocol": { "enum": ["rest", "websocket"] } }, + "required": ["protocol"] + }, + "then": { + "anyOf": [ + { "required": ["rules"] }, + { "required": ["access"] } + ] + } + }, + { + "if": { + "properties": { "protocol": { "const": "json-rpc" } }, + "required": ["protocol"] + }, + "then": { + "required": ["rules"], + "not": { "required": ["access"] } + } + }, + { + "if": { + "properties": { "protocol": { "const": "mcp" } }, + "required": ["protocol"] + }, + "then": { + "not": { "required": ["access"] }, + "anyOf": [ + { "required": ["rules"] }, + { + "required": ["mcp"], + "properties": { + "mcp": { + "required": ["allow_all_known_mcp_methods"], + "properties": { + "allow_all_known_mcp_methods": { "const": true } + } + } + } + } + ] + } + } + ] }, "rule": { "type": "object", "required": ["allow"], "additionalProperties": false, "properties": { - "allow": { + "allow": { "$ref": "#/$defs/l7Matcher" } + } + }, + "denyRule": { + "$ref": "#/$defs/l7Matcher" + }, + "l7Matcher": { + "type": "object", + "additionalProperties": false, + "properties": { + "method": { "type": "string", "minLength": 1 }, + "path": { "type": "string", "pattern": "^/" }, + "tool": { "$ref": "#/$defs/matcher" }, + "params": { + "type": "object", + "additionalProperties": { "$ref": "#/$defs/paramMatcher" } + } + } + }, + "matcher": { + "oneOf": [ + { "type": "string", "minLength": 1 }, + { "type": "object", - "required": ["method", "path"], "additionalProperties": false, "properties": { - "method": { - "type": "string", - "enum": [ - "GET", - "POST", - "PUT", - "PATCH", - "DELETE", - "HEAD", - "OPTIONS", - "WEBSOCKET_TEXT" - ] - }, - "path": { - "type": "string", - "pattern": "^/" + "any": { + "type": "array", + "items": { "type": "string", "minLength": 1 }, + "minItems": 1 } - } + }, + "required": ["any"] } + ] + }, + "paramMatcher": { + "oneOf": [ + { "$ref": "#/$defs/matcher" }, + { + "type": "object", + "additionalProperties": { "$ref": "#/$defs/paramMatcher" } + } + ] + }, + "jsonRpcOptions": { + "type": "object", + "additionalProperties": false, + "properties": { + "max_body_bytes": { "type": "integer", "minimum": 1, "maximum": 1048576 } + } + }, + "mcpOptions": { + "type": "object", + "additionalProperties": false, + "properties": { + "max_body_bytes": { "type": "integer", "minimum": 1, "maximum": 1048576 }, + "strict_tool_names": { "type": "boolean" }, + "allow_all_known_mcp_methods": { "type": "boolean" } } }, "binary": { diff --git a/scripts/brev-launchable-ci-cpu.sh b/scripts/brev-launchable-ci-cpu.sh index 178b08ad2de..67434e478eb 100755 --- a/scripts/brev-launchable-ci-cpu.sh +++ b/scripts/brev-launchable-ci-cpu.sh @@ -25,11 +25,14 @@ # # Usage (Brev launchable startup script — one-liner that curls this): # curl -fsSL https://raw.githubusercontent.com/NVIDIA/NemoClaw//scripts/brev-launchable-ci-cpu.sh | bash +# bash scripts/brev-launchable-ci-cpu.sh --print-openshell-version # resolve only # # Environment overrides: -# OPENSHELL_VERSION — OpenShell CLI release tag (default: v0.0.72) -# NEMOCLAW_REF — NemoClaw git ref to clone (default: main) -# NEMOCLAW_CLONE_DIR — Where to clone NemoClaw (default: ~/NemoClaw) +# OPENSHELL_VERSION — OpenShell CLI release tag (default: v0.0.72) +# NEMOCLAW_OPENSHELL_CHANNEL — Release channel (stable/dev/auto) +# NEMOCLAW_ACCEPT_DEV_UNVERIFIED_INSTALL — Required opt-in for the unverified dev channel +# NEMOCLAW_REF — NemoClaw git ref to clone (default: main) +# NEMOCLAW_CLONE_DIR — Where to clone NemoClaw (default: ~/NemoClaw) # # Related: # - Epic: https://github.com/NVIDIA/NemoClaw/issues/1326 @@ -38,11 +41,8 @@ set -euo pipefail # ── Configuration ──────────────────────────────────────────────────── -OPENSHELL_VERSION="${OPENSHELL_VERSION:-v0.0.72}" +OPENSHELL_VERSION="${OPENSHELL_VERSION:-}" NEMOCLAW_REF="${NEMOCLAW_REF:-main}" -TARGET_USER="${SUDO_USER:-$(id -un)}" -TARGET_HOME="$(getent passwd "$TARGET_USER" | cut -d: -f6)" -NEMOCLAW_CLONE_DIR="${NEMOCLAW_CLONE_DIR:-${TARGET_HOME}/NemoClaw}" LAUNCH_LOG="${LAUNCH_LOG:-/tmp/launch-plugin.log}" SENTINEL="/var/run/nemoclaw-launchable-ready" @@ -51,7 +51,7 @@ SENTINEL="/var/run/nemoclaw-launchable-ready" export DEBIAN_FRONTEND=noninteractive export NEEDRESTART_MODE=a -# ── Logging ────────────────────────────────────────────────────────── +# Logging mkdir -p "$(dirname "$LAUNCH_LOG")" exec > >(tee -a "$LAUNCH_LOG") 2>&1 @@ -70,11 +70,32 @@ assert_openshell_version() { fi } -assert_openshell_version "$OPENSHELL_VERSION" -if [[ "$OPENSHELL_VERSION" != v* ]]; then - OPENSHELL_VERSION="v${OPENSHELL_VERSION}" +if [ -z "$OPENSHELL_VERSION" ]; then + case "${NEMOCLAW_OPENSHELL_CHANNEL:-stable}" in + dev) OPENSHELL_VERSION="dev" ;; + stable | auto) OPENSHELL_VERSION="v0.0.72" ;; + *) fail "NEMOCLAW_OPENSHELL_CHANNEL must be one of: stable, dev, auto" ;; + esac +fi +if [ "${1:-}" = "--print-openshell-version" ]; then + printf '%s\n' "$OPENSHELL_VERSION" + exit 0 +fi +if [[ "$OPENSHELL_VERSION" = "dev" ]]; then + if [[ "${NEMOCLAW_ACCEPT_DEV_UNVERIFIED_INSTALL:-}" != "1" ]]; then + fail "Dev channel install skips SHA-256 verification. Set NEMOCLAW_ACCEPT_DEV_UNVERIFIED_INSTALL=1 to explicitly accept an unverified OpenShell dev-channel install." + fi + warn "Dev channel install skips SHA-256 verification. Use only in trusted environments." +else + assert_openshell_version "$OPENSHELL_VERSION" + if [[ "$OPENSHELL_VERSION" != v* ]]; then + OPENSHELL_VERSION="v${OPENSHELL_VERSION}" + fi fi OPENSHELL_VERSION_NO_V="${OPENSHELL_VERSION#v}" +TARGET_USER="${SUDO_USER:-$(id -un)}" +TARGET_HOME="$(getent passwd "$TARGET_USER" | cut -d: -f6)" +NEMOCLAW_CLONE_DIR="${NEMOCLAW_CLONE_DIR:-${TARGET_HOME}/NemoClaw}" # ── Retry helper ───────────────────────────────────────────────────── # Usage: retry 3 10 "description" command arg1 arg2 @@ -96,7 +117,7 @@ retry() { done } -# ── Wait for apt locks ─────────────────────────────────────────────── +# Wait for apt locks. # Brev VMs sometimes have unattended-upgrades running at boot. wait_for_apt_lock() { local max_wait=120 elapsed=0 @@ -177,7 +198,9 @@ install_openshell_cli_release() { retry 3 10 "download openshell" \ curl -fsSL -o "$tmpdir/$asset" \ "https://github.com/NVIDIA/OpenShell/releases/download/${OPENSHELL_VERSION}/${asset}" - verify_openshell_cli_asset "$tmpdir" "$asset" + if [[ "$OPENSHELL_VERSION" != "dev" ]]; then + verify_openshell_cli_asset "$tmpdir" "$asset" + fi tar xzf "$tmpdir/$asset" -C "$tmpdir" sudo install -m 755 "$tmpdir/openshell" /usr/local/bin/openshell rm -rf "$tmpdir" @@ -185,7 +208,6 @@ install_openshell_cli_release() { # ══════════════════════════════════════════════════════════════════════ # 1. System packages -# ══════════════════════════════════════════════════════════════════════ # Kill unattended-upgrades immediately — it grabs the apt lock on boot # and can block for 60-120s. Irrelevant on an ephemeral CI VM. sudo systemctl stop unattended-upgrades 2>/dev/null || true @@ -199,9 +221,7 @@ retry 3 10 "apt-get install" sudo apt-get install -y -qq \ ca-certificates curl git jq tar >/dev/null 2>&1 info "System packages installed" -# ══════════════════════════════════════════════════════════════════════ # 2. Docker -# ══════════════════════════════════════════════════════════════════════ if command -v docker >/dev/null 2>&1; then info "Docker already installed" else @@ -218,9 +238,7 @@ sudo usermod -aG docker "$TARGET_USER" 2>/dev/null || true # Docker socket permissions to work around stale group membership. info "Docker enabled ($(docker --version 2>/dev/null | head -c 40))" -# ══════════════════════════════════════════════════════════════════════ # 3. Node.js 22 -# ══════════════════════════════════════════════════════════════════════ node_major="" if command -v node >/dev/null 2>&1; then node_major="$(node -p 'process.versions.node.split(".")[0]' 2>/dev/null || true)" @@ -259,9 +277,7 @@ else info "Node.js $(node --version) installed" fi -# ══════════════════════════════════════════════════════════════════════ # 4. OpenShell CLI -# ══════════════════════════════════════════════════════════════════════ if command -v openshell >/dev/null 2>&1; then _installed_ver="$(openshell --version 2>&1 | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1 || echo '0.0.0')" _pinned_ver="$OPENSHELL_VERSION_NO_V" @@ -278,9 +294,7 @@ else info "OpenShell CLI installed: $(openshell --version 2>&1 || echo unknown)" fi -# ══════════════════════════════════════════════════════════════════════ # 5. Clone NemoClaw and install deps -# ══════════════════════════════════════════════════════════════════════ if [[ -d "$NEMOCLAW_CLONE_DIR/.git" ]]; then info "NemoClaw repo exists at $NEMOCLAW_CLONE_DIR — refreshing" git -C "$NEMOCLAW_CLONE_DIR" fetch origin "$NEMOCLAW_REF" diff --git a/scripts/checks/check-cloudflared-update.sh b/scripts/checks/check-cloudflared-update.sh new file mode 100755 index 00000000000..3f25aa2fc6f --- /dev/null +++ b/scripts/checks/check-cloudflared-update.sh @@ -0,0 +1,145 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# invalidState: the three reviewed E2E consumers drift to different cloudflared +# versions/digests, or their shared pin no longer matches the upstream asset. +# sourceBoundary: Cloudflare owns the release asset; NemoClaw owns all three +# workflow pins and independently verifies the downloaded bytes. +# whyNotSourceFix: upstream cannot enforce which release NemoClaw workflows use. +# regressionTest: cloudflared-update-check-workflow.test.ts covers three-pin +# parity, asset URL identity, digest mismatch, and update instructions. +# removalCondition: remove this checker when the three consumers share one +# machine-readable dependency manifest with equivalent live asset verification. + +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd -- "${SCRIPT_DIR}/../.." && pwd)" +E2E_WORKFLOW="${CLOUDFLARED_E2E_WORKFLOW:-${REPO_ROOT}/.github/workflows/e2e.yaml}" +RELEASE_API_URL="${CLOUDFLARED_RELEASE_API_URL:-https://api.github.com/repos/cloudflare/cloudflared/releases/latest}" +DOWNLOAD_BASE_URL="${CLOUDFLARED_DOWNLOAD_BASE_URL:-https://github.com/cloudflare/cloudflared/releases/download}" +CURL_BIN="${CLOUDFLARED_CURL_BIN:-curl}" +SHA256SUM_BIN="${CLOUDFLARED_SHA256SUM_BIN:-sha256sum}" + +fail() { + printf 'cloudflared update check failed: %s\n' "$*" >&2 + exit 1 +} + +for tool in "${CURL_BIN}" jq "${SHA256SUM_BIN}"; do + command -v "${tool}" >/dev/null 2>&1 || fail "required tool is unavailable: ${tool}" +done +[[ -r "${E2E_WORKFLOW}" ]] || fail "cannot read pin source: ${E2E_WORKFLOW}" + +version_pins=() +while IFS= read -r pin || [[ -n "${pin}" ]]; do + version_pins+=("${pin}") +done < <( + sed -nE 's/^[[:space:]]*CLOUDFLARED_VERSION:[[:space:]]*"([^"]+)".*$/\1/p' \ + "${E2E_WORKFLOW}" +) + +sha_pins=() +while IFS= read -r pin || [[ -n "${pin}" ]]; do + sha_pins+=("${pin}") +done < <( + sed -nE 's/^[[:space:]]*CLOUDFLARED_DEB_SHA256:[[:space:]]*"([0-9a-fA-F]+)".*$/\1/p' \ + "${E2E_WORKFLOW}" +) + +[[ "${#version_pins[@]}" -eq 3 ]] \ + || fail "expected exactly three CLOUDFLARED_VERSION pins in ${E2E_WORKFLOW}; found ${#version_pins[@]}" +[[ "${#sha_pins[@]}" -eq 3 ]] \ + || fail "expected exactly three CLOUDFLARED_DEB_SHA256 pins in ${E2E_WORKFLOW}; found ${#sha_pins[@]}" + +pinned_version="${version_pins[0]}" +pinned_sha="$(printf '%s' "${sha_pins[0]}" | tr '[:upper:]' '[:lower:]')" +for pin in "${version_pins[@]}"; do + [[ "${pin}" == "${pinned_version}" ]] \ + || fail "CLOUDFLARED_VERSION pins diverge in ${E2E_WORKFLOW}: ${version_pins[*]}" +done +for pin in "${sha_pins[@]}"; do + [[ "$(printf '%s' "${pin}" | tr '[:upper:]' '[:lower:]')" == "${pinned_sha}" ]] \ + || fail "CLOUDFLARED_DEB_SHA256 pins diverge in ${E2E_WORKFLOW}: ${sha_pins[*]}" +done +[[ "${pinned_version}" =~ ^[0-9]{4}\.[0-9]{1,2}\.[0-9]+$ ]] \ + || fail "invalid pinned cloudflared version: ${pinned_version}" +[[ "${pinned_sha}" =~ ^[0-9a-f]{64}$ ]] || fail "invalid pinned cloudflared SHA256" + +temp_dir="$(mktemp -d "${RUNNER_TEMP:-${TMPDIR:-/tmp}}/cloudflared-update-check.XXXXXX")" +trap 'rm -rf "${temp_dir}"' EXIT +release_json="${temp_dir}/latest-release.json" +cloudflared_deb="${temp_dir}/cloudflared-linux-amd64.deb" + +"${CURL_BIN}" \ + --fail \ + --silent \ + --show-error \ + --location \ + --retry 3 \ + --retry-all-errors \ + --retry-delay 2 \ + --header 'Accept: application/vnd.github+json' \ + --header 'X-GitHub-Api-Version: 2022-11-28' \ + --header 'User-Agent: NVIDIA-NemoClaw-cloudflared-update-check' \ + --output "${release_json}" \ + "${RELEASE_API_URL}" + +latest_version="$(jq -er '.tag_name | select(type == "string" and length > 0)' "${release_json}")" \ + || fail "latest release response has no tag_name" +asset_url="$( + jq -er 'first(.assets[]? | select(.name == "cloudflared-linux-amd64.deb") | .browser_download_url)' \ + "${release_json}" +)" || fail "latest release ${latest_version} has no cloudflared-linux-amd64.deb asset" + +[[ "${latest_version}" =~ ^[0-9]{4}\.[0-9]{1,2}\.[0-9]+$ ]] \ + || fail "latest release tag has an unexpected format: ${latest_version}" +expected_asset_url="${DOWNLOAD_BASE_URL%/}/${latest_version}/cloudflared-linux-amd64.deb" +[[ "${asset_url}" == "${expected_asset_url}" ]] \ + || fail "latest release returned an unexpected asset URL: ${asset_url}" + +"${CURL_BIN}" \ + --fail \ + --silent \ + --show-error \ + --location \ + --retry 3 \ + --retry-all-errors \ + --retry-delay 2 \ + --output "${cloudflared_deb}" \ + "${asset_url}" + +latest_sha="$("${SHA256SUM_BIN}" "${cloudflared_deb}" | awk '{print tolower($1)}')" +[[ "${latest_sha}" =~ ^[0-9a-f]{64}$ ]] || fail "could not compute the latest asset SHA256" + +version_lines="$(grep -n 'CLOUDFLARED_VERSION:' "${E2E_WORKFLOW}" | cut -d: -f1 | paste -sd, -)" +sha_lines="$(grep -n 'CLOUDFLARED_DEB_SHA256:' "${E2E_WORKFLOW}" | cut -d: -f1 | paste -sd, -)" +workflow_display="${E2E_WORKFLOW#"${REPO_ROOT}/"}" + +print_update_instructions() { + printf '%s\n' \ + 'cloudflared update required.' \ + "Pinned version: ${pinned_version}" \ + "Pinned linux-amd64.deb SHA256: ${pinned_sha}" \ + "Latest version: ${latest_version}" \ + "Latest linux-amd64.deb SHA256: ${latest_sha}" \ + 'Update locations:' \ + " ${workflow_display} CLOUDFLARED_VERSION lines: ${version_lines}" \ + " ${workflow_display} CLOUDFLARED_DEB_SHA256 lines: ${sha_lines}" \ + 'Set all three version/SHA256 pairs to the latest reviewed values, then rerun this check.' >&2 +} + +if [[ "${latest_version}" != "${pinned_version}" ]]; then + print_update_instructions + exit 1 +fi + +if [[ "${latest_sha}" != "${pinned_sha}" ]]; then + printf 'The current cloudflared release asset no longer matches its reviewed SHA256.\n' >&2 + print_update_instructions + exit 1 +fi + +printf '%s %s\n' "${pinned_sha}" "${cloudflared_deb}" | "${SHA256SUM_BIN}" -c - +printf 'cloudflared pin is current: version=%s sha256=%s\n' "${pinned_version}" "${pinned_sha}" diff --git a/scripts/checks/openshell-policy-mutation-read.ts b/scripts/checks/openshell-policy-mutation-read.ts index cae4139c5bf..31f2d37e35b 100644 --- a/scripts/checks/openshell-policy-mutation-read.ts +++ b/scripts/checks/openshell-policy-mutation-read.ts @@ -36,7 +36,7 @@ interface AuditedMutationRead { export const MUTATION_READS: readonly AuditedMutationRead[] = [ { relativePath: "src/lib/policy/index.ts", - expectedReadCalls: 4, + expectedReadCalls: 5, baseCommand: "runCapture(buildPolicyGetCommand(sandboxName))", unsafeBaseCommand: "runCapture(buildPolicyGetCommand(sandboxName), { ignoreError: true })", fullCommand: "runCapture(buildPolicyGetFullCommand(sandboxName), { ignoreError: true })", diff --git a/scripts/generate-openclaw-config.mts b/scripts/generate-openclaw-config.mts index c8a184275cf..a9876acaeda 100755 --- a/scripts/generate-openclaw-config.mts +++ b/scripts/generate-openclaw-config.mts @@ -1170,28 +1170,8 @@ export function buildConfig(env: Env = process.env): JsonObject { }; const pluginEntries: JsonObject = { - acpx: { enabled: false }, bonjour: { enabled: false }, - qqbot: { enabled: false }, }; - const bundledProviderPlugins: Record> = { - "amazon-bedrock": new Set(["amazon-bedrock", "bedrock"]), - "amazon-bedrock-mantle": new Set(["amazon-bedrock-mantle"]), - anthropic: new Set(["anthropic"]), - "anthropic-vertex": new Set(["anthropic-vertex"]), - fireworks: new Set(["fireworks"]), - google: new Set(["google", "google-gemini-cli"]), - kimi: new Set(["kimi"]), - lmstudio: new Set(["lmstudio"]), - ollama: new Set(["ollama", "ollama-local"]), - openai: new Set(["openai"]), - xai: new Set(["xai"]), - }; - for (const [pluginId, providerKeys] of Object.entries(bundledProviderPlugins)) { - if (!providerKeys.has(providerKey)) { - pluginEntries[pluginId] = { enabled: false }; - } - } const openclawOtel = buildOpenClawOtelConfig(env); if (openclawOtel) { pluginEntries["diagnostics-otel"] = { enabled: true }; diff --git a/scripts/install-openshell.sh b/scripts/install-openshell.sh index ecb81c1c932..9d0c07669f2 100755 --- a/scripts/install-openshell.sh +++ b/scripts/install-openshell.sh @@ -35,8 +35,8 @@ info "Detected $OS_LABEL ($ARCH_LABEL)" # Minimum version required for native messaging credential rewrite and # round-trippable base policies: WebSocket text frames, provider-shaped -# aliases, REST request bodies, and `policy get --base` for MCP/JSON-RPC-safe -# read-modify-write operations. +# aliases, REST request bodies, MCP/JSON-RPC L7 enforcement, and +# `policy get --base` for MCP/JSON-RPC-safe read-modify-write operations. MIN_VERSION="0.0.72" # Maximum version validated for this NemoClaw release. Newer OpenShell builds # may change sandbox semantics; upgrade NemoClaw before upgrading past this. @@ -54,6 +54,12 @@ case "$CHANNEL" in *) fail "NEMOCLAW_OPENSHELL_CHANNEL must be one of: stable, dev, auto" ;; esac +FORCE_INSTALL="${NEMOCLAW_OPENSHELL_FORCE_INSTALL:-0}" +case "$FORCE_INSTALL" in + 0 | 1) ;; + *) fail "NEMOCLAW_OPENSHELL_FORCE_INSTALL must be 0 or 1." ;; +esac + if [ "$CHANNEL" = "auto" ]; then RESOLVED_CHANNEL="stable" else @@ -61,6 +67,15 @@ else fi if [ "$RESOLVED_CHANNEL" = "dev" ]; then + # invalidState: a mutable dev artifact is consumed as if it were a verified + # stable release. sourceBoundary: OpenShell owns the moving dev tag; NemoClaw + # owns this explicit compatibility-only opt-in. whyNotSourceFix: NemoClaw + # cannot make that upstream tag immutable. regressionTest: + # test/install-openshell-version-check.test.ts covers rejection without the + # opt-in and acceptance with it. removalCondition: remove this path when dev + # compatibility testing ends or OpenShell publishes an independently + # verifiable immutable development channel. See the v0.0.72 compatibility + # review's "Dev Channel Opt-In" section. if [ "${NEMOCLAW_ACCEPT_DEV_UNVERIFIED_INSTALL:-}" != "1" ]; then fail "Dev channel install skips SHA-256 verification. Set NEMOCLAW_ACCEPT_DEV_UNVERIFIED_INSTALL=1 to explicitly accept an unverified OpenShell dev-channel install." fi @@ -174,13 +189,169 @@ version_gte() { return 0 } +installed_component_path() { + local openshell_bin="$1" + local component_name="$2" + local explicit_path="${3:-}" + if [ -n "$explicit_path" ]; then + printf '%s\n' "$explicit_path" + else + printf '%s/%s\n' "$(dirname "$openshell_bin")" "$component_name" + fi +} + +selected_sandbox_component_path() { + local openshell_bin="$1" + local explicit_path="${NEMOCLAW_OPENSHELL_SANDBOX_BIN:-}" + # Darwin uses the VM driver and ships no standalone sandbox supervisor. + # Ignore a leftover sibling unless the operator explicitly selected it. + if [ "$OS" = "Darwin" ] && [ -z "$explicit_path" ]; then + return 0 + fi + installed_component_path "$openshell_bin" openshell-sandbox "$explicit_path" +} + +canonical_file_path() { + local target="$1" + local link dir + local iterations=0 + [ -n "$target" ] || return 1 + case "$target" in + /*) ;; + *) target="$PWD/$target" ;; + esac + while [ -L "$target" ]; do + iterations=$((iterations + 1)) + [ "$iterations" -le 40 ] || return 1 + link="$(readlink "$target")" || return 1 + dir="$(cd -P "$(dirname "$target")" 2>/dev/null && pwd)" || return 1 + case "$link" in + /*) target="$link" ;; + *) target="$dir/$link" ;; + esac + done + dir="$(cd -P "$(dirname "$target")" 2>/dev/null && pwd)" || return 1 + printf '%s/%s\n' "$dir" "$(basename "$target")" +} + +component_shares_install_root() { + local openshell_bin="$1" + local component_bin="$2" + local canonical_openshell canonical_component + canonical_openshell="$(canonical_file_path "$openshell_bin")" || return 1 + canonical_component="$(canonical_file_path "$component_bin")" || return 1 + [ "$(dirname "$canonical_openshell")" = "$(dirname "$canonical_component")" ] +} + +file_sha256() { + local component_bin="$1" + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$component_bin" | awk '{print $1}' + elif command -v shasum >/dev/null 2>&1; then + shasum -a 256 "$component_bin" | awk '{print $1}' + else + return 1 + fi +} + +pinned_sandbox_build_version() { + local digest="$1" + case "$digest" in + # OpenShell v0.0.72 standalone sandbox binaries. These are bind-mounted + # into the supervisor container and can require a newer glibc than the + # host that runs the CLI/gateway, so `--version` is not always runnable. + f9f991a24d10772ad5d24ae27a8ea6baad8cac671695bd90fcd0355e0e0ad198 | \ + 32ca44fe7d9e6d332f2a753c6b8a1a6117b7388281dad9b5274d23ffc67e216f) + printf '%s\n' "0.0.72" + ;; + *) + return 1 + ;; + esac +} + +component_build_version() { + local component_bin="$1" + local component_role="${2:-component}" + local version_output version digest + if version_output="$("$component_bin" --version 2>/dev/null)"; then + version="$(printf '%s\n' "$version_output" \ + | grep -oE '[0-9]+\.[0-9]+\.[0-9]+[^[:space:]]*' \ + | head -1)" + if [ -n "$version" ]; then + printf '%s\n' "$version" + return 0 + fi + fi + + # Do not infer an identity from arbitrary embedded version strings. Only the + # exact pinned sandbox release artifacts may fall back when the host loader + # cannot execute their version probe (for example, GLIBC_2.39 on Brev). + [ "$component_role" = "sandbox" ] || return 1 + digest="$(file_sha256 "$component_bin")" || return 1 + pinned_sandbox_build_version "$digest" +} + +component_build_versions_match() { + local left="$1" + local right="$2" + local left_prefix right_prefix left_hash right_hash + [ "$left" = "$right" ] && return 0 + case "$left:$right" in + *+g*:*+g*) ;; + *) return 1 ;; + esac + left_prefix="${left%+g*}" + right_prefix="${right%+g*}" + left_hash="${left##*+g}" + right_hash="${right##*+g}" + [ "$left_prefix" = "$right_prefix" ] || return 1 + [[ "$left_hash" =~ ^[0-9a-fA-F]{7,}$ ]] || return 1 + [[ "$right_hash" =~ ^[0-9a-fA-F]{7,}$ ]] || return 1 + case "$left_hash" in "$right_hash"*) return 0 ;; esac + case "$right_hash" in "$left_hash"*) return 0 ;; esac + return 1 +} + +component_matches_cli_build() { + local openshell_bin="$1" + local component_bin="$2" + local component_role="${3:-component}" + local openshell_version component_version + openshell_version="$(component_build_version "$openshell_bin" cli)" + component_version="$(component_build_version "$component_bin" "$component_role")" + [ -n "$openshell_version" ] && [ -n "$component_version" ] \ + && component_build_versions_match "$openshell_version" "$component_version" +} + required_driver_bins_present() { + local openshell_bin="${1:-$(command -v openshell 2>/dev/null || true)}" + local gateway_bin sandbox_bin + [ -n "$openshell_bin" ] || return 1 + gateway_bin="$(installed_component_path "$openshell_bin" openshell-gateway "${NEMOCLAW_OPENSHELL_GATEWAY_BIN:-}")" + sandbox_bin="$(selected_sandbox_component_path "$openshell_bin")" + case "$OS" in + Linux) + [ -f "$gateway_bin" ] && [ -x "$gateway_bin" ] \ + && [ -f "$sandbox_bin" ] && [ -x "$sandbox_bin" ] + ;; + Darwin) + [ -f "$gateway_bin" ] && [ -x "$gateway_bin" ] + ;; + *) + return 0 + ;; + esac +} + +required_driver_bins_installed_in_dir() { + local dir="$1" case "$OS" in Linux) - command -v openshell-gateway >/dev/null 2>&1 && command -v openshell-sandbox >/dev/null 2>&1 + [ -x "$dir/openshell-gateway" ] && [ -x "$dir/openshell-sandbox" ] ;; Darwin) - command -v openshell-gateway >/dev/null 2>&1 + [ -x "$dir/openshell-gateway" ] ;; *) return 0 @@ -189,9 +360,43 @@ required_driver_bins_present() { } OPENSHELL_FEATURE_CHECK_ERROR="" +OPENSHELL_SANDBOX_MCP_FEATURE="allow_all_known_mcp_methods" + +openshell_required_feature_strings() { + local openshell_bin="$1" + local gateway_bin sandbox_bin candidate seen candidate_strings binary_strings + local -a candidates + + gateway_bin="$(installed_component_path "$openshell_bin" openshell-gateway "${NEMOCLAW_OPENSHELL_GATEWAY_BIN:-}")" + sandbox_bin="$(selected_sandbox_component_path "$openshell_bin")" + # Treat the CLI and its sibling release artifacts as one install. Arbitrary + # PATH hits must not be combined into a synthetic capability set. Advanced + # cross-prefix layouts remain available only through the explicit overrides. + candidates=("$openshell_bin" "$gateway_bin" "$sandbox_bin") + + seen=":" + binary_strings="" + for candidate in "${candidates[@]}"; do + [ -n "$candidate" ] || continue + [ -f "$candidate" ] || continue + case "$seen" in + *":$candidate:"*) continue ;; + esac + seen="${seen}${candidate}:" + candidate_strings="$(strings "$candidate" 2>/dev/null)" || return 1 + binary_strings="${binary_strings} +${candidate_strings}" + if [[ "$binary_strings" == *"request-body-credential-rewrite"* ]] \ + && [[ "$binary_strings" == *"websocket-credential-rewrite"* ]] \ + && [[ "$binary_strings" == *"$OPENSHELL_SANDBOX_MCP_FEATURE"* ]]; then + break + fi + done + printf '%s\n' "$binary_strings" +} openshell_has_required_messaging_features() { - local openshell_bin + local openshell_bin gateway_bin sandbox_bin sandbox_strings OPENSHELL_FEATURE_CHECK_ERROR="" openshell_bin="${1:-$(command -v openshell 2>/dev/null || true)}" if [ -z "$openshell_bin" ]; then @@ -202,23 +407,100 @@ openshell_has_required_messaging_features() { OPENSHELL_FEATURE_CHECK_ERROR="'strings' is required to verify OpenShell messaging credential rewrite support. Install binutils or an equivalent package and retry." return 2 fi + gateway_bin="$(installed_component_path "$openshell_bin" openshell-gateway "${NEMOCLAW_OPENSHELL_GATEWAY_BIN:-}")" + sandbox_bin="$(selected_sandbox_component_path "$openshell_bin")" + if [ ! -f "$openshell_bin" ] || [ ! -r "$openshell_bin" ] || [ ! -x "$openshell_bin" ]; then + OPENSHELL_FEATURE_CHECK_ERROR="The selected OpenShell CLI '$openshell_bin' is not a readable executable regular file." + return 1 + fi + if [ -n "${NEMOCLAW_OPENSHELL_GATEWAY_BIN:-}" ] \ + && { [ ! -f "$gateway_bin" ] || [ ! -r "$gateway_bin" ] || [ ! -x "$gateway_bin" ]; }; then + OPENSHELL_FEATURE_CHECK_ERROR="The explicit OpenShell gateway binary '$gateway_bin' is missing, unreadable, or not executable." + return 1 + fi + if [ -n "${NEMOCLAW_OPENSHELL_SANDBOX_BIN:-}" ] \ + && { [ ! -f "$sandbox_bin" ] || [ ! -r "$sandbox_bin" ] || [ ! -x "$sandbox_bin" ]; }; then + OPENSHELL_FEATURE_CHECK_ERROR="The explicit OpenShell sandbox binary '$sandbox_bin' is missing, unreadable, or not executable." + return 1 + fi + if [ -f "$gateway_bin" ] && { [ ! -r "$gateway_bin" ] || [ ! -x "$gateway_bin" ]; }; then + OPENSHELL_FEATURE_CHECK_ERROR="The selected OpenShell gateway is not readable and executable." + return 1 + fi + if [ -f "$sandbox_bin" ] && { [ ! -r "$sandbox_bin" ] || [ ! -x "$sandbox_bin" ]; }; then + OPENSHELL_FEATURE_CHECK_ERROR="The selected OpenShell sandbox is not readable and executable." + return 1 + fi + if [ -z "${NEMOCLAW_OPENSHELL_GATEWAY_BIN:-}" ] && [ -f "$gateway_bin" ] \ + && ! component_shares_install_root "$openshell_bin" "$gateway_bin"; then + OPENSHELL_FEATURE_CHECK_ERROR="The selected OpenShell gateway resolves outside the active CLI install root. Use an explicit component override for a deliberate cross-prefix layout." + return 1 + fi + if [ -z "${NEMOCLAW_OPENSHELL_SANDBOX_BIN:-}" ] && [ -f "$sandbox_bin" ] \ + && ! component_shares_install_root "$openshell_bin" "$sandbox_bin"; then + OPENSHELL_FEATURE_CHECK_ERROR="The selected OpenShell sandbox resolves outside the active CLI install root. Use an explicit component override for a deliberate cross-prefix layout." + return 1 + fi + if [ -f "$gateway_bin" ] && ! component_matches_cli_build "$openshell_bin" "$gateway_bin" gateway; then + OPENSHELL_FEATURE_CHECK_ERROR="The selected OpenShell gateway does not match the active CLI build. Install one coherent OpenShell release." + return 1 + fi + if [ -f "$sandbox_bin" ] && ! component_matches_cli_build "$openshell_bin" "$sandbox_bin" sandbox; then + OPENSHELL_FEATURE_CHECK_ERROR="The selected OpenShell sandbox does not match the active CLI build. Install one coherent OpenShell release." + return 1 + fi - # Keep this independent of a live gateway. `policy update --dry-run` still - # needs gateway metadata, but the CLI binary must contain the endpoint-option - # parser for request-body/WebSocket rewrite support released in OpenShell 0.0.39. + # OpenShell #1865 has no authoritative CLI/RPC capability query yet. Scan the + # release-coherent binary set selected beside the CLI (or by explicit + # component overrides) and fail closed; replace this when that API exists. + # Version alone is insufficient for moving dev builds. local binary_strings - binary_strings="$(strings "$openshell_bin" 2>/dev/null || true)" + if ! binary_strings="$(openshell_required_feature_strings "$openshell_bin")"; then + OPENSHELL_FEATURE_CHECK_ERROR="OpenShell selected binaries could not be read for capability verification." + return 1 + fi if [[ "$binary_strings" != *"request-body-credential-rewrite"* ]]; then - OPENSHELL_FEATURE_CHECK_ERROR="OpenShell binary is missing request-body-credential-rewrite support." + OPENSHELL_FEATURE_CHECK_ERROR="OpenShell installed binaries are missing request-body-credential-rewrite support." return 1 fi if [[ "$binary_strings" != *"websocket-credential-rewrite"* ]]; then - OPENSHELL_FEATURE_CHECK_ERROR="OpenShell binary is missing websocket-credential-rewrite support." + OPENSHELL_FEATURE_CHECK_ERROR="OpenShell installed binaries are missing websocket-credential-rewrite support." + return 1 + fi + if [[ "$binary_strings" != *"$OPENSHELL_SANDBOX_MCP_FEATURE"* ]]; then + OPENSHELL_FEATURE_CHECK_ERROR="OpenShell installed binaries are missing MCP/JSON-RPC L7 policy support." + return 1 + fi + + # MCP policy enforcement and credential replacement execute in + # openshell-sandbox. When that host artifact is present, require the native + # MCP policy marker from that exact binary. + if [ -z "$sandbox_bin" ] || [ ! -f "$sandbox_bin" ]; then + # VM drivers embed a compressed supervisor, so scanning the host driver is + # not authoritative. Docker/VM packaging can also keep the supervisor out + # of the host filesystem entirely. + # The MCP lifecycle's authoritative runtime check loads the exact generated + # protocol:mcp policy with --wait and exact-matches the effective state + # before it creates or updates any credential provider. + return 0 + fi + sandbox_strings="$(strings "$sandbox_bin" 2>/dev/null || true)" + if [[ "$sandbox_strings" != *"$OPENSHELL_SANDBOX_MCP_FEATURE"* ]]; then + OPENSHELL_FEATURE_CHECK_ERROR="OpenShell sandbox runtime is missing MCP/JSON-RPC L7 policy support." return 1 fi return 0 } +validate_explicit_component_override() { + local component_name="$1" + local component_path="$2" + [ -n "$component_path" ] || return 0 + if [ ! -f "$component_path" ] || [ ! -r "$component_path" ] || [ ! -x "$component_path" ]; then + fail "The explicit OpenShell $component_name binary '$component_path' is missing, unreadable, or not executable." + fi +} + require_openshell_messaging_features() { local openshell_bin="$1" openshell_has_required_messaging_features "$openshell_bin" \ @@ -305,6 +587,9 @@ repair_existing_macos_vm_driver() { return 1 } +validate_explicit_component_override gateway "${NEMOCLAW_OPENSHELL_GATEWAY_BIN:-}" +validate_explicit_component_override sandbox "${NEMOCLAW_OPENSHELL_SANDBOX_BIN:-}" + ACTIVE_OPENSHELL_BIN="" if command -v openshell >/dev/null 2>&1; then ACTIVE_OPENSHELL_BIN="$(command -v openshell 2>/dev/null || true)" @@ -314,9 +599,12 @@ if command -v openshell >/dev/null 2>&1; then if [ "$RESOLVED_CHANNEL" = "dev" ]; then if version_gte "$INSTALLED_VERSION" "$DEV_MIN_VERSION" \ && printf '%s\n' "$INSTALLED_VERSION_OUTPUT" | grep -qi 'dev'; then - if openshell_has_required_messaging_features; then - info "openshell already installed: $INSTALLED_VERSION_OUTPUT (dev channel)" - exit 0 + if required_driver_bins_present "$ACTIVE_OPENSHELL_BIN" && openshell_has_required_messaging_features "$ACTIVE_OPENSHELL_BIN"; then + if [ "$FORCE_INSTALL" != "1" ]; then + info "openshell already installed: $INSTALLED_VERSION_OUTPUT (dev channel)" + exit 0 + fi + warn "Current OpenShell dev build requested — refreshing the moving dev release instead of reusing the installed binary." else feature_status=$? if [ "$feature_status" = "2" ]; then @@ -324,17 +612,19 @@ if command -v openshell >/dev/null 2>&1; then fi fi fi - warn "openshell $INSTALLED_VERSION is not the required dev-channel messaging-rewrite build — upgrading..." + if [ "$FORCE_INSTALL" != "1" ]; then + warn "openshell $INSTALLED_VERSION is not the required dev-channel messaging-rewrite/MCP-L7 build — upgrading..." + fi else if version_gte "$INSTALLED_VERSION" "$MIN_VERSION"; then if ! version_gte "$MAX_VERSION" "$INSTALLED_VERSION"; then warn "openshell $INSTALLED_VERSION is above the maximum ($MAX_VERSION) supported by this NemoClaw release — reinstalling pinned OpenShell ${PIN_VERSION}..." - elif ! required_driver_bins_present; then + elif ! required_driver_bins_present "$ACTIVE_OPENSHELL_BIN"; then warn "openshell $INSTALLED_VERSION is missing Docker-driver binaries — reinstalling pinned OpenShell ${PIN_VERSION}..." - elif ! openshell_has_required_messaging_features; then - fail "${OPENSHELL_FEATURE_CHECK_ERROR:-openshell $INSTALLED_VERSION is missing required messaging credential rewrite support. Install an OpenShell build that includes provider aliases, WebSocket text rewrite, and request-body credential rewrite.}" + elif ! openshell_has_required_messaging_features "$ACTIVE_OPENSHELL_BIN"; then + fail "${OPENSHELL_FEATURE_CHECK_ERROR:-openshell $INSTALLED_VERSION is missing required messaging credential rewrite and MCP L7 policy support. Install an OpenShell build that includes provider aliases, WebSocket text rewrite, request-body credential rewrite, and MCP/JSON-RPC L7 policy enforcement.}" else - info "openshell already installed: $INSTALLED_VERSION (>= $MIN_VERSION, <= $MAX_VERSION, messaging rewrite and policy --base capable)" + info "openshell already installed: $INSTALLED_VERSION (>= $MIN_VERSION, <= $MAX_VERSION, messaging rewrite, MCP L7, and policy --base capable)" exit 0 fi else @@ -393,6 +683,16 @@ esac tmpdir="$(mktemp -d)" trap 'rm -rf "$tmpdir"' EXIT +select_sha_cmd() { + if command -v sha256sum >/dev/null 2>&1; then + SHA_CMD="sha256sum" + elif command -v shasum >/dev/null 2>&1; then + SHA_CMD="shasum -a 256" + else + fail "No SHA-256 tool available (sha256sum/shasum)" + fi +} + download_with_curl() { local name local -a curl_progress @@ -431,13 +731,7 @@ else fi info "Verifying SHA-256 checksum..." -if command -v sha256sum >/dev/null 2>&1; then - SHA_CMD="sha256sum" -elif command -v shasum >/dev/null 2>&1; then - SHA_CMD="shasum -a 256" -else - fail "No SHA-256 tool available (sha256sum/shasum)" -fi +select_sha_cmd for i in "${!ASSETS[@]}"; do asset_name="${ASSETS[$i]}" checksum_file="${CHECKSUM_FILES[$i]}" @@ -504,6 +798,8 @@ else fi fi +required_driver_bins_installed_in_dir "$target_dir" \ + || fail "OpenShell release '$RELEASE_TAG' did not install the required Docker-driver binaries." require_openshell_messaging_features "$target_dir/openshell" info "$("$target_dir/openshell" --version 2>&1 || echo openshell) installed" diff --git a/scripts/update-hermes-agent.sh b/scripts/update-hermes-agent.sh index a422eeb5128..60f6c90a6c3 100755 --- a/scripts/update-hermes-agent.sh +++ b/scripts/update-hermes-agent.sh @@ -94,16 +94,6 @@ for tool in curl python3 npm sha256sum tar sed realpath; do } done -image_ref_without_tag() { - local ref="$1" - local basename="${ref##*/}" - if [[ "$basename" == *:* ]]; then - printf '%s\n' "${ref%:*}" - return - fi - printf '%s\n' "$ref" -} - gh_api() { local url="$1" local -a auth=() @@ -208,6 +198,8 @@ installed_copy_schema_error() { for item in \ "validate-hermes-env-secret-boundary.py" \ "seed-hermes-dashboard-config.py" \ + "hermes-mcp-config-transaction.py" \ + "openshell-child-visible-credentials.v0.0.72.json" \ "HERMES_HOME=/sandbox/.hermes /usr/local/bin/hermes doctor --fix" \ "node --experimental-strip-types /opt/nemoclaw-hermes-config/generate-config.ts" \ "/sandbox/.hermes/dashboard-home"; do @@ -449,9 +441,8 @@ if [[ "$DO_REBUILD" == 1 ]]; then # locally built images have no registry digest to pin to — the ID-derived # tag guarantees the rebuild uses exactly the image built above. base_image_id="$(docker image inspect -f '{{.Id}}' "$BASE_REF")" - base_image_id_short="${base_image_id#sha256:}" - base_image_id_short="${base_image_id_short:0:12}" - pin_tag="$(image_ref_without_tag "$BASE_REF"):${TAG#v}-${base_image_id_short}" + base_image_id_hex="${base_image_id#sha256:}" + pin_tag="nemoclaw-hermes-sandbox-base-local:image-${base_image_id_hex}" docker tag "$BASE_REF" "$pin_tag" echo "" echo "Rebuilding sandbox against ${pin_tag} (image ID ${base_image_id})…" diff --git a/src/commands/sandbox/mcp.ts b/src/commands/sandbox/mcp.ts new file mode 100644 index 00000000000..14464f945f2 --- /dev/null +++ b/src/commands/sandbox/mcp.ts @@ -0,0 +1,38 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { dispatchMcpBridgeCommand } from "../../lib/actions/sandbox/mcp-bridge"; +import { NemoClawCommand } from "../../lib/cli/nemoclaw-oclif-command"; + +export default class SandboxMcpCommand extends NemoClawCommand { + static id = "sandbox:mcp"; + static strict = false; + static summary = "Manage MCP servers for a sandbox"; + static description = + "Manage OpenShell-enforced MCP Streamable HTTP servers for a sandbox. Credentials are registered as OpenShell providers and appear in sandbox config only as openshell:resolve:env placeholders."; + static usage = [" [args...]"]; + static examples = [ + "<%= config.bin %> sandbox mcp alpha list", + "<%= config.bin %> sandbox mcp alpha add github --url https://api.githubcopilot.com/mcp/ --env GITHUB_MCP_TOKEN", + "<%= config.bin %> sandbox mcp alpha status github --json", + "<%= config.bin %> sandbox mcp alpha remove github", + ]; + + public async run(): Promise { + this.parsed = true; + const [sandboxName, ...actionArgs] = this.argv; + if ( + !sandboxName || + sandboxName.trim() === "" || + sandboxName === "--help" || + sandboxName === "-h" + ) { + this.failWithLines( + ["Usage: nemoclaw mcp [args...]"], + 2, + ); + return; + } + await dispatchMcpBridgeCommand(sandboxName, actionArgs); + } +} diff --git a/src/commands/sandbox/oclif-command-adapters.test.ts b/src/commands/sandbox/oclif-command-adapters.test.ts index 71887f011ed..230b43a3bc8 100644 --- a/src/commands/sandbox/oclif-command-adapters.test.ts +++ b/src/commands/sandbox/oclif-command-adapters.test.ts @@ -260,8 +260,9 @@ describe("sandbox oclif command adapters", () => { timeout: "5m", reason: "debugging", policy: "permissive", + throwOnError: true, }); - expect(mocks.shieldsUp).toHaveBeenCalledWith("alpha"); + expect(mocks.shieldsUp).toHaveBeenCalledWith("alpha", { throwOnError: true }); expect(mocks.shieldsStatus).toHaveBeenCalledWith("alpha"); }); diff --git a/src/commands/sandbox/shields/down.ts b/src/commands/sandbox/shields/down.ts index 97ed6ecfc2b..6a29085fd04 100644 --- a/src/commands/sandbox/shields/down.ts +++ b/src/commands/sandbox/shields/down.ts @@ -2,11 +2,11 @@ // SPDX-License-Identifier: Apache-2.0 import { Flags } from "@oclif/core"; -import { NemoClawCommand } from "../../../lib/cli/nemoclaw-oclif-command"; - import { shieldsTimeoutDurationFlag } from "../../../lib/cli/duration-flags"; -import * as shields from "../../../lib/shields/index"; +import { NemoClawCommand } from "../../../lib/cli/nemoclaw-oclif-command"; import { sandboxNameArg } from "../../../lib/sandbox/command-support"; +import * as shields from "../../../lib/shields/index"; +import { withSandboxMutationLock } from "../../../lib/state/mcp-lifecycle-lock"; export default class ShieldsDownCommand extends NemoClawCommand { static id = "sandbox:shields:down"; @@ -24,10 +24,13 @@ export default class ShieldsDownCommand extends NemoClawCommand { public async run(): Promise { const { args, flags } = await this.parse(ShieldsDownCommand); - shields.shieldsDown(args.sandboxName, { - timeout: flags.timeout ?? null, - reason: flags.reason ?? null, - policy: flags.policy ?? "permissive", - }); + await withSandboxMutationLock(args.sandboxName, () => + shields.shieldsDown(args.sandboxName, { + timeout: flags.timeout ?? null, + reason: flags.reason ?? null, + policy: flags.policy ?? "permissive", + throwOnError: true, + }), + ); } } diff --git a/src/commands/sandbox/shields/status.ts b/src/commands/sandbox/shields/status.ts index a998da575cc..62ea0f8b490 100644 --- a/src/commands/sandbox/shields/status.ts +++ b/src/commands/sandbox/shields/status.ts @@ -2,9 +2,9 @@ // SPDX-License-Identifier: Apache-2.0 import { NemoClawCommand } from "../../../lib/cli/nemoclaw-oclif-command"; - -import * as shields from "../../../lib/shields/index"; import { sandboxNameArg } from "../../../lib/sandbox/command-support"; +import * as shields from "../../../lib/shields/index"; +import { withSandboxMutationLock } from "../../../lib/state/mcp-lifecycle-lock"; export default class ShieldsStatusCommand extends NemoClawCommand { static id = "sandbox:shields:status"; @@ -18,6 +18,6 @@ export default class ShieldsStatusCommand extends NemoClawCommand { public async run(): Promise { const { args } = await this.parse(ShieldsStatusCommand); - shields.shieldsStatus(args.sandboxName); + await withSandboxMutationLock(args.sandboxName, () => shields.shieldsStatus(args.sandboxName)); } } diff --git a/src/commands/sandbox/shields/up.ts b/src/commands/sandbox/shields/up.ts index d4235250297..f4cf4111978 100644 --- a/src/commands/sandbox/shields/up.ts +++ b/src/commands/sandbox/shields/up.ts @@ -2,9 +2,9 @@ // SPDX-License-Identifier: Apache-2.0 import { NemoClawCommand } from "../../../lib/cli/nemoclaw-oclif-command"; - -import * as shields from "../../../lib/shields/index"; import { sandboxNameArg } from "../../../lib/sandbox/command-support"; +import * as shields from "../../../lib/shields/index"; +import { withSandboxMutationLock } from "../../../lib/state/mcp-lifecycle-lock"; export default class ShieldsUpCommand extends NemoClawCommand { static id = "sandbox:shields:up"; @@ -18,6 +18,8 @@ export default class ShieldsUpCommand extends NemoClawCommand { public async run(): Promise { const { args } = await this.parse(ShieldsUpCommand); - shields.shieldsUp(args.sandboxName); + await withSandboxMutationLock(args.sandboxName, () => + shields.shieldsUp(args.sandboxName, { throwOnError: true }), + ); } } diff --git a/src/lib/actions/credentials-add.ts b/src/lib/actions/credentials-add.ts index ed3ed7b15cd..dffb95ad74c 100644 --- a/src/lib/actions/credentials-add.ts +++ b/src/lib/actions/credentials-add.ts @@ -180,6 +180,7 @@ export async function runCredentialsAddAction( } const result = runOpenshellProviderCommand(openshellArgs, { + env: Object.fromEntries(credentials.map((credential) => [credential, process.env[credential]])), ignoreError: true, stdio: ["ignore", "pipe", "pipe"], timeout: OPENSHELL_OPERATION_TIMEOUT_MS, diff --git a/src/lib/actions/gateway-drift-preflight.test.ts b/src/lib/actions/gateway-drift-preflight.test.ts index 988ee7a9fc5..0fd39b81d6f 100644 --- a/src/lib/actions/gateway-drift-preflight.test.ts +++ b/src/lib/actions/gateway-drift-preflight.test.ts @@ -5,6 +5,7 @@ import { createRequire } from "node:module"; import { afterEach, beforeEach, describe, expect, it, type MockInstance, vi } from "vitest"; +import { testTimeout } from "../../../test/helpers/timeouts"; import type { OpenShellStateRpcIssue } from "../adapters/openshell/gateway-drift"; type BackupAll = typeof import("./maintenance")["backupAll"]; @@ -114,7 +115,7 @@ describe("gateway drift preflight for maintenance actions", () => { ({ backupAll } = requireDist("./maintenance.js")); ({ upgradeSandboxes } = requireDist("./upgrade-sandboxes.js")); - }); + }, testTimeout(30_000)); afterEach(() => { for (const spy of spies) spy.mockRestore(); diff --git a/src/lib/actions/global.test.ts b/src/lib/actions/global.test.ts index d9afe7f0046..60bb8717694 100644 --- a/src/lib/actions/global.test.ts +++ b/src/lib/actions/global.test.ts @@ -78,7 +78,10 @@ describe("global cli action facade", () => { await runUpgradeSandboxesAction({ check: true }); expect(recoverHook).toHaveBeenCalledWith(); - expect(runOpenshellHook).toHaveBeenCalledWith(["provider", "list"], { timeout: 100 }); + expect(runOpenshellHook).toHaveBeenCalledWith( + ["provider", "list"], + expect.objectContaining({ timeout: 100, replaceEnv: true, env: expect.any(Object) }), + ); expect(upgradeHook).toHaveBeenCalledWith({ check: true }); }); @@ -87,6 +90,9 @@ describe("global cli action facade", () => { runOpenshellProviderCommand(["provider", "list"]); expect(mocks.recoverNamedGatewayRuntime).toHaveBeenCalledWith(); - expect(mocks.runOpenshell).toHaveBeenCalledWith(["provider", "list"], undefined); + expect(mocks.runOpenshell).toHaveBeenCalledWith( + ["provider", "list"], + expect.objectContaining({ replaceEnv: true, env: expect.any(Object) }), + ); }); }); diff --git a/src/lib/actions/global.ts b/src/lib/actions/global.ts index 54ae387443a..236e676b80d 100644 --- a/src/lib/actions/global.ts +++ b/src/lib/actions/global.ts @@ -8,6 +8,7 @@ import { } from "../domain/lifecycle/options"; import { recoverNamedGatewayRuntime as recoverNamedGatewayRuntimeAction } from "../gateway-runtime-action"; import type { OnboardFlags } from "../onboard/command-support"; +import { buildSubprocessEnv } from "../subprocess-env"; import { runDeployAction as executeDeployAction } from "./deploy"; import { backupAll as executeBackupAllAction, @@ -87,10 +88,20 @@ export function runOpenshellProviderCommand( timeout?: number; }, ) { + const explicitEnv = Object.fromEntries( + Object.entries(opts?.env ?? {}).filter( + (entry): entry is [string, string] => entry[1] !== undefined, + ), + ); + const providerOpts = { + ...opts, + env: buildSubprocessEnv(explicitEnv), + replaceEnv: true, + }; if (typeof runtimeHooks.runOpenshell === "function") { - return runtimeHooks.runOpenshell(args, opts); + return runtimeHooks.runOpenshell(args, providerOpts); } - return runOpenshell(args, opts); + return runOpenshell(args, providerOpts); } export function recordExtraProvider(name: string): boolean { diff --git a/src/lib/actions/inference-set.ts b/src/lib/actions/inference-set.ts index bffe3a28333..4af3566584b 100644 --- a/src/lib/actions/inference-set.ts +++ b/src/lib/actions/inference-set.ts @@ -26,6 +26,7 @@ import type { ConfigObject, ConfigValue } from "../security/credential-filter"; import { isConfigObject, isConfigValue } from "../security/credential-filter"; import { appendAuditEntry } from "../shields/audit"; import { withTimerBoundShieldsMutationLockAsync } from "../shields/timer-bound-lock"; +import { withSandboxMutationLock } from "../state/mcp-lifecycle-lock"; import * as onboardSession from "../state/onboard-session"; import type { SandboxEntry } from "../state/registry"; import * as registry from "../state/registry"; @@ -833,7 +834,9 @@ export async function runInferenceSet( // an async lock. The inner resolution still validates the live registry entry. const selected = resolveTargetSandbox(options.sandboxName, deps); deps.prepareRunOpenshell(); - return withTimerBoundShieldsMutationLockAsync(selected.sandboxName, "inference set", () => - runInferenceSetWithoutHostLock({ ...options, sandboxName: selected.sandboxName }, deps), + return withSandboxMutationLock(selected.sandboxName, () => + withTimerBoundShieldsMutationLockAsync(selected.sandboxName, "inference set", () => + runInferenceSetWithoutHostLock({ ...options, sandboxName: selected.sandboxName }, deps), + ), ); } diff --git a/src/lib/actions/onboard.ts b/src/lib/actions/onboard.ts index a427d7dc01c..2784c03666f 100644 --- a/src/lib/actions/onboard.ts +++ b/src/lib/actions/onboard.ts @@ -5,9 +5,14 @@ import { listAgents } from "../agent/defs"; import { runOnboardCommand } from "../onboard/command"; import type { OnboardFlags } from "../onboard/command-support"; -const { onboard: runOnboard } = require("../onboard") as { - onboard: (options?: unknown) => Promise; -}; +async function runOnboard(options?: unknown): Promise { + // Keep the monolithic legacy onboarding graph lazy so command metadata/help + // imports do not execute it. Resolve it only when the user invokes onboard. + const { onboard } = (await import("../onboard")) as unknown as { + onboard: (onboardOptions?: unknown) => Promise; + }; + await onboard(options); +} function buildOnboardCommandDeps(flags: OnboardFlags) { return { diff --git a/src/lib/actions/sandbox/destroy-confirmation.ts b/src/lib/actions/sandbox/destroy-confirmation.ts new file mode 100644 index 00000000000..08e5f96114d --- /dev/null +++ b/src/lib/actions/sandbox/destroy-confirmation.ts @@ -0,0 +1,51 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { resolveOpenshell } from "../../adapters/openshell/resolve"; +import { R, YW } from "../../cli/terminal-style"; +import { prompt as askPrompt } from "../../credentials/store"; +import type { DestroySandboxOptions } from "../../domain/lifecycle/options"; +import { + createSystemDeps as createSessionDeps, + getActiveSandboxSessions, +} from "../../state/sandbox-session"; + +function countActiveSandboxSessions(sandboxName: string): number { + const opsBin = resolveOpenshell(); + if (!opsBin) return 0; + try { + const result = getActiveSandboxSessions(sandboxName, createSessionDeps(opsBin)); + return result.detected ? result.sessions.length : 0; + } catch { + return 0; + } +} + +export async function confirmSandboxDestroy( + sandboxName: string, + options: DestroySandboxOptions, +): Promise { + // Preserve the existing best-effort session probe even for pre-confirmed + // destroys; callers historically performed it before checking --yes/--force. + const activeSessionCount = countActiveSandboxSessions(sandboxName); + if (options.yes === true || options.force === true) return true; + + console.log(` ${YW}Destroy sandbox '${sandboxName}'?${R}`); + if (activeSessionCount > 0) { + const plural = activeSessionCount > 1 ? "sessions" : "session"; + console.log( + ` ${YW}⚠ Active SSH ${plural} detected (${activeSessionCount} connection${activeSessionCount > 1 ? "s" : ""})${R}`, + ); + console.log( + ` Destroying will terminate ${activeSessionCount === 1 ? "the" : "all"} active ${plural} with a Broken pipe error.`, + ); + } + console.log(" This will permanently delete the sandbox and all workspace files inside it."); + console.log(" This cannot be undone."); + const answer = await askPrompt(" Type 'yes' to confirm, or press Enter to cancel [y/N]: "); + if (answer.trim().toLowerCase() === "y" || answer.trim().toLowerCase() === "yes") { + return true; + } + console.log(" Cancelled."); + return false; +} diff --git a/src/lib/actions/sandbox/destroy-execution.ts b/src/lib/actions/sandbox/destroy-execution.ts new file mode 100644 index 00000000000..5bfb0d7cc96 --- /dev/null +++ b/src/lib/actions/sandbox/destroy-execution.ts @@ -0,0 +1,240 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { R, YW } from "../../cli/terminal-style"; +import { getSandboxDeleteOutcome } from "../../domain/sandbox/destroy"; +import { + type DetachSandboxProvidersResult, + runSandboxProviderPreDeleteCleanup, +} from "../../onboard/sandbox-provider-cleanup"; +import { redact } from "../../security/redact"; +import { withTimerBoundShieldsMutationLockAsync } from "../../shields/timer-bound-lock"; +import { readTimerMarker } from "../../shields/timer-control"; +import type { SandboxEntry } from "../../state/registry"; +import type { DestroyRunOpenshell } from "./destroy-gateway"; +import { + finalizeMcpBridgesAfterSandboxDelete, + type McpDestroyPreparation, + prepareMcpBridgesForAbsentSandboxDestroy, + prepareMcpBridgesForDestroy, + restoreMcpBridgesAfterDestroyAbort, +} from "./mcp-bridge"; +import { wipeSandboxState } from "./wipe-state"; + +type SandboxDestroyExecutionInput = { + cleanupShieldsArtifacts: (sandboxName: string) => void; + force: boolean; + runOpenshell: DestroyRunOpenshell; + sandbox: SandboxEntry | null; + sandboxConfirmedAbsent: boolean; + sandboxName: string; +}; + +export type SandboxDestroyExecutionResult = + | { + ok: true; + alreadyGone: boolean; + deleteOutput: string; + deleteResult: ReturnType; + detachOutcome: DetachSandboxProvidersResult; + forcedLocalCleanup: boolean; + } + | { + ok: false; + deleteOutput: string; + exitCode: number; + gatewayUnreachable: boolean; + mcpOwnershipRequiresGateway: boolean; + mcpRecoveryFailure?: string; + }; + +type HardenedDeleteState = { + hardenedForDelete: boolean; + timerProcessToken?: string; +}; + +function emptyMcpDestroyPreparation(): McpDestroyPreparation { + return { + entries: [], + detachedProviderEntries: [], + scrubbedAdapterEntries: [], + destroyAlreadyPrepared: false, + destroyAlreadyPending: false, + }; +} + +async function prepareMcpDestroy( + sandboxName: string, + sandbox: SandboxEntry | null, + sandboxConfirmedAbsent: boolean, + force: boolean, +): Promise { + if (Object.keys(sandbox?.mcp?.bridges ?? {}).length === 0) { + return emptyMcpDestroyPreparation(); + } + const preparation = sandboxConfirmedAbsent + ? await prepareMcpBridgesForAbsentSandboxDestroy(sandboxName, { force }) + : await prepareMcpBridgesForDestroy(sandboxName); + if (sandboxConfirmedAbsent && preparation.entries.length > 0) { + console.warn( + ` ${YW}⚠${R} Sandbox '${sandboxName}' is already absent, so its retained-volume MCP adapter entry cannot be scrubbed in place. Exact OpenShell providers will be deleted so any stale credential placeholder cannot authenticate; same-name onboarding may need to replace stale MCP adapter config.`, + ); + } + return preparation; +} + +function wipeAndHardenLiveSandbox( + sandboxName: string, + sandboxConfirmedAbsent: boolean, +): HardenedDeleteState { + if (sandboxConfirmedAbsent) return { hardenedForDelete: false }; + + // Wipe before delete while the retained volume is still mounted. The caller + // holds the timer-bound lock across this phase and all following teardown. + wipeSandboxState(sandboxName); + const timerMarker = readTimerMarker(sandboxName); + if (!timerMarker) return { hardenedForDelete: false }; + + const timerProcessToken = /^[0-9a-f]{32}$/.test(timerMarker.processToken ?? "") + ? timerMarker.processToken + : undefined; + const { shieldsUp } = require("../../shields") as typeof import("../../shields"); + shieldsUp(sandboxName, { + throwOnError: true, + allowLegacyHermesProtocol: true, + }); + return { hardenedForDelete: true, timerProcessToken }; +} + +async function restoreMcpAfterDeleteAbort( + sandboxName: string, + preparation: McpDestroyPreparation, + hardened: HardenedDeleteState, +): Promise { + let recoveryFailure: string | undefined; + let openedRollbackWindow = false; + try { + if (hardened.hardenedForDelete && preparation.entries.length > 0) { + if (!hardened.timerProcessToken) { + throw new Error( + "Cannot open a bounded MCP rollback window because the active shields timer had no valid process token.", + ); + } + const { shieldsDown } = require("../../shields") as typeof import("../../shields"); + shieldsDown(sandboxName, { + reason: "restore MCP after refused sandbox delete", + timeout: "15m", + throwOnError: true, + allowLegacyHermesProtocol: true, + deferAutoRestoreWhileOwnerAlive: true, + processToken: hardened.timerProcessToken, + }); + openedRollbackWindow = true; + } + await restoreMcpBridgesAfterDestroyAbort(sandboxName, preparation); + } catch (error) { + recoveryFailure = error instanceof Error ? error.message : String(error); + } finally { + if (openedRollbackWindow) { + try { + const { shieldsUp } = require("../../shields") as typeof import("../../shields"); + shieldsUp(sandboxName, { + throwOnError: true, + allowLegacyHermesProtocol: true, + }); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + recoveryFailure = recoveryFailure + ? `${recoveryFailure}; shields re-lock failed: ${detail}` + : `shields re-lock failed: ${detail}`; + } + } + } + return recoveryFailure; +} + +async function finalizeMcpDestroy( + sandboxName: string, + preparation: McpDestroyPreparation, + force: boolean, +): Promise { + try { + await finalizeMcpBridgesAfterSandboxDelete(sandboxName, preparation, { force }); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + console.error( + ` Sandbox '${sandboxName}' is gone, but authenticated MCP provider cleanup is incomplete: ${detail}`, + ); + console.error( + " MCP cleanup state was preserved. Re-run destroy to finish without requiring the host MCP secret environment variable.", + ); + throw error; + } +} + +export async function executeSandboxDestroy({ + cleanupShieldsArtifacts, + force, + runOpenshell, + sandbox, + sandboxConfirmedAbsent, + sandboxName, +}: SandboxDestroyExecutionInput): Promise { + return withTimerBoundShieldsMutationLockAsync(sandboxName, "destroy sandbox", async () => { + const mcpPreparation = await prepareMcpDestroy( + sandboxName, + sandbox, + sandboxConfirmedAbsent, + force, + ); + // Prepared-only/incomplete adds have no external resources and are safely + // discarded during preparation. Remaining entries are the durable exact + // provider ownership manifest and must survive an unconfirmed delete. + const hasMcpOwnership = mcpPreparation.entries.length > 0; + const hardened = wipeAndHardenLiveSandbox(sandboxName, sandboxConfirmedAbsent); + const detachOutcome: DetachSandboxProvidersResult = sandboxConfirmedAbsent + ? { detached: [], failures: [] } + : runSandboxProviderPreDeleteCleanup(sandboxName, { runOpenshell, redact }); + const deleteResult = runOpenshell(["sandbox", "delete", sandboxName], { + ignoreError: true, + stdio: ["ignore", "pipe", "pipe"], + }); + const { + output: deleteOutput, + alreadyGone, + gatewayUnreachable, + } = getSandboxDeleteOutcome(deleteResult); + const forcedLocalCleanup = + deleteResult.status !== 0 && !alreadyGone && gatewayUnreachable && force && !hasMcpOwnership; + + if (deleteResult.status !== 0 && !alreadyGone && !forcedLocalCleanup) { + const mcpRecoveryFailure = sandboxConfirmedAbsent + ? undefined + : await restoreMcpAfterDeleteAbort(sandboxName, mcpPreparation, hardened); + return { + ok: false as const, + deleteOutput, + exitCode: deleteResult.status || 1, + gatewayUnreachable, + mcpOwnershipRequiresGateway: gatewayUnreachable && hasMcpOwnership, + mcpRecoveryFailure, + }; + } + + // The sandbox is confirmed gone, or --force is discarding only a local + // record that has no MCP ownership. Keep this under the lifecycle lock so + // stale timer state cannot target a same-name replacement. + cleanupShieldsArtifacts(sandboxName); + if (!forcedLocalCleanup) { + await finalizeMcpDestroy(sandboxName, mcpPreparation, force); + } + return { + ok: true as const, + detachOutcome, + deleteOutput, + deleteResult, + alreadyGone, + forcedLocalCleanup, + }; + }); +} diff --git a/src/lib/actions/sandbox/destroy-flow.test.ts b/src/lib/actions/sandbox/destroy-flow.test.ts index 3c951c9d2b4..3e3ce11d0c7 100644 --- a/src/lib/actions/sandbox/destroy-flow.test.ts +++ b/src/lib/actions/sandbox/destroy-flow.test.ts @@ -1,186 +1,47 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { createRequire } from "node:module"; - import { afterEach, beforeEach, describe, expect, it, type MockInstance, vi } from "vitest"; -type DestroySandbox = typeof import("./destroy")["destroySandbox"]; - -const requireDist = createRequire(import.meta.url); -const destroyModulePath = "./destroy.js"; - -type DestroyHarness = { - cleanupGatewaySpy: MockInstance; - destroySandbox: DestroySandbox; - events: string[]; - killTimerSpy: MockInstance; - killStaleProxySpy: MockInstance; - logSpy: MockInstance; - removeSandboxSpy: MockInstance; - runOpenshellSpy: MockInstance; - selectGatewaySpy: MockInstance; - stopAllSpy: MockInstance; - stopNimByNameSpy: MockInstance; - unloadOllamaModelsSpy: MockInstance; - shieldsUpSpy: MockInstance; -}; - -type DestroyHarnessOptions = { - activeTimer?: boolean; - deleteStatus?: number; - deleteOutput?: string; - registeredSandboxCount?: number; - shieldsUpError?: Error; -}; - -const sandboxEntry = { - name: "alpha", - provider: "ollama-local", - model: "nvidia/nemotron", - imageTag: null, - nimContainer: "alpha-nim", - gatewayName: "nemoclaw-19080", - gatewayPort: 19080, -}; - -function createDestroyHarness(options: DestroyHarnessOptions = {}): DestroyHarness { - delete require.cache[requireDist.resolve(destroyModulePath)]; - const events: string[] = []; - - const logSpy = vi.spyOn(console, "log").mockImplementation(() => undefined); - vi.spyOn(console, "error").mockImplementation(() => undefined); - vi.spyOn(console, "warn").mockImplementation(() => undefined); - - const resolve = requireDist("../../adapters/openshell/resolve.js"); - const runtime = requireDist("../../adapters/openshell/runtime.js"); - const destroyGateway = requireDist("./destroy-gateway.js"); - const sandboxProviderCleanup = requireDist("../../onboard/sandbox-provider-cleanup.js"); - const nim = requireDist("../../inference/nim.js"); - const ollamaProxy = requireDist("../../inference/ollama/proxy.js"); - const tunnelServices = requireDist("../../tunnel/services.js"); - const onboardSession = requireDist("../../state/onboard-session.js"); - const registry = requireDist("../../state/registry.js"); - const sandboxSession = requireDist("../../state/sandbox-session.js"); - const shields = requireDist("../../shields/index.js"); - const timerControl = requireDist("../../shields/timer-control.js"); - - vi.spyOn(resolve, "resolveOpenshell").mockReturnValue("/usr/bin/openshell"); - vi.spyOn(sandboxSession, "getActiveSandboxSessions").mockReturnValue({ - detected: true, - sessions: [{ pid: 1 }], - }); - vi.spyOn(registry, "getSandbox").mockReturnValue(sandboxEntry); - vi.spyOn(registry, "listSandboxes").mockReturnValue({ - sandboxes: Array.from({ length: options.registeredSandboxCount ?? 0 }, (_, i) => ({ - name: `sb-${i}`, - })), - }); - const removeSandboxSpy = vi.spyOn(registry, "removeSandbox").mockReturnValue(true); - vi.spyOn(onboardSession, "loadSession").mockReturnValue({ sandboxName: "alpha" }); - vi.spyOn(onboardSession, "updateSession").mockImplementation((mutator: unknown) => { - const session = { sandboxName: "alpha" }; - typeof mutator === "function" && (mutator as (value: typeof session) => void)(session); - return session; - }); - const runOpenshellSpy = vi.spyOn(runtime, "runOpenshell").mockImplementation((args: unknown) => { - const argv = Array.isArray(args) ? args : []; - switch (`${String(argv[0])} ${String(argv[1])}`) { - case "sandbox exec": - events.push("wipe"); - break; - case "sandbox delete": - events.push("delete"); - return { - status: options.deleteStatus ?? 0, - stdout: options.deleteOutput ?? "", - stderr: "", - }; - } - return { status: 0, stdout: "", stderr: "" }; - }); - vi.spyOn(runtime, "captureOpenshell").mockReturnValue({ status: 0, output: "" }); - const selectGatewaySpy = vi - .spyOn(destroyGateway, "selectGatewayForSandboxDestroy") - .mockImplementation(() => undefined); - const cleanupGatewaySpy = vi - .spyOn(destroyGateway, "cleanupGatewayAfterLastSandbox") - .mockImplementation(() => undefined); - vi.spyOn(sandboxProviderCleanup, "runSandboxProviderPreDeleteCleanup").mockImplementation(() => { - events.push("detach"); - return { failures: [] }; - }); - vi.spyOn(sandboxProviderCleanup, "emitProviderDetachResidualHint").mockImplementation( - () => undefined, - ); - const stopNimByNameSpy = vi - .spyOn(nim, "stopNimContainerByName") - .mockImplementation(() => undefined); - vi.spyOn(nim, "stopNimContainer").mockImplementation(() => undefined); - const killStaleProxySpy = vi - .spyOn(ollamaProxy, "killStaleProxy") - .mockImplementation(() => undefined); - const unloadOllamaModelsSpy = vi - .spyOn(ollamaProxy, "unloadOllamaModels") - .mockImplementation(() => undefined); - const stopAllSpy = vi.spyOn(tunnelServices, "stopAll").mockImplementation(() => undefined); - vi.spyOn(timerControl, "readTimerMarker").mockReturnValue( - options.activeTimer - ? { - pid: 4242, - sandboxName: "alpha", - snapshotPath: "/tmp/policy.yaml", - restoreAt: "2026-06-27T06:00:00.000Z", - processToken: "a".repeat(32), - } - : null, - ); - const shieldsUpSpy = vi.spyOn(shields, "shieldsUp").mockImplementation(() => { - events.push("harden"); - const shieldsUpError = options.shieldsUpError; - switch (shieldsUpError) { - case undefined: - break; - default: - throw shieldsUpError; - } - }); - const killTimerSpy = vi.spyOn(timerControl, "killTimer").mockImplementation(() => { - events.push("timer-cleanup"); - return { warnings: [] }; - }); - - logSpy.mockClear(); - - return { - cleanupGatewaySpy, - destroySandbox: requireDist(destroyModulePath).destroySandbox, - events, - killTimerSpy, - killStaleProxySpy, - logSpy, - removeSandboxSpy, - runOpenshellSpy, - selectGatewaySpy, - stopAllSpy, - stopNimByNameSpy, - unloadOllamaModelsSpy, - shieldsUpSpy, - }; -} +import { + expectAbsentSandboxMcpFinalize, + expectActiveTimerDestroyOrder, + expectFailedDeletePreservesHostState, + expectFailedHardeningStopsDelete, + expectFailedMcpFinalizePreservesRegistry, + expectFailedMcpRestorePreservesDestroyFailure, + expectMcpFinalizeAfterDelete, + expectMcpRestoreAfterDeleteFailure, + expectShieldsUpRefusalBeforeMutation, + expectStrictSandboxPresenceClassification, + expectSuccessfulLiveDestroy, +} from "../../../../test/helpers/destroy-flow-test-assertions"; +import { + createDestroyHarness, + resetDestroyModuleCache, +} from "../../../../test/helpers/destroy-flow-test-harness"; describe("destroySandbox flow", () => { let exitSpy: MockInstance; + let originalGatewayEnv: string | undefined; beforeEach(() => { + originalGatewayEnv = process.env.OPENSHELL_GATEWAY; exitSpy = vi.spyOn(process, "exit").mockImplementation(((code?: number | string | null) => { throw new Error(`process.exit(${code ?? 0})`); }) as never); }); afterEach(() => { + originalGatewayEnv === undefined + ? delete process.env.OPENSHELL_GATEWAY + : (process.env.OPENSHELL_GATEWAY = originalGatewayEnv); vi.restoreAllMocks(); - delete require.cache[requireDist.resolve(destroyModulePath)]; + resetDestroyModuleCache(); + }); + + it("trusts absence only from a successful, error-free sandbox list", { timeout: 15_000 }, () => { + expectStrictSandboxPresenceClassification(); }); it("selects the sandbox gateway, deletes live resources, cleans host state, and removes registry state", async () => { @@ -190,41 +51,60 @@ describe("destroySandbox flow", () => { harness.destroySandbox("alpha", { yes: true, cleanupGateway: true }), ).resolves.toBeUndefined(); - expect(harness.selectGatewaySpy).toHaveBeenCalledWith( - "alpha", - "nemoclaw-19080", - harness.runOpenshellSpy, - ); - expect(harness.stopNimByNameSpy).toHaveBeenCalledWith("alpha-nim"); - expect(harness.killStaleProxySpy).toHaveBeenCalledTimes(1); - expect(harness.runOpenshellSpy).toHaveBeenCalledWith( - ["sandbox", "delete", "alpha"], - expect.objectContaining({ ignoreError: true }), - ); - expect(harness.unloadOllamaModelsSpy).toHaveBeenCalledTimes(1); - expect(harness.removeSandboxSpy).toHaveBeenCalledWith("alpha"); - expect(harness.cleanupGatewaySpy).toHaveBeenCalledWith( - "nemoclaw-19080", - harness.runOpenshellSpy, - ); - expect(harness.logSpy.mock.calls.map((call) => String(call[0])).join("\n")).toContain( - "Sandbox 'alpha' destroyed", - ); - expect(exitSpy).not.toHaveBeenCalled(); + expectSuccessfulLiveDestroy(harness, exitSpy); }); it("stops before local cleanup when OpenShell fails to delete the live sandbox", async () => { - const harness = createDestroyHarness({ deleteStatus: 7, deleteOutput: "delete failed" }); + const harness = createDestroyHarness({ + deleteStatus: 7, + deleteOutput: "delete failed", + }); await expect(harness.destroySandbox("alpha", { yes: true })).rejects.toThrow("process.exit(7)"); - expect(harness.runOpenshellSpy).toHaveBeenCalledWith( - ["sandbox", "delete", "alpha"], - expect.objectContaining({ ignoreError: true }), + expectFailedDeletePreservesHostState(harness, exitSpy); + }); + + it("refuses shields-up Hermes MCP destroy before stopping services or preparing MCP state", async () => { + const harness = createDestroyHarness({ + agent: "hermes", + mcpServers: ["github"], + shieldsDown: false, + }); + + await expect(harness.destroySandbox("alpha", { yes: true })).rejects.toThrow( + "has shields up or an unreadable shields posture", ); - expect(harness.removeSandboxSpy).not.toHaveBeenCalled(); - expect(harness.cleanupGatewaySpy).not.toHaveBeenCalled(); - expect(exitSpy).toHaveBeenCalledWith(7); + + expectShieldsUpRefusalBeforeMutation(harness); + }); + + it("does not require mutable Hermes config for a prepared-only add", async () => { + const harness = createDestroyHarness({ + agent: "hermes", + mcpAddState: "prepared", + mcpServers: ["github"], + shieldsDown: false, + }); + + await expect(harness.destroySandbox("alpha", { yes: true })).resolves.toBeUndefined(); + + expect(harness.prepareMcpBridgesForDestroySpy).toHaveBeenCalledWith("alpha"); + }); + + it("does not require mutable Hermes config for absent-sandbox cleanup", async () => { + const harness = createDestroyHarness({ + agent: "hermes", + mcpServers: ["github"], + sandboxPresent: false, + shieldsDown: false, + }); + + await expect(harness.destroySandbox("alpha", { yes: true })).resolves.toBeUndefined(); + + expect(harness.prepareMcpBridgesForAbsentSandboxDestroySpy).toHaveBeenCalledWith("alpha", { + force: false, + }); }); it("does not stop shared host services when --force cleans up the last sandbox with the gateway down (#6046)", async () => { @@ -248,17 +128,35 @@ describe("destroySandbox flow", () => { expect(exitSpy).not.toHaveBeenCalled(); }); + it("fails closed and restores MCP state when --force cannot confirm sandbox deletion", async () => { + const harness = createDestroyHarness({ + activeTimer: true, + deleteStatus: 1, + deleteOutput: "error trying to connect: connection refused", + mcpServers: ["github"], + registeredSandboxCount: 1, + }); + + await expect(harness.destroySandbox("alpha", { force: true })).rejects.toThrow( + "process.exit(1)", + ); + + expectMcpRestoreAfterDeleteFailure(harness); + expect(harness.stopAllSpy).not.toHaveBeenCalled(); + expect(harness.cleanupGatewaySpy).not.toHaveBeenCalled(); + expect(exitSpy).toHaveBeenCalledWith(1); + const errorOutput = harness.errorSpy.mock.calls.map((call) => String(call[0])).join("\n"); + expect(errorOutput).toContain("MCP ownership required for exact provider cleanup"); + expect(errorOutput).toContain("--force cannot safely discard MCP ownership"); + expect(errorOutput).not.toContain("re-run with --force to remove the local sandbox record"); + }); + it("wipes while mutable, hardens an active timer window, then deletes and clears it", async () => { const harness = createDestroyHarness({ activeTimer: true }); await expect(harness.destroySandbox("alpha", { yes: true })).resolves.toBeUndefined(); - expect(harness.events).toEqual( - expect.arrayContaining(["wipe", "harden", "detach", "delete", "timer-cleanup"]), - ); - expect(harness.events.indexOf("wipe")).toBeLessThan(harness.events.indexOf("harden")); - expect(harness.events.indexOf("harden")).toBeLessThan(harness.events.indexOf("delete")); - expect(harness.events.indexOf("delete")).toBeLessThan(harness.events.indexOf("timer-cleanup")); + expectActiveTimerDestroyOrder(harness); }); it("does not delete when active-window hardening fails after the wipe", async () => { @@ -271,9 +169,67 @@ describe("destroySandbox flow", () => { "injected hardening failure", ); - expect(harness.events).toContain("wipe"); - expect(harness.events).toContain("harden"); - expect(harness.events).not.toContain("delete"); - expect(harness.killTimerSpy).not.toHaveBeenCalled(); + expectFailedHardeningStopsDelete(harness); + }); + + it("detaches MCP providers before delete and finalizes them only after delete succeeds", async () => { + const harness = createDestroyHarness({ mcpServers: ["github", "slack"] }); + + await harness.destroySandbox("alpha", { yes: true }); + + expectMcpFinalizeAfterDelete(harness); + }); + + it("restores MCP runtime state when sandbox delete fails", async () => { + const harness = createDestroyHarness({ + activeTimer: true, + deleteStatus: 7, + deleteOutput: "delete failed", + mcpServers: ["github"], + }); + + await expect(harness.destroySandbox("alpha", { yes: true })).rejects.toThrow("process.exit(7)"); + + expectMcpRestoreAfterDeleteFailure(harness); + }); + + it("relocks shields and preserves destroy failure when MCP rollback fails", async () => { + const harness = createDestroyHarness({ + activeTimer: true, + deleteStatus: 7, + deleteOutput: "delete failed", + mcpServers: ["github"], + restoreMcpError: "injected MCP restore failure", + }); + + await expect(harness.destroySandbox("alpha", { yes: true })).rejects.toThrow("process.exit(7)"); + + expectFailedMcpRestorePreservesDestroyFailure(harness); + }); + + it("preserves the registry when post-delete MCP cleanup fails, even with force", async () => { + const harness = createDestroyHarness({ + finalizeMcpError: "provider delete failed", + mcpServers: ["github"], + }); + + await expect(harness.destroySandbox("alpha", { yes: true, force: true })).rejects.toThrow( + "provider delete failed", + ); + + expectFailedMcpFinalizePreservesRegistry(harness); + }); + + it("finalizes exact MCP providers when the sandbox was already externally removed", async () => { + const harness = createDestroyHarness({ + deleteStatus: 1, + deleteOutput: "Error: sandbox alpha not found", + mcpServers: ["github"], + sandboxPresent: false, + }); + + await expect(harness.destroySandbox("alpha", { yes: true })).resolves.toBeUndefined(); + + expectAbsentSandboxMcpFinalize(harness); }); }); diff --git a/src/lib/actions/sandbox/destroy-preflight.ts b/src/lib/actions/sandbox/destroy-preflight.ts new file mode 100644 index 00000000000..2b8fb1ce449 --- /dev/null +++ b/src/lib/actions/sandbox/destroy-preflight.ts @@ -0,0 +1,81 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { OPENSHELL_PROBE_TIMEOUT_MS } from "../../adapters/openshell/timeouts"; +import type { SandboxEntry } from "../../state/registry"; +import * as registry from "../../state/registry"; +import { type DestroyRunOpenshell, selectGatewayForSandboxDestroy } from "./destroy-gateway"; +import { classifyDestroySandboxPresence } from "./destroy-presence"; +import { getSandboxTargetGatewayName } from "./gateway-target"; +import { assertMcpAdapterConfigMutationsAllowed } from "./mcp-bridge-runtime-capabilities"; + +export type SandboxDestroyPreflight = { + cleanupGatewayName: string; + runOpenshell: DestroyRunOpenshell; + sandbox: SandboxEntry | null; + sandboxConfirmedAbsent: boolean; +}; + +function stopSandboxInferenceResources(sandboxName: string, sandbox: SandboxEntry | null): void { + const nim = require("../../inference/nim") as { + stopNimContainer: (name: string, opts?: { silent?: boolean }) => void; + stopNimContainerByName: (name: string) => void; + }; + if (sandbox?.nimContainer) { + console.log(` Stopping NIM for '${sandboxName}'...`); + nim.stopNimContainerByName(sandbox.nimContainer); + } else { + // Older registry entries may not record the convention-named container. + nim.stopNimContainer(sandboxName, { silent: true }); + } + + // The Ollama auth proxy is per-sandbox. GPU model unload happens during + // post-delete host cleanup, after the live sandbox is confirmed gone. + if (sandbox?.provider?.includes("ollama")) { + const { killStaleProxy } = require("../../inference/ollama/proxy") as { + killStaleProxy: () => void; + }; + killStaleProxy(); + } +} + +export function prepareSandboxDestroy(sandboxName: string): SandboxDestroyPreflight { + const sandbox = registry.getSandbox(sandboxName); + console.log(` Deleting sandbox '${sandboxName}'...`); + const { runOpenshell } = require("../../adapters/openshell/runtime") as { + runOpenshell: DestroyRunOpenshell; + }; + + // Capture the sandbox gateway before destructive work, then pin every + // following OpenShell subprocess against that same registry-owned gateway. + const cleanupGatewayName = getSandboxTargetGatewayName(sandboxName); + selectGatewayForSandboxDestroy(sandboxName, cleanupGatewayName, runOpenshell); + process.env.OPENSHELL_GATEWAY = cleanupGatewayName; + + const sandboxPresence = classifyDestroySandboxPresence( + sandboxName, + runOpenshell(["sandbox", "list", "-o", "json"], { + ignoreError: true, + stdio: ["ignore", "pipe", "pipe"], + timeout: OPENSHELL_PROBE_TIMEOUT_MS, + }), + ); + const sandboxConfirmedAbsent = sandboxPresence === "absent"; + const mcpEntriesRequiringConfigMutation = Object.values(sandbox?.mcp?.bridges ?? {}).filter( + (entry) => entry.addState !== "prepared", + ); + if ( + !sandboxConfirmedAbsent && + sandbox && + !sandbox.mcp?.destroyPreparedAt && + !sandbox.mcp?.destroyPendingAt && + mcpEntriesRequiringConfigMutation.length > 0 + ) { + // Fail before stopping local services or mutating any MCP resource when + // the live adapter config cannot be changed safely. + assertMcpAdapterConfigMutationsAllowed(sandboxName, sandbox, mcpEntriesRequiringConfigMutation); + } + + stopSandboxInferenceResources(sandboxName, sandbox); + return { cleanupGatewayName, runOpenshell, sandbox, sandboxConfirmedAbsent }; +} diff --git a/src/lib/actions/sandbox/destroy-presence.ts b/src/lib/actions/sandbox/destroy-presence.ts new file mode 100644 index 00000000000..a03dad14f70 --- /dev/null +++ b/src/lib/actions/sandbox/destroy-presence.ts @@ -0,0 +1,46 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export type DestroySandboxPresence = "present" | "absent" | "unknown"; + +function isStrictSandboxListJsonRow(value: unknown): value is { name: string } { + if (!value || typeof value !== "object" || Array.isArray(value)) return false; + const row = value as Record; + const labels = row.labels; + return ( + typeof row.id === "string" && + typeof row.name === "string" && + row.name.length > 0 && + row.name.trim() === row.name && + !!labels && + typeof labels === "object" && + !Array.isArray(labels) && + Object.values(labels as Record).every((label) => typeof label === "string") && + typeof row.resource_version === "number" && + Number.isFinite(row.resource_version) && + typeof row.created_at === "string" && + typeof row.phase === "string" && + row.phase.length > 0 && + typeof row.current_policy_version === "number" && + Number.isFinite(row.current_policy_version) + ); +} + +export function classifyDestroySandboxPresence( + sandboxName: string, + result: { status: number | null; stdout?: string; stderr?: string }, +): DestroySandboxPresence { + if (result.status !== 0) return "unknown"; + const stderr = result.stderr?.trim() ?? ""; + if (stderr) return "unknown"; + let rows: unknown; + try { + rows = JSON.parse(result.stdout ?? ""); + } catch { + return "unknown"; + } + if (!Array.isArray(rows) || !rows.every(isStrictSandboxListJsonRow)) { + return "unknown"; + } + return rows.some((row) => row.name === sandboxName) ? "present" : "absent"; +} diff --git a/src/lib/actions/sandbox/destroy.ts b/src/lib/actions/sandbox/destroy.ts index 2ae692dbc8e..89d82061f6a 100644 --- a/src/lib/actions/sandbox/destroy.ts +++ b/src/lib/actions/sandbox/destroy.ts @@ -4,7 +4,6 @@ import fs from "node:fs"; import path from "node:path"; -import { resolveOpenshell } from "../../adapters/openshell/resolve"; import { OPENSHELL_PROBE_TIMEOUT_MS } from "../../adapters/openshell/timeouts"; import { CLI_NAME } from "../../cli/branding"; import { G, R, YW } from "../../cli/terminal-style"; @@ -14,35 +13,28 @@ import { normalizeDestroySandboxOptions, } from "../../domain/lifecycle/options"; import { - getSandboxDeleteOutcome, shouldCleanupGatewayAfterDestroy, shouldStopHostServicesAfterDestroy, } from "../../domain/sandbox/destroy"; import { emitProviderDetachResidualHint, - runSandboxProviderPreDeleteCleanup, SANDBOX_PROVIDER_SUFFIXES, } from "../../onboard/sandbox-provider-cleanup"; import { parseLiveSandboxNames } from "../../runtime-recovery"; -import { redact } from "../../security/redact"; -import { withTimerBoundShieldsMutationLock } from "../../shields/timer-bound-lock"; -import { killTimer as defaultKillShieldsTimer, readTimerMarker } from "../../shields/timer-control"; +import { killTimer as defaultKillShieldsTimer } from "../../shields/timer-control"; +import { withMcpLifecycleLock } from "../../state/mcp-lifecycle-lock"; import type { Session } from "../../state/onboard-session"; import * as onboardSession from "../../state/onboard-session"; import { resolveNemoclawStateDir } from "../../state/paths"; import * as registry from "../../state/registry"; -import { - createSystemDeps as createSessionDeps, - getActiveSandboxSessions, -} from "../../state/sandbox-session"; -import { - cleanupGatewayAfterLastSandbox, - type DestroyRunOpenshell, - selectGatewayForSandboxDestroy, -} from "./destroy-gateway"; -import { getSandboxTargetGatewayName } from "./gateway-target"; +import { confirmSandboxDestroy } from "./destroy-confirmation"; +import { executeSandboxDestroy } from "./destroy-execution"; +import { cleanupGatewayAfterLastSandbox } from "./destroy-gateway"; +import { prepareSandboxDestroy } from "./destroy-preflight"; import { type WipeSandboxStateDeps, wipeSandboxState } from "./wipe-state"; +export { classifyDestroySandboxPresence } from "./destroy-presence"; + type DockerRmi = (tag: string, opts?: { ignoreError?: boolean }) => { status: number | null }; type RemoveSandboxImageDeps = { @@ -302,171 +294,56 @@ export async function destroySandbox( sandboxName: string, options: string[] | DestroySandboxOptions = {}, ): Promise { - const normalized = normalizeDestroySandboxOptions(options); - const skipConfirm = normalized.yes === true || normalized.force === true; - - // Active session detection — enrich the confirmation prompt if sessions are active - let activeSessionCount = 0; - const opsBin = resolveOpenshell(); - if (opsBin) { - try { - const sessionResult = getActiveSandboxSessions(sandboxName, createSessionDeps(opsBin)); - if (sessionResult.detected) { - activeSessionCount = sessionResult.sessions.length; - } - } catch { - /* non-fatal */ - } - } - - if (!skipConfirm) { - console.log(` ${YW}Destroy sandbox '${sandboxName}'?${R}`); - if (activeSessionCount > 0) { - const plural = activeSessionCount > 1 ? "sessions" : "session"; - console.log( - ` ${YW}⚠ Active SSH ${plural} detected (${activeSessionCount} connection${activeSessionCount > 1 ? "s" : ""})${R}`, - ); - console.log( - ` Destroying will terminate ${activeSessionCount === 1 ? "the" : "all"} active ${plural} with a Broken pipe error.`, - ); - } - console.log(" This will permanently delete the sandbox and all workspace files inside it."); - console.log(" This cannot be undone."); - const answer = await askPrompt(" Type 'yes' to confirm, or press Enter to cancel [y/N]: "); - if (answer.trim().toLowerCase() !== "y" && answer.trim().toLowerCase() !== "yes") { - console.log(" Cancelled."); - return; - } - } - - const nim = require("../../inference/nim") as { - stopNimContainer: (sandboxName: string, opts?: { silent?: boolean }) => void; - stopNimContainerByName: (name: string) => void; - }; - const sb = registry.getSandbox(sandboxName); - if (sb && sb.nimContainer) { - console.log(` Stopping NIM for '${sandboxName}'...`); - nim.stopNimContainerByName(sb.nimContainer); - } else { - // Best-effort cleanup of convention-named NIM containers that may not - // be recorded in the registry (e.g. older sandboxes). Suppress output - // so the user doesn't see "No such container" noise when no NIM exists. - nim.stopNimContainer(sandboxName, { silent: true }); - } - - // The Ollama auth proxy is per-sandbox and only spawned when the provider - // is Ollama, so this guard scopes only `killStaleProxy()`. GPU unload is - // handled separately by `cleanupSandboxServices` above (which routes - // through `stopAll()` or directly into `unloadOllamaModels()` based on - // whether host services are being torn down). - if (sb?.provider?.includes("ollama")) { - const { killStaleProxy } = require("../../inference/ollama/proxy"); - killStaleProxy(); - } - - console.log(` Deleting sandbox '${sandboxName}'...`); - const { runOpenshell } = require("../../adapters/openshell/runtime") as { - runOpenshell: DestroyRunOpenshell; - }; - // Capture and select the sandbox's gateway before any destructive OpenShell - // operation. Provider cleanup and sandbox delete must address the gateway - // recorded for this sandbox, not whichever gateway happens to be active. - const cleanupGatewayName = getSandboxTargetGatewayName(sandboxName); - selectGatewayForSandboxDestroy(sandboxName, cleanupGatewayName, runOpenshell); + return withMcpLifecycleLock(sandboxName, () => destroySandboxUnlocked(sandboxName, options)); +} - const destructiveResult = withTimerBoundShieldsMutationLock( +async function destroySandboxUnlocked( + sandboxName: string, + options: string[] | DestroySandboxOptions = {}, +): Promise { + const normalized = normalizeDestroySandboxOptions(options); + if (!(await confirmSandboxDestroy(sandboxName, normalized))) return; + + const { cleanupGatewayName, runOpenshell, sandbox, sandboxConfirmedAbsent } = + prepareSandboxDestroy(sandboxName); + const destructiveResult = await executeSandboxDestroy({ + cleanupShieldsArtifacts: cleanupShieldsDestroyArtifacts, + force: normalized.force === true, + runOpenshell, + sandbox, + sandboxConfirmedAbsent, sandboxName, - "destroy sandbox", - () => { - // Wipe persistent state AFTER the gateway is selected so the exec targets - // the sandbox's recorded gateway (#5455 PRA-5), but BEFORE delete because - // `sandbox delete` unmounts the PVC and `rm -rf` could no longer reach it. - // Hold the same lock used by the auto-restore timer through wipe, provider - // detach, and delete. A timer that is already restoring finishes first; - // a waiting timer cannot mutate this sandbox or a same-name replacement. - wipeSandboxState(sandboxName); - - // The wipe needs the timed mutable posture so the sandbox user can - // remove manifest state. Convert it back to a verified locked posture - // immediately afterward and before delete. The outer owner carries the - // timer takeover token throughout, so a deadline/crash during the wipe - // can still reclaim it; after shieldsUp succeeds, delete failure or - // process death leaves a surviving sandbox hardened. - if (readTimerMarker(sandboxName)) { - const { shieldsUp: hardenShields } = - require("../../shields") as typeof import("../../shields"); - hardenShields(sandboxName, { - throwOnError: true, - allowLegacyHermesProtocol: true, - }); - } - - const lockedDetachOutcome = runSandboxProviderPreDeleteCleanup(sandboxName, { - runOpenshell, - redact, - }); - const lockedDeleteResult = runOpenshell(["sandbox", "delete", sandboxName], { - ignoreError: true, - stdio: ["ignore", "pipe", "pipe"], - }); - const { - output: deleteOutput, - alreadyGone: lockedAlreadyGone, - gatewayUnreachable, - } = getSandboxDeleteOutcome(lockedDeleteResult); - - // When the OpenShell gateway is down, every gateway call (including the - // final delete) gets a connection-refused/transport error. That used to - // abort destroy with no bypass, leaving no supported way to remove the - // sandbox record (#6046). Under --force, fall back to local cleanup; - // otherwise keep failing but point at the recovery paths. - const forcedLocalCleanup = - lockedDeleteResult.status !== 0 && - !lockedAlreadyGone && - gatewayUnreachable && - normalized.force === true; - - if (lockedDeleteResult.status !== 0 && !lockedAlreadyGone && !forcedLocalCleanup) { - // Any active timer was cleared only after shieldsUp verified the live - // sandbox was hardened. Preserve that locked state on delete failure; - // do not remove its local shields record as if deletion had succeeded. - return { - ok: false as const, - deleteOutput, - gatewayUnreachable, - exitCode: lockedDeleteResult.status || 1, - }; - } - - // Either the live sandbox is confirmed gone, or --force is discarding the - // local record for an unreachable gateway. In both cases the sandbox is - // no longer tracked locally, so revoke the timer and local shields state - // before releasing the lock so neither can target a subsequently created - // sandbox with the same name. - cleanupShieldsDestroyArtifacts(sandboxName); - return { - ok: true as const, - detachOutcome: lockedDetachOutcome, - deleteResult: lockedDeleteResult, - alreadyGone: lockedAlreadyGone, - forcedLocalCleanup, - deleteOutput, - }; - }, - ); + }); if (!destructiveResult.ok) { if (destructiveResult.deleteOutput) { console.error(` ${destructiveResult.deleteOutput}`); } - console.error(` Failed to destroy sandbox '${sandboxName}'.`); - if (destructiveResult.gatewayUnreachable) { + if (destructiveResult.mcpRecoveryFailure) { console.error( - ` The OpenShell gateway is unreachable. Start it (run '${CLI_NAME} ${sandboxName} status'),`, + ` Failed to restore MCP runtime state after the sandbox delete failed: ${destructiveResult.mcpRecoveryFailure}`, ); console.error( - ` or re-run with --force to remove the local sandbox record without the gateway.`, + ` MCP definitions and OpenShell providers were preserved; fix the reported cause and retry MCP restart or destroy.`, ); } + console.error(` Failed to destroy sandbox '${sandboxName}'.`); + if (destructiveResult.gatewayUnreachable) { + if (destructiveResult.mcpOwnershipRequiresGateway) { + console.error( + ` The OpenShell gateway is unreachable. Local state was preserved because it contains MCP ownership required for exact provider cleanup.`, + ); + console.error( + ` Start the gateway (run '${CLI_NAME} ${sandboxName} status'), then retry destroy; --force cannot safely discard MCP ownership.`, + ); + } else { + console.error( + ` The OpenShell gateway is unreachable. Start it (run '${CLI_NAME} ${sandboxName} status'),`, + ); + console.error( + ` or re-run with --force to remove the local sandbox record without the gateway.`, + ); + } + } process.exit(destructiveResult.exitCode); } const { detachOutcome, deleteResult, alreadyGone, forcedLocalCleanup, deleteOutput } = diff --git a/src/lib/actions/sandbox/doctor.ts b/src/lib/actions/sandbox/doctor.ts index eb00a6f2f12..5e8b244f054 100644 --- a/src/lib/actions/sandbox/doctor.ts +++ b/src/lib/actions/sandbox/doctor.ts @@ -432,7 +432,7 @@ function agentVersionDoctorCheck(sandboxName: string): DoctorCheck { } function shieldsDoctorCheck(sandboxName: string): DoctorCheck { - const posture = shields.getShieldsPosture(sandboxName, true); + const posture = shields.getShieldsPosture(sandboxName, false); const status: DoctorStatus = posture.mode === "locked" ? "ok" diff --git a/src/lib/actions/sandbox/mcp-bridge-adapter-deepagents.test.ts b/src/lib/actions/sandbox/mcp-bridge-adapter-deepagents.test.ts new file mode 100644 index 00000000000..c2ee05f178c --- /dev/null +++ b/src/lib/actions/sandbox/mcp-bridge-adapter-deepagents.test.ts @@ -0,0 +1,232 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +import type { McpBridgeEntry } from "../../state/registry"; +import { + buildDeepAgentsMcpRegisterCommand, + buildDeepAgentsMcpRemoveCommand, +} from "./mcp-bridge-adapter-deepagents"; +import { + buildDeepAgentsMcpStatusCommand, + DEEPAGENTS_MCP_CONFIG_PATH, +} from "./mcp-bridge-adapter-status"; + +const baseEntry: McpBridgeEntry = { + server: "github", + agent: "langchain-deepagents-code", + adapter: "deepagents-config", + url: "https://api.githubcopilot.com/mcp/", + env: ["GITHUB_TOKEN"], + providerName: "alpha-mcp-github", + policyName: "mcp-bridge-github", + addedAt: new Date(0).toISOString(), +}; + +function runDeepAgentsConfigCommand( + command: string, + initialConfig?: Record, +): { + status: number | null; + stdout: string; + stderr: string; + configExists: boolean; + config: Record | null; +} { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-deepagents-mcp-")); + const configPath = path.join(tmp, ".deepagents", ".mcp.json"); + const initializeConfig = + initialConfig === undefined + ? () => undefined + : () => { + fs.mkdirSync(path.dirname(configPath), { recursive: true }); + fs.writeFileSync(configPath, `${JSON.stringify(initialConfig, null, 2)}\n`, { + mode: 0o600, + }); + }; + initializeConfig(); + try { + const result = spawnSync( + "bash", + ["-c", command.replaceAll(DEEPAGENTS_MCP_CONFIG_PATH, configPath)], + { encoding: "utf-8", timeout: 5000 }, + ); + const configExists = fs.existsSync(configPath); + return { + status: result.status, + stdout: result.stdout, + stderr: result.stderr, + configExists, + config: configExists + ? (JSON.parse(fs.readFileSync(configPath, "utf-8")) as Record) + : null, + }; + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } +} + +describe("Deep Agents MCP config adapter", () => { + it("constructs a Deep Agents .mcp.json registration with placeholders", () => { + const command = buildDeepAgentsMcpRegisterCommand(baseEntry); + + expect(DEEPAGENTS_MCP_CONFIG_PATH).toBe("/sandbox/.deepagents/.mcp.json"); + expect(command).toContain(DEEPAGENTS_MCP_CONFIG_PATH); + expect(command).not.toContain('pathlib.Path("/sandbox/.mcp.json")'); + expect(command).toContain("mcpServers"); + expect(command).toContain('\\"type\\":\\"http\\"'); + expect(command).toContain("https://api.githubcopilot.com/mcp/"); + expect(command).toContain("openshell:resolve:env:GITHUB_TOKEN"); + expect(command).toContain("Invalid /sandbox/.deepagents/.mcp.json"); + expect(command).toContain("mcpServers must be an object"); + expect(command).toContain("already exists in /sandbox/.deepagents/.mcp.json"); + }); + + it("creates the Deep Agents config parent on first registration", () => { + const registration = runDeepAgentsConfigCommand(buildDeepAgentsMcpRegisterCommand(baseEntry)); + + expect(registration.status, registration.stderr).toBe(0); + expect(registration.configExists).toBe(true); + expect(registration.config).toEqual({ + mcpServers: { + github: { + type: "http", + url: "https://api.githubcopilot.com/mcp/", + headers: { + Authorization: "Bearer openshell:resolve:env:GITHUB_TOKEN", + }, + }, + }, + }); + }); + + it("rejects unowned config before registration mutates the file", () => { + const initialConfig = { ui: { theme: "dark" } }; + const registration = runDeepAgentsConfigCommand( + buildDeepAgentsMcpRegisterCommand(baseEntry), + initialConfig, + ); + + expect(registration.status).toBe(2); + expect(registration.stderr).toContain("only mcpServers is allowed"); + expect(registration.config).toEqual(initialConfig); + }); + + it("renders the complete registry-owned server projection", () => { + const jiraEntry: McpBridgeEntry = { + ...baseEntry, + server: "jira", + url: "https://mcp.atlassian.com/v1/", + env: ["JIRA_MCP_TOKEN"], + providerName: "alpha-mcp-jira", + policyName: "mcp-bridge-jira", + }; + const registration = runDeepAgentsConfigCommand( + buildDeepAgentsMcpRegisterCommand(jiraEntry, false, [baseEntry, jiraEntry]), + { + mcpServers: { + github: { + type: "http", + url: baseEntry.url, + headers: { + Authorization: "Bearer openshell:resolve:env:GITHUB_TOKEN", + }, + }, + }, + }, + ); + + expect(registration.status, registration.stderr).toBe(0); + expect(registration.config).toEqual({ + mcpServers: { + github: { + type: "http", + url: baseEntry.url, + headers: { Authorization: "Bearer openshell:resolve:env:GITHUB_TOKEN" }, + }, + jira: { + type: "http", + url: jiraEntry.url, + headers: { Authorization: "Bearer openshell:resolve:env:JIRA_MCP_TOKEN" }, + }, + }, + }); + }); + + it("fails Deep Agents removal on corrupt config unless forced", () => { + const normal = buildDeepAgentsMcpRemoveCommand(baseEntry); + const forced = buildDeepAgentsMcpRemoveCommand(baseEntry, true); + + expect(normal).toContain("Invalid /sandbox/.deepagents/.mcp.json"); + expect(normal).toContain('\\"force\\":false'); + expect(normal).toContain("raise SystemExit(2)"); + expect(normal).toContain("Refusing to remove modified MCP server"); + expect(forced).toContain('\\"force\\":true'); + }); + + it("treats every extra Deep Agents server field as ownership drift", () => { + const managedServer = { + type: "http", + url: baseEntry.url, + headers: { + Authorization: "Bearer openshell:resolve:env:GITHUB_TOKEN", + }, + }; + const driftedConfig = { + mcpServers: { + github: { + ...managedServer, + allowedTools: ["get_issue"], + }, + }, + }; + + const status = runDeepAgentsConfigCommand( + buildDeepAgentsMcpStatusCommand(baseEntry), + driftedConfig, + ); + expect(status.status, status.stderr).toBe(0); + expect(status.stdout.trim()).toBe("mismatch"); + + const remove = runDeepAgentsConfigCommand( + buildDeepAgentsMcpRemoveCommand(baseEntry), + driftedConfig, + ); + expect(remove.status).toBe(2); + expect(remove.stderr).toContain("Refusing to remove modified MCP server 'github'"); + expect(remove.config).toEqual(driftedConfig); + }); + + it("deletes an empty managed file but preserves unrelated Deep Agents config", () => { + const managedServer = { + type: "http", + url: baseEntry.url, + headers: { + Authorization: "Bearer openshell:resolve:env:GITHUB_TOKEN", + }, + }; + const onlyManagedServer = runDeepAgentsConfigCommand( + buildDeepAgentsMcpRemoveCommand(baseEntry), + { mcpServers: { github: managedServer } }, + ); + expect(onlyManagedServer.status, onlyManagedServer.stderr).toBe(0); + expect(onlyManagedServer.configExists).toBe(false); + + const withUnrelatedConfig = runDeepAgentsConfigCommand( + buildDeepAgentsMcpRemoveCommand(baseEntry), + { + mcpServers: { github: managedServer }, + ui: { theme: "dark" }, + }, + ); + expect(withUnrelatedConfig.status, withUnrelatedConfig.stderr).toBe(0); + expect(withUnrelatedConfig.configExists).toBe(true); + expect(withUnrelatedConfig.config).toEqual({ ui: { theme: "dark" } }); + }); +}); diff --git a/src/lib/actions/sandbox/mcp-bridge-adapter-deepagents.ts b/src/lib/actions/sandbox/mcp-bridge-adapter-deepagents.ts new file mode 100644 index 00000000000..649082b8956 --- /dev/null +++ b/src/lib/actions/sandbox/mcp-bridge-adapter-deepagents.ts @@ -0,0 +1,223 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { getSandbox, type McpBridgeEntry } from "../../state/registry"; +import { + type AdapterMutationOptions, + type AdapterRegistrationInspection, + inspectAdapterRegistrationCommand, +} from "./mcp-bridge-adapter-inspection"; +import { + buildDeepAgentsMcpStatusCommand, + DEEPAGENTS_MCP_CONFIG_PATH, + deepAgentsManagedServerConfig, + pythonJsonLiteral, +} from "./mcp-bridge-adapter-status"; +import { McpBridgeError } from "./mcp-bridge-contracts"; +import { redactBridgeSecretsForDisplay } from "./mcp-bridge-output"; +import { executeSandboxCommand } from "./process-recovery"; + +const DEEPAGENTS_MCP_CAPABILITY_MARKER = "NEMOCLAW_DEEPAGENTS_MCP_CAPABILITY=1"; +const DEEPAGENTS_MCP_CAPABILITY_COMMAND = + "/usr/local/bin/deepagents-code --nemoclaw-mcp-capability"; + +export function buildDeepAgentsMcpRegisterCommand( + entry: McpBridgeEntry, + replaceExisting = false, + managedEntries: readonly McpBridgeEntry[] = [entry], +): string { + const expectedServers = Object.fromEntries( + managedEntries + .map((managedEntry): [string, Record] => [ + managedEntry.server, + deepAgentsManagedServerConfig(managedEntry), + ]) + .sort(([left], [right]) => left.localeCompare(right)), + ); + const payload = { + server: entry.server, + expected: deepAgentsManagedServerConfig(entry), + expectedServers, + replaceExisting, + }; + return [ + "python3 - <<'PY'", + "import json, os, pathlib, sys", + `payload = json.loads(${pythonJsonLiteral(payload)})`, + `config_path = pathlib.Path(${JSON.stringify(DEEPAGENTS_MCP_CONFIG_PATH)})`, + "data = {}", + "if config_path.exists():", + " try:", + " data = json.loads(config_path.read_text(encoding='utf-8') or '{}')", + " except json.JSONDecodeError as exc:", + ` print(f'Invalid ${DEEPAGENTS_MCP_CONFIG_PATH}: {exc}', file=sys.stderr)`, + " raise SystemExit(2)", + "if not isinstance(data, dict):", + ` print('Invalid ${DEEPAGENTS_MCP_CONFIG_PATH}: expected a JSON object', file=sys.stderr)`, + " raise SystemExit(2)", + "if data and set(data) != {'mcpServers'}:", + ` print('Invalid ${DEEPAGENTS_MCP_CONFIG_PATH}: only mcpServers is allowed', file=sys.stderr)`, + " raise SystemExit(2)", + "servers = data.setdefault('mcpServers', {})", + "if not isinstance(servers, dict):", + ` print('Invalid ${DEEPAGENTS_MCP_CONFIG_PATH}: mcpServers must be an object', file=sys.stderr)`, + " raise SystemExit(2)", + "if payload['server'] in servers and not payload['replaceExisting']:", + ` print(f"MCP server '{payload['server']}' already exists in ${DEEPAGENTS_MCP_CONFIG_PATH} and is not managed by NemoClaw.", file=sys.stderr)`, + " raise SystemExit(2)", + "for name, current in servers.items():", + " if name == payload['server'] and payload['replaceExisting']:", + " continue", + " if payload['expectedServers'].get(name) != current:", + ` print(f"Invalid ${DEEPAGENTS_MCP_CONFIG_PATH}: MCP server '{name}' is not exact registry-owned state", file=sys.stderr)`, + " raise SystemExit(2)", + "data = {'mcpServers': payload['expectedServers']}", + "config_path.parent.mkdir(parents=True, exist_ok=True)", + "tmp = config_path.with_name(config_path.name + '.nemoclaw-mcp.tmp')", + "tmp.write_text(json.dumps(data, indent=2, sort_keys=True) + '\\n', encoding='utf-8')", + "os.chmod(tmp, 0o600)", + "os.replace(tmp, config_path)", + "os.chmod(config_path, 0o600)", + "PY", + ].join("\n"); +} + +export function buildDeepAgentsMcpRemoveCommand(entry: McpBridgeEntry, force = false): string { + const payload = { + server: entry.server, + expected: deepAgentsManagedServerConfig(entry), + force, + }; + return [ + "python3 - <<'PY'", + "import json, os, pathlib, sys", + `payload = json.loads(${pythonJsonLiteral(payload)})`, + `config_path = pathlib.Path(${JSON.stringify(DEEPAGENTS_MCP_CONFIG_PATH)})`, + "if not config_path.exists():", + " raise SystemExit(0)", + "try:", + " data = json.loads(config_path.read_text(encoding='utf-8') or '{}')", + "except json.JSONDecodeError as exc:", + ` print(f'Invalid ${DEEPAGENTS_MCP_CONFIG_PATH}: {exc}', file=sys.stderr)`, + " raise SystemExit(2)", + "if not isinstance(data, dict):", + ` print('Invalid ${DEEPAGENTS_MCP_CONFIG_PATH}: expected a JSON object', file=sys.stderr)`, + " raise SystemExit(2)", + "servers = data.get('mcpServers')", + "if servers is not None and not isinstance(servers, dict):", + ` print('Invalid ${DEEPAGENTS_MCP_CONFIG_PATH}: mcpServers must be an object', file=sys.stderr)`, + " raise SystemExit(2)", + "if isinstance(servers, dict):", + " present = payload['server'] in servers", + " current = servers.get(payload['server'])", + " if present and not payload['force']:", + " if current != payload['expected']:", + ` print(f"Refusing to remove modified MCP server '{payload['server']}' from ${DEEPAGENTS_MCP_CONFIG_PATH}. Use --force to remove it.", file=sys.stderr)`, + " raise SystemExit(2)", + " servers.pop(payload['server'], None)", + " if not servers:", + " data.pop('mcpServers', None)", + " if not data:", + " config_path.unlink()", + " raise SystemExit(0)", + "tmp = config_path.with_name(config_path.name + '.nemoclaw-mcp.tmp')", + "tmp.write_text(json.dumps(data, indent=2, sort_keys=True) + '\\n', encoding='utf-8')", + "os.chmod(tmp, 0o600)", + "os.replace(tmp, config_path)", + "os.chmod(config_path, 0o600)", + "PY", + ].join("\n"); +} + +export function inspectDeepAgentsAdapterRegistration( + sandboxName: string, + entry: McpBridgeEntry, +): AdapterRegistrationInspection { + return inspectAdapterRegistrationCommand( + sandboxName, + entry, + buildDeepAgentsMcpStatusCommand(entry), + ); +} + +export function assertDeepAgentsMcpMutationRuntimeCapability(sandboxName: string): void { + const result = executeSandboxCommand(sandboxName, DEEPAGENTS_MCP_CAPABILITY_COMMAND); + if (result?.status !== 0 || result.stdout.trim() !== DEEPAGENTS_MCP_CAPABILITY_MARKER) { + throw new McpBridgeError( + `LangChain Deep Agents Code sandbox '${sandboxName}' does not contain the managed MCP-aware launcher. Rebuild the sandbox before changing authenticated MCP state.`, + ); + } +} + +function runDeepAgentsAdapterCommand( + sandboxName: string, + entry: Pick, + command: string, + failureMessage: string, + options: AdapterMutationOptions = {}, +): void { + const result = executeSandboxCommand(sandboxName, command); + const output = redactBridgeSecretsForDisplay( + [result?.stdout, result?.stderr].filter(Boolean).join("\n").trim(), + entry, + options.envValues ?? {}, + ); + if (!result || result.status !== 0) { + if (options.bestEffort) return; + throw new McpBridgeError(output || failureMessage); + } +} + +function verifyDeepAgentsAdapterRegistration(sandboxName: string, entry: McpBridgeEntry): void { + const inspection = inspectDeepAgentsAdapterRegistration(sandboxName, entry); + if (inspection.state === "registered") return; + const detail = inspection.state === "error" ? inspection.detail : inspection.state; + throw new McpBridgeError( + `deepagents-config config verification failed after adding '${entry.server}': ${detail}.`, + ); +} + +function registryOwnedDeepAgentsEntries( + sandboxName: string, + entry: McpBridgeEntry, +): McpBridgeEntry[] { + const entries = new Map(); + const bridges = getSandbox(sandboxName)?.mcp?.bridges ?? {}; + for (const bridge of Object.values(bridges)) entries.set(bridge.server, bridge); + entries.set(entry.server, entry); + return [...entries.values()]; +} + +export function registerDeepAgentsAdapter( + sandboxName: string, + entry: McpBridgeEntry, + envValues: Record = {}, + replaceExisting = false, +): void { + runDeepAgentsAdapterCommand( + sandboxName, + entry, + buildDeepAgentsMcpRegisterCommand( + entry, + replaceExisting, + registryOwnedDeepAgentsEntries(sandboxName, entry), + ), + `Deep Agents Code MCP config registration failed for '${entry.server}'.`, + { envValues }, + ); + verifyDeepAgentsAdapterRegistration(sandboxName, entry); +} + +export function unregisterDeepAgentsAdapter( + sandboxName: string, + entry: McpBridgeEntry, + options: AdapterMutationOptions = {}, +): void { + runDeepAgentsAdapterCommand( + sandboxName, + entry, + buildDeepAgentsMcpRemoveCommand(entry, options.force === true), + `Deep Agents Code MCP config removal failed for '${entry.server}'.`, + options, + ); +} diff --git a/src/lib/actions/sandbox/mcp-bridge-adapter-hermes.test.ts b/src/lib/actions/sandbox/mcp-bridge-adapter-hermes.test.ts new file mode 100644 index 00000000000..8eb3efb8af8 --- /dev/null +++ b/src/lib/actions/sandbox/mcp-bridge-adapter-hermes.test.ts @@ -0,0 +1,67 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import type { McpBridgeEntry } from "../../state/registry"; +import { + buildHermesMcpExecArgs, + buildHermesMcpProbeCommand, + buildHermesMcpRegisterCommand, +} from "./mcp-bridge-adapter-hermes"; + +const baseEntry: McpBridgeEntry = { + server: "github", + agent: "hermes", + adapter: "hermes-config", + url: "https://api.githubcopilot.com/mcp/", + env: ["GITHUB_TOKEN"], + providerName: "alpha-mcp-github", + policyName: "mcp-bridge-github", + addedAt: new Date(0).toISOString(), +}; + +describe("Hermes MCP config adapter", () => { + it("constructs a Hermes config registration with placeholders", () => { + const command = buildHermesMcpRegisterCommand(baseEntry); + + expect(command.slice(0, 3)).toEqual([ + "/usr/local/lib/nemoclaw/hermes-mcp-config-transaction.py", + "add", + "--payload", + ]); + expect(JSON.parse(command[3] ?? "{}")).toEqual({ + server: "github", + url: "https://api.githubcopilot.com/mcp/", + headers: { Authorization: "Bearer openshell:resolve:env:GITHUB_TOKEN" }, + replace_existing: false, + }); + expect(buildHermesMcpExecArgs("hermes-box", command)).toEqual([ + "sandbox", + "exec", + "--name", + "hermes-box", + "--timeout", + "620", + "--no-tty", + "--", + ...command, + ]); + expect(buildHermesMcpProbeCommand()).toEqual([ + "/usr/local/lib/nemoclaw/hermes-mcp-config-transaction.py", + "probe", + ]); + expect(buildHermesMcpExecArgs("hermes-box", buildHermesMcpProbeCommand(), 30)).toEqual([ + "sandbox", + "exec", + "--name", + "hermes-box", + "--timeout", + "30", + "--no-tty", + "--", + "/usr/local/lib/nemoclaw/hermes-mcp-config-transaction.py", + "probe", + ]); + }); +}); diff --git a/src/lib/actions/sandbox/mcp-bridge-adapter-hermes.ts b/src/lib/actions/sandbox/mcp-bridge-adapter-hermes.ts new file mode 100644 index 00000000000..f7369f23813 --- /dev/null +++ b/src/lib/actions/sandbox/mcp-bridge-adapter-hermes.ts @@ -0,0 +1,315 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { runOpenshellProviderCommand } from "../../actions/global"; +import { waitUntil } from "../../core/wait"; +import { isShieldsDown } from "../../shields"; +import type { McpBridgeEntry } from "../../state/registry"; +import { classifyGatewayRestartFailure } from "./gateway-restart"; +import { + type AdapterMutationOptions, + type AdapterRegistrationInspection, + inspectAdapterRegistrationCommand, +} from "./mcp-bridge-adapter-inspection"; +import { buildHermesMcpStatusCommand, entryHeaders } from "./mcp-bridge-adapter-status"; +import { McpBridgeError } from "./mcp-bridge-contracts"; +import { commandOutput, redactBridgeSecretsForDisplay } from "./mcp-bridge-output"; +import { executeGatewaySupervisorAction } from "./process-recovery"; + +const HERMES_MCP_TRANSACTION_HELPER = "/usr/local/lib/nemoclaw/hermes-mcp-config-transaction.py"; +const HERMES_MCP_EXEC_TIMEOUT_SECONDS = 620; +const HERMES_MCP_PROBE_TIMEOUT_SECONDS = 30; +const HERMES_MCP_STARTUP_TIMEOUT_SECONDS = 90; +const HERMES_MCP_RECOVERY_TIMEOUT_MS = 210_000; +const HERMES_MCP_INITIAL_PROBE_ATTEMPTS = 3; +const HERMES_MCP_GATEWAY_NOT_READY = "Hermes gateway is not running for managed MCP reload"; +const HERMES_MCP_LIFECYCLE_NOT_READY = + "Hermes gateway is not running under the managed service lifecycle"; + +export function buildHermesMcpRegisterCommand( + entry: McpBridgeEntry, + replaceExisting = false, +): string[] { + const payload = { + server: entry.server, + url: entry.url, + headers: entryHeaders(entry), + replace_existing: replaceExisting, + }; + return [HERMES_MCP_TRANSACTION_HELPER, "add", "--payload", JSON.stringify(payload)]; +} + +function buildHermesMcpRemoveCommand(entry: McpBridgeEntry, force = false): string[] { + const payload = { + server: entry.server, + url: entry.url, + headers: entryHeaders(entry), + force, + }; + return [HERMES_MCP_TRANSACTION_HELPER, "remove", "--payload", JSON.stringify(payload)]; +} + +export function buildHermesMcpExecArgs( + sandboxName: string, + command: readonly string[], + timeoutSeconds = HERMES_MCP_EXEC_TIMEOUT_SECONDS, +): string[] { + return [ + "sandbox", + "exec", + "--name", + sandboxName, + "--timeout", + String(timeoutSeconds), + "--no-tty", + "--", + ...command, + ]; +} + +export function buildHermesMcpProbeCommand(): string[] { + return [HERMES_MCP_TRANSACTION_HELPER, "probe"]; +} + +export function inspectHermesAdapterRegistration( + sandboxName: string, + entry: McpBridgeEntry, +): AdapterRegistrationInspection { + return inspectAdapterRegistrationCommand(sandboxName, entry, buildHermesMcpStatusCommand(entry)); +} + +function parseLastJsonObject(output: string): Record | null { + for (const line of output.trim().split(/\r?\n/).reverse()) { + try { + const parsed = JSON.parse(line) as unknown; + if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) { + return parsed as Record; + } + } catch { + // OpenShell may frame diagnostics around the command's JSON line. + } + } + return null; +} + +/** Refuse an in-sandbox Hermes config mutation while config is locked. */ +export function assertHermesMcpConfigMutationAllowed(sandboxName: string): void { + if (isShieldsDown(sandboxName, false)) return; + throw new McpBridgeError( + `Hermes sandbox '${sandboxName}' has shields up or an unreadable shields posture. Run \`nemohermes ${sandboxName} shields down --timeout 15m --reason "MCP maintenance"\` before changing MCP configuration.`, + ); +} + +function isExactGatewayRecoveryCompletion( + result: ReturnType, +): boolean { + if (!result || result.status !== 0 || result.stderr.trim()) return false; + const lines = result.stdout.trim().split(/\r?\n/); + if (lines.length !== 2) return false; + const completion = lines[0]?.match( + /^v1 ([0-9a-f]{64}) complete (?:ok|already-running) ([0-9]+) ([1-9][0-9]*)$/, + ); + return completion !== null && lines[1] === `GATEWAY_PID=${completion[3]}`; +} + +/** + * Prove the running Hermes sandbox contains the packaged transaction helper + * and can invoke it through OpenShell current main's ordinary exec path before + * changing a global provider, policy, attachment, or adapter. + */ +export function assertHermesMcpMutationRuntimeCapability(sandboxName: string): void { + assertHermesMcpConfigMutationAllowed(sandboxName); + let lastDetail = ""; + const probe = (): boolean => { + let result: ReturnType; + try { + result = runOpenshellProviderCommand( + buildHermesMcpExecArgs( + sandboxName, + buildHermesMcpProbeCommand(), + HERMES_MCP_PROBE_TIMEOUT_SECONDS, + ), + { + ignoreError: true, + stdio: ["ignore", "pipe", "pipe"], + timeout: 45_000, + }, + ); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + throw new McpBridgeError( + `Hermes sandbox '${sandboxName}' cannot invoke the managed MCP transaction helper. Rebuild the sandbox before changing authenticated MCP state${detail ? `: ${detail}` : "."}`, + ); + } + const response = parseLastJsonObject(result.stdout || ""); + if (result.status === 0 && !result.error && response?.ok === true) return true; + lastDetail = commandOutput(result).trim(); + if (lastDetail === HERMES_MCP_GATEWAY_NOT_READY) return false; + if (lastDetail === HERMES_MCP_LIFECYCLE_NOT_READY) { + throw new McpBridgeError( + `Hermes sandbox '${sandboxName}' is not running the managed service lifecycle required for authenticated MCP changes. Run \`nemoclaw ${sandboxName} recover\` and retry.`, + ); + } + throw new McpBridgeError( + `Hermes sandbox '${sandboxName}' cannot invoke the managed MCP transaction helper. Rebuild the sandbox before changing authenticated MCP state${lastDetail ? `: ${lastDetail}` : "."}`, + ); + }; + + if ( + waitUntil(probe, { + maxAttempts: HERMES_MCP_INITIAL_PROBE_ATTEMPTS, + initialIntervalMs: 1_000, + maxIntervalMs: 1_000, + backoffFactor: 1, + }) + ) { + return; + } + + let recovery: ReturnType = null; + let recoveryFailureDetail = ""; + try { + recovery = executeGatewaySupervisorAction( + sandboxName, + "recover", + HERMES_MCP_RECOVERY_TIMEOUT_MS, + ); + } catch (error) { + recoveryFailureDetail = error instanceof Error ? error.message : String(error); + } + const recoveryCompleted = isExactGatewayRecoveryCompletion(recovery); + if (!recoveryCompleted) { + recoveryFailureDetail ||= recovery ? commandOutput(recovery).trim() : "no controller result"; + const classification = classifyGatewayRestartFailure(recovery); + const claimsInvalidCompletion = + recovery !== null && (recovery.status === 0 || recovery.stdout.trim().length > 0); + const terminalIntegrityFailure = + claimsInvalidCompletion || + classification.layer === "secret-boundary refusal" || + classification.layer === "unsafe config path" || + classification.layer === "config hash mismatch" || + classification.layer === "health timeout" || + recoveryFailureDetail.includes("SUPERVISOR_REBUILD_REQUIRED") || + recoveryFailureDetail.includes("SUPERVISOR_UNSAFE_CONTROL_DIR") || + recoveryFailureDetail.includes("SUPERVISOR_BUSY") || + recoveryFailureDetail.includes("SUPERVISOR_INVALID_") || + recoveryFailureDetail.includes("GATEWAY_GUARDS_MISSING"); + if (terminalIntegrityFailure) { + throw new McpBridgeError( + `Hermes sandbox '${sandboxName}' managed gateway recovery failed before MCP mutation: ${recoveryFailureDetail || classification.detail}.`, + ); + } + } + + // A privileged controller completion never authorizes mutation by itself. + // Even when transient controller unavailability lets the managed lifecycle + // finish naturally, the ordinary sandbox identity must freshly prove the + // packaged helper and a stable, trusted gateway topology before any MCP + // provider, policy, attachment, or adapter side effect. + if (!waitUntil(probe, HERMES_MCP_STARTUP_TIMEOUT_SECONDS, 1_000)) { + const recoveryDetail = recoveryFailureDetail + ? ` Managed recovery attempt did not complete: ${recoveryFailureDetail}.` + : ""; + throw new McpBridgeError( + `Hermes sandbox '${sandboxName}' cannot invoke the managed MCP transaction helper after managed gateway recovery. Rebuild the sandbox before changing authenticated MCP state${lastDetail ? `: ${lastDetail}` : "."}${recoveryDetail}`, + ); + } +} + +function runHermesAdapterCommand( + sandboxName: string, + entry: McpBridgeEntry, + command: readonly string[], + failureMessage: string, + options: AdapterMutationOptions & { requireReload?: boolean } = {}, +): void { + // OpenShell current main executes this fixed helper argv with ordinary + // workload authority. There is no listener, proxy, persistent service, or + // MCP traffic on this control path; argv carries only an OpenShell + // placeholder and endpoint metadata. + let result: ReturnType; + try { + result = runOpenshellProviderCommand(buildHermesMcpExecArgs(sandboxName, command), { + ignoreError: true, + stdio: ["ignore", "pipe", "pipe"], + // The remote supervisor enforces 620s; keep a small transport margin so + // remote termination is observed before this local subprocess is killed. + timeout: 645_000, + }); + } catch (error) { + if (options.bestEffort) return; + const detail = error instanceof Error ? error.message : String(error); + throw new McpBridgeError( + redactBridgeSecretsForDisplay(detail, entry, options.envValues ?? {}) || failureMessage, + ); + } + const output = redactBridgeSecretsForDisplay( + commandOutput(result, options.envValues ?? {}), + entry, + options.envValues ?? {}, + ); + if (result.status !== 0 || result.error) { + if (options.bestEffort) return; + const errorDetail = result.error + ? redactBridgeSecretsForDisplay(result.error.message, entry, options.envValues ?? {}) + : ""; + throw new McpBridgeError(errorDetail || output || failureMessage); + } + const stdout = result.stdout || ""; + const response = parseLastJsonObject(stdout); + if ( + response?.ok !== true || + typeof response.changed !== "boolean" || + typeof response.reloaded !== "boolean" + ) { + if (options.bestEffort) return; + throw new McpBridgeError( + `Hermes MCP lifecycle command returned an invalid response for '${entry.server}'.`, + ); + } + if (options.requireReload && response.reloaded !== true) { + if (options.bestEffort) return; + throw new McpBridgeError( + `Hermes gateway was not running, so MCP server '${entry.server}' was not loaded.`, + ); + } +} + +function verifyHermesAdapterRegistration(sandboxName: string, entry: McpBridgeEntry): void { + const inspection = inspectHermesAdapterRegistration(sandboxName, entry); + if (inspection.state === "registered") return; + const detail = inspection.state === "error" ? inspection.detail : inspection.state; + throw new McpBridgeError( + `hermes-config config verification failed after adding '${entry.server}': ${detail}.`, + ); +} + +export function registerHermesAdapter( + sandboxName: string, + entry: McpBridgeEntry, + envValues: Record = {}, + replaceExisting = false, +): void { + runHermesAdapterCommand( + sandboxName, + entry, + buildHermesMcpRegisterCommand(entry, replaceExisting), + `Hermes MCP config registration failed for '${entry.server}'.`, + { envValues, requireReload: true }, + ); + verifyHermesAdapterRegistration(sandboxName, entry); +} + +export function unregisterHermesAdapter( + sandboxName: string, + entry: McpBridgeEntry, + options: AdapterMutationOptions = {}, +): void { + runHermesAdapterCommand( + sandboxName, + entry, + buildHermesMcpRemoveCommand(entry, options.force === true), + `Hermes MCP config removal failed for '${entry.server}'.`, + options, + ); +} diff --git a/src/lib/actions/sandbox/mcp-bridge-adapter-inspection.test.ts b/src/lib/actions/sandbox/mcp-bridge-adapter-inspection.test.ts new file mode 100644 index 00000000000..e286d2562ba --- /dev/null +++ b/src/lib/actions/sandbox/mcp-bridge-adapter-inspection.test.ts @@ -0,0 +1,33 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import type { McpBridgeEntry } from "../../state/registry"; +import { parseAdapterRegistrationInspection } from "./mcp-bridge-adapter-inspection"; + +const baseEntry: McpBridgeEntry = { + server: "github", + agent: "openclaw", + adapter: "mcporter", + url: "https://api.githubcopilot.com/mcp/", + env: ["GITHUB_TOKEN"], + providerName: "alpha-mcp-github", + policyName: "mcp-bridge-github", + addedAt: new Date(0).toISOString(), +}; + +describe("MCP adapter registration inspection", () => { + it("uses stdout ownership state even when the adapter emits a runtime warning", () => { + expect( + parseAdapterRegistrationInspection( + { + status: 0, + stdout: "absent\n", + stderr: "(node:1200) [UNDICI-EHPA] Warning: EnvHttpProxyAgent is experimental", + }, + baseEntry, + ), + ).toEqual({ state: "absent" }); + }); +}); diff --git a/src/lib/actions/sandbox/mcp-bridge-adapter-inspection.ts b/src/lib/actions/sandbox/mcp-bridge-adapter-inspection.ts new file mode 100644 index 00000000000..165f70ecdff --- /dev/null +++ b/src/lib/actions/sandbox/mcp-bridge-adapter-inspection.ts @@ -0,0 +1,54 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { McpBridgeEntry } from "../../state/registry"; +import { redactBridgeSecretsForDisplay } from "./mcp-bridge-output"; +import { executeSandboxCommand, type SandboxCommandResult } from "./process-recovery"; + +export type AdapterRegistrationInspection = + | { state: "absent" | "registered" | "mismatch" } + | { state: "error"; detail: string }; + +export type AdapterMutationOptions = { + force?: boolean; + bestEffort?: boolean; + envValues?: Record; +}; + +export function parseAdapterRegistrationInspection( + result: SandboxCommandResult, + entry: McpBridgeEntry, +): AdapterRegistrationInspection { + const output = [result.stdout, result.stderr].filter(Boolean).join("\n").trim(); + if (result.status !== 0) { + return { + state: "error", + detail: + redactBridgeSecretsForDisplay(output, entry) || + `MCP adapter inspection exited ${result.status}.`, + }; + } + // Successful inspection commands write exactly one ownership state to + // stdout. Runtime warnings belong on stderr and must not replace that state. + const state = result.stdout.trim().split(/\r?\n/).at(-1)?.trim(); + if (state === "absent" || state === "registered" || state === "mismatch") { + return { state }; + } + return { + state: "error", + detail: redactBridgeSecretsForDisplay( + output || "MCP adapter inspection returned no state.", + entry, + ), + }; +} + +export function inspectAdapterRegistrationCommand( + sandboxName: string, + entry: McpBridgeEntry, + command: string, +): AdapterRegistrationInspection { + const result = executeSandboxCommand(sandboxName, command); + if (!result) return { state: "error", detail: "sandbox unreachable" }; + return parseAdapterRegistrationInspection(result, entry); +} diff --git a/src/lib/actions/sandbox/mcp-bridge-adapter-openclaw.test.ts b/src/lib/actions/sandbox/mcp-bridge-adapter-openclaw.test.ts new file mode 100644 index 00000000000..df596ddca36 --- /dev/null +++ b/src/lib/actions/sandbox/mcp-bridge-adapter-openclaw.test.ts @@ -0,0 +1,167 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +import type { McpBridgeEntry } from "../../state/registry"; +import { + buildOpenClawMcporterRegisterCommand, + buildOpenClawMcporterRemoveCommand, + MCPORTER_VERSION, +} from "./mcp-bridge-adapter-openclaw"; +import { + buildOpenClawMcporterInspectCommand, + mcporterHeadersMatchExpected, +} from "./mcp-bridge-adapter-status"; + +const baseEntry: McpBridgeEntry = { + server: "github", + agent: "openclaw", + adapter: "mcporter", + url: "https://api.githubcopilot.com/mcp/", + env: ["GITHUB_TOKEN"], + providerName: "alpha-mcp-github", + policyName: "mcp-bridge-github", + addedAt: new Date(0).toISOString(), +}; + +describe("OpenClaw mcporter MCP adapter", () => { + it("constructs a mcporter HTTP registration with OpenShell env placeholders", () => { + const command = buildOpenClawMcporterRegisterCommand(baseEntry); + + expect(command).toContain("'mcporter' 'config' 'add' 'github'"); + expect(command).toContain("'--url' 'https://api.githubcopilot.com/mcp/'"); + expect(command).toContain( + "'--header' 'Authorization=Bearer openshell:resolve:env:GITHUB_TOKEN'", + ); + expect(command).toContain("'--scope' 'home'"); + expect(command).toContain("already exists in mcporter config"); + expect(command).not.toContain("fake-secret"); + }); + + it("accepts only mcporter's synthesized HTTP Accept header in ownership checks", () => { + const expected = { + Authorization: "Bearer openshell:resolve:env:GITHUB_TOKEN", + }; + + expect( + mcporterHeadersMatchExpected( + { + ...expected, + accept: "application/json, text/event-stream", + }, + expected, + ), + ).toBe(true); + expect(mcporterHeadersMatchExpected(expected, expected)).toBe(true); + expect( + mcporterHeadersMatchExpected( + { + ...expected, + accept: "application/json", + }, + expected, + ), + ).toBe(false); + expect( + mcporterHeadersMatchExpected( + { + ...expected, + accept: "application/json, text/event-stream", + "x-unowned": "drift", + }, + expected, + ), + ).toBe(false); + expect( + mcporterHeadersMatchExpected( + { + Authorization: "Bearer changed", + accept: "application/json, text/event-stream", + }, + expected, + ), + ).toBe(false); + }); + + it("uses the normalized-header ownership rule in mcporter inspect and remove commands", () => { + const temp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mcporter-owner-")); + try { + const fakeMcporter = path.join(temp, "mcporter"); + const removeMarker = path.join(temp, "removed"); + fs.writeFileSync( + fakeMcporter, + [ + "#!/usr/bin/env node", + 'const fs = require("node:fs");', + 'const headers = JSON.parse(process.env.FAKE_MCPORTER_HEADERS || "{}");', + 'if (process.argv[3] === "get") {', + " process.stdout.write(JSON.stringify({", + ' name: "github", transport: "http",', + ' baseUrl: "https://api.githubcopilot.com/mcp/", headers,', + " }));", + " process.exit(0);", + "}", + 'if (process.argv[3] === "remove") {', + ' fs.writeFileSync(process.env.FAKE_MCPORTER_REMOVE_MARKER, "removed");', + " process.exit(0);", + "}", + "process.exit(3);", + ].join("\n"), + { mode: 0o755 }, + ); + const run = (command: string, headers: Record) => + spawnSync("/bin/sh", ["-c", command], { + encoding: "utf8", + env: { + ...process.env, + PATH: `${temp}:${process.env.PATH ?? ""}`, + FAKE_MCPORTER_HEADERS: JSON.stringify(headers), + FAKE_MCPORTER_REMOVE_MARKER: removeMarker, + }, + }); + const normalizedHeaders = { + Authorization: "Bearer openshell:resolve:env:GITHUB_TOKEN", + accept: "application/json, text/event-stream", + }; + + const inspect = run(buildOpenClawMcporterInspectCommand(baseEntry, true), normalizedHeaders); + expect(inspect.status).toBe(0); + expect(inspect.stdout.trim()).toBe("registered"); + + const remove = run(buildOpenClawMcporterRemoveCommand(baseEntry), normalizedHeaders); + expect(remove.status).toBe(0); + expect(fs.readFileSync(removeMarker, "utf8")).toBe("removed"); + + fs.rmSync(removeMarker, { force: true }); + const drifted = run(buildOpenClawMcporterRemoveCommand(baseEntry), { + ...normalizedHeaders, + "x-unowned": "drift", + }); + expect(drifted.status).toBe(2); + expect(drifted.stderr).toContain("Refusing to remove modified mcporter MCP server"); + expect(fs.existsSync(removeMarker)).toBe(false); + } finally { + fs.rmSync(temp, { recursive: true, force: true }); + } + }); + + it("does not fabricate Authorization headers for legacy entries without credentials", () => { + const command = buildOpenClawMcporterRegisterCommand({ + ...baseEntry, + env: [], + }); + + expect(command).not.toContain("Authorization="); + expect(command).toContain("'--url' 'https://api.githubcopilot.com/mcp/'"); + }); + + it("keeps the mcporter runtime pin visible for image tests", () => { + expect(MCPORTER_VERSION).toBe("0.7.3"); + }); +}); diff --git a/src/lib/actions/sandbox/mcp-bridge-adapter-openclaw.ts b/src/lib/actions/sandbox/mcp-bridge-adapter-openclaw.ts new file mode 100644 index 00000000000..46248c50613 --- /dev/null +++ b/src/lib/actions/sandbox/mcp-bridge-adapter-openclaw.ts @@ -0,0 +1,160 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { shellQuote } from "../../runner"; +import type { McpBridgeEntry } from "../../state/registry"; +import { + type AdapterMutationOptions, + type AdapterRegistrationInspection, + inspectAdapterRegistrationCommand, +} from "./mcp-bridge-adapter-inspection"; +import { + authorizationValue, + buildOpenClawMcporterInspectCommand, + entryHeaders, + mcporterHeaderMatcherSource, + pythonJsonLiteral, +} from "./mcp-bridge-adapter-status"; +import { McpBridgeError } from "./mcp-bridge-contracts"; +import { redactBridgeSecretsForDisplay } from "./mcp-bridge-output"; +import { executeSandboxCommand } from "./process-recovery"; + +export const MCPORTER_VERSION = "0.7.3"; + +function ensureMcporter(sandboxName: string): void { + const check = executeSandboxCommand(sandboxName, "command -v mcporter"); + if (check?.status === 0 && check.stdout.trim()) return; + throw new McpBridgeError( + `mcporter is not available in sandbox '${sandboxName}'. Rebuild with a NemoClaw image that includes mcporter@${MCPORTER_VERSION}.`, + ); +} + +export function buildOpenClawMcporterRegisterCommand( + entry: McpBridgeEntry, + replaceExisting = false, +): string { + const args = ["mcporter", "config", "add", entry.server, "--url", entry.url]; + const authorization = authorizationValue(entry); + if (authorization) args.push("--header", `Authorization=${authorization}`); + args.push("--scope", "home"); + const addCommand = args.map(shellQuote).join(" "); + if (replaceExisting) return addCommand; + const getCommand = ["mcporter", "config", "get", entry.server, "--json"] + .map(shellQuote) + .join(" "); + return [ + `if ${getCommand} >/dev/null 2>&1; then`, + ` echo ${shellQuote(`MCP server '${entry.server}' already exists in mcporter config and is not managed by NemoClaw.`)} >&2`, + " exit 2", + "fi", + addCommand, + ].join("\n"); +} + +export function buildOpenClawMcporterRemoveCommand(entry: McpBridgeEntry, force = false): string { + const payload = { + server: entry.server, + url: entry.url, + headers: entryHeaders(entry), + force, + }; + return [ + "node - <<'NODE'", + 'const { spawnSync } = require("node:child_process");', + `const expected = JSON.parse(${pythonJsonLiteral(payload)});`, + 'const get = spawnSync("mcporter", ["config", "get", expected.server, "--json"], { encoding: "utf8" });', + "if (get.error) { console.error(get.error.message); process.exit(3); }", + 'const getDetail = `${get.stderr || ""}\n${get.stdout || ""}`;', + "const absent = get.status !== 0 && /not\\s+found|does\\s+not\\s+exist|unknown\\s+server/i.test(getDetail);", + "if (absent) process.exit(0);", + "if (get.status !== 0) { console.error(getDetail.trim()); process.exit(3); }", + "let actual = null; try { actual = JSON.parse(get.stdout); } catch {}", + 'const headers = actual && actual.headers && typeof actual.headers === "object" ? actual.headers : {};', + mcporterHeaderMatcherSource(), + 'const registered = !!actual && actual.name === expected.server && actual.transport === "http" && actual.baseUrl === expected.url && mcporterHeadersMatchExpected(headers, expected.headers);', + "if (!registered && !expected.force) { console.error(`Refusing to remove modified mcporter MCP server '${expected.server}'. Use --force to remove it.`); process.exit(2); }", + 'const remove = spawnSync("mcporter", ["config", "remove", expected.server], { encoding: "utf8" });', + "if (remove.stdout) process.stdout.write(remove.stdout);", + "if (remove.stderr) process.stderr.write(remove.stderr);", + "if (remove.error) { console.error(remove.error.message); process.exit(3); }", + 'const removeDetail = `${remove.stderr || ""}\n${remove.stdout || ""}`;', + "if (remove.status !== 0 && /not\\s+found|does\\s+not\\s+exist|unknown\\s+server/i.test(removeDetail)) process.exit(0);", + "process.exit(remove.status === null ? 3 : remove.status);", + "NODE", + ].join("\n"); +} + +export function inspectOpenClawAdapterRegistration( + sandboxName: string, + entry: McpBridgeEntry, +): AdapterRegistrationInspection { + return inspectAdapterRegistrationCommand( + sandboxName, + entry, + buildOpenClawMcporterInspectCommand(entry, false), + ); +} + +export function registerOpenClawAdapter( + sandboxName: string, + entry: McpBridgeEntry, + envValues: Record = {}, + replaceExisting = false, +): void { + ensureMcporter(sandboxName); + const result = executeSandboxCommand( + sandboxName, + buildOpenClawMcporterRegisterCommand(entry, replaceExisting), + ); + const output = redactBridgeSecretsForDisplay( + [result?.stdout, result?.stderr].filter(Boolean).join("\n").trim(), + entry, + envValues, + ); + if (!result || result.status !== 0) { + throw new McpBridgeError(output || `mcporter config add failed for '${entry.server}'.`); + } + + // A zero exit from `config add` proves only that mcporter accepted the + // command. Re-read the persisted definition before claiming ownership so a + // changed mcporter normalization/schema cannot commit an entry that differs + // from the URL and opaque OpenShell placeholder NemoClaw intended. + const verification = executeSandboxCommand( + sandboxName, + buildOpenClawMcporterInspectCommand(entry, true), + ); + const verificationOutput = redactBridgeSecretsForDisplay( + [verification?.stdout, verification?.stderr].filter(Boolean).join("\n").trim(), + entry, + envValues, + ); + if ( + !verification || + verification.status !== 0 || + verification.stdout.trim().split(/\r?\n/).at(-1) !== "registered" + ) { + throw new McpBridgeError( + `mcporter config verification failed after adding '${entry.server}'${verificationOutput ? `: ${verificationOutput}` : "."}`, + ); + } +} + +export function unregisterOpenClawAdapter( + sandboxName: string, + entry: McpBridgeEntry, + options: AdapterMutationOptions = {}, +): void { + const result = executeSandboxCommand( + sandboxName, + buildOpenClawMcporterRemoveCommand(entry, options.force === true), + ); + const output = redactBridgeSecretsForDisplay( + [result?.stdout, result?.stderr].filter(Boolean).join("\n").trim(), + entry, + options.envValues ?? {}, + ); + if (!result || result.status !== 0) { + if (options.bestEffort) return; + throw new McpBridgeError(output || `mcporter config remove failed for '${entry.server}'.`); + } +} diff --git a/src/lib/actions/sandbox/mcp-bridge-adapter-registration.test.ts b/src/lib/actions/sandbox/mcp-bridge-adapter-registration.test.ts new file mode 100644 index 00000000000..13fe92a2d0c --- /dev/null +++ b/src/lib/actions/sandbox/mcp-bridge-adapter-registration.test.ts @@ -0,0 +1,116 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import type { AgentMcpAdapter } from "../../agent/defs"; +import type { McpBridgeEntry } from "../../state/registry"; + +const mocks = vi.hoisted(() => ({ + executeSandboxCommand: vi.fn(), + executeGatewaySupervisorAction: vi.fn(), + runOpenshellProviderCommand: vi.fn(), +})); + +vi.mock("./process-recovery", () => ({ + executeSandboxCommand: mocks.executeSandboxCommand, + executeGatewaySupervisorAction: mocks.executeGatewaySupervisorAction, +})); + +vi.mock("../../actions/global", () => ({ + runOpenshellProviderCommand: mocks.runOpenshellProviderCommand, +})); + +import { + buildDeepAgentsMcpStatusCommand, + buildHermesMcpStatusCommand, + registerAgentAdapter, +} from "./mcp-bridge-adapters"; + +const baseEntry: McpBridgeEntry = { + server: "github", + agent: "hermes", + adapter: "hermes-config", + url: "https://api.githubcopilot.com/mcp/", + env: ["GITHUB_TOKEN"], + providerName: "alpha-mcp-github", + policyName: "mcp-bridge-github", + addedAt: new Date(0).toISOString(), +}; + +const lifecycleSuccess = { + status: 0, + stdout: '{"changed":true,"ok":true,"reloaded":true}\n', + stderr: "", +}; + +const commandSuccess = { status: 0, stdout: "", stderr: "" }; +const registered = { status: 0, stdout: "registered\n", stderr: "" }; +const mismatch = { status: 0, stdout: "mismatch\n", stderr: "" }; + +interface AdapterCase { + name: string; + adapter: AgentMcpAdapter; + entry: McpBridgeEntry; + arrangeInspection: (result: typeof registered) => void; + statusCommand: (entry: McpBridgeEntry) => string; +} + +const adapterCases: AdapterCase[] = [ + { + name: "Hermes", + adapter: "hermes-config", + entry: baseEntry, + arrangeInspection: (result) => { + mocks.runOpenshellProviderCommand.mockReturnValue(lifecycleSuccess); + mocks.executeSandboxCommand.mockReturnValue(result); + }, + statusCommand: buildHermesMcpStatusCommand, + }, + { + name: "Deep Agents", + adapter: "deepagents-config", + entry: { + ...baseEntry, + agent: "langchain-deepagents-code", + adapter: "deepagents-config", + }, + arrangeInspection: (result) => { + mocks.executeSandboxCommand.mockReturnValueOnce(commandSuccess).mockReturnValueOnce(result); + }, + statusCommand: buildDeepAgentsMcpStatusCommand, + }, +]; + +describe.each(adapterCases)("$name MCP adapter registration", (adapterCase) => { + beforeEach(() => { + mocks.executeSandboxCommand.mockReset(); + mocks.executeGatewaySupervisorAction.mockReset(); + mocks.runOpenshellProviderCommand.mockReset(); + }); + + it("re-reads the persisted definition before registration succeeds", () => { + adapterCase.arrangeInspection(registered); + + expect(() => + registerAgentAdapter("alpha", adapterCase.adapter, adapterCase.entry, { + GITHUB_TOKEN: "host-only-secret", + }), + ).not.toThrow(); + + expect(mocks.executeSandboxCommand).toHaveBeenLastCalledWith( + "alpha", + adapterCase.statusCommand(adapterCase.entry), + ); + }); + + it("rejects a persisted definition that differs from the requested entry", () => { + adapterCase.arrangeInspection(mismatch); + + expect(() => + registerAgentAdapter("alpha", adapterCase.adapter, adapterCase.entry, { + GITHUB_TOKEN: "host-only-secret", + }), + ).toThrow(`${adapterCase.adapter} config verification failed after adding 'github': mismatch.`); + }); +}); diff --git a/src/lib/actions/sandbox/mcp-bridge-adapter-status.ts b/src/lib/actions/sandbox/mcp-bridge-adapter-status.ts new file mode 100644 index 00000000000..172034dfe8b --- /dev/null +++ b/src/lib/actions/sandbox/mcp-bridge-adapter-status.ts @@ -0,0 +1,162 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { McpBridgeEntry } from "../../state/registry"; + +// The pinned Deep Agents Code release auto-discovers this as the user-level MCP config. +// `/sandbox/.mcp.json` is project-level and is intentionally rejected by +// headless `dcode -n` unless project MCP has been separately trusted. +export const DEEPAGENTS_MCP_CONFIG_PATH = "/sandbox/.deepagents/.mcp.json"; +const DEFAULT_AUTH_HEADER = "Authorization"; +const DEFAULT_AUTH_SCHEME = "Bearer"; + +function authPlaceholder(entry: Pick): string | null { + const envName = entry.env[0]; + return envName ? `openshell:resolve:env:${envName}` : null; +} + +export function authorizationValue(entry: Pick): string | null { + const placeholder = authPlaceholder(entry); + return placeholder ? `${DEFAULT_AUTH_SCHEME} ${placeholder}` : null; +} + +export function entryHeaders(entry: Pick): Record { + const authorization = authorizationValue(entry); + return authorization ? { [DEFAULT_AUTH_HEADER]: authorization } : {}; +} + +export function pythonJsonLiteral(value: unknown): string { + return JSON.stringify(JSON.stringify(value)); +} + +/** + * mcporter@0.7.3 normalizes every HTTP definition returned by + * `config get --json` with an `accept: application/json, text/event-stream` + * header, even when that header is absent from the persisted config. Treat + * only that synthesized header as equivalent; every persisted/other header + * remains part of the ownership fingerprint. + * + * This function is also serialized into the in-sandbox inspection commands, + * so keep it self-contained (no references to module-scope values). + */ +export function mcporterHeadersMatchExpected( + actual: unknown, + expected: Record, +): boolean { + if (!actual || typeof actual !== "object" || Array.isArray(actual)) { + return false; + } + const actualHeaders = actual as Record; + for (const [name, value] of Object.entries(expected)) { + if (actualHeaders[name] !== value) return false; + } + const extraNames = Object.keys(actualHeaders).filter((name) => !Object.hasOwn(expected, name)); + if (extraNames.length === 0) return true; + if (extraNames.length !== 1) return false; + const [extraName] = extraNames; + return ( + extraName.toLowerCase() === "accept" && + actualHeaders[extraName] === "application/json, text/event-stream" + ); +} + +export function mcporterHeaderMatcherSource(): string { + return `const mcporterHeadersMatchExpected = ${mcporterHeadersMatchExpected.toString()};`; +} + +function hermesManagedServerConfig(entry: McpBridgeEntry): Record { + const headers = entryHeaders(entry); + return { + url: entry.url, + enabled: true, + timeout: 120, + connect_timeout: 60, + tools: { resources: true, prompts: true }, + ...(Object.keys(headers).length > 0 ? { headers } : {}), + }; +} + +export function deepAgentsManagedServerConfig(entry: McpBridgeEntry): Record { + const headers = entryHeaders(entry); + return { + type: "http", + url: entry.url, + ...(Object.keys(headers).length > 0 ? { headers } : {}), + }; +} + +export function buildHermesMcpStatusCommand(entry: McpBridgeEntry): string { + const payload = { + server: entry.server, + expected: hermesManagedServerConfig(entry), + }; + return [ + "/opt/hermes/.venv/bin/python - <<'PY'", + "import json, pathlib, yaml", + `payload = json.loads(${pythonJsonLiteral(payload)})`, + 'config_path = pathlib.Path("/sandbox/.hermes/config.yaml")', + "data = yaml.safe_load(config_path.read_text(encoding='utf-8')) if config_path.exists() else {}", + "servers = data.get('mcp_servers') if isinstance(data, dict) else None", + "present = isinstance(servers, dict) and payload['server'] in servers", + "server = servers.get(payload['server']) if present else None", + "ok = server == payload['expected']", + "print('registered' if ok else ('mismatch' if present else 'absent'))", + "PY", + ].join("\n"); +} + +export function buildDeepAgentsMcpStatusCommand(entry: McpBridgeEntry): string { + const payload = { + server: entry.server, + expected: deepAgentsManagedServerConfig(entry), + }; + return [ + "python3 - <<'PY'", + "import json, pathlib", + `payload = json.loads(${pythonJsonLiteral(payload)})`, + `config_path = pathlib.Path(${JSON.stringify(DEEPAGENTS_MCP_CONFIG_PATH)})`, + "try:", + " data = json.loads(config_path.read_text(encoding='utf-8') or '{}')", + "except Exception:", + " data = {}", + "servers = data.get('mcpServers') if isinstance(data, dict) else None", + "present = isinstance(servers, dict) and payload['server'] in servers", + "server = servers.get(payload['server']) if present else None", + "ok = server == payload['expected']", + "print('registered' if ok else ('mismatch' if present else 'absent'))", + "PY", + ].join("\n"); +} + +export function buildOpenClawMcporterInspectCommand( + entry: McpBridgeEntry, + failOnMismatch: boolean, +): string { + const payload = { + server: entry.server, + url: entry.url, + headers: entryHeaders(entry), + failOnMismatch, + }; + return [ + "node - <<'NODE'", + 'const { spawnSync } = require("node:child_process");', + `const expected = JSON.parse(${pythonJsonLiteral(payload)});`, + 'const result = spawnSync("mcporter", ["config", "get", expected.server, "--json"], { encoding: "utf8" });', + "if (result.error) { console.error(result.error.message); process.exit(3); }", + "if (result.status !== 0) {", + ' const detail = `${result.stderr || ""}\n${result.stdout || ""}`;', + " if (/not\\s+found|does\\s+not\\s+exist|unknown\\s+server/i.test(detail)) { console.log('absent'); process.exit(0); }", + " console.error(detail.trim() || `mcporter config get exited ${result.status}`);", + " process.exit(3);", + "}", + "let actual = null;", + "try { actual = JSON.parse(result.stdout); } catch {}", + 'const headers = actual && actual.headers && typeof actual.headers === "object" ? actual.headers : {};', + mcporterHeaderMatcherSource(), + 'const registered = !!actual && actual.name === expected.server && actual.transport === "http" && actual.baseUrl === expected.url && mcporterHeadersMatchExpected(headers, expected.headers);', + 'console.log(registered ? "registered" : "mismatch");', + "if (!registered && expected.failOnMismatch) process.exit(2);", + "NODE", + ].join("\n"); +} diff --git a/src/lib/actions/sandbox/mcp-bridge-adapters.ts b/src/lib/actions/sandbox/mcp-bridge-adapters.ts new file mode 100644 index 00000000000..64ebd2af1f6 --- /dev/null +++ b/src/lib/actions/sandbox/mcp-bridge-adapters.ts @@ -0,0 +1,157 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { AgentMcpAdapter } from "../../agent/defs"; +import type { McpBridgeEntry } from "../../state/registry"; +import { + assertDeepAgentsMcpMutationRuntimeCapability, + inspectDeepAgentsAdapterRegistration, + registerDeepAgentsAdapter, + unregisterDeepAgentsAdapter, +} from "./mcp-bridge-adapter-deepagents"; +import { + assertHermesMcpConfigMutationAllowed, + assertHermesMcpMutationRuntimeCapability, + inspectHermesAdapterRegistration, + registerHermesAdapter, + unregisterHermesAdapter, +} from "./mcp-bridge-adapter-hermes"; +import type { + AdapterMutationOptions, + AdapterRegistrationInspection, +} from "./mcp-bridge-adapter-inspection"; +import { + inspectOpenClawAdapterRegistration, + registerOpenClawAdapter, + unregisterOpenClawAdapter, +} from "./mcp-bridge-adapter-openclaw"; + +export { + buildDeepAgentsMcpRegisterCommand, + buildDeepAgentsMcpRemoveCommand, +} from "./mcp-bridge-adapter-deepagents"; +export { + buildHermesMcpExecArgs, + buildHermesMcpProbeCommand, + buildHermesMcpRegisterCommand, +} from "./mcp-bridge-adapter-hermes"; +export { + type AdapterRegistrationInspection, + parseAdapterRegistrationInspection, +} from "./mcp-bridge-adapter-inspection"; +export { + buildOpenClawMcporterRegisterCommand, + buildOpenClawMcporterRemoveCommand, + MCPORTER_VERSION, +} from "./mcp-bridge-adapter-openclaw"; +export { + buildDeepAgentsMcpStatusCommand, + buildHermesMcpStatusCommand, + buildOpenClawMcporterInspectCommand, + DEEPAGENTS_MCP_CONFIG_PATH, + mcporterHeadersMatchExpected, +} from "./mcp-bridge-adapter-status"; + +export function inspectAgentAdapterRegistration( + sandboxName: string, + adapter: AgentMcpAdapter, + entry: McpBridgeEntry, +): AdapterRegistrationInspection { + switch (adapter) { + case "mcporter": + return inspectOpenClawAdapterRegistration(sandboxName, entry); + case "hermes-config": + return inspectHermesAdapterRegistration(sandboxName, entry); + case "deepagents-config": + return inspectDeepAgentsAdapterRegistration(sandboxName, entry); + } +} + +/** + * Refuse an in-sandbox adapter config mutation while Hermes config is locked. + * This host-side check intentionally runs before provider, policy, attachment, + * or adapter work; the transaction helper repeats the file-level check to + * close posture drift between this preflight and the actual config write. + * + * Deep Agents and OpenClaw do not use the Hermes shields contract. In + * particular, teardown of a legacy Deep Agents entry must remain possible on + * an image that predates the managed launcher capability marker. + */ +export function assertAgentMcpConfigMutationAllowed( + sandboxName: string, + adapter: AgentMcpAdapter, +): void { + if (adapter === "hermes-config") assertHermesMcpConfigMutationAllowed(sandboxName); +} + +export function assertAgentMcpMutationRuntimeCapability( + sandboxName: string, + adapter: AgentMcpAdapter, +): void { + switch (adapter) { + case "deepagents-config": + assertDeepAgentsMcpMutationRuntimeCapability(sandboxName); + return; + case "hermes-config": + assertHermesMcpMutationRuntimeCapability(sandboxName); + return; + case "mcporter": + return; + } +} + +/** + * Validate the runtime needed to scrub an existing adapter definition. + * Hermes teardown still uses its managed transaction helper and therefore + * requires the full helper/lifecycle probe. Deep Agents teardown executes the + * ownership-checked config scrub directly and must remain available to images + * that predate the new launcher marker. + */ +export function assertAgentMcpTeardownRuntimeCapability( + sandboxName: string, + adapter: AgentMcpAdapter, +): void { + assertAgentMcpConfigMutationAllowed(sandboxName, adapter); + if (adapter === "hermes-config") { + assertAgentMcpMutationRuntimeCapability(sandboxName, adapter); + } +} + +export function registerAgentAdapter( + sandboxName: string, + adapter: AgentMcpAdapter, + entry: McpBridgeEntry, + envValues: Record = {}, + options: { replaceExisting?: boolean } = {}, +): void { + switch (adapter) { + case "mcporter": + registerOpenClawAdapter(sandboxName, entry, envValues, options.replaceExisting === true); + return; + case "hermes-config": + registerHermesAdapter(sandboxName, entry, envValues, options.replaceExisting === true); + return; + case "deepagents-config": + registerDeepAgentsAdapter(sandboxName, entry, envValues, options.replaceExisting === true); + return; + } +} + +export function unregisterAgentAdapter( + sandboxName: string, + adapter: AgentMcpAdapter, + entry: McpBridgeEntry, + options: AdapterMutationOptions = {}, +): void { + switch (adapter) { + case "mcporter": + unregisterOpenClawAdapter(sandboxName, entry, options); + return; + case "hermes-config": + unregisterHermesAdapter(sandboxName, entry, options); + return; + case "deepagents-config": + unregisterDeepAgentsAdapter(sandboxName, entry, options); + return; + } +} diff --git a/src/lib/actions/sandbox/mcp-bridge-add-restart.ts b/src/lib/actions/sandbox/mcp-bridge-add-restart.ts new file mode 100644 index 00000000000..810f078ae67 --- /dev/null +++ b/src/lib/actions/sandbox/mcp-bridge-add-restart.ts @@ -0,0 +1,391 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import crypto from "node:crypto"; + +import type { AgentMcpAdapter } from "../../agent/defs"; +import * as policies from "../../policy"; +import { withMcpLifecycleLock } from "../../state/mcp-lifecycle-lock"; +import type { McpBridgeEntry } from "../../state/registry"; +import * as registry from "../../state/registry"; +import { + assertAgentMcpConfigMutationAllowed, + assertAgentMcpMutationRuntimeCapability, + inspectAgentAdapterRegistration, + registerAgentAdapter, + unregisterAgentAdapter, +} from "./mcp-bridge-adapters"; +import { type McpBridgeAddOptions, McpBridgeError } from "./mcp-bridge-contracts"; +import { + applyGeneratedPolicy, + buildMcpBridgePolicyKey, + buildMcpBridgePolicyName, + buildMcpBridgePolicyYaml, + removeGeneratedPolicy, +} from "./mcp-bridge-policy"; +import { + assertMcpProviderRecoverable, + assertNoAttachedProviderCredentialCollision, + attachProvider, + deleteProvider, + detachMissingProviderReference, + detachProvider, + inspectMcpProvider, + type McpCredentialRevisionObservation, + observeMcpCredentialRevision, + providerMatchesCredential, + providerShapeDetail, + upsertMcpProvider, + waitForAttachedMcpCredential, + waitForDetachedMcpCredential, +} from "./mcp-bridge-provider"; +import { + assertMcpDestroyNotPending, + assertNoDerivedResourceCollision, + bridgeState, + ensureSandboxGatewaySelected, + getBridgeAdapter, + getSandboxAgent, + getSandboxOrThrow, + nowIso, + writeBridgeEntry, +} from "./mcp-bridge-state"; +import { + assertAuthenticatedCredentialReference, + buildMcpBridgeProviderName, + normalizeMcpServerUrl, + resolveCredentialEnv, + uniqueEnvNames, + validateMcpServerName, + validateMcpServerUrlResolvedTarget, + validateSandboxName, +} from "./mcp-bridge-validation"; + +function sameMcpAddIntent(existing: McpBridgeEntry, requested: McpBridgeEntry): boolean { + return ( + existing.server === requested.server && + existing.agent === requested.agent && + existing.adapter === requested.adapter && + existing.url === requested.url && + existing.providerName === requested.providerName && + existing.policyName === requested.policyName && + existing.env.length === requested.env.length && + existing.env.every((name, index) => name === requested.env[index]) + ); +} + +function assertPreparedMcpAddResourcesAbsent( + sandboxName: string, + adapter: AgentMcpAdapter, + entry: McpBridgeEntry, + resolvedAddresses?: readonly string[], +): void { + const adapterInspection = inspectAgentAdapterRegistration(sandboxName, adapter, entry); + if (adapterInspection.state !== "absent") { + const detail = + adapterInspection.state === "error" + ? adapterInspection.detail + : `server name is already ${adapterInspection.state}`; + throw new McpBridgeError( + `MCP add preflight for '${entry.server}' found an existing ${adapter} adapter entry: ${detail}. The durable add manifest was preserved without claiming it.`, + ); + } + + const providerInspection = inspectMcpProvider(entry.providerName); + if (providerInspection.exists !== false) { + const detail = + providerInspection.exists === null + ? (providerInspection.error ?? "provider inspection failed") + : (providerShapeDetail(providerInspection, entry.env[0]) ?? "provider already exists"); + throw new McpBridgeError( + `MCP add preflight for '${entry.server}' could not prove provider '${entry.providerName}' absent: ${detail}. The durable add manifest was preserved without claiming it.`, + ); + } + + const existingPolicy = registry + .getCustomPolicies(sandboxName) + .find((policy) => policy.name === entry.policyName); + if (existingPolicy) { + throw new McpBridgeError( + `MCP add preflight for '${entry.server}' found an existing policy ownership record '${entry.policyName}'. The durable add manifest was preserved without claiming it.`, + ); + } + const policyContent = buildMcpBridgePolicyYaml( + entry.server, + entry.url, + adapter, + resolvedAddresses, + ); + const policyState = policies.getPresetContentGatewayState(sandboxName, policyContent); + if (policyState !== "absent") { + throw new McpBridgeError( + `MCP add preflight for '${entry.server}' could not prove generated policy key '${buildMcpBridgePolicyKey(entry.server)}' absent (state: ${policyState ?? "unreachable"}). The durable add manifest was preserved without claiming it.`, + ); + } +} + +export async function addMcpBridge( + sandboxName: string, + options: McpBridgeAddOptions, +): Promise { + return withMcpLifecycleLock(sandboxName, () => addMcpBridgeUnlocked(sandboxName, options)); +} + +async function addMcpBridgeUnlocked( + sandboxName: string, + options: McpBridgeAddOptions, +): Promise { + validateSandboxName(sandboxName); + validateMcpServerName(options.server); + assertAuthenticatedCredentialReference(options.env); + const normalizedUrl = normalizeMcpServerUrl(options.url); + const resolvedAddresses = await validateMcpServerUrlResolvedTarget(new URL(normalizedUrl)); + const sandbox = getSandboxOrThrow(sandboxName); + assertMcpDestroyNotPending(sandbox); + const agent = getSandboxAgent(sandbox); + const adapter = getBridgeAdapter(agent); + const existingEntry = bridgeState(sandbox)[options.server]; + if (existingEntry && !existingEntry.addState) { + throw new McpBridgeError( + `MCP server '${options.server}' already exists on sandbox '${sandboxName}'.`, + ); + } + + const envNames = uniqueEnvNames(options.env); + const envCollision = Object.values(bridgeState(sandbox)).find( + (entry) => + entry.server !== options.server && entry.env.some((envName) => envNames.includes(envName)), + ); + if (envCollision) { + const duplicate = envCollision.env.find((envName) => envNames.includes(envName)); + throw new McpBridgeError( + `Credential key '${duplicate}' is already attached through MCP server '${envCollision.server}'. OpenShell static credential keys must be unique within a sandbox; use a distinct host environment name.`, + 2, + ); + } + const providerName = + envNames.length > 0 + ? (existingEntry?.providerName ?? + buildMcpBridgeProviderName( + sandboxName, + options.server, + crypto.randomBytes(8).toString("hex"), + )) + : undefined; + const adapterEnvValues = resolveCredentialEnv(options.env); + if (!existingEntry && !Object.hasOwn(adapterEnvValues, envNames[0])) { + throw new McpBridgeError( + `Host environment variable '${envNames[0]}' is required to create MCP provider '${providerName}'.`, + 1, + ); + } + const policyName = buildMcpBridgePolicyName(options.server); + assertNoDerivedResourceCollision(sandbox, options.server, providerName, policyName); + const requestedEntry: McpBridgeEntry = { + server: options.server, + agent: agent.name, + adapter, + url: normalizedUrl, + env: envNames, + ...(providerName ? { providerName } : {}), + policyName, + addedAt: existingEntry?.addedAt ?? nowIso(), + addState: existingEntry?.addState ?? "prepared", + }; + + if (existingEntry && !sameMcpAddIntent(existingEntry, requestedEntry)) { + throw new McpBridgeError( + `MCP server '${options.server}' has an incomplete add transaction with different URL, credential, agent, or derived resources. Re-run the original add command or remove it with --force before changing the definition.`, + 2, + ); + } + + let entry: McpBridgeEntry = existingEntry + ? { ...existingEntry, env: [...existingEntry.env] } + : requestedEntry; + const resumingPreflightedAdd = existingEntry?.addState === "preflighted"; + if (existingEntry?.addState === "prepared" && !Object.hasOwn(adapterEnvValues, entry.env[0])) { + throw new McpBridgeError( + `Host environment variable '${entry.env[0]}' is required to create MCP provider '${entry.providerName}'.`, + 1, + ); + } + // Hermes config posture is host-visible, so reject before even the durable + // prepared manifest is written. The in-sandbox helper repeats the check at + // the actual config write so a concurrent posture change still fails closed. + assertAgentMcpConfigMutationAllowed(sandboxName, adapter); + // This is the durable ownership manifest for every resource created below. + // It intentionally precedes gateway selection and all OpenShell mutations, + // so process death can never leave an unowned provider/policy/adapter entry. + if (!existingEntry) writeBridgeEntry(sandboxName, entry); + + let providerCreated = false; + let providerAttachAttempted = false; + let policyApplied = false; + let adapterMutationAttempted = false; + let previousCredentialRevision: McpCredentialRevisionObservation | undefined; + try { + await ensureSandboxGatewaySelected(sandboxName); + let detachedMissingProviderReference = false; + if (resumingPreflightedAdd) { + const providerInspection = inspectMcpProvider(entry.providerName); + if (providerInspection.exists === null) { + throw new McpBridgeError( + providerInspection.error ?? + `Could not inspect OpenShell provider '${entry.providerName}' before resuming MCP add.`, + ); + } + if (providerInspection.exists === false) { + // A provider can disappear while its sandbox-spec attachment remains. + // OpenShell cannot start any sandbox child while that dangling name is + // present, so detaching the already-missing provider reference is the + // one recovery side effect that must precede the image capability + // probe. It neither reads nor replaces credential material, and the + // durable add manifest retains ownership if the later probe fails. + detachMissingProviderReference(sandboxName, entry); + detachedMissingProviderReference = true; + } + } + assertAgentMcpMutationRuntimeCapability(sandboxName, adapter); + if (detachedMissingProviderReference) { + waitForDetachedMcpCredential(sandboxName, entry); + } + if (resumingPreflightedAdd && !Object.hasOwn(adapterEnvValues, entry.env[0])) { + try { + // A retry may reuse an exact provider without re-exporting its secret, + // but recreating a missing provider cannot. This check and any owned + // policy cleanup happen only after the running-image capability probe. + assertMcpProviderRecoverable(entry); + } catch (error) { + removeGeneratedPolicy(sandboxName, entry, { bestEffort: true }); + throw error; + } + } + + if (entry.addState === "prepared") { + assertPreparedMcpAddResourcesAbsent(sandboxName, adapter, entry, resolvedAddresses); + entry = { ...entry, addState: "preflighted" }; + // This second durable boundary proves the derived resource names and the + // adapter slot were absent before any side effect. After a crash, retries + // may therefore reuse only missing or exact resources, never drift. + writeBridgeEntry(sandboxName, entry); + } + const adapterInspection = inspectAgentAdapterRegistration(sandboxName, adapter, entry); + if ( + adapterInspection.state !== "absent" && + !(resumingPreflightedAdd && adapterInspection.state === "registered") + ) { + const detail = + adapterInspection.state === "error" + ? adapterInspection.detail + : `server name is already ${adapterInspection.state}`; + throw new McpBridgeError( + `MCP server '${entry.server}' cannot be registered in the ${adapter} adapter: ${detail}.`, + ); + } + // Credential keys are sandbox-global. Prove this key is not already + // supplied by a foreign attachment before opening its MCP route, then check + // again after provider creation to close the intervening race. + assertNoAttachedProviderCredentialCollision(sandboxName, entry); + // Loading the real protocol:mcp policy with --wait is the authoritative + // running-supervisor capability check. Do it before any host credential is + // created or updated so unsupported runtimes fail without that side effect. + applyGeneratedPolicy(sandboxName, entry, resolvedAddresses); + policyApplied = true; + const providerResult = upsertMcpProvider(providerName ?? "", options.env, { + // A first mutation must still observe the absence proven above. Only a + // retry of the durable preflighted transaction may encounter an exact + // provider whose immutable ID was already persisted by this add. + allowExisting: resumingPreflightedAdd, + expectedProviderId: entry.providerId, + prepareMutation: (action) => { + // A fresh create has no prior revision to compare. Observe only the + // bounded placeholder classification for an actual update, after the + // running supervisor has accepted the authenticated MCP policy. + if (action === "update") { + previousCredentialRevision = observeMcpCredentialRevision(sandboxName, entry); + } + }, + }); + providerCreated = providerResult.action === "created"; + const providerId = providerResult.inspection.id; + if (!providerId) { + throw new McpBridgeError( + `OpenShell did not return a stable provider ID for '${providerName}'. Refusing later MCP side effects.`, + ); + } + if (entry.providerId !== providerId) { + entry = { ...entry, providerId }; + // The immutable OpenShell identity is the ownership boundary for every + // later lifecycle action. Persist it before policy, attachment, or + // adapter mutations. A process death before this write fails closed. + writeBridgeEntry(sandboxName, entry); + } + assertNoAttachedProviderCredentialCollision(sandboxName, entry); + if (providerResult.action === "updated" && previousCredentialRevision === undefined) { + throw new McpBridgeError( + `Could not retain the prior OpenShell credential revision for provider '${entry.providerName}'.`, + ); + } + providerAttachAttempted = true; + attachProvider(sandboxName, entry); + waitForAttachedMcpCredential(sandboxName, entry, { + ...(providerResult.action === "updated" + ? { + previousRevision: previousCredentialRevision, + } + : {}), + }); + // The adapter was proven absent above, so cleanup is safe even when a + // command commits config and then fails during its runtime reload. + adapterMutationAttempted = true; + registerAgentAdapter(sandboxName, adapter, entry, adapterEnvValues, { + // An exact adapter entry is evidence of a post-commit process death. + // Replacing it is idempotent and, for Hermes, re-verifies runtime reload. + replaceExisting: resumingPreflightedAdd && adapterInspection.state === "registered", + }); + const { addState: _completedAddState, ...committedEntry } = entry; + writeBridgeEntry(sandboxName, committedEntry); + } catch (error) { + const rollbackProviderInspection = + (providerAttachAttempted || providerCreated) && entry.providerId + ? inspectMcpProvider(providerName) + : undefined; + const rollbackProviderOwned = + !!rollbackProviderInspection && + providerMatchesCredential(rollbackProviderInspection, entry.env[0], entry.providerId); + if (adapterMutationAttempted) { + unregisterAgentAdapter(sandboxName, adapter, entry, { + force: false, + bestEffort: true, + envValues: adapterEnvValues, + }); + } + const detachOutcome = providerAttachAttempted + ? detachProvider(sandboxName, entry, { bestEffort: true }) + : "absent"; + let reservationCleanupProved = !providerAttachAttempted; + if (providerAttachAttempted && detachOutcome !== "unknown") { + try { + waitForDetachedMcpCredential(sandboxName, entry); + reservationCleanupProved = true; + } catch { + reservationCleanupProved = false; + } + } + if (policyApplied && reservationCleanupProved) + removeGeneratedPolicy(sandboxName, entry, { + bestEffort: true, + }); + if (providerCreated && rollbackProviderOwned && reservationCleanupProved) { + const beforeDelete = inspectMcpProvider(providerName); + if (providerMatchesCredential(beforeDelete, entry.env[0], entry.providerId)) { + deleteProvider(entry, { allowMissing: true, bestEffort: true }); + } + } + // Exception rollback is best-effort and process death skips it entirely. + // Keep the durable add manifest until a retry converges or `mcp remove` + // proves and cleans each exact resource. + throw error; + } +} diff --git a/src/lib/actions/sandbox/mcp-bridge-contracts.ts b/src/lib/actions/sandbox/mcp-bridge-contracts.ts new file mode 100644 index 00000000000..f4ad498cda1 --- /dev/null +++ b/src/lib/actions/sandbox/mcp-bridge-contracts.ts @@ -0,0 +1,70 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { AgentMcpAdapter } from "../../agent/defs"; + +export const MCP_BRIDGE_POLICY_SOURCE = "generated:nemoclaw-mcp-bridge"; +export class McpBridgeError extends Error { + constructor( + message: string, + readonly exitCode = 1, + ) { + super(message); + this.name = "McpBridgeError"; + } +} + +export interface ParsedEnvReference { + name: string; + value?: string; +} + +export interface ParsedMcpAddArgs { + server: string; + url: string; + env: ParsedEnvReference[]; +} + +export interface McpBridgeAddOptions extends ParsedMcpAddArgs {} + +export interface McpBridgeStatus { + server: string; + agent: string; + warnings: string[]; + support: { + supported: boolean; + mode: "bridge" | "disabled"; + adapter?: AgentMcpAdapter; + reason?: string; + }; + url?: string; + env: { + names: string[]; + missing: string[]; + ready: boolean; + }; + provider: { + name?: string; + registryPresent: boolean; + gatewayPresent: boolean | null; + attached: boolean | null; + credentialReady: boolean | null; + detail?: string; + }; + policy: { + name?: string; + registryPresent: boolean; + gatewayPresent: boolean | null; + }; + adapter: { + registered: boolean | null; + detail?: string; + }; + addState?: "prepared" | "preflighted"; + addedAt?: string; + updatedAt?: string; +} + +export function isAgentMcpAdapter(value: unknown): value is AgentMcpAdapter { + return value === "mcporter" || value === "hermes-config" || value === "deepagents-config"; +} diff --git a/src/lib/actions/sandbox/mcp-bridge-destroy-preflight.ts b/src/lib/actions/sandbox/mcp-bridge-destroy-preflight.ts new file mode 100644 index 00000000000..7dde510f8c1 --- /dev/null +++ b/src/lib/actions/sandbox/mcp-bridge-destroy-preflight.ts @@ -0,0 +1,171 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { McpBridgeEntry, SandboxEntry } from "../../state/registry"; +import * as registry from "../../state/registry"; +import { McpBridgeError } from "./mcp-bridge-contracts"; +import { + assertGeneratedPolicyRegistrationMutationSafe, + removeGeneratedPolicy, +} from "./mcp-bridge-policy"; +import { + inspectMcpProvider, + type McpProviderInspection, + providerMatchesCredential, + providerShapeDetail, +} from "./mcp-bridge-provider"; +import { + bridgeState, + ensureSandboxGatewaySelected, + getSandboxOrThrow, + setBridgeState, +} from "./mcp-bridge-state"; +import { assertAuthenticatedBridgeEntry, validateSandboxName } from "./mcp-bridge-validation"; + +export interface McpDestroyPreparation { + entries: McpBridgeEntry[]; + detachedProviderEntries: McpBridgeEntry[]; + scrubbedAdapterEntries: McpBridgeEntry[]; + /** True when phase one was completed by an earlier destroy process. */ + destroyAlreadyPrepared: boolean; + /** True when a previous destroy already confirmed the sandbox was absent. */ + destroyAlreadyPending: boolean; +} + +export function cloneMcpBridgeEntry(entry: McpBridgeEntry): McpBridgeEntry { + return { ...entry, env: [...entry.env] }; +} + +function mcpBridgeEntriesEqual(left: McpBridgeEntry, right: McpBridgeEntry): boolean { + return ( + left.server === right.server && + left.agent === right.agent && + left.adapter === right.adapter && + left.url === right.url && + left.providerName === right.providerName && + left.providerId === right.providerId && + left.policyName === right.policyName && + left.addedAt === right.addedAt && + left.updatedAt === right.updatedAt && + left.addState === right.addState && + left.env.length === right.env.length && + left.env.every((name, index) => name === right.env[index]) + ); +} + +export async function discardSafeIncompleteMcpAdds( + sandboxName: string, + sandbox: SandboxEntry, + options: { sandboxAbsent?: boolean } = {}, +): Promise { + const bridges = bridgeState(sandbox); + const providerlessCandidates = Object.values(bridges).filter( + (entry) => entry.addState === "preflighted" && !entry.providerId, + ); + if (providerlessCandidates.length > 0) await ensureSandboxGatewaySelected(sandboxName); + const remainingEntries: Array<[string, McpBridgeEntry]> = []; + const providerlessPreflighted: McpBridgeEntry[] = []; + for (const [server, entry] of Object.entries(bridges)) { + if (entry.addState === "prepared") continue; + if (entry.addState === "preflighted" && !entry.providerId) { + assertAuthenticatedBridgeEntry(entry); + const inspection = inspectMcpProvider(entry.providerName); + if (inspection.exists === false) { + providerlessPreflighted.push(entry); + continue; + } + } + remainingEntries.push([server, entry]); + } + const remaining = Object.fromEntries(remainingEntries); + if (Object.keys(remaining).length === Object.keys(bridges).length) return sandbox; + for (const entry of providerlessPreflighted) { + if (options.sandboxAbsent) { + const ownedRegistration = assertGeneratedPolicyRegistrationMutationSafe(sandboxName, entry); + if (ownedRegistration) registry.removeCustomPolicyByName(sandboxName, entry.policyName); + } else { + removeGeneratedPolicy(sandboxName, entry); + } + } + // A prepared add precedes all external side effects, so destroy drops only + // its local manifest and never inspects same-name global resources. + setBridgeState(sandboxName, remaining); + return getSandboxOrThrow(sandboxName); +} + +export function assertMcpDestroySnapshotCurrent( + sandboxName: string, + entries: readonly McpBridgeEntry[], +): SandboxEntry { + const sandbox = getSandboxOrThrow(sandboxName); + const current = bridgeState(sandbox); + const expectedServers = new Set(entries.map((entry) => entry.server)); + if ( + Object.keys(current).length !== expectedServers.size || + entries.some( + (entry) => !current[entry.server] || !mcpBridgeEntriesEqual(current[entry.server], entry), + ) + ) { + throw new McpBridgeError( + `MCP bridge definitions changed while sandbox '${sandboxName}' was being destroyed. Cleanup state was preserved; re-run destroy to reconcile the current definitions.`, + ); + } + return sandbox; +} + +export function inspectExactMcpDestroyProvider( + entry: McpBridgeEntry, + options: { allowMissing: boolean; force?: boolean }, +): McpProviderInspection { + assertAuthenticatedBridgeEntry(entry); + if (!entry.providerId) { + throw new McpBridgeError( + `MCP server '${entry.server}' has no stable OpenShell provider ID. Refusing destructive cleanup of same-name provider '${entry.providerName}'. Remove the legacy bridge with --force only after independently cleaning that provider.`, + ); + } + const inspection = inspectMcpProvider(entry.providerName); + if (inspection.exists === null) { + throw new McpBridgeError( + inspection.error ?? `Could not inspect OpenShell provider '${entry.providerName}'.`, + ); + } + if (!inspection.exists) { + if (options.allowMissing) return inspection; + throw new McpBridgeError( + `OpenShell provider '${entry.providerName}' is missing. Refusing to destroy sandbox state because a failed sandbox delete could not restore authenticated MCP without the preserved provider credential.`, + ); + } + if (!providerMatchesCredential(inspection, entry.env[0], entry.providerId)) { + const forceDetail = options.force + ? " --force does not delete a non-matching global provider because it may be owned by another workflow." + : ""; + throw new McpBridgeError( + `OpenShell provider '${entry.providerName}' no longer exactly matches MCP server '${entry.server}'. ${providerShapeDetail(inspection, entry.env[0], entry.providerId)}${forceDetail}`, + ); + } + return inspection; +} + +/** Build cleanup state after a gateway-pinned list proves the sandbox absent. */ +export async function prepareMcpBridgesForAbsentSandboxDestroy( + sandboxName: string, + options: { force?: boolean } = {}, +): Promise { + validateSandboxName(sandboxName); + const sandbox = await discardSafeIncompleteMcpAdds(sandboxName, getSandboxOrThrow(sandboxName), { + sandboxAbsent: true, + }); + const entries = Object.values(bridgeState(sandbox)).map(cloneMcpBridgeEntry); + const destroyAlreadyPrepared = !!sandbox.mcp?.destroyPreparedAt; + const destroyAlreadyPending = !!sandbox.mcp?.destroyPendingAt; + for (const entry of entries) { + inspectExactMcpDestroyProvider(entry, { allowMissing: true, force: options.force }); + } + return { + entries, + detachedProviderEntries: [], + scrubbedAdapterEntries: [], + destroyAlreadyPrepared, + destroyAlreadyPending, + }; +} diff --git a/src/lib/actions/sandbox/mcp-bridge-destroy.ts b/src/lib/actions/sandbox/mcp-bridge-destroy.ts new file mode 100644 index 00000000000..5a393e8fccd --- /dev/null +++ b/src/lib/actions/sandbox/mcp-bridge-destroy.ts @@ -0,0 +1,362 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { McpBridgeEntry } from "../../state/registry"; +import * as registry from "../../state/registry"; +import { registerAgentAdapter, unregisterAgentAdapter } from "./mcp-bridge-adapters"; +import { + isAgentMcpAdapter, + MCP_BRIDGE_POLICY_SOURCE, + McpBridgeError, +} from "./mcp-bridge-contracts"; +import type { McpDestroyPreparation } from "./mcp-bridge-destroy-preflight"; +import { + assertMcpDestroySnapshotCurrent, + cloneMcpBridgeEntry, + discardSafeIncompleteMcpAdds, + inspectExactMcpDestroyProvider, +} from "./mcp-bridge-destroy-preflight"; +import { + attachProvider, + deleteProvider, + detachProvider, + inspectMcpProvider, + waitForAttachedMcpCredential, + waitForDetachedMcpCredential, +} from "./mcp-bridge-provider"; +import { restoreExistingMcpBridgeRuntime } from "./mcp-bridge-restart"; +import { + assertMcpAdapterConfigMutationsAllowed, + assertMcpAdapterTeardownRuntimeCapabilities, +} from "./mcp-bridge-runtime-capabilities"; +import { + bridgeState, + ensureSandboxGatewaySelected, + getBridgeAdapter, + getSandboxAgent, + getSandboxOrThrow, + nowIso, +} from "./mcp-bridge-state"; +import { validateSandboxName } from "./mcp-bridge-validation"; + +export type { McpDestroyPreparation } from "./mcp-bridge-destroy-preflight"; +export { + cloneMcpBridgeEntry, + discardSafeIncompleteMcpAdds, + inspectExactMcpDestroyProvider, + prepareMcpBridgesForAbsentSandboxDestroy, +} from "./mcp-bridge-destroy-preflight"; + +/** + * Phase one of sandbox destroy. Remove the adapter entry from the retained + * sandbox volume and detach exact MCP providers while preserving the global + * provider objects (and therefore their host-only credentials), generated + * policy, and registry cleanup manifest. Any failure restores adapter and + * attachment state before returning. + */ +export async function prepareMcpBridgesForDestroy( + sandboxName: string, +): Promise { + validateSandboxName(sandboxName); + const currentSandbox = getSandboxOrThrow(sandboxName); + const entriesRequiringExternalCleanup = Object.values(bridgeState(currentSandbox)).filter( + (entry) => entry.addState !== "prepared", + ); + // Run the host-visible config preflight before + // discardSafeIncompleteMcpAdds, which may remove an owned policy for a + // providerless preflighted add. That cleanup has no adapter/provider to + // probe; complete entries get the teardown runtime probe after retry markers. + assertMcpAdapterConfigMutationsAllowed( + sandboxName, + currentSandbox, + entriesRequiringExternalCleanup, + ); + const sandbox = await discardSafeIncompleteMcpAdds(sandboxName, currentSandbox); + const entries = Object.values(bridgeState(sandbox)).map(cloneMcpBridgeEntry); + const destroyAlreadyPrepared = !!sandbox.mcp?.destroyPreparedAt; + const destroyAlreadyPending = !!sandbox.mcp?.destroyPendingAt; + const incompleteAdd = entries.find((entry) => entry.addState === "preflighted"); + if (incompleteAdd) { + throw new McpBridgeError( + `MCP server '${incompleteAdd.server}' has an incomplete add transaction. Re-run the original mcp add command or remove it with --force before destroying the live sandbox.`, + ); + } + if (entries.length === 0) { + return { + entries: [], + detachedProviderEntries: [], + scrubbedAdapterEntries: [], + destroyAlreadyPrepared, + destroyAlreadyPending, + }; + } + + // A pending marker is written only after OpenShell confirmed deletion. On + // retry, a provider may therefore already be absent due to partial cleanup; + // the retained entries are the durable, idempotent cleanup manifest. + for (const entry of entries) { + inspectExactMcpDestroyProvider(entry, { + allowMissing: destroyAlreadyPending, + }); + } + if (destroyAlreadyPending) { + return { + entries, + detachedProviderEntries: [], + scrubbedAdapterEntries: [], + destroyAlreadyPrepared, + destroyAlreadyPending: true, + }; + } + if (destroyAlreadyPrepared) { + // Phase one completed before a prior process stopped. The sandbox may be + // live with its adapter scrubbed/provider detached, or it may already be + // gone. In either case, repeating delete is the next idempotent step. + return { + entries, + detachedProviderEntries: entries.map(cloneMcpBridgeEntry), + scrubbedAdapterEntries: entries.map(cloneMcpBridgeEntry), + destroyAlreadyPrepared: true, + destroyAlreadyPending: false, + }; + } + + await ensureSandboxGatewaySelected(sandboxName); + assertMcpAdapterTeardownRuntimeCapabilities(sandboxName, sandbox, entries); + const detached: McpBridgeEntry[] = []; + const scrubbedAdapters: McpBridgeEntry[] = []; + try { + for (const entry of entries) { + const adapter = isAgentMcpAdapter(entry.adapter) + ? entry.adapter + : getBridgeAdapter(getSandboxAgent(sandbox)); + unregisterAgentAdapter(sandboxName, adapter, entry, { + envValues: {}, + }); + scrubbedAdapters.push(entry); + } + for (const entry of entries) { + inspectExactMcpDestroyProvider(entry, { allowMissing: false }); + const detachOutcome = detachProvider(sandboxName, entry); + if (detachOutcome === "unknown") { + throw new McpBridgeError( + `Could not prove provider detach for MCP server '${entry.server}'.`, + ); + } + waitForDetachedMcpCredential(sandboxName, entry); + // Both an acknowledged detach and a freshly-proven absent binding are + // rollback responsibilities until destroyPreparedAt is durable. This + // closes retry-after-process-death gaps where an earlier attempt already + // detached one entry before a later entry fails. + detached.push(entry); + } + const marked = registry.updateSandbox(sandboxName, { + mcp: { + bridges: Object.fromEntries( + entries.map((entry) => [entry.server, cloneMcpBridgeEntry(entry)]), + ), + destroyPreparedAt: nowIso(), + }, + }); + if (!marked) { + throw new McpBridgeError( + `Could not persist prepared MCP destroy state for sandbox '${sandboxName}'.`, + ); + } + } catch (error) { + const rollbackFailures: string[] = []; + for (const entry of [...detached].reverse()) { + try { + inspectExactMcpDestroyProvider(entry, { allowMissing: false }); + attachProvider(sandboxName, entry); + // Reattach preserves the provider value, so presence is sufficient; + // still wait before reloading an adapter that may connect immediately. + waitForAttachedMcpCredential(sandboxName, entry); + } catch (rollbackError) { + rollbackFailures.push( + rollbackError instanceof Error ? rollbackError.message : String(rollbackError), + ); + } + } + for (const entry of scrubbedAdapters) { + try { + const adapter = isAgentMcpAdapter(entry.adapter) + ? entry.adapter + : getBridgeAdapter(getSandboxAgent(sandbox)); + registerAgentAdapter( + sandboxName, + adapter, + entry, + {}, + { + replaceExisting: true, + }, + ); + } catch (rollbackError) { + rollbackFailures.push( + rollbackError instanceof Error ? rollbackError.message : String(rollbackError), + ); + } + } + const current = registry.getSandbox(sandboxName); + if (current?.mcp?.destroyPreparedAt) { + try { + registry.updateSandbox(sandboxName, { + mcp: { + bridges: Object.fromEntries( + entries.map((entry) => [entry.server, cloneMcpBridgeEntry(entry)]), + ), + }, + }); + } catch (rollbackError) { + rollbackFailures.push( + rollbackError instanceof Error ? rollbackError.message : String(rollbackError), + ); + } + } + const detail = error instanceof Error ? error.message : String(error); + throw new McpBridgeError( + rollbackFailures.length > 0 + ? `${detail}\nMCP destroy rollback could not reattach: ${rollbackFailures.join("; ")}` + : detail, + ); + } + return { + entries, + detachedProviderEntries: detached, + scrubbedAdapterEntries: scrubbedAdapters, + destroyAlreadyPrepared: false, + destroyAlreadyPending: false, + }; +} + +/** Restore all MCP runtime state after OpenShell refused to delete the sandbox. */ +export async function restoreMcpBridgesAfterDestroyAbort( + sandboxName: string, + preparation: McpDestroyPreparation, +): Promise { + if (preparation.entries.length === 0 || preparation.destroyAlreadyPending) { + return; + } + const preparedSandbox = assertMcpDestroySnapshotCurrent(sandboxName, preparation.entries); + const destroyPreparedAt = preparedSandbox.mcp?.destroyPreparedAt ?? nowIso(); + const cleared = registry.updateSandbox(sandboxName, { + mcp: { + bridges: Object.fromEntries( + preparation.entries.map((entry) => [entry.server, cloneMcpBridgeEntry(entry)]), + ), + }, + }); + if (!cleared) { + throw new McpBridgeError( + `Could not clear prepared MCP destroy state for sandbox '${sandboxName}' before runtime restoration.`, + ); + } + try { + // Reattach only the exact existing providers. This restoration path never + // reads host secret values and therefore cannot rotate preserved credentials. + for (const entry of preparation.entries) + inspectExactMcpDestroyProvider(entry, { allowMissing: false }); + await restoreExistingMcpBridgeRuntime(sandboxName, preparation.entries, { + lifecyclePhase: "teardown-rollback", + }); + } catch (error) { + let markerRestoreFailure = ""; + try { + const restored = registry.updateSandbox(sandboxName, { + mcp: { + bridges: Object.fromEntries( + preparation.entries.map((entry) => [entry.server, cloneMcpBridgeEntry(entry)]), + ), + destroyPreparedAt, + }, + }); + if (!restored) markerRestoreFailure = "sandbox registry entry disappeared"; + } catch (restoreError) { + markerRestoreFailure = + restoreError instanceof Error ? restoreError.message : String(restoreError); + } + const detail = error instanceof Error ? error.message : String(error); + throw new McpBridgeError( + markerRestoreFailure + ? `${detail}; could not restore the MCP destroy retry marker: ${markerRestoreFailure}` + : detail, + ); + } +} + +/** + * Phase two of sandbox destroy, called only after OpenShell confirmed the + * sandbox is gone. Delete exact matching global providers, then clear the MCP + * bridge manifest and owned custom-policy records in one registry update. + */ +export async function finalizeMcpBridgesAfterSandboxDelete( + sandboxName: string, + preparation: McpDestroyPreparation, + options: { force?: boolean } = {}, +): Promise { + const entries = preparation.entries; + if (entries.length === 0) return; + + await ensureSandboxGatewaySelected(sandboxName); + + const sandbox = assertMcpDestroySnapshotCurrent(sandboxName, entries); + if (!sandbox.mcp?.destroyPendingAt) { + const marked = registry.updateSandbox(sandboxName, { + mcp: { + bridges: Object.fromEntries( + entries.map((entry) => [entry.server, cloneMcpBridgeEntry(entry)]), + ), + destroyPendingAt: nowIso(), + }, + }); + if (!marked) { + throw new McpBridgeError( + `Could not persist MCP destroy cleanup state for sandbox '${sandboxName}'. No MCP providers were deleted.`, + ); + } + assertMcpDestroySnapshotCurrent(sandboxName, entries); + } + + // Inspect every provider before deleting any so ownership drift cannot + // produce a predictable partial cleanup. Missing is safe only now that the + // durable pending marker proves the sandbox was already deleted. + const inspections = entries.map((entry) => + inspectExactMcpDestroyProvider(entry, { + allowMissing: true, + force: options.force, + }), + ); + for (const [index, entry] of entries.entries()) { + if (!inspections[index]?.exists) continue; + const beforeDelete = inspectExactMcpDestroyProvider(entry, { + allowMissing: true, + force: options.force, + }); + if (!beforeDelete.exists) continue; + deleteProvider(entry, { allowMissing: true }); + const after = inspectMcpProvider(entry.providerName); + if (after.exists !== false) { + throw new McpBridgeError( + after.error ?? + `OpenShell provider '${entry.providerName}' still exists after delete. MCP cleanup state was preserved for retry.`, + ); + } + } + + const finalSandbox = assertMcpDestroySnapshotCurrent(sandboxName, entries); + const ownedPolicyNames = new Set(entries.map((entry) => entry.policyName)); + const remainingCustomPolicies = (finalSandbox.customPolicies ?? []).filter( + (policy) => + !(ownedPolicyNames.has(policy.name) && policy.sourcePath === MCP_BRIDGE_POLICY_SOURCE), + ); + const cleared = registry.updateSandbox(sandboxName, { + mcp: undefined, + customPolicies: remainingCustomPolicies.length > 0 ? remainingCustomPolicies : undefined, + }); + if (!cleared) { + throw new McpBridgeError( + `MCP providers were deleted, but cleanup state for sandbox '${sandboxName}' could not be cleared. Re-run destroy; missing providers are accepted while cleanup is pending.`, + ); + } +} diff --git a/src/lib/actions/sandbox/mcp-bridge-input-runtime.test.ts b/src/lib/actions/sandbox/mcp-bridge-input-runtime.test.ts new file mode 100644 index 00000000000..cd2813feaeb --- /dev/null +++ b/src/lib/actions/sandbox/mcp-bridge-input-runtime.test.ts @@ -0,0 +1,115 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { + addMcpBridge, + buildMcpBridgeProviderArgs, + dispatchMcpBridgeCommand, + redactCredentialValuesForDisplay, + resolveCredentialEnv, +} from "./mcp-bridge"; + +describe("MCP input runtime boundaries", () => { + it("rejects unauthenticated direct add callers before sandbox or network side effects", async () => { + await expect( + addMcpBridge("missing-sandbox", { + server: "github", + url: "https://mcp.example.test/mcp", + env: [], + }), + ).rejects.toThrow(/requires exactly one --env KEY/); + await expect( + addMcpBridge("missing-sandbox", { + server: "github", + url: "https://mcp.example.test/mcp", + env: [{ name: "GCP_PROJECT_ID", value: "host-only-secret" }], + }), + ).rejects.toThrow(/materialized as a raw child-process value/); + }); + + it("resolves host env values without requiring them for provider reuse", () => { + const prior = process.env.MCP_BRIDGE_TEST_TOKEN; + process.env.MCP_BRIDGE_TEST_TOKEN = "secret-value"; + try { + expect(resolveCredentialEnv([{ name: "MCP_BRIDGE_TEST_TOKEN" }])).toEqual({ + MCP_BRIDGE_TEST_TOKEN: "secret-value", + }); + } finally { + prior === undefined + ? delete process.env.MCP_BRIDGE_TEST_TOKEN + : (process.env.MCP_BRIDGE_TEST_TOKEN = prior); + } + expect(resolveCredentialEnv([{ name: "MCP_BRIDGE_TEST_TOKEN_NOT_SET" }])).toEqual({}); + }); + + it("redacts inline credential values from provider failure output", () => { + const output = redactCredentialValuesForDisplay( + "provider failed for --credential TOKEN=inline-secret-value", + { TOKEN: "inline-secret-value" }, + ); + expect(output).toContain("provider failed for --credential"); + expect(output).not.toContain("inline-secret-value"); + }); + + it("passes MCP provider credentials by environment name, not argv value", () => { + const args = buildMcpBridgeProviderArgs( + "create", + "alpha-mcp-github", + [{ name: "TOKEN", value: "inline-secret-value" }], + { TOKEN: "inline-secret-value" }, + ); + + expect(args).toEqual([ + "provider", + "create", + "--name", + "alpha-mcp-github", + "--type", + "generic", + "--credential", + "TOKEN", + ]); + expect(args.join(" ")).not.toContain("inline-secret-value"); + expect(args.join(" ")).not.toContain("TOKEN=inline-secret-value"); + }); + + it("rejects surplus positional arguments before sandbox side effects", async () => { + const priorExitCode = process.exitCode; + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + try { + process.exitCode = undefined; + await dispatchMcpBridgeCommand("missing-sandbox", ["list", "extra"]); + expect(process.exitCode).toBe(2); + expect(errorSpy).toHaveBeenCalledWith( + expect.stringContaining("Usage: nemoclaw mcp list [--json]"), + ); + + process.exitCode = undefined; + await dispatchMcpBridgeCommand("missing-sandbox", ["remove", "one", "two"]); + expect(process.exitCode).toBe(2); + expect(errorSpy).toHaveBeenCalledWith( + expect.stringContaining("Usage: nemoclaw mcp remove [--force]"), + ); + } finally { + errorSpy.mockRestore(); + process.exitCode = priorExitCode; + } + }); + + it("documents force cleanup without promising residual registry removal", async () => { + const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + try { + await dispatchMcpBridgeCommand("missing-sandbox", ["remove", "--help"]); + expect(logSpy).toHaveBeenCalledWith( + expect.stringContaining( + "Best-effort owned cleanup; preserves registry state when residuals remain", + ), + ); + expect(logSpy).not.toHaveBeenCalledWith(expect.stringContaining("stale registry removal")); + } finally { + logSpy.mockRestore(); + } + }); +}); diff --git a/src/lib/actions/sandbox/mcp-bridge-input-targets.test.ts b/src/lib/actions/sandbox/mcp-bridge-input-targets.test.ts new file mode 100644 index 00000000000..34dd3b66272 --- /dev/null +++ b/src/lib/actions/sandbox/mcp-bridge-input-targets.test.ts @@ -0,0 +1,130 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import dns from "node:dns/promises"; + +import { describe, expect, it, vi } from "vitest"; + +import { addMcpBridge, normalizeMcpServerUrl } from "./mcp-bridge"; +import { validateMcpServerUrlResolvedTarget } from "./mcp-bridge-validation"; + +describe("MCP URL target validation", () => { + it("sorts and deduplicates public DNS pins deterministically", async () => { + const lookup = vi.spyOn(dns, "lookup").mockResolvedValue([ + { address: "2606:4700:4700::1111", family: 6 }, + { address: "8.8.8.8", family: 4 }, + { address: "8.8.8.8", family: 4 }, + ] as never); + try { + await expect( + validateMcpServerUrlResolvedTarget(new URL("https://mcp.example.test/mcp")), + ).resolves.toEqual(["2606:4700:4700::1111", "8.8.8.8"]); + } finally { + lookup.mockRestore(); + } + }); + + it("rejects private DNS answers and OpenShell host aliases before DNS", async () => { + const lookup = vi + .spyOn(dns, "lookup") + .mockResolvedValueOnce([{ address: "127.0.0.1", family: 4 }] as never); + try { + await expect( + validateMcpServerUrlResolvedTarget(new URL("https://mcp.example.test/mcp")), + ).rejects.toThrow(/resolves to private, local, or special-use address '127\.0\.0\.1'/); + await expect( + validateMcpServerUrlResolvedTarget(new URL("https://host.openshell.internal:31337/mcp")), + ).rejects.toThrow(/does not expose an attested driver gateway address/); + expect(lookup).toHaveBeenCalledOnce(); + } finally { + lookup.mockRestore(); + } + }); + + it("rejects hostile OpenShell alias registrations before sandbox or network side effects", async () => { + const lookup = vi.spyOn(dns, "lookup"); + try { + for (const host of [ + "host.openshell.internal", + "host.docker.internal", + "host.containers.internal", + ]) { + await expect( + addMcpBridge("missing-sandbox", { + server: "local", + url: `https://${host}:31337/mcp`, + env: [{ name: "SAFE_MCP_TOKEN", value: "host-only-secret" }], + }), + ).rejects.toThrow(/does not expose an attested driver gateway address/); + } + expect(lookup).not.toHaveBeenCalled(); + } finally { + lookup.mockRestore(); + } + }); + + it("rejects malformed percent paths before DNS or sandbox side effects", async () => { + const lookup = vi.spyOn(dns, "lookup"); + try { + for (const path of ["%", "%GG", "%2"]) { + await expect( + addMcpBridge("missing-sandbox", { + server: "malformed", + url: `https://mcp.example.test/${path}`, + env: [{ name: "SAFE_MCP_TOKEN", value: "host-only-secret" }], + }), + ).rejects.toThrow(/percent characters/); + } + expect(lookup).not.toHaveBeenCalled(); + } finally { + lookup.mockRestore(); + } + }); + + it("rejects local, private, and OpenShell host-alias URL targets", () => { + expect(() => normalizeMcpServerUrl("https://localhost:31337/mcp")).toThrow( + /private, local, or special-use IP/, + ); + expect(() => normalizeMcpServerUrl("https://127.0.0.1:31337/mcp")).toThrow( + /private, local, or special-use IP/, + ); + for (const host of ["2130706433", "0177.0.0.1", "0x7f.0.0.1", "localhost."]) { + expect(() => normalizeMcpServerUrl(`https://${host}:31337/mcp`)).toThrow( + /private, local, or special-use IP/, + ); + } + expect(() => normalizeMcpServerUrl("https://169.254.169.254/latest")).toThrow( + /private, local, or special-use IP/, + ); + expect(() => normalizeMcpServerUrl("https://[::1]:31337/mcp")).toThrow( + /IPv6-literal MCP server URLs are not supported/, + ); + expect(() => normalizeMcpServerUrl("https://[::ffff:a00:1]:31337/mcp")).toThrow( + /IPv6-literal MCP server URLs are not supported/, + ); + expect(() => normalizeMcpServerUrl("https://[::ffff:127.0.0.1]:31337/mcp")).toThrow( + /IPv6-literal MCP server URLs are not supported/, + ); + expect(() => normalizeMcpServerUrl("https://[::ffff:7f00:1]:31337/mcp")).toThrow( + /IPv6-literal MCP server URLs are not supported/, + ); + expect(() => normalizeMcpServerUrl("http://mcp.example.test/mcp")).toThrow(/must use https/); + expect(normalizeMcpServerUrl("https://8.8.8.8/mcp")).toBe("https://8.8.8.8/mcp"); + expect(() => normalizeMcpServerUrl("https://[2606:4700::1]/mcp")).toThrow( + /IPv6-literal MCP server URLs are not supported/, + ); + expect(() => normalizeMcpServerUrl("http://host.openshell.internal:31337/mcp")).toThrow( + /must use https/, + ); + for (const host of [ + "host.openshell.internal", + "host.openshell.internal.", + "host.docker.internal", + "host.containers.internal", + ]) { + expect(() => normalizeMcpServerUrl(`https://${host}:31337/mcp`)).toThrow( + /does not expose an attested driver gateway address/, + ); + } + }); +}); diff --git a/src/lib/actions/sandbox/mcp-bridge-input-validation.test.ts b/src/lib/actions/sandbox/mcp-bridge-input-validation.test.ts new file mode 100644 index 00000000000..8ba82d212df --- /dev/null +++ b/src/lib/actions/sandbox/mcp-bridge-input-validation.test.ts @@ -0,0 +1,253 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + SUBPROCESS_ENV_ALLOWED_NAMES, + SUBPROCESS_ENV_ALLOWED_PREFIXES, +} from "../../subprocess-env"; +import { + buildMcpBridgeProviderArgs, + MCP_SERVER_URL_MAX_LENGTH, + normalizeMcpServerUrl, + parseMcpAddArgs, + resolveCredentialEnv, +} from "./mcp-bridge"; +import childVisibleCredentialManifest from "./openshell-child-visible-credentials.v0.0.72.json"; + +describe("MCP CLI input validation", () => { + it("parses server, URL, and env references", () => { + const parsed = parseMcpAddArgs([ + "github", + "--url", + "https://api.githubcopilot.com/mcp/", + "--env", + "GITHUB_TOKEN", + ]); + + expect(parsed).toEqual({ + server: "github", + url: "https://api.githubcopilot.com/mcp/", + env: [{ name: "GITHUB_TOKEN" }], + }); + }); + + it("rejects inline env values that would leak through process arguments", () => { + expect(() => + parseMcpAddArgs(["srv", "--url=https://mcp.example.test/rpc", "--env=TOKEN=a=b=c"]), + ).toThrow(/process arguments and shell history/); + }); + + it("rejects OpenShell child-environment compatibility keys as MCP credentials", () => { + expect(childVisibleCredentialManifest).toMatchObject({ + openshellVersion: "0.0.72", + openshellCommit: "8cb16de9eae4c44d7d31e1493747d8c10abb5963", + }); + expect(childVisibleCredentialManifest.rawChildValueKeys).toEqual([ + "GCP_PROJECT_ID", + "GOOGLE_CLOUD_PROJECT", + "CLOUD_ML_REGION", + "GCP_LOCATION", + "GCP_SERVICE_ACCOUNT_EMAIL", + "GOOSE_PROVIDER", + "ANTHROPIC_VERTEX_PROJECT_ID", + "VERTEX_LOCATION", + ]); + for (const name of childVisibleCredentialManifest.rawChildValueKeys) { + expect(() => + parseMcpAddArgs(["github", "--url", "https://mcp.example.test/mcp", "--env", name]), + ).toThrow(/materialized as a raw child-process value/); + expect(() => resolveCredentialEnv([{ name, value: "host-only-secret" }])).toThrow( + /preserve the host-only credential boundary/, + ); + expect(() => + buildMcpBridgeProviderArgs("create", "provider", [{ name }], { + [name]: "host-only-secret", + }), + ).toThrow(/materialized as a raw child-process value/); + } + + expect(childVisibleCredentialManifest.rewrittenChildValueKeys).toEqual([ + "GCE_METADATA_HOST", + "GCE_METADATA_IP", + "METADATA_SERVER_DETECTION", + ]); + for (const name of childVisibleCredentialManifest.rewrittenChildValueKeys) { + expect(() => + parseMcpAddArgs(["github", "--url", "https://mcp.example.test/mcp", "--env", name]), + ).toThrow(/rewritten by OpenShell's Google Cloud metadata compatibility path/); + } + }); + + it("rejects host subprocess control and allowlist names as MCP credentials", () => { + for (const name of SUBPROCESS_ENV_ALLOWED_NAMES) { + expect(childVisibleCredentialManifest.runtimeControlKeys).toContain(name); + } + for (const prefix of SUBPROCESS_ENV_ALLOWED_PREFIXES) { + expect(childVisibleCredentialManifest.runtimeControlPrefixes).toContain(prefix); + } + for (const name of [ + "PATH", + "HOME", + "HTTP_PROXY", + "SSL_CERT_FILE", + "KUBECONFIG", + "LC_ALL", + "XDG_CONFIG_HOME", + "OPENSHELL_GATEWAY", + "GRPC_TRACE", + ]) { + expect(() => + parseMcpAddArgs(["github", "--url", "https://mcp.example.test/mcp", "--env", name]), + ).toThrow(/reserved for host subprocess control/); + } + }); + + it("rejects sandbox runtime-control names as MCP credentials", () => { + for (const name of [ + "BASH_ENV", + "ALL_PROXY", + "all_proxy", + "API_SERVER_KEY", + "DENO_CERT", + "grpc_proxy", + "NEMOCLAW_DASHBOARD_PORT", + "OPENCLAW_GATEWAY_URL", + "OPENAI_BASE_URL", + "HERMES_HOME", + "DEEPAGENTS_CONFIG_PATH", + "LANGCHAIN_TRACING_V2", + "ENV", + "LD_PRELOAD", + "DYLD_INSERT_LIBRARIES", + "GLIBC_TUNABLES", + "NODE_OPTIONS", + "PYTHONHOME", + "PYTHONPATH", + "RUBYOPT", + "PERL5OPT", + "JAVA_TOOL_OPTIONS", + "_JAVA_OPTIONS", + "CLASSPATH", + "VIRTUAL_ENV", + "UV_PROJECT_ENVIRONMENT", + ]) { + expect(() => + parseMcpAddArgs(["github", "--url", "https://mcp.example.test/mcp", "--env", name]), + ).toThrow(/reserved for sandbox runtime control/); + expect(() => resolveCredentialEnv([{ name, value: "host-only-secret" }])).toThrow( + /could alter or prevent agent commands/, + ); + expect(() => + buildMcpBridgeProviderArgs("create", "provider", [{ name }], { + [name]: "host-only-secret", + }), + ).toThrow(/reserved for sandbox runtime control/); + } + }); + + it("rejects host stdio commands", () => { + expect(() => + parseMcpAddArgs([ + "github", + "--env", + "GITHUB_TOKEN", + "--", + "npx", + "@modelcontextprotocol/server-github", + ]), + ).toThrow(/Host stdio MCP commands are not supported/); + }); + + it("requires an HTTPS MCP URL", () => { + expect(() => parseMcpAddArgs(["github"])).toThrow(/--url/); + expect(() => parseMcpAddArgs(["github", "--url", "stdio://github"])).toThrow(/https/); + }); + + it("normalizes URLs without persisting credentials", () => { + expect(normalizeMcpServerUrl("https://mcp.example.test")).toBe("https://mcp.example.test/"); + expect(() => normalizeMcpServerUrl("https://user:pass@mcp.example.test/mcp")).toThrow( + /must not embed credentials/, + ); + expect(() => normalizeMcpServerUrl("https://mcp.example.test/mcp?token=secret")).toThrow( + /must not include a query string/, + ); + expect(() => normalizeMcpServerUrl("https://mcp.example.test/mcp?")).toThrow( + /must not include a query string/, + ); + expect(() => normalizeMcpServerUrl("https://mcp.example.test/mcp#credential")).toThrow( + /must not include a fragment/, + ); + expect(() => normalizeMcpServerUrl("https://mcp.example.test/mcp#")).toThrow( + /must not include a fragment/, + ); + for (const token of [ + "nvapi-abcdefghijklmnop", + "ghp_abcdefghijklmnop", + "sk-abcdefghijklmnopqrstuvwxyz", + "sk-abcdefghijklmnopqrstuvwxyz.json", + `bot1234567890:${"A".repeat(35)}`, + `bot1234567890:${"A".repeat(34)}-`, + `1234567890:${"B".repeat(35)}`, + `${"A".repeat(24)}.${"B".repeat(6)}.${"C".repeat(26)}-`, + ]) { + expect(() => normalizeMcpServerUrl(`https://mcp.example.test/mcp/${token}`)).toThrow( + /paths must not contain secret-shaped credential material.*full URL is persisted/i, + ); + } + for (const path of ["/botanical/mcp", "/bottom/mcp", "/api/bots/mcp"]) { + expect(normalizeMcpServerUrl(`https://mcp.example.test${path}`)).toBe( + `https://mcp.example.test${path}`, + ); + } + expect(() => normalizeMcpServerUrl("https://*.example.test/mcp")).toThrow( + /hosts must be literal/, + ); + expect(() => normalizeMcpServerUrl("https://mcp.example.test:0/mcp")).toThrow( + /port must be between 1 and 65535/, + ); + for (const path of [ + "/mcp/**", + "/mcp/%2A%2A", + "/a/%2e%2e/mcp", + "/mcp/%2fadmin", + "/mcp/%", + "/mcp/%GG", + "/mcp/%2", + "/mcp;version=1", + "/mcp/[admin]", + "/mcp\\admin", + "/mcp//admin", + "/mcp/café", + ]) { + expect(() => normalizeMcpServerUrl(`https://mcp.example.test${path}`)).toThrow( + /literal and canonical/, + ); + } + }); + + it("bounds persisted MCP endpoint URLs consistently across adapters", () => { + const prefix = "https://mcp.example.test/"; + const maxLengthUrl = prefix.padEnd(MCP_SERVER_URL_MAX_LENGTH, "a"); + expect(normalizeMcpServerUrl(maxLengthUrl)).toBe(maxLengthUrl); + expect(() => normalizeMcpServerUrl(`${maxLengthUrl}a`)).toThrow(/at most 2048 characters/); + }); + + it("requires exactly one bearer credential reference", () => { + expect(() => parseMcpAddArgs(["github", "--url", "https://mcp.example.test/mcp"])).toThrow( + /requires exactly one --env KEY/, + ); + expect(() => + parseMcpAddArgs([ + "github", + "--url", + "https://mcp.example.test/mcp", + "--env", + "TOKEN_ONE", + "--env", + "TOKEN_TWO", + ]), + ).toThrow(/requires exactly one --env KEY/); + }); +}); diff --git a/src/lib/actions/sandbox/mcp-bridge-output.test.ts b/src/lib/actions/sandbox/mcp-bridge-output.test.ts new file mode 100644 index 00000000000..11ed0b2a573 --- /dev/null +++ b/src/lib/actions/sandbox/mcp-bridge-output.test.ts @@ -0,0 +1,136 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import type { McpBridgeEntry } from "../../state/registry"; +import { commandOutput, redactBridgeSecretsForDisplay } from "./mcp-bridge-output"; + +const baseEntry: McpBridgeEntry = { + server: "github", + agent: "openclaw", + adapter: "mcporter", + url: "https://api.githubcopilot.com/mcp/", + env: ["GITHUB_TOKEN"], + providerName: "alpha-mcp-github", + policyName: "mcp-bridge-github", + addedAt: new Date(0).toISOString(), +}; + +describe("MCP adapter output redaction", () => { + it("redacts credential values from adapter display output", () => { + const prior = process.env.GITHUB_TOKEN; + process.env.GITHUB_TOKEN = "real-host-secret"; + try { + const redacted = redactBridgeSecretsForDisplay( + "failed header Authorization=Bearer real-host-secret raw real-host-secret", + baseEntry, + ); + + expect(redacted).toBe("failed header Authorization=Bearer ***REDACTED***"); + } finally { + prior === undefined ? delete process.env.GITHUB_TOKEN : (process.env.GITHUB_TOKEN = prior); + } + }); + + it("redacts inline credential values that were not exported in host env", () => { + const prior = process.env.GITHUB_TOKEN; + delete process.env.GITHUB_TOKEN; + try { + const redacted = redactBridgeSecretsForDisplay( + "adapter echoed Authorization=Bearer inline-provider-secret and inline-provider-secret", + baseEntry, + { GITHUB_TOKEN: "inline-provider-secret" }, + ); + + expect(redacted).toBe("adapter echoed Authorization=Bearer ***REDACTED***"); + } finally { + prior === undefined ? delete process.env.GITHUB_TOKEN : (process.env.GITHUB_TOKEN = prior); + } + }); + + it("redacts resolved Authorization bearer values even without host env access", () => { + const prior = process.env.GITHUB_TOKEN; + delete process.env.GITHUB_TOKEN; + try { + const redacted = redactBridgeSecretsForDisplay( + '{"headers":{"Authorization":"Bearer resolved-provider-secret"},"raw":"Authorization: Bearer another-secret","status":"kept"}', + baseEntry, + ); + + expect(redacted).toBe( + '{"headers":{"Authorization":"Bearer ***REDACTED***"},"raw":"Authorization: Bearer ***REDACTED***","status":"kept"}', + ); + expect(JSON.parse(redacted)).toMatchObject({ status: "kept" }); + } finally { + prior === undefined ? delete process.env.GITHUB_TOKEN : (process.env.GITHUB_TOKEN = prior); + } + }); + + it("redacts overlapping raw values longest-first and removes display controls", () => { + const redacted = redactBridgeSecretsForDisplay( + "Authorization: raw-long-secret raw-long-secret raw\u001b[31m", + { env: ["LONG", "SHORT"] }, + { LONG: "raw-long-secret", SHORT: "raw" }, + ); + + expect(redacted).toBe("Authorization: ***REDACTED***"); + expect(redacted).not.toContain("secret"); + expect(redacted).not.toContain("\u001b"); + }); + + it("fully redacts generic bearer and authorization values", () => { + const redacted = redactBridgeSecretsForDisplay( + 'Bearer opaque-value Authorization="second-value"', + ); + + expect(redacted).toBe('Bearer ***REDACTED*** Authorization="***REDACTED***"'); + expect(redacted).not.toContain("opaque-value"); + expect(redacted).not.toContain("second-value"); + }); + + it("bounds generic values to one line while preserving quoted structured output", () => { + const redacted = redactBridgeSecretsForDisplay( + [ + "Authorization: Bearer alpha beta, gamma", + "next line kept", + '{"Authorization":"Bearer quoted secret,with,commas","status":"kept"}', + "MCP_TOKEN='assignment secret,with commas' status=kept", + ].join("\n"), + ); + + expect(redacted).toBe( + [ + "Authorization: Bearer ***REDACTED***", + "next line kept", + '{"Authorization":"Bearer ***REDACTED***","status":"kept"}', + "MCP_TOKEN='***REDACTED***' status=kept", + ].join("\n"), + ); + }); + + it("removes display controls before recognizing and redacting sensitive keys", () => { + const redacted = redactBridgeSecretsForDisplay( + "Authori\u001bzation: Bearer alpha\u0000 beta\nnext line kept", + ); + + expect(redacted).toBe("Authorization: Bearer ***REDACTED***\nnext line kept"); + expect(redacted).not.toMatch(/[\u0000-\u0008\u000b\u000c\u000e-\u001f]/); + }); + + it("strips ANSI before redacting split credential values from command output", () => { + const secret = "ansi-split-secret"; + const redacted = commandOutput( + { + status: 0, + stdout: `\u001b[2mId:\u001b[0m provider-id\nraw ${secret.slice(0, 5)}\u001b[31m${secret.slice(5)}\u001b[0m`, + stderr: "", + }, + { MCP_TOKEN: secret }, + ); + + expect(redacted).toBe("Id: provider-id\nraw ***REDACTED***"); + expect(redacted).not.toContain(secret); + expect(redacted).not.toContain("\u001b"); + }); +}); diff --git a/src/lib/actions/sandbox/mcp-bridge-output.ts b/src/lib/actions/sandbox/mcp-bridge-output.ts new file mode 100644 index 00000000000..36ba3206879 --- /dev/null +++ b/src/lib/actions/sandbox/mcp-bridge-output.ts @@ -0,0 +1,194 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { stripAnsi } from "../../adapters/openshell/client"; +import { redactStandaloneSecretsFull } from "../../security/redact"; +import type { McpBridgeEntry } from "../../state/registry"; + +export type OpenShellCommandResult = { + status: number | null; + stdout?: string | Buffer | null; + stderr?: string | Buffer | null; +}; + +const UNSAFE_DISPLAY_CONTROL_CHARS = /[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f-\u009f]/g; +const MCP_REDACTION_MARKER = "***REDACTED***"; +const MCP_SENSITIVE_VALUE_CANDIDATE = + /(?:(?:(["'])([A-Za-z_][A-Za-z0-9_-]*)\1|([A-Za-z_][A-Za-z0-9_-]*))\s*[:=]\s*|\bBearer\s+)/gi; + +type SensitiveValueCandidate = { + index: number; + end: number; + prefix: string; + key?: string; +}; + +function isSensitiveOutputKey(key: string): boolean { + return /authorization|api[_-]?key|token|secret|password|credential/i.test(key); +} + +function nextSensitiveValueCandidate( + line: string, + fromIndex: number, +): SensitiveValueCandidate | undefined { + const candidates = new RegExp( + MCP_SENSITIVE_VALUE_CANDIDATE.source, + MCP_SENSITIVE_VALUE_CANDIDATE.flags, + ); + candidates.lastIndex = fromIndex; + for (let match = candidates.exec(line); match; match = candidates.exec(line)) { + const key = match[2] ?? match[3]; + if (key && !isSensitiveOutputKey(key)) continue; + return { + index: match.index, + end: candidates.lastIndex, + prefix: match[0], + ...(key ? { key } : {}), + }; + } + return undefined; +} + +function enclosingQuoteAt(line: string, index: number): '"' | "'" | undefined { + let quote: '"' | "'" | undefined; + let escaped = false; + for (let cursor = 0; cursor < index; cursor++) { + const character = line[cursor]; + if (escaped) { + escaped = false; + continue; + } + if (character === "\\") { + escaped = true; + continue; + } + if (character !== '"' && character !== "'") continue; + quote = quote === character ? undefined : (quote ?? character); + } + return quote; +} + +function closingQuoteIndex(line: string, fromIndex: number, quote: '"' | "'"): number { + let escaped = false; + for (let cursor = fromIndex; cursor < line.length; cursor++) { + const character = line[cursor]; + if (escaped) { + escaped = false; + continue; + } + if (character === "\\") { + escaped = true; + continue; + } + if (character === quote) return cursor; + } + return -1; +} + +function redactSensitiveValuesOnLine(line: string): string { + let output = ""; + let cursor = 0; + while (cursor < line.length) { + const candidate = nextSensitiveValueCandidate(line, cursor); + if (!candidate) { + output += line.slice(cursor); + break; + } + + output += line.slice(cursor, candidate.index) + candidate.prefix; + let valueStart = candidate.end; + if (candidate.key) { + const bearer = /^Bearer\s+/i.exec(line.slice(valueStart)); + if (bearer) { + output += bearer[0]; + valueStart += bearer[0].length; + } + } + + const openingQuote = line[valueStart]; + if (openingQuote === '"' || openingQuote === "'") { + const closingQuote = closingQuoteIndex(line, valueStart + 1, openingQuote); + const quotedBearer = candidate.key + ? /^Bearer\s+/i.exec( + line.slice(valueStart + 1, closingQuote < 0 ? undefined : closingQuote), + ) + : null; + output += `${openingQuote}${quotedBearer?.[0] ?? ""}${MCP_REDACTION_MARKER}`; + if (closingQuote < 0) break; + output += openingQuote; + cursor = closingQuote + 1; + continue; + } + + const enclosingQuote = enclosingQuoteAt(line, candidate.index); + const enclosingQuoteEnd = enclosingQuote + ? closingQuoteIndex(line, valueStart, enclosingQuote) + : -1; + const followingCandidate = nextSensitiveValueCandidate(line, valueStart); + let valueEnd = + enclosingQuoteEnd >= 0 ? enclosingQuoteEnd : (followingCandidate?.index ?? line.length); + if (enclosingQuoteEnd < 0 && followingCandidate) { + while (valueEnd > valueStart && /\s/.test(line[valueEnd - 1] ?? "")) valueEnd--; + } + output += MCP_REDACTION_MARKER; + cursor = valueEnd; + } + return output; +} + +function explicitCredentialValues( + entry: Pick | undefined, + envValues: Record, +): string[] { + const values = [ + ...(entry?.env.map((name) => envValues[name] ?? process.env[name] ?? "") ?? []), + ...Object.values(envValues), + ]; + return [...new Set(values.filter(Boolean))].sort((left, right) => right.length - left.length); +} + +function redactMcpOutput( + text: string, + entry: Pick | undefined, + envValues: Record, +): string { + // Preserve the semantic text before removing standalone control bytes. + // Otherwise an SGR label such as `\x1b[2mId:\x1b[0m` becomes + // `[2mId:[0m`, which is safe to display but no longer parseable. + let output = stripAnsi(text || ""); + for (const value of explicitCredentialValues(entry, envValues)) { + output = output.replaceAll(value, MCP_REDACTION_MARKER); + } + output = output.replace(UNSAFE_DISPLAY_CONTROL_CHARS, ""); + output = output + .split(/(\r\n|\n|\r)/) + .map((part) => (/^(?:\r\n|\n|\r)$/.test(part) ? part : redactSensitiveValuesOnLine(part))) + .join(""); + return redactStandaloneSecretsFull(output); +} + +export function redactBridgeSecretsForDisplay( + text: string, + entry?: Pick, + envValues: Record = {}, +): string { + return redactMcpOutput(text, entry, envValues); +} + +export function redactCredentialValuesForDisplay( + value: string, + envValues: Record, +): string { + return redactMcpOutput(value, undefined, envValues); +} + +export function commandOutput( + result: OpenShellCommandResult, + envValues: Record = {}, +): string { + const stdout = + typeof result.stdout === "string" ? result.stdout : (result.stdout?.toString() ?? ""); + const stderr = + typeof result.stderr === "string" ? result.stderr : (result.stderr?.toString() ?? ""); + return redactMcpOutput(`${stderr}${stdout}`, undefined, envValues).replace(/\r/g, "").trim(); +} diff --git a/src/lib/actions/sandbox/mcp-bridge-policy-render.ts b/src/lib/actions/sandbox/mcp-bridge-policy-render.ts new file mode 100644 index 00000000000..94f2463bf80 --- /dev/null +++ b/src/lib/actions/sandbox/mcp-bridge-policy-render.ts @@ -0,0 +1,126 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import YAML from "yaml"; + +import type { AgentMcpAdapter } from "../../agent/defs"; +import { parseMcpUrl, validateMcpServerName } from "./mcp-bridge-validation"; + +export const MCP_BRIDGE_POLICY_MAX_BODY_BYTES = 131_072; +export const MCP_BRIDGE_ALLOWED_METHODS = [ + "initialize", + "notifications/initialized", + "ping", + "tools/list", + "tools/call", + "resources/list", + "resources/read", + "resources/templates/list", + "resources/subscribe", + "resources/unsubscribe", + "prompts/list", + "prompts/get", + "tasks/list", + "tasks/get", + "tasks/update", + "tasks/result", + "tasks/cancel", + "completion/complete", + "logging/setLevel", + "server/discover", + "messages/listen", + "notifications/cancelled", + "notifications/progress", + "notifications/roots/list_changed", + "notifications/elicitation/complete", +] as const; + +export function buildMcpBridgePolicyName(server: string): string { + validateMcpServerName(server); + return `mcp-bridge-${server.toLowerCase().replace(/_/g, "-")}`; +} + +export function buildMcpBridgePolicyKey(server: string): string { + return buildMcpBridgePolicyName(server).replace(/-/g, "_"); +} + +function endpointPort(url: URL): number { + if (url.port) return Number.parseInt(url.port, 10); + return url.protocol === "https:" ? 443 : 80; +} + +function endpointPath(url: URL): string { + return url.pathname || "/"; +} + +function binariesForAdapter(adapter: AgentMcpAdapter): Array<{ path: string }> { + switch (adapter) { + case "mcporter": + return [ + { path: "/usr/local/bin/mcporter" }, + { path: "/usr/bin/mcporter" }, + { path: "/usr/local/bin/openclaw" }, + // npm entrypoints are #!/usr/bin/env node scripts. OpenShell binds + // policy to /proc//exe and ancestors, not spoofable argv paths. + { path: "/usr/local/bin/node" }, + { path: "/usr/bin/node" }, + ]; + case "hermes-config": + return [ + { path: "/usr/local/bin/hermes" }, + // Hermes is a Python console script; /proc//exe resolves the venv + // interpreter to the system Python binary after the wrapper execs it. + { path: "/usr/bin/python3*" }, + { path: "/opt/hermes/.venv/bin/python*" }, + ]; + case "deepagents-config": + return [{ path: "/usr/local/bin/dcode" }, { path: "/opt/venv/bin/python3*" }]; + } +} + +function allowedIpsForEndpoint( + resolvedAddresses: readonly string[] | undefined, +): string[] | undefined { + // OpenShell resolves this hostname for every new connection, validates every + // current answer against allowed_ips, and connects to that validated list. + return resolvedAddresses && resolvedAddresses.length > 0 ? [...resolvedAddresses] : undefined; +} + +export function buildMcpBridgePolicyYaml( + server: string, + url: string, + adapter: AgentMcpAdapter, + resolvedAddresses?: readonly string[], +): string { + const parsed = parseMcpUrl(url); + const key = buildMcpBridgePolicyKey(server); + const allowedIps = allowedIpsForEndpoint(resolvedAddresses); + return YAML.stringify({ + preset: { + name: buildMcpBridgePolicyName(server), + description: `Generated MCP policy for ${server}`, + }, + network_policies: { + [key]: { + name: key, + endpoints: [ + { + host: parsed.hostname, + port: endpointPort(parsed), + path: endpointPath(parsed), + protocol: "mcp", + enforcement: "enforce", + ...(allowedIps ? { allowed_ips: allowedIps } : {}), + mcp: { + max_body_bytes: MCP_BRIDGE_POLICY_MAX_BODY_BYTES, + strict_tool_names: true, + allow_all_known_mcp_methods: false, + }, + rules: MCP_BRIDGE_ALLOWED_METHODS.map((method) => ({ allow: { method } })), + }, + ], + binaries: binariesForAdapter(adapter), + }, + }, + }); +} diff --git a/src/lib/actions/sandbox/mcp-bridge-policy.test.ts b/src/lib/actions/sandbox/mcp-bridge-policy.test.ts new file mode 100644 index 00000000000..424e9613106 --- /dev/null +++ b/src/lib/actions/sandbox/mcp-bridge-policy.test.ts @@ -0,0 +1,231 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, describe, expect, it, vi } from "vitest"; +import YAML from "yaml"; + +import * as policies from "../../policy"; +import * as registry from "../../state/registry"; +import { + buildMcpBridgePolicyName, + buildMcpBridgePolicyYaml, + buildMcpBridgeProviderName, + MCP_BRIDGE_ALLOWED_METHODS, + MCP_BRIDGE_POLICY_MAX_BODY_BYTES, +} from "./mcp-bridge"; +import { applyGeneratedPolicy } from "./mcp-bridge-policy"; + +describe("MCP OpenShell policy", () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("refuses to apply a generated policy without exact public address pins", () => { + expect(() => + applyGeneratedPolicy( + "alpha", + { + server: "github", + agent: "openclaw", + adapter: "mcporter", + url: "https://api.githubcopilot.com/mcp", + env: ["GITHUB_MCP_TOKEN"], + providerName: "alpha-mcp-github-0123456789abcdef", + policyName: "mcp-bridge-github", + addedAt: "2026-06-01T00:00:00.000Z", + }, + [], + ), + ).toThrow(/without exact public address pins/); + }); + + it("pins DNS answers while constraining the generic mcporter Node grant", () => { + const policyName = buildMcpBridgePolicyName("GitHub_Server"); + const policy = YAML.parse( + buildMcpBridgePolicyYaml("GitHub_Server", "https://api.githubcopilot.com/mcp", "mcporter", [ + "8.8.8.8", + "2606:4700:4700::1111", + ]), + ) as { + preset: { name: string }; + network_policies: Record< + string, + { + endpoints: Array<{ + host: string; + port: number; + path: string; + protocol: string; + mcp: { + max_body_bytes: number; + strict_tool_names?: boolean; + allow_all_known_mcp_methods?: boolean; + }; + allowed_ips?: string[]; + rules?: Array<{ allow: { method: string } }>; + }>; + binaries: Array<{ path: string }>; + } + >; + }; + const entry = policy.network_policies.mcp_bridge_github_server; + + expect(policyName).toBe("mcp-bridge-github-server"); + expect(policy.preset.name).toBe(policyName); + expect(entry.endpoints[0]).toMatchObject({ + host: "api.githubcopilot.com", + port: 443, + path: "/mcp", + protocol: "mcp", + enforcement: "enforce", + mcp: { + max_body_bytes: MCP_BRIDGE_POLICY_MAX_BODY_BYTES, + strict_tool_names: true, + allow_all_known_mcp_methods: false, + }, + }); + expect(entry.endpoints[0].rules).toEqual( + MCP_BRIDGE_ALLOWED_METHODS.map((method) => ({ + allow: { method }, + })), + ); + expect(entry.endpoints[0].allowed_ips).toEqual(["8.8.8.8", "2606:4700:4700::1111"]); + expect(entry.binaries.map((binary) => binary.path)).toEqual([ + "/usr/local/bin/mcporter", + "/usr/bin/mcporter", + "/usr/local/bin/openclaw", + "/usr/local/bin/node", + "/usr/bin/node", + ]); + expect(entry.endpoints[0].mcp).toEqual({ + max_body_bytes: MCP_BRIDGE_POLICY_MAX_BODY_BYTES, + strict_tool_names: true, + allow_all_known_mcp_methods: false, + }); + }); + + it("applies internally generated DNS pins outside the user-supplied preset path", () => { + vi.spyOn(registry, "getCustomPolicies").mockReturnValue([]); + vi.spyOn(registry, "addCustomPolicy").mockReturnValue(true); + vi.spyOn(policies, "getPresetContentGatewayState") + .mockReturnValueOnce("absent") + .mockReturnValueOnce("match"); + const applyPresetContent = vi.spyOn(policies, "applyPresetContent").mockReturnValue(true); + + applyGeneratedPolicy( + "alpha", + { + server: "github", + agent: "openclaw", + adapter: "mcporter", + url: "https://api.githubcopilot.com/mcp", + env: ["GITHUB_MCP_TOKEN"], + providerName: "alpha-mcp-github-0123456789abcdef", + policyName: "mcp-bridge-github", + addedAt: "2026-06-01T00:00:00.000Z", + }, + ["8.8.8.8"], + ); + + const [, , generatedContent, options] = applyPresetContent.mock.calls[0]; + expect(generatedContent).toContain("allowed_ips:"); + expect(options).toEqual({ + expectedExistingNetworkPolicyContent: null, + nonFatal: true, + skipRegistryUpdate: true, + }); + }); + + it("pins the current OpenShell main client-to-server MCP method profile", () => { + expect(MCP_BRIDGE_ALLOWED_METHODS).toEqual([ + "initialize", + "notifications/initialized", + "ping", + "tools/list", + "tools/call", + "resources/list", + "resources/read", + "resources/templates/list", + "resources/subscribe", + "resources/unsubscribe", + "prompts/list", + "prompts/get", + "tasks/list", + "tasks/get", + "tasks/update", + "tasks/result", + "tasks/cancel", + "completion/complete", + "logging/setLevel", + "server/discover", + "messages/listen", + "notifications/cancelled", + "notifications/progress", + "notifications/roots/list_changed", + "notifications/elicitation/complete", + ]); + }); + + it("emits only fields supported by OpenShell current main", () => { + const policy = YAML.parse( + buildMcpBridgePolicyYaml("srv", "https://mcp.example.test/mcp", "mcporter"), + ) as { network_policies: Record> }> }; + const endpoint = policy.network_policies.mcp_bridge_srv.endpoints[0]; + expect(endpoint).not.toHaveProperty("credential_keys"); + expect(endpoint).not.toHaveProperty("tls"); + }); + + it("refuses to generate authenticated policies for unpinnable OpenShell host aliases", () => { + for (const host of [ + "host.openshell.internal", + "host.openshell.internal.", + "host.docker.internal", + "host.containers.internal", + ]) { + expect(() => + buildMcpBridgePolicyYaml("local", `https://${host}:31337/mcp`, "mcporter"), + ).toThrow(/does not expose an attested driver gateway address/); + } + }); + + it("scopes binaries to the selected agent adapter", () => { + const hermes = YAML.parse( + buildMcpBridgePolicyYaml("srv", "https://mcp.example.test/mcp", "hermes-config"), + ) as { + network_policies: Record }>; + }; + const deepAgents = YAML.parse( + buildMcpBridgePolicyYaml("srv", "https://mcp.example.test/mcp", "deepagents-config"), + ) as { + network_policies: Record }>; + }; + + expect(hermes.network_policies.mcp_bridge_srv.binaries.map((b) => b.path)).toEqual([ + "/usr/local/bin/hermes", + "/usr/bin/python3*", + "/opt/hermes/.venv/bin/python*", + ]); + expect(deepAgents.network_policies.mcp_bridge_srv.binaries.map((b) => b.path)).toEqual([ + "/usr/local/bin/dcode", + "/opt/venv/bin/python3*", + ]); + }); + + it("uses stable collision-resistant provider names with a length guard", () => { + expect(buildMcpBridgeProviderName("alpha", "github-server")).toBe("alpha-mcp-github-server"); + const caseNormalized = buildMcpBridgeProviderName("alpha", "GitHub-Server"); + const underscoreNormalized = buildMcpBridgeProviderName("alpha", "github_server"); + expect(caseNormalized).toMatch(/^alpha-mcp-github-server-[a-f0-9]{16}$/); + expect(underscoreNormalized).toMatch(/^alpha-mcp-github-server-[a-f0-9]{16}$/); + expect(new Set([caseNormalized, underscoreNormalized, "alpha-mcp-github-server"]).size).toBe(3); + const long = buildMcpBridgeProviderName( + "sandbox-name-with-a-long-prefix", + "ServerNameThatWouldOtherwiseExceedTheProviderNameLimit", + ); + expect(long.length).toBeLessThanOrEqual(63); + expect(long).toMatch(/^sandbox-name-with-a-long-prefix-mcp-servername-[a-f0-9]{16}$/); + expect(buildMcpBridgeProviderName("alpha", "github-server", "0123456789abcdef")).toBe( + "alpha-mcp-github-server-0123456789abcdef", + ); + }); +}); diff --git a/src/lib/actions/sandbox/mcp-bridge-policy.ts b/src/lib/actions/sandbox/mcp-bridge-policy.ts new file mode 100644 index 00000000000..8a99f478341 --- /dev/null +++ b/src/lib/actions/sandbox/mcp-bridge-policy.ts @@ -0,0 +1,323 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import * as policies from "../../policy"; +import type { McpBridgeEntry } from "../../state/registry"; +import * as registry from "../../state/registry"; +import { + isAgentMcpAdapter, + MCP_BRIDGE_POLICY_SOURCE, + McpBridgeError, +} from "./mcp-bridge-contracts"; +import { buildMcpBridgePolicyKey, buildMcpBridgePolicyYaml } from "./mcp-bridge-policy-render"; + +export { + buildMcpBridgePolicyKey, + buildMcpBridgePolicyName, + buildMcpBridgePolicyYaml, + MCP_BRIDGE_ALLOWED_METHODS, + MCP_BRIDGE_POLICY_MAX_BODY_BYTES, +} from "./mcp-bridge-policy-render"; + +type GeneratedPolicyRegistrationState = { + policy: registry.CustomPolicyEntry; + state: "match" | "absent" | "drift" | null; + confirmed: boolean; +}; + +function withoutPendingContent( + policy: registry.CustomPolicyEntry, + content = policy.content, +): registry.CustomPolicyEntry { + const { pendingContent: _pendingContent, ...confirmed } = policy; + return { ...confirmed, content }; +} + +function persistGeneratedPolicyRegistration( + sandboxName: string, + policy: registry.CustomPolicyEntry, +): void { + if (!registry.addCustomPolicy(sandboxName, policy)) { + throw new McpBridgeError( + `Could not persist ownership for generated MCP policy '${policy.name}'.`, + ); + } +} + +/** + * Resolve a crash-interrupted generated-policy transition against the effective + * gateway policy. `content` remains the last confirmed value while + * `pendingContent` reserves the desired value, so either side of the mutation + * can be recognized safely after process death. + */ +function reconcileGeneratedPolicyRegistration( + sandboxName: string, + policy: registry.CustomPolicyEntry, +): GeneratedPolicyRegistrationState { + const pendingContent = policy.pendingContent; + if (pendingContent === undefined) { + return { + policy, + state: policies.getPresetContentGatewayState(sandboxName, policy.content), + confirmed: true, + }; + } + if (!pendingContent) { + return { policy, state: "drift", confirmed: false }; + } + + const pendingState = policies.getPresetContentGatewayState(sandboxName, pendingContent); + if (pendingState === "match") { + const confirmedPolicy = withoutPendingContent(policy, pendingContent); + persistGeneratedPolicyRegistration(sandboxName, confirmedPolicy); + return { policy: confirmedPolicy, state: "match", confirmed: true }; + } + + // A new add has no older confirmed value; content equals the reservation. + // Only an absent key is safe to retry. + if (pendingContent === policy.content) { + return { policy, state: pendingState, confirmed: false }; + } + + const confirmedState = policies.getPresetContentGatewayState(sandboxName, policy.content); + if (confirmedState === "match" || (confirmedState === "absent" && pendingState === "absent")) { + const confirmedPolicy = withoutPendingContent(policy); + persistGeneratedPolicyRegistration(sandboxName, confirmedPolicy); + return { policy: confirmedPolicy, state: confirmedState, confirmed: true }; + } + return { policy, state: confirmedState === null ? null : "drift", confirmed: false }; +} + +export function applyGeneratedPolicy( + sandboxName: string, + entry: McpBridgeEntry, + resolvedAddresses: readonly string[], +): void { + if (resolvedAddresses.length === 0) { + throw new McpBridgeError( + `Refusing to apply generated MCP policy '${entry.policyName}' without exact public address pins.`, + ); + } + const adapter = isAgentMcpAdapter(entry.adapter) ? entry.adapter : "mcporter"; + const content = buildMcpBridgePolicyYaml(entry.server, entry.url, adapter, resolvedAddresses); + const policyKey = buildMcpBridgePolicyKey(entry.server); + const sameNamePolicy = registry + .getCustomPolicies(sandboxName) + .find((policy) => policy.name === entry.policyName); + if (sameNamePolicy && sameNamePolicy.sourcePath !== MCP_BRIDGE_POLICY_SOURCE) { + throw new McpBridgeError( + `Generated MCP policy '${entry.policyName}' conflicts with an unowned same-name registry record. Refusing to replace operator-owned policy state.`, + ); + } + const registeredPolicy = sameNamePolicy; + let previousPolicy: registry.CustomPolicyEntry | undefined; + let previousPolicyConfirmed = false; + let ownsExistingPolicyKey = false; + if (registeredPolicy) { + const reconciled = reconcileGeneratedPolicyRegistration(sandboxName, registeredPolicy); + previousPolicy = reconciled.policy; + previousPolicyConfirmed = reconciled.confirmed; + const previousState = reconciled.state; + if (previousState !== "absent" && previousState !== "match") { + throw new McpBridgeError( + `Generated MCP policy '${entry.policyName}' has drifted or could not be inspected against its recorded content. Refusing to replace the live key.`, + ); + } + // A prior ownership record may have been reserved immediately before a + // process died, so an absent key is safe to create. A present key is safe + // to replace only after its full content matches that ownership record. + ownsExistingPolicyKey = previousState === "match"; + } else { + const unownedState = policies.getPresetContentGatewayState(sandboxName, content); + if (unownedState !== "absent") { + throw new McpBridgeError( + `Generated MCP policy key '${policyKey}' is already present or could not be inspected without a NemoClaw ownership record.`, + ); + } + } + + // Preserve the last confirmed content while reserving a changed desired + // value. For a brand-new key, content and pendingContent are intentionally + // equal so an absent live key remains recognizable as an uncommitted add. + let reservation: registry.CustomPolicyEntry; + if ( + previousPolicy && + previousPolicy.content === content && + (previousPolicy.pendingContent === undefined || previousPolicy.pendingContent === content) + ) { + reservation = previousPolicy; + } else if (previousPolicy) { + reservation = { ...withoutPendingContent(previousPolicy), pendingContent: content }; + persistGeneratedPolicyRegistration(sandboxName, reservation); + } else { + reservation = { + name: entry.policyName, + content, + pendingContent: content, + sourcePath: MCP_BRIDGE_POLICY_SOURCE, + }; + persistGeneratedPolicyRegistration(sandboxName, reservation); + } + // `custom` denotes user-supplied preset content and intentionally rejects + // `allowed_ips`. This content is generated from validated MCP inputs and the + // ownership reservation above; `skipRegistryUpdate` avoids a second write. + const ok = policies.applyPresetContent(sandboxName, entry.policyName, content, { + expectedExistingNetworkPolicyContent: + ownsExistingPolicyKey && previousPolicy ? previousPolicy.content : null, + nonFatal: true, + skipRegistryUpdate: true, + }); + // `policy set --wait` proves that a submitted revision loaded, but OpenShell + // also returns success for unchanged and concurrently superseded revisions. + // Confirm that the effective policy still contains our exact generated entry. + const activeState = policies.getPresetContentGatewayState(sandboxName, content); + if (ok !== false && activeState === "match") { + persistGeneratedPolicyRegistration(sandboxName, withoutPendingContent(reservation, content)); + return; + } + + if (previousPolicyConfirmed && previousPolicy) { + const previousState = policies.getPresetContentGatewayState( + sandboxName, + previousPolicy.content, + ); + if (previousState === "match" || (previousState === "absent" && activeState === "absent")) { + persistGeneratedPolicyRegistration(sandboxName, withoutPendingContent(previousPolicy)); + } + } else if (activeState === "absent") { + registry.removeCustomPolicyByName(sandboxName, entry.policyName); + } + const detail = + activeState === "match" ? "the update command failed" : `effective state: ${activeState}`; + throw new McpBridgeError( + `Failed to activate generated MCP policy '${entry.policyName}' (${detail}).`, + ); +} + +function generatedPolicyContent(entry: McpBridgeEntry): string { + const adapter = isAgentMcpAdapter(entry.adapter) ? entry.adapter : "mcporter"; + return buildMcpBridgePolicyYaml(entry.server, entry.url, adapter); +} + +export function assertGeneratedPolicyMutationSafe( + sandboxName: string, + entry: McpBridgeEntry, +): void { + const registeredPolicy = assertGeneratedPolicyRegistrationMutationSafe(sandboxName, entry); + const owned = registeredPolicy !== undefined; + const reconciled = registeredPolicy + ? reconcileGeneratedPolicyRegistration(sandboxName, registeredPolicy) + : undefined; + const content = reconciled?.policy.content ?? generatedPolicyContent(entry); + const state = reconciled?.state ?? policies.getPresetContentGatewayState(sandboxName, content); + if (state === "absent") return; + if (!owned || state !== "match") { + throw new McpBridgeError( + `Generated MCP policy '${entry.policyName}' is unowned, unreachable, or drifted. Refusing to mutate the adapter, provider, or same-key live policy until ownership is resolved.`, + ); + } +} + +/** Check registry ownership without consulting a sandbox already proven absent. */ +export function assertGeneratedPolicyRegistrationMutationSafe( + sandboxName: string, + entry: McpBridgeEntry, +): registry.CustomPolicyEntry | undefined { + const registeredPolicy = registry + .getCustomPolicies(sandboxName) + .find((policy) => policy.name === entry.policyName); + const owned = registeredPolicy?.sourcePath === MCP_BRIDGE_POLICY_SOURCE; + if (registeredPolicy && !owned) { + throw new McpBridgeError( + `Generated MCP policy '${entry.policyName}' conflicts with an unowned same-name registry record. Refusing to mutate the adapter, provider, or live policy.`, + ); + } + return owned ? registeredPolicy : undefined; +} + +export function removeGeneratedPolicy( + sandboxName: string, + entry: McpBridgeEntry, + options: { bestEffort?: boolean } = {}, +): void { + const policyName = entry.policyName; + const registeredPolicy = registry + .getCustomPolicies(sandboxName) + .find((policy) => policy.name === policyName); + const ownsRegistration = registeredPolicy?.sourcePath === MCP_BRIDGE_POLICY_SOURCE; + const reconciled = + registeredPolicy && ownsRegistration + ? reconcileGeneratedPolicyRegistration(sandboxName, registeredPolicy) + : undefined; + const effectiveRegistration = reconciled?.policy ?? registeredPolicy; + const content = effectiveRegistration?.content ?? generatedPolicyContent(entry); + const gatewayState = + reconciled?.state ?? policies.getPresetContentGatewayState(sandboxName, content); + if (gatewayState === "absent") { + if (ownsRegistration) { + registry.removeCustomPolicyByName(sandboxName, policyName); + } + return; + } + if (!ownsRegistration || gatewayState !== "match") { + if (options.bestEffort) return; + throw new McpBridgeError( + `Generated MCP policy '${policyName}' is unowned, unreachable, or no longer matches its registered content. Refusing to delete same-key policy state.`, + ); + } + const ok = policies.removePreset(sandboxName, policyName, { + nonFatal: true, + // Keep ownership durable across a crash or superseded OpenShell revision. + // It is cleared only after the exact live key is proven absent below. + skipRegistryUpdate: true, + }); + // OpenShell can acknowledge a superseded policy revision as success. Confirm + // the exact generated key is absent before discarding its ownership record. + const activeState = policies.getPresetContentGatewayState(sandboxName, content); + if (activeState === "absent") { + registry.removeCustomPolicyByName(sandboxName, policyName); + return; + } + // Keep (or defensively restore) the last reconciled ownership record when + // exact post-state is not proven. + if (ownsRegistration && effectiveRegistration) { + persistGeneratedPolicyRegistration(sandboxName, effectiveRegistration); + } + if (options.bestEffort) return; + const detail = ok ? `effective state: ${activeState}` : "the removal command failed"; + throw new McpBridgeError(`Failed to remove generated MCP policy '${policyName}' (${detail}).`); +} + +export function getRegisteredGeneratedPolicy( + sandboxName: string, + entry: McpBridgeEntry | undefined, +): ReturnType[number] | undefined { + if (!entry?.policyName) return undefined; + return registry + .getCustomPolicies(sandboxName) + .find( + (policy) => + policy.name === entry.policyName && policy.sourcePath === MCP_BRIDGE_POLICY_SOURCE, + ); +} + +export function getPolicyPresence( + sandboxName: string, + entry: McpBridgeEntry | undefined, +): boolean | null { + if (!entry?.policyName) return false; + const registeredPolicy = getRegisteredGeneratedPolicy(sandboxName, entry); + if (!registeredPolicy) return null; + const confirmedState = policies.getPresetContentGatewayState( + sandboxName, + registeredPolicy.content, + ); + if (confirmedState === "match") return true; + const pendingContent = registeredPolicy.pendingContent; + if (typeof pendingContent !== "string" || pendingContent.length === 0) { + return confirmedState === null ? null : false; + } + const pendingState = policies.getPresetContentGatewayState(sandboxName, pendingContent); + if (pendingState === "match") return true; + return confirmedState === null || pendingState === null ? null : false; +} diff --git a/src/lib/actions/sandbox/mcp-bridge-provider-attachments.ts b/src/lib/actions/sandbox/mcp-bridge-provider-attachments.ts new file mode 100644 index 00000000000..6a548cde324 --- /dev/null +++ b/src/lib/actions/sandbox/mcp-bridge-provider-attachments.ts @@ -0,0 +1,206 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Provider attachment mutations are guarded by immutable provider identity and + * credential-shape inspection before and after each OpenShell command. Keep + * this compensation until attachment mutations expose an immutable-ID CAS API. + */ + +import { runOpenshellProviderCommand } from "../../actions/global"; +import { stripAnsi } from "../../adapters/openshell/client"; +import type { McpBridgeEntry } from "../../state/registry"; +import { McpBridgeError } from "./mcp-bridge-contracts"; +import { commandOutput, type OpenShellCommandResult } from "./mcp-bridge-output"; +import { + inspectMcpProvider, + inspectMcpProviderAttachments, + type McpProviderAttachment, + type McpProviderAttachmentInspection, + providerMatchesCredential, + providerShapeDetail, +} from "./mcp-bridge-provider-inspection"; +import { + assertAuthenticatedBridgeEntry, + assertPersistedAuthenticatedBridgeEntry, +} from "./mcp-bridge-validation"; + +function exactAttachment( + sandboxName: string, + entry: McpBridgeEntry, +): { inspection: McpProviderAttachmentInspection; attachment?: McpProviderAttachment } { + const inspection = inspectMcpProviderAttachments(sandboxName); + return { + inspection, + attachment: inspection.attachments?.find( + (attachment) => attachment.name === entry.providerName, + ), + }; +} + +function attachmentMatchesCurrentProviderSnapshot( + attachment: McpProviderAttachment | undefined, + entry: McpBridgeEntry, +): boolean { + return ( + !!attachment && + attachment.providerId === entry.providerId && + entry.env.length === 1 && + attachment.credentialKeys.length === 1 && + attachment.credentialKeys[0] === entry.env[0] + ); +} + +export function attachProvider(sandboxName: string, entry: McpBridgeEntry): void { + if (!entry.providerName) return; + assertAuthenticatedBridgeEntry(entry); + if (!entry.providerId) { + throw new McpBridgeError( + `MCP server '${entry.server}' has no stable OpenShell provider ID. Refusing to attach same-name provider '${entry.providerName}'.`, + ); + } + const inspection = inspectMcpProvider(entry.providerName); + if (inspection.exists === false) { + throw new McpBridgeError( + `OpenShell provider '${entry.providerName}' disappeared before attach.`, + ); + } + if (!providerMatchesCredential(inspection, entry.env[0], entry.providerId)) { + throw new McpBridgeError( + `OpenShell provider '${entry.providerName}' changed before attach. ${providerShapeDetail(inspection, entry.env[0], entry.providerId)} Refusing to mutate it.`, + ); + } + if (!inspection.id || !inspection.resourceVersion) { + throw new McpBridgeError(`OpenShell provider '${entry.providerName}' has incomplete metadata.`); + } + const result = runOpenshellProviderCommand( + ["sandbox", "provider", "attach", sandboxName, entry.providerName], + { ignoreError: true, stdio: ["ignore", "pipe", "pipe"] }, + ) as OpenShellCommandResult; + if (result.status !== 0) { + const output = commandOutput(result); + const afterError = exactAttachment(sandboxName, entry); + if (attachmentMatchesCurrentProviderSnapshot(afterError.attachment, entry)) return; + throw new McpBridgeError( + output || + afterError.inspection.error || + `Failed to attach MCP provider '${entry.providerName}'.`, + ); + } + const after = exactAttachment(sandboxName, entry); + if (!attachmentMatchesCurrentProviderSnapshot(after.attachment, entry)) { + throw new McpBridgeError( + after.inspection.error ?? + `OpenShell did not persist the expected provider identity and credential shape for '${entry.providerName}' after attach.`, + ); + } +} + +export function providerDetachChangedState(status: number | null, output: string): boolean { + return ( + status === 0 && + !/\bwas\s+not\s+attached\b|\balready\s+detached\b|\bNotAttached\b/i.test(stripAnsi(output)) + ); +} + +export type ProviderDetachOutcome = "detached" | "absent" | "unknown"; + +export function detachProvider( + sandboxName: string, + entry: McpBridgeEntry, + options: { bestEffort?: boolean } = {}, +): ProviderDetachOutcome { + if (!entry.providerName) return "absent"; + assertPersistedAuthenticatedBridgeEntry(entry); + if (!entry.providerId) { + if (options.bestEffort) return "unknown"; + throw new McpBridgeError( + `MCP server '${entry.server}' has no recorded provider ID for prechecked detach.`, + ); + } + const before = exactAttachment(sandboxName, entry); + if (!before.inspection.attachments) { + if (options.bestEffort) return "unknown"; + throw new McpBridgeError( + before.inspection.error ?? `Could not inspect provider attachment '${entry.providerName}'.`, + ); + } + if (!before.attachment) return "absent"; + if ( + before.attachment.providerId !== entry.providerId || + before.attachment.credentialKeys.length !== 1 || + before.attachment.credentialKeys[0] !== entry.env[0] + ) { + if (options.bestEffort) return "unknown"; + throw new McpBridgeError( + `Provider attachment '${entry.providerName}' does not match MCP server '${entry.server}'. Expected stable provider ID '${entry.providerId}', found '${before.attachment.providerId ?? "missing"}', with credential keys '${before.attachment.credentialKeys.join(", ") || "none"}'.`, + ); + } + const result = runOpenshellProviderCommand( + ["sandbox", "provider", "detach", sandboxName, entry.providerName], + { + ignoreError: true, + stdio: ["ignore", "pipe", "pipe"], + suppressOutput: true, + } as Record, + ) as OpenShellCommandResult; + const output = commandOutput(result); + const after = exactAttachment(sandboxName, entry); + if (after.inspection.attachments && !after.attachment) { + return providerDetachChangedState(result.status, output) ? "detached" : "absent"; + } + if (options.bestEffort) return "unknown"; + throw new McpBridgeError( + output || + after.inspection.error || + `OpenShell did not confirm removal of provider attachment '${entry.providerName}'.`, + ); +} + +/** + * Remove a dangling provider name from the sandbox spec after the provider + * object itself has been independently proven absent. OpenShell main cannot + * list attachments while a referenced provider is missing, but its detach + * command removes the name directly from the sandbox spec under CAS. + */ +export function detachMissingProviderReference( + sandboxName: string, + entry: McpBridgeEntry, +): ProviderDetachOutcome { + if (!entry.providerName) return "absent"; + assertPersistedAuthenticatedBridgeEntry(entry); + const before = inspectMcpProvider(entry.providerName); + if (before.exists !== false) { + const detail = + before.exists === null + ? (before.error ?? "provider inspection failed") + : `provider ID '${before.id ?? "unparseable"}' is present`; + throw new McpBridgeError( + `OpenShell provider '${entry.providerName}' is not provably absent before dangling-reference cleanup: ${detail}.`, + ); + } + const result = runOpenshellProviderCommand( + ["sandbox", "provider", "detach", sandboxName, entry.providerName], + { ignoreError: true, stdio: ["ignore", "pipe", "pipe"] }, + ) as OpenShellCommandResult; + const output = commandOutput(result); + if (result.status !== 0) { + throw new McpBridgeError( + output || `Failed to remove dangling provider reference '${entry.providerName}'.`, + ); + } + const afterProvider = inspectMcpProvider(entry.providerName); + if (afterProvider.exists !== false) { + throw new McpBridgeError( + afterProvider.error ?? + `A same-name provider appeared while removing dangling reference '${entry.providerName}'. Refusing to create or adopt it.`, + ); + } + const cleanOutput = stripAnsi(output); + if (!/\bDetached provider\b|\bwas not attached to sandbox\b/i.test(cleanOutput)) { + throw new McpBridgeError( + `OpenShell returned an unrecognized result while removing dangling provider reference '${entry.providerName}'.`, + ); + } + return providerDetachChangedState(result.status, output) ? "detached" : "absent"; +} diff --git a/src/lib/actions/sandbox/mcp-bridge-provider-inspection.ts b/src/lib/actions/sandbox/mcp-bridge-provider-inspection.ts new file mode 100644 index 00000000000..9a3f76e3f9a --- /dev/null +++ b/src/lib/actions/sandbox/mcp-bridge-provider-inspection.ts @@ -0,0 +1,289 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { runOpenshellProviderCommand } from "../../actions/global"; +import { stripAnsi } from "../../adapters/openshell/client"; +import type { McpBridgeEntry } from "../../state/registry"; +import { McpBridgeError } from "./mcp-bridge-contracts"; +import { commandOutput, type OpenShellCommandResult } from "./mcp-bridge-output"; +import { + assertAuthenticatedBridgeEntry, + normalizeMcpServerUrl, + validateMcpServerUrlResolvedTarget, +} from "./mcp-bridge-validation"; + +export type McpProviderInspection = { + exists: boolean | null; + id: string | null; + resourceVersion: number | null; + type: string | null; + credentialKeys: string[] | null; + error?: string; +}; + +export type McpProviderAttachment = { + name: string; + providerId: string | null; + credentialKeys: string[]; +}; + +export type McpProviderAttachmentInspection = { + attachments: McpProviderAttachment[] | null; + error?: string; +}; + +const MCP_PROVIDER_ID_RE = /^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$/; + +export function parseMcpProviderMetadata(output: string): Omit { + const clean = stripAnsi(output).replace(/\r/g, ""); + const idMatch = clean.match(/^\s*Id:\s*(\S.*?)\s*$/m); + const resourceVersionMatch = clean.match(/^\s*Resource version:\s*(\d+)\s*$/m); + const typeMatch = clean.match(/^\s*Type:\s*(\S.*?)\s*$/m); + const credentialMatch = clean.match(/^\s*Credential keys:\s*(.*?)\s*$/m); + const rawId = idMatch?.[1]?.trim(); + const parsedResourceVersion = resourceVersionMatch + ? Number.parseInt(resourceVersionMatch[1] ?? "", 10) + : null; + const rawKeys = credentialMatch?.[1]?.trim(); + return { + id: rawId && MCP_PROVIDER_ID_RE.test(rawId) ? rawId : null, + resourceVersion: + parsedResourceVersion !== null && Number.isSafeInteger(parsedResourceVersion) + ? parsedResourceVersion + : null, + type: typeMatch?.[1]?.trim() || null, + credentialKeys: + rawKeys === undefined + ? null + : rawKeys === "" || rawKeys === "" + ? [] + : rawKeys.split(",").map((key) => key.trim()), + }; +} + +export function inspectMcpProvider(providerName: string | undefined): McpProviderInspection { + if (!providerName) { + return { + exists: false, + id: null, + resourceVersion: null, + type: null, + credentialKeys: null, + }; + } + const result = runOpenshellProviderCommand(["provider", "get", providerName], { + ignoreError: true, + stdio: ["ignore", "pipe", "pipe"], + }) as OpenShellCommandResult; + if (result.status !== 0) { + const output = commandOutput(result); + if (/not\s+found|NotFound|does\s+not\s+exist|unknown\s+provider/i.test(output)) { + return { + exists: false, + id: null, + resourceVersion: null, + type: null, + credentialKeys: null, + }; + } + return { + exists: null, + id: null, + resourceVersion: null, + type: null, + credentialKeys: null, + error: output || `Could not inspect OpenShell provider '${providerName}'.`, + }; + } + return { + exists: true, + ...parseMcpProviderMetadata(commandOutput(result)), + }; +} + +export function parseMcpProviderAttachmentNames(output: string): string[] { + const clean = stripAnsi(output).replace(/\r/g, "").trim(); + if (/^No providers attached to sandbox\b/m.test(clean)) return []; + const lines = clean + .split("\n") + .map((line) => line.trim()) + .filter(Boolean); + const headerIndex = lines.findIndex((line) => + /^NAME\s+TYPE\s+CREDENTIAL_KEYS\s+CONFIG_KEYS$/.test(line), + ); + if (headerIndex < 0) throw new Error("missing provider attachment table header"); + return lines.slice(headerIndex + 1).map((line) => { + const match = line.match(/^(\S+)\s+(\S+)\s+(\d+)\s+(\d+)$/); + if (!match?.[1]) throw new Error("invalid provider attachment table row"); + return match[1]; + }); +} + +export function inspectMcpProviderAttachments( + sandboxName: string, +): McpProviderAttachmentInspection { + const result = runOpenshellProviderCommand(["sandbox", "provider", "list", sandboxName], { + ignoreError: true, + stdio: ["ignore", "pipe", "pipe"], + }) as OpenShellCommandResult; + const output = commandOutput(result); + if (result.status !== 0) { + return { attachments: null, error: output || "provider attachment inspection failed" }; + } + try { + const clean = stripAnsi(output).replace(/\r/g, "").trim(); + if (/^No providers attached to sandbox\b/m.test(clean)) return { attachments: [] }; + const names = parseMcpProviderAttachmentNames(clean); + const attachments = names.map((name) => { + const provider = inspectMcpProvider(name); + if ( + provider.exists !== true || + !provider.id || + !provider.resourceVersion || + !provider.type || + !provider.credentialKeys + ) { + throw new Error( + provider.error ?? `attached provider '${name}' disappeared or has incomplete metadata`, + ); + } + return { + name, + providerId: provider.id, + credentialKeys: provider.credentialKeys, + }; + }); + return { attachments }; + } catch (error) { + return { + attachments: null, + error: `OpenShell returned invalid provider attachment metadata: ${error instanceof Error ? error.message : String(error)}`, + }; + } +} + +export function assertNoAttachedProviderCredentialCollision( + sandboxName: string, + entry: McpBridgeEntry, +): void { + const inspection = inspectMcpProviderAttachments(sandboxName); + if (!inspection.attachments) { + throw new McpBridgeError( + inspection.error ?? `Could not inspect providers attached to sandbox '${sandboxName}'.`, + ); + } + const credentialKey = entry.env[0]; + const collision = inspection.attachments.find( + (attachment) => + attachment.credentialKeys.includes(credentialKey) && + !(attachment.name === entry.providerName && attachment.providerId === entry.providerId), + ); + if (collision) { + throw new McpBridgeError( + `Credential key '${credentialKey}' is already supplied by attached provider '${collision.name}' with ID '${collision.providerId ?? "missing"}'. Refusing to reserve the key for MCP before provider activation.`, + ); + } +} + +export function providerMatchesCredential( + inspection: McpProviderInspection, + expectedCredential: string | undefined, + expectedProviderId: string | undefined, +): boolean { + return ( + inspection.exists === true && + expectedProviderId !== undefined && + inspection.id === expectedProviderId && + inspection.resourceVersion !== null && + inspection.type === "generic" && + expectedCredential !== undefined && + inspection.credentialKeys?.length === 1 && + inspection.credentialKeys[0] === expectedCredential + ); +} + +export function providerShapeDetail( + inspection: McpProviderInspection, + expectedCredential: string | undefined, + expectedProviderId?: string, +): string | undefined { + if (inspection.exists === null) return inspection.error ?? "provider inspection failed"; + const id = inspection.id ?? "unparseable"; + if (!expectedProviderId) { + return inspection.exists + ? `The registry entry has no stable OpenShell provider ID; live provider ID is '${id}'.` + : "The registry entry has no stable OpenShell provider ID."; + } + if (!inspection.exists) return undefined; + if (providerMatchesCredential(inspection, expectedCredential, expectedProviderId)) { + return undefined; + } + if (inspection.id !== expectedProviderId) { + return `Expected stable provider ID '${expectedProviderId}', found '${id}'.`; + } + if (inspection.resourceVersion === null) { + return "OpenShell provider metadata did not include a valid resource version."; + } + const type = inspection.type ?? "unparseable"; + const keys = inspection.credentialKeys?.join(", ") || "none or unparseable"; + return `Expected generic provider with only credential key '${expectedCredential ?? ""}', found type '${type}' with keys '${keys}'.`; +} + +export function assertMcpProviderRecoverable(entry: McpBridgeEntry): McpProviderInspection { + assertAuthenticatedBridgeEntry(entry); + if (!entry.providerId) { + throw new McpBridgeError( + `MCP server '${entry.server}' has no stable OpenShell provider ID. Refusing to adopt or mutate same-name provider '${entry.providerName}'; remove the legacy bridge with --force and recreate it after independently cleaning the provider.`, + ); + } + const expectedCredential = entry.env[0]; + const inspection = inspectMcpProvider(entry.providerName); + if (inspection.exists === null) { + throw new McpBridgeError( + inspection.error ?? `Could not inspect OpenShell provider '${entry.providerName}'.`, + ); + } + if (inspection.exists) { + if (!providerMatchesCredential(inspection, expectedCredential, entry.providerId)) { + throw new McpBridgeError( + `OpenShell provider '${entry.providerName}' no longer exactly matches MCP server '${entry.server}'. ${providerShapeDetail(inspection, expectedCredential, entry.providerId)}`, + ); + } + return inspection; + } + if (!process.env[expectedCredential]) { + throw new McpBridgeError( + `OpenShell provider '${entry.providerName}' is missing. Export host environment variable '${expectedCredential}' before retrying so the authenticated MCP provider can be recreated.`, + ); + } + return inspection; +} + +export async function preflightMcpEntryTargets( + entries: readonly McpBridgeEntry[], +): Promise> { + for (const entry of entries) assertAuthenticatedBridgeEntry(entry); + const results = await Promise.all( + entries.map(async (entry) => { + const normalized = normalizeMcpServerUrl(entry.url); + if (normalized !== entry.url) { + throw new McpBridgeError( + `MCP server '${entry.server}' has a non-canonical stored URL. Remove it with --force and add it again before lifecycle operations.`, + ); + } + const addresses = await validateMcpServerUrlResolvedTarget(new URL(normalized)); + return [entry.server, addresses] as const; + }), + ); + return new Map(results); +} + +export function providerAttached( + sandboxName: string, + providerName: string | undefined, +): boolean | null { + if (!providerName) return null; + const inspection = inspectMcpProviderAttachments(sandboxName); + if (!inspection.attachments) return null; + return inspection.attachments.some((attachment) => attachment.name === providerName); +} diff --git a/src/lib/actions/sandbox/mcp-bridge-provider-mutation.ts b/src/lib/actions/sandbox/mcp-bridge-provider-mutation.ts new file mode 100644 index 00000000000..32895b5435f --- /dev/null +++ b/src/lib/actions/sandbox/mcp-bridge-provider-mutation.ts @@ -0,0 +1,240 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * OpenShell v0.0.72 provider mutations have no compare-and-swap operation, so + * another client can race between NemoClaw's preinspection and mutation. A + * nonzero mutation result is therefore ambiguous and always fails closed; + * NemoClaw never infers success from a later resource-version increase. + * Randomized provider names, the MCP lifecycle lock, and mandatory + * postinspection of immutable identity, credential shape, and resource version + * constrain this TOCTOU boundary. Remove the compensation when OpenShell + * exposes provider CAS or immutable provider IDs as mutation targets. + */ + +import { runOpenshellProviderCommand } from "../../actions/global"; +import type { McpBridgeEntry } from "../../state/registry"; +import { McpBridgeError, type ParsedEnvReference } from "./mcp-bridge-contracts"; +import { commandOutput, type OpenShellCommandResult } from "./mcp-bridge-output"; +import { + inspectMcpProvider, + type McpProviderInspection, + providerMatchesCredential, + providerShapeDetail, +} from "./mcp-bridge-provider-inspection"; +import { + assertPersistedAuthenticatedBridgeEntry, + resolveCredentialEnv, + uniqueEnvNames, + validateMcpCredentialEnvName, +} from "./mcp-bridge-validation"; + +export type { ProviderDetachOutcome } from "./mcp-bridge-provider-attachments"; +export { + attachProvider, + detachMissingProviderReference, + detachProvider, + providerDetachChangedState, +} from "./mcp-bridge-provider-attachments"; + +export function buildMcpBridgeProviderArgs( + action: "create" | "update", + providerName: string, + env: readonly ParsedEnvReference[], + envValues: Record, +): string[] { + const args = + action === "create" + ? ["provider", "create", "--name", providerName, "--type", "generic"] + : ["provider", "update", providerName]; + for (const entry of env) { + validateMcpCredentialEnvName(entry.name); + const value = envValues[entry.name]; + if (value !== undefined && value !== "") { + args.push("--credential", entry.name); + } + } + return args; +} + +export function upsertMcpProvider( + providerName: string, + env: readonly ParsedEnvReference[], + options: { + allowExisting: boolean; + expectedProviderId?: string; + prepareMutation?: (action: "create" | "update") => void; + }, +): { + action: "created" | "updated" | "reused" | "none"; + inspection: McpProviderInspection; +} { + const envNames = uniqueEnvNames(env); + if (envNames.length === 0) { + return { + action: "none", + inspection: { + exists: false, + id: null, + resourceVersion: null, + type: null, + credentialKeys: null, + }, + }; + } + const envValues = resolveCredentialEnv(env); + const inspection = inspectMcpProvider(providerName); + if (inspection.exists === null) { + throw new McpBridgeError( + inspection.error ?? `Could not inspect OpenShell provider '${providerName}'.`, + ); + } + if (inspection.exists && !options.allowExisting) { + throw new McpBridgeError( + `OpenShell provider '${providerName}' already exists but is not owned by a registered MCP bridge. Remove or rename that provider before retrying.`, + ); + } + if (inspection.exists && !options.expectedProviderId) { + throw new McpBridgeError( + `OpenShell provider '${providerName}' already exists, but the incomplete MCP add has no stable provider ID and cannot safely adopt it. Remove that provider independently, then retry the original mcp add command.`, + ); + } + if ( + inspection.exists && + !providerMatchesCredential(inspection, envNames[0], options.expectedProviderId) + ) { + throw new McpBridgeError( + `OpenShell provider '${providerName}' no longer exactly matches MCP server credential '${envNames[0]}'. ${providerShapeDetail(inspection, envNames[0], options.expectedProviderId)} Remove the stale provider and run mcp restart with the credential exported.`, + ); + } + if (Object.keys(envValues).length === 0) { + if (inspection.exists) return { action: "reused", inspection }; + throw new McpBridgeError( + `Host environment variable '${envNames[0]}' is required to create MCP provider '${providerName}'.`, + 1, + ); + } + const action = inspection.exists ? "update" : "create"; + // Let callers establish policy and revision proofs only after the actual + // mutation kind is known. The immediate reinspection below closes races + // that occur while those fail-closed prerequisites are being prepared. + options.prepareMutation?.(action); + // invalidState: another OpenShell client replaces a mutable provider name + // between inspection and mutation. sourceBoundary: OpenShell owns provider + // compare-and-swap; v0.0.72 exposes no provider CAS flags. whyNotSourceFix: + // NemoClaw cannot atomically mutate the upstream store, so it uses randomized + // names, a lifecycle mutex, and immutable-ID/resource-version reinspection. + // regressionTest: mcp-provider-ownership.test.ts simulates a concurrent + // resource-version writer and requires the ambiguous update to fail closed. + // removalCondition: use native immutable provider IDs/CAS once OpenShell + // exposes them, then remove this inspect-mutate-inspect compensation. + const beforeMutation = inspectMcpProvider(providerName); + if (action === "create" && beforeMutation.exists !== false) { + const detail = + beforeMutation.exists === null + ? (beforeMutation.error ?? "provider inspection failed") + : "a same-name provider appeared after preflight"; + throw new McpBridgeError( + `OpenShell provider '${providerName}' changed before create: ${detail}. Refusing to mutate it.`, + ); + } + if ( + action === "update" && + !providerMatchesCredential(beforeMutation, envNames[0], options.expectedProviderId) + ) { + throw new McpBridgeError( + `OpenShell provider '${providerName}' changed before update. ${providerShapeDetail(beforeMutation, envNames[0], options.expectedProviderId)} Refusing to mutate it.`, + ); + } + const result = runOpenshellProviderCommand( + buildMcpBridgeProviderArgs(action, providerName, env, envValues), + { + ignoreError: true, + env: envValues, + stdio: ["ignore", "pipe", "pipe"], + }, + ) as OpenShellCommandResult; + if (result.status !== 0) { + // Never infer that our update committed from a later resource-version + // increase: a concurrent writer can advance the same provider after our + // command failed. A non-zero result is ambiguous and must fail closed. + throw new McpBridgeError( + commandOutput(result, envValues) || `Failed to ${action} MCP provider '${providerName}'.`, + ); + } + const after = inspectMcpProvider(providerName); + if (after.exists !== true || !after.id) { + throw new McpBridgeError( + after.error ?? + `OpenShell did not return a stable provider ID after ${action} for '${providerName}'. Refusing later MCP side effects.`, + ); + } + const expectedProviderId = action === "create" ? after.id : options.expectedProviderId; + if ( + !after.resourceVersion || + !providerMatchesCredential(after, envNames[0], expectedProviderId) || + (action === "update" && after.resourceVersion <= (beforeMutation.resourceVersion ?? 0)) + ) { + throw new McpBridgeError( + `OpenShell provider '${providerName}' changed during ${action}. ${providerShapeDetail(after, envNames[0], expectedProviderId)} Refusing later MCP side effects.`, + ); + } + return { action: action === "create" ? "created" : "updated", inspection: after }; +} + +function inspectMcpProviderForDeletion( + entry: McpBridgeEntry, + options: { allowMissing?: boolean; bestEffort?: boolean } = {}, +): McpProviderInspection | null { + if (!entry.providerName) return null; + try { + assertPersistedAuthenticatedBridgeEntry(entry); + if (!entry.providerId) { + throw new McpBridgeError( + `MCP server '${entry.server}' has no stable OpenShell provider ID. Refusing to delete same-name provider '${entry.providerName}'.`, + ); + } + const inspection = inspectMcpProvider(entry.providerName); + if (inspection.exists === false) { + if (options.allowMissing) return inspection; + throw new McpBridgeError( + `OpenShell provider '${entry.providerName}' disappeared before delete.`, + ); + } + if (!providerMatchesCredential(inspection, entry.env[0], entry.providerId)) { + throw new McpBridgeError( + `OpenShell provider '${entry.providerName}' changed before delete. ${providerShapeDetail(inspection, entry.env[0], entry.providerId)} Refusing to mutate it.`, + ); + } + return inspection; + } catch (error) { + if (options.bestEffort) return null; + throw error; + } +} + +export function deleteProvider( + entry: McpBridgeEntry, + options: { allowMissing?: boolean; bestEffort?: boolean } = {}, +): void { + if (!entry.providerName) return; + const inspection = inspectMcpProviderForDeletion(entry, options); + if (!inspection?.exists || !inspection.id || !inspection.resourceVersion) return; + const result = runOpenshellProviderCommand(["provider", "delete", entry.providerName], { + ignoreError: true, + stdio: ["ignore", "pipe", "pipe"], + suppressOutput: true, + } as Record) as OpenShellCommandResult; + if (result.status !== 0) { + const output = commandOutput(result); + if (options.allowMissing && /not\s+found|NotFound/i.test(output)) return; + if (options.bestEffort) return; + throw new McpBridgeError(output || `Failed to delete MCP provider '${entry.providerName}'.`); + } + const after = inspectMcpProvider(entry.providerName); + if (after.exists !== false && !options.bestEffort) { + throw new McpBridgeError( + after.error ?? `OpenShell provider '${entry.providerName}' still exists after delete.`, + ); + } +} diff --git a/src/lib/actions/sandbox/mcp-bridge-provider-readiness.ts b/src/lib/actions/sandbox/mcp-bridge-provider-readiness.ts new file mode 100644 index 00000000000..946054a0e05 --- /dev/null +++ b/src/lib/actions/sandbox/mcp-bridge-provider-readiness.ts @@ -0,0 +1,197 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { waitUntil } from "../../core/wait"; +import { shellQuote } from "../../runner"; +import type { McpBridgeEntry } from "../../state/registry"; +import { McpBridgeError } from "./mcp-bridge-contracts"; +import { + assertAuthenticatedBridgeEntry, + assertPersistedAuthenticatedBridgeEntry, + validateMcpCredentialEnvName, +} from "./mcp-bridge-validation"; +import { executeSandboxExecCommand } from "./process-recovery"; + +const MCP_CREDENTIAL_REVISION_OBSERVATION_RE = /^(?:absent|canonical|v[0-9]{1,20})$/; + +export type McpCredentialRevisionObservation = "absent" | "canonical" | `v${number}`; + +/** + * Provider synchronization proofs must observe a fresh OpenShell-mediated exec + * environment. A direct Docker exec does not receive OpenShell provider state + * and could otherwise make an absent credential look successfully revoked. + */ +function executeMcpCredentialProofCommand( + sandboxName: string, + command: string, +): ReturnType { + // OpenShell current main rejects CR/LF in each sandbox-exec argv element. + // Transport the proof as base64 so the `sh -c` argument remains one line; + // the decoded script still runs only inside the sandbox and contains no raw + // credential value. + const encodedCommand = Buffer.from(command, "utf8").toString("base64"); + const transportCommand = [ + "command -v base64 >/dev/null 2>&1 || { echo NEMOCLAW_BASE64_MISSING >&2; exit 127; }", + `decoded="$(printf '%s' '${encodedCommand}' | base64 -d)" || exit 1`, + `printf '%s' "$decoded" | sh`, + ].join("; "); + return executeSandboxExecCommand(sandboxName, transportCommand, undefined, { + allowLocalDockerFallback: false, + }); +} + +function mcpCredentialPlaceholderValidatorShell(envName: string): string[] { + validateMcpCredentialEnvName(envName); + const canonical = `openshell:resolve:env:${envName}`; + const revisionPrefix = "openshell:resolve:env:v"; + const revisionSuffix = `_${envName}`; + return [ + `canonical=${shellQuote(canonical)}`, + `prefix=${shellQuote(revisionPrefix)}`, + `suffix=${shellQuote(revisionSuffix)}`, + "valid_placeholder() {", + ' candidate="$1"', + ' [ "$candidate" = "$canonical" ] && return 0', + ' versioned="${candidate#"$prefix"}"', + ' [ "$versioned" != "$candidate" ] || return 1', + ' revision="${versioned%"$suffix"}"', + ' [ "$revision" != "$versioned" ] || return 1', + ' [ "$versioned" = "$revision$suffix" ] || return 1', + ' case "$revision" in ""|*[!0-9]*) return 1 ;; esac', + ' [ "${#revision}" -le 20 ] || return 1', + "}", + ]; +} + +/** + * Emit only a bounded classification of the OpenShell placeholder observed by + * a fresh exec. Raw environment values are never written or printed. Keeping + * the observation on stdout lets the trusted host compare revisions without + * relying on sandbox-writable state. + */ +export function buildMcpCredentialRevisionObservationCommand(envName: string): string { + return [ + ...mcpCredentialPlaceholderValidatorShell(envName), + `if [ -z "\${${envName}+x}" ]; then`, + " printf '%s\\n' absent", + " exit 0", + "fi", + `value="\${${envName}}"`, + 'valid_placeholder "$value" || exit 1', + 'if [ "$value" = "$canonical" ]; then', + " printf '%s\\n' canonical", + " exit 0", + "fi", + 'versioned="${value#"$prefix"}"', + 'revision="${versioned%"$suffix"}"', + "printf 'v%s\\n' \"$revision\"", + ].join("\n"); +} + +function parseMcpCredentialRevisionObservation( + output: string, +): McpCredentialRevisionObservation | null { + const observation = output.trim(); + return MCP_CREDENTIAL_REVISION_OBSERVATION_RE.test(observation) + ? (observation as McpCredentialRevisionObservation) + : null; +} + +function tryObserveMcpCredentialRevision( + sandboxName: string, + envName: string, +): McpCredentialRevisionObservation | null { + const result = executeMcpCredentialProofCommand( + sandboxName, + buildMcpCredentialRevisionObservationCommand(envName), + ); + if (!result || result.status !== 0) return null; + return parseMcpCredentialRevisionObservation(result.stdout); +} + +export function observeMcpCredentialRevision( + sandboxName: string, + entry: McpBridgeEntry, +): McpCredentialRevisionObservation { + assertAuthenticatedBridgeEntry(entry); + const observation = tryObserveMcpCredentialRevision(sandboxName, entry.env[0]); + if (observation === null) { + throw new McpBridgeError( + `Could not observe the current OpenShell credential revision for sandbox '${sandboxName}'.`, + ); + } + return observation; +} + +export function waitForAttachedMcpCredential( + sandboxName: string, + entry: McpBridgeEntry, + options: { previousRevision?: McpCredentialRevisionObservation } = {}, +): void { + assertAuthenticatedBridgeEntry(entry); + const envName = entry.env[0]; + if ( + options.previousRevision !== undefined && + !MCP_CREDENTIAL_REVISION_OBSERVATION_RE.test(options.previousRevision) + ) { + throw new McpBridgeError("Invalid prior MCP credential revision observation."); + } + const timeoutSeconds = Number.parseInt( + process.env.NEMOCLAW_MCP_PROVIDER_SYNC_TIMEOUT_SECONDS ?? "30", + 10, + ); + const ready = waitUntil( + () => { + // Each exec is a fresh OpenShell process. Only the bounded placeholder + // classification crosses back to the host, where the comparison cannot + // be influenced by a same-UID sandbox process rewriting a snapshot file. + const observation = tryObserveMcpCredentialRevision(sandboxName, envName); + return ( + observation !== null && + observation !== "absent" && + (options.previousRevision === undefined || observation !== options.previousRevision) + ); + }, + Number.isFinite(timeoutSeconds) && timeoutSeconds > 0 ? timeoutSeconds : 30, + 1_000, + ); + if (!ready) { + throw new McpBridgeError( + `OpenShell did not synchronize the expected credential revision for placeholder '${envName}' into sandbox '${sandboxName}' after provider attachment or update.`, + ); + } +} + +export function buildMcpCredentialDetachedCommand(envName: string): string { + validateMcpCredentialEnvName(envName); + return `[ -z "\${${envName}+x}" ]`; +} + +export function waitForDetachedMcpCredential(sandboxName: string, entry: McpBridgeEntry): void { + assertPersistedAuthenticatedBridgeEntry(entry); + const envName = entry.env[0]; + try { + validateMcpCredentialEnvName(envName); + } catch { + // The exact provider attachment post-state was already checked by the + // detach operation. Do not start a fresh child under a legacy loader, + // shell, or compatibility env name merely to repeat that proof. + return; + } + const timeoutSeconds = Number.parseInt( + process.env.NEMOCLAW_MCP_PROVIDER_SYNC_TIMEOUT_SECONDS ?? "30", + 10, + ); + const revoked = waitUntil( + () => + executeMcpCredentialProofCommand(sandboxName, buildMcpCredentialDetachedCommand(envName)) + ?.status === 0, + Number.isFinite(timeoutSeconds) && timeoutSeconds > 0 ? timeoutSeconds : 30, + 1_000, + ); + if (!revoked) { + throw new McpBridgeError( + `OpenShell did not confirm credential '${envName}' was revoked from fresh execs in sandbox '${sandboxName}' after detach. Preserving MCP policy and ownership state.`, + ); + } +} diff --git a/src/lib/actions/sandbox/mcp-bridge-provider.test.ts b/src/lib/actions/sandbox/mcp-bridge-provider.test.ts new file mode 100644 index 00000000000..05be966a37e --- /dev/null +++ b/src/lib/actions/sandbox/mcp-bridge-provider.test.ts @@ -0,0 +1,270 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; + +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + buildMcpCredentialRevisionObservationCommand, + parseMcpProviderAttachmentNames, + parseMcpProviderMetadata, + providerDetachChangedState, +} from "./mcp-bridge"; +import { commandOutput } from "./mcp-bridge-output"; +import { + observeMcpCredentialRevision, + waitForAttachedMcpCredential, + waitForDetachedMcpCredential, +} from "./mcp-bridge-provider"; +import * as processRecovery from "./process-recovery"; + +function decodeMcpProofTransport(command: string): string { + const match = command.match(/printf '%s' '([A-Za-z0-9+/=]+)' \| base64 -d/); + return match?.[1] ? Buffer.from(match[1], "base64").toString("utf8") : ""; +} + +describe("OpenShell MCP provider state", () => { + afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + }); + + it("parses provider type and credential keys without values", () => { + expect( + parseMcpProviderMetadata(` +Provider: + + Id: 11111111-2222-4333-8444-555555555555 + Name: alpha-mcp-github + Type: generic + Resource version: 7 + Credential keys: GITHUB_TOKEN + Config keys: +`), + ).toEqual({ + id: "11111111-2222-4333-8444-555555555555", + resourceVersion: 7, + type: "generic", + credentialKeys: ["GITHUB_TOKEN"], + }); + expect(parseMcpProviderMetadata("Type: generic\nCredential keys: \n")).toEqual({ + id: null, + resourceVersion: null, + type: "generic", + credentialKeys: [], + }); + }); + + it("parses ANSI-decorated OpenShell provider metadata after redaction", () => { + const output = commandOutput({ + status: 0, + stdout: [ + "\u001b[2mProvider:\u001b[0m", + "\u001b[2m Id:\u001b[0m 11111111-2222-4333-8444-555555555555", + "\u001b[2m Type:\u001b[0m generic", + "\u001b[2m Resource version:\u001b[0m 7", + "\u001b[2m Credential keys:\u001b[0m GITHUB_TOKEN", + ].join("\n"), + stderr: "", + }); + + expect(parseMcpProviderMetadata(output)).toEqual({ + id: "11111111-2222-4333-8444-555555555555", + resourceVersion: 7, + type: "generic", + credentialKeys: ["GITHUB_TOKEN"], + }); + expect(output).not.toContain("\u001b"); + expect(output).not.toMatch(/\[[0-9;]*m/); + }); + + it("distinguishes a real detach from OpenShell's idempotent success", () => { + expect( + providerDetachChangedState(0, "✓ Detached provider alpha-mcp-github from sandbox alpha"), + ).toBe(true); + expect( + providerDetachChangedState(0, "Provider alpha-mcp-github was not attached to sandbox alpha."), + ).toBe(false); + }); + + it("parses the stock OpenShell sandbox provider table", () => { + expect( + parseMcpProviderAttachmentNames(` +NAME TYPE CREDENTIAL_KEYS CONFIG_KEYS +alpha-mcp-github generic 1 0 +alpha-mcp-slack generic 1 0 +`), + ).toEqual(["alpha-mcp-github", "alpha-mcp-slack"]); + expect(parseMcpProviderAttachmentNames("No providers attached to sandbox alpha.\n")).toEqual( + [], + ); + expect(() => parseMcpProviderAttachmentNames("unexpected output\n")).toThrow( + /attachment table header/, + ); + }); + + it("emits only bounded credential revision observations", () => { + const command = buildMcpCredentialRevisionObservationCommand("GITHUB_TOKEN"); + for (const [value, observation] of [ + [undefined, "absent"], + ["openshell:resolve:env:GITHUB_TOKEN", "canonical"], + ["openshell:resolve:env:v11_GITHUB_TOKEN", "v11"], + ["openshell:resolve:env:v0_GITHUB_TOKEN", "v0"], + ] as const) { + const result = spawnSync("/bin/sh", ["-c", command], { + encoding: "utf8", + env: value === undefined ? {} : { GITHUB_TOKEN: value }, + }); + expect(result.status, value).toBe(0); + expect(result.stdout.trim()).toBe(observation); + expect(result.stderr).toBe(""); + } + + for (const value of [ + "raw-secret", + "openshell:resolve:env:v_GITHUB_TOKEN", + "openshell:resolve:env:v11_OTHER_TOKEN", + "openshell:resolve:env:v11x_GITHUB_TOKEN", + `openshell:resolve:env:v${"1".repeat(21)}_GITHUB_TOKEN`, + ]) { + const result = spawnSync("/bin/sh", ["-c", command], { + encoding: "utf8", + env: { GITHUB_TOKEN: value }, + }); + expect(result.status, value).not.toBe(0); + expect(result.stdout).toBe(""); + expect(result.stderr).toBe(""); + } + expect(command).not.toMatch(/\/tmp|snapshot|cat\s|exec\s+[0-9]*>/); + }); + + it("uses an OpenShell-only exec for provider credential proofs", () => { + const exec = vi.spyOn(processRecovery, "executeSandboxExecCommand").mockReturnValue({ + status: 0, + stdout: "v11", + stderr: "", + }); + + expect( + observeMcpCredentialRevision("alpha", { + server: "github", + agent: "openclaw", + adapter: "mcporter", + url: "https://mcp.example.test/mcp", + env: ["GITHUB_TOKEN"], + providerName: "alpha-mcp-github-0123456789abcdef", + providerId: "11111111-2222-4333-8444-555555555555", + policyName: "mcp-bridge-github", + addedAt: "2026-06-01T00:00:00.000Z", + }), + ).toBe("v11"); + const proofCommand = exec.mock.calls[0]?.[1] ?? ""; + expect(proofCommand).not.toMatch(/[\r\n]/); + expect(proofCommand).toContain("base64 -d"); + expect(decodeMcpProofTransport(proofCommand)).toContain("GITHUB_TOKEN"); + expect(decodeMcpProofTransport(proofCommand)).not.toMatch(/\/tmp|snapshot/); + expect(exec).toHaveBeenCalledWith("alpha", proofCommand, undefined, { + allowLocalDockerFallback: false, + }); + const decodeFailure = spawnSync("/bin/sh", ["-c", proofCommand.replace("base64 -d", "false")]); + expect(decodeFailure.status).not.toBe(0); + + exec.mockReturnValue({ status: 0, stdout: "raw-secret", stderr: "" }); + expect(() => + observeMcpCredentialRevision("alpha", { + server: "github", + agent: "openclaw", + adapter: "mcporter", + url: "https://mcp.example.test/mcp", + env: ["GITHUB_TOKEN"], + providerName: "alpha-mcp-github-0123456789abcdef", + providerId: "11111111-2222-4333-8444-555555555555", + policyName: "mcp-bridge-github", + addedAt: "2026-06-01T00:00:00.000Z", + }), + ).toThrow(/Could not observe the current OpenShell credential revision/); + }); + + it("uses a newline-free OpenShell transport for attachment readiness", () => { + const exec = vi.spyOn(processRecovery, "executeSandboxExecCommand").mockReturnValue({ + status: 0, + stdout: "canonical", + stderr: "", + }); + + waitForAttachedMcpCredential("alpha", { + server: "github", + agent: "openclaw", + adapter: "mcporter", + url: "https://mcp.example.test/mcp", + env: ["GITHUB_TOKEN"], + providerName: "alpha-mcp-github-0123456789abcdef", + providerId: "11111111-2222-4333-8444-555555555555", + policyName: "mcp-bridge-github", + addedAt: "2026-06-01T00:00:00.000Z", + }); + + const proofCommand = exec.mock.calls[0]?.[1] ?? ""; + expect(proofCommand).not.toMatch(/[\r\n]/); + expect(decodeMcpProofTransport(proofCommand)).toContain("valid_placeholder"); + expect(decodeMcpProofTransport(proofCommand)).toContain("GITHUB_TOKEN"); + }); + + it("fails detach verification when the strict OpenShell exec is unavailable", () => { + vi.stubEnv("NEMOCLAW_MCP_PROVIDER_SYNC_TIMEOUT_SECONDS", "1"); + const exec = vi.spyOn(processRecovery, "executeSandboxExecCommand").mockReturnValue(null); + vi.spyOn(Date, "now").mockReturnValueOnce(0).mockReturnValueOnce(0).mockReturnValue(1_000); + + expect(() => + waitForDetachedMcpCredential("alpha", { + server: "github", + agent: "openclaw", + adapter: "mcporter", + url: "https://mcp.example.test/mcp", + env: ["GITHUB_TOKEN"], + providerName: "alpha-mcp-github-0123456789abcdef", + providerId: "11111111-2222-4333-8444-555555555555", + policyName: "mcp-bridge-github", + addedAt: "2026-06-01T00:00:00.000Z", + }), + ).toThrow(/did not confirm credential 'GITHUB_TOKEN' was revoked/); + + const proofCommand = exec.mock.calls[0]?.[1] ?? ""; + expect(proofCommand).not.toMatch(/[\r\n]/); + expect(decodeMcpProofTransport(proofCommand)).toContain("GITHUB_TOKEN+x"); + expect(exec).toHaveBeenCalledWith("alpha", proofCommand, undefined, { + allowLocalDockerFallback: false, + }); + }); + + it("requires a changed credential revision after provider updates", () => { + const entry = { + server: "github", + agent: "openclaw", + adapter: "mcporter", + url: "https://mcp.example.test/mcp", + env: ["GITHUB_TOKEN"], + providerName: "alpha-mcp-github-0123456789abcdef", + providerId: "11111111-2222-4333-8444-555555555555", + policyName: "mcp-bridge-github", + addedAt: "2026-06-01T00:00:00.000Z", + }; + const exec = vi.spyOn(processRecovery, "executeSandboxExecCommand").mockReturnValue({ + status: 0, + stdout: "v12", + stderr: "", + }); + + waitForAttachedMcpCredential("alpha", entry, { previousRevision: "v11" }); + expect(exec).toHaveBeenCalledTimes(1); + + vi.stubEnv("NEMOCLAW_MCP_PROVIDER_SYNC_TIMEOUT_SECONDS", "1"); + exec.mockClear(); + exec.mockReturnValue({ status: 0, stdout: "v11", stderr: "" }); + vi.spyOn(Date, "now").mockReturnValueOnce(0).mockReturnValueOnce(0).mockReturnValue(1_000); + expect(() => waitForAttachedMcpCredential("alpha", entry, { previousRevision: "v11" })).toThrow( + /did not synchronize the expected credential revision/, + ); + expect(exec).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/lib/actions/sandbox/mcp-bridge-provider.ts b/src/lib/actions/sandbox/mcp-bridge-provider.ts new file mode 100644 index 00000000000..7592a5fa890 --- /dev/null +++ b/src/lib/actions/sandbox/mcp-bridge-provider.ts @@ -0,0 +1,38 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export type { + McpProviderAttachment, + McpProviderAttachmentInspection, + McpProviderInspection, +} from "./mcp-bridge-provider-inspection"; +export { + assertMcpProviderRecoverable, + assertNoAttachedProviderCredentialCollision, + inspectMcpProvider, + inspectMcpProviderAttachments, + parseMcpProviderAttachmentNames, + parseMcpProviderMetadata, + preflightMcpEntryTargets, + providerAttached, + providerMatchesCredential, + providerShapeDetail, +} from "./mcp-bridge-provider-inspection"; +export type { ProviderDetachOutcome } from "./mcp-bridge-provider-mutation"; +export { + attachProvider, + buildMcpBridgeProviderArgs, + deleteProvider, + detachMissingProviderReference, + detachProvider, + providerDetachChangedState, + upsertMcpProvider, +} from "./mcp-bridge-provider-mutation"; +export type { McpCredentialRevisionObservation } from "./mcp-bridge-provider-readiness"; +export { + buildMcpCredentialDetachedCommand, + buildMcpCredentialRevisionObservationCommand, + observeMcpCredentialRevision, + waitForAttachedMcpCredential, + waitForDetachedMcpCredential, +} from "./mcp-bridge-provider-readiness"; diff --git a/src/lib/actions/sandbox/mcp-bridge-rebuild.ts b/src/lib/actions/sandbox/mcp-bridge-rebuild.ts new file mode 100644 index 00000000000..118417218d8 --- /dev/null +++ b/src/lib/actions/sandbox/mcp-bridge-rebuild.ts @@ -0,0 +1,265 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { McpBridgeEntry } from "../../state/registry"; +import { registerAgentAdapter, unregisterAgentAdapter } from "./mcp-bridge-adapters"; +import { isAgentMcpAdapter, McpBridgeError } from "./mcp-bridge-contracts"; +import { + cloneMcpBridgeEntry, + discardSafeIncompleteMcpAdds, + inspectExactMcpDestroyProvider, +} from "./mcp-bridge-destroy"; +import { + assertGeneratedPolicyMutationSafe, + assertGeneratedPolicyRegistrationMutationSafe, +} from "./mcp-bridge-policy"; +import { + assertMcpProviderRecoverable, + attachProvider, + detachProvider, + preflightMcpEntryTargets, + waitForAttachedMcpCredential, + waitForDetachedMcpCredential, +} from "./mcp-bridge-provider"; +import { restoreExistingMcpBridgeRuntime } from "./mcp-bridge-restart"; +import { + assertMcpAdapterConfigMutationsAllowed, + assertMcpAdapterTeardownRuntimeCapabilities, +} from "./mcp-bridge-runtime-capabilities"; +import { + assertMcpDestroyNotPending, + bridgeState, + ensureSandboxGatewaySelected, + getBridgeAdapter, + getSandboxAgent, + getSandboxOrThrow, + setBridgeState, +} from "./mcp-bridge-state"; +import { assertAuthenticatedBridgeEntry, validateSandboxName } from "./mcp-bridge-validation"; + +export interface McpRebuildPreparation { + entries: McpBridgeEntry[]; + detachedProviderEntries: McpBridgeEntry[]; + scrubbedAdapterEntries: McpBridgeEntry[]; +} + +async function getCompleteMcpRebuildEntries( + sandboxName: string, + options: { sandboxAbsent?: boolean } = {}, +): Promise { + validateSandboxName(sandboxName); + const currentSandbox = getSandboxOrThrow(sandboxName); + assertMcpDestroyNotPending(currentSandbox); + if (!options.sandboxAbsent) { + const entriesRequiringExternalCleanup = Object.values(bridgeState(currentSandbox)).filter( + (entry) => entry.addState !== "prepared", + ); + // This host-visible config preflight must precede + // discardSafeIncompleteMcpAdds, which can remove an owned policy for a + // providerless preflighted add. That cleanup has no adapter/provider to + // probe; complete entries get the teardown runtime probe below. + assertMcpAdapterConfigMutationsAllowed( + sandboxName, + currentSandbox, + entriesRequiringExternalCleanup, + ); + } + const sandbox = await discardSafeIncompleteMcpAdds(sandboxName, currentSandbox, options); + const entries = Object.values(bridgeState(sandbox)).map(cloneMcpBridgeEntry); + const incompleteAdd = entries.find((entry) => entry.addState); + if (incompleteAdd) { + throw new McpBridgeError( + `MCP server '${incompleteAdd.server}' has an incomplete add transaction (${incompleteAdd.addState}). Re-run the original mcp add command or remove it with --force before rebuilding the sandbox.`, + ); + } + return entries; +} + +/** + * Preserve MCP intent for stale-registry recovery after OpenShell has already + * proved the sandbox absent. There is no sandbox process or retained adapter + * to scrub, so this path validates targets and provider recoverability without + * attempting sandbox exec or changing provider attachment state. + */ +export async function prepareMcpBridgesForAbsentSandboxRebuild( + sandboxName: string, +): Promise { + const entries = await getCompleteMcpRebuildEntries(sandboxName, { sandboxAbsent: true }); + if (entries.length === 0) { + return { + entries: [], + detachedProviderEntries: [], + scrubbedAdapterEntries: [], + }; + } + await preflightMcpEntryTargets(entries); + await ensureSandboxGatewaySelected(sandboxName); + for (const entry of entries) { + assertGeneratedPolicyRegistrationMutationSafe(sandboxName, entry); + } + for (const entry of entries) assertMcpProviderRecoverable(entry); + return { + entries, + detachedProviderEntries: [], + scrubbedAdapterEntries: [], + }; +} + +export async function prepareMcpBridgesForRebuild( + sandboxName: string, +): Promise { + const sandbox = getSandboxOrThrow(sandboxName); + const entries = await getCompleteMcpRebuildEntries(sandboxName); + if (entries.length === 0) { + return { + entries: [], + detachedProviderEntries: [], + scrubbedAdapterEntries: [], + }; + } + await preflightMcpEntryTargets(entries); + await ensureSandboxGatewaySelected(sandboxName); + for (const entry of entries) assertGeneratedPolicyMutationSafe(sandboxName, entry); + assertMcpAdapterTeardownRuntimeCapabilities(sandboxName, sandbox, entries); + for (const entry of entries) assertMcpProviderRecoverable(entry); + const detached: McpBridgeEntry[] = []; + const scrubbedAdapters: McpBridgeEntry[] = []; + try { + for (const entry of entries) { + const adapter = isAgentMcpAdapter(entry.adapter) + ? entry.adapter + : getBridgeAdapter(getSandboxAgent(sandbox)); + // `/sandbox` may be a retained PVC. Scrub before delete so a replacement + // Hermes/agent cannot boot with a stale placeholder while its provider + // is intentionally detached during recreate. + unregisterAgentAdapter(sandboxName, adapter, entry, { envValues: {} }); + scrubbedAdapters.push(entry); + } + for (const entry of entries) { + // Keep the provider and its host-only credentials for the replacement + // sandbox, but detach it before OpenShell deletes the old attachment. + inspectExactMcpDestroyProvider(entry, { allowMissing: false }); + const detachOutcome = detachProvider(sandboxName, entry); + if (detachOutcome === "unknown") { + throw new McpBridgeError( + `Could not prove provider detach for MCP server '${entry.server}'.`, + ); + } + waitForDetachedMcpCredential(sandboxName, entry); + // A binding already absent on retry was still detached by this rebuild + // transaction (possibly before a prior process died), so it must be + // reattached if sandbox deletion later aborts. + detached.push(entry); + } + } catch (error) { + const rollbackFailures: string[] = []; + for (const entry of detached.reverse()) { + try { + inspectExactMcpDestroyProvider(entry, { allowMissing: false }); + attachProvider(sandboxName, entry); + // Reattach preserves the provider value, so presence is sufficient; + // still wait before reloading an adapter that may connect immediately. + waitForAttachedMcpCredential(sandboxName, entry); + } catch (rollbackError) { + rollbackFailures.push( + rollbackError instanceof Error ? rollbackError.message : String(rollbackError), + ); + } + } + for (const entry of scrubbedAdapters) { + try { + const adapter = isAgentMcpAdapter(entry.adapter) + ? entry.adapter + : getBridgeAdapter(getSandboxAgent(sandbox)); + registerAgentAdapter( + sandboxName, + adapter, + entry, + {}, + { + replaceExisting: true, + }, + ); + } catch (rollbackError) { + rollbackFailures.push( + rollbackError instanceof Error ? rollbackError.message : String(rollbackError), + ); + } + } + const detail = error instanceof Error ? error.message : String(error); + throw new McpBridgeError( + rollbackFailures.length > 0 + ? `${detail}\nMCP rebuild rollback could not reattach: ${rollbackFailures.join("; ")}` + : detail, + ); + } + return { + entries, + detachedProviderEntries: detached, + scrubbedAdapterEntries: scrubbedAdapters, + }; +} + +export async function reattachMcpProvidersAfterRebuildAbort( + sandboxName: string, + entries: readonly McpBridgeEntry[], + scrubbedAdapterEntries: readonly McpBridgeEntry[] = [], +): Promise { + if (entries.length === 0 && scrubbedAdapterEntries.length === 0) return; + await ensureSandboxGatewaySelected(sandboxName); + const sandbox = getSandboxOrThrow(sandboxName); + assertMcpAdapterTeardownRuntimeCapabilities(sandboxName, sandbox, [ + ...entries, + ...scrubbedAdapterEntries, + ]); + + const failures: string[] = []; + for (const entry of entries) { + try { + // Rebuild abort helpers are exported and may run after a long sandbox + // delete attempt; re-prove the immutable provider identity immediately + // before reattaching by its mutable name. + assertMcpProviderRecoverable(entry); + attachProvider(sandboxName, entry); + waitForAttachedMcpCredential(sandboxName, entry); + } catch (error) { + failures.push(error instanceof Error ? error.message : String(error)); + } + } + for (const entry of scrubbedAdapterEntries) { + try { + const adapter = isAgentMcpAdapter(entry.adapter) + ? entry.adapter + : getBridgeAdapter(getSandboxAgent(sandbox)); + registerAgentAdapter( + sandboxName, + adapter, + entry, + {}, + { + replaceExisting: true, + }, + ); + } catch (error) { + failures.push(error instanceof Error ? error.message : String(error)); + } + } + if (failures.length > 0) { + throw new McpBridgeError(failures.join("; ")); + } +} + +export async function restoreMcpBridgesAfterRebuild( + sandboxName: string, + entries: readonly McpBridgeEntry[], +): Promise { + if (entries.length === 0) return; + for (const entry of entries) assertAuthenticatedBridgeEntry(entry); + const bridges = Object.fromEntries( + entries.map((entry) => [entry.server, { ...entry, env: [...entry.env] }]), + ); + // Persist the recovery contract before touching the gateway. If refresh + // fails, `mcp restart` remains retryable after the operator fixes the cause. + setBridgeState(sandboxName, bridges); + await restoreExistingMcpBridgeRuntime(sandboxName, entries); +} diff --git a/src/lib/actions/sandbox/mcp-bridge-remove.ts b/src/lib/actions/sandbox/mcp-bridge-remove.ts new file mode 100644 index 00000000000..e28a473b4a9 --- /dev/null +++ b/src/lib/actions/sandbox/mcp-bridge-remove.ts @@ -0,0 +1,321 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { AgentMcpAdapter } from "../../agent/defs"; +import { withMcpLifecycleLock } from "../../state/mcp-lifecycle-lock"; +import type { McpBridgeEntry } from "../../state/registry"; +import { + assertAgentMcpConfigMutationAllowed, + assertAgentMcpTeardownRuntimeCapability, + unregisterAgentAdapter, +} from "./mcp-bridge-adapters"; +import { isAgentMcpAdapter, McpBridgeError } from "./mcp-bridge-contracts"; +import { assertGeneratedPolicyMutationSafe, removeGeneratedPolicy } from "./mcp-bridge-policy"; +import { + deleteProvider, + detachMissingProviderReference, + detachProvider, + inspectMcpProvider, + providerMatchesCredential, + providerShapeDetail, + waitForDetachedMcpCredential, +} from "./mcp-bridge-provider"; +import { + assertMcpDestroyNotPending, + bridgeState, + ensureSandboxGatewaySelected, + getBridgeAdapter, + getSandboxAgent, + getSandboxOrThrow, + removeBridgeEntry, +} from "./mcp-bridge-state"; +import { + assertAuthenticatedBridgeEntry, + assertPersistedAuthenticatedBridgeEntry, + resolvePersistedCredentialEnvForRedaction, + validateMcpServerName, + validateSandboxName, +} from "./mcp-bridge-validation"; + +function requiresProviderDetachBeforeAdapterCleanup(entry: McpBridgeEntry): boolean { + assertPersistedAuthenticatedBridgeEntry(entry); + try { + assertAuthenticatedBridgeEntry(entry); + return false; + } catch { + // Older durable entries can contain names that current builds reject + // because OpenShell exposes or interprets them in every fresh child. Such + // a provider must be detached before any adapter capability or mutation + // command is allowed to start inside the sandbox. + return true; + } +} + +function assertExactMcpRemoveProvider( + entry: McpBridgeEntry, + options: { allowMissing: boolean; force?: boolean }, +): void { + assertPersistedAuthenticatedBridgeEntry(entry); + if (!entry.providerId) { + throw new McpBridgeError( + `MCP server '${entry.server}' has no stable OpenShell provider ID. Refusing destructive cleanup of same-name provider '${entry.providerName}'. Remove the legacy bridge with --force only after independently cleaning that provider.`, + ); + } + const inspection = inspectMcpProvider(entry.providerName); + if (inspection.exists === null) { + throw new McpBridgeError( + inspection.error ?? `Could not inspect OpenShell provider '${entry.providerName}'.`, + ); + } + if (!inspection.exists) { + if (options.allowMissing) return; + throw new McpBridgeError( + `OpenShell provider '${entry.providerName}' is missing. Refusing to destroy sandbox state because a failed sandbox delete could not restore authenticated MCP without the preserved provider credential.`, + ); + } + if (!providerMatchesCredential(inspection, entry.env[0], entry.providerId)) { + const forceDetail = options.force + ? " --force does not delete a non-matching global provider because it may be owned by another workflow." + : ""; + throw new McpBridgeError( + `OpenShell provider '${entry.providerName}' no longer exactly matches MCP server '${entry.server}'. ${providerShapeDetail(inspection, entry.env[0], entry.providerId)}${forceDetail}`, + ); + } +} + +export async function removeMcpBridge( + sandboxName: string, + server: string, + options: { force?: boolean; allowResidual?: boolean } = {}, +): Promise { + return withMcpLifecycleLock(sandboxName, () => + removeMcpBridgeUnlocked(sandboxName, server, options), + ); +} + +async function removeMcpBridgeUnlocked( + sandboxName: string, + server: string, + options: { force?: boolean; allowResidual?: boolean } = {}, +): Promise { + validateSandboxName(sandboxName); + validateMcpServerName(server); + const sandbox = getSandboxOrThrow(sandboxName); + assertMcpDestroyNotPending(sandbox); + const entry = bridgeState(sandbox)[server]; + if (!entry) { + if (!options.force) { + throw new McpBridgeError(`MCP server '${server}' not found on sandbox '${sandboxName}'.`); + } + console.log(` No MCP server '${server}' is registered on sandbox '${sandboxName}'.`); + return; + } + if (entry.addState === "prepared") { + // `prepared` is persisted before gateway selection and is advanced only + // after adapter/provider/policy absence has been proven. It therefore owns + // no external resources and can be cancelled without touching same-name + // state another workflow may own. + removeBridgeEntry(sandboxName, server); + console.log(` Cancelled incomplete MCP add for '${server}' on sandbox '${sandboxName}'.`); + return; + } + // Cleanup follows the adapter persisted with the bridge. Requiring the + // sandbox's current agent to still advertise MCP support would strand old + // resources after an agent/capability migration. + const adapter = isAgentMcpAdapter(entry.adapter) + ? entry.adapter + : getBridgeAdapter(getSandboxAgent(sandbox)); + const detachBeforeAdapterCleanup = entry.providerName + ? requiresProviderDetachBeforeAdapterCleanup(entry) + : false; + // Teardown must remain available for a backward-compatible Deep Agents MCP + // entry on an image that predates the managed launcher marker. Hermes still + // performs its host-side shields preflight here, before any provider, policy, + // attachment, or adapter side effect. + assertAgentMcpConfigMutationAllowed(sandboxName, adapter); + await ensureSandboxGatewaySelected(sandboxName); + assertGeneratedPolicyMutationSafe(sandboxName, entry); + const failures: string[] = []; + let providerOwnershipProved = !entry.providerName; + let providerWasMissing = false; + if (entry.providerName) { + if (!entry.providerId) { + const inspection = inspectMcpProvider(entry.providerName); + if (inspection.exists === false) { + // With no live provider there is no global object to adopt or destroy. + // This lets an operator independently remove a legacy/orphan provider, + // then use MCP remove to clear only the exact adapter/policy manifest. + providerOwnershipProved = true; + providerWasMissing = true; + } else { + const detail = + inspection.exists === null + ? (inspection.error ?? `Could not inspect OpenShell provider '${entry.providerName}'.`) + : `MCP server '${entry.server}' has no stable OpenShell provider ID. Refusing to detach or delete same-name provider '${entry.providerName}'.`; + if (!options.force) throw new McpBridgeError(detail); + failures.push(detail); + } + } else { + const inspection = inspectMcpProvider(entry.providerName); + if (inspection.exists === false) { + providerOwnershipProved = true; + providerWasMissing = true; + } else if ( + inspection.exists === true && + entry.env.length === 1 && + providerMatchesCredential(inspection, entry.env[0], entry.providerId) + ) { + providerOwnershipProved = true; + } else { + const detail = + inspection.exists === null + ? (inspection.error ?? `Could not inspect OpenShell provider '${entry.providerName}'.`) + : `OpenShell provider '${entry.providerName}' has drifted or lacks a complete registered credential binding. ${providerShapeDetail(inspection, entry.env[0], entry.providerId) ?? ""}`; + if (!options.force) { + throw new McpBridgeError(detail); + } + // Force is allowed to continue cleaning resources whose ownership is + // independently provable, but it never broadens ownership of a global + // provider merely because the local bridge registry names it. + failures.push(detail); + } + } + } + + let missingProviderReferenceDetached = false; + if (providerWasMissing && providerOwnershipProved && entry.providerName) { + try { + detachMissingProviderReference(sandboxName, entry); + missingProviderReferenceDetached = true; + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + if (!options.force) throw new McpBridgeError(detail); + failures.push(detail); + } + } + + let providerDetachedBeforeAdapterCleanup = false; + if (detachBeforeAdapterCleanup && providerOwnershipProved && entry.providerName) { + try { + const detachOutcome = providerWasMissing + ? missingProviderReferenceDetached + ? "detached" + : "unknown" + : detachProvider(sandboxName, entry); + providerDetachedBeforeAdapterCleanup = detachOutcome !== "unknown"; + if (!providerDetachedBeforeAdapterCleanup) { + throw new McpBridgeError( + `Provider detach state for '${entry.providerName}' is unknown; refusing to start an adapter child while legacy credential '${entry.env[0]}' may still be attached.`, + ); + } + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + if (!options.force) throw new McpBridgeError(detail); + failures.push(detail); + } + } + + // A dangling provider name can prevent fresh sandbox execs on OpenShell + // main, so clear that host-side spec reference before mutating the in-sandbox + // adapter. + const adapterEnvValues = resolvePersistedCredentialEnvForRedaction(entry.env); + let adapterCleanupProved = !detachBeforeAdapterCleanup || providerDetachedBeforeAdapterCleanup; + if (adapterCleanupProved) { + try { + // For a legacy unsafe credential, the exact provider reference was + // necessarily detached above before this first sandbox child. Otherwise + // this probe precedes every provider/policy/adapter side effect. Hermes + // retains its helper/lifecycle validation; Deep Agents intentionally + // skips only the marker that an older image cannot expose. + assertAgentMcpTeardownRuntimeCapability(sandboxName, adapter); + unregisterAgentAdapter( + sandboxName, + (entry.adapter as AgentMcpAdapter | undefined) ?? adapter, + entry, + { force: options.force === true, envValues: adapterEnvValues }, + ); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + if (!options.force) throw new McpBridgeError(detail); + adapterCleanupProved = false; + failures.push(detail); + } + } + let reservationCleanupProved = !entry.providerName && adapterCleanupProved; + if (adapterCleanupProved && providerOwnershipProved && entry.providerName) { + try { + // OpenShell main cannot list a sandbox whose spec references a missing + // provider. Remove that dangling name directly before using the normal + // table-backed detach path for a provider that still exists. + const detachOutcome = providerWasMissing + ? missingProviderReferenceDetached + ? "detached" + : "unknown" + : providerDetachedBeforeAdapterCleanup + ? "detached" + : detachProvider(sandboxName, entry); + if (detachOutcome !== "unknown") { + // A missing provider has no credential left to revoke. Its stock CLI + // detach result is authoritative for the sandbox-spec reference, and + // skipping a fresh-exec probe lets cleanup proceed even if another + // unrelated provider reference is also dangling. + if (!providerWasMissing && !providerDetachedBeforeAdapterCleanup) { + waitForDetachedMcpCredential(sandboxName, entry); + } + reservationCleanupProved = true; + } + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + if (!options.force) throw new McpBridgeError(detail); + failures.push(detail); + } + } + if (reservationCleanupProved) { + try { + removeGeneratedPolicy(sandboxName, entry); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + if (!options.force) throw new McpBridgeError(detail); + failures.push(detail); + } + } else { + failures.push( + `Provider detach state for '${entry.providerName}' is unknown; preserved the MCP policy and ownership manifest.`, + ); + } + if ( + reservationCleanupProved && + providerOwnershipProved && + !providerWasMissing && + entry.providerName + ) { + try { + // Recheck immediately before the mutable-name delete to narrow the + // replacement window. OpenShell main does not expose an atomic + // identity-conditioned delete, so concurrent direct provider mutation + // remains outside this lifecycle command's safety boundary. + assertExactMcpRemoveProvider(entry, { + allowMissing: false, + force: options.force, + }); + deleteProvider(entry, { + allowMissing: options.force === true || entry.addState === "preflighted", + }); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + if (!options.force) throw new McpBridgeError(detail); + failures.push(detail); + } + } + if (failures.length > 0) { + console.warn(` MCP force cleanup warnings:\n${failures.join("\n")}`); + if (!options.allowResidual) { + throw new McpBridgeError( + `MCP force cleanup left residual resources for '${server}'. The registry entry was preserved so cleanup can be retried.`, + ); + } + return; + } + removeBridgeEntry(sandboxName, server); + console.log(` Removed MCP server '${server}' from sandbox '${sandboxName}'.`); +} diff --git a/src/lib/actions/sandbox/mcp-bridge-render.ts b/src/lib/actions/sandbox/mcp-bridge-render.ts new file mode 100644 index 00000000000..e7bc2d11b02 --- /dev/null +++ b/src/lib/actions/sandbox/mcp-bridge-render.ts @@ -0,0 +1,84 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { AgentDefinition } from "../../agent/defs"; +import type { McpBridgeStatus } from "./mcp-bridge-contracts"; + +export function renderMcpBridgeList( + sandboxName: string, + statuses: McpBridgeStatus[], + agent: AgentDefinition, +): void { + console.log(""); + if (agent.mcpCapability.support !== "bridge") { + console.log(` MCP support: disabled for ${agent.displayName}`); + if (agent.mcpCapability.reason) console.log(` ${agent.mcpCapability.reason}`); + } + if (statuses.length === 0) { + console.log(` No MCP servers for sandbox '${sandboxName}'.`); + console.log(""); + return; + } + console.log(` MCP servers for sandbox '${sandboxName}':`); + for (const status of statuses) { + const policy = status.policy.gatewayPresent ? "policy" : "policy?"; + const provider = + status.provider.registryPresent && + status.provider.gatewayPresent && + status.provider.attached === true && + status.provider.credentialReady === true + ? "provider" + : "provider?"; + const env = status.env.names.length > 0 ? status.env.names.join(", ") : "(none)"; + console.log( + ` ${status.server.padEnd(18)} ${policy.padEnd(8)} ${provider.padEnd(10)} env: ${env}${status.addState ? ` add:${status.addState}` : ""}`, + ); + } + console.log(""); +} + +export function renderMcpBridgeStatus( + sandboxName: string, + statuses: McpBridgeStatus[], + agent: AgentDefinition, +): void { + if (statuses.length === 0) { + console.log(""); + console.log(` MCP servers for sandbox '${sandboxName}': none`); + console.log(` agent: ${agent.name}`); + console.log(` support: ${agent.mcpCapability.support}`); + if (agent.mcpCapability.reason) console.log(` reason: ${agent.mcpCapability.reason}`); + console.log(""); + return; + } + for (const status of statuses) { + console.log(""); + console.log(` MCP server: ${status.server}`); + console.log(` agent: ${status.agent}`); + console.log(` support: ${status.support.mode}`); + if (status.support.reason) console.log(` reason: ${status.support.reason}`); + if (status.url) console.log(` endpoint: ${status.url}`); + if (status.addState) console.log(` add transaction: incomplete (${status.addState})`); + console.log( + ` provider: ${status.provider.registryPresent ? status.provider.name : "(none)"}`, + ); + console.log( + ` provider attached: ${status.provider.attached === null ? "unknown" : status.provider.attached ? "yes" : "no"}`, + ); + console.log( + ` provider credentials: ${status.provider.credentialReady === null ? "unknown" : status.provider.credentialReady ? "ready" : "drifted or missing"}`, + ); + if (status.provider.detail) console.log(` provider detail: ${status.provider.detail}`); + console.log( + ` policy: ${status.policy.gatewayPresent === null ? "unknown" : status.policy.gatewayPresent ? "present" : "missing"}`, + ); + console.log( + ` adapter: ${status.adapter.registered === null ? "unknown" : status.adapter.registered ? "registered" : "missing"}`, + ); + console.log( + ` env: ${status.env.ready ? "ready" : status.env.missing.length > 0 ? `missing ${status.env.missing.join(", ")}` : "not ready"}`, + ); + for (const warning of status.warnings) console.log(` warning: ${warning}`); + } + console.log(""); +} diff --git a/src/lib/actions/sandbox/mcp-bridge-restart.ts b/src/lib/actions/sandbox/mcp-bridge-restart.ts new file mode 100644 index 00000000000..24a41b38c84 --- /dev/null +++ b/src/lib/actions/sandbox/mcp-bridge-restart.ts @@ -0,0 +1,215 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { AgentMcpAdapter } from "../../agent/defs"; +import { withMcpLifecycleLock } from "../../state/mcp-lifecycle-lock"; +import type { McpBridgeEntry } from "../../state/registry"; +import { registerAgentAdapter } from "./mcp-bridge-adapters"; +import { McpBridgeError } from "./mcp-bridge-contracts"; +import { applyGeneratedPolicy, assertGeneratedPolicyMutationSafe } from "./mcp-bridge-policy"; +import { + assertMcpProviderRecoverable, + assertNoAttachedProviderCredentialCollision, + attachProvider, + detachMissingProviderReference, + type McpCredentialRevisionObservation, + type McpProviderInspection, + observeMcpCredentialRevision, + preflightMcpEntryTargets, + upsertMcpProvider, + waitForAttachedMcpCredential, + waitForDetachedMcpCredential, +} from "./mcp-bridge-provider"; +import { + assertMcpAdapterConfigMutationsAllowed, + assertMcpAdapterMutationRuntimeCapabilities, + assertMcpAdapterTeardownRuntimeCapabilities, +} from "./mcp-bridge-runtime-capabilities"; +import { + assertMcpDestroyNotPending, + bridgeState, + ensureSandboxGatewaySelected, + getBridgeAdapter, + getSandboxAgent, + getSandboxOrThrow, + nowIso, + writeBridgeEntry, +} from "./mcp-bridge-state"; +import { + assertAuthenticatedBridgeEntry, + resolveCredentialEnv, + validateSandboxName, +} from "./mcp-bridge-validation"; + +function resolvedTargetPins( + resolvedByServer: ReadonlyMap, + entry: McpBridgeEntry, +): string[] { + const addresses = resolvedByServer.get(entry.server); + if (!addresses || addresses.length === 0) { + throw new McpBridgeError( + `MCP server '${entry.server}' has no validated public address pins. Refusing policy mutation.`, + ); + } + return addresses; +} + +export async function restartMcpBridge(sandboxName: string, server?: string): Promise { + return withMcpLifecycleLock(sandboxName, () => restartMcpBridgeUnlocked(sandboxName, server)); +} + +async function restartMcpBridgeUnlocked(sandboxName: string, server?: string): Promise { + validateSandboxName(sandboxName); + const sandbox = getSandboxOrThrow(sandboxName); + assertMcpDestroyNotPending(sandbox); + const agent = getSandboxAgent(sandbox); + const adapter = getBridgeAdapter(agent); + const bridges = bridgeState(sandbox); + const targets = server ? [[server, bridges[server]] as const] : Object.entries(bridges); + if (targets.length === 0) { + console.log(` No MCP servers for sandbox '${sandboxName}'.`); + return; + } + for (const [name, entry] of targets) { + if (!entry) { + throw new McpBridgeError(`MCP server '${name}' not found on sandbox '${sandboxName}'.`); + } + if (entry.addState) { + throw new McpBridgeError( + `MCP server '${name}' has an incomplete add transaction (${entry.addState}). Re-run mcp add with the same URL and --env ${entry.env[0] ?? "KEY"}, or remove it with --force.`, + ); + } + assertAuthenticatedBridgeEntry(entry); + } + const targetEntries = targets + .map(([, entry]) => entry) + .filter((entry): entry is McpBridgeEntry => !!entry); + // Hermes shields posture is host-visible. Refuse before DNS, gateway + // recovery/selection, provider inspection, or any lifecycle mutation. + assertMcpAdapterConfigMutationsAllowed(sandboxName, sandbox, targetEntries); + const resolvedByServer = await preflightMcpEntryTargets(targetEntries); + await ensureSandboxGatewaySelected(sandboxName); + // Prove every policy key is absent or still matches its recorded ownership + // before inspecting or updating any provider. `applyGeneratedPolicy` repeats + // this check immediately before mutation to close the preflight-to-apply race. + for (const entry of targetEntries) assertGeneratedPolicyMutationSafe(sandboxName, entry); + const providerInspectionByServer = new Map(); + for (const entry of targetEntries) { + providerInspectionByServer.set(entry.server, assertMcpProviderRecoverable(entry)); + } + const missingProviderEntries = targetEntries.filter( + (entry) => providerInspectionByServer.get(entry.server)?.exists === false, + ); + // Detach every dangling name before asking the supervisor for a fresh exec. + // Provider environment resolution can remain blocked while any missing name + // is still present in the sandbox spec. These references name providers + // already proven absent; no live credential is removed before the runtime + // capability probe, and the durable bridge manifest is retained on failure. + for (const entry of missingProviderEntries) { + detachMissingProviderReference(sandboxName, entry); + } + assertMcpAdapterMutationRuntimeCapabilities(sandboxName, sandbox, targetEntries); + for (const entry of missingProviderEntries) { + waitForDetachedMcpCredential(sandboxName, entry); + } + for (const [name, storedEntry] of targets) { + // Validated as a complete authenticated entry before gateway side effects. + if (!storedEntry) continue; + let entry = storedEntry; + const envRefs = entry.env.map((envName) => ({ name: envName })); + const adapterEnvValues = resolveCredentialEnv(envRefs); + const resolvedAddresses = resolvedTargetPins(resolvedByServer, entry); + let previousCredentialRevision: McpCredentialRevisionObservation | undefined; + assertNoAttachedProviderCredentialCollision(sandboxName, entry); + // Revalidate the actual running supervisor before rotating, recreating, + // attaching, or re-registering an authenticated provider. + applyGeneratedPolicy(sandboxName, entry, resolvedAddresses); + const providerResult = upsertMcpProvider(entry.providerName ?? "", envRefs, { + allowExisting: true, + expectedProviderId: entry.providerId, + prepareMutation: (action) => { + if (action === "update") { + previousCredentialRevision = observeMcpCredentialRevision(sandboxName, entry); + } + }, + }); + const providerId = providerResult.inspection.id; + if (!providerId) { + throw new McpBridgeError( + `OpenShell did not return a stable provider ID for '${entry.providerName}'. Refusing later MCP side effects.`, + ); + } + const refreshedEntry = + providerId === entry.providerId ? entry : { ...entry, providerId, updatedAt: nowIso() }; + if (refreshedEntry !== entry) { + // A missing owned provider may be recreated during restart. Record the + // replacement object's immutable ID before policy/attach/adapter work. + writeBridgeEntry(sandboxName, refreshedEntry); + entry = refreshedEntry; + } + assertNoAttachedProviderCredentialCollision(sandboxName, entry); + if (providerResult.action === "updated" && previousCredentialRevision === undefined) { + throw new McpBridgeError( + `Could not retain the prior OpenShell credential revision for provider '${entry.providerName}'.`, + ); + } + attachProvider(sandboxName, entry); + waitForAttachedMcpCredential(sandboxName, entry, { + ...(providerResult.action === "updated" + ? { previousRevision: previousCredentialRevision } + : {}), + }); + registerAgentAdapter( + sandboxName, + (entry.adapter as AgentMcpAdapter | undefined) ?? adapter, + entry, + adapterEnvValues, + { replaceExisting: true }, + ); + writeBridgeEntry(sandboxName, { + ...entry, + adapter: (entry.adapter as AgentMcpAdapter | undefined) ?? adapter, + updatedAt: nowIso(), + }); + console.log(` Refreshed MCP server '${name}'.`); + } +} + +export async function restoreExistingMcpBridgeRuntime( + sandboxName: string, + entries: readonly McpBridgeEntry[], + options: { lifecyclePhase?: "active-mutation" | "teardown-rollback" } = {}, +): Promise { + if (entries.length === 0) return; + for (const entry of entries) assertAuthenticatedBridgeEntry(entry); + const resolvedByServer = await preflightMcpEntryTargets(entries); + await ensureSandboxGatewaySelected(sandboxName); + const sandbox = getSandboxOrThrow(sandboxName); + assertMcpDestroyNotPending(sandbox); + if (options.lifecyclePhase === "teardown-rollback") { + // A failed delete/rebuild must be able to restore a backward-compatible + // Deep Agents entry on the same old image it just scrubbed. New/rebuilt + // images use the default path and must prove the current marker before any + // policy, provider, attachment, or adapter mutation. + assertMcpAdapterTeardownRuntimeCapabilities(sandboxName, sandbox, entries); + } else { + assertMcpAdapterMutationRuntimeCapabilities(sandboxName, sandbox, entries); + } + for (const entry of entries) { + assertGeneratedPolicyMutationSafe(sandboxName, entry); + const provider = assertMcpProviderRecoverable(entry); + if (provider.exists !== true) { + throw new McpBridgeError( + `OpenShell provider '${entry.providerName}' is missing. Runtime restoration refuses to create or rotate credentials; run explicit MCP restart after exporting '${entry.env[0]}'.`, + ); + } + assertNoAttachedProviderCredentialCollision(sandboxName, entry); + applyGeneratedPolicy(sandboxName, entry, resolvedTargetPins(resolvedByServer, entry)); + attachProvider(sandboxName, entry); + waitForAttachedMcpCredential(sandboxName, entry); + const adapter = + (entry.adapter as AgentMcpAdapter | undefined) ?? getBridgeAdapter(getSandboxAgent(sandbox)); + registerAgentAdapter(sandboxName, adapter, entry, {}, { replaceExisting: true }); + writeBridgeEntry(sandboxName, { ...entry, adapter, updatedAt: nowIso() }); + } +} diff --git a/src/lib/actions/sandbox/mcp-bridge-runtime-capabilities.ts b/src/lib/actions/sandbox/mcp-bridge-runtime-capabilities.ts new file mode 100644 index 00000000000..aa73e878be4 --- /dev/null +++ b/src/lib/actions/sandbox/mcp-bridge-runtime-capabilities.ts @@ -0,0 +1,59 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { AgentMcpAdapter } from "../../agent/defs"; +import type { McpBridgeEntry, SandboxEntry } from "../../state/registry"; +import { + assertAgentMcpConfigMutationAllowed, + assertAgentMcpMutationRuntimeCapability, + assertAgentMcpTeardownRuntimeCapability, +} from "./mcp-bridge-adapters"; +import { isAgentMcpAdapter } from "./mcp-bridge-contracts"; +import { getBridgeAdapter, getSandboxAgent } from "./mcp-bridge-state"; + +function adaptersForEntries( + sandbox: SandboxEntry, + entries: readonly McpBridgeEntry[], +): Set { + return new Set( + entries.map((entry) => + isAgentMcpAdapter(entry.adapter) ? entry.adapter : getBridgeAdapter(getSandboxAgent(sandbox)), + ), + ); +} + +export function assertMcpAdapterMutationRuntimeCapabilities( + sandboxName: string, + sandbox: SandboxEntry, + entries: readonly McpBridgeEntry[], +): void { + for (const adapter of adaptersForEntries(sandbox, entries)) { + assertAgentMcpMutationRuntimeCapability(sandboxName, adapter); + } +} + +/** + * Prove host-visible config mutability without requiring a capability marker + * from the image being torn down. Deep Agents entries created by an older + * NemoClaw release remain safe to scrub because their exact persisted adapter + * definition is still ownership-checked by unregisterAgentAdapter. + */ +export function assertMcpAdapterConfigMutationsAllowed( + sandboxName: string, + sandbox: SandboxEntry, + entries: readonly McpBridgeEntry[], +): void { + for (const adapter of adaptersForEntries(sandbox, entries)) { + assertAgentMcpConfigMutationAllowed(sandboxName, adapter); + } +} + +export function assertMcpAdapterTeardownRuntimeCapabilities( + sandboxName: string, + sandbox: SandboxEntry, + entries: readonly McpBridgeEntry[], +): void { + for (const adapter of adaptersForEntries(sandbox, entries)) { + assertAgentMcpTeardownRuntimeCapability(sandboxName, adapter); + } +} diff --git a/src/lib/actions/sandbox/mcp-bridge-state.ts b/src/lib/actions/sandbox/mcp-bridge-state.ts new file mode 100644 index 00000000000..e1c0963ae98 --- /dev/null +++ b/src/lib/actions/sandbox/mcp-bridge-state.ts @@ -0,0 +1,158 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { type AgentDefinition, type AgentMcpAdapter, loadAgent } from "../../agent/defs"; +import { recoverNamedGatewayRuntime } from "../../gateway-runtime-action"; +import type { McpBridgeEntry, SandboxEntry } from "../../state/registry"; +import * as registry from "../../state/registry"; +import { getSandboxTargetGatewayName } from "./gateway-target"; +import { + isAgentMcpAdapter, + MCP_BRIDGE_POLICY_SOURCE, + McpBridgeError, +} from "./mcp-bridge-contracts"; + +export function nowIso(): string { + return new Date().toISOString(); +} + +export function getSandboxOrThrow(sandboxName: string): SandboxEntry { + const sandbox = registry.getSandbox(sandboxName); + if (!sandbox) { + throw new McpBridgeError(`Sandbox '${sandboxName}' not found.`, 1); + } + return sandbox; +} + +function getSandboxAgentName(sandbox: SandboxEntry): string { + return sandbox.agent || "openclaw"; +} + +export function getSandboxAgent(sandbox: SandboxEntry): AgentDefinition { + return loadAgent(getSandboxAgentName(sandbox)); +} + +function unsupportedMessage(agent: AgentDefinition): string { + const reason = agent.mcpCapability.reason + ? ` ${agent.mcpCapability.reason}` + : " MCP support is disabled for this agent."; + return `${agent.displayName} does not support managed MCP servers yet.${reason} Issue #566 tracks future design.`; +} + +function assertBridgeSupported(agent: AgentDefinition): void { + if (agent.mcpCapability.support === "bridge") return; + throw new McpBridgeError(unsupportedMessage(agent), 1); +} + +export function getBridgeAdapter(agent: AgentDefinition): AgentMcpAdapter { + assertBridgeSupported(agent); + const adapter = agent.mcpCapability.adapter; + if (!adapter) { + throw new McpBridgeError( + `${agent.displayName} declares MCP support but does not declare an adapter.`, + 1, + ); + } + return adapter; +} + +export function getEntryAdapter( + entry: Pick | undefined, + agent: AgentDefinition, +): AgentMcpAdapter | null { + if (entry && isAgentMcpAdapter(entry.adapter)) return entry.adapter; + return agent.mcpCapability.support === "bridge" && agent.mcpCapability.adapter + ? agent.mcpCapability.adapter + : null; +} + +export function bridgeState(sandbox: SandboxEntry): Record { + return sandbox.mcp?.bridges ?? {}; +} + +export function setBridgeState(sandboxName: string, bridges: Record): void { + const mcpState = registry.getSandbox(sandboxName)?.mcp; + const destroyPreparedAt = mcpState?.destroyPreparedAt; + const destroyPendingAt = mcpState?.destroyPendingAt; + const hasDestroyState = !!destroyPreparedAt || !!destroyPendingAt; + const updated = registry.updateSandbox(sandboxName, { + mcp: + Object.keys(bridges).length > 0 || hasDestroyState + ? { + bridges, + ...(destroyPreparedAt ? { destroyPreparedAt } : {}), + ...(destroyPendingAt ? { destroyPendingAt } : {}), + } + : undefined, + }); + if (!updated) { + throw new McpBridgeError(`Could not persist MCP lifecycle state for sandbox '${sandboxName}'.`); + } +} + +export function assertMcpDestroyNotPending(sandbox: SandboxEntry): void { + if (!sandbox.mcp?.destroyPreparedAt && !sandbox.mcp?.destroyPendingAt) return; + throw new McpBridgeError( + `Sandbox '${sandbox.name}' has an incomplete MCP destroy transaction. Re-run the sandbox destroy command to finish cleanup before using MCP commands.`, + ); +} + +export function assertNoDerivedResourceCollision( + sandbox: SandboxEntry, + server: string, + providerName: string | undefined, + policyName: string, +): void { + const conflictingCustomPolicy = sandbox.customPolicies?.find( + (policy) => policy.name === policyName && policy.sourcePath !== MCP_BRIDGE_POLICY_SOURCE, + ); + if (conflictingCustomPolicy || sandbox.policies?.includes(policyName)) { + throw new McpBridgeError( + `Generated MCP policy name '${policyName}' conflicts with an existing non-MCP policy. Choose a different server name.`, + 2, + ); + } + for (const entry of Object.values(bridgeState(sandbox))) { + if (entry.server === server) continue; + const providerCollision = + providerName !== undefined && + entry.providerName !== undefined && + entry.providerName === providerName; + if (providerCollision || entry.policyName === policyName) { + throw new McpBridgeError( + `MCP server '${server}' conflicts with existing server '${entry.server}' after OpenShell resource-name normalization. Choose a name that differs beyond case, hyphens, and underscores.`, + 2, + ); + } + } +} + +export function writeBridgeEntry(sandboxName: string, entry: McpBridgeEntry): void { + const sandbox = getSandboxOrThrow(sandboxName); + const bridges = { ...bridgeState(sandbox), [entry.server]: entry }; + setBridgeState(sandboxName, bridges); +} + +export function removeBridgeEntry(sandboxName: string, server: string): void { + const sandbox = getSandboxOrThrow(sandboxName); + const bridges = { ...bridgeState(sandbox) }; + delete bridges[server]; + setBridgeState(sandboxName, bridges); +} + +export async function ensureSandboxGatewaySelected(sandboxName: string): Promise { + const gatewayName = getSandboxTargetGatewayName(sandboxName); + const recovery = await recoverNamedGatewayRuntime({ + gatewayName, + }); + if (!recovery.recovered || recovery.after.state !== "healthy_named") { + throw new McpBridgeError( + `Could not select healthy OpenShell gateway '${gatewayName}' for sandbox '${sandboxName}' (before: ${recovery.before.state}, after: ${recovery.after.state}). Refusing to mutate MCP resources on another gateway.`, + ); + } + // Pin every subsequent OpenShell subprocess in this lifecycle operation to + // the sandbox's recorded gateway. The globally selected gateway is mutable + // shared metadata and another NemoClaw process may select a sibling between + // this health check and the provider/policy mutation. + process.env.OPENSHELL_GATEWAY = gatewayName; +} diff --git a/src/lib/actions/sandbox/mcp-bridge-status-boundaries.test.ts b/src/lib/actions/sandbox/mcp-bridge-status-boundaries.test.ts new file mode 100644 index 00000000000..6fa85d3f282 --- /dev/null +++ b/src/lib/actions/sandbox/mcp-bridge-status-boundaries.test.ts @@ -0,0 +1,176 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, describe, expect, it } from "vitest"; + +const sourceRequireHook = path.resolve("test/helpers/onboard-script-mocks.cjs"); +const sourceNodeOptions = [process.env.NODE_OPTIONS, `--require=${sourceRequireHook}`] + .filter(Boolean) + .join(" "); +const tempHomes = new Set(); + +function createTempHome(prefix: string): string { + const home = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); + tempHomes.add(home); + return home; +} + +afterEach(() => { + tempHomes.forEach((home) => fs.rmSync(home, { recursive: true, force: true })); + tempHomes.clear(); +}); + +describe("cross-agent MCP status boundaries", () => { + it("reports unsupported persisted boundaries without starting an unsafe sandbox child", () => { + const home = createTempHome("nemoclaw-mcp-status-risk-"); + const script = String.raw` +process.env.HOME = ${JSON.stringify(home)}; +process.env.LD_PRELOAD = "/tmp/legacy-attached-loader.so"; +const registry = require("./src/lib/state/registry.js"); +const gatewayRuntime = require("./src/lib/gateway-runtime-action.js"); +const globalActions = require("./src/lib/actions/global.js"); +const policies = require("./src/lib/policy/index.js"); +const processRecovery = require("./src/lib/actions/sandbox/process-recovery.js"); +gatewayRuntime.recoverNamedGatewayRuntime = async () => ({ + recovered: true, + attempted: false, + before: { state: "healthy_named" }, + after: { state: "healthy_named" }, +}); +globalActions.runOpenshellProviderCommand = (args) => { + if (args[0] === "provider" && args[1] === "get") { + return { + status: 0, + stdout: "Id: 11111111-2222-4333-8444-555555555555\nType: generic\nResource version: 4\nCredential keys: LD_PRELOAD\n", + stderr: "", + }; + } + if (args[0] === "sandbox" && args[1] === "provider" && args[2] === "list") { + return { + status: 0, + stdout: "NAME TYPE CREDENTIAL_KEYS CONFIG_KEYS\nalpha-mcp-fake generic 1 0\n", + stderr: "", + }; + } + throw new Error("Unexpected OpenShell call: " + args.join(" ")); +}; +policies.getPresetContentGatewayState = () => "match"; +processRecovery.executeSandboxCommand = () => { + throw new Error("unsafe sandbox child must not start while LD_PRELOAD is attached"); +}; +registry.registerSandbox({ + name: "alpha", + agent: "openclaw", + mcp: { bridges: { fake: { + server: "fake", + agent: "openclaw", + adapter: "mcporter", + url: "https://host.openshell.internal:31337/mcp", + env: ["LD_PRELOAD"], + providerName: "alpha-mcp-fake", + providerId: "11111111-2222-4333-8444-555555555555", + policyName: "mcp-bridge-fake", + addedAt: "2026-06-01T00:00:00.000Z", + } } }, +}); +registry.addCustomPolicy("alpha", { + name: "mcp-bridge-fake", + content: "network_policies: {}\n", + sourcePath: "generated:nemoclaw-mcp-bridge", +}); +const bridge = require("./src/lib/actions/sandbox/mcp-bridge.js"); +(async () => { + const [status] = await bridge.statusMcpBridge("alpha", "fake"); + const lines = []; + const originalLog = console.log; + console.log = (...args) => lines.push(args.join(" ")); + try { + await bridge.dispatchMcpBridgeCommand("alpha", ["status", "fake"]); + } finally { + console.log = originalLog; + } + process.stdout.write(JSON.stringify({ status, text: lines.join("\n") })); +})().catch((error) => { + console.error(error); + process.exit(1); +}); +`; + const result = spawnSync(process.execPath, ["-e", script], { + cwd: process.cwd(), + encoding: "utf8", + env: { ...process.env, HOME: home, NODE_OPTIONS: sourceNodeOptions }, + }); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + const payload = JSON.parse(result.stdout) as { + status: { + warnings: string[]; + provider: { attached: boolean | null }; + adapter: { registered: boolean | null; detail?: string }; + }; + text: string; + }; + expect(payload.status.provider.attached).toBe(true); + expect(payload.status.adapter).toEqual({ + registered: null, + detail: expect.stringMatching(/inspection was skipped.*legacy credential/i), + }); + expect(payload.status.warnings).toEqual([ + expect.stringMatching(/provider at sandbox scope.*endpoint-exclusive credential binding/i), + expect.stringMatching(/persisted MCP URL no longer satisfies.*remove this server/i), + expect.stringMatching( + /persisted MCP credential name no longer satisfies.*remove this server/i, + ), + ]); + expect(payload.text).toMatch( + /warning: OpenShell currently attaches this credential provider at sandbox scope/i, + ); + expect(payload.text).toMatch(/warning: This persisted MCP URL no longer satisfies/i); + expect(payload.text).toMatch( + /warning: This persisted MCP credential name no longer satisfies/i, + ); + }); + + it("reports Hermes bridge support in status JSON without requiring servers", () => { + const home = createTempHome("nemoclaw-mcp-status-"); + const script = ` +process.env.HOME = ${JSON.stringify(home)}; +const registry = require("./src/lib/state/registry.js"); +const bridge = require("./src/lib/actions/sandbox/mcp-bridge.js"); +registry.registerSandbox({ name: "hermes-sandbox", agent: "hermes" }); +bridge.dispatchMcpBridgeCommand("hermes-sandbox", ["status", "--json"]).then( + () => process.exit(0), + (error) => { + console.error(error); + process.exit(1); + }, +); +`; + const result = spawnSync(process.execPath, ["-e", script], { + cwd: process.cwd(), + encoding: "utf8", + env: { ...process.env, HOME: home, NODE_OPTIONS: sourceNodeOptions }, + }); + + expect(result.status).toBe(0); + const payload = JSON.parse(result.stdout.trim()) as { + sandbox: string; + agent: string; + support: { supported: boolean; mode: string; reason?: string }; + bridges: unknown[]; + }; + expect(payload.sandbox).toBe("hermes-sandbox"); + expect(payload.agent).toBe("hermes"); + expect(payload.support).toMatchObject({ + supported: true, + mode: "bridge", + adapter: "hermes-config", + }); + expect(payload.bridges).toEqual([]); + }); +}); diff --git a/src/lib/actions/sandbox/mcp-bridge-status-removal.test.ts b/src/lib/actions/sandbox/mcp-bridge-status-removal.test.ts new file mode 100644 index 00000000000..463457f59cb --- /dev/null +++ b/src/lib/actions/sandbox/mcp-bridge-status-removal.test.ts @@ -0,0 +1,182 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, describe, expect, it } from "vitest"; + +const sourceRequireHook = path.resolve("test/helpers/onboard-script-mocks.cjs"); +const sourceNodeOptions = [process.env.NODE_OPTIONS, `--require=${sourceRequireHook}`] + .filter(Boolean) + .join(" "); +const tempHomes = new Set(); + +function createTempHome(prefix: string): string { + const home = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); + tempHomes.add(home); + return home; +} + +afterEach(() => { + tempHomes.forEach((home) => fs.rmSync(home, { recursive: true, force: true })); + tempHomes.clear(); +}); + +describe("cross-agent MCP removal", () => { + it("removes a persisted bridge without requiring the current agent to support MCP", () => { + const home = createTempHome("nemoclaw-mcp-remove-"); + const script = ` +process.env.HOME = ${JSON.stringify(home)}; +const registry = require("./src/lib/state/registry.js"); +const agentDefs = require("./src/lib/agent/defs.js"); +const globalActions = require("./src/lib/actions/global.js"); +const gatewayRuntime = require("./src/lib/gateway-runtime-action.js"); +const policies = require("./src/lib/policy/index.js"); +const processRecovery = require("./src/lib/actions/sandbox/process-recovery.js"); +agentDefs.loadAgent = () => { throw new Error("current agent must not be consulted"); }; +gatewayRuntime.recoverNamedGatewayRuntime = async () => ({ + recovered: true, + attempted: false, + before: { state: "healthy_named" }, + after: { state: "healthy_named" }, +}); +globalActions.runOpenshellProviderCommand = (args) => { + if (args.join(" ") === "status --output json") { + return { + status: 0, + stdout: "ready", + stderr: "", + }; + } + throw new Error("Unexpected OpenShell call: " + args.join(" ")); +}; +policies.removePreset = () => true; +policies.getPresetContentGatewayState = () => "absent"; +processRecovery.executeSandboxCommand = () => ({ status: 0, stdout: "", stderr: "" }); +processRecovery.executeSandboxExecCommand = () => ({ status: 0, stdout: "", stderr: "" }); +registry.registerSandbox({ + name: "legacy-sandbox", + agent: "legacy-disabled", + mcp: { bridges: { github: { + server: "github", + url: "https://host.openshell.internal:31337/mcp", + env: [], + policyName: "mcp-bridge-github", + adapter: "mcporter", + createdAt: "2026-06-01T00:00:00.000Z", + updatedAt: "2026-06-01T00:00:00.000Z", + } } }, +}); +registry.addCustomPolicy("legacy-sandbox", { + name: "mcp-bridge-github", + content: "network_policies:\\n mcp_bridge_github:\\n endpoints: []\\n", + sourcePath: "generated:nemoclaw-mcp-bridge", + appliedAt: "2026-06-01T00:00:00.000Z", +}); +const bridge = require("./src/lib/actions/sandbox/mcp-bridge.js"); +bridge.removeMcpBridge("legacy-sandbox", "github").then( + () => { + process.stdout.write(JSON.stringify(registry.getSandbox("legacy-sandbox"))); + process.exit(0); + }, + (error) => { + console.error(error); + process.exit(1); + }, +); +`; + const result = spawnSync(process.execPath, ["-e", script], { + cwd: process.cwd(), + encoding: "utf8", + env: { ...process.env, HOME: home, NODE_OPTIONS: sourceNodeOptions }, + }); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + const jsonStart = result.stdout.indexOf("{"); + const sandbox = JSON.parse(result.stdout.slice(jsonStart)) as { + mcp?: unknown; + }; + expect(sandbox.mcp).toBeUndefined(); + }); + + it("preserves the registry entry when force cleanup leaves residual policy state", () => { + const home = createTempHome("nemoclaw-mcp-residual-"); + const script = ` +process.env.HOME = ${JSON.stringify(home)}; +const registry = require("./src/lib/state/registry.js"); +const agentDefs = require("./src/lib/agent/defs.js"); +const globalActions = require("./src/lib/actions/global.js"); +const gatewayRuntime = require("./src/lib/gateway-runtime-action.js"); +const policies = require("./src/lib/policy/index.js"); +const processRecovery = require("./src/lib/actions/sandbox/process-recovery.js"); +agentDefs.loadAgent = () => { throw new Error("current agent must not be consulted"); }; +gatewayRuntime.recoverNamedGatewayRuntime = async () => ({ + recovered: true, + attempted: false, + before: { state: "healthy_named" }, + after: { state: "healthy_named" }, +}); +globalActions.runOpenshellProviderCommand = (args) => { + if (args.join(" ") === "status --output json") { + return { + status: 0, + stdout: "ready", + stderr: "", + }; + } + throw new Error("Unexpected OpenShell call: " + args.join(" ")); +}; +policies.removePreset = () => false; +policies.getPresetContentGatewayState = () => "match"; +processRecovery.executeSandboxCommand = () => ({ status: 0, stdout: "", stderr: "" }); +processRecovery.executeSandboxExecCommand = () => ({ status: 0, stdout: "", stderr: "" }); +registry.registerSandbox({ + name: "legacy-sandbox", + agent: "legacy-disabled", + mcp: { bridges: { github: { + server: "github", + url: "https://mcp.example.test/mcp", + env: [], + policyName: "mcp-bridge-github", + adapter: "mcporter", + createdAt: "2026-06-01T00:00:00.000Z", + updatedAt: "2026-06-01T00:00:00.000Z", + } } }, +}); +registry.addCustomPolicy("legacy-sandbox", { + name: "mcp-bridge-github", + content: "network_policies:\\n mcp_bridge_github:\\n name: managed\\n endpoints: []\\n", + sourcePath: "generated:nemoclaw-mcp-bridge", + appliedAt: "2026-06-01T00:00:00.000Z", +}); +const bridge = require("./src/lib/actions/sandbox/mcp-bridge.js"); +bridge.removeMcpBridge("legacy-sandbox", "github", { force: true }).then( + () => process.exit(1), + (error) => { + process.stdout.write(JSON.stringify({ + message: error.message, + sandbox: registry.getSandbox("legacy-sandbox"), + })); + process.exit(0); + }, +); +`; + const result = spawnSync(process.execPath, ["-e", script], { + cwd: process.cwd(), + encoding: "utf8", + env: { ...process.env, HOME: home, NODE_OPTIONS: sourceNodeOptions }, + }); + + expect(result.status).toBe(0); + const jsonStart = result.stdout.indexOf("{"); + const payload = JSON.parse(result.stdout.slice(jsonStart)) as { + message: string; + sandbox: { mcp?: { bridges?: Record } }; + }; + expect(payload.message).toContain("registry entry was preserved"); + expect(payload.sandbox.mcp?.bridges).toHaveProperty("github"); + }); +}); diff --git a/src/lib/actions/sandbox/mcp-bridge-status-state.test.ts b/src/lib/actions/sandbox/mcp-bridge-status-state.test.ts new file mode 100644 index 00000000000..46376419e01 --- /dev/null +++ b/src/lib/actions/sandbox/mcp-bridge-status-state.test.ts @@ -0,0 +1,262 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, describe, expect, it } from "vitest"; + +const sourceRequireHook = path.resolve("test/helpers/onboard-script-mocks.cjs"); +const sourceNodeOptions = [process.env.NODE_OPTIONS, `--require=${sourceRequireHook}`] + .filter(Boolean) + .join(" "); +const tempHomes = new Set(); + +function createTempHome(prefix: string): string { + const home = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); + tempHomes.add(home); + return home; +} + +afterEach(() => { + tempHomes.forEach((home) => fs.rmSync(home, { recursive: true, force: true })); + tempHomes.clear(); +}); + +describe("cross-agent MCP status state", () => { + it("rejects duplicate static credential keys across bridges in one sandbox", () => { + const home = createTempHome("nemoclaw-mcp-env-key-"); + const script = ` +process.env.HOME = ${JSON.stringify(home)}; +const registry = require("./src/lib/state/registry.js"); +registry.registerSandbox({ + name: "openclaw-sandbox", + agent: "openclaw", + mcp: { bridges: { first: { + server: "first", + url: "https://8.8.8.8/mcp", + env: ["SHARED_MCP_TOKEN"], + providerName: "nemoclaw-mcp-openclaw-sandbox-first", + policyName: "mcp-bridge-first", + adapter: "mcporter", + createdAt: "2026-06-01T00:00:00.000Z", + updatedAt: "2026-06-01T00:00:00.000Z", + } } }, +}); +const bridge = require("./src/lib/actions/sandbox/mcp-bridge.js"); +bridge.addMcpBridge("openclaw-sandbox", { + server: "second", + url: "https://8.8.8.8/mcp", + env: [{ name: "SHARED_MCP_TOKEN" }], +}).then( + () => process.exit(1), + (error) => { + process.stdout.write(error.message); + process.exit(0); + }, +); +`; + const result = spawnSync(process.execPath, ["-e", script], { + cwd: process.cwd(), + encoding: "utf8", + env: { ...process.env, HOME: home, NODE_OPTIONS: sourceNodeOptions }, + }); + + expect(result.status).toBe(0); + expect(result.stdout).toContain("already attached through MCP server 'first'"); + }); + + it("preserves destroy transaction markers when the last bridge is removed", () => { + const home = createTempHome("nemoclaw-mcp-destroy-state-"); + const script = ` +process.env.HOME = ${JSON.stringify(home)}; +const registry = require("./src/lib/state/registry.js"); +const state = require("./src/lib/actions/sandbox/mcp-bridge-state.js"); +const markers = ["destroyPreparedAt", "destroyPendingAt"]; +for (const [index, marker] of markers.entries()) { + const name = "destroy-state-" + index; + registry.registerSandbox({ + name, + agent: "openclaw", + mcp: { + bridges: { github: { + server: "github", + agent: "openclaw", + adapter: "mcporter", + url: "https://mcp.example.test/mcp", + env: [], + policyName: "mcp-bridge-github", + addedAt: "2026-06-01T00:00:00.000Z", + } }, + [marker]: "2026-06-27T01:00:00.000Z", + }, + }); + state.removeBridgeEntry(name, "github"); +} +process.stdout.write(JSON.stringify(markers.map((_, index) => registry.getSandbox("destroy-state-" + index)))); +`; + const result = spawnSync(process.execPath, ["-e", script], { + cwd: process.cwd(), + encoding: "utf8", + env: { ...process.env, HOME: home, NODE_OPTIONS: sourceNodeOptions }, + }); + + expect(result.status).toBe(0); + const sandboxes = JSON.parse(result.stdout) as Array<{ + mcp: { + bridges: Record; + destroyPreparedAt?: string; + destroyPendingAt?: string; + }; + }>; + expect(sandboxes[0]?.mcp).toEqual({ + bridges: {}, + destroyPreparedAt: "2026-06-27T01:00:00.000Z", + }); + expect(sandboxes[1]?.mcp).toEqual({ + bridges: {}, + destroyPendingAt: "2026-06-27T01:00:00.000Z", + }); + }); + + it("validates requested server names and does not read inherited bridge keys", () => { + const home = createTempHome("nemoclaw-mcp-status-key-"); + const script = ` +process.env.HOME = ${JSON.stringify(home)}; +const registry = require("./src/lib/state/registry.js"); +const status = require("./src/lib/actions/sandbox/mcp-bridge-status.js"); +registry.registerSandbox({ name: "openclaw-sandbox", agent: "openclaw" }); +(async () => { + let invalid; + try { + await status.statusMcpBridge("openclaw-sandbox", "__proto__"); + } catch (error) { + invalid = { message: error.message, exitCode: error.exitCode }; + } + const inherited = await status.statusMcpBridge("openclaw-sandbox", "constructor"); + process.stdout.write(JSON.stringify({ invalid, inherited })); +})().catch((error) => { + console.error(error); + process.exit(1); +}); +`; + const result = spawnSync(process.execPath, ["-e", script], { + cwd: process.cwd(), + encoding: "utf8", + env: { ...process.env, HOME: home, NODE_OPTIONS: sourceNodeOptions }, + }); + + expect(result.status).toBe(0); + const payload = JSON.parse(result.stdout) as { + invalid: { message: string; exitCode: number }; + inherited: Array<{ + server: string; + provider: { registryPresent: boolean }; + adapter: { registered: boolean | null }; + }>; + }; + expect(payload.invalid.exitCode).toBe(2); + expect(payload.invalid.message).toContain("Invalid MCP server name '__proto__'"); + expect(payload.inherited).toHaveLength(1); + expect(payload.inherited[0]).toMatchObject({ + server: "constructor", + provider: { registryPresent: false }, + adapter: { registered: null }, + }); + }); + + it("reports each bridge from its persisted adapter or agent capability", () => { + const home = createTempHome("nemoclaw-mcp-status-agent-"); + const script = ` +process.env.HOME = ${JSON.stringify(home)}; +const registry = require("./src/lib/state/registry.js"); +const agentDefs = require("./src/lib/agent/defs.js"); +const gatewayRuntime = require("./src/lib/gateway-runtime-action.js"); +const processRecovery = require("./src/lib/actions/sandbox/process-recovery.js"); +agentDefs.loadAgent = (name) => { + if (name === "current-disabled") { + return { + name, + displayName: "Current Disabled", + mcpCapability: { support: "disabled", reason: "current agent is disabled" }, + }; + } + if (name === "persisted-enabled") { + return { + name, + displayName: "Persisted Enabled", + mcpCapability: { support: "bridge", adapter: "deepagents-config" }, + }; + } + throw new Error("Unexpected agent lookup: " + name); +}; +gatewayRuntime.recoverNamedGatewayRuntime = async () => ({ + recovered: true, + attempted: false, + before: { state: "healthy_named" }, + after: { state: "healthy_named" }, +}); +processRecovery.executeSandboxCommand = () => ({ status: 0, stdout: "registered", stderr: "" }); +registry.registerSandbox({ + name: "persisted-status", + agent: "current-disabled", + mcp: { bridges: { + direct: { + server: "direct", + agent: "persisted-unknown", + adapter: "mcporter", + url: "https://mcp.example.test/direct", + env: [], + policyName: "mcp-bridge-direct", + addedAt: "2026-06-01T00:00:00.000Z", + }, + legacy: { + server: "legacy", + agent: "persisted-enabled", + url: "https://mcp.example.test/legacy", + env: [], + policyName: "mcp-bridge-legacy", + addedAt: "2026-06-01T00:00:00.000Z", + }, + } }, +}); +const status = require("./src/lib/actions/sandbox/mcp-bridge-status.js"); +status.statusMcpBridge("persisted-status").then( + (bridges) => process.stdout.write(JSON.stringify(bridges)), + (error) => { + console.error(error); + process.exit(1); + }, +); +`; + const result = spawnSync(process.execPath, ["-e", script], { + cwd: process.cwd(), + encoding: "utf8", + env: { ...process.env, HOME: home, NODE_OPTIONS: sourceNodeOptions }, + }); + + expect(result.status).toBe(0); + const bridges = JSON.parse(result.stdout) as Array<{ + server: string; + agent: string; + support: { supported: boolean; mode: string; adapter?: string; reason?: string }; + adapter: { registered: boolean | null }; + }>; + expect(bridges).toHaveLength(2); + expect(bridges[0]).toMatchObject({ + server: "direct", + agent: "persisted-unknown", + support: { supported: true, mode: "bridge", adapter: "mcporter" }, + adapter: { registered: true }, + }); + expect(bridges[0]?.support.reason).toBeUndefined(); + expect(bridges[1]).toMatchObject({ + server: "legacy", + agent: "persisted-enabled", + support: { supported: true, mode: "bridge", adapter: "deepagents-config" }, + adapter: { registered: true }, + }); + }); +}); diff --git a/src/lib/actions/sandbox/mcp-bridge-status.ts b/src/lib/actions/sandbox/mcp-bridge-status.ts new file mode 100644 index 00000000000..8ea71accd58 --- /dev/null +++ b/src/lib/actions/sandbox/mcp-bridge-status.ts @@ -0,0 +1,258 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { type AgentDefinition, type AgentMcpAdapter, loadAgent } from "../../agent/defs"; +import type { McpBridgeEntry } from "../../state/registry"; +import { + buildDeepAgentsMcpStatusCommand, + buildHermesMcpStatusCommand, + buildOpenClawMcporterInspectCommand, +} from "./mcp-bridge-adapters"; +import { isAgentMcpAdapter, type McpBridgeStatus } from "./mcp-bridge-contracts"; +import { redactBridgeSecretsForDisplay } from "./mcp-bridge-output"; +import { getPolicyPresence, getRegisteredGeneratedPolicy } from "./mcp-bridge-policy"; +import { + inspectMcpProvider, + providerAttached, + providerMatchesCredential, + providerShapeDetail, +} from "./mcp-bridge-provider"; +import { + bridgeState, + ensureSandboxGatewaySelected, + getSandboxAgent, + getSandboxOrThrow, +} from "./mcp-bridge-state"; +import { + assertAuthenticatedBridgeEntry, + normalizeMcpServerUrl, + resolvePersistedCredentialEnvForRedaction, + validateMcpServerName, + validateSandboxName, +} from "./mcp-bridge-validation"; +import { executeSandboxCommand } from "./process-recovery"; + +export interface McpBridgeJsonSummary { + sandbox: string; + agent: string; + support: McpBridgeStatus["support"]; + bridges: McpBridgeStatus[]; +} + +const SANDBOX_SCOPED_PROVIDER_WARNING = + "OpenShell currently attaches this credential provider at sandbox scope, not exclusively to this MCP endpoint. Keep other inspected routes for the same adapter binary at least as restrictive until OpenShell supports endpoint-exclusive credential binding plus Host, scheme, and query enforcement."; +const UNSUPPORTED_STORED_URL_WARNING = + "This persisted MCP URL no longer satisfies the authenticated endpoint boundary. Restart and rebuild fail closed for it; remove this server (use --force if cleanup is partial), then add a normal public HTTPS DNS endpoint."; +const UNSUPPORTED_STORED_CREDENTIAL_WARNING = + "This persisted MCP credential name no longer satisfies the host-only credential boundary. Restart and rebuild fail closed for it; remove this server, then add it again with a dedicated service credential name."; + +function storedUrlWarning(entry: McpBridgeEntry): string | undefined { + try { + return normalizeMcpServerUrl(entry.url) === entry.url + ? undefined + : UNSUPPORTED_STORED_URL_WARNING; + } catch { + return UNSUPPORTED_STORED_URL_WARNING; + } +} + +function storedCredentialWarning(entry: McpBridgeEntry): string | undefined { + try { + assertAuthenticatedBridgeEntry(entry); + return undefined; + } catch { + return UNSUPPORTED_STORED_CREDENTIAL_WARNING; + } +} + +function getAdapterRegistration( + sandboxName: string, + adapter: AgentMcpAdapter | undefined, + entry: McpBridgeEntry | undefined, +): McpBridgeStatus["adapter"] { + if (!entry) return { registered: null }; + if (!adapter) return { registered: null, detail: "MCP adapter is not declared" }; + const command = + adapter === "mcporter" + ? buildOpenClawMcporterInspectCommand(entry, false) + : adapter === "hermes-config" + ? buildHermesMcpStatusCommand(entry) + : buildDeepAgentsMcpStatusCommand(entry); + const result = executeSandboxCommand(sandboxName, command); + if (!result) return { registered: null, detail: "sandbox unreachable" }; + if (result.status === 0) { + const output = result.stdout.trim(); + if (output === "registered") return { registered: true }; + return { registered: false, detail: output || "not found" }; + } + const envValues = resolvePersistedCredentialEnvForRedaction(entry.env); + return { + registered: false, + detail: redactBridgeSecretsForDisplay( + result.stderr || result.stdout || "not found", + entry, + envValues, + ), + }; +} + +export async function statusMcpBridge( + sandboxName: string, + server?: string, +): Promise { + validateSandboxName(sandboxName); + if (server !== undefined) validateMcpServerName(server); + const sandbox = getSandboxOrThrow(sandboxName); + const agent = getSandboxAgent(sandbox); + const bridges = bridgeState(sandbox); + if (Object.keys(bridges).length > 0) { + await ensureSandboxGatewaySelected(sandboxName); + } + const selectedEntry = + server !== undefined && Object.hasOwn(bridges, server) ? bridges[server] : undefined; + const entries: Array<[string, McpBridgeEntry | undefined]> = + server !== undefined ? [[server, selectedEntry]] : Object.entries(bridges); + if (server !== undefined && !selectedEntry) { + return [ + { + server, + agent: agent.name, + warnings: [], + support: { + supported: agent.mcpCapability.support === "bridge", + mode: agent.mcpCapability.support, + ...(agent.mcpCapability.adapter ? { adapter: agent.mcpCapability.adapter } : {}), + ...(agent.mcpCapability.reason ? { reason: agent.mcpCapability.reason } : {}), + }, + env: { names: [], missing: [], ready: false }, + provider: { + registryPresent: false, + gatewayPresent: false, + attached: null, + credentialReady: null, + }, + policy: { registryPresent: false, gatewayPresent: false }, + adapter: { registered: null }, + }, + ]; + } + + return entries.map(([name, entry]) => { + const support = entry ? getPersistedBridgeSupport(entry) : getSupportSummary(agent); + const registeredPolicy = getRegisteredGeneratedPolicy(sandboxName, entry); + const hasCredentialBinding = + !!entry && + Array.isArray(entry.env) && + entry.env.length === 1 && + !!entry.providerName && + !!entry.providerId; + const missingEnv = entry + ? entry.env.filter( + (envName: string) => process.env[envName] === undefined || process.env[envName] === "", + ) + : []; + const expectedCredential = entry?.env.length === 1 ? entry.env[0] : undefined; + const providerInspection = inspectMcpProvider(entry?.providerName); + const providerCredentialReady = providerMatchesCredential( + providerInspection, + expectedCredential, + entry?.providerId, + ); + const providerDetail = providerShapeDetail( + providerInspection, + expectedCredential, + entry?.providerId, + ); + const attached = providerAttached(sandboxName, entry?.providerName); + const warnings: string[] = []; + if (attached === true) warnings.push(SANDBOX_SCOPED_PROVIDER_WARNING); + let credentialWarning: string | undefined; + if (entry) { + const urlWarning = storedUrlWarning(entry); + if (urlWarning) warnings.push(urlWarning); + credentialWarning = storedCredentialWarning(entry); + if (credentialWarning) warnings.push(credentialWarning); + } + const unsafeCredentialMayBeAttached = + !!credentialWarning && !!entry?.providerName && attached !== false; + return { + server: name, + agent: entry?.agent ?? agent.name, + warnings, + support, + ...(entry ? { url: entry.url } : {}), + ...(entry?.addState ? { addState: entry.addState } : {}), + env: { + names: entry?.env ?? [], + missing: missingEnv, + ready: + hasCredentialBinding && + !entry?.addState && + (providerInspection.exists ? providerCredentialReady : missingEnv.length === 0), + }, + provider: { + name: entry?.providerName, + registryPresent: !!entry?.providerName, + gatewayPresent: entry?.providerName ? providerInspection.exists : null, + attached, + credentialReady: entry ? providerCredentialReady : null, + ...(providerDetail ? { detail: providerDetail } : {}), + }, + policy: { + name: entry?.policyName, + registryPresent: !!registeredPolicy, + gatewayPresent: getPolicyPresence(sandboxName, entry), + }, + adapter: unsafeCredentialMayBeAttached + ? { + registered: null, + detail: + "Adapter inspection was skipped because the unsupported legacy credential may still be attached to fresh sandbox children.", + } + : getAdapterRegistration(sandboxName, support.adapter, entry), + ...(entry?.addedAt ? { addedAt: entry.addedAt } : {}), + ...(entry?.updatedAt ? { updatedAt: entry.updatedAt } : {}), + }; + }); +} + +function getPersistedBridgeSupport(entry: McpBridgeEntry): McpBridgeStatus["support"] { + if (isAgentMcpAdapter(entry.adapter)) { + return { + supported: true, + mode: "bridge", + adapter: entry.adapter, + }; + } + try { + return getSupportSummary(loadAgent(entry.agent)); + } catch { + return { + supported: false, + mode: "disabled", + reason: `Persisted agent '${entry.agent}' is unavailable.`, + }; + } +} + +function getSupportSummary(agent: AgentDefinition): McpBridgeStatus["support"] { + return { + supported: agent.mcpCapability.support === "bridge", + mode: agent.mcpCapability.support, + ...(agent.mcpCapability.adapter ? { adapter: agent.mcpCapability.adapter } : {}), + ...(agent.mcpCapability.reason ? { reason: agent.mcpCapability.reason } : {}), + }; +} + +export function buildJsonSummary( + sandboxName: string, + agent: AgentDefinition, + statuses: McpBridgeStatus[], +): McpBridgeJsonSummary { + return { + sandbox: sandboxName, + agent: agent.name, + support: getSupportSummary(agent), + bridges: statuses, + }; +} diff --git a/src/lib/actions/sandbox/mcp-bridge-url-validation.ts b/src/lib/actions/sandbox/mcp-bridge-url-validation.ts new file mode 100644 index 00000000000..fe1f38b56cb --- /dev/null +++ b/src/lib/actions/sandbox/mcp-bridge-url-validation.ts @@ -0,0 +1,198 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { resolveHostAddresses } from "../../adapters/dns/resolve"; +import { + isBlockedMcpUrlTargetHost, + isOpenShellMcpHostAlias, + MCP_SERVER_URL_MAX_LENGTH, +} from "../../security/mcp-url-target"; +import { TOKEN_PREFIX_PATTERNS } from "../../security/secret-patterns"; +import { McpBridgeError } from "./mcp-bridge-contracts"; + +export { MCP_SERVER_URL_MAX_LENGTH } from "../../security/mcp-url-target"; + +const MCP_PATH_CREDENTIAL_PATTERNS = TOKEN_PREFIX_PATTERNS.map( + // Validation rejects a token contained anywhere in a persisted segment. + // Redaction's word boundaries are inappropriate here because '-' is a valid + // final Telegram/Discord token character but is not a RegExp "word" byte. + (pattern) => new RegExp(pattern.source.replaceAll("\\b", ""), pattern.flags.replace("g", "")), +); + +/** Reject self-identifying credentials in persisted endpoint path segments. */ +function hasSecretShapedMcpPathSegment(pathname: string): boolean { + return pathname.split("/").some((segment) => { + if (!segment) return false; + return MCP_PATH_CREDENTIAL_PATTERNS.some((pattern) => pattern.test(segment)); + }); +} + +function rejectUnsupportedOpenShellMcpHostAlias(hostname: string): void { + if (!isOpenShellMcpHostAlias(hostname)) return; + // invalidState: a host alias is accepted without an attested gateway address, + // forcing broad private-range policy instead of an exact destination pin. + // sourceBoundary: the pinned OpenShell release owns gateway-address discovery. + // whyNotSourceFix: v0.0.72 exposes no attested driver gateway address. + // regressionTest: URL validation and all three live adapters reject aliases. + // removalCondition: remove only after a reviewed OpenShell capability exposes + // an attested address; a future version number alone is not that capability. + throw new McpBridgeError( + `Authenticated MCP OpenShell host alias '${hostname}' is unavailable with OpenShell v0.0.72 because that release does not expose an attested driver gateway address for exact policy pinning. Use a normal HTTPS DNS endpoint with public address records.`, + 2, + ); +} + +function validateMcpServerUrlTarget(parsed: URL): void { + if (isBlockedMcpUrlTargetHost(parsed.hostname)) { + throw new McpBridgeError( + `MCP server URL host '${parsed.hostname}' is a private, local, or special-use IP address. Use a normal HTTPS DNS endpoint with public address records.`, + 2, + ); + } +} + +export function normalizeMcpServerUrl(rawUrl: string): string { + if (rawUrl.length > MCP_SERVER_URL_MAX_LENGTH) { + throw new McpBridgeError( + `MCP server URL must be at most ${MCP_SERVER_URL_MAX_LENGTH} characters.`, + 2, + ); + } + let parsed: URL; + try { + parsed = new URL(rawUrl); + } catch { + throw new McpBridgeError(`Invalid MCP server URL '${rawUrl}'.`, 2); + } + if (parsed.protocol !== "https:") { + throw new McpBridgeError( + "Authenticated MCP server URLs must use https:// so the configured MCP client uses TLS when OpenShell forwards credential-bearing requests.", + 2, + ); + } + if (!parsed.hostname) { + throw new McpBridgeError("MCP server URL must include a hostname.", 2); + } + if (/[*{};]/.test(parsed.hostname)) { + throw new McpBridgeError( + "MCP server URL hosts must be literal; wildcard and glob hostnames are not supported.", + 2, + ); + } + if (parsed.hostname.startsWith("[") && parsed.hostname.endsWith("]")) { + // invalidState: an IPv6 literal reaches an OpenShell parser that cannot + // represent and enforce its exact proxy target safely. + // sourceBoundary: the pinned OpenShell proxy parser owns literal support. + // whyNotSourceFix: v0.0.72 does not support this target form. + // regressionTest: host/Hermes parity rejects private and public IPv6 literals. + // removalCondition: remove only with reviewed parser support and parity proof; + // never infer the capability from semver alone. + throw new McpBridgeError( + "IPv6-literal MCP server URLs are not supported by the current OpenShell proxy target parser. Use a DNS hostname with public A/AAAA records.", + 2, + ); + } + if (parsed.username || parsed.password) { + throw new McpBridgeError( + "MCP server URL must not embed credentials. Use --env KEY so OpenShell resolves host-only credentials.", + 2, + ); + } + if (rawUrl.includes("?") || parsed.search) { + throw new McpBridgeError( + "MCP server URLs must not include a query string because URLs are persisted and displayed. Put credentials in --env and use a stable endpoint path.", + 2, + ); + } + if (rawUrl.includes("#") || parsed.hash) { + throw new McpBridgeError( + "MCP server URLs must not include a fragment because fragments are not sent to the server.", + 2, + ); + } + if (parsed.port === "0") { + throw new McpBridgeError("MCP server URL port must be between 1 and 65535.", 2); + } + if ( + rawUrl.includes("%") || + parsed.pathname.includes("%") || + rawUrl.includes("\\") || + /\/{2,}/.test(parsed.pathname) || + /[\*\[\]\{\};]/.test(parsed.pathname) + ) { + throw new McpBridgeError( + "MCP server URL paths must be literal and canonical; percent characters, backslashes, semicolons, and glob metacharacters are not supported.", + 2, + ); + } + if (hasSecretShapedMcpPathSegment(parsed.pathname)) { + throw new McpBridgeError( + "MCP server URL paths must not contain secret-shaped credential material because the full URL is persisted and displayed. Put the bearer credential in --env KEY.", + 2, + ); + } + rejectUnsupportedOpenShellMcpHostAlias(parsed.hostname); + validateMcpServerUrlTarget(parsed); + if (parsed.hostname.endsWith(".")) { + throw new McpBridgeError( + "MCP server URL hostnames must use canonical spelling without a trailing dot.", + 2, + ); + } + if (!parsed.pathname) parsed.pathname = "/"; + const normalized = parsed.toString(); + if (normalized.length > MCP_SERVER_URL_MAX_LENGTH) { + throw new McpBridgeError( + `MCP server URL must be at most ${MCP_SERVER_URL_MAX_LENGTH} characters after normalization.`, + 2, + ); + } + return normalized; +} + +export async function validateMcpServerUrlResolvedTarget(parsed: URL): Promise { + // invalidState: a hostname is public at add time but later rebinds to an + // unpinned address. sourceBoundary: NemoClaw pins the add-time public answers; + // OpenShell v0.0.72 resolves, validates every answer against allowed_ips, and + // connects with that same SocketAddr list. whyNotSourceFix: duplicating DNS + // resolution here before each remote connection would create a second, + // non-authoritative TOCTOU boundary outside OpenShell's data plane. + // regressionTest: e2e/support/mcp-bridge-sandbox.test.ts pins the exact + // upstream source contract, and live/mcp-bridge.test.ts remaps DNS and proves + // a 403 plus zero upstream requests for all three adapters. + // removalCondition: revisit only when the pinned OpenShell implementation or + // its allowed_ips resolve-validate-connect contract changes. + rejectUnsupportedOpenShellMcpHostAlias(parsed.hostname); + if (isBlockedMcpUrlTargetHost(parsed.hostname)) { + validateMcpServerUrlTarget(parsed); + } + let addresses: Array<{ address: string }>; + try { + addresses = await resolveHostAddresses(parsed.hostname); + } catch (error) { + const detail = error instanceof Error && error.message ? ` ${error.message}` : ""; + throw new McpBridgeError( + `MCP server URL host '${parsed.hostname}' could not be resolved before policy registration.${detail}`, + 2, + ); + } + if (addresses.length === 0) { + throw new McpBridgeError( + `MCP server URL host '${parsed.hostname}' resolved without any addresses before policy registration.`, + 2, + ); + } + for (const { address } of addresses) { + if (isBlockedMcpUrlTargetHost(address)) { + throw new McpBridgeError( + `MCP server URL host '${parsed.hostname}' resolves to private, local, or special-use address '${address}'. Use a normal HTTPS DNS endpoint with public address records.`, + 2, + ); + } + } + return [...new Set(addresses.map(({ address }) => address.toLowerCase()))].sort(); +} + +export function parseMcpUrl(rawUrl: string): URL { + return new URL(normalizeMcpServerUrl(rawUrl)); +} diff --git a/src/lib/actions/sandbox/mcp-bridge-validation.ts b/src/lib/actions/sandbox/mcp-bridge-validation.ts new file mode 100644 index 00000000000..6fff7bceef0 --- /dev/null +++ b/src/lib/actions/sandbox/mcp-bridge-validation.ts @@ -0,0 +1,277 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import crypto from "node:crypto"; + +import type { McpBridgeEntry } from "../../state/registry"; +import { isSubprocessEnvNameAllowed } from "../../subprocess-env"; +import { + McpBridgeError, + type ParsedEnvReference, + type ParsedMcpAddArgs, +} from "./mcp-bridge-contracts"; +import { normalizeMcpServerUrl } from "./mcp-bridge-url-validation"; +// This static import is intentionally fail-closed: TypeScript/build packaging +// must reject a missing or malformed security manifest instead of letting the +// CLI start with a weakened credential-name denylist. Input, package, image, +// and workflow contracts pin its structure, installed path, and version. +import childVisibleCredentialManifest from "./openshell-child-visible-credentials.v0.0.72.json"; + +export { + MCP_SERVER_URL_MAX_LENGTH, + normalizeMcpServerUrl, + parseMcpUrl, + validateMcpServerUrlResolvedTarget, +} from "./mcp-bridge-url-validation"; + +const VALID_SERVER_RE = /^[A-Za-z][A-Za-z0-9_-]{0,63}$/; +const VALID_ENV_RE = /^[A-Za-z_][A-Za-z0-9_]{0,127}$/; +const VALID_SANDBOX_RE = /^[a-z0-9]([a-z0-9-]*[a-z0-9])?$/; +// invalidState: an MCP bearer name aliases a child-visible or process-control +// key and exposes or executes the provider value outside the intended request. +// sourceBoundary: the versioned JSON manifest pins OpenShell-owned keys to the +// shipped source commit; NemoClaw owns host and agent runtime-control rejects. +// whyNotSourceFix: v0.0.72 exposes provider keys to every fresh sandbox exec +// and does not advertise safe credential-name capabilities at runtime. +// regressionTest: the mcp-bridge-input validation/runtime suites check every +// pinned and runtime key; package contracts require version alignment. +// removalCondition: replace these rejects when OpenShell offers endpoint-only +// credentials plus a machine-readable child-environment capability manifest. +const OPENSHELL_RAW_CHILD_ENV_KEYS = new Set(childVisibleCredentialManifest.rawChildValueKeys); +const OPENSHELL_REWRITTEN_CHILD_ENV_KEYS = new Set( + childVisibleCredentialManifest.rewrittenChildValueKeys, +); +// OpenShell attaches provider keys to every fresh sandbox exec. A placeholder +// under one of these names can alter a loader, shell, or supported agent +// runtime before the requested command starts (for example, PYTHONHOME makes +// Python fail during initialization). Require operators to use a dedicated +// service credential alias instead of a process-control name. +const SANDBOX_RUNTIME_CONTROL_ENV_KEYS = new Set(childVisibleCredentialManifest.runtimeControlKeys); +const SANDBOX_RUNTIME_CONTROL_ENV_PREFIXES = childVisibleCredentialManifest.runtimeControlPrefixes; +const MCP_PROVIDER_HASH_BYTES = 8; +export function validateSandboxName(name: string): void { + if (!name || name.length > 63 || !VALID_SANDBOX_RE.test(name)) { + throw new McpBridgeError( + `Invalid sandbox name '${name}'. Names must be 1-63 lowercase alphanumeric characters with optional internal hyphens.`, + 2, + ); + } +} + +export function validateMcpServerName(name: string): void { + if (!VALID_SERVER_RE.test(name)) { + throw new McpBridgeError( + `Invalid MCP server name '${name}'. Names must start with a letter and contain only letters, digits, hyphens, and underscores.`, + 2, + ); + } +} + +export function validateMcpCredentialEnvName(name: string): void { + validatePersistedMcpCredentialEnvName(name); + if (isSubprocessEnvNameAllowed(name)) { + throw new McpBridgeError( + `MCP credential environment name '${name}' is reserved for host subprocess control and could be forwarded outside the provider mutation. Use a dedicated secret name such as MY_SERVICE_MCP_TOKEN.`, + 2, + ); + } + if (OPENSHELL_RAW_CHILD_ENV_KEYS.has(name)) { + throw new McpBridgeError( + `MCP credential environment name '${name}' is materialized as a raw child-process value by OpenShell's Google Cloud compatibility path. Use a distinct secret name to preserve the host-only credential boundary.`, + 2, + ); + } + if (OPENSHELL_REWRITTEN_CHILD_ENV_KEYS.has(name)) { + throw new McpBridgeError( + `MCP credential environment name '${name}' is rewritten by OpenShell's Google Cloud metadata compatibility path. Use a distinct secret name so credential attachment remains deterministic.`, + 2, + ); + } + if ( + SANDBOX_RUNTIME_CONTROL_ENV_KEYS.has(name) || + SANDBOX_RUNTIME_CONTROL_ENV_PREFIXES.some((prefix) => name.startsWith(prefix)) + ) { + throw new McpBridgeError( + `MCP credential environment name '${name}' is reserved for sandbox runtime control and could alter or prevent agent commands. Use a dedicated secret name such as MY_SERVICE_MCP_TOKEN.`, + 2, + ); + } +} + +/** Validate syntax only for cleanup of durable entries created by older builds. */ +export function validatePersistedMcpCredentialEnvName(name: string): void { + if (!VALID_ENV_RE.test(name)) { + throw new McpBridgeError( + `Invalid environment variable name '${name}'. Names must match [A-Za-z_][A-Za-z0-9_]*.`, + 2, + ); + } +} + +export function parseMcpAddArgs(argv: string[]): ParsedMcpAddArgs { + const env: ParsedEnvReference[] = []; + let server = ""; + let url = ""; + + for (let i = 0; i < argv.length; i++) { + const token = argv[i]; + if (token === "--") { + throw new McpBridgeError( + "Host stdio MCP commands are not supported. Use --url so OpenShell can enforce MCP traffic and provider credentials.", + 2, + ); + } + if (token === "--env" || token === "-e") { + const raw = argv[++i] ?? ""; + const eq = raw.indexOf("="); + const name = eq >= 0 ? raw.slice(0, eq) : raw; + validateMcpCredentialEnvName(name); + if (eq >= 0) { + throw new McpBridgeError( + "Inline --env KEY=VALUE is not accepted because it exposes the secret in the NemoClaw process arguments and shell history. Export KEY, then pass --env KEY.", + 2, + ); + } + env.push({ name }); + continue; + } + if (token?.startsWith("--env=")) { + const raw = token.slice("--env=".length); + const eq = raw.indexOf("="); + const name = eq >= 0 ? raw.slice(0, eq) : raw; + validateMcpCredentialEnvName(name); + if (eq >= 0) { + throw new McpBridgeError( + "Inline --env KEY=VALUE is not accepted because it exposes the secret in the NemoClaw process arguments and shell history. Export KEY, then pass --env KEY.", + 2, + ); + } + env.push({ name }); + continue; + } + if (token === "--url") { + url = normalizeMcpServerUrl(argv[++i] ?? ""); + continue; + } + if (token?.startsWith("--url=")) { + url = normalizeMcpServerUrl(token.slice("--url=".length)); + continue; + } + if (token?.startsWith("-")) { + throw new McpBridgeError(`Unknown mcp add option: ${token}`, 2); + } + if (!server) { + server = token ?? ""; + validateMcpServerName(server); + continue; + } + throw new McpBridgeError( + "Usage: nemoclaw mcp add --url --env KEY", + 2, + ); + } + + if (!server) { + throw new McpBridgeError( + "Usage: nemoclaw mcp add --url --env KEY", + 2, + ); + } + if (!url) { + throw new McpBridgeError("MCP server URL is required. Pass --url .", 2); + } + if (env.length !== 1) { + throw new McpBridgeError( + "Authenticated MCP requires exactly one --env KEY bearer credential reference.", + 2, + ); + } + + return { server, url, env }; +} + +export function uniqueEnvNames(env: readonly ParsedEnvReference[] | readonly string[]): string[] { + const names = env.map((entry) => (typeof entry === "string" ? entry : entry.name)); + return [...new Set(names)]; +} + +export function assertAuthenticatedCredentialReference(env: readonly ParsedEnvReference[]): void { + if (env.length !== 1) { + throw new McpBridgeError( + "Authenticated MCP requires exactly one --env KEY bearer credential reference.", + 2, + ); + } + validateMcpCredentialEnvName(env[0].name); +} + +export function assertPersistedAuthenticatedBridgeEntry(entry: McpBridgeEntry): void { + if (!Array.isArray(entry.env) || entry.env.length !== 1 || !entry.providerName) { + throw new McpBridgeError( + `MCP server '${entry.server}' has no complete authenticated credential binding. Remove it with --force, then add it again with --env KEY.`, + 2, + ); + } + validatePersistedMcpCredentialEnvName(entry.env[0]); +} + +export function assertAuthenticatedBridgeEntry(entry: McpBridgeEntry): void { + assertPersistedAuthenticatedBridgeEntry(entry); + validateMcpCredentialEnvName(entry.env[0]); +} + +/** + * Read values only for local display redaction while cleaning legacy state. + * Never pass this map to a subprocess environment or provider mutation. + */ +export function resolvePersistedCredentialEnvForRedaction( + envNames: readonly string[], +): Record { + const resolved: Record = {}; + for (const name of envNames) { + validatePersistedMcpCredentialEnvName(name); + const value = process.env[name]; + if (value !== undefined && value !== "") resolved[name] = value; + } + return resolved; +} + +export function resolveCredentialEnv(env: readonly ParsedEnvReference[]): Record { + const resolved: Record = {}; + for (const entry of env) { + validateMcpCredentialEnvName(entry.name); + const value = entry.value ?? process.env[entry.name]; + if (value !== undefined && value !== "") { + resolved[entry.name] = value; + } + } + return resolved; +} + +export function buildMcpBridgeProviderName( + sandboxName: string, + server: string, + instanceId?: string, +): string { + validateSandboxName(sandboxName); + validateMcpServerName(server); + if (instanceId !== undefined && !/^[a-f0-9]{16}$/.test(instanceId)) { + throw new McpBridgeError("Invalid MCP provider instance ID."); + } + const serverSlug = server + .toLowerCase() + .replace(/_/g, "-") + .replace(/[^a-z0-9-]/g, "-"); + const rawBase = `${sandboxName}-mcp-${server}${instanceId ? `-${instanceId}` : ""}`; + const base = `${sandboxName}-mcp-${serverSlug}${instanceId ? `-${instanceId}` : ""}` + .replace(/-+/g, "-") + .replace(/^-|-$/g, ""); + if (base.length <= 63 && base === rawBase) return base; + const hash = crypto + .createHash("sha256") + .update(`${sandboxName}:${server}:${instanceId ?? "stable"}`) + .digest("hex") + .slice(0, MCP_PROVIDER_HASH_BYTES * 2); + const suffix = `-${hash}`; + return `${base.slice(0, 63 - suffix.length).replace(/-+$/g, "")}${suffix}`; +} diff --git a/src/lib/actions/sandbox/mcp-bridge.ts b/src/lib/actions/sandbox/mcp-bridge.ts new file mode 100644 index 00000000000..f56d7965ef9 --- /dev/null +++ b/src/lib/actions/sandbox/mcp-bridge.ts @@ -0,0 +1,318 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { McpBridgeEntry } from "../../state/registry"; +import { addMcpBridge as addMcpBridgeLifecycle } from "./mcp-bridge-add-restart"; +import { type McpBridgeAddOptions, McpBridgeError } from "./mcp-bridge-contracts"; +import { + finalizeMcpBridgesAfterSandboxDelete as finalizeMcpBridgesAfterSandboxDeleteLifecycle, + prepareMcpBridgesForAbsentSandboxDestroy as prepareMcpBridgesForAbsentSandboxDestroyLifecycle, + prepareMcpBridgesForDestroy as prepareMcpBridgesForDestroyLifecycle, + restoreMcpBridgesAfterDestroyAbort as restoreMcpBridgesAfterDestroyAbortLifecycle, +} from "./mcp-bridge-destroy"; +import { redactBridgeSecretsForDisplay } from "./mcp-bridge-output"; +import { + prepareMcpBridgesForAbsentSandboxRebuild as prepareMcpBridgesForAbsentSandboxRebuildLifecycle, + prepareMcpBridgesForRebuild as prepareMcpBridgesForRebuildLifecycle, + reattachMcpProvidersAfterRebuildAbort as reattachMcpProvidersAfterRebuildAbortLifecycle, + restoreMcpBridgesAfterRebuild as restoreMcpBridgesAfterRebuildLifecycle, +} from "./mcp-bridge-rebuild"; +import { removeMcpBridge as removeMcpBridgeLifecycle } from "./mcp-bridge-remove"; +import { renderMcpBridgeList, renderMcpBridgeStatus } from "./mcp-bridge-render"; +import { restartMcpBridge as restartMcpBridgeLifecycle } from "./mcp-bridge-restart"; +import { getSandboxAgent, getSandboxOrThrow } from "./mcp-bridge-state"; +import { buildJsonSummary, statusMcpBridge } from "./mcp-bridge-status"; +import { parseMcpAddArgs } from "./mcp-bridge-validation"; + +export { + buildDeepAgentsMcpRegisterCommand, + buildDeepAgentsMcpRemoveCommand, + buildDeepAgentsMcpStatusCommand, + buildHermesMcpExecArgs, + buildHermesMcpProbeCommand, + buildHermesMcpRegisterCommand, + buildOpenClawMcporterInspectCommand, + buildOpenClawMcporterRegisterCommand, + buildOpenClawMcporterRemoveCommand, + DEEPAGENTS_MCP_CONFIG_PATH, + MCPORTER_VERSION, + mcporterHeadersMatchExpected, + parseAdapterRegistrationInspection, +} from "./mcp-bridge-adapters"; +export type { + McpBridgeAddOptions, + McpBridgeStatus, + ParsedEnvReference, + ParsedMcpAddArgs, +} from "./mcp-bridge-contracts"; +export { MCP_BRIDGE_POLICY_SOURCE, McpBridgeError } from "./mcp-bridge-contracts"; +export { + redactBridgeSecretsForDisplay, + redactCredentialValuesForDisplay, +} from "./mcp-bridge-output"; +export { + buildMcpBridgePolicyName, + buildMcpBridgePolicyYaml, + MCP_BRIDGE_ALLOWED_METHODS, + MCP_BRIDGE_POLICY_MAX_BODY_BYTES, +} from "./mcp-bridge-policy"; +export { + buildMcpBridgeProviderArgs, + buildMcpCredentialRevisionObservationCommand, + detachMissingProviderReference, + parseMcpProviderAttachmentNames, + parseMcpProviderMetadata, + providerDetachChangedState, +} from "./mcp-bridge-provider"; +export { + buildMcpBridgeProviderName, + MCP_SERVER_URL_MAX_LENGTH, + normalizeMcpServerUrl, + parseMcpAddArgs, + resolveCredentialEnv, + validateMcpCredentialEnvName, + validateMcpServerName, +} from "./mcp-bridge-validation"; +export { statusMcpBridge }; + +export interface McpDestroyPreparation { + entries: McpBridgeEntry[]; + detachedProviderEntries: McpBridgeEntry[]; + scrubbedAdapterEntries: McpBridgeEntry[]; + /** True when phase one was completed by an earlier destroy process. */ + destroyAlreadyPrepared: boolean; + /** True when a previous destroy already confirmed the sandbox was absent. */ + destroyAlreadyPending: boolean; +} + +export interface McpRebuildPreparation { + entries: McpBridgeEntry[]; + detachedProviderEntries: McpBridgeEntry[]; + scrubbedAdapterEntries: McpBridgeEntry[]; +} + +export async function addMcpBridge( + sandboxName: string, + options: McpBridgeAddOptions, +): Promise { + return addMcpBridgeLifecycle(sandboxName, options); +} + +export async function restartMcpBridge(sandboxName: string, server?: string): Promise { + return restartMcpBridgeLifecycle(sandboxName, server); +} + +export async function removeMcpBridge( + sandboxName: string, + server: string, + options: { force?: boolean; allowResidual?: boolean } = {}, +): Promise { + return removeMcpBridgeLifecycle(sandboxName, server, options); +} + +export async function prepareMcpBridgesForAbsentSandboxDestroy( + sandboxName: string, + options: { force?: boolean } = {}, +): Promise { + return prepareMcpBridgesForAbsentSandboxDestroyLifecycle(sandboxName, options); +} + +export async function prepareMcpBridgesForDestroy( + sandboxName: string, +): Promise { + return prepareMcpBridgesForDestroyLifecycle(sandboxName); +} + +export async function restoreMcpBridgesAfterDestroyAbort( + sandboxName: string, + preparation: McpDestroyPreparation, +): Promise { + return restoreMcpBridgesAfterDestroyAbortLifecycle(sandboxName, preparation); +} + +export async function finalizeMcpBridgesAfterSandboxDelete( + sandboxName: string, + preparation: McpDestroyPreparation, + options: { force?: boolean } = {}, +): Promise { + return finalizeMcpBridgesAfterSandboxDeleteLifecycle(sandboxName, preparation, options); +} + +export async function prepareMcpBridgesForAbsentSandboxRebuild( + sandboxName: string, +): Promise { + return prepareMcpBridgesForAbsentSandboxRebuildLifecycle(sandboxName); +} + +export async function prepareMcpBridgesForRebuild( + sandboxName: string, +): Promise { + return prepareMcpBridgesForRebuildLifecycle(sandboxName); +} + +export async function reattachMcpProvidersAfterRebuildAbort( + sandboxName: string, + entries: readonly McpBridgeEntry[], + scrubbedAdapterEntries: readonly McpBridgeEntry[] = [], +): Promise { + return reattachMcpProvidersAfterRebuildAbortLifecycle( + sandboxName, + entries, + scrubbedAdapterEntries, + ); +} + +export async function restoreMcpBridgesAfterRebuild( + sandboxName: string, + entries: readonly McpBridgeEntry[], +): Promise { + return restoreMcpBridgesAfterRebuildLifecycle(sandboxName, entries); +} + +function parseJsonFlag(args: string[]): { json: boolean; rest: string[] } { + return { + json: args.includes("--json"), + rest: args.filter((arg) => arg !== "--json"), + }; +} + +function requireNoExtraArgs(args: string[], usage: string): void { + if (args.length > 0) throw new McpBridgeError(usage, 2); +} + +function requireAtMostOneArg(args: string[], usage: string): string | undefined { + if (args.length > 1) throw new McpBridgeError(usage, 2); + return args[0]; +} + +function hasHelpFlag(args: readonly string[]): boolean { + return args.includes("--help") || args.includes("-h"); +} + +function renderMcpHelp(subcommand: string): void { + switch (subcommand) { + case "add": + console.log(`USAGE + nemoclaw mcp add --url --env KEY + +FLAGS + --url URL MCP Streamable HTTP endpoint + --env KEY Required host credential reference registered with OpenShell + +SECURITY + Credentials are registered as an OpenShell provider and appear inside the + sandbox only as openshell:resolve:env:KEY placeholders. OpenShell resolves + them at egress while enforcing the generated protocol: mcp policy.`); + return; + case "list": + console.log(`USAGE + nemoclaw mcp list [--json] + +FLAGS + --json Emit sandbox, support, and MCP server state as JSON`); + return; + case "status": + console.log(`USAGE + nemoclaw mcp status [server] [--json] + +FLAGS + --json Emit MCP server status as JSON`); + return; + case "restart": + console.log(`USAGE + nemoclaw mcp restart [server]`); + return; + case "remove": + console.log(`USAGE + nemoclaw mcp remove [--force] + +FLAGS + --force Best-effort owned cleanup; preserves registry state when residuals remain`); + return; + default: + console.log(`USAGE + nemoclaw mcp [args...]`); + } +} + +export async function dispatchMcpBridgeCommand( + sandboxName: string, + actionArgs: string[], +): Promise { + const [subcommand = "list", ...rest] = actionArgs; + try { + if (subcommand === "--help" || subcommand === "-h") { + renderMcpHelp("mcp"); + return; + } + if (hasHelpFlag(rest)) { + renderMcpHelp(subcommand); + return; + } + switch (subcommand) { + case "add": { + const options = parseMcpAddArgs(rest); + await addMcpBridge(sandboxName, options); + console.log(` MCP server '${options.server}' added to sandbox '${sandboxName}'.`); + return; + } + case "list": { + const { json, rest: listRest } = parseJsonFlag(rest); + requireNoExtraArgs(listRest, "Usage: nemoclaw mcp list [--json]"); + const sandbox = getSandboxOrThrow(sandboxName); + const agent = getSandboxAgent(sandbox); + const statuses = await statusMcpBridge(sandboxName); + if (json) + console.log(JSON.stringify(buildJsonSummary(sandboxName, agent, statuses), null, 2)); + else renderMcpBridgeList(sandboxName, statuses, agent); + return; + } + case "status": { + const { json, rest: statusRest } = parseJsonFlag(rest); + const server = requireAtMostOneArg( + statusRest, + "Usage: nemoclaw mcp status [server] [--json]", + ); + const sandbox = getSandboxOrThrow(sandboxName); + const agent = getSandboxAgent(sandbox); + const statuses = await statusMcpBridge(sandboxName, server); + if (json) { + console.log( + JSON.stringify( + server ? statuses[0] : buildJsonSummary(sandboxName, agent, statuses), + null, + 2, + ), + ); + } else renderMcpBridgeStatus(sandboxName, statuses, agent); + return; + } + case "restart": { + const server = requireAtMostOneArg(rest, "Usage: nemoclaw mcp restart [server]"); + await restartMcpBridge(sandboxName, server); + return; + } + case "remove": { + const force = rest.includes("--force"); + const names = rest.filter((arg) => arg !== "--force"); + const server = names[0]; + if (!server || names.length > 1) + throw new McpBridgeError("Usage: nemoclaw mcp remove [--force]", 2); + await removeMcpBridge(sandboxName, server, { force }); + return; + } + default: + throw new McpBridgeError( + "Usage: nemoclaw mcp [args...]", + 2, + ); + } + } catch (error) { + if (error instanceof McpBridgeError) { + console.error(` ${redactBridgeSecretsForDisplay(error.message)}`); + process.exitCode = error.exitCode; + return; + } + throw error; + } +} diff --git a/src/lib/actions/sandbox/openshell-child-visible-credentials.v0.0.72.json b/src/lib/actions/sandbox/openshell-child-visible-credentials.v0.0.72.json new file mode 100644 index 00000000000..7dd671d2f68 --- /dev/null +++ b/src/lib/actions/sandbox/openshell-child-visible-credentials.v0.0.72.json @@ -0,0 +1,108 @@ +{ + "openshellVersion": "0.0.72", + "openshellCommit": "8cb16de9eae4c44d7d31e1493747d8c10abb5963", + "sources": [ + "crates/openshell-core/src/google_cloud.rs", + "crates/openshell-core/src/provider_credentials.rs" + ], + "nemoclawSources": [ + "src/lib/subprocess-env.ts", + "src/lib/actions/sandbox/mcp-bridge-validation.ts", + "agents/hermes/mcp-config-transaction.py" + ], + "rawChildValueKeys": [ + "GCP_PROJECT_ID", + "GOOGLE_CLOUD_PROJECT", + "CLOUD_ML_REGION", + "GCP_LOCATION", + "GCP_SERVICE_ACCOUNT_EMAIL", + "GOOSE_PROVIDER", + "ANTHROPIC_VERTEX_PROJECT_ID", + "VERTEX_LOCATION" + ], + "rewrittenChildValueKeys": [ + "GCE_METADATA_HOST", + "GCE_METADATA_IP", + "METADATA_SERVER_DETECTION" + ], + "runtimeControlKeys": [ + "_JAVA_OPTIONS", + "ALL_PROXY", + "all_proxy", + "API_SERVER_KEY", + "BASH_ENV", + "BASHOPTS", + "CDPATH", + "CLASSPATH", + "CONDA_PREFIX", + "CURL_CA_BUNDLE", + "DENO_CERT", + "DOCKER_HOST", + "ENV", + "GCONV_PATH", + "GIT_SSL_CAINFO", + "GIT_SSL_CAPATH", + "GLOBIGNORE", + "grpc_proxy", + "HOME", + "HOSTNAME", + "HTTP_PROXY", + "http_proxy", + "HTTPS_PROXY", + "https_proxy", + "IFS", + "KUBECONFIG", + "LANG", + "LOCPATH", + "LOGNAME", + "NLSPATH", + "NODE_ENV", + "NODE_EXTRA_CA_CERTS", + "NO_PROXY", + "no_proxy", + "PATH", + "PROMPT_COMMAND", + "PS4", + "REQUESTS_CA_BUNDLE", + "RUST_BACKTRACE", + "RUST_LOG", + "SHELL", + "SHELLOPTS", + "SSH_AUTH_SOCK", + "SSL_CERT_DIR", + "SSL_CERT_FILE", + "TEMP", + "TERM", + "TMP", + "TMPDIR", + "USER", + "VIRTUAL_ENV", + "ZDOTDIR" + ], + "runtimeControlPrefixes": [ + "DEEPAGENTS_", + "DYLD_", + "GATEWAY_", + "GLIBC_", + "GRPC_", + "HERMES_", + "JAVA_", + "JDK_", + "LANGCHAIN_", + "LANGGRAPH_", + "LANGSMITH_", + "LC_", + "LD_", + "MALLOC_", + "NEMOCLAW_", + "NODE_", + "OPENAI_", + "OPENCLAW_", + "OPENSHELL_", + "PERL", + "PYTHON", + "RUBY", + "UV_", + "XDG_" + ] +} diff --git a/src/lib/actions/sandbox/policy-channel-lock.test.ts b/src/lib/actions/sandbox/policy-channel-lock.test.ts new file mode 100644 index 00000000000..0a6ce4be6d9 --- /dev/null +++ b/src/lib/actions/sandbox/policy-channel-lock.test.ts @@ -0,0 +1,42 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const lockMocks = vi.hoisted(() => ({ + withMcpLifecycleLock: vi.fn(async (_sandboxName: string, operation: () => unknown) => + operation(), + ), + withSandboxMutationLock: vi.fn(async () => undefined), +})); + +vi.mock("../../state/mcp-lifecycle-lock", () => lockMocks); + +import { + addSandboxChannel, + addSandboxPolicy, + removeSandboxChannel, + removeSandboxPolicy, + startSandboxChannel, + stopSandboxChannel, +} from "./policy-channel"; + +describe("policy and channel sandbox mutation locking", () => { + beforeEach(() => { + lockMocks.withSandboxMutationLock.mockClear(); + }); + + it.each([ + ["policy add", () => addSandboxPolicy("alpha")], + ["policy remove", () => removeSandboxPolicy("alpha")], + ["channel add", () => addSandboxChannel("alpha")], + ["channel remove", () => removeSandboxChannel("alpha")], + ["channel start", () => startSandboxChannel("alpha")], + ["channel stop", () => stopSandboxChannel("alpha")], + ])("routes %s through the shared per-sandbox lock", async (_label, action) => { + await action(); + + expect(lockMocks.withSandboxMutationLock).toHaveBeenCalledOnce(); + expect(lockMocks.withSandboxMutationLock).toHaveBeenCalledWith("alpha", expect.any(Function)); + }); +}); diff --git a/src/lib/actions/sandbox/policy-channel.ts b/src/lib/actions/sandbox/policy-channel.ts index 39faa945ba2..6c7c4e10b5a 100644 --- a/src/lib/actions/sandbox/policy-channel.ts +++ b/src/lib/actions/sandbox/policy-channel.ts @@ -59,6 +59,7 @@ import { knownChannelNames, persistChannelTokens, } from "../../sandbox/channels"; +import { withSandboxMutationLock } from "../../state/mcp-lifecycle-lock"; import * as registry from "../../state/registry"; import { isDockerRuntimeDown, printDockerRuntimeDownGuidance } from "./gateway-failure-classifier"; import { ensureMessagingHostForwardAfterRebuild } from "./messaging-host-forward-lifecycle"; @@ -94,6 +95,13 @@ const YW = useColor ? "\x1b[1;33m" : ""; export async function addSandboxPolicy( sandboxName: string, options: PolicyAddOptions = {}, +): Promise { + return withSandboxMutationLock(sandboxName, () => addSandboxPolicyUnlocked(sandboxName, options)); +} + +async function addSandboxPolicyUnlocked( + sandboxName: string, + options: PolicyAddOptions, ): Promise { const { dryRun, skipConfirm, source, presetArg } = parsePolicyAddOptions(options); @@ -916,6 +924,15 @@ function safeLoadOnboardSession(): ReturnType export async function addSandboxChannel( sandboxName: string, options: ChannelMutationOptions = {}, +): Promise { + return withSandboxMutationLock(sandboxName, () => + addSandboxChannelUnlocked(sandboxName, options), + ); +} + +async function addSandboxChannelUnlocked( + sandboxName: string, + options: ChannelMutationOptions, ): Promise { const dryRun = Boolean(options.dryRun); const force = Boolean(options.force); @@ -1290,6 +1307,15 @@ export function removeChannelPresetIfPresent(sandboxName: string, channelName: s export async function removeSandboxChannel( sandboxName: string, options: ChannelMutationOptions = {}, +): Promise { + return withSandboxMutationLock(sandboxName, () => + removeSandboxChannelUnlocked(sandboxName, options), + ); +} + +async function removeSandboxChannelUnlocked( + sandboxName: string, + options: ChannelMutationOptions, ): Promise { const dryRun = Boolean(options.dryRun); const rawChannelArg = options.channel; @@ -1456,19 +1482,32 @@ export async function stopSandboxChannel( sandboxName: string, options: ChannelMutationOptions = {}, ): Promise { - await sandboxChannelsSetEnabled(sandboxName, options, true); + await withSandboxMutationLock(sandboxName, () => + sandboxChannelsSetEnabled(sandboxName, options, true), + ); } export async function startSandboxChannel( sandboxName: string, options: ChannelMutationOptions = {}, ): Promise { - await sandboxChannelsSetEnabled(sandboxName, options, false); + await withSandboxMutationLock(sandboxName, () => + sandboxChannelsSetEnabled(sandboxName, options, false), + ); } export async function removeSandboxPolicy( sandboxName: string, options: PolicyRemoveOptions = {}, +): Promise { + return withSandboxMutationLock(sandboxName, () => + removeSandboxPolicyUnlocked(sandboxName, options), + ); +} + +async function removeSandboxPolicyUnlocked( + sandboxName: string, + options: PolicyRemoveOptions, ): Promise { const dryRun = Boolean(options.dryRun); const skipConfirm = Boolean( diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index f3806aa36b4..e3876574430 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -22,6 +22,7 @@ import { import { createTempSshConfig } from "../../sandbox/temp-ssh-config"; import { withTimerBoundShieldsMutationLock } from "../../shields/timer-bound-lock"; import * as registry from "../../state/registry"; +import { buildSubprocessEnv } from "../../subprocess-env"; import { ensureHermesDashboardPortForwardIfEnabled, ensureSandboxPortForward, @@ -67,6 +68,10 @@ export type SandboxCommandResult = { stderr: string; }; +export type SandboxExecCommandOptions = { + allowLocalDockerFallback?: boolean; +}; + const DEFAULT_SANDBOX_EXEC_TIMEOUT_MS = 15000; type AuxiliaryRecoveryResult = { @@ -130,7 +135,12 @@ export function executeSandboxCommand( `openshell-${sandboxName}`, command, ], - { encoding: "utf-8", stdio: ["ignore", "pipe", "pipe"], timeout: 15000 }, + { + encoding: "utf-8", + env: buildSubprocessEnv(), + stdio: ["ignore", "pipe", "pipe"], + timeout: 15000, + }, ); return { status: result.status ?? 1, @@ -180,6 +190,7 @@ function executeLocalDockerSandboxCommand( try { const result = dockerSpawnSync(argv, { encoding: "utf-8", + env: buildSubprocessEnv(), stdio: ["ignore", "pipe", "pipe"], timeout, }); @@ -193,6 +204,7 @@ export function executeSandboxExecCommand( sandboxName: string, command: string, timeout = DEFAULT_SANDBOX_EXEC_TIMEOUT_MS, + options: SandboxExecCommandOptions = {}, ): SandboxCommandResult | null { const markedCommand = buildSandboxExecMarkedCommand(command); const effectiveTimeout = resolveSandboxExecTimeout(timeout); @@ -203,7 +215,7 @@ export function executeSandboxExecCommand( { cwd: ROOT, encoding: "utf-8", - env: process.env, + env: buildSubprocessEnv(), stdio: ["ignore", "pipe", "pipe"], timeout: effectiveTimeout, }, @@ -213,6 +225,7 @@ export function executeSandboxExecCommand( } catch { // OpenShell transport failed; try the trusted direct-container fallback. } + if (options.allowLocalDockerFallback === false) return null; // Keep the fallback outside the OpenShell try/catch so a fail-closed identity // refusal cannot be caught and retried against changing container state. return executeLocalDockerSandboxCommand(sandboxName, markedCommand, effectiveTimeout); diff --git a/src/lib/actions/sandbox/rebuild-backup-phase.ts b/src/lib/actions/sandbox/rebuild-backup-phase.ts new file mode 100644 index 00000000000..147b8d0af68 --- /dev/null +++ b/src/lib/actions/sandbox/rebuild-backup-phase.ts @@ -0,0 +1,90 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { SandboxMessagingPlan } from "../../messaging"; +import { type WebSearchConfig, webSearchProviderForConfig } from "../../inference/web-search"; +import { mergeRebuildMessagingPolicyPresets } from "../../onboard/messaging-policy-presets"; +import { resolveRecreatePolicyPresets } from "../../onboard/policy-preset-persistence"; +import { isStaleBuiltinWebSearchPolicyPreset } from "../../onboard/policy-selection"; +import type { RebuildBail, RebuildLog } from "./rebuild-credential-preflight"; +import { backupSandboxStateForRebuild, type RebuildSandboxEntry } from "./rebuild-flow-helpers"; + +export type RebuildBackupManifest = Exclude< + ReturnType, + undefined +>; + +export interface RebuildBackupPhaseInput { + sandboxName: string; + sandboxEntry: RebuildSandboxEntry; + staleRecovery: boolean; + preparedRecoveryManifest: RebuildBackupManifest; + messagingPlan: SandboxMessagingPlan | null; + webSearchConfig: WebSearchConfig | null; + log: RebuildLog; + bail: RebuildBail; + relockShieldsIfNeeded: (sandboxStillExists: boolean) => boolean; +} + +export interface RebuildBackupPhaseResult { + backupManifest: RebuildBackupManifest; + policyPresets: string[]; + sessionPolicyPresets: string[] | null; +} + +export function runRebuildBackupPhase( + input: RebuildBackupPhaseInput, +): RebuildBackupPhaseResult | null { + const backupManifest = + input.preparedRecoveryManifest ?? + backupSandboxStateForRebuild( + input.sandboxName, + input.sandboxEntry, + input.staleRecovery, + input.log, + input.relockShieldsIfNeeded, + input.bail, + ); + if (backupManifest === undefined) return null; + + const registryPolicyPresets = Array.isArray(input.sandboxEntry.policies) + ? input.sandboxEntry.policies.filter( + (value: unknown): value is string => typeof value === "string", + ) + : []; + const disabledChannels = [...(input.messagingPlan?.disabledChannels ?? [])]; + const enabledChannelIds = (input.messagingPlan?.channels ?? []) + .filter((channel) => !channel.disabled) + .map((channel) => channel.channelId); + const mergedPolicyPresets = mergeRebuildMessagingPolicyPresets( + backupManifest?.policyPresets, + registryPolicyPresets, + enabledChannelIds, + disabledChannels, + ); + const customPresetNames = new Set( + (input.sandboxEntry.customPolicies ?? []).map((policy) => policy.name), + ); + const policyPresets = mergedPolicyPresets.filter( + (name) => + !isStaleBuiltinWebSearchPolicyPreset(name, { + webSearchConfig: input.webSearchConfig, + customPresetNames, + }) && !(customPresetNames.has(name) && ["brave", "tavily", "nous-web"].includes(name)), + ); + if (input.webSearchConfig) { + const activePreset = webSearchProviderForConfig(input.webSearchConfig); + if (!customPresetNames.has(activePreset) && !policyPresets.includes(activePreset)) { + policyPresets.push(activePreset); + } + } + const sessionPolicyPresets = resolveRecreatePolicyPresets( + policyPresets, + input.sandboxEntry.policyPresetsFinalized === true, + (input.sandboxEntry.customPolicies?.length ?? 0) > 0, + {}, + true, + ).policyPresets; + + return { backupManifest, policyPresets, sessionPolicyPresets }; +} diff --git a/src/lib/actions/sandbox/rebuild-config-hash.ts b/src/lib/actions/sandbox/rebuild-config-hash.ts new file mode 100644 index 00000000000..e55c9cfdb9c --- /dev/null +++ b/src/lib/actions/sandbox/rebuild-config-hash.ts @@ -0,0 +1,44 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { R, YW } from "../../cli/terminal-style"; +import { shellQuote } from "../../runner"; +import { redact } from "../../security/redact"; +import { executeSandboxCommand } from "./process-recovery"; + +export function buildRefreshMutableOpenClawConfigHashCommand( + configDir = "/sandbox/.openclaw", +): string { + return [ + `config_dir=${shellQuote(configDir)}`, + 'config_file="${config_dir}/openclaw.json"', + 'hash_file="${config_dir}/.config-hash"', + '[ -d "$config_dir" ] || exit 0', + '[ ! -L "$config_dir" ] || { echo "refusing symlinked OpenClaw config dir: $config_dir" >&2; exit 10; }', + '[ ! -L "$config_file" ] || { echo "refusing symlinked OpenClaw config file: $config_file" >&2; exit 11; }', + '[ ! -L "$hash_file" ] || { echo "refusing symlinked OpenClaw config hash: $hash_file" >&2; exit 12; }', + 'owner="$(stat -c "%U" "$config_dir" 2>/dev/null || echo unknown)"', + '[ "$owner" != "root" ] || exit 0', + '[ -f "$config_file" ] || exit 0', + 'cd "$config_dir" || exit 13', + "sha256sum openclaw.json > .config-hash", + "chmod 660 .config-hash 2>/dev/null || true", + ].join("; "); +} + +export function refreshMutableOpenClawConfigHashAfterPostRestoreWrites( + sandboxName: string, + log: (msg: string) => void, +): boolean { + const result = executeSandboxCommand(sandboxName, buildRefreshMutableOpenClawConfigHashCommand()); + if (result && result.status === 0) { + log("Mutable OpenClaw config hash refreshed after post-restore config writes"); + return true; + } + + const detail = result + ? [result.stderr, result.stdout].filter(Boolean).join("; ") || `exit ${result.status}` + : "could not obtain sandbox SSH config"; + console.error(` ${YW}⚠${R} Mutable OpenClaw config hash was not refreshed: ${redact(detail)}`); + return false; +} diff --git a/src/lib/actions/sandbox/rebuild-credential-preflight.ts b/src/lib/actions/sandbox/rebuild-credential-preflight.ts new file mode 100644 index 00000000000..0b4249ae648 --- /dev/null +++ b/src/lib/actions/sandbox/rebuild-credential-preflight.ts @@ -0,0 +1,199 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { runOpenshell } from "../../adapters/openshell/runtime"; +import { CLI_NAME } from "../../cli/branding"; +import { R, RD } from "../../cli/terminal-style"; +import type { RebuildSandboxEntry } from "./rebuild-flow-helpers"; +import { + checkRebuildGatewayProviderOrBail, + shouldVerifyRebuildGatewayProvider, +} from "./rebuild-provider-preflight"; +import { getRebuildCredentialEnvFromRegistry } from "./rebuild-resume-config"; + +const onboardModule = require("../../onboard") as { + hydrateCredentialEnv: (name: string) => string | null; +}; +const hermesProviderAuth = require("../../hermes-provider-auth") as { + HERMES_PROVIDER_NAME: string; + HERMES_INFERENCE_CREDENTIAL_ENV: string; + HERMES_NOUS_API_KEY_CREDENTIAL_ENV: string; + inspectHermesProviderBinding: (runOpenshellFn: typeof runOpenshell) => { + exists: boolean; + credentialKeys: string[] | null; + }; + registerHermesInferenceProvider: ( + apiKey: string, + runOpenshellFn: typeof runOpenshell, + credentialEnv?: string, + baseUrl?: string, + ) => void; +}; + +export type RebuildBail = (message: string, code?: number) => never; +export type RebuildLog = (message: string) => void; + +function normalizeHermesRebuildAuthMethod(value: unknown): "oauth" | "api_key" | null { + const normalized = String(value || "") + .trim() + .toLowerCase() + .replace(/[\s-]+/g, "_"); + if (!normalized) return null; + if (normalized === "oauth" || normalized === "nous_oauth" || normalized === "nous_portal_oauth") { + return "oauth"; + } + if ( + normalized === "api" || + normalized === "key" || + normalized === "api_key" || + normalized === "apikey" || + normalized === "nous_api_key" + ) { + return "api_key"; + } + return null; +} + +function nonEmptyString(value: unknown): string | null { + const normalized = String(value || "").trim(); + return normalized || null; +} + +function preflightHermesProviderCredentials( + persistedAuthMethod: unknown, + credentialEnv: string | null, + log: RebuildLog, +): boolean { + const authMethod = + normalizeHermesRebuildAuthMethod(persistedAuthMethod) || + (credentialEnv === hermesProviderAuth.HERMES_NOUS_API_KEY_CREDENTIAL_ENV ? "api_key" : null); + const expectedCredentialEnv = + authMethod === "api_key" + ? hermesProviderAuth.HERMES_NOUS_API_KEY_CREDENTIAL_ENV + : hermesProviderAuth.HERMES_INFERENCE_CREDENTIAL_ENV; + const binding = hermesProviderAuth.inspectHermesProviderBinding(runOpenshell); + + if (binding.exists) { + const matches = + binding.credentialKeys?.length === 1 && binding.credentialKeys[0] === expectedCredentialEnv; + if (matches) { + log("Hermes Provider rebuild preflight: credential binding matches"); + return true; + } + log("Hermes Provider rebuild preflight: credential binding does not match"); + console.error(""); + console.error( + ` ${RD}Rebuild preflight failed:${R} the shared Hermes Provider credential binding has changed.`, + ); + console.error( + " Expected exactly the credential binding recorded for this sandbox; re-run Hermes onboarding to reconcile it.", + ); + console.error(" Sandbox is untouched — no data was lost."); + return false; + } + + if (authMethod === "api_key") { + const envKey = nonEmptyString( + process.env[hermesProviderAuth.HERMES_NOUS_API_KEY_CREDENTIAL_ENV], + ); + log( + `Hermes Provider rebuild preflight: OpenShell provider missing; API key env=${envKey ? "present" : "missing"}`, + ); + if (envKey) { + try { + console.log( + " Hermes Provider is not registered in OpenShell; registering it from the configured exported API-key environment variable before rebuild.", + ); + hermesProviderAuth.registerHermesInferenceProvider( + envKey, + runOpenshell, + hermesProviderAuth.HERMES_NOUS_API_KEY_CREDENTIAL_ENV, + ); + const registered = hermesProviderAuth.inspectHermesProviderBinding(runOpenshell); + return ( + registered.credentialKeys?.length === 1 && + registered.credentialKeys[0] === hermesProviderAuth.HERMES_NOUS_API_KEY_CREDENTIAL_ENV + ); + } catch (err) { + log( + `Hermes Provider rebuild preflight: failed to register OpenShell provider: ${ + err instanceof Error ? err.message : String(err) + }`, + ); + } + } + } + + console.error(""); + console.error( + ` ${RD}Rebuild preflight failed:${R} Hermes Provider is not registered in OpenShell.`, + ); + console.error(" Hermes Provider credentials must be stored in OpenShell, not host-side files."); + if (authMethod === "api_key") { + console.error( + ` Export the Hermes Provider API key and rerun rebuild, or re-run ${CLI_NAME} onboard to register it.`, + ); + } else { + console.error( + ` Re-run ${CLI_NAME} onboard interactively to authorize Hermes Provider and register it with OpenShell.`, + ); + } + console.error(""); + console.error(" Sandbox is untouched — no data was lost."); + return false; +} + +export function preflightRebuildCredentials( + sb: RebuildSandboxEntry, + log: RebuildLog, + bail: RebuildBail, +): boolean { + const rebuildCredentialEnv = getRebuildCredentialEnvFromRegistry(sb.provider, sb.credentialEnv); + const rebuildProvider = sb.provider; + + if (rebuildProvider === hermesProviderAuth.HERMES_PROVIDER_NAME) { + if (!preflightHermesProviderCredentials(sb.hermesAuthMethod, rebuildCredentialEnv, log)) { + bail("Missing Hermes Provider credentials"); + return false; + } + return true; + } + + if (!rebuildCredentialEnv) { + if (!checkRebuildGatewayProviderOrBail(rebuildProvider, rebuildCredentialEnv, log, bail)) { + return false; + } + log( + "Preflight credential check: no credentialEnv in session (local inference or missing session)", + ); + return true; + } + + const credentialValue = onboardModule.hydrateCredentialEnv(rebuildCredentialEnv); + log( + `Preflight credential check: ${rebuildCredentialEnv} → ${credentialValue ? "present" : "MISSING"}`, + ); + if (!checkRebuildGatewayProviderOrBail(rebuildProvider, rebuildCredentialEnv, log, bail)) { + return false; + } + if (!credentialValue && shouldVerifyRebuildGatewayProvider(rebuildProvider)) { + log( + `Preflight credential check: provider '${rebuildProvider}' registered in gateway — skipping env check for ${rebuildCredentialEnv}`, + ); + return true; + } + if (credentialValue) return true; + + console.error(""); + console.error(` ${RD}Rebuild preflight failed:${R} provider credential not found.`); + console.error(` The non-interactive recreate step requires ${rebuildCredentialEnv},`); + console.error(" but it is not set in the environment."); + console.error(""); + console.error(" To fix, do one of:"); + console.error(` export ${rebuildCredentialEnv}=`); + console.error(` ${CLI_NAME} onboard # re-enter the key interactively`); + console.error(""); + console.error(" Sandbox is untouched — no data was lost."); + bail(`Missing credential: ${rebuildCredentialEnv}`); + return false; +} diff --git a/src/lib/actions/sandbox/rebuild-custom-image-preflight.test.ts b/src/lib/actions/sandbox/rebuild-custom-image-preflight.test.ts new file mode 100644 index 00000000000..d94a1a0c345 --- /dev/null +++ b/src/lib/actions/sandbox/rebuild-custom-image-preflight.test.ts @@ -0,0 +1,104 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it, vi } from "vitest"; + +import { ROOT } from "../../runner"; +import { preflightRebuildImage } from "./rebuild-custom-image-preflight"; + +function input(fromDockerfile: string | null) { + return { + agent: null, + fromDockerfile, + model: "model", + provider: "ollama-local", + preferredInferenceApi: null, + compatibleEndpointReasoning: null, + webSearchConfig: null, + hermesToolGateways: [], + sandboxGpuConfig: { + mode: "0" as const, + hostGpuDetected: false, + hostGpuPlatform: null, + sandboxGpuEnabled: false, + sandboxGpuDevice: null, + errors: [], + }, + gatewayPort: 8080, + chatUiUrl: "http://127.0.0.1:18789", + }; +} + +describe("preflightRebuildImage", () => { + it("prebuilds the managed OpenClaw image instead of deferring its first build until delete", async () => { + const buildImage = vi.fn(() => ({ status: 0 }) as never); + const cleanupBuildCtx = vi.fn(() => true); + const stageBuildContext = vi.fn(() => ({ + buildCtx: "/tmp/rebuild-managed-context", + stagedDockerfile: "/tmp/rebuild-managed-context/Dockerfile", + cleanupBuildCtx, + })); + const result = await preflightRebuildImage(input(null), { + stageBuildContext, + prepareDockerfilePatch: vi.fn(async () => ({ buildId: "1", resolvedBaseImage: null })), + buildImage, + removeImage: vi.fn(), + }); + + expect(result.ok).toBe(true); + expect(stageBuildContext).toHaveBeenCalledWith( + expect.objectContaining({ root: ROOT, agent: null }), + ); + expect(buildImage).toHaveBeenCalledOnce(); + expect(cleanupBuildCtx).toHaveBeenCalledOnce(); + }); + + it.each([ + ["malformed syntax", "THIS IS NOT A DOCKERFILE"], + ["missing COPY context", "FROM scratch\nCOPY missing.txt /missing.txt\n"], + ])("fails before delete for %s", async (_label, dockerfileContents) => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-custom-preflight-")); + const dockerfile = path.join(dir, "Dockerfile.custom"); + fs.writeFileSync(dockerfile, dockerfileContents); + const removeImage = vi.fn(); + try { + const result = await preflightRebuildImage(input(dockerfile), { + prepareDockerfilePatch: vi.fn(async () => ({ buildId: "1", resolvedBaseImage: null })), + buildImage: vi.fn(() => ({ status: 1, stderr: "dockerfile validation failed" }) as never), + removeImage, + }); + expect(result).toEqual({ ok: false, detail: "dockerfile validation failed" }); + expect(removeImage).toHaveBeenCalledOnce(); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + + it("builds and removes the exact staged custom context on success", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-custom-preflight-")); + const dockerfile = path.join(dir, "Dockerfile.custom"); + fs.writeFileSync(dockerfile, "FROM scratch\n"); + const buildImage = vi.fn(() => ({ status: 0 }) as never); + const removeImage = vi.fn(); + try { + const result = await preflightRebuildImage(input(dockerfile), { + prepareDockerfilePatch: vi.fn(async () => ({ buildId: "1", resolvedBaseImage: null })), + buildImage, + removeImage, + }); + expect(result.ok).toBe(true); + expect(buildImage).toHaveBeenCalledWith( + expect.stringContaining("Dockerfile"), + expect.stringMatching(/^nemoclaw-rebuild-preflight:/), + expect.any(String), + expect.objectContaining({ ignoreError: true }), + ); + expect(removeImage).toHaveBeenCalledOnce(); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/src/lib/actions/sandbox/rebuild-custom-image-preflight.ts b/src/lib/actions/sandbox/rebuild-custom-image-preflight.ts new file mode 100644 index 00000000000..f1ee9c0e9b8 --- /dev/null +++ b/src/lib/actions/sandbox/rebuild-custom-image-preflight.ts @@ -0,0 +1,111 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { dockerBuild, dockerRmi } from "../../adapters/docker"; +import type { AgentDefinition } from "../../agent/defs"; +import { createAgentSandbox } from "../../agent/onboard"; +import type { WebSearchConfig } from "../../inference/web-search"; +import { stageCreateSandboxBuildContext } from "../../onboard/build-context-stage"; +import { prepareSandboxDockerfilePatch } from "../../onboard/sandbox-dockerfile-patch-flow"; +import type { SandboxGpuConfig } from "../../onboard/sandbox-gpu-mode"; +import { ROOT } from "../../runner"; +import { OPENCLAW_SANDBOX_BASE_IMAGE, SANDBOX_BASE_TAG } from "../../sandbox-base-image"; + +type PreflightInput = { + agent: AgentDefinition | null; + fromDockerfile: string | null; + model: string; + provider: string | null; + preferredInferenceApi: string | null; + compatibleEndpointReasoning: "true" | "false" | null; + webSearchConfig: WebSearchConfig | null; + hermesToolGateways: string[]; + sandboxGpuConfig: SandboxGpuConfig; + gatewayPort: number; + chatUiUrl: string; +}; + +type PreflightDeps = { + stageBuildContext?: typeof stageCreateSandboxBuildContext; + prepareDockerfilePatch?: typeof prepareSandboxDockerfilePatch; + buildImage?: typeof dockerBuild; + removeImage?: typeof dockerRmi; +}; + +export type RebuildImagePreflightResult = + | { ok: true; imageTag: string | null } + | { ok: false; detail: string }; + +function resultDetail(result: { stderr?: unknown; stdout?: unknown; status?: unknown }): string { + return ( + [result.stderr, result.stdout] + .filter((value): value is string => typeof value === "string" && value.trim().length > 0) + .join("; ") || `docker build exited with status ${String(result.status ?? "unknown")}` + ); +} + +export async function preflightRebuildImage( + input: PreflightInput, + deps: PreflightDeps = {}, +): Promise { + const stage = deps.stageBuildContext ?? stageCreateSandboxBuildContext; + const preparePatch = deps.prepareDockerfilePatch ?? prepareSandboxDockerfilePatch; + const buildImage = deps.buildImage ?? dockerBuild; + const removeImage = deps.removeImage ?? dockerRmi; + let cleanup: (() => boolean) | null = null; + let imageTag: string | null = null; + const previousReasoning = process.env.NEMOCLAW_REASONING; + try { + if (input.provider === "compatible-endpoint") { + process.env.NEMOCLAW_REASONING = input.compatibleEndpointReasoning ?? "false"; + } else { + delete process.env.NEMOCLAW_REASONING; + } + const staged = stage({ + root: ROOT, + fromDockerfile: input.fromDockerfile, + agent: input.agent, + createAgentSandbox, + log: () => {}, + warn: () => {}, + error: () => {}, + exit: (code): never => { + throw new Error(`custom build-context staging exited with code ${String(code ?? 1)}`); + }, + }); + cleanup = staged.cleanupBuildCtx; + await preparePatch({ + agent: input.agent, + fromDockerfile: input.fromDockerfile, + sandboxBaseImage: OPENCLAW_SANDBOX_BASE_IMAGE, + sandboxBaseTag: SANDBOX_BASE_TAG, + stagedDockerfile: staged.stagedDockerfile, + model: input.model, + chatUiUrl: input.chatUiUrl, + provider: input.provider, + preferredInferenceApi: input.preferredInferenceApi, + webSearchConfig: input.webSearchConfig, + hermesToolGateways: input.hermesToolGateways, + sandboxGpuConfig: input.sandboxGpuConfig, + gatewayPort: input.gatewayPort, + log: () => {}, + warn: () => {}, + }); + imageTag = `nemoclaw-rebuild-preflight:${String(process.pid)}-${String(Date.now())}`; + const result = buildImage(staged.stagedDockerfile, imageTag, staged.buildCtx, { + ignoreError: true, + suppressOutput: true, + stdio: ["ignore", "pipe", "pipe"], + }); + return result.status === 0 + ? { ok: true, imageTag } + : { ok: false, detail: resultDetail(result) }; + } catch (err) { + return { ok: false, detail: err instanceof Error ? err.message : String(err) }; + } finally { + if (imageTag) removeImage(imageTag, { ignoreError: true, suppressOutput: true }); + cleanup?.(); + if (previousReasoning === undefined) delete process.env.NEMOCLAW_REASONING; + else process.env.NEMOCLAW_REASONING = previousReasoning; + } +} diff --git a/src/lib/actions/sandbox/rebuild-dcode-orchestrator.ts b/src/lib/actions/sandbox/rebuild-dcode-orchestrator.ts index 73cb450a9d6..59b76db74a0 100644 --- a/src/lib/actions/sandbox/rebuild-dcode-orchestrator.ts +++ b/src/lib/actions/sandbox/rebuild-dcode-orchestrator.ts @@ -40,10 +40,15 @@ export type DcodeRebuildOrchestrator = { run(action: () => Promise): Promise; runSync(action: () => T): T; preflightCredentials(): Promise; - prepareImage(resumeConfig: RebuildResumeConfig, skipLiveRoute: boolean): Promise; + prepareImage( + resumeConfig: RebuildResumeConfig, + skipLiveRoute: boolean, + gatewayPort: number, + ): Promise; revalidateBeforeDelete( resumeConfig: RebuildResumeConfig, skipLiveRoute: boolean, + gatewayPort: number, ): Promise; clearManagedCustomDockerfile(session: Session): void; storedDockerfile(sessionMatchesSandbox: boolean, session: Session | null): string | null; @@ -102,7 +107,7 @@ export function createDcodeRebuildOrchestrator( } return deps.preflightCredentials(sandboxName, entry, log, scope.bail); }), - prepareImage: (resumeConfig, skipLiveRoute) => + prepareImage: (resumeConfig, skipLiveRoute, gatewayPort) => run(async () => { if (!scope.enabled) return deps.ensureAgentBaseImage(rebuildAgent, scope.bail); const replacement = await prepareDcodeReplacementBeforeMutation({ @@ -110,6 +115,7 @@ export function createDcodeRebuildOrchestrator( entry, resumeConfig, skipLiveRoute, + gatewayPort, log, bail: scope.bail, checkGatewaySchema: () => deps.checkGatewaySchema(sandboxName, scope.bail), @@ -121,7 +127,7 @@ export function createDcodeRebuildOrchestrator( scope.adopt(replacement); return true; }), - revalidateBeforeDelete: (resumeConfig, skipLiveRoute) => + revalidateBeforeDelete: (resumeConfig, skipLiveRoute, gatewayPort) => run(async () => { if (!scope.enabled) return true; const replacement = scope.preparedReplacement; @@ -131,6 +137,7 @@ export function createDcodeRebuildOrchestrator( entry, resumeConfig, skipLiveRoute, + gatewayPort, log, bail: scope.bail, checkGatewaySchema: () => deps.checkGatewaySchema(sandboxName, scope.bail), diff --git a/src/lib/actions/sandbox/rebuild-dcode-preflight.ts b/src/lib/actions/sandbox/rebuild-dcode-preflight.ts index 2272379a0eb..4de339a0d68 100644 --- a/src/lib/actions/sandbox/rebuild-dcode-preflight.ts +++ b/src/lib/actions/sandbox/rebuild-dcode-preflight.ts @@ -54,6 +54,8 @@ export type DcodeReplacementPreflightInput = { entry: RebuildSandboxEntry; resumeConfig: RebuildResumeConfig; skipLiveRoute: boolean; + /** Authoritative persisted gateway port carried by the rebuild target. */ + gatewayPort?: number; log(message: string): void; bail: DcodeRebuildPreflightBail; checkGatewaySchema(): boolean; @@ -178,9 +180,10 @@ function resolveTarget( entry: RebuildSandboxEntry, resumeConfig: RebuildResumeConfig, bail: DcodeRebuildPreflightBail, + gatewayPort?: number, ): ResolvedDcodeRebuildTarget { try { - return resolveDcodeRebuildTarget(entry, resumeConfig); + return resolveDcodeRebuildTarget(entry, resumeConfig, gatewayPort); } catch (error) { return fail(error instanceof Error ? error.message : String(error), bail); } @@ -222,12 +225,13 @@ function requireCurrentTarget( target: ResolvedDcodeRebuildTarget, resumeConfig: RebuildResumeConfig, bail: DcodeRebuildPreflightBail, + gatewayPort?: number, ): void { const currentEntry = registry.getSandbox(sandboxName) as RebuildSandboxEntry | null; if (!currentEntry || !isDeepStrictEqual(currentEntry, entry)) { fail("the recorded sandbox target changed during preflight", bail); } - const currentTarget = resolveTarget(currentEntry, resumeConfig, bail); + const currentTarget = resolveTarget(currentEntry, resumeConfig, bail, gatewayPort); if (!isDeepStrictEqual(currentTarget, target)) { fail("the resolved DCode target changed during preflight", bail); } @@ -350,7 +354,7 @@ function disposePreparation( export async function prepareDcodeReplacementBeforeMutation( input: DcodeReplacementPreflightInput, ): Promise { - const { sandboxName, entry, resumeConfig, skipLiveRoute, log, bail } = input; + const { sandboxName, entry, resumeConfig, skipLiveRoute, gatewayPort, log, bail } = input; let buildContext: PreparedDcodeRebuildImage | null = null; let pinnedBase: PinnedDcodeBaseImage | null = null; let transferred = false; @@ -363,7 +367,7 @@ export async function prepareDcodeReplacementBeforeMutation( } const session = requireManagedDcodeSession(sandboxName, bail); - const target = resolveTarget(entry, resumeConfig, bail); + const target = resolveTarget(entry, resumeConfig, bail, gatewayPort); if (!skipLiveRoute) requireInferenceRoute(sandboxName, target, bail); pinnedBase = buildPinnedDcodeBaseImage(bail); @@ -376,6 +380,7 @@ export async function prepareDcodeReplacementBeforeMutation( model: target.model, preferredInferenceApi: target.preferredInferenceApi, sandboxGpuConfig, + gatewayPort, }), ); if (!imageResult.ok) fail(imageResult.detail, bail); @@ -386,7 +391,7 @@ export async function prepareDcodeReplacementBeforeMutation( } if (!input.checkGatewaySchema()) return null; if (!skipLiveRoute) requireInferenceRoute(sandboxName, target, bail); - requireCurrentTarget(sandboxName, entry, target, resumeConfig, bail); + requireCurrentTarget(sandboxName, entry, target, resumeConfig, bail, gatewayPort); if (!verifyPreparedDcodeRebuildImage(buildContext) || !pinnedBase.verify()) { fail("the prepared DCode replacement inputs changed during preflight", bail); } @@ -410,8 +415,9 @@ export async function prepareDcodeReplacementBeforeMutation( export async function revalidateDcodeReplacementAtMutationEdge( input: DcodeReplacementPreflightInput & { replacement: PreparedDcodeReplacement }, ): Promise { - const { sandboxName, entry, resumeConfig, skipLiveRoute, log, bail, replacement } = input; - const target = resolveTarget(entry, resumeConfig, bail); + const { sandboxName, entry, resumeConfig, skipLiveRoute, gatewayPort, log, bail, replacement } = + input; + const target = resolveTarget(entry, resumeConfig, bail, gatewayPort); if (replacement.gatewayName !== target.gatewayName) { fail("the prepared DCode gateway changed before deletion", bail); } @@ -420,7 +426,7 @@ export async function revalidateDcodeReplacementAtMutationEdge( } if (!input.checkGatewaySchema()) return false; if (!skipLiveRoute) requireInferenceRoute(sandboxName, target, bail); - requireCurrentTarget(sandboxName, entry, target, resumeConfig, bail); + requireCurrentTarget(sandboxName, entry, target, resumeConfig, bail, gatewayPort); if (!replacement.verify()) { fail("the prepared DCode replacement inputs changed before deletion", bail); } diff --git a/src/lib/actions/sandbox/rebuild-destroy-phase.ts b/src/lib/actions/sandbox/rebuild-destroy-phase.ts new file mode 100644 index 00000000000..ec9b1a06b12 --- /dev/null +++ b/src/lib/actions/sandbox/rebuild-destroy-phase.ts @@ -0,0 +1,128 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { runOpenshell } from "../../adapters/openshell/runtime"; +import { G, R } from "../../cli/terminal-style"; +import { getSandboxDeleteOutcome } from "../../domain/sandbox/destroy"; +import * as nim from "../../inference/nim"; +import * as registry from "../../state/registry"; +import { removeSandboxRegistryEntry } from "./destroy"; +import type { RebuildBackupManifest } from "./rebuild-backup-phase"; +import type { RebuildBail, RebuildLog } from "./rebuild-credential-preflight"; +import { type RebuildSandboxEntry, warnUnpreservedUserManagedFiles } from "./rebuild-flow-helpers"; +import { prepareMcpBeforeBestEffortNimStop } from "./rebuild-mcp-order"; +import { + type McpRebuildPreparation, + prepareMcpForRebuild, + reattachMcpAfterDeleteFailure, +} from "./rebuild-mcp-phase"; + +export interface RebuildDestroyPhaseInput { + sandboxName: string; + sandboxEntry: RebuildSandboxEntry; + staleRecovery: boolean; + backupManifest: RebuildBackupManifest; + log: RebuildLog; + bail: RebuildBail; + relockShieldsIfNeeded: (sandboxStillExists: boolean) => boolean; + onDeleted: () => void; +} + +/** + * Detach owned MCP state, stop inference, and delete the old sandbox. + * Boundary coverage: rebuild-flow.test.ts exercises success, stale recovery, + * delete failure, provider reattach failure, and MCP-bearing registry retention. + */ +export async function runRebuildDestroyPhase( + input: RebuildDestroyPhaseInput, +): Promise { + const { + sandboxName, + staleRecovery, + backupManifest, + log, + bail, + relockShieldsIfNeeded, + onDeleted, + } = input; + + // Step 3: Delete sandbox without tearing down gateway or session. + // sandboxDestroy() cleans up the gateway when it's the last sandbox and + // nulls session.sandboxName — both break the immediate onboard --resume. + console.log(" Deleting old sandbox..."); + const sbMeta = registry.getSandbox(sandboxName); + log( + `Registry entry: agent=${sbMeta?.agent}, agentVersion=${sbMeta?.agentVersion}, nimContainer=${sbMeta?.nimContainer}`, + ); + const mcpPreparation = await prepareMcpBeforeBestEffortNimStop({ + prepareMcp: () => prepareMcpForRebuild(sandboxName, staleRecovery, relockShieldsIfNeeded, bail), + stopNim: () => { + if (sbMeta && sbMeta.nimContainer) { + log(`Stopping NIM container: ${sbMeta.nimContainer}`); + nim.stopNimContainerByName(sbMeta.nimContainer); + } else { + // Best-effort cleanup — see comment in sandboxDestroy. + nim.stopNimContainer(sandboxName, { silent: true }); + } + }, + log, + }); + if (!mcpPreparation) return null; + // MCP preparation removes only adapter entries whose exact ownership + // fingerprints match the registry. Probe afterward so a Deep Agents + // `.mcp.json` containing only NemoClaw-managed entries is not mislabeled as + // unpreserved user state; any file that remains still needs the warning. + if (!staleRecovery) warnUnpreservedUserManagedFiles(sandboxName, log); + const rebuildMcpEntries = mcpPreparation.entries; + const rebuildDetachedMcpProviderEntries = mcpPreparation.detachedProviderEntries; + const rebuildScrubbedMcpAdapterEntries = mcpPreparation.scrubbedAdapterEntries; + + log(`Running: openshell sandbox delete ${sandboxName}`); + const deleteResult = runOpenshell(["sandbox", "delete", sandboxName], { + ignoreError: true, + stdio: ["ignore", "pipe", "pipe"], + }); + const { alreadyGone } = getSandboxDeleteOutcome(deleteResult); + log(`Delete result: exit=${deleteResult.status}, alreadyGone=${alreadyGone}`); + if (deleteResult.status !== 0 && !alreadyGone) { + console.error(" Failed to delete sandbox. Aborting rebuild."); + const mcpRecoveryFailure = await reattachMcpAfterDeleteFailure( + sandboxName, + rebuildDetachedMcpProviderEntries, + rebuildScrubbedMcpAdapterEntries, + ); + if (mcpRecoveryFailure) { + console.error( + ` Failed to reattach MCP providers to the existing sandbox: ${mcpRecoveryFailure}`, + ); + } + if (backupManifest) { + console.error(" State backup is preserved at: " + backupManifest.backupPath); + } + relockShieldsIfNeeded(true); + bail( + mcpRecoveryFailure + ? `Failed to delete sandbox; MCP provider recovery also failed: ${mcpRecoveryFailure}` + : "Failed to delete sandbox.", + deleteResult.status || 1, + ); + return null; + } + onDeleted(); + if (rebuildMcpEntries.length === 0) { + removeSandboxRegistryEntry(sandboxName); + } else { + // The registry entry is the durable MCP rebuild transaction. The inner + // onboard run observes that the sandbox is absent, carries the MCP state + // into the replacement registration, and never enters generic live + // recreation. Keeping it here closes every process-death window between + // successful delete and fresh registry registration. + log("Preserving MCP-bearing registry entry across sandbox recreation"); + } + log( + `Registry after remove: ${JSON.stringify(registry.listSandboxes().sandboxes.map((s: { name: string }) => s.name))}`, + ); + console.log(` ${G}\u2713${R} Old sandbox deleted`); + + return mcpPreparation; +} diff --git a/src/lib/actions/sandbox/rebuild-durable-config.test.ts b/src/lib/actions/sandbox/rebuild-durable-config.test.ts new file mode 100644 index 00000000000..fe1d2628aa0 --- /dev/null +++ b/src/lib/actions/sandbox/rebuild-durable-config.test.ts @@ -0,0 +1,194 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { createSession } from "../../state/onboard-session"; +import { resolveRebuildDurableConfig } from "./rebuild-durable-config"; + +describe("resolveRebuildDurableConfig", () => { + it("uses a legacy built-in Brave policy for a nonmatching session", () => { + const session = createSession({ sandboxName: "other", webSearchConfig: null }); + const config = resolveRebuildDurableConfig( + "alpha", + { name: "alpha", policies: ["brave"], nemoclawVersion: "0.1.0" }, + session, + ); + expect(config.webSearchConfig).toEqual({ fetchEnabled: true, provider: "brave" }); + }); + + it("does not mistake a legacy custom policy named brave for web search", () => { + const config = resolveRebuildDurableConfig( + "alpha", + { + name: "alpha", + policies: ["brave"], + customPolicies: [{ name: "brave", content: "allow: []" }], + nemoclawVersion: "0.1.0", + }, + createSession({ sandboxName: "other" }), + ); + expect(config.webSearchConfig).toBeNull(); + }); + + it("keeps an explicit durable web-search disable authoritative", () => { + const config = resolveRebuildDurableConfig( + "alpha", + { + name: "alpha", + policies: ["brave"], + webSearchEnabled: false, + fromDockerfile: null, + }, + createSession({ sandboxName: "alpha", webSearchConfig: { fetchEnabled: true } }), + ); + expect(config.webSearchConfig).toBeNull(); + }); + + it("fails closed for an ambiguous legacy image without its matching session", () => { + const config = resolveRebuildDurableConfig( + "alpha", + { name: "alpha", nemoclawVersion: null }, + createSession({ sandboxName: "other" }), + ); + expect(config.fromDockerfileError).toContain("cannot distinguish"); + }); + + it("accepts explicit managed-image provenance for an old agent runtime", () => { + const config = resolveRebuildDurableConfig( + "alpha", + { + name: "alpha", + agentVersion: "2026.3.11", + nemoclawVersion: null, + fromDockerfile: null, + }, + createSession({ sandboxName: "other" }), + ); + expect(config.fromDockerfile).toBeNull(); + expect(config.fromDockerfileError).toBeNull(); + }); + + it("does not treat a same-name null image session as proof of a legacy managed image", () => { + const config = resolveRebuildDurableConfig( + "alpha", + { name: "alpha", provider: "ollama-local", model: "model", nemoclawVersion: null }, + createSession({ sandboxName: "alpha", provider: "ollama-local", model: "model" }), + ); + expect(config.fromDockerfileError).toContain("cannot distinguish"); + }); + + it("fails closed for corrupt durable web-search state", () => { + const config = resolveRebuildDurableConfig( + "alpha", + { name: "alpha", webSearchEnabled: "false" as never, fromDockerfile: null }, + null, + ); + expect(config.webSearchError).toContain("not boolean"); + }); + + it("preserves an explicit durable Tavily provider", () => { + const config = resolveRebuildDurableConfig( + "alpha", + { + name: "alpha", + webSearchEnabled: true, + webSearchProvider: "tavily", + fromDockerfile: null, + }, + createSession({ sandboxName: "other" }), + ); + expect(config.webSearchConfig).toEqual({ fetchEnabled: true, provider: "tavily" }); + expect(config.webSearchError).toBeNull(); + }); + + it("backfills a legacy enabled provider from the matching Tavily session", () => { + const config = resolveRebuildDurableConfig( + "alpha", + { + name: "alpha", + provider: "compatible-endpoint", + model: "model", + webSearchEnabled: true, + fromDockerfile: null, + }, + createSession({ + sandboxName: "alpha", + provider: "compatible-endpoint", + model: "model", + webSearchConfig: { fetchEnabled: true, provider: "tavily" }, + }), + ); + expect(config.webSearchConfig).toEqual({ fetchEnabled: true, provider: "tavily" }); + }); + + it("does not infer managed Tavily from the DCode interpreter opt-in preset", () => { + const config = resolveRebuildDurableConfig( + "alpha", + { + name: "alpha", + agent: "langchain-deepagents-code", + policies: ["tavily"], + nemoclawVersion: "0.1.0", + }, + createSession({ sandboxName: "other" }), + ); + expect(config.webSearchConfig).toBeNull(); + }); + + it("fails closed for an invalid durable web-search provider", () => { + const config = resolveRebuildDurableConfig( + "alpha", + { + name: "alpha", + webSearchEnabled: true, + webSearchProvider: "other" as never, + fromDockerfile: null, + }, + null, + ); + expect(config.webSearchError).toContain("webSearchProvider"); + }); + + it.each([ + ["NOUS_API_KEY", "api_key"], + ["OPENAI_API_KEY", "oauth"], + ] as const)("recovers legacy Hermes auth from %s", (credentialEnv, expected) => { + const config = resolveRebuildDurableConfig( + "alpha", + { + name: "alpha", + provider: "hermes-provider", + credentialEnv, + nemoclawVersion: "0.1.0", + }, + createSession({ sandboxName: "other" }), + ); + expect(config.hermesAuthMethod).toBe(expected); + expect(config.hermesAuthMethodError).toBeNull(); + }); + + it("fails closed when legacy Hermes auth has no durable clue", () => { + const config = resolveRebuildDurableConfig( + "alpha", + { name: "alpha", provider: "hermes-provider", nemoclawVersion: "0.1.0" }, + createSession({ sandboxName: "other" }), + ); + expect(config.hermesAuthMethodError).toContain("cannot determine"); + }); + + it("does not borrow Hermes auth from a same-name conflicting selection", () => { + const config = resolveRebuildDurableConfig( + "alpha", + { name: "alpha", provider: "hermes-provider", model: "target", nemoclawVersion: "0.1.0" }, + createSession({ + sandboxName: "alpha", + provider: "hermes-provider", + model: "different", + hermesAuthMethod: "oauth", + }), + ); + expect(config.hermesAuthMethod).toBeNull(); + expect(config.hermesAuthMethodError).toContain("cannot determine"); + }); +}); diff --git a/src/lib/actions/sandbox/rebuild-durable-config.ts b/src/lib/actions/sandbox/rebuild-durable-config.ts new file mode 100644 index 00000000000..cb685ef0e0a --- /dev/null +++ b/src/lib/actions/sandbox/rebuild-durable-config.ts @@ -0,0 +1,242 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import path from "node:path"; +import { + HERMES_DASHBOARD_ENABLE_ENV, + HERMES_DASHBOARD_INTERNAL_PORT_ENV, + HERMES_DASHBOARD_PORT_ENV, + HERMES_DASHBOARD_TUI_ENV, +} from "../../hermes-dashboard"; +import { + HERMES_INFERENCE_CREDENTIAL_ENV, + HERMES_NOUS_API_KEY_CREDENTIAL_ENV, + HERMES_PROVIDER_NAME, +} from "../../hermes-provider-auth"; +import { + isWebSearchProvider, + type WebSearchConfig, + type WebSearchProvider, + webSearchProviderForConfig, +} from "../../inference/web-search"; +import { resolveHermesDashboardOnboardState } from "../../onboard/hermes-dashboard"; +import type { Session } from "../../state/onboard-session"; +import type { RebuildSandboxEntry } from "./rebuild-flow-helpers"; +import type { RebuildResumeConfig } from "./rebuild-resume-config"; + +export type RebuildDurableConfig = { + fromDockerfile: string | null; + fromDockerfileError: string | null; + hermesAuthMethod: "oauth" | "api_key" | null; + hermesAuthMethodError: string | null; + webSearchConfig: WebSearchConfig | null; + webSearchError: string | null; +}; + +export const REBUILD_HERMES_DASHBOARD_ENV_KEYS = [ + HERMES_DASHBOARD_ENABLE_ENV, + HERMES_DASHBOARD_PORT_ENV, + HERMES_DASHBOARD_INTERNAL_PORT_ENV, + HERMES_DASHBOARD_TUI_ENV, +] as const; + +export type RebuildHermesDashboardEnv = Partial< + Record<(typeof REBUILD_HERMES_DASHBOARD_ENV_KEYS)[number], string> +>; + +export type RebuildHermesDashboardResolution = + | { ok: true; env: RebuildHermesDashboardEnv } + | { ok: false; reason: string }; + +function validDashboardPort(value: unknown): value is number { + return typeof value === "number" && Number.isInteger(value) && value >= 1024 && value <= 65535; +} + +export function resolveRebuildHermesDashboardEnv( + rebuildAgent: string | null, + entry: RebuildSandboxEntry, + controlUiPort: number | null, +): RebuildHermesDashboardResolution { + if ( + entry.hermesDashboardEnabled !== undefined && + typeof entry.hermesDashboardEnabled !== "boolean" + ) { + return { ok: false, reason: "recorded hermesDashboardEnabled value is not boolean" }; + } + if (rebuildAgent !== "hermes" || entry.hermesDashboardEnabled !== true) { + return { ok: true, env: { [HERMES_DASHBOARD_ENABLE_ENV]: "0" } }; + } + if (!validDashboardPort(entry.hermesDashboardPort)) { + return { ok: false, reason: "recorded Hermes dashboard port is invalid or missing" }; + } + if (!validDashboardPort(entry.hermesDashboardInternalPort)) { + return { ok: false, reason: "recorded Hermes dashboard internal port is invalid or missing" }; + } + if (entry.hermesDashboardTui !== undefined && typeof entry.hermesDashboardTui !== "boolean") { + return { ok: false, reason: "recorded hermesDashboardTui value is not boolean" }; + } + const env: RebuildHermesDashboardEnv = { + [HERMES_DASHBOARD_ENABLE_ENV]: "1", + [HERMES_DASHBOARD_PORT_ENV]: String(entry.hermesDashboardPort), + [HERMES_DASHBOARD_INTERNAL_PORT_ENV]: String(entry.hermesDashboardInternalPort), + [HERMES_DASHBOARD_TUI_ENV]: entry.hermesDashboardTui === true ? "1" : "0", + }; + try { + resolveHermesDashboardOnboardState({ + agentName: rebuildAgent, + effectivePort: controlUiPort ?? 0, + env, + fail: (message): never => { + throw new Error(message); + }, + }); + } catch (err) { + return { ok: false, reason: err instanceof Error ? err.message : String(err) }; + } + return { ok: true, env }; +} + +function normalizeHermesAuthMethod(value: unknown): "oauth" | "api_key" | null { + return value === "oauth" || value === "api_key" ? value : null; +} + +export function resolveRebuildDurableConfig( + sandboxName: string, + entry: RebuildSandboxEntry, + session: Session | null, + resolvedSelection: { provider: string | null; model: string | null } = { + provider: entry.provider ?? null, + model: entry.model ?? null, + }, +): RebuildDurableConfig { + const matchingSession = + session?.sandboxName === sandboxName && + (!resolvedSelection.provider || session.provider === resolvedSelection.provider) && + (!resolvedSelection.model || session.model === resolvedSelection.model) + ? session + : null; + const legacyBravePolicy = + entry.policies?.includes("brave") === true && + !entry.customPolicies?.some((policy) => policy.name === "brave"); + const legacyTavilyPolicy = + entry.agent !== "langchain-deepagents-code" && + entry.policies?.includes("tavily") === true && + !entry.customPolicies?.some((policy) => policy.name === "tavily"); + const recordedWebSearchProvider = entry.webSearchProvider; + const webSearchEnabled = + typeof entry.webSearchEnabled === "boolean" + ? entry.webSearchEnabled + : isWebSearchProvider(recordedWebSearchProvider) || + matchingSession?.webSearchConfig?.fetchEnabled === true || + legacyBravePolicy || + legacyTavilyPolicy; + let webSearchError: string | null = null; + if (entry.webSearchEnabled !== undefined && typeof entry.webSearchEnabled !== "boolean") { + webSearchError = "recorded webSearchEnabled value is not boolean"; + } else if ( + recordedWebSearchProvider !== undefined && + recordedWebSearchProvider !== null && + !isWebSearchProvider(recordedWebSearchProvider) + ) { + webSearchError = "recorded webSearchProvider value is invalid"; + } else if (!webSearchEnabled && isWebSearchProvider(recordedWebSearchProvider)) { + webSearchError = "recorded webSearchProvider is set while web search is disabled"; + } + let webSearchProvider: WebSearchProvider | null = null; + if (webSearchEnabled && !webSearchError) { + webSearchProvider = isWebSearchProvider(recordedWebSearchProvider) + ? recordedWebSearchProvider + : matchingSession?.webSearchConfig?.fetchEnabled === true + ? webSearchProviderForConfig(matchingSession.webSearchConfig) + : legacyTavilyPolicy + ? "tavily" + : "brave"; + } + const recordedFromDockerfile: unknown = + entry.fromDockerfile !== undefined + ? entry.fromDockerfile + : (matchingSession?.metadata?.fromDockerfile ?? null); + const fromDockerfileError = + recordedFromDockerfile !== null && + recordedFromDockerfile !== undefined && + (typeof recordedFromDockerfile !== "string" || recordedFromDockerfile.length === 0) + ? "recorded value is not a non-empty path" + : entry.fromDockerfile === undefined && !recordedFromDockerfile && !entry.nemoclawVersion + ? "legacy registry entry cannot distinguish a managed image from a custom --from image" + : null; + let hermesAuthMethod = + entry.hermesAuthMethod !== undefined + ? normalizeHermesAuthMethod(entry.hermesAuthMethod) + : normalizeHermesAuthMethod(matchingSession?.hermesAuthMethod); + if ( + entry.hermesAuthMethod === undefined && + !matchingSession && + resolvedSelection.provider === HERMES_PROVIDER_NAME + ) { + if (entry.credentialEnv === HERMES_NOUS_API_KEY_CREDENTIAL_ENV) hermesAuthMethod = "api_key"; + if (entry.credentialEnv === HERMES_INFERENCE_CREDENTIAL_ENV) hermesAuthMethod = "oauth"; + } + const hermesAuthMethodError = + resolvedSelection.provider === HERMES_PROVIDER_NAME && hermesAuthMethod === null + ? "cannot determine the recorded Hermes Provider authentication method" + : null; + + return { + fromDockerfile: + typeof recordedFromDockerfile === "string" && recordedFromDockerfile + ? recordedFromDockerfile + : null, + fromDockerfileError, + hermesAuthMethod, + hermesAuthMethodError, + webSearchConfig: + webSearchEnabled && webSearchProvider + ? { fetchEnabled: true, provider: webSearchProvider } + : null, + webSearchError, + }; +} + +export function resolveRebuildDockerfile( + fromDockerfile: string | null, +): { ok: true; path: string | null } | { ok: false; path: string; reason: string } { + if (!fromDockerfile) return { ok: true, path: null }; + const resolved = path.resolve(fromDockerfile); + try { + if (!fs.statSync(resolved).isFile()) { + return { ok: false, path: resolved, reason: "path is not a regular file" }; + } + fs.accessSync(resolved, fs.constants.R_OK); + } catch (err) { + return { + ok: false, + path: resolved, + reason: err instanceof Error ? err.message : String(err), + }; + } + return { ok: true, path: resolved }; +} + +export function validatedRebuildRegistryUpdate( + resume: RebuildResumeConfig, + durable: RebuildDurableConfig, + fromDockerfile: string | null, + credentialEnv: string | null, +): Partial { + return { + provider: resume.provider, + model: resume.model, + endpointUrl: resume.endpointUrl, + credentialEnv, + preferredInferenceApi: resume.preferredInferenceApi, + compatibleEndpointReasoning: resume.compatibleEndpointReasoning, + nimContainer: resume.nimContainer, + webSearchEnabled: durable.webSearchConfig?.fetchEnabled === true, + webSearchProvider: durable.webSearchConfig + ? webSearchProviderForConfig(durable.webSearchConfig) + : null, + fromDockerfile, + hermesAuthMethod: durable.hermesAuthMethod, + }; +} diff --git a/src/lib/actions/sandbox/rebuild-env-isolation.test.ts b/src/lib/actions/sandbox/rebuild-env-isolation.test.ts index 24956edb006..5efc28a5218 100644 --- a/src/lib/actions/sandbox/rebuild-env-isolation.test.ts +++ b/src/lib/actions/sandbox/rebuild-env-isolation.test.ts @@ -47,6 +47,19 @@ describe("AMBIENT_RECREATE_ENV_VARS contract PRA-4 (#5735)", () => { "NEMOCLAW_PROVIDER_KEY", "NEMOCLAW_ENDPOINT_URL", "NEMOCLAW_MODEL", + "NEMOCLAW_COMPAT_MODEL", + "NEMOCLAW_CLOUD_EXPERIMENTAL_MODEL", + "NEMOCLAW_PREFERRED_API", + "NEMOCLAW_REASONING", + "NEMOCLAW_VLLM_MODEL", + "NEMOCLAW_VLLM_EXTRA_ARGS_JSON", + "NEMOCLAW_FROM_DOCKERFILE", + "NEMOCLAW_WEB_SEARCH_PROVIDER", + "NEMOCLAW_POLICY_TIER", + "NEMOCLAW_POLICY_MODE", + "NEMOCLAW_POLICY_PRESETS", + "NEMOCLAW_SANDBOX_GPU", + "NEMOCLAW_SANDBOX_GPU_DEVICE", ]); }); }); @@ -97,6 +110,19 @@ describe("isolateAmbientRecreateEnv", () => { NEMOCLAW_AGENT: "langchain-deepagents-code", NEMOCLAW_PROVIDER_KEY: "sk-bogus", NEMOCLAW_MODEL: "some-model", + NEMOCLAW_COMPAT_MODEL: "some-compat-model", + NEMOCLAW_PREFERRED_API: "openai-responses", + NEMOCLAW_REASONING: "false", + NEMOCLAW_VLLM_MODEL: "ambient-vllm-model", + NEMOCLAW_VLLM_EXTRA_ARGS_JSON: '{"ambient":true}', + NEMOCLAW_FROM_DOCKERFILE: "/tmp/unrelated.Dockerfile", + NEMOCLAW_WEB_SEARCH_PROVIDER: "tavily", + NEMOCLAW_POLICY_TIER: "permissive", + NEMOCLAW_POLICY_MODE: "customize", + NEMOCLAW_POLICY_PRESETS: "ambient-preset", + NEMOCLAW_SANDBOX_GPU: "0", + NEMOCLAW_SANDBOX_GPU_DEVICE: "9", + NVIDIA_INFERENCE_API_KEY: "hosted-source-key", // not part of the selection set — must be left untouched NVIDIA_API_KEY: "nvapi-keep-me", }; @@ -106,6 +132,7 @@ describe("isolateAmbientRecreateEnv", () => { for (const name of AMBIENT_RECREATE_ENV_VARS) { expect(env[name]).toBeUndefined(); } + expect(env.NVIDIA_INFERENCE_API_KEY).toBe("hosted-source-key"); expect(env.NVIDIA_API_KEY).toBe("nvapi-keep-me"); restore(); @@ -113,6 +140,19 @@ describe("isolateAmbientRecreateEnv", () => { expect(env.NEMOCLAW_AGENT).toBe("langchain-deepagents-code"); expect(env.NEMOCLAW_PROVIDER_KEY).toBe("sk-bogus"); expect(env.NEMOCLAW_MODEL).toBe("some-model"); + expect(env.NEMOCLAW_COMPAT_MODEL).toBe("some-compat-model"); + expect(env.NEMOCLAW_PREFERRED_API).toBe("openai-responses"); + expect(env.NEMOCLAW_REASONING).toBe("false"); + expect(env.NEMOCLAW_VLLM_MODEL).toBe("ambient-vllm-model"); + expect(env.NEMOCLAW_VLLM_EXTRA_ARGS_JSON).toBe('{"ambient":true}'); + expect(env.NEMOCLAW_FROM_DOCKERFILE).toBe("/tmp/unrelated.Dockerfile"); + expect(env.NEMOCLAW_WEB_SEARCH_PROVIDER).toBe("tavily"); + expect(env.NEMOCLAW_POLICY_TIER).toBe("permissive"); + expect(env.NEMOCLAW_POLICY_MODE).toBe("customize"); + expect(env.NEMOCLAW_POLICY_PRESETS).toBe("ambient-preset"); + expect(env.NEMOCLAW_SANDBOX_GPU).toBe("0"); + expect(env.NEMOCLAW_SANDBOX_GPU_DEVICE).toBe("9"); + expect(env.NVIDIA_INFERENCE_API_KEY).toBe("hosted-source-key"); expect(env.NVIDIA_API_KEY).toBe("nvapi-keep-me"); // A var that was never set stays unset after restore. expect("NEMOCLAW_PROVIDER" in env).toBe(false); diff --git a/src/lib/actions/sandbox/rebuild-env-isolation.ts b/src/lib/actions/sandbox/rebuild-env-isolation.ts index 4c3819c1a8a..c0db1aacbd4 100644 --- a/src/lib/actions/sandbox/rebuild-env-isolation.ts +++ b/src/lib/actions/sandbox/rebuild-env-isolation.ts @@ -17,6 +17,17 @@ // - NEMOCLAW_PROVIDER_KEY → src/lib/onboard/provider-key-bridge.ts / providers.ts // - NEMOCLAW_ENDPOINT_URL → src/lib/onboard.ts (remote endpoint override) // - NEMOCLAW_MODEL → src/lib/onboard.ts (model override) +// - NEMOCLAW_COMPAT_MODEL / NEMOCLAW_CLOUD_EXPERIMENTAL_MODEL +// → src/lib/onboard/providers.ts (hosted model aliases) +// - NEMOCLAW_PREFERRED_API → src/lib/onboard/setup-nim-selection.ts +// - NEMOCLAW_REASONING → src/lib/onboard/reasoning-mode.ts +// - NEMOCLAW_VLLM_MODEL / NEMOCLAW_VLLM_EXTRA_ARGS_JSON +// → src/lib/onboard/setup-nim-vllm.ts +// - NEMOCLAW_FROM_DOCKERFILE → src/lib/onboard/entry-options.ts +// - NEMOCLAW_POLICY_TIER / NEMOCLAW_POLICY_MODE / NEMOCLAW_POLICY_PRESETS +// → src/lib/onboard/policy-tier-env.ts / policy selection +// - NEMOCLAW_SANDBOX_GPU / NEMOCLAW_SANDBOX_GPU_DEVICE +// → src/lib/onboard/sandbox-gpu-mode.ts // This list MUST stay in sync with those reads; a contract test in // rebuild-env-isolation.test.ts pins the exact set so adding a new // onboard-selection env var forces a conscious update here. @@ -30,6 +41,19 @@ export const AMBIENT_RECREATE_ENV_VARS = [ "NEMOCLAW_PROVIDER_KEY", "NEMOCLAW_ENDPOINT_URL", "NEMOCLAW_MODEL", + "NEMOCLAW_COMPAT_MODEL", + "NEMOCLAW_CLOUD_EXPERIMENTAL_MODEL", + "NEMOCLAW_PREFERRED_API", + "NEMOCLAW_REASONING", + "NEMOCLAW_VLLM_MODEL", + "NEMOCLAW_VLLM_EXTRA_ARGS_JSON", + "NEMOCLAW_FROM_DOCKERFILE", + "NEMOCLAW_WEB_SEARCH_PROVIDER", + "NEMOCLAW_POLICY_TIER", + "NEMOCLAW_POLICY_MODE", + "NEMOCLAW_POLICY_PRESETS", + "NEMOCLAW_SANDBOX_GPU", + "NEMOCLAW_SANDBOX_GPU_DEVICE", ] as const; /** @@ -43,7 +67,6 @@ export const AMBIENT_RECREATE_ENV_VARS = [ */ export function sanitizeEnvValueForDisplay(value: string, maxLength = 80): string { const stripped = value - // biome-ignore lint/suspicious/noControlCharactersInRegex: deliberately stripping control chars from untrusted env input before display. .replace(/[\u0000-\u001f\u007f-\u009f]/g, " ") .replace(/\s+/g, " ") .trim(); diff --git a/src/lib/actions/sandbox/rebuild-flow-helpers.test.ts b/src/lib/actions/sandbox/rebuild-flow-helpers.test.ts index fa38fbfe1ed..eda288d0ca9 100644 --- a/src/lib/actions/sandbox/rebuild-flow-helpers.test.ts +++ b/src/lib/actions/sandbox/rebuild-flow-helpers.test.ts @@ -20,6 +20,11 @@ function loadRebuildFlowHelpers(): RebuildFlowHelpersModule { return requireDist(rebuildFlowHelpersPath); } +// Warm the CommonJS dependency graph outside the first test's timeout. Tests +// still reload this entry module after installing dependency spies. +loadRebuildFlowHelpers(); +delete require.cache[requireDist.resolve(rebuildFlowHelpersPath)]; + function loadSandboxState(): SandboxStateModule { return requireDist(sandboxStatePath); } @@ -74,7 +79,167 @@ function makeBail(): (msg: string, code?: number) => never { }; } -describe("backupSandboxStateForRebuild — user-managed file warning", () => { +describe("rebuild target gateway preflight", () => { + const priorGateway = process.env.OPENSHELL_GATEWAY; + + afterEach(() => { + vi.restoreAllMocks(); + switch (priorGateway) { + case undefined: + delete process.env.OPENSHELL_GATEWAY; + break; + default: + process.env.OPENSHELL_GATEWAY = priorGateway; + } + }); + + it("health-checks and pins the sandbox's persisted gateway", async () => { + const gatewayRuntime = requireDist("../../gateway-runtime-action.js"); + const recover = vi.spyOn(gatewayRuntime, "recoverNamedGatewayRuntime").mockResolvedValue({ + recovered: true, + before: { state: "connected_other" }, + after: { state: "healthy_named" }, + attempted: true, + }); + const { ensureRebuildTargetGatewaySelected } = loadRebuildFlowHelpers(); + + await expect( + ensureRebuildTargetGatewaySelected( + "alpha", + { name: "alpha", gatewayName: "nemoclaw-19080", gatewayPort: 19080 }, + () => undefined, + makeBail(), + ), + ).resolves.toBe(true); + + expect(recover).toHaveBeenCalledWith({ gatewayName: "nemoclaw-19080" }); + expect(process.env.OPENSHELL_GATEWAY).toBe("nemoclaw-19080"); + }); + + it("fails closed when the target gateway cannot become healthy", async () => { + const gatewayRuntime = requireDist("../../gateway-runtime-action.js"); + vi.spyOn(gatewayRuntime, "recoverNamedGatewayRuntime").mockResolvedValue({ + recovered: false, + before: { state: "connected_other" }, + after: { state: "missing_named" }, + attempted: true, + }); + const { ensureRebuildTargetGatewaySelected } = loadRebuildFlowHelpers(); + + await expect( + ensureRebuildTargetGatewaySelected( + "alpha", + { name: "alpha", gatewayName: "nemoclaw-19080", gatewayPort: 19080 }, + () => undefined, + makeBail(), + ), + ).rejects.toThrow("Could not select healthy gateway 'nemoclaw-19080'"); + }); +}); + +describe("rebuild agent base image preflight", () => { + const overrideEnvVar = "NEMOCLAW_HERMES_SANDBOX_BASE_IMAGE_REF"; + let priorOverride: string | undefined; + + beforeEach(() => { + priorOverride = process.env[overrideEnvVar]; + delete process.env[overrideEnvVar]; + }); + + afterEach(() => { + vi.restoreAllMocks(); + const original = priorOverride; + const restoreOverride = + original === undefined + ? () => Reflect.deleteProperty(process.env, overrideEnvVar) + : () => Reflect.set(process.env, overrideEnvVar, original); + restoreOverride(); + }); + + function mockBaseImagePreflight(imageRef: string) { + const agentDefs = requireDist("../../agent/defs.js"); + const agentOnboard = requireDist("../../agent/onboard.js"); + vi.spyOn(agentDefs, "loadAgent").mockReturnValue({ name: "hermes" }); + const ensureAgentBaseImage = vi + .spyOn(agentOnboard, "ensureAgentBaseImage") + .mockReturnValue({ imageTag: imageRef, built: true }); + const pinAgentSandboxBaseImageRef = vi + .spyOn(agentOnboard, "pinAgentSandboxBaseImageRef") + .mockImplementation((_agentName, ref) => String(ref)); + return { ensureAgentBaseImage, pinAgentSandboxBaseImageRef }; + } + + it("forces a repository-local build and returns its exact ref when no override exists", () => { + const imageRef = "nemoclaw-hermes-sandbox-base-local:12345678"; + const { ensureAgentBaseImage } = mockBaseImagePreflight(imageRef); + const { ensureRebuildAgentBaseImage } = loadRebuildFlowHelpers(); + + const result = ensureRebuildAgentBaseImage("hermes", makeBail()); + + expect(ensureAgentBaseImage).toHaveBeenCalledWith(expect.objectContaining({ name: "hermes" }), { + forceBaseImageRebuild: true, + }); + expect(result).toEqual({ ok: true, imageRef, overrideEnvVar }); + }); + + it("resolves an explicit caller override instead of replacing it during preflight", () => { + process.env[overrideEnvVar] = "nemoclaw-hermes-sandbox-base-local:caller"; + const mutableRef = "nemoclaw-hermes-sandbox-base-local:resolved"; + const immutableRef = `nemoclaw-hermes-sandbox-base-local:image-${"a".repeat(64)}`; + const { ensureAgentBaseImage, pinAgentSandboxBaseImageRef } = + mockBaseImagePreflight(mutableRef); + pinAgentSandboxBaseImageRef.mockReturnValue(immutableRef); + const { ensureRebuildAgentBaseImage } = loadRebuildFlowHelpers(); + + const result = ensureRebuildAgentBaseImage("hermes", makeBail()); + + expect(ensureAgentBaseImage).toHaveBeenCalledWith(expect.objectContaining({ name: "hermes" }), { + forceBaseImageRebuild: false, + }); + expect(pinAgentSandboxBaseImageRef).toHaveBeenCalledWith("hermes", mutableRef); + expect(result).toEqual({ ok: true, imageRef: immutableRef, overrideEnvVar }); + }); + + it("pins the preflighted ref only for recreation and restores caller state", () => { + const { pinRebuildAgentBaseImageForRecreate } = loadRebuildFlowHelpers(); + const env: NodeJS.ProcessEnv = { + [overrideEnvVar]: "nemoclaw-hermes-sandbox-base-local:image-caller", + }; + const restore = pinRebuildAgentBaseImageForRecreate( + { + ok: true, + imageRef: "nemoclaw-hermes-sandbox-base-local:image-resolved", + overrideEnvVar, + }, + env, + ); + + expect(env[overrideEnvVar]).toBe("nemoclaw-hermes-sandbox-base-local:image-resolved"); + restore(); + expect(env[overrideEnvVar]).toBe("nemoclaw-hermes-sandbox-base-local:image-caller"); + restore(); + expect(env[overrideEnvVar]).toBe("nemoclaw-hermes-sandbox-base-local:image-caller"); + }); + + it("removes a scoped recreation pin when the caller had no override", () => { + const { pinRebuildAgentBaseImageForRecreate } = loadRebuildFlowHelpers(); + const env: NodeJS.ProcessEnv = {}; + const restore = pinRebuildAgentBaseImageForRecreate( + { + ok: true, + imageRef: "nemoclaw-hermes-sandbox-base-local:12345678", + overrideEnvVar, + }, + env, + ); + + expect(env[overrideEnvVar]).toBe("nemoclaw-hermes-sandbox-base-local:12345678"); + restore(); + expect(Object.hasOwn(env, overrideEnvVar)).toBe(false); + }); +}); + +describe("warnUnpreservedUserManagedFiles", () => { let warnSpy: MockInstance; let logSpy: MockInstance; let errorSpy: MockInstance; @@ -99,40 +264,27 @@ describe("backupSandboxStateForRebuild — user-managed file warning", () => { vi.restoreAllMocks(); }); - it( - "emits warning when user-managed files exist in the sandbox", - testTimeoutOptions(15_000), - () => { - probeSpy.mockReturnValue({ - declared: [".env", ".mcp.json"], - existing: [".env", ".mcp.json"], - }); + it("warns directly before a rebuild replaces user-managed MCP files", () => { + probeSpy.mockReturnValue({ + declared: [".env", ".mcp.json"], + existing: [".env", ".mcp.json"], + }); - const { backupSandboxStateForRebuild } = loadRebuildFlowHelpers(); - const result = backupSandboxStateForRebuild( - "alpha", - makeSandboxEntry(), - false, - () => undefined, - () => true, - makeBail(), - ); - - expect(result).toBeTruthy(); - expect(backupSpy).toHaveBeenCalledOnce(); - expect(probeSpy).toHaveBeenCalledOnce(); - expect(probeSpy).toHaveBeenCalledWith("alpha"); - - const warnLines = warnSpy.mock.calls.map((args: unknown[]) => String(args[0])); - expect(warnLines.some((line: string) => line.includes("not preserved by rebuild"))).toBe( - true, - ); - expect(warnLines.some((line: string) => line.includes(".env, .mcp.json"))).toBe(true); - expect(warnLines.some((line: string) => line.includes("Re-add them after rebuild"))).toBe( - true, - ); - }, - ); + const { warnUnpreservedUserManagedFiles } = loadRebuildFlowHelpers(); + warnUnpreservedUserManagedFiles("alpha", () => undefined); + + expect(probeSpy).toHaveBeenCalledOnce(); + expect(probeSpy).toHaveBeenCalledWith("alpha"); + + const warnLines = warnSpy.mock.calls.map((args: unknown[]) => String(args[0])); + expect( + warnLines.some((line: string) => line.includes("will not be preserved if rebuild replaces")), + ).toBe(true); + expect(warnLines.some((line: string) => line.includes(".env, .mcp.json"))).toBe(true); + expect(warnLines.some((line: string) => line.includes("After a successful rebuild"))).toBe( + true, + ); + }); it("emits no warning when probe returns no existing user-managed files", () => { probeSpy.mockReturnValue({ @@ -140,39 +292,23 @@ describe("backupSandboxStateForRebuild — user-managed file warning", () => { existing: [], }); - const { backupSandboxStateForRebuild } = loadRebuildFlowHelpers(); - const result = backupSandboxStateForRebuild( - "alpha", - makeSandboxEntry(), - false, - () => undefined, - () => true, - makeBail(), - ); + const { warnUnpreservedUserManagedFiles } = loadRebuildFlowHelpers(); + warnUnpreservedUserManagedFiles("alpha", () => undefined); - expect(result).toBeTruthy(); expect(probeSpy).toHaveBeenCalledOnce(); const warnLines = warnSpy.mock.calls.map((args: unknown[]) => String(args[0])); - expect(warnLines.some((line: string) => line.includes("not preserved by rebuild"))).toBe(false); + expect(warnLines.some((line: string) => line.includes("will not be preserved"))).toBe(false); }); it("emits no warning when agent declares no user-managed files", () => { probeSpy.mockReturnValue({ declared: [], existing: [] }); - const { backupSandboxStateForRebuild } = loadRebuildFlowHelpers(); - const result = backupSandboxStateForRebuild( - "alpha", - makeSandboxEntry(), - false, - () => undefined, - () => true, - makeBail(), - ); + const { warnUnpreservedUserManagedFiles } = loadRebuildFlowHelpers(); + warnUnpreservedUserManagedFiles("alpha", () => undefined); - expect(result).toBeTruthy(); expect(probeSpy).toHaveBeenCalledOnce(); const warnLines = warnSpy.mock.calls.map((args: unknown[]) => String(args[0])); - expect(warnLines.some((line: string) => line.includes("not preserved by rebuild"))).toBe(false); + expect(warnLines.some((line: string) => line.includes("will not be preserved"))).toBe(false); }); it("skips probe when staleRecovery short-circuits the backup", () => { @@ -191,11 +327,7 @@ describe("backupSandboxStateForRebuild — user-managed file warning", () => { expect(probeSpy).not.toHaveBeenCalled(); }); - it("surfaces a user-visible warning when the probe errors but does not fail the backup", () => { - probeSpy.mockImplementation(() => { - throw new Error("ssh boom"); - }); - + it("does not probe during backup before managed MCP adapter entries are scrubbed", () => { const { backupSandboxStateForRebuild } = loadRebuildFlowHelpers(); const result = backupSandboxStateForRebuild( "alpha", @@ -207,6 +339,18 @@ describe("backupSandboxStateForRebuild — user-managed file warning", () => { ); expect(result).toBeTruthy(); + expect(backupSpy).toHaveBeenCalledOnce(); + expect(probeSpy).not.toHaveBeenCalled(); + }); + + it("surfaces a user-visible warning when the post-scrub probe errors", () => { + probeSpy.mockImplementation(() => { + throw new Error("ssh boom"); + }); + + const { warnUnpreservedUserManagedFiles } = loadRebuildFlowHelpers(); + expect(() => warnUnpreservedUserManagedFiles("alpha", () => undefined)).not.toThrow(); + const warnLines = warnSpy.mock.calls.map((args: unknown[]) => String(args[0])); expect( warnLines.some((line: string) => diff --git a/src/lib/actions/sandbox/rebuild-flow-helpers.ts b/src/lib/actions/sandbox/rebuild-flow-helpers.ts index 3676c35f8f9..6f08df18a43 100644 --- a/src/lib/actions/sandbox/rebuild-flow-helpers.ts +++ b/src/lib/actions/sandbox/rebuild-flow-helpers.ts @@ -5,9 +5,19 @@ import { detectOpenShellStateRpcResultIssue, printOpenShellStateRpcIssue, } from "../../adapters/openshell/gateway-drift"; -import { ensureAgentBaseImage } from "../../agent/onboard"; +import { loadAgent } from "../../agent/defs"; +import { + ensureAgentBaseImage, + getAgentSandboxBaseImageEnvVar, + pinAgentSandboxBaseImageRef, +} from "../../agent/onboard"; +import { CLI_NAME } from "../../cli/branding"; import { RD as _RD, G, R, YW } from "../../cli/terminal-style"; -import { getNamedGatewayLifecycleState } from "../../gateway-runtime-action"; +import { + getNamedGatewayLifecycleState, + recoverNamedGatewayRuntime, +} from "../../gateway-runtime-action"; +import { resolveSandboxGatewayName } from "../../onboard/gateway-binding"; import { captureSandboxListWithGatewayRecovery, printSandboxListFailureWithRecoveryContext, @@ -17,9 +27,6 @@ import * as shields from "../../shields"; import * as registry from "../../state/registry"; import * as sandboxState from "../../state/sandbox"; import * as userManagedFilesProbe from "../../state/user-managed-files-probe"; -import { loadAgent } from "../../agent/defs"; -import { CLI_NAME } from "../../cli/branding"; -import { resolveSandboxGatewayName } from "../../onboard/gateway-binding"; import { getReconciledSandboxGatewayState, printGatewayLifecycleHint, @@ -34,6 +41,43 @@ export type RebuildLiveState = { staleRegistrySnapshot: ReturnType | null; }; +export type RebuildAgentBaseImagePreflight = { + ok: boolean; + imageRef: string | null; + overrideEnvVar: string | null; +}; + +/** + * Select, health-check, and process-pin the gateway recorded for this sandbox + * before any provider or credential preflight. OpenShell's global selection is + * shared mutable metadata; OPENSHELL_GATEWAY keeps every later subprocess in + * this rebuild on the target even if another process selects a sibling gateway. + */ +export async function ensureRebuildTargetGatewaySelected( + sandboxName: string, + sb: RebuildSandboxEntry, + log: (message: string) => void, + bail: (message: string, code?: number) => never, +): Promise { + const gatewayName = resolveSandboxGatewayName(sb); + const recovery = await recoverNamedGatewayRuntime({ gatewayName }); + if (!recovery.recovered || recovery.after.state !== "healthy_named") { + console.error(""); + console.error( + ` ${_RD}Rebuild preflight failed:${R} could not select the target gateway '${gatewayName}'.`, + ); + console.error( + ` Gateway state before: ${recovery.before.state}; after: ${recovery.after.state}.`, + ); + console.error(" Sandbox is untouched — no data was lost."); + bail(`Could not select healthy gateway '${gatewayName}' for sandbox '${sandboxName}'`); + return false; + } + process.env.OPENSHELL_GATEWAY = gatewayName; + log(`Pinned rebuild subprocesses to target gateway '${gatewayName}'`); + return true; +} + export async function resolveRebuildLiveState( sandboxName: string, sb: RebuildSandboxEntry, @@ -49,7 +93,9 @@ export async function resolveRebuildLiveState( log( `openshell sandbox list exit=${isLive.status}, output=${(isLive.output || "").substring(0, 200)}`, ); - const liveListIssue = detectOpenShellStateRpcResultIssue(isLive); + const liveListIssue = detectOpenShellStateRpcResultIssue(isLive, { + gatewayName: recordedGateway, + }); if (liveListIssue) { printOpenShellStateRpcIssue(liveListIssue, { action: `rebuilding sandbox '${sandboxName}'`, @@ -136,9 +182,9 @@ export async function resolveRebuildLiveState( export function openRebuildShieldsWindowForState( sandboxName: string, - staleRecovery: boolean, + recoveryRecreate: boolean, ): { rebuildShieldsWindow: RebuildShieldsWindow | null; staleSandboxWasLocked: boolean } { - if (staleRecovery) { + if (recoveryRecreate) { return { staleSandboxWasLocked: !shields.isShieldsDown(sandboxName), rebuildShieldsWindow: { relocked: false, wasLocked: false }, @@ -153,12 +199,20 @@ export function openRebuildShieldsWindowForState( export function ensureRebuildAgentBaseImage( rebuildAgent: string | null, bail: (msg: string, code?: number) => never, -): boolean { - if (!rebuildAgent) return true; +): RebuildAgentBaseImagePreflight { + if (!rebuildAgent) return { ok: true, imageRef: null, overrideEnvVar: null }; const agentDef = loadAgent(rebuildAgent); + const overrideEnvVar = getAgentSandboxBaseImageEnvVar(agentDef.name); + const hasExplicitOverride = Boolean(process.env[overrideEnvVar]?.trim()); try { - ensureAgentBaseImage(agentDef, { forceBaseImageRebuild: true }); - return true; + const result = ensureAgentBaseImage(agentDef, { + forceBaseImageRebuild: !hasExplicitOverride, + }); + const imageRef = + hasExplicitOverride && result.imageTag + ? pinAgentSandboxBaseImageRef(agentDef.name, result.imageTag) + : result.imageTag; + return { ok: true, imageRef, overrideEnvVar }; } catch (err) { const message = err instanceof Error ? err.message : String(err); console.error(""); @@ -167,10 +221,32 @@ export function ensureRebuildAgentBaseImage( console.error(""); console.error(" Sandbox is untouched — no data was lost."); bail(message); - return false; + return { ok: false, imageRef: null, overrideEnvVar: null }; } } +export function pinRebuildAgentBaseImageForRecreate( + preflight: RebuildAgentBaseImagePreflight, + env: NodeJS.ProcessEnv = process.env, +): () => void { + const { imageRef, overrideEnvVar } = preflight; + if (!preflight.ok || !imageRef || !overrideEnvVar) return () => undefined; + + const hadPriorValue = Object.hasOwn(env, overrideEnvVar); + const priorValue = env[overrideEnvVar]; + env[overrideEnvVar] = imageRef; + let restored = false; + return () => { + if (restored) return; + restored = true; + if (hadPriorValue && priorValue !== undefined) { + env[overrideEnvVar] = priorValue; + } else { + delete env[overrideEnvVar]; + } + }; +} + export function backupSandboxStateForRebuild( sandboxName: string, sb: RebuildSandboxEntry, @@ -221,11 +297,19 @@ export function backupSandboxStateForRebuild( ); } console.log(` Backup: ${backupManifest.backupPath}`); - warnUnpreservedUserManagedFiles(sandboxName, log); return backupManifest; } -function warnUnpreservedUserManagedFiles(sandboxName: string, log: (msg: string) => void): void { +/** + * Warn only after MCP rebuild preparation has scrubbed NemoClaw-owned adapter + * entries. In particular, a managed-only Deep Agents `.mcp.json` is removed by + * that transaction; if the file still exists at this point it contains + * additional user-owned content that the state backup intentionally excludes. + */ +export function warnUnpreservedUserManagedFiles( + sandboxName: string, + log: (msg: string) => void, +): void { let probe: userManagedFilesProbe.UserManagedFilesProbe; try { probe = userManagedFilesProbe.probeUserManagedFiles(sandboxName); @@ -247,7 +331,7 @@ function warnUnpreservedUserManagedFiles(sandboxName: string, log: (msg: string) return; } console.warn( - ` ${YW}⚠${R} User-managed files in sandbox not preserved by rebuild: ${probe.existing.join(", ")}`, + ` ${YW}⚠${R} User-managed files will not be preserved if rebuild replaces this sandbox: ${probe.existing.join(", ")}`, ); - console.warn(" Re-add them after rebuild, or manage them from the host."); + console.warn(" After a successful rebuild, re-add them or manage them from the host."); } diff --git a/src/lib/actions/sandbox/rebuild-flow-test-fixtures.ts b/src/lib/actions/sandbox/rebuild-flow-test-fixtures.ts new file mode 100644 index 00000000000..04f0cf8702f --- /dev/null +++ b/src/lib/actions/sandbox/rebuild-flow-test-fixtures.ts @@ -0,0 +1,91 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export function makeActiveTeamsMessagingPlan() { + return { + schemaVersion: 1, + sandboxName: "alpha", + agent: "openclaw", + workflow: "rebuild", + channels: [ + { + channelId: "teams", + displayName: "Microsoft Teams", + authMode: "token-paste", + active: true, + selected: true, + configured: true, + disabled: false, + inputs: [ + { + channelId: "teams", + inputId: "appId", + kind: "config", + required: true, + sourceEnv: "MSTEAMS_APP_ID", + statePath: "teamsConfig.appId", + value: "teams-app-id", + }, + { + channelId: "teams", + inputId: "clientSecret", + kind: "secret", + required: true, + sourceEnv: "MSTEAMS_APP_PASSWORD", + credentialAvailable: true, + }, + { + channelId: "teams", + inputId: "tenantId", + kind: "config", + required: true, + sourceEnv: "MSTEAMS_TENANT_ID", + statePath: "teamsConfig.tenantId", + value: "teams-tenant-id", + }, + { + channelId: "teams", + inputId: "webhookPort", + kind: "config", + required: false, + sourceEnv: "MSTEAMS_PORT", + statePath: "teamsConfig.webhookPort", + value: "3978", + }, + ], + hostForward: { + channelId: "teams", + port: 3978, + label: "Microsoft Teams webhook", + }, + hooks: [], + }, + ], + disabledChannels: [], + credentialBindings: [], + networkPolicy: { presets: ["teams"], entries: [] }, + agentRender: [], + buildSteps: [], + stateUpdates: [], + healthChecks: [], + }; +} + +export function makePreparedRecoveryManifest() { + return { + version: 1, + sandboxName: "alpha", + timestamp: "2026-07-01T06-50-42-044Z", + agentType: "openclaw", + agentVersion: "0.1.0", + expectedVersion: "0.2.0", + stateDirs: ["workspace"], + backedUpDirs: ["workspace"], + stateFiles: [], + dir: "/sandbox/.openclaw", + backupPath: "/tmp/rebuild-backups/alpha/2026-07-01T06-50-42-044Z", + blueprintDigest: null, + policyPresets: ["npm"], + customPolicies: [], + }; +} diff --git a/src/lib/actions/sandbox/rebuild-flow.test.ts b/src/lib/actions/sandbox/rebuild-flow.test.ts index fcb0f4baf82..301fb433f68 100644 --- a/src/lib/actions/sandbox/rebuild-flow.test.ts +++ b/src/lib/actions/sandbox/rebuild-flow.test.ts @@ -1,673 +1,14 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { afterEach, beforeEach, describe, expect, it } from "vitest"; - -import { - createRebuildFlowHarness, - makePreparedRecoveryManifest, - resetRebuildFlowTestEnvironment, - restoreRebuildFlowTestEnvironment, - snapshotEnv, -} from "../../../../test/helpers/rebuild-flow-harness"; - -function makeActiveTeamsMessagingPlan() { - return { - schemaVersion: 1, - sandboxName: "alpha", - agent: "openclaw", - workflow: "rebuild", - channels: [ - { - channelId: "teams", - displayName: "Microsoft Teams", - authMode: "token-paste", - active: true, - selected: true, - configured: true, - disabled: false, - inputs: [ - { - channelId: "teams", - inputId: "appId", - kind: "config", - required: true, - sourceEnv: "MSTEAMS_APP_ID", - statePath: "teamsConfig.appId", - value: "teams-app-id", - }, - { - channelId: "teams", - inputId: "clientSecret", - kind: "secret", - required: true, - sourceEnv: "MSTEAMS_APP_PASSWORD", - credentialAvailable: true, - }, - { - channelId: "teams", - inputId: "tenantId", - kind: "config", - required: true, - sourceEnv: "MSTEAMS_TENANT_ID", - statePath: "teamsConfig.tenantId", - value: "teams-tenant-id", - }, - { - channelId: "teams", - inputId: "webhookPort", - kind: "config", - required: false, - sourceEnv: "MSTEAMS_PORT", - statePath: "teamsConfig.webhookPort", - value: "3978", - }, - ], - hostForward: { - channelId: "teams", - port: 3978, - label: "Microsoft Teams webhook", - }, - hooks: [], - }, - ], - disabledChannels: [], - credentialBindings: [], - networkPolicy: { presets: ["teams"], entries: [] }, - agentRender: [], - buildSteps: [], - stateUpdates: [], - healthChecks: [], - }; -} - -describe("rebuildSandbox flow", () => { - beforeEach(resetRebuildFlowTestEnvironment); - afterEach(restoreRebuildFlowTestEnvironment); - - it("backs up, recreates, restores, reapplies policy, and relocks on a successful OpenClaw rebuild", async () => { - const harness = createRebuildFlowHarness({ - applyPreset: () => true, - }); - - await expect( - harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), - ).resolves.toBeUndefined(); - - expect(harness.backupSandboxStateSpy).toHaveBeenCalledWith("alpha"); - expect(harness.runOpenshellSpy).toHaveBeenCalledWith( - ["sandbox", "delete", "alpha"], - expect.objectContaining({ ignoreError: true }), - ); - expect(harness.onboardSpy).toHaveBeenCalledWith( - expect.objectContaining({ - resume: true, - nonInteractive: true, - recreateSandbox: true, - autoYes: true, - }), - ); - expect(harness.restoreSandboxStateSpy).toHaveBeenCalledWith( - "alpha", - "/tmp/nemoclaw-rebuild-backup", - ); - expect(harness.applyPresetSpy).toHaveBeenCalledWith("alpha", "npm"); - expect(harness.applyPresetSpy).toHaveBeenCalledWith("alpha", "bad"); - expect(harness.applyPresetSpy).toHaveBeenCalledWith("alpha", "throw"); - expect(harness.registryUpdateSpy).toHaveBeenCalledWith("alpha", { - agentVersion: "0.2.0", - policies: ["npm", "bad", "throw"], - }); - expect(harness.executeSandboxCommandSpy).toHaveBeenCalledWith("alpha", "openclaw doctor --fix"); - expect(harness.relockSpy).toHaveBeenCalledWith("alpha", expect.any(Object), true, "nemoclaw"); - expect(process.env.NEMOCLAW_SANDBOX_NAME).toBe("alpha"); - expect(harness.logSpy.mock.calls.map((call) => String(call[0])).join("\n")).toContain( - "rebuilt successfully", - ); - }); - - it("restores the validated pre-upgrade manifest without taking a second backup (#6114)", async () => { - const harness = createRebuildFlowHarness({ - applyPreset: () => true, - sandboxListOutput: "alpha Error", - }); - const recoveryManifest = makePreparedRecoveryManifest(); - - await expect( - harness.rebuildSandbox("alpha", ["--yes"], { - throwOnError: true, - recoveryManifest, - }), - ).resolves.toBeUndefined(); - - expect(harness.backupSandboxStateSpy).not.toHaveBeenCalled(); - expect(harness.runOpenshellSpy).toHaveBeenCalledWith( - ["sandbox", "delete", "alpha"], - expect.objectContaining({ ignoreError: true }), - ); - expect(harness.restoreSandboxStateSpy).toHaveBeenCalledWith( - "alpha", - recoveryManifest.backupPath, - ); - }); - - it("rejects a mismatched prepared manifest before deleting the sandbox (#6114)", async () => { - const harness = createRebuildFlowHarness({ - recoveryManifestValidation: () => ({ - ok: false, - reason: "manifest sandbox 'beta' does not match 'alpha'", - }), - }); - - await expect( - harness.rebuildSandbox("alpha", ["--yes"], { - throwOnError: true, - recoveryManifest: makePreparedRecoveryManifest(), - }), - ).rejects.toThrow("Invalid recovery manifest"); - - expect(harness.backupSandboxStateSpy).not.toHaveBeenCalled(); - expect(harness.runOpenshellSpy).not.toHaveBeenCalledWith( - ["sandbox", "delete", "alpha"], - expect.anything(), - ); - expect(harness.onboardSpy).not.toHaveBeenCalled(); - }); - - it("revalidates the prepared manifest immediately before deleting the sandbox (#6114)", async () => { - let validationCount = 0; - const harness = createRebuildFlowHarness({ - recoveryManifestValidation: (manifest) => { - validationCount++; - return validationCount === 1 - ? { ok: true as const, manifest } - : { ok: false as const, reason: "persisted backup identity changed during validation" }; - }, - }); - - await expect( - harness.rebuildSandbox("alpha", ["--yes"], { - throwOnError: true, - recoveryManifest: makePreparedRecoveryManifest(), - }), - ).rejects.toThrow("Invalid recovery manifest"); - - expect(validationCount).toBe(2); - expect(harness.backupSandboxStateSpy).not.toHaveBeenCalled(); - expect(harness.runOpenshellSpy).not.toHaveBeenCalledWith( - ["sandbox", "delete", "alpha"], - expect.anything(), - ); - expect(harness.onboardSpy).not.toHaveBeenCalled(); - }); - - it("rejects same-agent registry configuration drift before deleting the sandbox (#6114)", async () => { - const harness = createRebuildFlowHarness({ - preDeleteSandboxEntry: { - name: "alpha", - provider: "compatible-endpoint", - model: "new-model", - policies: ["npm", "github"], - agent: null, - agentVersion: "0.1.0", - nemoclawVersion: "0.0.71", - }, - }); - - await expect( - harness.rebuildSandbox("alpha", ["--yes"], { - throwOnError: true, - recoveryManifest: makePreparedRecoveryManifest(), - }), - ).rejects.toThrow("Recovery registry configuration changed during preflight"); - - expect(harness.backupSandboxStateSpy).not.toHaveBeenCalled(); - expect(harness.runOpenshellSpy).not.toHaveBeenCalledWith( - ["sandbox", "delete", "alpha"], - expect.anything(), - ); - }); - - it("uses the single refreshed registry snapshot for recreate rollback (#6114)", async () => { - const harness = createRebuildFlowHarness({ - preDeleteDefaultSandbox: "beta", - onboard: () => { - throw new Error("recreate failed"); - }, - }); - - await expect( - harness.rebuildSandbox("alpha", ["--yes"], { - throwOnError: true, - recoveryManifest: makePreparedRecoveryManifest(), - }), - ).rejects.toThrow("Recreate failed"); - - expect(harness.restoreSandboxEntrySpy).toHaveBeenCalledWith( - expect.objectContaining({ name: "alpha", agentVersion: "0.1.0" }), - { reclaimDefault: null }, - ); - }); - - it("rejects a latest-backup change immediately before deleting the sandbox (#6114)", async () => { - const harness = createRebuildFlowHarness({ - preDeleteLatestManifest: { - ...makePreparedRecoveryManifest(), - timestamp: "2026-07-01T07-00-00-000Z", - backupPath: "/tmp/rebuild-backups/alpha/2026-07-01T07-00-00-000Z", - }, - }); - - await expect( - harness.rebuildSandbox("alpha", ["--yes"], { - throwOnError: true, - recoveryManifest: makePreparedRecoveryManifest(), - }), - ).rejects.toThrow("Recovery backup identity changed during preflight"); - - expect(harness.backupSandboxStateSpy).not.toHaveBeenCalled(); - expect(harness.runOpenshellSpy).not.toHaveBeenCalledWith( - ["sandbox", "delete", "alpha"], - expect.anything(), - ); - }); - - it("restores the registry entry when prepared-backup recreation fails (#6114)", async () => { - const harness = createRebuildFlowHarness({ - onboard: () => { - throw new Error("recreate failed"); - }, - }); - const recoveryManifest = makePreparedRecoveryManifest(); - - await expect( - harness.rebuildSandbox("alpha", ["--yes"], { - throwOnError: true, - recoveryManifest, - }), - ).rejects.toThrow("Recreate failed"); - - expect(harness.backupSandboxStateSpy).not.toHaveBeenCalled(); - expect(harness.restoreSandboxEntrySpy).toHaveBeenCalledWith( - expect.objectContaining({ name: "alpha", agentVersion: "0.1.0" }), - { reclaimDefault: "alpha" }, - ); - expect(harness.restoreSandboxStateSpy).not.toHaveBeenCalled(); - }); - - it("restores enabled messaging presets while pruning disabled ones from final policies", async () => { - const disabledSlackPlan = { - schemaVersion: 1, - sandboxName: "alpha", - agent: "openclaw", - workflow: "rebuild", - channels: [ - { channelId: "telegram", disabled: false }, - { channelId: "discord", disabled: false }, - { channelId: "whatsapp", disabled: false }, - { channelId: "wechat", disabled: false }, - { channelId: "slack", disabled: true }, - ], - disabledChannels: ["slack"], - credentialBindings: [], - networkPolicy: { presets: [], entries: [] }, - agentRender: [], - buildSteps: [], - stateUpdates: [], - healthChecks: [], - }; - const harness = createRebuildFlowHarness({ - applyPreset: () => true, - backupPolicyPresets: ["slack", "npm", "pypi", "telegram"], - buildMessagingRebuildPlan: () => disabledSlackPlan, - }); - - await expect( - harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), - ).resolves.toBeUndefined(); - - expect(harness.applyPresetSpy.mock.calls.map((call) => call[1])).toEqual([ - "npm", - "pypi", - "telegram", - "discord", - "whatsapp", - "wechat", - ]); - expect(harness.registryUpdateSpy).toHaveBeenCalledWith("alpha", { - agentVersion: "0.2.0", - policies: ["npm", "pypi", "telegram", "discord", "whatsapp", "wechat"], - }); - }); - - it("prunes the disabled Teams preset from the final registry policies after rebuild", async () => { - const disabledTeamsPlan = { - schemaVersion: 1, - sandboxName: "alpha", - agent: "openclaw", - workflow: "rebuild", - channels: [], - disabledChannels: ["teams"], - credentialBindings: [], - networkPolicy: { presets: [], entries: [] }, - agentRender: [], - buildSteps: [], - stateUpdates: [], - healthChecks: [], - }; - const harness = createRebuildFlowHarness({ - applyPreset: () => true, - backupPolicyPresets: ["teams", "npm"], - buildMessagingRebuildPlan: () => disabledTeamsPlan, - }); - - await expect( - harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), - ).resolves.toBeUndefined(); - - expect(harness.applyPresetSpy).toHaveBeenCalledWith("alpha", "npm"); - expect(harness.applyPresetSpy).not.toHaveBeenCalledWith("alpha", "teams"); - expect(harness.registryUpdateSpy).toHaveBeenCalledWith("alpha", { - agentVersion: "0.2.0", - policies: ["npm"], - }); - }); - - it("aborts before backup/delete when messaging manifest staging fails", async () => { - const harness = createRebuildFlowHarness({ - buildMessagingRebuildPlan: () => { - throw new Error("manifest boom"); - }, - }); - - await expect( - harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), - ).rejects.toThrow("manifest boom"); - - const errors = harness.errorSpy.mock.calls.map((call) => String(call[0])).join("\n"); - expect(errors).toContain("messaging manifest plan could not be staged"); - expect(errors).toContain("Sandbox is untouched"); - expect(harness.backupSandboxStateSpy).not.toHaveBeenCalled(); - expect(harness.runOpenshellSpy).not.toHaveBeenCalledWith( - ["sandbox", "delete", "alpha"], - expect.anything(), - ); - expect(harness.onboardSpy).not.toHaveBeenCalled(); - }); - - it("starts the active Teams host forward after a successful rebuild", async () => { - const plan = makeActiveTeamsMessagingPlan(); - const harness = createRebuildFlowHarness({ - applyPreset: () => true, - buildMessagingRebuildPlan: () => plan, - }); - - await expect( - harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), - ).resolves.toBeUndefined(); - - expect(harness.ensureMessagingHostForwardAfterRebuildSpy).toHaveBeenCalledWith("alpha", plan); - expect( - harness.ensureMessagingHostForwardAfterRebuildSpy.mock.invocationCallOrder[0], - ).toBeGreaterThan(harness.onboardSpy.mock.invocationCallOrder[0]); - }); - - it("finishes the rebuild while surfacing incomplete post-restore work", async () => { - const harness = createRebuildFlowHarness({ - executeSandboxCommand: () => ({ status: 1, stdout: "", stderr: "hash refresh failed" }), - repairMutableConfigPerms: () => ({ - applied: false, - skipReason: "unreadable", - reason: "cannot stat mutable config", - }), - restoreSandboxState: () => ({ - success: false, - restoredDirs: ["workspace"], - restoredFiles: [], - failedDirs: ["config"], - failedFiles: ["user.md"], - }), - }); - - await expect( - harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), - ).resolves.toBeUndefined(); - - const output = harness.logSpy.mock.calls.map((call) => String(call[0])).join("\n"); - expect(output).toContain("rebuilt but some post-restore steps were incomplete"); - expect(output).toContain("State restore was incomplete"); - expect(output).toContain("Mutable config permissions were not verified"); - expect(output).toContain("Mutable OpenClaw config hash was not refreshed"); - expect(harness.applyPresetSpy).toHaveBeenCalledWith("alpha", "bad"); - expect(harness.applyPresetSpy).toHaveBeenCalledWith("alpha", "throw"); - expect(harness.errorSpy).toHaveBeenCalledWith(expect.stringContaining("bad, throw")); - expect(harness.relockSpy).toHaveBeenCalledWith("alpha", expect.any(Object), true, "nemoclaw"); - expect(harness.registryUpdateSpy).toHaveBeenCalledWith("alpha", { - agentVersion: "0.2.0", - policies: ["npm"], - }); - expect(output).toContain("Policy presets failed to reapply: bad, throw"); - }); - - it("isolates ambient onboard-selection env during recreate, then restores it (#5735)", async () => { - // Simulate an installer that just onboarded an unrelated Deep Agents - // sandbox and left its selection env in the process before - // `upgrade-sandboxes --auto` rebuilds an existing OpenClaw (registry agent - // null) sandbox. - const restoreEnv = snapshotEnv(["NEMOCLAW_AGENT", "NEMOCLAW_PROVIDER_KEY"]); - process.env.NEMOCLAW_AGENT = "langchain-deepagents-code"; - process.env.NEMOCLAW_PROVIDER_KEY = "sk-bogus-installer-key"; - - let envSeenInsideOnboard: { - agent: string | undefined; - providerKey: string | undefined; - } | null = null; - - try { - const harness = createRebuildFlowHarness({ - applyPreset: () => true, - onboard: () => { - // onboard --resume's agent/provider/credential resolution reads these - // directly from process.env; they must be gone during recreate so the - // pinned registry session wins. - envSeenInsideOnboard = { - agent: process.env.NEMOCLAW_AGENT, - providerKey: process.env.NEMOCLAW_PROVIDER_KEY, - }; - }, - }); - - await expect( - harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), - ).resolves.toBeUndefined(); - - expect(envSeenInsideOnboard).toEqual({ agent: undefined, providerKey: undefined }); - // The mismatch (env agent != registry agent) is surfaced before delete. - const logged = harness.logSpy.mock.calls.map((call) => String(call[0])).join("\n"); - expect(logged).toContain("Ignoring ambient NEMOCLAW_AGENT='langchain-deepagents-code'"); - // The caller's env is left exactly as it was after the rebuild. - expect(process.env.NEMOCLAW_AGENT).toBe("langchain-deepagents-code"); - expect(process.env.NEMOCLAW_PROVIDER_KEY).toBe("sk-bogus-installer-key"); - } finally { - restoreEnv(); - } - }); - - it("recreates a matching-session custom-endpoint sandbox from a validated session endpoint while ignoring hostile ambient values for PRA-4 (#5735)", async () => { - // Matching session (sandboxName === target) with a custom endpoint recorded - // in that session. Hostile ambient NEMOCLAW_ENDPOINT_URL/PROVIDER/MODEL must - // be absent during recreate so onboard --resume uses the validated session - // endpoint selected by prepareRebuildResumeConfig. - const restoreEnv = snapshotEnv([ - "NEMOCLAW_ENDPOINT_URL", - "NEMOCLAW_PROVIDER", - "NEMOCLAW_MODEL", - "COMPATIBLE_API_KEY", - ]); - process.env.NEMOCLAW_ENDPOINT_URL = "https://attacker.example.test/v1"; - process.env.NEMOCLAW_PROVIDER = "build"; - process.env.NEMOCLAW_MODEL = "attacker-model"; - process.env.COMPATIBLE_API_KEY = "compat-key"; // pass credential preflight - - let envSeenInsideOnboard: Record | null = null; - try { - const harness = createRebuildFlowHarness({ - applyPreset: () => true, - sandboxEntry: { provider: "compatible-endpoint", model: "session-model" }, - onboard: () => { - envSeenInsideOnboard = { - endpoint: process.env.NEMOCLAW_ENDPOINT_URL, - provider: process.env.NEMOCLAW_PROVIDER, - model: process.env.NEMOCLAW_MODEL, - }; - }, - }); - // The custom endpoint lives only in this sandbox's own matching session; - // it is canonicalized at the pre-delete rebuild boundary before rewrite. - harness.session.endpointUrl = "https://my-custom-endpoint.example/v1?x=1#frag"; - - await expect( - harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), - ).resolves.toBeUndefined(); - - // Ambient selection env was isolated during the recreate. - expect(envSeenInsideOnboard).toEqual({ - endpoint: undefined, - provider: undefined, - model: undefined, - }); - expect(harness.session.endpointUrl).toBe("https://my-custom-endpoint.example/v1"); - // Provider/model come from the registry entry, not the ambient values. - expect(harness.session.provider).toBe("compatible-endpoint"); - expect(harness.session.model).toBe("session-model"); - // Caller env restored afterward. - expect(process.env.NEMOCLAW_ENDPOINT_URL).toBe("https://attacker.example.test/v1"); - expect(process.env.NEMOCLAW_PROVIDER).toBe("build"); - expect(process.env.NEMOCLAW_MODEL).toBe("attacker-model"); - } finally { - restoreEnv(); - } - }); - - it("aborts before backup/delete when a custom-endpoint target has no matching session (#5735)", async () => { - // Installer flow: the loaded onboard session belongs to a different - // (just-created) sandbox, and the target uses a custom OpenAI-compatible - // provider whose base URL is only in its own session. Recreating it would - // either fail or reconfigure against the wrong endpoint after deletion — so - // rebuild must fail closed with the sandbox intact. - const restoreEnv = snapshotEnv(["COMPATIBLE_API_KEY"]); - process.env.COMPATIBLE_API_KEY = "compat-key"; // pass credential preflight first - try { - const harness = createRebuildFlowHarness({ - sandboxEntry: { provider: "compatible-endpoint", model: "custom-model" }, - sessionSandboxName: "some-other-sandbox", - }); - - await expect( - harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), - ).rejects.toThrow("Cannot determine recreate endpoint"); - - const errors = harness.errorSpy.mock.calls.map((call) => String(call[0])).join("\n"); - expect(errors).toContain("cannot determine the inference endpoint"); - expect(errors).toContain("Sandbox is untouched"); - expect(harness.backupSandboxStateSpy).not.toHaveBeenCalled(); - expect(harness.runOpenshellSpy).not.toHaveBeenCalledWith( - ["sandbox", "delete", "alpha"], - expect.anything(), - ); - expect(harness.onboardSpy).not.toHaveBeenCalled(); - } finally { - restoreEnv(); - } - }); - - it("rebuilds a known-remote target even when the session belongs to another sandbox (#5735)", async () => { - // The same non-matching-session scenario but with a provider that has a - // canonical endpoint (NVIDIA Endpoints): the endpoint is re-derivable from - // registry, so the rebuild proceeds (no abort) and pins it. - const restoreEnv = snapshotEnv(["NVIDIA_INFERENCE_API_KEY"]); - process.env.NVIDIA_INFERENCE_API_KEY = "nvapi-key"; // pass credential preflight - try { - const harness = createRebuildFlowHarness({ - applyPreset: () => true, - sandboxEntry: { provider: "nvidia-prod", model: "nvidia/nemotron" }, - sessionSandboxName: "some-other-sandbox", - }); - // A stale endpoint carried over from the unrelated session must be - // repinned from the nvidia-prod canonical config, not reused as-is. - const staleEndpoint = "https://stale.example.test/v1"; - harness.session.endpointUrl = staleEndpoint; - - await expect( - harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), - ).resolves.toBeUndefined(); - - expect(harness.onboardSpy).toHaveBeenCalled(); - expect(harness.session.endpointUrl).not.toBe(staleEndpoint); - expect(harness.runOpenshellSpy).toHaveBeenCalledWith( - ["sandbox", "delete", "alpha"], - expect.objectContaining({ ignoreError: true }), - ); - } finally { - restoreEnv(); - } - }); - - it("does not abort a routed (nvidia-router) target with a non-matching session (#5735)", async () => { - // nvidia-router derives its endpoint from the blueprint, not the session, so - // the endpoint preflight must not treat it like a custom endpoint and abort. - const harness = createRebuildFlowHarness({ - applyPreset: () => true, - sandboxEntry: { provider: "nvidia-router", model: "router-model" }, - sessionSandboxName: "some-other-sandbox", - }); - - await expect( - harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), - ).resolves.toBeUndefined(); - - expect(harness.runOpenshellSpy).toHaveBeenCalledWith( - ["sandbox", "delete", "alpha"], - expect.objectContaining({ ignoreError: true }), - ); - expect(harness.onboardSpy).toHaveBeenCalled(); - }); - - it("marks recreate onboarding failures as terminal and preserves retry cleanup", async () => { - const harness = createRebuildFlowHarness({ - onboard: (session) => { - session.lastStepStarted = "sandbox"; - throw new Error("inner recreate boom"); - }, - }); - - await expect( - harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), - ).rejects.toThrow("Recreate failed"); - - expect(harness.releaseOnboardLockSpy).toHaveBeenCalled(); - expect(harness.markStepFailedSpy).toHaveBeenCalledWith( - "sandbox", - "Rebuild recreate failed", - expect.objectContaining({ updateMachine: true }), - ); - expect(harness.session).toMatchObject({ - status: "failed", - failure: { step: "sandbox", message: "Rebuild recreate failed" }, - machine: { state: "failed" }, - steps: { sandbox: { status: "failed", error: "Rebuild recreate failed" } }, - }); - expect(harness.relockSpy).toHaveBeenCalledWith("alpha", expect.any(Object), false, "nemoclaw"); - expect(process.env.NEMOCLAW_SANDBOX_NAME).toBe("alpha"); - - // #5735 (PRA-T2): preconditions (credential/endpoint) passed, so the - // delete proceeded; when onboard() then fails for a residual runtime reason, - // the operator must get a clear fatal recovery path with the preserved - // backup — not a silent loss. Precondition-class failures are caught before - // delete by prepareRebuildResumeConfig (covered by the abort tests above). - const errors = harness.errorSpy.mock.calls.map((call) => String(call[0])).join("\n"); - expect(errors).toContain("Recreate failed after sandbox was destroyed"); - expect(errors).toContain("Backup is preserved at: /tmp/nemoclaw-rebuild-backup"); - expect(errors).toContain("onboard --resume"); - }); -}); +import { registerRebuildFlowLifecycleTests } from "../../../../test/helpers/rebuild-flow-lifecycle-cases"; +import { registerRebuildFlowRecoveryTests } from "../../../../test/helpers/rebuild-flow-recovery-cases"; +import { registerRebuildFlowTargetCredentialsTests } from "../../../../test/helpers/rebuild-flow-target-credentials-cases"; +import { registerRebuildFlowTargetImageTests } from "../../../../test/helpers/rebuild-flow-target-image-cases"; +import { registerRebuildFlowTargetSessionTests } from "../../../../test/helpers/rebuild-flow-target-session-cases"; + +registerRebuildFlowLifecycleTests(); +registerRebuildFlowRecoveryTests(); +registerRebuildFlowTargetSessionTests(); +registerRebuildFlowTargetCredentialsTests(); +registerRebuildFlowTargetImageTests(); diff --git a/src/lib/actions/sandbox/rebuild-gateway-drift.test.ts b/src/lib/actions/sandbox/rebuild-gateway-drift.test.ts index 9671234f92a..ea879563862 100644 --- a/src/lib/actions/sandbox/rebuild-gateway-drift.test.ts +++ b/src/lib/actions/sandbox/rebuild-gateway-drift.test.ts @@ -19,6 +19,8 @@ const sandboxSession = requireDist("../../state/sandbox-session.js"); const onboardSession = requireDist("../../state/onboard-session.js"); const sandboxVersion = requireDist("../../sandbox/version.js"); const agentRuntime = requireDist("../../agent/runtime.js"); +const rebuildUsageNotice = requireDist("./rebuild-usage-notice.js"); +const rebuildImagePreflight = requireDist("./rebuild-custom-image-preflight.js"); const { rebuildSandbox } = requireDist("./rebuild.js") as { rebuildSandbox: RebuildSandbox; }; @@ -72,7 +74,11 @@ describe("rebuild gateway drift preflight", () => { .mockReturnValue({ status: 0, output: "" } as never); recoverNamedGatewayRuntimeSpy = vi .spyOn(gatewayRuntime, "recoverNamedGatewayRuntime") - .mockResolvedValue({ recovered: true }); + .mockResolvedValue({ + recovered: true, + before: { state: "healthy_named" }, + after: { state: "healthy_named" }, + }); spies.push( detectPreflightIssueSpy, @@ -88,15 +94,29 @@ describe("rebuild gateway drift preflight", () => { policies: [], nimContainer: null, agent: null, + nemoclawVersion: "0.1.0", + dashboardPort: 18789, + gatewayName: "nemoclaw", + gatewayPort: 8080, } as never), + vi.spyOn(registry, "updateSandbox").mockReturnValue(true), vi.spyOn(resolve, "resolveOpenshell").mockReturnValue(null), vi.spyOn(sandboxSession, "getActiveSandboxSessions").mockReturnValue({ detected: false, sessions: [], }), vi.spyOn(onboardSession, "loadSession").mockReturnValue(null), + vi.spyOn(onboardSession, "acquireOnboardLock").mockReturnValue({ acquired: true }), + vi.spyOn(onboardSession, "releaseOnboardLock").mockImplementation(() => undefined), vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue(null), vi.spyOn(agentRuntime, "getAgentDisplayName").mockReturnValue("OpenClaw"), + vi + .spyOn(requireDist("../../onboard.js"), "preflightAuthoritativeRebuildTarget") + .mockResolvedValue(undefined), + vi + .spyOn(rebuildImagePreflight, "preflightRebuildImage") + .mockResolvedValue({ ok: true, imageTag: null }), + vi.spyOn(rebuildUsageNotice, "ensureRebuildUsageNoticeAccepted").mockResolvedValue(true), checkAgentVersionSpy, ); }); @@ -219,7 +239,11 @@ describe("rebuild gateway drift preflight", () => { .mockReturnValue({ status: 0, output: "" } as never); recoverNamedGatewayRuntimeSpy = vi .spyOn(gatewayRuntime, "recoverNamedGatewayRuntime") - .mockResolvedValue({ recovered: true }); + .mockResolvedValue({ + recovered: true, + before: { state: "healthy_named" }, + after: { state: "healthy_named" }, + }); checkAgentVersionSpy = vi .spyOn(sandboxVersion, "checkAgentVersion") .mockReturnValue({ expectedVersion: "0.1.0", sandboxVersion: "0.0.1" } as never); @@ -245,15 +269,25 @@ describe("rebuild gateway drift preflight", () => { agent: null, gatewayName: "nemoclaw-12345", gatewayPort: 12345, + nemoclawVersion: "0.1.0", + dashboardPort: 18789, } as never), + vi.spyOn(registry, "updateSandbox").mockReturnValue(true), vi.spyOn(resolve, "resolveOpenshell").mockReturnValue(null), vi.spyOn(sandboxSession, "getActiveSandboxSessions").mockReturnValue({ detected: false, sessions: [], }), vi.spyOn(onboardSession, "loadSession").mockReturnValue(null), + vi.spyOn(onboardSession, "acquireOnboardLock").mockReturnValue({ acquired: true }), + vi.spyOn(onboardSession, "releaseOnboardLock").mockImplementation(() => undefined), vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue(null), vi.spyOn(agentRuntime, "getAgentDisplayName").mockReturnValue("OpenClaw"), + vi.spyOn(onboardMod, "preflightAuthoritativeRebuildTarget").mockResolvedValue(undefined), + vi + .spyOn(rebuildImagePreflight, "preflightRebuildImage") + .mockResolvedValue({ ok: true, imageTag: null }), + vi.spyOn(rebuildUsageNotice, "ensureRebuildUsageNoticeAccepted").mockResolvedValue(true), checkAgentVersionSpy, vi.spyOn(destroy, "removeSandboxRegistryEntry").mockImplementation(() => undefined), vi.spyOn(onboardMod, "onboard").mockRejectedValue(new Error("recreate-stub")), @@ -274,7 +308,7 @@ describe("rebuild gateway drift preflight", () => { expect(listCalls).toBe(2); }); - it("does not recover generic sandbox list failures", async () => { + it("does not retry gateway recovery for generic sandbox list failures", async () => { detectPreflightIssueSpy.mockReturnValue(null); captureOpenshellSpy.mockReturnValue({ status: 1, output: "unknown option: sandbox list" }); @@ -282,7 +316,8 @@ describe("rebuild gateway drift preflight", () => { "Failed to query running sandboxes from OpenShell.", ); - expect(recoverNamedGatewayRuntimeSpy).not.toHaveBeenCalled(); + expect(recoverNamedGatewayRuntimeSpy).toHaveBeenCalledTimes(2); + expect(recoverNamedGatewayRuntimeSpy).toHaveBeenCalledWith({ gatewayName: "nemoclaw" }); expect(captureOpenshellSpy).toHaveBeenCalledTimes(1); }); }); diff --git a/src/lib/actions/sandbox/rebuild-gpu-opt-out.test.ts b/src/lib/actions/sandbox/rebuild-gpu-opt-out.test.ts index 9fbaf11bb60..16316d03082 100644 --- a/src/lib/actions/sandbox/rebuild-gpu-opt-out.test.ts +++ b/src/lib/actions/sandbox/rebuild-gpu-opt-out.test.ts @@ -3,7 +3,11 @@ import { describe, expect, it } from "vitest"; -import { buildRebuildRecreateOnboardOpts, rebuildShouldOptOutGpu } from "./rebuild-gpu-opt-out"; +import { + buildRebuildRecreateOnboardOpts, + getRebuildSandboxGpuOverrides, + rebuildShouldOptOutGpu, +} from "./rebuild-gpu-opt-out"; describe("rebuildShouldOptOutGpu", () => { it("returns false when the registry entry is null", () => { @@ -102,25 +106,69 @@ describe("rebuildShouldOptOutGpu", () => { }); }); +describe("getRebuildSandboxGpuOverrides", () => { + it("pins forced GPU mode and its recorded device", () => { + expect( + getRebuildSandboxGpuOverrides({ + sandboxGpuMode: "1", + sandboxGpuEnabled: true, + sandboxGpuDevice: "nvidia.com/gpu=2", + }), + ).toEqual({ + sandboxGpu: "enable", + sandboxGpuDevice: "nvidia.com/gpu=2", + sessionGpuPassthrough: true, + }); + }); + + it("pins opt-out while keeping auto distinct from cached enabled state", () => { + expect(getRebuildSandboxGpuOverrides({ sandboxGpuMode: "0" })).toEqual({ + sandboxGpu: "disable", + sandboxGpuDevice: null, + sessionGpuPassthrough: false, + }); + expect( + getRebuildSandboxGpuOverrides({ sandboxGpuMode: "auto", sandboxGpuEnabled: true }), + ).toEqual({ + sandboxGpu: null, + sandboxGpuDevice: null, + sessionGpuPassthrough: false, + }); + }); + + it("does not treat legacy effective-enabled fields as forced sandbox GPU", () => { + expect(getRebuildSandboxGpuOverrides({ sandboxGpuEnabled: true, gpuEnabled: true })).toEqual({ + sandboxGpu: null, + sandboxGpuDevice: null, + sessionGpuPassthrough: false, + }); + }); +}); + describe("buildRebuildRecreateOnboardOpts", () => { const baseArgs = { rebuildAgent: "openclaw", storedFromDockerfile: null, autoYes: true, + usageNoticeAccepted: true as const, }; + const dashboard = { dashboardPort: 18789 }; it("forwards noGpu:true when the recorded sandboxGpuMode is the explicit opt-out '0'", () => { const opts = buildRebuildRecreateOnboardOpts({ ...baseArgs, - sb: { sandboxGpuMode: "0", sandboxGpuEnabled: false }, + sb: { ...dashboard, sandboxGpuMode: "0", sandboxGpuEnabled: false }, }); expect(opts.noGpu).toBe(true); expect(opts).toMatchObject({ resume: true, nonInteractive: true, recreateSandbox: true, + authoritativeResumeConfig: true, agent: "openclaw", fromDockerfile: null, + sandboxGpu: "disable", + sandboxGpuDevice: null, autoYes: true, }); }); @@ -128,7 +176,7 @@ describe("buildRebuildRecreateOnboardOpts", () => { it("forwards noGpu:true for legacy entries with gpuEnabled:false and no sandboxGpuMode", () => { const opts = buildRebuildRecreateOnboardOpts({ ...baseArgs, - sb: { gpuEnabled: false }, + sb: { ...dashboard, gpuEnabled: false }, }); expect(opts.noGpu).toBe(true); }); @@ -136,30 +184,41 @@ describe("buildRebuildRecreateOnboardOpts", () => { it("omits noGpu for auto-mode CPU fallback so resume stays auto", () => { const opts = buildRebuildRecreateOnboardOpts({ ...baseArgs, - sb: { sandboxGpuMode: "auto", sandboxGpuEnabled: false }, + sb: { ...dashboard, sandboxGpuMode: "auto", sandboxGpuEnabled: false }, }); expect(opts).not.toHaveProperty("noGpu"); + expect(opts.sandboxGpu).toBeNull(); + expect(opts.sandboxGpuDevice).toBeNull(); }); it("omits noGpu when sandboxGpuMode is '1'", () => { const opts = buildRebuildRecreateOnboardOpts({ ...baseArgs, - sb: { sandboxGpuMode: "1", sandboxGpuEnabled: true }, + sb: { + ...dashboard, + sandboxGpuMode: "1", + sandboxGpuEnabled: true, + sandboxGpuDevice: "nvidia.com/gpu=2", + }, }); expect(opts).not.toHaveProperty("noGpu"); + expect(opts.sandboxGpu).toBe("enable"); + expect(opts.sandboxGpuDevice).toBe("nvidia.com/gpu=2"); }); - it("omits noGpu when no sandbox entry is captured", () => { - const opts = buildRebuildRecreateOnboardOpts({ ...baseArgs, sb: null }); - expect(opts).not.toHaveProperty("noGpu"); + it("fails closed when a dashboard-managed sandbox has no durable port", () => { + expect(() => buildRebuildRecreateOnboardOpts({ ...baseArgs, sb: null })).toThrow( + "without its persisted dashboard port", + ); }); it("preserves storedFromDockerfile and autoYes regardless of GPU opt-out", () => { const opts = buildRebuildRecreateOnboardOpts({ - sb: { sandboxGpuMode: "0" }, + sb: { ...dashboard, sandboxGpuMode: "0" }, rebuildAgent: "hermes", storedFromDockerfile: "/sandbox/.openclaw/Dockerfile.custom", autoYes: false, + usageNoticeAccepted: true, }); expect(opts.agent).toBe("hermes"); expect(opts.fromDockerfile).toBe("/sandbox/.openclaw/Dockerfile.custom"); diff --git a/src/lib/actions/sandbox/rebuild-gpu-opt-out.ts b/src/lib/actions/sandbox/rebuild-gpu-opt-out.ts index 514b557c540..39f42c5bf5e 100644 --- a/src/lib/actions/sandbox/rebuild-gpu-opt-out.ts +++ b/src/lib/actions/sandbox/rebuild-gpu-opt-out.ts @@ -1,13 +1,23 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { loadAgent } from "../../agent/defs"; +import { shouldManageDashboardForAgent } from "../../onboard/dashboard-runtime"; +import { + resolveGatewayPortFromName, + resolveSandboxGatewayName, +} from "../../onboard/gateway-binding"; import type { PreparedDcodeRebuildHandoff } from "../../onboard/prepared-dcode-rebuild"; import { normalizeSandboxGpuMode } from "../../onboard/sandbox-gpu-mode"; export type RebuildGpuOptOutEntry = { sandboxGpuMode?: string | null; sandboxGpuEnabled?: boolean; + sandboxGpuDevice?: string | null; gpuEnabled?: boolean; + dashboardPort?: number | null; + gatewayName?: string | null; + gatewayPort?: number | null; }; // Modern source of truth is the persisted `sandboxGpuMode` string ("0" / "1" / @@ -29,12 +39,51 @@ export function rebuildShouldOptOutGpu(sb: RebuildGpuOptOutEntry | null | undefi return sb.gpuEnabled === false; } +export function getRebuildSandboxGpuOverrides(sb: RebuildGpuOptOutEntry | null | undefined): { + sandboxGpu: "enable" | "disable" | null; + sandboxGpuDevice: string | null; + sessionGpuPassthrough: boolean; +} { + const mode = normalizeSandboxGpuMode(sb?.sandboxGpuMode); + if (mode === "1") { + return { + sandboxGpu: "enable", + sandboxGpuDevice: sb?.sandboxGpuDevice?.trim() || null, + sessionGpuPassthrough: true, + }; + } + if (mode === "0") { + return { sandboxGpu: "disable", sandboxGpuDevice: null, sessionGpuPassthrough: false }; + } + if (hasRecordedGpuMode(sb?.sandboxGpuMode) && mode === null) { + throw new Error(`Invalid recorded sandbox GPU mode '${String(sb?.sandboxGpuMode)}'.`); + } + if (mode === "auto") { + // A false cached value keeps resume's legacy fallback from converting + // recorded auto mode into forced enable after the old registry row is + // temporarily removed. Fresh preflight recomputes actual auto detection. + return { sandboxGpu: null, sandboxGpuDevice: null, sessionGpuPassthrough: false }; + } + if (sb?.gpuEnabled === false) { + return { sandboxGpu: "disable", sandboxGpuDevice: null, sessionGpuPassthrough: false }; + } + return { sandboxGpu: null, sandboxGpuDevice: null, sessionGpuPassthrough: false }; +} + export type RebuildRecreateOnboardOpts = { resume: true; nonInteractive: true; recreateSandbox: true; + authoritativeResumeConfig: true; + acceptThirdPartySoftware: true; agent: string | null | undefined; fromDockerfile: string | null; + sandboxGpu: "enable" | "disable" | null; + sandboxGpuDevice: string | null; + controlUiPort: number | null; + targetGatewayName: string; + targetGatewayPort: number; + onboardLockAlreadyHeld: true; preparedDcodeRebuild?: PreparedDcodeRebuildHandoff; autoYes: boolean; noGpu?: true; @@ -46,13 +95,44 @@ export function buildRebuildRecreateOnboardOpts(args: { storedFromDockerfile: string | null; preparedDcodeRebuild?: PreparedDcodeRebuildHandoff; autoYes: boolean; + usageNoticeAccepted: true; }): RebuildRecreateOnboardOpts { + const gpuOverrides = getRebuildSandboxGpuOverrides(args.sb); + const targetGatewayName = resolveSandboxGatewayName(args.sb); + const targetGatewayPort = resolveGatewayPortFromName(targetGatewayName); + if (targetGatewayPort === null) { + throw new Error(`Cannot resolve persisted gateway port for '${targetGatewayName}'.`); + } + const dashboardPort = args.sb?.dashboardPort; + if ( + dashboardPort !== undefined && + dashboardPort !== null && + (!Number.isInteger(dashboardPort) || dashboardPort < 0 || dashboardPort > 65535) + ) { + throw new Error(`Invalid persisted dashboard port '${String(dashboardPort)}'.`); + } + const managesDashboard = shouldManageDashboardForAgent( + loadAgent(args.rebuildAgent || "openclaw"), + ); + if (managesDashboard && (!dashboardPort || dashboardPort < 1)) { + throw new Error( + "Cannot recreate a dashboard-managed sandbox without its persisted dashboard port.", + ); + } return { resume: true, nonInteractive: true, recreateSandbox: true, + authoritativeResumeConfig: true, + acceptThirdPartySoftware: args.usageNoticeAccepted, agent: args.rebuildAgent, fromDockerfile: args.storedFromDockerfile, + sandboxGpu: gpuOverrides.sandboxGpu, + sandboxGpuDevice: gpuOverrides.sandboxGpuDevice, + controlUiPort: managesDashboard ? (dashboardPort ?? null) : null, + targetGatewayName, + targetGatewayPort, + onboardLockAlreadyHeld: true, ...(args.preparedDcodeRebuild ? { preparedDcodeRebuild: args.preparedDcodeRebuild } : {}), autoYes: args.autoYes, ...(rebuildShouldOptOutGpu(args.sb) ? { noGpu: true as const } : {}), diff --git a/src/lib/actions/sandbox/rebuild-managed-image-preflight.ts b/src/lib/actions/sandbox/rebuild-managed-image-preflight.ts index 4e803cb7da8..5f457dadf86 100644 --- a/src/lib/actions/sandbox/rebuild-managed-image-preflight.ts +++ b/src/lib/actions/sandbox/rebuild-managed-image-preflight.ts @@ -7,6 +7,7 @@ import path from "node:path"; import { dockerBuild, dockerRmi } from "../../adapters/docker"; import type { AgentDefinition } from "../../agent/defs"; +import { GATEWAY_PORT } from "../../core/ports"; import { createAgentSandbox } from "../../agent/onboard"; import { type PreparedSandboxBuildContext, @@ -28,6 +29,7 @@ export type ManagedDcodeRebuildImageInput = { provider: string; preferredInferenceApi: string | null; sandboxGpuConfig: SandboxGpuConfig; + gatewayPort?: number; }; export type ManagedDcodeRebuildImageDeps = { @@ -256,6 +258,7 @@ export async function prepareManagedDcodeRebuildImage( webSearchConfig: null, hermesToolGateways: [], sandboxGpuConfig: input.sandboxGpuConfig, + gatewayPort: input.gatewayPort ?? GATEWAY_PORT, log: () => {}, warn: () => {}, }); diff --git a/src/lib/actions/sandbox/rebuild-mcp-order.test.ts b/src/lib/actions/sandbox/rebuild-mcp-order.test.ts new file mode 100644 index 00000000000..181dbcb068c --- /dev/null +++ b/src/lib/actions/sandbox/rebuild-mcp-order.test.ts @@ -0,0 +1,51 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { prepareMcpBeforeBestEffortNimStop } from "./rebuild-mcp-order"; + +describe("rebuild MCP and local NIM ordering", () => { + it("does not stop NIM when MCP preservation fails or aborts", async () => { + const stopNim = vi.fn(); + await expect( + prepareMcpBeforeBestEffortNimStop({ + prepareMcp: async () => null, + stopNim, + log: vi.fn(), + }), + ).resolves.toBeNull(); + expect(stopNim).not.toHaveBeenCalled(); + + await expect( + prepareMcpBeforeBestEffortNimStop({ + prepareMcp: async () => { + throw new Error("policy drift"); + }, + stopNim, + log: vi.fn(), + }), + ).rejects.toThrow("policy drift"); + expect(stopNim).not.toHaveBeenCalled(); + }); + + it("stops NIM only after MCP preservation and treats stop as best effort", async () => { + const order: string[] = []; + const log = vi.fn(); + await expect( + prepareMcpBeforeBestEffortNimStop({ + prepareMcp: async () => { + order.push("mcp-prepared"); + return { entries: 1 }; + }, + stopNim: () => { + order.push("nim-stop"); + throw new Error("runtime unavailable"); + }, + log, + }), + ).resolves.toEqual({ entries: 1 }); + expect(order).toEqual(["mcp-prepared", "nim-stop"]); + expect(log).toHaveBeenCalledWith(expect.stringContaining("runtime unavailable")); + }); +}); diff --git a/src/lib/actions/sandbox/rebuild-mcp-order.ts b/src/lib/actions/sandbox/rebuild-mcp-order.ts new file mode 100644 index 00000000000..69b0f78c916 --- /dev/null +++ b/src/lib/actions/sandbox/rebuild-mcp-order.ts @@ -0,0 +1,23 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** Keep local inference available until MCP preservation has fully succeeded. */ +export async function prepareMcpBeforeBestEffortNimStop(options: { + prepareMcp(): Promise; + stopNim(): void; + log(message: string): void; +}): Promise { + const preparation = await options.prepareMcp(); + if (preparation === null) return null; + + try { + options.stopNim(); + } catch (error) { + // NIM stop already uses ignoreError. Preserve that best-effort contract if + // the local runtime still throws; recreate force-removes the old name. + options.log( + `Best-effort NIM stop failed; continuing rebuild: ${error instanceof Error ? error.message : String(error)}`, + ); + } + return preparation; +} diff --git a/src/lib/actions/sandbox/rebuild-mcp-phase.ts b/src/lib/actions/sandbox/rebuild-mcp-phase.ts new file mode 100644 index 00000000000..8b3c09e2e68 --- /dev/null +++ b/src/lib/actions/sandbox/rebuild-mcp-phase.ts @@ -0,0 +1,128 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { CLI_NAME } from "../../cli/branding"; +import { G, R, YW } from "../../cli/terminal-style"; +import * as registry from "../../state/registry"; +import { + prepareMcpBridgesForAbsentSandboxRebuild, + prepareMcpBridgesForRebuild, + reattachMcpProvidersAfterRebuildAbort, + restoreMcpBridgesAfterRebuild, +} from "./mcp-bridge"; +import type { RebuildBail } from "./rebuild-credential-preflight"; +import type { RebuildSandboxEntry } from "./rebuild-flow-helpers"; + +export type McpRebuildPreparation = Awaited>; + +export async function prepareMcpForRebuild( + sandboxName: string, + staleRecovery: boolean, + relockShieldsIfNeeded: (sandboxStillExists: boolean) => boolean, + bail: (message: string, code?: number) => never, +): Promise { + try { + return await (staleRecovery + ? prepareMcpBridgesForAbsentSandboxRebuild(sandboxName) + : prepareMcpBridgesForRebuild(sandboxName)); + } catch (error) { + relockShieldsIfNeeded(!staleRecovery); + bail( + `Failed to preserve MCP bridges before rebuild: ${error instanceof Error ? error.message : String(error)}`, + ); + return null; + } +} + +export async function reattachMcpAfterDeleteFailure( + sandboxName: string, + entries: McpRebuildPreparation["detachedProviderEntries"], + scrubbedAdapterEntries: McpRebuildPreparation["scrubbedAdapterEntries"], +): Promise { + try { + await reattachMcpProvidersAfterRebuildAbort(sandboxName, entries, scrubbedAdapterEntries); + return undefined; + } catch (error) { + return error instanceof Error ? error.message : String(error); + } +} + +export function restoreMcpRegistryForRebuildRetry( + staleRecovery: boolean, + entries: McpRebuildPreparation["entries"], + original: RebuildSandboxEntry, + log: (message: string) => void, +): void { + if (staleRecovery || entries.length === 0) return; + try { + // MCP-bearing rebuilds deliberately preserve the registry entry instead of + // removing it. Restore any metadata overwritten by a partial onboard, but + // leave the current default pointer alone: a concurrent `nemoclaw use` + // selection must win because this rebuild never moved that pointer. + registry.restoreSandboxEntry(original); + log("Recreate failed: restored MCP-bearing registry entry for stale recovery retry"); + } catch (error) { + log(`Failed to restore MCP-bearing registry entry after recreate failure: ${String(error)}`); + } +} + +export function printMcpRebuildRetryCommand( + sandboxName: string, + entries: McpRebuildPreparation["entries"], +): void { + if (entries.length > 0) { + console.error(` 2. Run: ${CLI_NAME} ${sandboxName} rebuild --yes`); + console.error( + ` This will recreate sandbox '${sandboxName}' and restore its MCP bridges.`, + ); + return; + } + console.error(` 2. Run: ${CLI_NAME} onboard --resume`); + console.error(` This will recreate sandbox '${sandboxName}'.`); +} + +export async function restoreMcpAfterRebuild( + sandboxName: string, + entries: McpRebuildPreparation["entries"], +): Promise { + if (entries.length === 0) return true; + console.log(" Restoring MCP bridges..."); + try { + await restoreMcpBridgesAfterRebuild(sandboxName, entries); + console.log(` ${G}✓${R} MCP bridges restored`); + return true; + } catch (error) { + console.error( + ` ${YW}⚠${R} MCP bridge restore incomplete: ${error instanceof Error ? error.message : String(error)}`, + ); + return false; + } +} + +export function postRestoreCompleted(status: { + messagingHostForwardUnverified: boolean; + mcpBridgeRestoreUnverified: boolean; + mutableConfigHashRefreshUnverified: boolean; + mutablePermsRepairUnverified: boolean; + policyPresetRestoreIncomplete: boolean; + restoreSucceeded: boolean; +}): boolean { + return ( + status.restoreSucceeded && + !status.mutablePermsRepairUnverified && + !status.mutableConfigHashRefreshUnverified && + !status.messagingHostForwardUnverified && + !status.mcpBridgeRestoreUnverified && + !status.policyPresetRestoreIncomplete + ); +} + +export function printMcpRestoreRecovery( + sandboxName: string, + mcpBridgeRestoreUnverified: boolean, +): void { + if (!mcpBridgeRestoreUnverified) return; + console.log( + ` MCP bridge definitions were preserved but not fully refreshed — fix the reported cause, then run \`${CLI_NAME} ${sandboxName} mcp restart\``, + ); +} diff --git a/src/lib/actions/sandbox/rebuild-messaging-phase.ts b/src/lib/actions/sandbox/rebuild-messaging-phase.ts new file mode 100644 index 00000000000..e9219704452 --- /dev/null +++ b/src/lib/actions/sandbox/rebuild-messaging-phase.ts @@ -0,0 +1,162 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { runOpenshell } from "../../adapters/openshell/runtime"; +import { loadAgent } from "../../agent/defs"; +import { RD as _RD, D, G, R } from "../../cli/terminal-style"; +import type { + MessagingHookApplyRequest, + MessagingHookOutputMap, + MessagingOpenShellRunner, + SandboxMessagingPlan, +} from "../../messaging"; +import { + createBuiltInChannelManifestRegistry, + createBuiltInRenderTemplateResolver, + isMessagingSupportedAgent, + listSupportedMessagingChannelIdsForAgent, + MessagingSetupApplier, + MessagingWorkflowPlanner, + tryGetMessagingAgentId, +} from "../../messaging"; +import type { SandboxEntry } from "../../state/registry"; +import type { RebuildBail } from "./rebuild-credential-preflight"; + +/** Build and stage the manifest-derived messaging recreate contract. */ +export async function stageMessagingManifestPlanForRebuild( + sandboxName: string, + sandboxEntry: SandboxEntry, + rebuildAgent: string | null, + log: (message: string) => void, +): Promise { + const agent = loadAgent(rebuildAgent || "openclaw"); + const manifestRegistry = createBuiltInChannelManifestRegistry(); + const manifests = manifestRegistry.list(); + const agentId = tryGetMessagingAgentId(agent, manifests); + if (agentId === null) { + MessagingSetupApplier.clearPlanEnv(); + log( + `Messaging manifest rebuild plan skipped: agent '${agent.name}' is not supported by any channel manifest`, + ); + return null; + } + if (!isMessagingSupportedAgent(agent, manifests)) { + MessagingSetupApplier.clearPlanEnv(); + log( + `Messaging manifest rebuild plan skipped: agent '${agent.name}' has no supported messaging channels`, + ); + return null; + } + const supportedChannelIds = listSupportedMessagingChannelIdsForAgent(manifests, agentId); + const planner = new MessagingWorkflowPlanner( + manifestRegistry, + undefined, + createBuiltInRenderTemplateResolver(), + ); + const plan = await planner.buildRebuildPlanFromSandboxEntry({ + sandboxName, + agent: agentId, + sandboxEntry, + supportedChannelIds, + }); + if (!plan) { + MessagingSetupApplier.clearPlanEnv(); + log("Messaging manifest rebuild plan: no configured channels"); + return null; + } + MessagingSetupApplier.writePlanToEnv(plan); + if (plan.channels.length === 0) { + log("Messaging manifest rebuild plan staged: no configured channels"); + return plan; + } + log( + `Messaging manifest rebuild plan staged: ${plan.channels + .map((channel) => channel.channelId) + .join(",")}`, + ); + return plan; +} + +/** Stage the manifest plan while preserving rebuild's fail-before-delete boundary. */ +export async function stageRebuildMessagingPlanOrBail( + sandboxName: string, + sandboxEntry: SandboxEntry, + rebuildAgent: string | null, + log: (message: string) => void, + bail: RebuildBail, +): Promise { + try { + return await stageMessagingManifestPlanForRebuild(sandboxName, sandboxEntry, rebuildAgent, log); + } catch (err) { + // Source boundary: persisted registry messaging plans and current channel + // manifests are host-side inputs. If they drift or become invalid, rebuild + // must fail here before backup/delete; remove this boundary only if manifest + // staging becomes total over all persisted registry states. + const message = err instanceof Error ? err.message : String(err); + console.error(""); + console.error( + ` ${_RD}Rebuild preflight failed:${R} messaging manifest plan could not be staged.`, + ); + console.error(` ${message}`); + console.error(""); + console.error(" Sandbox is untouched — no data was lost."); + bail(message); + return null; + } +} + +const runMessagingOpenshell: MessagingOpenShellRunner = (args, options = {}) => + runOpenshell([...args], { + env: options.env as NodeJS.ProcessEnv | undefined, + ignoreError: options.ignoreError, + input: options.input, + stdio: options.stdio as never, + }); + +function hookOutputsFromBuildSteps( + plan: SandboxMessagingPlan, + request: MessagingHookApplyRequest, +): { readonly outputs: MessagingHookOutputMap } { + const outputs: Record = {}; + for (const step of plan.buildSteps) { + if ( + step.channelId !== request.channelId || + step.hookId !== request.hookId || + step.value === undefined + ) { + continue; + } + outputs[step.outputId] = { kind: step.kind, value: step.value }; + } + return { outputs }; +} + +/** Reapply OpenClaw messaging files that doctor may have rewritten. */ +export async function reapplyMessagingManifestAfterOpenClawDoctor( + sandboxName: string, + plan: SandboxMessagingPlan | null, + log: (message: string) => void, +): Promise { + if (!plan || plan.agent !== "openclaw") { + log("Messaging manifest reapply skipped: no OpenClaw messaging plan"); + return; + } + + try { + log("Reapplying messaging manifest render and post-agent-install hooks after doctor"); + const result = await MessagingSetupApplier.applyAgentConfigAtOpenShell(plan, { + runOpenshell: runMessagingOpenshell, + runHook: (request) => hookOutputsFromBuildSteps(plan, request), + }); + log( + `messaging manifest reapply: targets=${result.appliedTargets.join(",")}, hooks=${result.appliedHooks.join(",")}`, + ); + if (result.appliedTargets.length > 0 || result.appliedHooks.length > 0) { + console.log(` ${G}\u2713${R} Messaging manifest config reapplied`); + } + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + log(`Messaging manifest reapply failed: ${message}`); + console.log(` ${D}Messaging manifest config reapply skipped (${message})${R}`); + } +} diff --git a/src/lib/actions/sandbox/rebuild-pipeline.ts b/src/lib/actions/sandbox/rebuild-pipeline.ts new file mode 100644 index 00000000000..1f83235d2da --- /dev/null +++ b/src/lib/actions/sandbox/rebuild-pipeline.ts @@ -0,0 +1,238 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { RebuildSandboxOptions } from "../../domain/lifecycle/options"; +import { BRAVE_API_KEY_ENV, TAVILY_API_KEY_ENV } from "../../inference/web-search"; +import { MESSAGING_SETUP_APPLIER_ENV_KEY } from "../../messaging/applier/types"; +import { MESSAGING_CHANNEL_CONFIG_ENV_KEYS } from "../../messaging-channel-config"; +import { DOCKER_GPU_PATCH_NETWORK_ENV } from "../../onboard/docker-gpu-patch"; +import { withMcpLifecycleLock } from "../../state/mcp-lifecycle-lock"; +import * as registry from "../../state/registry"; +import { runRebuildBackupPhase } from "./rebuild-backup-phase"; +import { buildRefreshMutableOpenClawConfigHashCommand } from "./rebuild-config-hash"; +import { runRebuildDestroyPhase } from "./rebuild-destroy-phase"; +import { REBUILD_HERMES_DASHBOARD_ENV_KEYS } from "./rebuild-durable-config"; +import { stageMessagingManifestPlanForRebuild } from "./rebuild-messaging-phase"; +import { runRebuildPostRestorePhase } from "./rebuild-post-restore-phase"; +import { runRebuildPreflightPhase } from "./rebuild-preflight-phase"; +import { + type RebuildSandboxExecutionOptions, + revalidatePreparedRecoveryBeforeDelete, +} from "./rebuild-prepared-recovery"; +import { runRebuildRecreatePhase } from "./rebuild-recreate-phase"; +import { runRebuildRestorePhase } from "./rebuild-restore-phase"; +import { runRebuildShieldsPhase } from "./rebuild-shields-phase"; + +export { buildRefreshMutableOpenClawConfigHashCommand, stageMessagingManifestPlanForRebuild }; + +/** + * Rebuild a live sandbox while preserving registered agent state and policies. + * + * The facade scopes mutable process environment and serializes the typed phase + * pipeline with the MCP lifecycle lock. + */ +export async function rebuildSandbox( + sandboxName: string, + options: string[] | RebuildSandboxOptions = {}, + opts: RebuildSandboxExecutionOptions = {}, +): Promise { + return withMcpLifecycleLock(sandboxName, async () => { + const scopedEnvKeys = [ + BRAVE_API_KEY_ENV, + TAVILY_API_KEY_ENV, + MESSAGING_SETUP_APPLIER_ENV_KEY, + "OPENSHELL_GATEWAY", + DOCKER_GPU_PATCH_NETWORK_ENV, + ...REBUILD_HERMES_DASHBOARD_ENV_KEYS, + ...MESSAGING_CHANNEL_CONFIG_ENV_KEYS, + ]; + const savedEnv = scopedEnvKeys.map((key) => [key, process.env[key]] as const); + try { + await rebuildSandboxUnlocked(sandboxName, options, opts); + } finally { + for (const key of scopedEnvKeys) delete process.env[key]; + Object.assign( + process.env, + Object.fromEntries( + savedEnv.filter((entry): entry is [string, string] => entry[1] !== undefined), + ), + ); + } + }); +} + +async function rebuildSandboxUnlocked( + sandboxName: string, + options: string[] | RebuildSandboxOptions, + opts: RebuildSandboxExecutionOptions, +): Promise { + const preflight = await runRebuildPreflightPhase(sandboxName, options, opts); + if (!preflight) return; + const { + sandboxEntry, + rebuildAgent, + versionCheck, + targetConfig, + recreateOptions, + messagingPlan, + baseImagePreflight, + liveState, + recoveryManifest: validatedRecoveryManifest, + dcodePreflight, + releaseOnboardLock, + log, + bail, + } = preflight; + const { + resumeConfig, + sessionSnapshot, + sessionMatchesSandbox, + durableConfig, + hermesToolGateways, + hasHermesToolGateways, + credentialEnv, + fromDockerfile, + } = targetConfig; + const { staleRecovery } = liveState; + let recoveryManifest = validatedRecoveryManifest; + const preparedBackupRecovery = recoveryManifest !== null; + const recoveryRecreate = staleRecovery || preparedBackupRecovery; + let recoveryRegistrySnapshot = preparedBackupRecovery + ? JSON.parse(JSON.stringify(registry.load())) + : liveState.staleRegistrySnapshot; + try { + const shieldsPhase = runRebuildShieldsPhase( + sandboxName, + recoveryRecreate, + releaseOnboardLock, + bail, + ); + if (!shieldsPhase) return; + const { + window: rebuildShieldsWindow, + staleSandboxWasLocked, + relock: relockShieldsIfNeeded, + } = shieldsPhase; + let sandboxStillExists = true; + + try { + const preDeleteRecovery = revalidatePreparedRecoveryBeforeDelete( + sandboxName, + sandboxEntry, + recoveryManifest, + recoveryRegistrySnapshot, + bail, + ); + recoveryManifest = preDeleteRecovery.manifest; + recoveryRegistrySnapshot = preDeleteRecovery.registrySnapshot; + + const backup = runRebuildBackupPhase({ + sandboxName, + sandboxEntry, + staleRecovery, + preparedRecoveryManifest: recoveryManifest, + messagingPlan, + webSearchConfig: durableConfig.webSearchConfig, + log, + bail, + relockShieldsIfNeeded, + }); + if (!backup) return; + + // DCode's retained replacement and live inference route must still match at + // the last safe point. This check intentionally precedes MCP adapter scrub, + // provider detach, NIM stop, and sandbox deletion in the destroy phase. + if ( + !(await dcodePreflight.revalidateBeforeDelete( + resumeConfig, + recoveryRecreate, + recreateOptions.targetGatewayPort, + )) + ) { + return; + } + + const mcpPreparation = await runRebuildDestroyPhase({ + sandboxName, + sandboxEntry, + staleRecovery, + backupManifest: backup.backupManifest, + log, + bail, + relockShieldsIfNeeded, + onDeleted: () => { + sandboxStillExists = false; + }, + }); + if (!mcpPreparation) return; + + const restoreDcodeGpuPatchNetwork = dcodePreflight.applyDockerGpuPatchNetwork(); + let recreated: boolean; + try { + recreated = await runRebuildRecreatePhase({ + sandboxName, + sandboxEntry, + sessionSnapshot, + sessionMatchesSandbox, + durableConfig, + resumeConfig, + recreateOptions, + fromDockerfile, + rebuildAgent, + messagingPlan, + rebuildsHermesSandbox: rebuildAgent === "hermes", + hermesToolGateways, + hasHermesToolGateways, + sessionPolicyPresets: backup.sessionPolicyPresets, + credentialEnv, + baseImagePreflight, + recoveryRecreate, + recoveryRegistrySnapshot, + backupManifest: backup.backupManifest, + mcpEntries: mcpPreparation.entries, + rebuildShieldsWindow, + relockShieldsIfNeeded, + onCreated: () => { + sandboxStillExists = true; + }, + log, + bail, + }); + } finally { + restoreDcodeGpuPatchNetwork(); + } + if (!recreated) return; + + const restored = runRebuildRestorePhase({ + sandboxName, + backupManifest: backup.backupManifest, + policyPresets: backup.policyPresets, + log, + }); + await runRebuildPostRestorePhase({ + sandboxName, + sandboxEntry, + messagingPlan, + backupManifest: backup.backupManifest, + mcpEntries: mcpPreparation.entries, + restoreSucceeded: restored.restoreSucceeded, + restoredPresets: restored.restoredPresets, + failedPresets: restored.failedPresets, + staleRecovery, + recoveryRecreate, + preparedBackupRecovery, + staleSandboxWasLocked, + versionCheck, + relockShieldsIfNeeded, + log, + bail, + }); + } finally { + if (!rebuildShieldsWindow.relocked) relockShieldsIfNeeded(sandboxStillExists); + } + } finally { + dcodePreflight.cleanup(); + process.removeListener("exit", releaseOnboardLock); + releaseOnboardLock(); + } +} diff --git a/src/lib/actions/sandbox/rebuild-post-restore-phase.ts b/src/lib/actions/sandbox/rebuild-post-restore-phase.ts new file mode 100644 index 00000000000..9b00930a761 --- /dev/null +++ b/src/lib/actions/sandbox/rebuild-post-restore-phase.ts @@ -0,0 +1,215 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { loadAgent } from "../../agent/defs"; +import * as agentRuntime from "../../agent/runtime"; +import { CLI_NAME } from "../../cli/branding"; +import { D, G, R, YW } from "../../cli/terminal-style"; +import type { SandboxMessagingPlan } from "../../messaging"; +import type * as sandboxVersion from "../../sandbox/version"; +import * as shields from "../../shields"; +import * as registry from "../../state/registry"; +import { ensureMessagingHostForwardAfterRebuild } from "./messaging-host-forward-lifecycle"; +import { executeSandboxCommand } from "./process-recovery"; +import type { RebuildBackupManifest } from "./rebuild-backup-phase"; +import { refreshMutableOpenClawConfigHashAfterPostRestoreWrites } from "./rebuild-config-hash"; +import type { RebuildBail, RebuildLog } from "./rebuild-credential-preflight"; +import type { RebuildSandboxEntry } from "./rebuild-flow-helpers"; +import { + type McpRebuildPreparation, + postRestoreCompleted, + printMcpRestoreRecovery, + restoreMcpAfterRebuild, +} from "./rebuild-mcp-phase"; +import { reapplyMessagingManifestAfterOpenClawDoctor } from "./rebuild-messaging-phase"; + +export interface RebuildPostRestorePhaseInput { + sandboxName: string; + sandboxEntry: RebuildSandboxEntry; + messagingPlan: SandboxMessagingPlan | null; + backupManifest: RebuildBackupManifest; + mcpEntries: McpRebuildPreparation["entries"]; + restoreSucceeded: boolean; + restoredPresets: string[]; + failedPresets: string[]; + staleRecovery: boolean; + recoveryRecreate: boolean; + preparedBackupRecovery: boolean; + staleSandboxWasLocked: boolean; + versionCheck: ReturnType; + relockShieldsIfNeeded: (sandboxStillExists: boolean) => boolean; + log: RebuildLog; + bail: RebuildBail; +} + +/** + * Repair agent state, restore MCP/forwarding, reconcile the registry, and report + * the final transaction result. Boundary coverage: rebuild-flow.test.ts and + * rebuild-config-hash.test.ts cover the complete/incomplete post-restore paths. + */ +export async function runRebuildPostRestorePhase( + input: RebuildPostRestorePhaseInput, +): Promise { + const { + sandboxName, + sandboxEntry: sb, + messagingPlan, + backupManifest, + mcpEntries, + restoreSucceeded, + restoredPresets, + failedPresets, + staleRecovery, + recoveryRecreate, + preparedBackupRecovery, + staleSandboxWasLocked, + versionCheck, + relockShieldsIfNeeded, + log, + bail, + } = input; + const rebuiltAgent = agentRuntime.getSessionAgent(sandboxName); + const rebuiltAgentName = agentRuntime.getAgentDisplayName(rebuiltAgent); + const agentDef = rebuiltAgent ? loadAgent(rebuiltAgent.name) : loadAgent("openclaw"); + let mutablePermsRepairUnverified = false; + let mutableConfigHashRefreshUnverified = false; + let messagingHostForwardUnverified = false; + const policyPresetRestoreIncomplete = failedPresets.length > 0; + + if (agentDef.name === "openclaw") { + log("Running openclaw doctor --fix inside sandbox for post-upgrade structure repair"); + const doctorResult = executeSandboxCommand(sandboxName, "openclaw doctor --fix"); + log( + `doctor --fix: exit=${doctorResult?.status}, stdout=${(doctorResult?.stdout || "").substring(0, 200)}`, + ); + if (doctorResult && doctorResult.status === 0) { + console.log(` ${G}\u2713${R} Post-upgrade structure check passed`); + } else { + console.log( + ` ${D}Post-upgrade structure check skipped (doctor returned ${doctorResult?.status ?? "null"})${R}`, + ); + } + + await reapplyMessagingManifestAfterOpenClawDoctor(sandboxName, messagingPlan, log); + log("Refreshing mutable OpenClaw config hash after post-restore config writes"); + if (!refreshMutableOpenClawConfigHashAfterPostRestoreWrites(sandboxName, log)) { + mutableConfigHashRefreshUnverified = true; + } + + log("Restoring mutable OpenClaw config permissions after post-restore config writes"); + let permRepair: ReturnType | null = null; + try { + permRepair = shields.repairMutableConfigPerms(sandboxName); + } catch (error) { + mutablePermsRepairUnverified = true; + console.error( + ` ${YW}\u26a0${R} Mutable config permission repair errored: ${error instanceof Error ? error.message : String(error)}`, + ); + } + if (permRepair === null) { + // The thrown error was reported above. + } else if (!permRepair.applied) { + if (permRepair.skipReason === "unreadable") { + mutablePermsRepairUnverified = true; + console.error( + ` ${YW}\u26a0${R} Mutable config permissions not restored: ${permRepair.reason}`, + ); + } else { + log(`Mutable config permission repair skipped: ${permRepair.reason}`); + } + } else if (permRepair.verified) { + console.log(` ${G}\u2713${R} Mutable config permissions restored`); + } else { + mutablePermsRepairUnverified = true; + console.error( + ` ${YW}\u26a0${R} Mutable config permission repair incomplete: ${permRepair.errors.join("; ")}`, + ); + } + } + + const mcpBridgeRestoreUnverified = !(await restoreMcpAfterRebuild(sandboxName, mcpEntries)); + const policyPresetsFinalized = + sb.policyPresetsFinalized === true && + failedPresets.length === 0 && + (sb.customPolicies?.length ?? 0) === 0 + ? true + : undefined; + registry.updateSandbox(sandboxName, { + agentVersion: agentDef.expectedVersion || null, + policies: restoredPresets, + policyTier: sb.policyTier ?? null, + policyPresetsFinalized, + }); + log( + `Registry updated: agentVersion=${agentDef.expectedVersion}, policies=[${restoredPresets.join(",")}], policyPresetsFinalized=${String(policyPresetsFinalized === true)}`, + ); + + if (!relockShieldsIfNeeded(true)) { + bail("Failed to re-apply shields lockdown."); + return; + } + if (!ensureMessagingHostForwardAfterRebuild(sandboxName, messagingPlan)) { + messagingHostForwardUnverified = true; + } + + console.log(""); + const postRestoreComplete = postRestoreCompleted({ + messagingHostForwardUnverified, + mcpBridgeRestoreUnverified, + mutableConfigHashRefreshUnverified, + mutablePermsRepairUnverified, + policyPresetRestoreIncomplete, + restoreSucceeded, + }); + if (postRestoreComplete) { + console.log(` ${G}\u2713${R} Sandbox '${sandboxName}' rebuilt successfully`); + if (staleRecovery && !backupManifest) { + console.log( + ` ${D}Recovered from a stale registry entry \u2014 no prior workspace state was available to restore.${R}`, + ); + } + if (versionCheck.expectedVersion) { + console.log(` Now running: ${rebuiltAgentName} v${versionCheck.expectedVersion}`); + } + } else { + console.log( + ` ${YW}\u26a0${R} Sandbox '${sandboxName}' rebuilt but some post-restore steps were incomplete`, + ); + if (!restoreSucceeded && backupManifest) { + console.log( + ` State restore was incomplete \u2014 backup available at: ${backupManifest.backupPath}`, + ); + } + if (mutablePermsRepairUnverified) { + console.log( + ` Mutable config permissions were not verified \u2014 run \`${CLI_NAME} ${sandboxName} doctor --fix\` to restore the OpenClaw config permission contract`, + ); + } + if (mutableConfigHashRefreshUnverified) { + console.log( + ` Mutable OpenClaw config hash was not refreshed \u2014 restart the sandbox or re-run \`${CLI_NAME} ${sandboxName} rebuild\` before relying on config integrity checks`, + ); + } + if (messagingHostForwardUnverified) { + console.log( + ` Messaging webhook forward was not verified \u2014 run \`${CLI_NAME} ${sandboxName} connect\` after resolving the port conflict`, + ); + } + printMcpRestoreRecovery(sandboxName, mcpBridgeRestoreUnverified); + if (policyPresetRestoreIncomplete) { + console.log( + ` Policy presets failed to reapply: ${failedPresets.join(", ")} \u2014 re-apply manually with \`${CLI_NAME} ${sandboxName} policy-add\``, + ); + } + } + if (recoveryRecreate && staleSandboxWasLocked) { + console.log( + ` ${YW}\u26a0${R} Shields were previously enabled but the recreated sandbox starts unlocked \u2014 run \`${CLI_NAME} ${sandboxName} shields up\` to restore lockdown.`, + ); + } + if (preparedBackupRecovery && !postRestoreComplete) { + bail( + `Prepared backup recovery for '${sandboxName}' completed with unverified post-restore state.`, + ); + } +} diff --git a/src/lib/actions/sandbox/rebuild-preflight-confirmation.ts b/src/lib/actions/sandbox/rebuild-preflight-confirmation.ts new file mode 100644 index 00000000000..d40b9628f25 --- /dev/null +++ b/src/lib/actions/sandbox/rebuild-preflight-confirmation.ts @@ -0,0 +1,136 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { resolveOpenshell } from "../../adapters/openshell/resolve"; +import * as agentRuntime from "../../agent/runtime"; +import { B, D, R, YW } from "../../cli/terminal-style"; +import { prompt as askPrompt } from "../../credentials/store"; +import { + normalizeRebuildSandboxOptions, + type RebuildSandboxOptions, +} from "../../domain/lifecycle/options"; +import * as sandboxVersion from "../../sandbox/version"; +import { redact } from "../../security/redact"; +import { + createSystemDeps as createSessionDeps, + getActiveSandboxSessions, +} from "../../state/sandbox-session"; +import { type RebuildBail, type RebuildLog } from "./rebuild-credential-preflight"; +import { printRebuildPreflightFailure } from "./rebuild-preflight-error"; +import { ensureRebuildUsageNoticeAccepted } from "./rebuild-usage-notice"; + +export type RebuildVersionCheck = ReturnType; + +export function createRebuildCommandContext( + options: string[] | RebuildSandboxOptions, + opts: { throwOnError?: boolean }, +): { bail: RebuildBail; log: RebuildLog; skipConfirm: boolean } { + const normalized = normalizeRebuildSandboxOptions(options); + const verbose = normalized.verbose === true || process.env.NEMOCLAW_REBUILD_VERBOSE === "1"; + return { + log: verbose + ? (message: string) => + console.error(` ${D}[rebuild ${new Date().toISOString()}] ${redact(message)}${R}`) + : () => {}, + skipConfirm: normalized.yes === true || normalized.force === true, + bail: opts.throwOnError + ? (message: string) => { + throw new Error(message); + } + : (_message: string, code = 1) => process.exit(code), + }; +} + +export function countActiveSandboxSessionsForRebuild(sandboxName: string): number { + const opsBinRebuild = resolveOpenshell(); + // Source boundary: active-session detection depends on host process listing + // and the OpenShell binary being installed. A failed/unavailable detector is + // not evidence of active sessions, and rebuild's safety preflights still run + // before destructive work. Keep the prior fail-open prompt behavior here; + // remove this fallback only if session detection becomes a required, typed + // OpenShell API that can distinguish "zero sessions" from "unavailable". + if (!opsBinRebuild) return 0; + try { + const result = getActiveSandboxSessions(sandboxName, createSessionDeps(opsBinRebuild)); + return result.detected ? result.sessions.length : 0; + } catch { + return 0; + } +} + +export function getRebuildAgentDisplayName(sandboxName: string): string { + return agentRuntime.getAgentDisplayName(agentRuntime.getSessionAgent(sandboxName)); +} + +async function confirmSandboxRebuildIfNeeded( + skipConfirm: boolean, + activeSessionCount: number, +): Promise { + if (skipConfirm) return true; + if (activeSessionCount > 0) { + const plural = activeSessionCount > 1 ? "sessions" : "session"; + console.log( + ` ${YW}⚠ Active SSH ${plural} detected (${activeSessionCount} connection${activeSessionCount > 1 ? "s" : ""})${R}`, + ); + console.log( + ` Rebuilding will terminate ${activeSessionCount === 1 ? "the" : "all"} active ${plural} with a Broken pipe error.`, + ); + console.log(""); + } + console.log(" This will:"); + console.log(" 1. Back up workspace state"); + console.log(" 2. Destroy and recreate the sandbox with the current image"); + console.log(" 3. Restore workspace state into the new sandbox"); + console.log(""); + const answer = await askPrompt(" Proceed? [y/N]: "); + if (answer.trim().toLowerCase() !== "y" && answer.trim().toLowerCase() !== "yes") { + console.log(" Cancelled."); + return false; + } + return true; +} + +async function ensureRebuildUsageNoticeOrBail(bail: RebuildBail): Promise { + let accepted = false; + try { + accepted = await ensureRebuildUsageNoticeAccepted({ + stdinIsTty: process.stdin?.isTTY === true, + }); + } catch (err) { + printRebuildPreflightFailure( + "the current third-party software notice could not be recorded.", + err instanceof Error ? err.message : String(err), + "Third-party software notice preflight failed", + bail, + ); + } + if (accepted) return; + printRebuildPreflightFailure( + "the current third-party software notice was not accepted.", + "Accept the current notice before rebuilding.", + "Third-party software notice was not accepted", + bail, + ); +} + +export async function confirmRebuildIntent( + sandboxName: string, + agentName: string, + skipConfirm: boolean, + activeSessionCount: number, + bail: RebuildBail, +): Promise { + const versionCheck = sandboxVersion.checkAgentVersion(sandboxName); + console.log(""); + console.log(` ${B}Rebuild sandbox '${sandboxName}'${R}`); + if (versionCheck.sandboxVersion) { + console.log(` Current: ${agentName} v${versionCheck.sandboxVersion}`); + } + if (versionCheck.expectedVersion) { + console.log(` Target: ${agentName} v${versionCheck.expectedVersion}`); + } + console.log(""); + if (!(await confirmSandboxRebuildIfNeeded(skipConfirm, activeSessionCount))) return null; + await ensureRebuildUsageNoticeOrBail(bail); + return versionCheck; +} diff --git a/src/lib/actions/sandbox/rebuild-preflight-error.ts b/src/lib/actions/sandbox/rebuild-preflight-error.ts new file mode 100644 index 00000000000..993314f361e --- /dev/null +++ b/src/lib/actions/sandbox/rebuild-preflight-error.ts @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { RD as _RD, R } from "../../cli/terminal-style"; +import type { RebuildBail } from "./rebuild-credential-preflight"; + +export function printRebuildPreflightFailure( + summary: string, + detail: string, + bailMessage: string, + bail: RebuildBail, +): void { + console.error(""); + console.error(` ${_RD}Rebuild preflight failed:${R} ${summary}`); + console.error(` ${detail}`); + console.error(" Sandbox is untouched — no data was lost."); + bail(bailMessage); +} diff --git a/src/lib/actions/sandbox/rebuild-preflight-guards.ts b/src/lib/actions/sandbox/rebuild-preflight-guards.ts new file mode 100644 index 00000000000..11037a9a777 --- /dev/null +++ b/src/lib/actions/sandbox/rebuild-preflight-guards.ts @@ -0,0 +1,94 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + detectOpenShellStateRpcPreflightIssue, + printOpenShellStateRpcIssue, +} from "../../adapters/openshell/gateway-drift"; +import { CLI_NAME } from "../../cli/branding"; +import { resolveSandboxGatewayName } from "../../onboard/gateway-binding"; +import * as onboardSession from "../../state/onboard-session"; +import * as registry from "../../state/registry"; +import type { RebuildBail } from "./rebuild-credential-preflight"; +import type { RebuildSandboxEntry } from "./rebuild-flow-helpers"; +import { printRebuildPreflightFailure } from "./rebuild-preflight-error"; + +export function checkRebuildGatewaySchemaPreflight( + sandboxName: string, + sb: RebuildSandboxEntry, + bail: RebuildBail, +): boolean { + const issue = detectOpenShellStateRpcPreflightIssue({ + gatewayName: resolveSandboxGatewayName(sb), + }); + if (issue) { + printOpenShellStateRpcIssue(issue, { + action: `rebuilding sandbox '${sandboxName}'`, + command: `${CLI_NAME} ${sandboxName} rebuild`, + }); + bail("OpenShell gateway schema mismatch."); + return false; + } + return true; +} + +export function getRebuildSandboxEntryOrBail( + sandboxName: string, + bail: RebuildBail, +): RebuildSandboxEntry | null { + const sb = registry.getSandbox(sandboxName) as RebuildSandboxEntry | null; + if (!sb) { + console.error(` Sandbox '${sandboxName}' not found in registry.`); + bail(`Sandbox '${sandboxName}' not found in registry.`); + return null; + } + return sb; +} + +export function isSingleAgentRebuildSupported( + sb: registry.SandboxEntry & { agents?: unknown[] }, + bail: RebuildBail, +): boolean { + if (sb.agents && sb.agents.length > 1) { + console.error(" Multi-agent sandbox rebuild is not yet supported."); + console.error(` Back up state manually and recreate with \`${CLI_NAME} onboard\`.`); + bail("Multi-agent sandbox rebuild is not yet supported."); + return false; + } + return true; +} + +export function acquireRebuildOnboardLock(sandboxName: string, bail: RebuildBail): () => void { + const lock = onboardSession.acquireOnboardLock( + `${CLI_NAME} ${sandboxName} rebuild --authoritative-resume`, + ); + if (!lock.acquired) { + console.error(` Another ${CLI_NAME} onboarding run is already in progress.`); + if (lock.holderPid) console.error(` Lock holder PID: ${lock.holderPid}`); + console.error(" Sandbox is untouched — no data was lost."); + bail("Could not acquire onboard lock before rebuild"); + } + let released = false; + const release = () => { + if (released) return; + released = true; + onboardSession.releaseOnboardLock(); + }; + process.once("exit", release); + return release; +} + +export function assertRebuildEntryUnchanged( + sandboxName: string, + confirmedEntrySnapshot: string, + bail: RebuildBail, +): void { + const lockedEntry = registry.getSandbox(sandboxName); + if (lockedEntry && JSON.stringify(lockedEntry) === confirmedEntrySnapshot) return; + printRebuildPreflightFailure( + "the sandbox configuration changed while rebuild confirmation was pending.", + "Review the current sandbox state and rerun rebuild.", + "Sandbox configuration changed before rebuild lock acquisition", + bail, + ); +} diff --git a/src/lib/actions/sandbox/rebuild-preflight-phase.ts b/src/lib/actions/sandbox/rebuild-preflight-phase.ts new file mode 100644 index 00000000000..3335fbcdc96 --- /dev/null +++ b/src/lib/actions/sandbox/rebuild-preflight-phase.ts @@ -0,0 +1,172 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { RebuildSandboxOptions } from "../../domain/lifecycle/options"; +import type { SandboxMessagingPlan } from "../../messaging"; +import type { RebuildManifest } from "../../state/sandbox"; +import { + preflightRebuildCredentials, + type RebuildBail, + type RebuildLog, +} from "./rebuild-credential-preflight"; +import { + createDcodeRebuildOrchestrator, + type DcodeRebuildOrchestrator, + isDcodeRebuildAgent, +} from "./rebuild-dcode-orchestrator"; +import { + type RebuildAgentBaseImagePreflight, + type RebuildLiveState, + type RebuildSandboxEntry, + resolveRebuildLiveState, +} from "./rebuild-flow-helpers"; +import type { RebuildRecreateOnboardOpts } from "./rebuild-gpu-opt-out"; +import { + confirmRebuildIntent, + countActiveSandboxSessionsForRebuild, + createRebuildCommandContext, + getRebuildAgentDisplayName, + type RebuildVersionCheck, +} from "./rebuild-preflight-confirmation"; +import { + acquireRebuildOnboardLock, + assertRebuildEntryUnchanged, + checkRebuildGatewaySchemaPreflight, + getRebuildSandboxEntryOrBail, + isSingleAgentRebuildSupported, +} from "./rebuild-preflight-guards"; +import { prepareRebuildTargetPreflights } from "./rebuild-preflight-target-phase"; +import { + type RebuildSandboxExecutionOptions, + validatePreparedRecoveryManifest, +} from "./rebuild-prepared-recovery"; +import type { RebuildTargetConfig } from "./rebuild-target-preflight"; + +export interface RebuildPreflightPhaseResult { + sandboxEntry: RebuildSandboxEntry; + rebuildAgent: string | null; + versionCheck: RebuildVersionCheck; + targetConfig: RebuildTargetConfig; + recreateOptions: RebuildRecreateOnboardOpts; + messagingPlan: SandboxMessagingPlan | null; + baseImagePreflight: RebuildAgentBaseImagePreflight; + liveState: RebuildLiveState; + recoveryManifest: RebuildManifest | null; + dcodePreflight: DcodeRebuildOrchestrator; + releaseOnboardLock: () => void; + log: RebuildLog; + bail: RebuildBail; +} + +/** + * Validate and pin the complete recreate contract while the old sandbox remains + * intact. The returned onboard lock stays held across every destructive phase. + * Boundary coverage: rebuild-flow-*.test.ts exercises the fail-closed + * preflights, confirmation, stale recovery, credential/image/GPU checks, and + * registry drift. + */ +export async function runRebuildPreflightPhase( + sandboxName: string, + options: string[] | RebuildSandboxOptions = {}, + opts: RebuildSandboxExecutionOptions = {}, +): Promise { + const { log, bail, skipConfirm } = createRebuildCommandContext(options, opts); + const activeSessionCount = countActiveSandboxSessionsForRebuild(sandboxName); + const sandboxEntry = getRebuildSandboxEntryOrBail(sandboxName, bail); + if (!sandboxEntry) return null; + const confirmedEntrySnapshot = JSON.stringify(sandboxEntry); + const recoveryManifest = validatePreparedRecoveryManifest( + sandboxName, + sandboxEntry, + opts.recoveryManifest, + bail, + ); + if (!isSingleAgentRebuildSupported(sandboxEntry, bail)) return null; + + const rebuildAgent = sandboxEntry.agent || null; + const agentName = getRebuildAgentDisplayName(sandboxName); + const dcodePreflight = createDcodeRebuildOrchestrator({ + sandboxName, + entry: sandboxEntry, + rebuildAgent, + log, + bail, + deps: { + checkGatewaySchema: (name, scopedBail) => + checkRebuildGatewaySchemaPreflight(name, sandboxEntry, scopedBail), + preflightCredentials: (_name, entry, scopedLog, scopedBail) => + preflightRebuildCredentials(entry, scopedLog, scopedBail), + // Non-DCode rebuilds stay on the existing typed base-image preflight. + // The orchestrator only calls this dependency when its DCode scope is disabled. + ensureAgentBaseImage: () => true, + }, + }); + let retainDcodePreflight = false; + try { + if ( + !isDcodeRebuildAgent(rebuildAgent) && + !checkRebuildGatewaySchemaPreflight(sandboxName, sandboxEntry, bail) + ) { + return null; + } + const versionCheck = await confirmRebuildIntent( + sandboxName, + agentName, + skipConfirm, + activeSessionCount, + bail, + ); + if (!versionCheck) return null; + + const releaseOnboardLock = acquireRebuildOnboardLock(sandboxName, bail); + let retainOnboardLock = false; + try { + assertRebuildEntryUnchanged(sandboxName, confirmedEntrySnapshot, bail); + const preparedTarget = await prepareRebuildTargetPreflights({ + sandboxName, + sandboxEntry, + rebuildAgent, + // Reaching this point means either --yes was supplied or confirmation + // succeeded, matching the previous `skipConfirm || confirmed` contract. + autoYes: true, + log, + bail, + }); + if (!preparedTarget) return null; + + const liveState = await resolveRebuildLiveState(sandboxName, sandboxEntry, log, bail); + if (!liveState) return null; + if (isDcodeRebuildAgent(rebuildAgent)) { + const recoveryRecreate = liveState.staleRecovery || recoveryManifest !== null; + const imageReady = await dcodePreflight.prepareImage( + preparedTarget.targetConfig.resumeConfig, + recoveryRecreate, + preparedTarget.recreateOptions.targetGatewayPort, + ); + if (!imageReady || !dcodePreflight.preparedReplacement) return null; + preparedTarget.recreateOptions.preparedDcodeRebuild = dcodePreflight.preparedReplacement; + } + retainOnboardLock = true; + retainDcodePreflight = true; + return { + sandboxEntry, + rebuildAgent, + versionCheck, + ...preparedTarget, + liveState, + recoveryManifest, + dcodePreflight, + releaseOnboardLock, + log, + bail, + }; + } finally { + if (!retainOnboardLock) { + process.removeListener("exit", releaseOnboardLock); + releaseOnboardLock(); + } + } + } finally { + if (!retainDcodePreflight) dcodePreflight.cleanup(); + } +} diff --git a/src/lib/actions/sandbox/rebuild-preflight-target-phase.ts b/src/lib/actions/sandbox/rebuild-preflight-target-phase.ts new file mode 100644 index 00000000000..4110d3e4019 --- /dev/null +++ b/src/lib/actions/sandbox/rebuild-preflight-target-phase.ts @@ -0,0 +1,142 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { CLI_NAME } from "../../cli/branding"; +import type { SandboxMessagingPlan } from "../../messaging"; +import * as registry from "../../state/registry"; +import { getSandboxTargetGatewayName } from "./gateway-target"; +import type { RebuildBail, RebuildLog } from "./rebuild-credential-preflight"; +import { isDcodeRebuildAgent } from "./rebuild-dcode-orchestrator"; +import { validatedRebuildRegistryUpdate } from "./rebuild-durable-config"; +import { + ensureRebuildAgentBaseImage, + ensureRebuildTargetGatewaySelected, + pinRebuildAgentBaseImageForRecreate, + type RebuildAgentBaseImagePreflight, + type RebuildSandboxEntry, +} from "./rebuild-flow-helpers"; +import type { RebuildRecreateOnboardOpts } from "./rebuild-gpu-opt-out"; +import { preflightRebuildMessagingConflicts } from "./rebuild-messaging-conflict-preflight"; +import { stageRebuildMessagingPlanOrBail } from "./rebuild-messaging-phase"; +import { checkRebuildGatewaySchemaPreflight } from "./rebuild-preflight-guards"; +import { + hydrateMessagingConfigForRebuild, + preflightAuthoritativeOnboardRuntime, + preflightRebuildTargetRuntime, + prepareRebuildRecreateOptions, + prepareRebuildTargetConfig, + type RebuildTargetConfig, + stageRebuildHermesDashboardConfig, +} from "./rebuild-target-preflight"; + +export interface RebuildPreparedTarget { + targetConfig: RebuildTargetConfig; + recreateOptions: RebuildRecreateOnboardOpts; + messagingPlan: SandboxMessagingPlan | null; + baseImagePreflight: RebuildAgentBaseImagePreflight; +} + +/** Resolve, validate, and persist the complete non-destructive recreate target. */ +export async function prepareRebuildTargetPreflights(args: { + sandboxName: string; + sandboxEntry: RebuildSandboxEntry; + rebuildAgent: string | null; + autoYes: boolean; + log: RebuildLog; + bail: RebuildBail; +}): Promise { + const { sandboxName, sandboxEntry, rebuildAgent, autoYes, log, bail } = args; + hydrateMessagingConfigForRebuild(sandboxName, log); + if (!(await ensureRebuildTargetGatewaySelected(sandboxName, sandboxEntry, log, bail))) + return null; + + const targetConfig = prepareRebuildTargetConfig( + sandboxName, + sandboxEntry, + rebuildAgent, + log, + bail, + ); + if (!targetConfig) return null; + const { resumeConfig, durableConfig, credentialEnv, fromDockerfile } = targetConfig; + const recreateOptions = prepareRebuildRecreateOptions( + sandboxEntry, + rebuildAgent, + fromDockerfile, + autoYes, + bail, + ); + if (!recreateOptions) return null; + if ( + !stageRebuildHermesDashboardConfig( + rebuildAgent, + sandboxEntry, + recreateOptions.controlUiPort, + bail, + ) + ) { + return null; + } + + const messagingPlan = await stageRebuildMessagingPlanOrBail( + sandboxName, + sandboxEntry, + rebuildAgent, + log, + bail, + ); + // Detect cross-sandbox credential conflicts immediately after staging the + // exact rebuild plan, before host/runtime probes and every destructive phase. + await preflightRebuildMessagingConflicts(messagingPlan, { + sandboxName, + gatewayName: getSandboxTargetGatewayName(sandboxName), + registry, + cliName: () => CLI_NAME, + log: (message) => console.log(message), + error: (message) => console.error(message), + bail, + }); + if ( + !(await preflightAuthoritativeOnboardRuntime(sandboxName, resumeConfig, recreateOptions, bail)) + ) { + return null; + } + if (!(await ensureRebuildTargetGatewaySelected(sandboxName, sandboxEntry, log, bail))) + return null; + if (!checkRebuildGatewaySchemaPreflight(sandboxName, sandboxEntry, bail)) return null; + + const rebuildsDcodeSandbox = isDcodeRebuildAgent(rebuildAgent); + const baseImagePreflight = rebuildsDcodeSandbox + ? { ok: true, imageRef: null, overrideEnvVar: null } + : ensureRebuildAgentBaseImage(rebuildAgent, bail); + if (!baseImagePreflight.ok) return null; + const restoreBaseImageOverride = pinRebuildAgentBaseImageForRecreate(baseImagePreflight); + let targetRuntimeReady = false; + try { + targetRuntimeReady = await preflightRebuildTargetRuntime( + targetConfig, + sandboxEntry, + recreateOptions, + log, + bail, + { skipImagePreflight: rebuildsDcodeSandbox }, + ); + } finally { + restoreBaseImageOverride(); + } + if (!targetRuntimeReady) return null; + + const validatedRegistryUpdate = validatedRebuildRegistryUpdate( + resumeConfig, + durableConfig, + fromDockerfile, + credentialEnv, + ); + if (!registry.updateSandbox(sandboxName, validatedRegistryUpdate)) { + bail("Sandbox registry entry disappeared during rebuild preflight"); + return null; + } + Object.assign(sandboxEntry, validatedRegistryUpdate); + + return { targetConfig, recreateOptions, messagingPlan, baseImagePreflight }; +} diff --git a/src/lib/actions/sandbox/rebuild-prepared-recovery.ts b/src/lib/actions/sandbox/rebuild-prepared-recovery.ts new file mode 100644 index 00000000000..6ababf3eeda --- /dev/null +++ b/src/lib/actions/sandbox/rebuild-prepared-recovery.ts @@ -0,0 +1,129 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { isDeepStrictEqual } from "node:util"; + +import { RD as _RD, R } from "../../cli/terminal-style"; +import * as registry from "../../state/registry"; +import * as sandboxState from "../../state/sandbox"; +import type { RebuildBail } from "./rebuild-credential-preflight"; +import type { RebuildSandboxEntry } from "./rebuild-flow-helpers"; + +export interface RebuildSandboxExecutionOptions { + throwOnError?: boolean; + /** Internal installer recovery input; never exposed as a CLI option. */ + recoveryManifest?: sandboxState.RebuildManifest; +} + +function failPreparedRecoveryPreDelete( + detail: string, + errorMessage: string, + bail: RebuildBail, +): never { + console.error(""); + console.error(` ${_RD}Recovery pre-delete check failed:${R} ${detail}.`); + console.error(" Sandbox is untouched — no data was lost."); + return bail(errorMessage); +} + +export function validatePreparedRecoveryManifest( + sandboxName: string, + sandboxEntry: RebuildSandboxEntry, + candidate: sandboxState.RebuildManifest | undefined, + bail: RebuildBail, +): sandboxState.RebuildManifest | null { + if (!candidate) return null; + const validation = sandboxState.validateRebuildRecoveryManifest( + sandboxName, + sandboxEntry.agent, + candidate, + ); + if (!validation.ok) { + console.error(""); + console.error(` ${_RD}Recovery preflight failed:${R} ${validation.reason}.`); + console.error(" Sandbox is untouched — no data was lost."); + bail(`Invalid recovery manifest: ${validation.reason}`); + return null; + } + if (!sandboxState.hasPositiveManagedImageEvidence(sandboxEntry)) { + console.error(""); + console.error( + ` ${_RD}Recovery preflight failed:${R} registry has no NemoClaw-managed image fingerprint.`, + ); + console.error(" Pre-fingerprint and custom-image sandboxes are not recreated automatically."); + console.error(" Sandbox is untouched — no data was lost."); + bail("Recovery registry entry has no NemoClaw-managed image fingerprint."); + return null; + } + return validation.manifest; +} + +export function revalidatePreparedRecoveryBeforeDelete( + sandboxName: string, + initialEntry: RebuildSandboxEntry, + candidate: sandboxState.RebuildManifest | null, + registrySnapshot: registry.SandboxRegistry | null, + bail: RebuildBail, +): { + manifest: sandboxState.RebuildManifest | null; + registrySnapshot: registry.SandboxRegistry | null; +} { + if (!candidate) return { manifest: null, registrySnapshot }; + + const refreshedRegistrySnapshot = JSON.parse( + JSON.stringify(registry.load()), + ) as registry.SandboxRegistry; + const currentEntry = refreshedRegistrySnapshot.sandboxes[sandboxName]; + if (!currentEntry) { + return failPreparedRecoveryPreDelete( + "registry entry no longer exists", + "Recovery registry identity changed during preflight.", + bail, + ); + } + if (!isDeepStrictEqual(currentEntry, initialEntry)) { + return failPreparedRecoveryPreDelete( + "registered sandbox configuration changed during preflight", + "Recovery registry configuration changed during preflight.", + bail, + ); + } + + const latestManifest = sandboxState.getLatestBackup(sandboxName); + if ( + !latestManifest || + latestManifest.timestamp !== candidate.timestamp || + latestManifest.backupPath !== candidate.backupPath + ) { + return failPreparedRecoveryPreDelete( + "latest prepared backup changed during preflight", + "Recovery backup identity changed during preflight.", + bail, + ); + } + + const validation = sandboxState.validateRebuildRecoveryManifest( + sandboxName, + currentEntry.agent, + latestManifest, + ); + if (!validation.ok) { + return failPreparedRecoveryPreDelete( + validation.reason, + `Invalid recovery manifest: ${validation.reason}`, + bail, + ); + } + if (!sandboxState.hasPositiveManagedImageEvidence(currentEntry)) { + return failPreparedRecoveryPreDelete( + "registry no longer has a NemoClaw-managed image fingerprint", + "Recovery registry entry has no NemoClaw-managed image fingerprint.", + bail, + ); + } + + return { + manifest: validation.manifest, + registrySnapshot: refreshedRegistrySnapshot, + }; +} diff --git a/src/lib/actions/sandbox/rebuild-recreate-phase.ts b/src/lib/actions/sandbox/rebuild-recreate-phase.ts new file mode 100644 index 00000000000..5e832cf66ee --- /dev/null +++ b/src/lib/actions/sandbox/rebuild-recreate-phase.ts @@ -0,0 +1,260 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { CLI_NAME } from "../../cli/branding"; +import { RD as _RD, R } from "../../cli/terminal-style"; +import type { SandboxMessagingPlan } from "../../messaging"; +import { markLastStartedStepFailed } from "../../onboard/exit-step-failure"; +import * as shields from "../../shields"; +import type { Session } from "../../state/onboard-session"; +import * as onboardSession from "../../state/onboard-session"; +import * as registry from "../../state/registry"; +import type { RebuildBackupManifest } from "./rebuild-backup-phase"; +import type { RebuildBail, RebuildLog } from "./rebuild-credential-preflight"; +import type { RebuildDurableConfig } from "./rebuild-durable-config"; +import { isolateAmbientRecreateEnv } from "./rebuild-env-isolation"; +import { + pinRebuildAgentBaseImageForRecreate, + type RebuildAgentBaseImagePreflight, + type RebuildSandboxEntry, +} from "./rebuild-flow-helpers"; +import { + getRebuildSandboxGpuOverrides, + type RebuildRecreateOnboardOpts, +} from "./rebuild-gpu-opt-out"; +import { + type McpRebuildPreparation, + printMcpRebuildRetryCommand, + restoreMcpRegistryForRebuildRetry, +} from "./rebuild-mcp-phase"; +import type { RebuildResumeConfig } from "./rebuild-resume-config"; +import { printRebuildShieldsRecovery, type RebuildShieldsWindow } from "./rebuild-shields"; + +export interface RebuildRecreatePhaseInput { + sandboxName: string; + sandboxEntry: RebuildSandboxEntry; + sessionSnapshot: Session | null; + sessionMatchesSandbox: boolean; + durableConfig: RebuildDurableConfig; + resumeConfig: RebuildResumeConfig; + recreateOptions: RebuildRecreateOnboardOpts; + fromDockerfile: string | null; + rebuildAgent: string | null; + messagingPlan: SandboxMessagingPlan | null; + rebuildsHermesSandbox: boolean; + hermesToolGateways: string[]; + hasHermesToolGateways: boolean; + sessionPolicyPresets: string[] | null; + credentialEnv: string | null; + baseImagePreflight: RebuildAgentBaseImagePreflight; + recoveryRecreate: boolean; + recoveryRegistrySnapshot: ReturnType | null; + backupManifest: RebuildBackupManifest; + mcpEntries: McpRebuildPreparation["entries"]; + rebuildShieldsWindow: RebuildShieldsWindow; + relockShieldsIfNeeded: (sandboxStillExists: boolean) => boolean; + onCreated: () => void; + log: RebuildLog; + bail: RebuildBail; +} + +/** + * Recreate the deleted sandbox from its validated registry-derived contract. + * Boundary coverage: rebuild-flow.test.ts exercises success, process-exit and + * thrown failures, stale/MCP retry restoration, session pinning, and env isolation. + */ +export async function runRebuildRecreatePhase(input: RebuildRecreatePhaseInput): Promise { + const { + sandboxName, + sandboxEntry: sb, + sessionSnapshot: sessionBefore, + sessionMatchesSandbox, + durableConfig: rebuildDurableConfig, + resumeConfig, + recreateOptions, + fromDockerfile: storedFromDockerfile, + rebuildAgent, + messagingPlan: rebuildMessagingPlan, + rebuildsHermesSandbox, + hermesToolGateways: rebuildHermesToolGateways, + hasHermesToolGateways: hasRebuildHermesToolGateways, + sessionPolicyPresets: rebuildSessionPolicyPresets, + credentialEnv: rebuildCredentialEnv, + baseImagePreflight: rebuildBaseImagePreflight, + recoveryRecreate, + recoveryRegistrySnapshot, + backupManifest, + mcpEntries: rebuildMcpEntries, + rebuildShieldsWindow, + relockShieldsIfNeeded, + onCreated, + log, + bail, + } = input; + + console.log(""); + console.log(" Creating new sandbox with current image..."); + + const rebuildGpuOverrides = getRebuildSandboxGpuOverrides(sb); + log( + `Session before update: sandboxName=${sessionBefore?.sandboxName}, status=${sessionBefore?.status}, resumable=${sessionBefore?.resumable}, provider=${sessionBefore?.provider}, model=${sessionBefore?.model}, sessionMatch=${sessionMatchesSandbox}`, + ); + + onboardSession.updateSession((s: Session) => { + Object.assign( + s, + onboardSession.createSession({ + mode: "non-interactive", + hermesAuthMethod: rebuildDurableConfig.hermesAuthMethod, + webSearchConfig: rebuildDurableConfig.webSearchConfig, + telegramConfig: sessionMatchesSandbox ? sessionBefore?.telegramConfig : null, + wechatConfig: sessionMatchesSandbox ? sessionBefore?.wechatConfig : null, + migratedLegacyValueHashes: sessionMatchesSandbox + ? sessionBefore?.migratedLegacyValueHashes + : null, + routerPid: resumeConfig.provider === "nvidia-router" ? sessionBefore?.routerPid : undefined, + routerCredentialHash: + resumeConfig.provider === "nvidia-router" ? sessionBefore?.routerCredentialHash : null, + metadata: { + gatewayName: recreateOptions.targetGatewayName, + fromDockerfile: storedFromDockerfile, + }, + }), + ); + s.steps.preflight.status = "complete"; + s.steps.preflight.startedAt = null; + s.steps.preflight.completedAt = s.updatedAt; + s.steps.preflight.error = null; + s.steps.gateway.status = "complete"; + s.steps.gateway.startedAt = null; + s.steps.gateway.completedAt = s.updatedAt; + s.steps.gateway.error = null; + s.sandboxName = sandboxName; + s.resumable = true; + s.status = "in_progress"; + s.agent = rebuildAgent; + s.messagingPlan = rebuildMessagingPlan; + s.hermesToolGateways = rebuildsHermesSandbox ? rebuildHermesToolGateways : []; + s.policyPresets = rebuildSessionPolicyPresets; + s.gpuPassthrough = rebuildGpuOverrides.sessionGpuPassthrough; + s.metadata.fromDockerfile = storedFromDockerfile; + s.provider = resumeConfig.provider; + s.model = resumeConfig.model; + s.nimContainer = resumeConfig.nimContainer; + s.credentialEnv = rebuildCredentialEnv; + s.preferredInferenceApi = resumeConfig.preferredInferenceApi; + s.compatibleEndpointReasoning = resumeConfig.compatibleEndpointReasoning; + s.endpointUrl = resumeConfig.endpointUrl; + return s; + }); + const sessionAfter = onboardSession.loadSession(); + log( + `Session after update: sandboxName=${sessionAfter?.sandboxName}, status=${sessionAfter?.status}, resumable=${sessionAfter?.resumable}, provider=${sessionAfter?.provider}, model=${sessionAfter?.model}`, + ); + log( + `Recreate env will target NEMOCLAW_SANDBOX_NAME=${sandboxName}; NEMOCLAW_RECREATE_SANDBOX=${process.env.NEMOCLAW_RECREATE_SANDBOX}`, + ); + log( + `Calling onboard({ resume: true, nonInteractive: true, recreateSandbox: true, fromDockerfile: ${storedFromDockerfile} })`, + ); + + // Intercept process.exit so a failed inner onboard can preserve the backup + // and durable retry state instead of terminating the outer transaction. + const { onboard } = require("../../onboard") as { + onboard: (options: RebuildRecreateOnboardOpts) => Promise; + }; + let onboardFailed = false; + let onboardExitCode = 1; + const savedExit = process.exit; + process.exit = ((code) => { + onboardFailed = true; + onboardExitCode = typeof code === "number" ? code : 1; + const error = new Error(`onboard exited with code ${onboardExitCode}`); + error.name = "RebuildOnboardExit"; + throw error; + }) as typeof process.exit; + + const restoreAmbientRecreateEnv = isolateAmbientRecreateEnv(); + const previousSandboxName = process.env.NEMOCLAW_SANDBOX_NAME; + process.env.NEMOCLAW_SANDBOX_NAME = sandboxName; + const restoreRebuildBaseImageOverride = + pinRebuildAgentBaseImageForRecreate(rebuildBaseImagePreflight); + try { + await onboard(recreateOptions); + log("onboard() returned successfully"); + } catch (error) { + onboardFailed = true; + const message = error instanceof Error ? error.message : String(error); + const name = error instanceof Error ? error.name : ""; + if (name !== "RebuildOnboardExit") log(`onboard() threw: ${message}`); + } finally { + process.exit = savedExit; + restoreRebuildBaseImageOverride(); + restoreAmbientRecreateEnv(); + if (previousSandboxName === undefined) delete process.env.NEMOCLAW_SANDBOX_NAME; + else process.env.NEMOCLAW_SANDBOX_NAME = previousSandboxName; + } + + if (!onboardFailed) onCreated(); + if (onboardFailed) { + try { + markLastStartedStepFailed(onboardSession, "Rebuild recreate failed"); + } catch { + /* best effort */ + } + + const snapshotEntry = recoveryRegistrySnapshot?.sandboxes?.[sandboxName]; + if (recoveryRecreate && snapshotEntry) { + try { + registry.restoreSandboxEntry(snapshotEntry, { + reclaimDefault: + recoveryRegistrySnapshot?.defaultSandbox === sandboxName ? sandboxName : null, + }); + log("Recovery recreate failed: restored preserved registry entry for retry"); + } catch (error) { + log(`Failed to restore registry entry after recovery recreate failure: ${String(error)}`); + } + } + restoreMcpRegistryForRebuildRetry(recoveryRecreate, rebuildMcpEntries, sb, log); + + console.error(""); + if (recoveryRecreate) { + console.error(` ${_RD}Recovery recreate failed.${R}`); + console.error( + " Your local registry entry has been preserved — you can retry once the issue above is fixed.", + ); + } else { + console.error(` ${_RD}Recreate failed after sandbox was destroyed.${R}`); + } + if (backupManifest) console.error(` Backup is preserved at: ${backupManifest.backupPath}`); + console.error(""); + console.error(" To recover manually:"); + console.error(" 1. Fix the issue above (missing credential, Docker problem, etc.)"); + printMcpRebuildRetryCommand(sandboxName, rebuildMcpEntries); + if (backupManifest) { + console.error(" 3. Then restore your workspace state:"); + console.error( + ` ${CLI_NAME} ${sandboxName} snapshot restore "${backupManifest.timestamp}"`, + ); + } + printRebuildShieldsRecovery(sandboxName, rebuildShieldsWindow, CLI_NAME); + console.error(""); + relockShieldsIfNeeded(false); + bail( + backupManifest + ? `Recreate failed (sandbox destroyed). Backup: ${backupManifest.backupPath}` + : "Recreate failed (stale-sandbox recovery).", + onboardExitCode, + ); + return false; + } + + if (recoveryRecreate) shields.clearShieldsState(sandboxName); + const preservedRegistryFields = { + ...(hasRebuildHermesToolGateways ? { hermesToolGateways: [...rebuildHermesToolGateways] } : {}), + }; + if (Object.keys(preservedRegistryFields).length > 0) { + registry.updateSandbox(sandboxName, preservedRegistryFields); + } + return true; +} diff --git a/src/lib/actions/sandbox/rebuild-restore-phase.ts b/src/lib/actions/sandbox/rebuild-restore-phase.ts new file mode 100644 index 00000000000..5790dd0fc20 --- /dev/null +++ b/src/lib/actions/sandbox/rebuild-restore-phase.ts @@ -0,0 +1,83 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { CLI_NAME } from "../../cli/branding"; +import { G, R, YW } from "../../cli/terminal-style"; +import * as policies from "../../policy"; +import * as sandboxState from "../../state/sandbox"; +import type { RebuildBackupManifest } from "./rebuild-backup-phase"; +import type { RebuildLog } from "./rebuild-credential-preflight"; + +export interface RebuildRestorePhaseInput { + sandboxName: string; + backupManifest: RebuildBackupManifest; + policyPresets: string[]; + log: RebuildLog; +} + +export interface RebuildRestorePhaseResult { + restoreSucceeded: boolean; + restoredPresets: string[]; + failedPresets: string[]; +} + +/** + * Restore preserved workspace state and gateway-owned built-in policy presets. + * Boundary coverage: rebuild-flow.test.ts exercises full/partial state restore, + * stale recovery, successful presets, and incomplete preset recovery reporting. + */ +export function runRebuildRestorePhase(input: RebuildRestorePhaseInput): RebuildRestorePhaseResult { + const { sandboxName, backupManifest, policyPresets, log } = input; + let restoreSucceeded = true; + if (backupManifest) { + console.log(""); + console.log(" Restoring workspace state..."); + log(`Restoring from: ${backupManifest.backupPath} into sandbox: ${sandboxName}`); + const restore = sandboxState.restoreSandboxState(sandboxName, backupManifest.backupPath); + log( + `Restore result: success=${restore.success}, restored=${restore.restoredDirs.join(",")}; files=${restore.restoredFiles.join(",")}, failed=${restore.failedDirs.join(",")}; failedFiles=${restore.failedFiles.join(",")}`, + ); + restoreSucceeded = restore.success; + if (!restore.success) { + console.error(` Partial restore: ${restore.restoredDirs.join(", ") || "none"}`); + console.error(` Failed: ${restore.failedDirs.join(", ")}`); + if (restore.failedFiles.length > 0) { + console.error(` Failed files: ${restore.failedFiles.join(", ")}`); + } + console.error(` Manual restore available from: ${backupManifest.backupPath}`); + } else { + console.log( + ` ${G}\u2713${R} State restored (${restore.restoredDirs.length} directories, ${restore.restoredFiles.length} files)`, + ); + } + } + + const restoredPresets: string[] = []; + const failedPresets: string[] = []; + if (policyPresets.length > 0) { + console.log(""); + console.log(" Restoring policy presets..."); + log(`Policy presets to restore: [${policyPresets.join(",")}]`); + for (const presetName of policyPresets) { + try { + log(`Applying preset: ${presetName}`); + const applied = policies.applyPreset(sandboxName, presetName); + if (applied) restoredPresets.push(presetName); + else failedPresets.push(presetName); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + log(`Failed to apply preset '${presetName}': ${message}`); + failedPresets.push(presetName); + } + } + if (restoredPresets.length > 0) { + console.log(` ${G}\u2713${R} Policy presets restored: ${restoredPresets.join(", ")}`); + } + if (failedPresets.length > 0) { + console.error(` ${YW}\u26a0${R} Failed to restore presets: ${failedPresets.join(", ")}`); + console.error(` Re-apply manually with: ${CLI_NAME} ${sandboxName} policy-add`); + } + } + + return { restoreSucceeded, restoredPresets, failedPresets }; +} diff --git a/src/lib/actions/sandbox/rebuild-resume-config.test.ts b/src/lib/actions/sandbox/rebuild-resume-config.test.ts index 4251212ef0a..8218dfec1c6 100644 --- a/src/lib/actions/sandbox/rebuild-resume-config.test.ts +++ b/src/lib/actions/sandbox/rebuild-resume-config.test.ts @@ -137,9 +137,58 @@ describe("getRebuildEndpointFromRegistry", () => { }); describe("prepareRebuildResumeConfig", () => { + it("recovers a complete legacy selection only from the target's matching session", () => { + vi.spyOn(onboardSession, "loadSession").mockReturnValue({ + sandboxName: "alpha", + provider: "nvidia-prod", + model: "nvidia/legacy-model", + credentialEnv: "NVIDIA_INFERENCE_API_KEY", + preferredInferenceApi: "openai-completions", + }); + const config = prepareRebuildResumeConfig("alpha", entry(), null, noopLog, throwingBail); + expect(config).toMatchObject({ + provider: "nvidia-prod", + model: "nvidia/legacy-model", + credentialEnv: "NVIDIA_INFERENCE_API_KEY", + preferredInferenceApi: "openai-completions", + }); + }); + + it("surfaces the legacy local credential migration while clearing the stale key", () => { + vi.spyOn(onboardSession, "loadSession").mockReturnValue({ + sandboxName: "alpha", + provider: "ollama-local", + model: "llama3.2", + credentialEnv: "OPENAI_API_KEY", + }); + const log = vi.fn(); + const consoleLog = vi.spyOn(console, "log").mockImplementation(() => undefined); + + const config = prepareRebuildResumeConfig( + "alpha", + entry({ provider: "ollama-local", model: "llama3.2" }), + null, + log, + throwingBail, + ); + + expect(config?.credentialEnv).toBeNull(); + expect(consoleLog).toHaveBeenCalledWith(expect.stringContaining("GH #2519")); + expect(log).toHaveBeenCalledWith(expect.stringContaining("clearing for rebuild")); + }); + + it("fails closed when neither registry nor matching session has a complete selection", () => { + vi.spyOn(onboardSession, "loadSession").mockReturnValue({ sandboxName: "other" }); + expect(() => prepareRebuildResumeConfig("alpha", entry(), null, noopLog, throwingBail)).toThrow( + "Cannot determine recorded inference provider and model", + ); + }); + it("validates and canonicalizes a matching custom-endpoint session endpoint", () => { vi.spyOn(onboardSession, "loadSession").mockReturnValue({ sandboxName: "alpha", + provider: "compatible-endpoint", + model: "m", endpointUrl: " http://127.0.0.1:19999/v1/?x=1#frag ", }); const config = prepareRebuildResumeConfig( @@ -185,6 +234,8 @@ describe("prepareRebuildResumeConfig", () => { it("ignores target-scoped explicit env when the custom-endpoint session matches the sandbox", () => { vi.spyOn(onboardSession, "loadSession").mockReturnValue({ sandboxName: "alpha", + provider: "compatible-endpoint", + model: "m", endpointUrl: "https://session.example.test/v1?x=1#frag", }); const restore = snapshotEnv([ @@ -228,6 +279,8 @@ describe("prepareRebuildResumeConfig", () => { it("fails closed for a matching custom-endpoint session with an invalid endpoint", () => { vi.spyOn(onboardSession, "loadSession").mockReturnValue({ sandboxName: "alpha", + provider: "compatible-endpoint", + model: "m", endpointUrl: "https://user:pass@example.test/v1", }); expect(() => @@ -241,6 +294,24 @@ describe("prepareRebuildResumeConfig", () => { ).toThrow("Cannot validate recreate endpoint"); }); + it("does not borrow a custom endpoint from a conflicting same-sandbox selection", () => { + vi.spyOn(onboardSession, "loadSession").mockReturnValue({ + sandboxName: "alpha", + provider: "nvidia-prod", + model: "different-model", + endpointUrl: "https://wrong.example.test/v1", + }); + expect(() => + prepareRebuildResumeConfig( + "alpha", + entry({ provider: "compatible-endpoint", model: "m" }), + null, + noopLog, + throwingBail, + ), + ).toThrow("Cannot validate recreate endpoint"); + }); + it("pins the canonical endpoint when the session belongs to another sandbox", () => { vi.spyOn(onboardSession, "loadSession").mockReturnValue({ sandboxName: "other" }); const config = prepareRebuildResumeConfig( @@ -403,6 +474,7 @@ describe("prepareRebuildResumeConfig", () => { endpointUrl: "http://127.0.0.1:19999/v1", credentialEnv: "COMPATIBLE_API_KEY", preferredInferenceApi: "openai-completions", + compatibleEndpointReasoning: "true", }), null, noopLog, @@ -413,11 +485,52 @@ describe("prepareRebuildResumeConfig", () => { model: "m", credentialEnv: "COMPATIBLE_API_KEY", preferredInferenceApi: "openai-completions", + compatibleEndpointReasoning: "true", pinEndpoint: true, endpointUrl: "http://127.0.0.1:19999/v1", }); }); + it("does not borrow compatible-endpoint reasoning from an unrelated session", () => { + vi.spyOn(onboardSession, "loadSession").mockReturnValue({ + sandboxName: "other", + compatibleEndpointReasoning: "true", + }); + const config = prepareRebuildResumeConfig( + "alpha", + entry({ + provider: "compatible-endpoint", + model: "m", + endpointUrl: "https://example.test/v1", + }), + null, + noopLog, + throwingBail, + ); + expect(config?.compatibleEndpointReasoning).toBeNull(); + }); + + it("uses the target session as a legacy reasoning fallback", () => { + vi.spyOn(onboardSession, "loadSession").mockReturnValue({ + sandboxName: "alpha", + provider: "compatible-endpoint", + model: "m", + compatibleEndpointReasoning: "false", + }); + const config = prepareRebuildResumeConfig( + "alpha", + entry({ + provider: "compatible-endpoint", + model: "m", + endpointUrl: "https://example.test/v1", + }), + null, + noopLog, + throwingBail, + ); + expect(config?.compatibleEndpointReasoning).toBe("false"); + }); + it("fails closed for invalid durable custom endpoint metadata before delete", () => { vi.spyOn(onboardSession, "loadSession").mockReturnValue({ sandboxName: "other" }); expect(() => @@ -453,7 +566,13 @@ describe("prepareRebuildResumeConfig", () => { const prior = process.env.NEMOCLAW_AGENT; process.env.NEMOCLAW_AGENT = "langchain-deepagents-code"; try { - const config = prepareRebuildResumeConfig("alpha", entry(), null, noopLog, throwingBail); + const config = prepareRebuildResumeConfig( + "alpha", + entry({ provider: "nvidia-prod", model: "nvidia/test" }), + null, + noopLog, + throwingBail, + ); expect(config?.ambient.agentMismatch).toEqual({ envAgent: "langchain-deepagents-code", registryAgent: "openclaw", diff --git a/src/lib/actions/sandbox/rebuild-resume-config.ts b/src/lib/actions/sandbox/rebuild-resume-config.ts index 86957e98043..14b10e484fd 100644 --- a/src/lib/actions/sandbox/rebuild-resume-config.ts +++ b/src/lib/actions/sandbox/rebuild-resume-config.ts @@ -182,11 +182,12 @@ function getExplicitTargetEndpointFromEnv( */ export interface RebuildResumeConfig { readonly agent: string | null; - readonly provider: string | null; - readonly model: string | null; + readonly provider: string; + readonly model: string; readonly nimContainer: string | null; readonly credentialEnv: string | null; readonly preferredInferenceApi: string | null; + readonly compatibleEndpointReasoning: "true" | "false" | null; /** * Whether this endpoint was derived without trusting the matching onboard * session. Kept for preflight/tests; rebuild writes `endpointUrl` @@ -243,16 +244,67 @@ export function prepareRebuildResumeConfig( const session = onboardSession.loadSession(); const sessionMatchesSandbox = session?.sandboxName === sandboxName; const registrySelection = normalizeInferenceSelection(sb); + const matchingSessionSelection = sessionMatchesSandbox + ? normalizeInferenceSelection(session) + : null; + const sessionSelectionMatchesRegistry = Boolean( + matchingSessionSelection && + (!registrySelection.provider || + matchingSessionSelection.provider === registrySelection.provider) && + (!registrySelection.model || matchingSessionSelection.model === registrySelection.model), + ); + const legacySelection = sessionSelectionMatchesRegistry ? matchingSessionSelection : null; + const trustedSelection = normalizeInferenceSelection({ + provider: registrySelection.provider ?? legacySelection?.provider, + model: registrySelection.model ?? legacySelection?.model, + endpointUrl: registrySelection.endpointUrl, + credentialEnv: registrySelection.credentialEnv ?? legacySelection?.credentialEnv, + preferredInferenceApi: + registrySelection.preferredInferenceApi ?? legacySelection?.preferredInferenceApi, + compatibleEndpointReasoning: + registrySelection.compatibleEndpointReasoning ?? legacySelection?.compatibleEndpointReasoning, + nimContainer: registrySelection.nimContainer ?? legacySelection?.nimContainer, + }); + if (!trustedSelection.provider || !trustedSelection.model) { + console.error(""); + console.error( + ` ${_RD}Rebuild preflight failed:${R} cannot determine the recorded inference provider and model.`, + ); + console.error( + ` Neither the '${sandboxName}' registry entry nor its own matching onboard session contains a complete selection.`, + ); + console.error(" Sandbox is untouched — no data was lost."); + bail("Cannot determine recorded inference provider and model for recreate"); + return null; + } + // Compatibility boundary for GH #2519: pre-fix local-provider sessions + // could persist credentialEnv="OPENAI_API_KEY" even though local inference + // never required a host credential. Only recognize the target sandbox's own + // matching selection; a stale session for another provider or sandbox must + // not influence the authoritative recreate config. + if ( + legacySelection?.credentialEnv === "OPENAI_API_KEY" && + isLocalInferenceProvider(trustedSelection.provider) + ) { + console.log( + ` ${D}Note: migrating ${trustedSelection.provider} sandbox off OPENAI_API_KEY (GH #2519). ` + + `Local inference does not require a host API key.${R}`, + ); + log( + `Preflight: legacy ${trustedSelection.provider} sandbox detected (credentialEnv=OPENAI_API_KEY) — clearing for rebuild`, + ); + } + const compatibleEndpointReasoning = trustedSelection.compatibleEndpointReasoning; const rebuildEndpoint = getRebuildEndpointFromRegistry( - registrySelection.provider, + trustedSelection.provider, registrySelection.endpointUrl, ); const explicitTargetEndpoint = !sessionMatchesSandbox && !rebuildEndpoint.known ? getExplicitTargetEndpointFromEnv( sandboxName, - registrySelection.provider, - registrySelection.model, + trustedSelection.provider, + trustedSelection.model, ) : null; @@ -271,15 +323,14 @@ export function prepareRebuildResumeConfig( // sandbox stays live. if ( !sessionMatchesSandbox && - registrySelection.provider && - !isLocalInferenceProvider(registrySelection.provider) && - registrySelection.provider !== hermesProviderAuth.HERMES_PROVIDER_NAME && + !isLocalInferenceProvider(trustedSelection.provider) && + trustedSelection.provider !== hermesProviderAuth.HERMES_PROVIDER_NAME && !rebuildEndpoint.known && !explicitTargetEndpoint ) { console.error(""); console.error( - ` ${_RD}Rebuild preflight failed:${R} cannot determine the inference endpoint for provider '${registrySelection.provider}'.`, + ` ${_RD}Rebuild preflight failed:${R} cannot determine the inference endpoint for provider '${trustedSelection.provider}'.`, ); console.error( ` The custom endpoint for '${sandboxName}' is recorded only in its own onboard session,`, @@ -290,7 +341,7 @@ export function prepareRebuildResumeConfig( console.error(""); console.error(" Sandbox is untouched — no data was lost."); bail( - `Cannot determine recreate endpoint for provider '${registrySelection.provider}' without a matching session`, + `Cannot determine recreate endpoint for provider '${trustedSelection.provider}' without a matching session`, ); return null; } @@ -302,34 +353,40 @@ export function prepareRebuildResumeConfig( // canonicalization. // 3. The target sandbox's own matching session endpoint, validated below. let endpointUrl = rebuildEndpoint.known ? rebuildEndpoint.endpointUrl : explicitTargetEndpoint; - if (!endpointUrl && !rebuildEndpoint.known && sessionMatchesSandbox) { + if ( + !endpointUrl && + !rebuildEndpoint.known && + sessionMatchesSandbox && + sessionSelectionMatchesRegistry + ) { endpointUrl = canonicalCustomEndpointUrl(session?.endpointUrl); - if (!endpointUrl) { - console.error(""); - console.error( - ` ${_RD}Rebuild preflight failed:${R} cannot validate the inference endpoint for provider '${registrySelection.provider}'.`, - ); - console.error( - ` The custom endpoint for '${sandboxName}' is missing or invalid in its onboard session.`, - ); - console.error(" Sandbox is untouched — no data was lost."); - bail( - `Cannot validate recreate endpoint for provider '${registrySelection.provider}' from matching session`, - ); - return null; - } + } + if (!endpointUrl && !rebuildEndpoint.known) { + console.error(""); + console.error( + ` ${_RD}Rebuild preflight failed:${R} cannot validate the inference endpoint for provider '${trustedSelection.provider}'.`, + ); + console.error( + ` The custom endpoint for '${sandboxName}' is missing, invalid, or belongs to a conflicting onboard selection.`, + ); + console.error(" Sandbox is untouched — no data was lost."); + bail( + `Cannot validate recreate endpoint for provider '${trustedSelection.provider}' from matching session`, + ); + return null; } return { agent: rebuildAgent, - provider: registrySelection.provider, - model: registrySelection.model, - nimContainer: registrySelection.nimContainer, + provider: trustedSelection.provider, + model: trustedSelection.model, + nimContainer: trustedSelection.nimContainer, credentialEnv: getRebuildCredentialEnvFromRegistry( - registrySelection.provider, - registrySelection.credentialEnv, + trustedSelection.provider, + trustedSelection.credentialEnv, ), - preferredInferenceApi: registrySelection.preferredInferenceApi, + preferredInferenceApi: trustedSelection.preferredInferenceApi, + compatibleEndpointReasoning, pinEndpoint: rebuildEndpoint.known || explicitTargetEndpoint !== null, endpointUrl, ambient, diff --git a/src/lib/actions/sandbox/rebuild-resume-snapshot.test.ts b/src/lib/actions/sandbox/rebuild-resume-snapshot.test.ts index 379b5f1de4e..9bd79df1d65 100644 --- a/src/lib/actions/sandbox/rebuild-resume-snapshot.test.ts +++ b/src/lib/actions/sandbox/rebuild-resume-snapshot.test.ts @@ -26,24 +26,31 @@ describe("rebuild resume snapshot repair", () => { const observed = { handoffOptions: null as Record | null, preRepairMachineState: null as string | null, + preRepairPreflightStatus: null as string | null, + preRepairGatewayStatus: null as string | null, preRepairStatus: null as string | null, preRepairResumable: null as boolean | null, repairedMachineState: null as string | null, + sandboxEnvInsideOnboard: null as string | null, }; beforeEach(() => { spies = []; observed.handoffOptions = null; observed.preRepairMachineState = null; + observed.preRepairPreflightStatus = null; + observed.preRepairGatewayStatus = null; observed.preRepairStatus = null; observed.preRepairResumable = null; observed.repairedMachineState = null; + observed.sandboxEnvInsideOnboard = null; delete require.cache[requireDist.resolve(rebuildModulePath)]; errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); logSpy = vi.spyOn(console, "log").mockImplementation(() => undefined); const gatewayDrift = requireDist("../../adapters/openshell/gateway-drift.js"); + const gatewayRuntime = requireDist("../../gateway-runtime-action.js"); const openshellRuntime = requireDist("../../adapters/openshell/runtime.js"); const sandboxList = requireDist("../../openshell-sandbox-list.js"); const resolve = requireDist("../../adapters/openshell/resolve.js"); @@ -58,6 +65,8 @@ describe("rebuild resume snapshot repair", () => { const sandboxVersion = requireDist("../../sandbox/version.js"); const destroy = requireDist("./destroy.js"); const rebuildShields = requireDist("./rebuild-shields.js"); + const rebuildImagePreflight = requireDist("./rebuild-custom-image-preflight.js"); + const rebuildUsageNotice = requireDist("./rebuild-usage-notice.js"); const nim = requireDist("../../inference/nim.js"); session = onboardSession.createSession({ @@ -90,6 +99,11 @@ describe("rebuild resume snapshot repair", () => { spies.push( vi.spyOn(gatewayDrift, "detectOpenShellStateRpcPreflightIssue").mockReturnValue(null), vi.spyOn(gatewayDrift, "detectOpenShellStateRpcResultIssue").mockReturnValue(null), + vi.spyOn(gatewayRuntime, "recoverNamedGatewayRuntime").mockResolvedValue({ + recovered: true, + before: { state: "healthy_named" }, + after: { state: "healthy_named" }, + }), vi.spyOn(sandboxList, "captureSandboxListWithGatewayRecovery").mockResolvedValue({ result: { status: 0, output: "alpha Ready" }, }), @@ -101,6 +115,7 @@ describe("rebuild resume snapshot repair", () => { vi.spyOn(agentRuntime, "getAgentDisplayName").mockReturnValue("OpenClaw"), vi.spyOn(onboardSession, "loadSession").mockImplementation(loadSession), vi.spyOn(onboardSession, "updateSession").mockImplementation(updateSession), + vi.spyOn(onboardSession, "acquireOnboardLock").mockReturnValue({ acquired: true }), vi.spyOn(onboardSession, "releaseOnboardLock").mockImplementation(() => undefined), vi.spyOn(onboardSession, "markStepFailed").mockImplementation(() => loadSession()), vi.spyOn(registry, "getSandbox").mockReturnValue({ @@ -110,7 +125,12 @@ describe("rebuild resume snapshot repair", () => { policies: [], agent: null, nimContainer: null, + nemoclawVersion: "0.1.0", + dashboardPort: 18789, + gatewayName: "nemoclaw", + gatewayPort: 8080, } as never), + vi.spyOn(registry, "updateSandbox").mockReturnValue(true), vi.spyOn(registry, "listSandboxes").mockReturnValue({ sandboxes: [] } as never), vi.spyOn(sandboxSession, "getActiveSandboxSessions").mockReturnValue({ detected: false, @@ -141,14 +161,24 @@ describe("rebuild resume snapshot repair", () => { vi.spyOn(destroy, "removeSandboxRegistryEntry").mockImplementation(() => undefined), vi.spyOn(nim, "stopNimContainer").mockImplementation(() => undefined), vi.spyOn(nim, "stopNimContainerByName").mockImplementation(() => undefined), + vi.spyOn(nim, "detectGpu").mockReturnValue(null), + vi.spyOn(onboardMod, "preflightAuthoritativeRebuildTarget").mockResolvedValue(undefined), + vi.spyOn(rebuildImagePreflight, "preflightRebuildImage").mockResolvedValue({ + ok: true, + imageTag: null, + }), + vi.spyOn(rebuildUsageNotice, "ensureRebuildUsageNoticeAccepted").mockResolvedValue(true), vi.spyOn(onboardMod, "onboard").mockImplementation(async (options: unknown) => { observed.handoffOptions = options as Record; const reopened = onboardSession.loadSession(); observed.preRepairMachineState = reopened.machine.state; + observed.preRepairPreflightStatus = reopened.steps.preflight.status; + observed.preRepairGatewayStatus = reopened.steps.gateway.status; observed.preRepairStatus = reopened.status; observed.preRepairResumable = reopened.resumable; resumeRepair.repairResumeMachineSnapshot(reopened, "2026-06-01T00:01:00.000Z"); observed.repairedMachineState = reopened.machine.state; + observed.sandboxEnvInsideOnboard = process.env.NEMOCLAW_SANDBOX_NAME ?? null; throw new Error("stop-after-resume-repair-probe"); }), ); @@ -168,7 +198,7 @@ describe("rebuild resume snapshot repair", () => { delete require.cache[requireDist.resolve(rebuildModulePath)]; }); - it("reopens complete sessions so onboard resume repair can restore the resumable state", async () => { + it("replaces complete history with a target-scoped resume snapshot", async () => { await expect(rebuildSandbox("alpha", ["--yes"], { throwOnError: true })).rejects.toThrow( "Recreate failed", ); @@ -177,12 +207,21 @@ describe("rebuild resume snapshot repair", () => { resume: true, nonInteractive: true, recreateSandbox: true, + authoritativeResumeConfig: true, + acceptThirdPartySoftware: true, + controlUiPort: 18789, + targetGatewayName: "nemoclaw", + targetGatewayPort: 8080, + onboardLockAlreadyHeld: true, autoYes: true, }); - expect(observed.preRepairMachineState).toBe("complete"); + expect(observed.preRepairMachineState).toBe("init"); + expect(observed.preRepairPreflightStatus).toBe("complete"); + expect(observed.preRepairGatewayStatus).toBe("complete"); expect(observed.preRepairStatus).toBe("in_progress"); expect(observed.preRepairResumable).toBe(true); - expect(observed.repairedMachineState).toBe("provider_selection"); - expect(process.env.NEMOCLAW_SANDBOX_NAME).toBe("alpha"); - }); + expect(observed.repairedMachineState).toBe("init"); + expect(observed.sandboxEnvInsideOnboard).toBe("alpha"); + expect(process.env.NEMOCLAW_SANDBOX_NAME).toBe(originalSandboxName); + }, 15_000); }); diff --git a/src/lib/actions/sandbox/rebuild-shields-finally.test.ts b/src/lib/actions/sandbox/rebuild-shields-finally.test.ts index c6f6908dbfe..42a841003af 100644 --- a/src/lib/actions/sandbox/rebuild-shields-finally.test.ts +++ b/src/lib/actions/sandbox/rebuild-shields-finally.test.ts @@ -32,12 +32,16 @@ describe("rebuild shields relock guard", () => { const sandboxList = requireDist("../../openshell-sandbox-list.js"); const resolve = requireDist("../../adapters/openshell/resolve.js"); const agentRuntime = requireDist("../../agent/runtime.js"); + const onboardMod = requireDist("../../onboard.js"); const onboardSession = requireDist("../../state/onboard-session.js"); const registry = requireDist("../../state/registry.js"); const sandboxState = requireDist("../../state/sandbox.js"); const sandboxSession = requireDist("../../state/sandbox-session.js"); const sandboxVersion = requireDist("../../sandbox/version.js"); const rebuildShields = requireDist("./rebuild-shields.js"); + const rebuildImagePreflight = requireDist("./rebuild-custom-image-preflight.js"); + const rebuildUsageNotice = requireDist("./rebuild-usage-notice.js"); + const nim = requireDist("../../inference/nim.js"); relockSpy = vi .spyOn(rebuildShields, "relockRebuildShieldsWindow") @@ -52,9 +56,11 @@ describe("rebuild shields relock guard", () => { spies.push( vi.spyOn(gatewayDrift, "detectOpenShellStateRpcPreflightIssue").mockReturnValue(null), vi.spyOn(gatewayDrift, "detectOpenShellStateRpcResultIssue").mockReturnValue(null), - vi - .spyOn(gatewayRuntime, "recoverNamedGatewayRuntime") - .mockResolvedValue({ recovered: false }), + vi.spyOn(gatewayRuntime, "recoverNamedGatewayRuntime").mockResolvedValue({ + recovered: true, + before: { state: "connected_other" }, + after: { state: "healthy_named" }, + }), sandboxListRecoverySpy.mockResolvedValue({ result: { status: 0, output: "alpha Ready" }, }), @@ -62,6 +68,8 @@ describe("rebuild shields relock guard", () => { vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue(null), vi.spyOn(agentRuntime, "getAgentDisplayName").mockReturnValue("OpenClaw"), vi.spyOn(onboardSession, "loadSession").mockReturnValue(null), + vi.spyOn(onboardSession, "acquireOnboardLock").mockReturnValue({ acquired: true }), + vi.spyOn(onboardSession, "releaseOnboardLock").mockImplementation(() => undefined), vi.spyOn(registry, "getSandbox").mockReturnValue({ name: "alpha", provider: "ollama-local", @@ -69,9 +77,12 @@ describe("rebuild shields relock guard", () => { policies: [], agent: null, nimContainer: null, + nemoclawVersion: "0.1.0", gatewayName: "nemoclaw-8090", gatewayPort: 8090, + dashboardPort: 18789, } as never), + vi.spyOn(registry, "updateSandbox").mockReturnValue(true), vi.spyOn(sandboxSession, "getActiveSandboxSessions").mockReturnValue({ detected: false, sessions: [], @@ -80,6 +91,13 @@ describe("rebuild shields relock guard", () => { expectedVersion: "0.1.0", sandboxVersion: "0.0.1", } as never), + vi.spyOn(nim, "detectGpu").mockReturnValue(null), + vi.spyOn(onboardMod, "preflightAuthoritativeRebuildTarget").mockResolvedValue(undefined), + vi.spyOn(rebuildImagePreflight, "preflightRebuildImage").mockResolvedValue({ + ok: true, + imageTag: null, + }), + vi.spyOn(rebuildUsageNotice, "ensureRebuildUsageNoticeAccepted").mockResolvedValue(true), vi.spyOn(rebuildShields, "openRebuildShieldsWindow").mockReturnValue(rebuildWindow), relockSpy, vi.spyOn(sandboxState, "backupSandboxState").mockImplementation(() => { @@ -105,5 +123,5 @@ describe("rebuild shields relock guard", () => { expect(relockSpy).toHaveBeenCalledWith("alpha", rebuildWindow, true, expect.any(String)); expect(sandboxListRecoverySpy).toHaveBeenCalledWith({ gatewayName: "nemoclaw-8090" }); expect(rebuildWindow.relocked).toBe(true); - }); + }, 15_000); }); diff --git a/src/lib/actions/sandbox/rebuild-shields-phase.ts b/src/lib/actions/sandbox/rebuild-shields-phase.ts new file mode 100644 index 00000000000..c96c54c07c0 --- /dev/null +++ b/src/lib/actions/sandbox/rebuild-shields-phase.ts @@ -0,0 +1,49 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { CLI_NAME } from "../../cli/branding"; +import type { RebuildBail } from "./rebuild-credential-preflight"; +import { openRebuildShieldsWindowForState } from "./rebuild-flow-helpers"; +import { type RebuildShieldsWindow, relockRebuildShieldsWindow } from "./rebuild-shields"; + +export interface RebuildShieldsPhaseResult { + window: RebuildShieldsWindow; + staleSandboxWasLocked: boolean; + relock: (sandboxStillExists: boolean) => boolean; +} + +/** + * Open the mutable rebuild window while preserving fail-safe lock cleanup. + * Boundary coverage: rebuild-shields-finally.test.ts and rebuild-flow.test.ts. + */ +export function runRebuildShieldsPhase( + sandboxName: string, + recoveryRecreate: boolean, + releaseOnboardLock: () => void, + bail: RebuildBail, +): RebuildShieldsPhaseResult | null { + let window: RebuildShieldsWindow | null; + let staleSandboxWasLocked: boolean; + try { + ({ rebuildShieldsWindow: window, staleSandboxWasLocked } = openRebuildShieldsWindowForState( + sandboxName, + recoveryRecreate, + )); + } catch (error) { + process.removeListener("exit", releaseOnboardLock); + releaseOnboardLock(); + throw error; + } + if (!window) { + process.removeListener("exit", releaseOnboardLock); + releaseOnboardLock(); + bail("Failed to auto-unlock shields."); + return null; + } + return { + window, + staleSandboxWasLocked, + relock: (sandboxStillExists: boolean) => + relockRebuildShieldsWindow(sandboxName, window, sandboxStillExists, CLI_NAME), + }; +} diff --git a/src/lib/actions/sandbox/rebuild-target-config.ts b/src/lib/actions/sandbox/rebuild-target-config.ts new file mode 100644 index 00000000000..716008ec75a --- /dev/null +++ b/src/lib/actions/sandbox/rebuild-target-config.ts @@ -0,0 +1,166 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { loadAgent } from "../../agent/defs"; +import { webSearchProviderForConfig } from "../../inference/web-search"; +import type { Session } from "../../state/onboard-session"; +import * as onboardSession from "../../state/onboard-session"; +import type { RebuildBail } from "./rebuild-credential-preflight"; +import { isDcodeRebuildAgent } from "./rebuild-dcode-orchestrator"; +import { + type RebuildDurableConfig, + resolveRebuildDockerfile, + resolveRebuildDurableConfig, +} from "./rebuild-durable-config"; +import type { RebuildSandboxEntry } from "./rebuild-flow-helpers"; +import { printRebuildPreflightFailure } from "./rebuild-preflight-error"; +import { prepareRebuildResumeConfig, type RebuildResumeConfig } from "./rebuild-resume-config"; + +const hermesProviderAuth = require("../../hermes-provider-auth") as { + HERMES_PROVIDER_NAME: string; + HERMES_INFERENCE_CREDENTIAL_ENV: string; + HERMES_NOUS_API_KEY_CREDENTIAL_ENV: string; +}; + +export type RebuildTargetConfig = { + resumeConfig: RebuildResumeConfig; + sessionSnapshot: Session | null; + sessionMatchesSandbox: boolean; + durableConfig: RebuildDurableConfig; + hermesToolGateways: string[]; + hasHermesToolGateways: boolean; + credentialEnv: string | null; + fromDockerfile: string | null; + agentDefinition: ReturnType | null; +}; + +function stringListOrNull(value: unknown): string[] | null { + if (!Array.isArray(value)) return null; + return value.filter((item: unknown): item is string => typeof item === "string"); +} + +function resolveRebuildHermesToolGateways( + rebuildAgent: string | null, + sb: RebuildSandboxEntry, + session: Session | null, + sessionMatchesSandbox: boolean, +): { gateways: string[]; recorded: boolean } { + if (rebuildAgent !== "hermes") return { gateways: [], recorded: false }; + const registryGateways = stringListOrNull(sb.hermesToolGateways); + const sessionGateways = sessionMatchesSandbox + ? stringListOrNull(session?.hermesToolGateways) + : null; + return { + gateways: registryGateways ?? sessionGateways ?? [], + recorded: registryGateways !== null || sessionGateways !== null, + }; +} + +function validateRebuildDurableConfig( + durableConfig: RebuildDurableConfig, + resumeConfig: RebuildResumeConfig, + bail: RebuildBail, +): boolean { + if (durableConfig.webSearchError) { + printRebuildPreflightFailure( + "recorded web-search state is invalid.", + durableConfig.webSearchError, + "Recorded web-search state is invalid", + bail, + ); + return false; + } + if (durableConfig.fromDockerfileError) { + printRebuildPreflightFailure( + "recorded custom Dockerfile is invalid.", + durableConfig.fromDockerfileError, + "Recorded custom Dockerfile is invalid", + bail, + ); + return false; + } + if ( + durableConfig.hermesAuthMethodError || + (resumeConfig.provider === hermesProviderAuth.HERMES_PROVIDER_NAME && + durableConfig.hermesAuthMethod === null) + ) { + printRebuildPreflightFailure( + "Hermes auth state is incomplete.", + durableConfig.hermesAuthMethodError ?? + "cannot determine the recorded Hermes Provider authentication method", + "Cannot determine recorded Hermes Provider authentication method", + bail, + ); + return false; + } + return true; +} + +export function prepareRebuildTargetConfig( + sandboxName: string, + sb: RebuildSandboxEntry, + rebuildAgent: string | null, + log: (message: string) => void, + bail: RebuildBail, +): RebuildTargetConfig | null { + const resumeConfig = prepareRebuildResumeConfig(sandboxName, sb, rebuildAgent, log, bail); + if (!resumeConfig) return null; + const sessionSnapshot = onboardSession.loadSession(); + const sessionMatchesSandbox = sessionSnapshot?.sandboxName === sandboxName; + const durableConfig = resolveRebuildDurableConfig(sandboxName, sb, sessionSnapshot, { + provider: resumeConfig.provider, + model: resumeConfig.model, + }); + if (!validateRebuildDurableConfig(durableConfig, resumeConfig, bail)) return null; + if (isDcodeRebuildAgent(rebuildAgent) && durableConfig.fromDockerfile) { + printRebuildPreflightFailure( + "the managed DCode registry entry conflicts with a recorded custom Dockerfile.", + "Managed DCode rebuilds must use the verified managed image path.", + "Managed DCode rebuild cannot use a recorded custom Dockerfile", + bail, + ); + return null; + } + + const dockerfile = resolveRebuildDockerfile(durableConfig.fromDockerfile); + if (!dockerfile.ok) { + printRebuildPreflightFailure( + "recorded custom Dockerfile is unavailable.", + `${dockerfile.path}: ${dockerfile.reason}`, + "Recorded custom Dockerfile is unavailable", + bail, + ); + return null; + } + + const hermesGateways = resolveRebuildHermesToolGateways( + rebuildAgent, + sb, + sessionSnapshot, + sessionMatchesSandbox, + ); + const hermesToolGateways = + rebuildAgent === "hermes" && + durableConfig.webSearchConfig && + webSearchProviderForConfig(durableConfig.webSearchConfig) === "tavily" + ? hermesGateways.gateways.filter((gateway) => gateway !== "nous-web") + : hermesGateways.gateways; + const credentialEnv = + resumeConfig.provider === hermesProviderAuth.HERMES_PROVIDER_NAME + ? durableConfig.hermesAuthMethod === "api_key" + ? hermesProviderAuth.HERMES_NOUS_API_KEY_CREDENTIAL_ENV + : hermesProviderAuth.HERMES_INFERENCE_CREDENTIAL_ENV + : resumeConfig.credentialEnv; + + return { + resumeConfig, + sessionSnapshot, + sessionMatchesSandbox, + durableConfig, + hermesToolGateways, + hasHermesToolGateways: hermesGateways.recorded, + credentialEnv, + fromDockerfile: dockerfile.path, + agentDefinition: rebuildAgent && rebuildAgent !== "openclaw" ? loadAgent(rebuildAgent) : null, + }; +} diff --git a/src/lib/actions/sandbox/rebuild-target-preflight.ts b/src/lib/actions/sandbox/rebuild-target-preflight.ts new file mode 100644 index 00000000000..f46289ef311 --- /dev/null +++ b/src/lib/actions/sandbox/rebuild-target-preflight.ts @@ -0,0 +1,22 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Compatibility facade for rebuild target preflight. The implementation is + * separated by concern so config resolution, runtime validation, and mutable + * staging remain independently reviewable. + */ +export { printRebuildPreflightFailure } from "./rebuild-preflight-error"; +export { + prepareRebuildTargetConfig, + type RebuildTargetConfig, +} from "./rebuild-target-config"; +export { + preflightAuthoritativeOnboardRuntime, + preflightRebuildTargetRuntime, +} from "./rebuild-target-runtime"; +export { + hydrateMessagingConfigForRebuild, + prepareRebuildRecreateOptions, + stageRebuildHermesDashboardConfig, +} from "./rebuild-target-staging"; diff --git a/src/lib/actions/sandbox/rebuild-target-runtime.ts b/src/lib/actions/sandbox/rebuild-target-runtime.ts new file mode 100644 index 00000000000..3e6e32cc7fc --- /dev/null +++ b/src/lib/actions/sandbox/rebuild-target-runtime.ts @@ -0,0 +1,212 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import * as nim from "../../inference/nim"; +import { + webSearchEnvFor, + webSearchLabelFor, + webSearchProviderForConfig, +} from "../../inference/web-search"; +import { shouldManageDashboardForAgent } from "../../onboard/dashboard-runtime"; +import { isLinuxDockerDriverGatewayEnabled } from "../../onboard/docker-driver-platform"; +import { enforceDockerGpuPatchPreserveNetwork } from "../../onboard/docker-gpu-local-inference"; +import { resolveSandboxGpuConfig } from "../../onboard/sandbox-gpu-mode"; +import { agentSupportsWebSearchProvider } from "../../onboard/web-search-support"; +import { redact } from "../../security/redact"; +import { + preflightRebuildCredentials, + type RebuildBail, + type RebuildLog, +} from "./rebuild-credential-preflight"; +import * as rebuildImagePreflight from "./rebuild-custom-image-preflight"; +import type { RebuildDurableConfig } from "./rebuild-durable-config"; +import type { RebuildSandboxEntry } from "./rebuild-flow-helpers"; +import type { RebuildRecreateOnboardOpts } from "./rebuild-gpu-opt-out"; +import { printRebuildPreflightFailure } from "./rebuild-preflight-error"; +import type { RebuildResumeConfig } from "./rebuild-resume-config"; +import type { RebuildTargetConfig } from "./rebuild-target-config"; + +const onboardModule = require("../../onboard") as { + ensureValidatedWebSearchCredential: ( + config: NonNullable, + nonInteractive?: boolean, + ) => Promise; + preflightAuthoritativeRebuildTarget: ( + options: RebuildRecreateOnboardOpts & { + model: string; + provider: string; + sandboxName: string; + }, + ) => Promise; +}; + +async function preflightRebuildWebSearchCredential( + durableConfig: RebuildDurableConfig, + bail: RebuildBail, +): Promise { + const config = durableConfig.webSearchConfig; + if (!config) return true; + const provider = webSearchProviderForConfig(config); + const label = webSearchLabelFor(provider); + try { + const credential = await onboardModule.ensureValidatedWebSearchCredential(config, true); + if (typeof credential !== "string" || !credential.trim()) { + throw new Error(`${label} credential validation did not return a usable key.`); + } + return true; + } catch (err) { + printRebuildPreflightFailure( + `${label} credential is invalid.`, + err instanceof Error ? err.message : String(err), + `${label} credential preflight failed`, + bail, + ); + return false; + } +} + +export async function preflightRebuildTargetRuntime( + target: RebuildTargetConfig, + sb: RebuildSandboxEntry, + recreateOptions: RebuildRecreateOnboardOpts, + log: RebuildLog, + bail: RebuildBail, + options: { skipImagePreflight?: boolean } = {}, +): Promise { + const webSearchConfig = target.durableConfig.webSearchConfig; + const webSearchProvider = webSearchConfig ? webSearchProviderForConfig(webSearchConfig) : null; + if ( + webSearchProvider && + !agentSupportsWebSearchProvider( + target.agentDefinition, + webSearchProvider, + target.fromDockerfile, + ) + ) { + const label = webSearchLabelFor(webSearchProvider); + printRebuildPreflightFailure( + `the recorded agent/image does not support ${label}.`, + "Recreate with a supported image before enabling recorded web-search state.", + `Recorded ${label} is unsupported by the rebuild image`, + bail, + ); + return false; + } + if (webSearchProvider) { + const credentialEnv = webSearchEnvFor(webSearchProvider); + const collidingBridge = Object.values(sb.mcp?.bridges ?? {}).find((entry) => + entry.env.includes(credentialEnv), + ); + if (collidingBridge) { + printRebuildPreflightFailure( + `the recorded ${webSearchLabelFor(webSearchProvider)} credential is also owned by MCP server '${collidingBridge.server}'.`, + `Use a distinct credential name; ${credentialEnv} cannot be shared across managed providers.`, + "Web Search and MCP credential ownership conflict", + bail, + ); + return false; + } + } + + const managesDashboard = shouldManageDashboardForAgent(target.agentDefinition); + const gpuEnv = { ...process.env }; + delete gpuEnv.NEMOCLAW_SANDBOX_GPU; + delete gpuEnv.NEMOCLAW_SANDBOX_GPU_DEVICE; + const sandboxGpuConfig = resolveSandboxGpuConfig(nim.detectGpu(), { + flag: recreateOptions.sandboxGpu, + device: recreateOptions.sandboxGpuDevice, + env: gpuEnv, + }); + if (sandboxGpuConfig.errors.length > 0) { + printRebuildPreflightFailure( + "the recorded sandbox GPU state cannot be recreated.", + sandboxGpuConfig.errors.join(" "), + "Recorded sandbox GPU state is invalid", + bail, + ); + return false; + } + try { + await enforceDockerGpuPatchPreserveNetwork(target.resumeConfig.provider, sandboxGpuConfig, { + dockerDriverGateway: isLinuxDockerDriverGatewayEnabled(), + gatewayPort: recreateOptions.targetGatewayPort, + log, + }); + } catch (err) { + printRebuildPreflightFailure( + "the recorded GPU network path is not reachable.", + err instanceof Error ? err.message : String(err), + "Sandbox GPU network preflight failed", + bail, + ); + return false; + } + + if (!options.skipImagePreflight) { + const customImage = await rebuildImagePreflight.preflightRebuildImage({ + agent: target.agentDefinition, + fromDockerfile: target.fromDockerfile, + model: target.resumeConfig.model, + provider: target.resumeConfig.provider, + preferredInferenceApi: target.resumeConfig.preferredInferenceApi, + compatibleEndpointReasoning: target.resumeConfig.compatibleEndpointReasoning, + webSearchConfig: target.durableConfig.webSearchConfig, + hermesToolGateways: target.hermesToolGateways, + sandboxGpuConfig, + gatewayPort: recreateOptions.targetGatewayPort, + chatUiUrl: managesDashboard + ? `http://127.0.0.1:${String(recreateOptions.controlUiPort)}` + : "", + }); + if (!customImage.ok) { + printRebuildPreflightFailure( + "the replacement sandbox image did not build.", + redact(customImage.detail), + "Replacement sandbox image preflight failed", + bail, + ); + return false; + } + } + if (!(await preflightRebuildWebSearchCredential(target.durableConfig, bail))) return false; + + // Credential preflight must use the same trusted selection. Legacy registry + // rows may recover provider/model from their own matching onboard session; + // checking the raw row first would miss that remote credential requirement. + return preflightRebuildCredentials( + { + ...sb, + provider: target.resumeConfig.provider, + model: target.resumeConfig.model, + credentialEnv: target.credentialEnv, + hermesAuthMethod: target.durableConfig.hermesAuthMethod, + }, + log, + bail, + ); +} + +export async function preflightAuthoritativeOnboardRuntime( + sandboxName: string, + resumeConfig: RebuildResumeConfig, + recreateOptions: RebuildRecreateOnboardOpts, + bail: RebuildBail, +): Promise { + try { + await onboardModule.preflightAuthoritativeRebuildTarget({ + ...recreateOptions, + model: resumeConfig.model, + provider: resumeConfig.provider, + sandboxName, + }); + return true; + } catch (err) { + printRebuildPreflightFailure( + "the replacement onboarding host/runtime checks did not pass.", + err instanceof Error ? err.message : String(err), + "Replacement onboarding preflight failed", + bail, + ); + return false; + } +} diff --git a/src/lib/actions/sandbox/rebuild-target-staging.ts b/src/lib/actions/sandbox/rebuild-target-staging.ts new file mode 100644 index 00000000000..4bea2ffd555 --- /dev/null +++ b/src/lib/actions/sandbox/rebuild-target-staging.ts @@ -0,0 +1,80 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { hydrateMessagingChannelConfig } from "../../messaging-channel-config"; +import { getStoredMessagingChannelConfig } from "../../onboard/messaging-config"; +import * as onboardSession from "../../state/onboard-session"; +import type { RebuildBail } from "./rebuild-credential-preflight"; +import { + REBUILD_HERMES_DASHBOARD_ENV_KEYS, + resolveRebuildHermesDashboardEnv, +} from "./rebuild-durable-config"; +import type { RebuildSandboxEntry } from "./rebuild-flow-helpers"; +import { + buildRebuildRecreateOnboardOpts, + type RebuildRecreateOnboardOpts, +} from "./rebuild-gpu-opt-out"; +import { printRebuildPreflightFailure } from "./rebuild-preflight-error"; + +export function prepareRebuildRecreateOptions( + sb: RebuildSandboxEntry, + rebuildAgent: string | null, + storedFromDockerfile: string | null, + autoYes: boolean, + bail: RebuildBail, +): RebuildRecreateOnboardOpts | null { + try { + return buildRebuildRecreateOnboardOpts({ + sb, + rebuildAgent, + storedFromDockerfile, + autoYes, + usageNoticeAccepted: true, + }); + } catch (err) { + printRebuildPreflightFailure( + "the recorded recreate target is invalid.", + err instanceof Error ? err.message : String(err), + "Recorded recreate target is invalid", + bail, + ); + return null; + } +} + +export function stageRebuildHermesDashboardConfig( + rebuildAgent: string | null, + sb: RebuildSandboxEntry, + controlUiPort: number | null, + bail: RebuildBail, +): boolean { + const resolved = resolveRebuildHermesDashboardEnv(rebuildAgent, sb, controlUiPort); + if (!resolved.ok) { + printRebuildPreflightFailure( + "the recorded Hermes dashboard state is invalid.", + resolved.reason, + "Recorded Hermes dashboard state is invalid", + bail, + ); + return false; + } + for (const key of REBUILD_HERMES_DASHBOARD_ENV_KEYS) { + const value = resolved.env[key]; + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + return true; +} + +export function hydrateMessagingConfigForRebuild( + sandboxName: string, + log: (msg: string) => void, +): void { + const rebuildSession = onboardSession.loadSession(); + const hydratedMessagingConfig = hydrateMessagingChannelConfig( + getStoredMessagingChannelConfig(sandboxName, rebuildSession), + ); + if (hydratedMessagingConfig) { + log(`Stashed messaging config for rebuild: ${Object.keys(hydratedMessagingConfig).join(",")}`); + } +} diff --git a/src/lib/actions/sandbox/rebuild-usage-notice.test.ts b/src/lib/actions/sandbox/rebuild-usage-notice.test.ts new file mode 100644 index 00000000000..77d6f7d039b --- /dev/null +++ b/src/lib/actions/sandbox/rebuild-usage-notice.test.ts @@ -0,0 +1,42 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { NOTICE_ACCEPT_ENV } from "../../onboard/usage-notice"; +import { ensureRebuildUsageNoticeAccepted } from "./rebuild-usage-notice"; + +describe("ensureRebuildUsageNoticeAccepted", () => { + it("does not treat rebuild confirmation as notice acceptance", async () => { + const ensureConsent = vi.fn().mockResolvedValue(false); + + await expect( + ensureRebuildUsageNoticeAccepted({ ensureConsent, env: {}, stdinIsTty: false }), + ).resolves.toBe(false); + expect(ensureConsent).toHaveBeenCalledWith( + expect.objectContaining({ nonInteractive: true, acceptedByFlag: false }), + ); + }); + + it("honors only the dedicated non-interactive acceptance env", async () => { + const ensureConsent = vi.fn().mockResolvedValue(true); + + await ensureRebuildUsageNoticeAccepted({ + ensureConsent, + env: { [NOTICE_ACCEPT_ENV]: "1" }, + stdinIsTty: false, + }); + expect(ensureConsent).toHaveBeenCalledWith( + expect.objectContaining({ nonInteractive: true, acceptedByFlag: true }), + ); + }); + + it("keeps an attached terminal interactive unless explicitly configured otherwise", async () => { + const ensureConsent = vi.fn().mockResolvedValue(true); + + await ensureRebuildUsageNoticeAccepted({ ensureConsent, env: {}, stdinIsTty: true }); + expect(ensureConsent).toHaveBeenCalledWith( + expect.objectContaining({ nonInteractive: false, acceptedByFlag: false }), + ); + }); +}); diff --git a/src/lib/actions/sandbox/rebuild-usage-notice.ts b/src/lib/actions/sandbox/rebuild-usage-notice.ts new file mode 100644 index 00000000000..58990100f19 --- /dev/null +++ b/src/lib/actions/sandbox/rebuild-usage-notice.ts @@ -0,0 +1,31 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { prompt } from "../../credentials/store"; +import { ensureUsageNoticeConsent, NOTICE_ACCEPT_ENV } from "../../onboard/usage-notice"; + +type EnsureConsent = typeof ensureUsageNoticeConsent; + +export type RebuildUsageNoticeDeps = { + ensureConsent?: EnsureConsent; + env?: NodeJS.ProcessEnv; + stdinIsTty?: boolean; +}; + +/** + * Resolve the current notice before rebuild enters its destructive window. + * Destructive `--yes` is deliberately not legal-notice consent: unattended + * callers must have the current saved acceptance or set the dedicated env. + */ +export async function ensureRebuildUsageNoticeAccepted( + deps: RebuildUsageNoticeDeps = {}, +): Promise { + const env = deps.env ?? process.env; + const stdinIsTty = deps.stdinIsTty ?? process.stdin?.isTTY === true; + return (deps.ensureConsent ?? ensureUsageNoticeConsent)({ + nonInteractive: env.NEMOCLAW_NON_INTERACTIVE === "1" || !stdinIsTty, + acceptedByFlag: String(env[NOTICE_ACCEPT_ENV] || "") === "1", + promptFn: prompt, + writeLine: console.error, + }); +} diff --git a/src/lib/actions/sandbox/rebuild.ts b/src/lib/actions/sandbox/rebuild.ts index 3030dfd587d..d41de1a6be9 100644 --- a/src/lib/actions/sandbox/rebuild.ts +++ b/src/lib/actions/sandbox/rebuild.ts @@ -1,1460 +1,9 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { isDeepStrictEqual } from "node:util"; - -import { CLI_NAME } from "../../cli/branding"; -import { prompt as askPrompt } from "../../credentials/store"; -import { - normalizeRebuildSandboxOptions, - type RebuildSandboxOptions, -} from "../../domain/lifecycle/options"; - -const { hydrateCredentialEnv } = require("../../onboard") as { - hydrateCredentialEnv: (name: string) => string | null; -}; -const hermesProviderAuth = require("../../hermes-provider-auth") as { - HERMES_PROVIDER_NAME: string; - HERMES_NOUS_API_KEY_CREDENTIAL_ENV: string; - isHermesProviderRegistered: (runOpenshellFn: typeof runOpenshell) => boolean; - registerHermesInferenceProvider: ( - apiKey: string, - runOpenshellFn: typeof runOpenshell, - credentialEnv?: string, - baseUrl?: string, - ) => void; -}; - -import { - detectOpenShellStateRpcPreflightIssue, - printOpenShellStateRpcIssue, -} from "../../adapters/openshell/gateway-drift"; -import { resolveOpenshell } from "../../adapters/openshell/resolve"; -import { runOpenshell } from "../../adapters/openshell/runtime"; -import { loadAgent } from "../../agent/defs"; -import * as agentRuntime from "../../agent/runtime"; -import { RD as _RD, B, D, G, R, YW } from "../../cli/terminal-style"; -import { getSandboxDeleteOutcome } from "../../domain/sandbox/destroy"; -import * as nim from "../../inference/nim"; -import type { - MessagingHookApplyRequest, - MessagingHookOutputMap, - MessagingOpenShellRunner, - SandboxMessagingPlan, -} from "../../messaging"; -import { - createBuiltInChannelManifestRegistry, - createBuiltInRenderTemplateResolver, - isMessagingSupportedAgent, - listSupportedMessagingChannelIdsForAgent, - MessagingSetupApplier, - MessagingWorkflowPlanner, - tryGetMessagingAgentId, -} from "../../messaging"; -import { hydrateMessagingChannelConfig } from "../../messaging-channel-config"; -import { markLastStartedStepFailed } from "../../onboard/exit-step-failure"; -import { getStoredMessagingChannelConfig } from "../../onboard/messaging-config"; -import { mergeRebuildMessagingPolicyPresets } from "../../onboard/messaging-policy-presets"; -import * as policies from "../../policy"; -import { shellQuote } from "../../runner"; -import * as sandboxVersion from "../../sandbox/version"; -import { redact } from "../../security/redact"; -import * as shields from "../../shields"; -import type { Session } from "../../state/onboard-session"; -import * as onboardSession from "../../state/onboard-session"; -import * as registry from "../../state/registry"; -import * as sandboxState from "../../state/sandbox"; -import { - createSystemDeps as createSessionDeps, - getActiveSandboxSessions, -} from "../../state/sandbox-session"; -import { removeSandboxRegistryEntry } from "./destroy"; -import { getSandboxTargetGatewayName } from "./gateway-target"; -import { ensureMessagingHostForwardAfterRebuild } from "./messaging-host-forward-lifecycle"; -import { executeSandboxCommand } from "./process-recovery"; -import { createDcodeRebuildOrchestrator, isDcodeRebuildAgent } from "./rebuild-dcode-orchestrator"; -import { isolateAmbientRecreateEnv } from "./rebuild-env-isolation"; -import { - backupSandboxStateForRebuild, - ensureRebuildAgentBaseImage, - openRebuildShieldsWindowForState, - type RebuildSandboxEntry, - resolveRebuildLiveState, -} from "./rebuild-flow-helpers"; -import { buildRebuildRecreateOnboardOpts } from "./rebuild-gpu-opt-out"; -import { preflightRebuildMessagingConflicts } from "./rebuild-messaging-conflict-preflight"; -import { - checkRebuildGatewayProviderOrBail, - shouldVerifyRebuildGatewayProvider, -} from "./rebuild-provider-preflight"; -import { - getRebuildCredentialEnvFromRegistry, - isLocalInferenceProvider, - prepareRebuildResumeConfig, -} from "./rebuild-resume-config"; -import { printRebuildShieldsRecovery, relockRebuildShieldsWindow } from "./rebuild-shields"; - -export function buildRefreshMutableOpenClawConfigHashCommand( - configDir = "/sandbox/.openclaw", -): string { - return [ - `config_dir=${shellQuote(configDir)}`, - 'config_file="${config_dir}/openclaw.json"', - 'hash_file="${config_dir}/.config-hash"', - '[ -d "$config_dir" ] || exit 0', - '[ ! -L "$config_dir" ] || { echo "refusing symlinked OpenClaw config dir: $config_dir" >&2; exit 10; }', - '[ ! -L "$config_file" ] || { echo "refusing symlinked OpenClaw config file: $config_file" >&2; exit 11; }', - '[ ! -L "$hash_file" ] || { echo "refusing symlinked OpenClaw config hash: $hash_file" >&2; exit 12; }', - 'owner="$(stat -c "%U" "$config_dir" 2>/dev/null || echo unknown)"', - '[ "$owner" != "root" ] || exit 0', - '[ -f "$config_file" ] || exit 0', - 'cd "$config_dir" || exit 13', - "sha256sum openclaw.json > .config-hash", - "chmod 660 .config-hash 2>/dev/null || true", - ].join("; "); -} - -function refreshMutableOpenClawConfigHashAfterPostRestoreWrites( - sandboxName: string, - log: (msg: string) => void, -): boolean { - const result = executeSandboxCommand(sandboxName, buildRefreshMutableOpenClawConfigHashCommand()); - if (result && result.status === 0) { - log("Mutable OpenClaw config hash refreshed after post-restore config writes"); - return true; - } - - const detail = result - ? [result.stderr, result.stdout].filter(Boolean).join("; ") || `exit ${result.status}` - : "could not obtain sandbox SSH config"; - console.error(` ${YW}⚠${R} Mutable OpenClaw config hash was not refreshed: ${redact(detail)}`); - return false; -} - -/** - * Emit timestamped rebuild diagnostics when verbose rebuild logging is enabled. - */ -function _rebuildLog(msg: string) { - console.error(` ${D}[rebuild ${new Date().toISOString()}] ${redact(msg)}${R}`); -} - -function normalizeHermesRebuildAuthMethod(value: unknown): "oauth" | "api_key" | null { - const normalized = String(value || "") - .trim() - .toLowerCase() - .replace(/[\s-]+/g, "_"); - if (!normalized) return null; - if (normalized === "oauth" || normalized === "nous_oauth" || normalized === "nous_portal_oauth") { - return "oauth"; - } - if ( - normalized === "api" || - normalized === "key" || - normalized === "api_key" || - normalized === "apikey" || - normalized === "nous_api_key" - ) { - return "api_key"; - } - return null; -} - -function nonEmptyString(value: unknown): string | null { - const normalized = String(value || "").trim(); - return normalized || null; -} - -function preflightHermesProviderCredentials( - session: Session | null, - credentialEnv: string | null, - log: (msg: string) => void, -): boolean { - const authMethod = - normalizeHermesRebuildAuthMethod(session?.hermesAuthMethod) || - (credentialEnv === hermesProviderAuth.HERMES_NOUS_API_KEY_CREDENTIAL_ENV ? "api_key" : null); - - if (hermesProviderAuth.isHermesProviderRegistered(runOpenshell)) { - log("Hermes Provider rebuild preflight: provider is registered in OpenShell"); - return true; - } - - if (authMethod === "api_key") { - const envKey = - nonEmptyString(process.env[hermesProviderAuth.HERMES_NOUS_API_KEY_CREDENTIAL_ENV]) || - nonEmptyString(process.env.NEMOCLAW_PROVIDER_KEY); - log( - `Hermes Provider rebuild preflight: OpenShell provider missing; API key env=${envKey ? "present" : "missing"}`, - ); - if (envKey) { - try { - hermesProviderAuth.registerHermesInferenceProvider( - envKey, - runOpenshell, - hermesProviderAuth.HERMES_NOUS_API_KEY_CREDENTIAL_ENV, - ); - return true; - } catch (err) { - log( - `Hermes Provider rebuild preflight: failed to register OpenShell provider: ${err instanceof Error ? err.message : String(err)}`, - ); - } - } - } - - console.error(""); - console.error( - ` ${_RD}Rebuild preflight failed:${R} Hermes Provider is not registered in OpenShell.`, - ); - console.error(" Hermes Provider credentials must be stored in OpenShell, not host-side files."); - if (authMethod === "api_key") { - console.error( - ` Export the Hermes Provider API key and rerun rebuild, or re-run ${CLI_NAME} onboard to register it.`, - ); - } else { - console.error( - ` Re-run ${CLI_NAME} onboard interactively to authorize Hermes Provider and register it with OpenShell.`, - ); - } - console.error(""); - console.error(" Sandbox is untouched — no data was lost."); - return false; -} - -export async function stageMessagingManifestPlanForRebuild( - sandboxName: string, - sandboxEntry: registry.SandboxEntry, - rebuildAgent: string | null, - log: (msg: string) => void, -): Promise { - const agent = loadAgent(rebuildAgent || "openclaw"); - const manifestRegistry = createBuiltInChannelManifestRegistry(); - const manifests = manifestRegistry.list(); - const agentId = tryGetMessagingAgentId(agent, manifests); - if (agentId === null) { - MessagingSetupApplier.clearPlanEnv(); - log( - `Messaging manifest rebuild plan skipped: agent '${agent.name}' is not supported by any channel manifest`, - ); - return null; - } - if (!isMessagingSupportedAgent(agent, manifests)) { - MessagingSetupApplier.clearPlanEnv(); - log( - `Messaging manifest rebuild plan skipped: agent '${agent.name}' has no supported messaging channels`, - ); - return null; - } - const supportedChannelIds = listSupportedMessagingChannelIdsForAgent(manifests, agentId); - const planner = new MessagingWorkflowPlanner( - manifestRegistry, - undefined, - createBuiltInRenderTemplateResolver(), - ); - const plan = await planner.buildRebuildPlanFromSandboxEntry({ - sandboxName, - agent: agentId, - sandboxEntry, - supportedChannelIds, - }); - if (!plan) { - MessagingSetupApplier.clearPlanEnv(); - log("Messaging manifest rebuild plan: no configured channels"); - return null; - } - MessagingSetupApplier.writePlanToEnv(plan); - if (plan.channels.length === 0) { - log("Messaging manifest rebuild plan staged: no configured channels"); - return plan; - } - log( - `Messaging manifest rebuild plan staged: ${plan.channels - .map((channel) => channel.channelId) - .join(",")}`, - ); - return plan; -} - -const runMessagingOpenshell: MessagingOpenShellRunner = (args, options = {}) => - runOpenshell([...args], { - env: options.env as NodeJS.ProcessEnv | undefined, - ignoreError: options.ignoreError, - input: options.input, - stdio: options.stdio as never, - }); - -function hookOutputsFromBuildSteps( - plan: SandboxMessagingPlan, - request: MessagingHookApplyRequest, -): { readonly outputs: MessagingHookOutputMap } { - const outputs: Record = {}; - for (const step of plan.buildSteps) { - if ( - step.channelId !== request.channelId || - step.hookId !== request.hookId || - step.value === undefined - ) { - continue; - } - outputs[step.outputId] = { - kind: step.kind, - value: step.value, - }; - } - return { outputs }; -} - -function countActiveSandboxSessionsForRebuild(sandboxName: string): number { - const opsBinRebuild = resolveOpenshell(); - // Source boundary: active-session detection depends on host process listing - // and the OpenShell binary being installed. A failed/unavailable detector is - // not evidence of active sessions, and rebuild's safety preflights still run - // before destructive work. Keep the prior fail-open prompt behavior here; - // remove this fallback only if session detection becomes a required, typed - // OpenShell API that can distinguish "zero sessions" from "unavailable". - if (!opsBinRebuild) return 0; - - try { - const sessionResult = getActiveSandboxSessions(sandboxName, createSessionDeps(opsBinRebuild)); - return sessionResult.detected ? sessionResult.sessions.length : 0; - } catch { - return 0; - } -} - -async function confirmSandboxRebuildIfNeeded( - skipConfirm: boolean, - rebuildActiveSessionCount: number, -): Promise { - if (skipConfirm) return true; - - if (rebuildActiveSessionCount > 0) { - const plural = rebuildActiveSessionCount > 1 ? "sessions" : "session"; - console.log( - ` ${YW}⚠ Active SSH ${plural} detected (${rebuildActiveSessionCount} connection${rebuildActiveSessionCount > 1 ? "s" : ""})${R}`, - ); - console.log( - ` Rebuilding will terminate ${rebuildActiveSessionCount === 1 ? "the" : "all"} active ${plural} with a Broken pipe error.`, - ); - console.log(""); - } - console.log(" This will:"); - console.log(" 1. Back up workspace state"); - console.log(" 2. Destroy and recreate the sandbox with the current image"); - console.log(" 3. Restore workspace state into the new sandbox"); - console.log(""); - const answer = await askPrompt(" Proceed? [y/N]: "); - if (answer.trim().toLowerCase() !== "y" && answer.trim().toLowerCase() !== "yes") { - console.log(" Cancelled."); - return false; - } - return true; -} - -function checkRebuildGatewaySchemaPreflight( - sandboxName: string, - bail: (msg: string, code?: number) => never, -): boolean { - const gatewayPreflightIssue = detectOpenShellStateRpcPreflightIssue(); - if (gatewayPreflightIssue) { - printOpenShellStateRpcIssue(gatewayPreflightIssue, { - action: `rebuilding sandbox '${sandboxName}'`, - command: `${CLI_NAME} ${sandboxName} rebuild`, - }); - bail("OpenShell gateway schema mismatch."); - return false; - } - return true; -} - -function getRebuildSandboxEntryOrBail( - sandboxName: string, - bail: (msg: string, code?: number) => never, -): RebuildSandboxEntry | null { - const sb = registry.getSandbox(sandboxName) as RebuildSandboxEntry | null; - if (!sb) { - console.error(` Sandbox '${sandboxName}' not found in registry.`); - bail(`Sandbox '${sandboxName}' not found in registry.`); - return null; - } - return sb; -} - -function isSingleAgentRebuildSupported( - sb: registry.SandboxEntry & { agents?: unknown[] }, - bail: (msg: string, code?: number) => never, -): boolean { - if (sb.agents && sb.agents.length > 1) { - console.error(" Multi-agent sandbox rebuild is not yet supported."); - console.error(` Back up state manually and recreate with \`${CLI_NAME} onboard\`.`); - bail("Multi-agent sandbox rebuild is not yet supported."); - return false; - } - return true; -} - -async function stageRebuildMessagingPlanOrBail( - sandboxName: string, - sb: RebuildSandboxEntry, - rebuildAgent: string | null, - log: (msg: string) => void, - bail: (msg: string, code?: number) => never, -): Promise { - try { - return await stageMessagingManifestPlanForRebuild(sandboxName, sb, rebuildAgent, log); - } catch (err) { - // Source boundary: persisted registry messaging plans and current channel - // manifests are host-side inputs. If they drift or become invalid, rebuild - // must fail here before backup/delete; remove this boundary only if manifest - // staging becomes total over all persisted registry states. - const message = err instanceof Error ? err.message : String(err); - console.error(""); - console.error( - ` ${_RD}Rebuild preflight failed:${R} messaging manifest plan could not be staged.`, - ); - console.error(` ${message}`); - console.error(""); - console.error(" Sandbox is untouched — no data was lost."); - bail(message); - return null; - } -} - -function preflightRebuildCredentials( - sandboxName: string, - sb: RebuildSandboxEntry, - log: (msg: string) => void, - bail: (msg: string, code?: number) => never, -): boolean { - const session = onboardSession.loadSession(); - const sessionMatchesTarget = session?.sandboxName === sandboxName; - // The target registry entry is authoritative when a matching legacy session - // omitted credentialEnv; rebuild rewrites provider/model from this entry later, - // so remote registry providers must still fail closed before backup/delete. - const registryCredentialEnv = getRebuildCredentialEnvFromRegistry(sb.provider, sb.credentialEnv); - let rebuildCredentialEnv = registryCredentialEnv; - if (sessionMatchesTarget && registryCredentialEnv === null) { - rebuildCredentialEnv = session?.credentialEnv || null; - } - if (!sessionMatchesTarget && session?.sandboxName) { - log( - `Preflight warning: session belongs to '${session.sandboxName}', not '${sandboxName}' — using registry credential env ${rebuildCredentialEnv || "(none)"}`, - ); - console.log( - ` ${D}Note: onboard session belongs to '${session.sandboxName}', not '${sandboxName}'. ` + - `Using the '${sandboxName}' registry entry for credential preflight.${R}`, - ); - } - - const rebuildProvider = sb.provider; - - // Compatibility boundary for GH #2519: pre-fix local-provider sessions could - // persist credentialEnv="OPENAI_API_KEY" even though current local-provider - // write paths persist null. Only a session for this sandbox plus a local - // target registry provider may bypass the key; keep until legacy sessions are - // no longer supported by rebuild migration tests. - if ( - sessionMatchesTarget && - isLocalInferenceProvider(sb.provider) && - rebuildCredentialEnv === "OPENAI_API_KEY" - ) { - console.log( - ` ${D}Note: migrating ${sb.provider} sandbox off OPENAI_API_KEY (GH #2519). ` + - `Local inference does not require a host API key.${R}`, - ); - log( - `Preflight: legacy ${sb.provider} sandbox detected (credentialEnv=OPENAI_API_KEY) — clearing for rebuild`, - ); - rebuildCredentialEnv = null; - } - - if (rebuildProvider === hermesProviderAuth.HERMES_PROVIDER_NAME) { - if ( - !preflightHermesProviderCredentials( - sessionMatchesTarget ? session : null, - rebuildCredentialEnv, - log, - ) - ) { - bail("Missing Hermes Provider credentials"); - return false; - } - rebuildCredentialEnv = null; - } - - if (!rebuildCredentialEnv) { - if (!checkRebuildGatewayProviderOrBail(rebuildProvider, rebuildCredentialEnv, log, bail)) { - return false; - } - log( - "Preflight credential check: no credentialEnv in session (local inference or missing session)", - ); - return true; - } - - const credentialValue = hydrateCredentialEnv(rebuildCredentialEnv); - log( - `Preflight credential check: ${rebuildCredentialEnv} → ${credentialValue ? "present" : "MISSING"}`, - ); - if (!checkRebuildGatewayProviderOrBail(rebuildProvider, rebuildCredentialEnv, log, bail)) { - return false; - } - if (!credentialValue && shouldVerifyRebuildGatewayProvider(rebuildProvider)) { - log( - `Preflight credential check: provider '${rebuildProvider}' registered in gateway — skipping env check for ${rebuildCredentialEnv}`, - ); - return true; - } - if (credentialValue) return true; - - console.error(""); - console.error(` ${_RD}Rebuild preflight failed:${R} provider credential not found.`); - console.error(` The non-interactive recreate step requires ${rebuildCredentialEnv},`); - console.error(" but it is not set in the environment."); - console.error(""); - console.error(" To fix, do one of:"); - console.error(` export ${rebuildCredentialEnv}=`); - console.error(` ${CLI_NAME} onboard # re-enter the key interactively`); - console.error(""); - console.error(" Sandbox is untouched — no data was lost."); - bail(`Missing credential: ${rebuildCredentialEnv}`); - return false; -} - -function hydrateMessagingConfigForRebuild(sandboxName: string, log: (msg: string) => void): void { - const rebuildSession = onboardSession.loadSession(); - const hydratedMessagingConfig = hydrateMessagingChannelConfig( - getStoredMessagingChannelConfig(sandboxName, rebuildSession), - ); - if (hydratedMessagingConfig) { - log(`Stashed messaging config for rebuild: ${Object.keys(hydratedMessagingConfig).join(",")}`); - } -} - -function printRebuildVersionSummary( - sandboxName: string, - agentName: string, - versionCheck: ReturnType, -): void { - console.log(""); - console.log(` ${B}Rebuild sandbox '${sandboxName}'${R}`); - if (versionCheck.sandboxVersion) { - console.log(` Current: ${agentName} v${versionCheck.sandboxVersion}`); - } - if (versionCheck.expectedVersion) { - console.log(` Target: ${agentName} v${versionCheck.expectedVersion}`); - } - console.log(""); -} - -async function reapplyMessagingManifestAfterOpenClawDoctor( - sandboxName: string, - plan: SandboxMessagingPlan | null, - log: (msg: string) => void, -): Promise { - if (!plan || plan.agent !== "openclaw") { - log("Messaging manifest reapply skipped: no OpenClaw messaging plan"); - return; - } - - try { - log("Reapplying messaging manifest render and post-agent-install hooks after doctor"); - const result = await MessagingSetupApplier.applyAgentConfigAtOpenShell(plan, { - runOpenshell: runMessagingOpenshell, - runHook: (request) => hookOutputsFromBuildSteps(plan, request), - }); - log( - `messaging manifest reapply: targets=${result.appliedTargets.join(",")}, hooks=${result.appliedHooks.join(",")}`, - ); - if (result.appliedTargets.length > 0 || result.appliedHooks.length > 0) { - console.log(` ${G}✓${R} Messaging manifest config reapplied`); - } - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - log(`Messaging manifest reapply failed: ${message}`); - console.log(` ${D}Messaging manifest config reapply skipped (${message})${R}`); - } -} - -/** - * Rebuild a live sandbox while preserving registered agent state and policies. - * - * Agent sandboxes force-refresh their base image before backup/delete so local - * `Dockerfile.base` changes fail before destructive work and are applied to the - * recreated sandbox image. - */ -interface RebuildSandboxExecutionOptions { - throwOnError?: boolean; - /** Internal installer recovery input; never exposed as a CLI option. */ - recoveryManifest?: sandboxState.RebuildManifest; -} - -type RebuildBail = (message: string, code?: number) => never; - -function failPreparedRecoveryPreDelete( - detail: string, - errorMessage: string, - bail: RebuildBail, -): never { - console.error(""); - console.error(` ${_RD}Recovery pre-delete check failed:${R} ${detail}.`); - console.error(" Sandbox is untouched — no data was lost."); - return bail(errorMessage); -} - -function revalidatePreparedRecoveryBeforeDelete( - sandboxName: string, - initialEntry: RebuildSandboxEntry, - candidate: sandboxState.RebuildManifest | null, - registrySnapshot: registry.SandboxRegistry | null, - bail: RebuildBail, -): { - manifest: sandboxState.RebuildManifest | null; - registrySnapshot: registry.SandboxRegistry | null; -} { - if (!candidate) return { manifest: null, registrySnapshot }; - - const refreshedRegistrySnapshot = JSON.parse( - JSON.stringify(registry.load()), - ) as registry.SandboxRegistry; - const currentEntry = refreshedRegistrySnapshot.sandboxes[sandboxName]; - if (!currentEntry) { - return failPreparedRecoveryPreDelete( - "registry entry no longer exists", - "Recovery registry identity changed during preflight.", - bail, - ); - } - if (!isDeepStrictEqual(currentEntry, initialEntry)) { - return failPreparedRecoveryPreDelete( - "registered sandbox configuration changed during preflight", - "Recovery registry configuration changed during preflight.", - bail, - ); - } - - const latestManifest = sandboxState.getLatestBackup(sandboxName); - if ( - !latestManifest || - latestManifest.timestamp !== candidate.timestamp || - latestManifest.backupPath !== candidate.backupPath - ) { - return failPreparedRecoveryPreDelete( - "latest prepared backup changed during preflight", - "Recovery backup identity changed during preflight.", - bail, - ); - } - - const validation = sandboxState.validateRebuildRecoveryManifest( - sandboxName, - currentEntry.agent, - latestManifest, - ); - if (!validation.ok) { - return failPreparedRecoveryPreDelete( - validation.reason, - `Invalid recovery manifest: ${validation.reason}`, - bail, - ); - } - if (!sandboxState.hasPositiveManagedImageEvidence(currentEntry)) { - return failPreparedRecoveryPreDelete( - "registry no longer has a NemoClaw-managed image fingerprint", - "Recovery registry entry has no NemoClaw-managed image fingerprint.", - bail, - ); - } - - return { - manifest: validation.manifest, - registrySnapshot: refreshedRegistrySnapshot, - }; -} - -export async function rebuildSandbox( - sandboxName: string, - options: string[] | RebuildSandboxOptions = {}, - opts: RebuildSandboxExecutionOptions = {}, -): Promise { - const normalized = normalizeRebuildSandboxOptions(options); - const verbose = normalized.verbose === true || process.env.NEMOCLAW_REBUILD_VERBOSE === "1"; - const log: (msg: string) => void = verbose ? _rebuildLog : () => {}; - const skipConfirm = normalized.yes === true || normalized.force === true; - // When called from upgradeSandboxes in a loop, throwOnError prevents - // process.exit from aborting the entire batch on the first failure. - const bail: RebuildBail = opts.throwOnError - ? (msg: string, _code = 1) => { - throw new Error(msg); - } - : (_msg: string, code = 1) => process.exit(code); - - // Active session detection — enrich the confirmation prompt if sessions are active - const rebuildActiveSessionCount = countActiveSandboxSessionsForRebuild(sandboxName); - - const sb = getRebuildSandboxEntryOrBail(sandboxName, bail); - if (!sb) return; - - let recoveryManifest: sandboxState.RebuildManifest | null = null; - if (opts.recoveryManifest) { - const validation = sandboxState.validateRebuildRecoveryManifest( - sandboxName, - sb.agent, - opts.recoveryManifest, - ); - if (!validation.ok) { - console.error(""); - console.error(` ${_RD}Recovery preflight failed:${R} ${validation.reason}.`); - console.error(" Sandbox is untouched — no data was lost."); - bail(`Invalid recovery manifest: ${validation.reason}`); - return; - } - if (!sandboxState.hasPositiveManagedImageEvidence(sb)) { - console.error(""); - console.error( - ` ${_RD}Recovery preflight failed:${R} registry has no NemoClaw-managed image fingerprint.`, - ); - console.error( - " Pre-fingerprint and custom-image sandboxes are not recreated automatically.", - ); - console.error(" Sandbox is untouched — no data was lost."); - bail("Recovery registry entry has no NemoClaw-managed image fingerprint."); - return; - } - recoveryManifest = validation.manifest; - } - - // Multi-agent guard (temporary — until swarm lands) - if (!isSingleAgentRebuildSupported(sb, bail)) return; - - const rebuildAgent = sb.agent || null; - const rebuildsDcodeSandbox = isDcodeRebuildAgent(rebuildAgent); - const agent = agentRuntime.getSessionAgent(sandboxName); - const agentName = agentRuntime.getAgentDisplayName(agent); - - if (!rebuildsDcodeSandbox && !checkRebuildGatewaySchemaPreflight(sandboxName, bail)) return; - - // Hydrate non-secret messaging config before the rebuild touches anything - // destructive. The manifest plan in registry is the durable source; legacy - // session channel fields are read only as compatibility fallback by - // getStoredMessagingChannelConfig(). - hydrateMessagingConfigForRebuild(sandboxName, log); - - // Version check — show what's changing - const versionCheck = sandboxVersion.checkAgentVersion(sandboxName); - printRebuildVersionSummary(sandboxName, agentName, versionCheck); - - const rebuildConfirmed = await confirmSandboxRebuildIfNeeded( - skipConfirm, - rebuildActiveSessionCount, - ); - if (!rebuildConfirmed) return; - - const dcodePreflight = createDcodeRebuildOrchestrator({ - sandboxName, - entry: sb, - rebuildAgent, - log, - bail, - deps: { - checkGatewaySchema: checkRebuildGatewaySchemaPreflight, - preflightCredentials: preflightRebuildCredentials, - ensureAgentBaseImage: ensureRebuildAgentBaseImage, - }, - }); - - // Step 0: Preflight — verify recreate preconditions BEFORE destroying - // anything. The most common rebuild failure is a missing provider credential - // when onboard runs in non-interactive mode. Checking now lets us abort with - // the sandbox still intact. See #2273. - const credentialsReady = await dcodePreflight.preflightCredentials(); - if (!credentialsReady) { - dcodePreflight.cleanup(); - return; - } - - // #5735 (PRA-6/PRA-9): resolve and validate the entire recreate config — agent, - // provider, model, credential, endpoint — from the registry/session BEFORE any - // destructive backup/delete, and surface/neutralize ambient onboard-selection - // env that would otherwise steer the resume away from the recorded sandbox. - // Fails closed (sandbox untouched) when a precondition cannot be satisfied. - const resumeConfig = dcodePreflight.runSync(() => - prepareRebuildResumeConfig(sandboxName, sb, rebuildAgent, log, dcodePreflight.bail), - ); - if (!resumeConfig) { - dcodePreflight.cleanup(); - return; - } - - const rebuildMessagingPlan = await dcodePreflight.run(() => - stageRebuildMessagingPlanOrBail(sandboxName, sb, rebuildAgent, log, dcodePreflight.bail), - ); - - // #5954: detect cross-sandbox messaging credential conflicts (e.g. another - // sandbox already polling the same Teams app) BEFORE any destructive - // backup/delete. This guard previously ran only in the recreate - // (onboard --resume) phase — after the sandbox was destroyed — so a conflict - // left the sandbox permanently lost. Running it here keeps it intact. - await dcodePreflight.run(() => - preflightRebuildMessagingConflicts(rebuildMessagingPlan, { - sandboxName, - gatewayName: getSandboxTargetGatewayName(sandboxName), - registry, - cliName: () => CLI_NAME, - // The conflict warning explains why the rebuild aborts, so it must reach - // the user regardless of the verbose flag (unlike the diagnostic `log`). - log: (message: string) => console.log(message), - error: (message: string) => console.error(message), - bail: dcodePreflight.bail, - }), - ); - - // Step 1: Ensure sandbox is live for backup, or identify stale-sandbox recovery. - const liveState = await dcodePreflight.run(() => - resolveRebuildLiveState(sandboxName, sb, log, dcodePreflight.bail), - ); - if (!liveState) { - dcodePreflight.cleanup(); - return; - } - const { staleRecovery } = liveState; - const preparedBackupRecovery = recoveryManifest !== null; - const recoveryRecreate = staleRecovery || preparedBackupRecovery; - // A prepared pre-upgrade backup can recover a sandbox that still appears in - // OpenShell but is stuck in Provisioning/Error. Capture the same registry - // rollback state used by missing-live-sandbox recovery before deletion. - let recoveryRegistrySnapshot = dcodePreflight.runSync(() => - preparedBackupRecovery - ? JSON.parse(JSON.stringify(registry.load())) - : liveState.staleRegistrySnapshot, - ); - - // DCode prebuilds and seals the managed replacement inputs; other agents retain the - // existing base-image-only preflight. - const imageReady = await dcodePreflight.prepareImage(resumeConfig, recoveryRecreate); - if (!imageReady) { - dcodePreflight.cleanup(); - return; - } - - // On stale-sandbox recovery the live sandbox is gone, so the normal - // unlock→recreate→relock cycle cannot run. Track stale lock state and defer - // clearing old shields state until recreate succeeds (#4497). - const { rebuildShieldsWindow, staleSandboxWasLocked } = dcodePreflight.runSync(() => - openRebuildShieldsWindowForState(sandboxName, recoveryRecreate), - ); - if (!rebuildShieldsWindow) return dcodePreflight.bail("Failed to auto-unlock shields."); - - const relockShieldsIfNeeded = (sandboxStillExists: boolean): boolean => - relockRebuildShieldsWindow(sandboxName, rebuildShieldsWindow, sandboxStillExists, CLI_NAME); - - let sandboxStillExists = true; - - try { - // Re-read the prepared manifest immediately before the destructive phase. - // Base-image builds and other preflight work can take long enough that the - // on-disk backup may have been replaced since the initial validation. - const preDeleteRecovery = revalidatePreparedRecoveryBeforeDelete( - sandboxName, - sb, - recoveryManifest, - recoveryRegistrySnapshot, - bail, - ); - recoveryManifest = preDeleteRecovery.manifest; - recoveryRegistrySnapshot = preDeleteRecovery.registrySnapshot; - - // Step 2: Backup (skipped on stale-sandbox recovery -- no live state exists) - // Installer recovery already has a validated pre-upgrade backup. Reuse it - // instead of trying to reach a non-Ready sandbox to create a second backup. - const backupManifest = - recoveryManifest ?? - backupSandboxStateForRebuild( - sandboxName, - sb, - staleRecovery, - log, - relockShieldsIfNeeded, - bail, - ); - if (backupManifest === undefined) return; - - // Backup can take long enough for the recorded target, gateway route, or - // retained build inputs to drift. DCode fails closed at the deletion edge; - // a harmless backup may remain, but the live sandbox is preserved. - if (!(await dcodePreflight.revalidateBeforeDelete(resumeConfig, recoveryRecreate))) return; - - // Step 3: Delete sandbox without tearing down gateway or session. - // sandboxDestroy() cleans up the gateway when it's the last sandbox and - // nulls session.sandboxName — both break the immediate onboard --resume. - console.log(" Deleting old sandbox..."); - const sbMeta = registry.getSandbox(sandboxName); - log( - `Registry entry: agent=${sbMeta?.agent}, agentVersion=${sbMeta?.agentVersion}, nimContainer=${sbMeta?.nimContainer}`, - ); - if (sbMeta && sbMeta.nimContainer) { - log(`Stopping NIM container: ${sbMeta.nimContainer}`); - nim.stopNimContainerByName(sbMeta.nimContainer); - } else { - // Best-effort cleanup — see comment in sandboxDestroy. - nim.stopNimContainer(sandboxName, { silent: true }); - } - - log(`Running: openshell sandbox delete ${sandboxName}`); - const deleteResult = runOpenshell(["sandbox", "delete", sandboxName], { - ignoreError: true, - stdio: ["ignore", "pipe", "pipe"], - }); - const { alreadyGone } = getSandboxDeleteOutcome(deleteResult); - log(`Delete result: exit=${deleteResult.status}, alreadyGone=${alreadyGone}`); - if (deleteResult.status !== 0 && !alreadyGone) { - console.error(" Failed to delete sandbox. Aborting rebuild."); - if (backupManifest) { - console.error(" State backup is preserved at: " + backupManifest.backupPath); - } - relockShieldsIfNeeded(true); - bail("Failed to delete sandbox.", deleteResult.status || 1); - return; - } - sandboxStillExists = false; - removeSandboxRegistryEntry(sandboxName); - log( - `Registry after remove: ${JSON.stringify(registry.listSandboxes().sandboxes.map((s: { name: string }) => s.name))}`, - ); - console.log(` ${G}\u2713${R} Old sandbox deleted`); - - // Step 4: Recreate via onboard --resume - console.log(""); - console.log(" Creating new sandbox with current image..."); - - // Force the sandbox name so onboard recreates with the same name. - // Mark session resumable and point at this sandbox; set env var as fallback. - const sessionBefore = onboardSession.loadSession(); - const sessionMatchesSandbox = sessionBefore?.sandboxName === sandboxName; - const rebuildsHermesSandbox = rebuildAgent === "hermes"; - let registryHermesToolGateways: string[] | null = null; - if (rebuildsHermesSandbox && Array.isArray(sb.hermesToolGateways)) { - registryHermesToolGateways = sb.hermesToolGateways.filter( - (value: unknown): value is string => typeof value === "string", - ); - } - const sessionHermesToolGateways = - rebuildsHermesSandbox && - sessionMatchesSandbox && - Array.isArray(sessionBefore?.hermesToolGateways) - ? sessionBefore.hermesToolGateways.filter( - (value: unknown): value is string => typeof value === "string", - ) - : null; - const rebuildHermesToolGateways = rebuildsHermesSandbox - ? (registryHermesToolGateways ?? sessionHermesToolGateways ?? []) - : []; - const hasRebuildHermesToolGateways = - rebuildsHermesSandbox && - (registryHermesToolGateways !== null || sessionHermesToolGateways !== null); - log( - `Session before update: sandboxName=${sessionBefore?.sandboxName}, status=${sessionBefore?.status}, resumable=${sessionBefore?.resumable}, provider=${sessionBefore?.provider}, model=${sessionBefore?.model}, sessionMatch=${sessionMatchesSandbox}`, - ); - - // Sync the session's agent field with the registry so onboard --resume - // rebuilds the correct sandbox type. Without this, a stale session.agent - // from a previous onboard of a *different* agent type would be picked up - // by resolveAgentName() and the wrong Dockerfile would be used. (#2201) - onboardSession.updateSession((s: Session) => { - s.sandboxName = sandboxName; - s.resumable = true; - s.status = "in_progress"; - s.agent = rebuildAgent; - s.messagingPlan = rebuildMessagingPlan; - s.hermesToolGateways = rebuildsHermesSandbox ? rebuildHermesToolGateways : []; - // Persist inference selection from the about-to-be-removed registry entry - // so onboard --resume can recreate with the same provider/model in - // non-interactive mode. Without this the registry is gone by the time - // setupNim runs, leaving no recovery source. Assign explicitly (with a - // null fallback) so a missing registry value doesn't silently leave a - // stale session entry from an earlier sandbox in place. - // #5735: apply the recreate config resolved + validated BEFORE delete by - // prepareRebuildResumeConfig, so onboard --resume recreates the recorded - // sandbox in non-interactive mode. Provider/model/credential/endpoint come - // from the about-to-be-removed registry entry or a validated matching - // custom-endpoint session, never ambient env. Assign explicitly so missing - // values cannot leave stale entries from an earlier sandbox in place. - s.provider = resumeConfig.provider; - s.model = resumeConfig.model; - s.nimContainer = resumeConfig.nimContainer; - s.credentialEnv = resumeConfig.credentialEnv; - s.preferredInferenceApi = resumeConfig.preferredInferenceApi; - dcodePreflight.clearManagedCustomDockerfile(s); - // `onboard --resume` uses the session as the recreate contract. Always - // overwrite the endpoint from the preflighted registry-derived config, - // even when the pre-existing session currently matches this sandbox name: - // stale recovery can be retrying after an earlier failed recreate left a - // partial session behind. Leaving the old endpoint in that case can silently - // steer the recreate to the wrong provider URL. `prepareRebuildResumeConfig` - // already validates whether this endpoint is recoverable before any - // destructive work, so this is the safest source boundary (#4497/#5869). - s.endpointUrl = resumeConfig.endpointUrl; - return s; - }); - process.env.NEMOCLAW_SANDBOX_NAME = sandboxName; - - const sessionAfter = onboardSession.loadSession(); - log( - `Session after update: sandboxName=${sessionAfter?.sandboxName}, status=${sessionAfter?.status}, resumable=${sessionAfter?.resumable}, provider=${sessionAfter?.provider}, model=${sessionAfter?.model}`, - ); - log( - `Env: NEMOCLAW_SANDBOX_NAME=${process.env.NEMOCLAW_SANDBOX_NAME}, NEMOCLAW_RECREATE_SANDBOX=${process.env.NEMOCLAW_RECREATE_SANDBOX}`, - ); - - // Forward the stored --from Dockerfile path so onboard --resume uses the - // same custom image. Without this, the conflict check rejects the resume - // because requestedFrom (null) !== recordedFrom (the stored path). (#2301) - // Only read from the session when it belongs to this sandbox to avoid - // using config from a different sandbox's onboard run. - const storedFromDockerfile = dcodePreflight.storedDockerfile( - sessionMatchesSandbox, - sessionAfter, - ); - log( - `Calling onboard({ resume: true, nonInteractive: true, recreateSandbox: true, fromDockerfile: ${storedFromDockerfile} })`, - ); - - // Intercept process.exit during onboard so we can attempt rollback - // instead of dying with the sandbox destroyed. onboard() has ~87 - // process.exit() calls that would otherwise kill the process with no - // chance to recover. See #2273. - // - // NOTE: Throwing from the overridden process.exit unwinds onboard's - // call stack, which skips process.once("exit") listeners (lock - // release, build context cleanup, session failure marking). We - // manually release the lock and mark the session failed in the - // onboardFailed block below. - const { onboard } = require("../../onboard"); - let onboardFailed = false; - let onboardExitCode = 1; - const _savedExit = process.exit; - process.exit = ((code) => { - onboardFailed = true; - onboardExitCode = typeof code === "number" ? code : 1; - // Throw a sentinel to unwind the onboard call stack. - // The catch block below handles it. - const err = new Error(`onboard exited with code ${onboardExitCode}`); - err.name = "RebuildOnboardExit"; - throw err; - }) as typeof process.exit; - - // Reaching here means the user already consented to the destructive - // rebuild (either via --yes/--force or by answering "y" at the prompt). - // Propagate that consent so the size-confirm gate inside the - // non-interactive onboard does not abort after the old sandbox has - // been deleted. The recreate path also inherits the original sandbox's - // no-GPU intent so the inner `onboard --resume` does not enforce the - // Docker CDI GPU preflight on hosts without an NVIDIA GPU. - const recreateOpts = buildRebuildRecreateOnboardOpts({ - sb, - rebuildAgent, - storedFromDockerfile, - preparedDcodeRebuild: dcodePreflight.preparedReplacement ?? undefined, - autoYes: skipConfirm || rebuildConfirmed, - }); - // #5735: isolate ambient onboard-selection env only for the duration of the - // recreate. The session was just pinned to the registry agent/provider/ - // model/credential above, so removing NEMOCLAW_AGENT/PROVIDER/PROVIDER_KEY/ - // ENDPOINT_URL/MODEL forces onboard --resume to recreate from that pinned - // config (and the already-registered gateway provider) instead of an - // unrelated onboard's values. Restored in finally so a bulk rebuild loop - // and the caller's process env are left untouched. - const restoreAmbientRecreateEnv = isolateAmbientRecreateEnv(); - const restoreDockerGpuPatchNetwork = dcodePreflight.applyDockerGpuPatchNetwork(); - try { - await onboard(recreateOpts); - log("onboard() returned successfully"); - } catch (err) { - onboardFailed = true; - const message = err instanceof Error ? err.message : String(err); - const name = err instanceof Error ? err.name : ""; - if (name !== "RebuildOnboardExit") { - log(`onboard() threw: ${message}`); - } - } finally { - process.exit = _savedExit; - restoreAmbientRecreateEnv(); - restoreDockerGpuPatchNetwork(); - } - - if (!onboardFailed) { - sandboxStillExists = true; - } - - if (onboardFailed) { - // Clean up onboard's internal state that normally runs in - // process.once("exit") listeners — those never fire because we - // threw from the overridden process.exit instead of actually - // exiting. Without this the onboard lock file stays on disk and - // blocks the next onboard/rebuild invocation. - try { - onboardSession.releaseOnboardLock(); - } catch { - /* best effort */ - } - try { - markLastStartedStepFailed(onboardSession, "Rebuild recreate failed"); - } catch { - /* best effort */ - } - - // Recovery already removed the registry entry before the recreate. If the - // recreate failed, restore the captured entry so the recommended - // `rebuild --yes` (and `connect`) - // remain retryable instead of failing at dispatch with "not found in - // registry" (#4497). Restore unconditionally — overwriting any partial entry - // a failed `onboard` may have registered — so the original metadata - // (defaultSandbox, customPolicies, every field) wins, not a half-written - // recreate entry. The restore targets only this sandbox under the registry - // lock, leaving other sandboxes' concurrent changes intact. - const snapshotEntry = recoveryRegistrySnapshot?.sandboxes?.[sandboxName]; - if (recoveryRecreate && snapshotEntry) { - try { - registry.restoreSandboxEntry(snapshotEntry, { - reclaimDefault: - recoveryRegistrySnapshot?.defaultSandbox === sandboxName ? sandboxName : null, - }); - log("Recovery recreate failed: restored preserved registry entry for retry"); - } catch (err) { - log(`Failed to restore registry entry after recovery recreate failure: ${String(err)}`); - } - } - - console.error(""); - if (recoveryRecreate) { - console.error(` ${_RD}Recovery recreate failed.${R}`); - console.error( - " Your local registry entry has been preserved — you can retry once the issue above is fixed.", - ); - } else { - console.error(` ${_RD}Recreate failed after sandbox was destroyed.${R}`); - } - if (backupManifest) { - console.error(` Backup is preserved at: ${backupManifest.backupPath}`); - } - console.error(""); - console.error(" To recover manually:"); - console.error(` 1. Fix the issue above (missing credential, Docker problem, etc.)`); - console.error(` 2. Run: ${CLI_NAME} onboard --resume`); - console.error(` This will recreate sandbox '${sandboxName}'.`); - if (backupManifest) { - console.error(` 3. Then restore your workspace state:`); - console.error( - ` ${CLI_NAME} ${sandboxName} snapshot restore "${backupManifest.timestamp}"`, - ); - } - printRebuildShieldsRecovery(sandboxName, rebuildShieldsWindow, CLI_NAME); - console.error(""); - relockShieldsIfNeeded(false); - bail( - backupManifest - ? `Recreate failed (sandbox destroyed). Backup: ${backupManifest.backupPath}` - : "Recreate failed (stale-sandbox recovery).", - onboardExitCode, - ); - return; - } - - // Recreate succeeded. Reset the prior shields state so the freshly recreated - // (mutable) sandbox reports its true posture. Deferred until here so a failed - // recreate above leaves the lockdown record intact for a retry (#4497). - if (recoveryRecreate) { - shields.clearShieldsState(sandboxName); - } - - const preservedRegistryFields = { - ...(hasRebuildHermesToolGateways - ? { hermesToolGateways: [...rebuildHermesToolGateways] } - : {}), - }; - if (Object.keys(preservedRegistryFields).length > 0) { - registry.updateSandbox(sandboxName, preservedRegistryFields); - } - - // Step 5: Restore (skipped on stale-sandbox recovery -- no backup exists) - let restoreSucceeded = true; - if (backupManifest) { - console.log(""); - console.log(" Restoring workspace state..."); - log(`Restoring from: ${backupManifest.backupPath} into sandbox: ${sandboxName}`); - const restore = sandboxState.restoreSandboxState(sandboxName, backupManifest.backupPath); - log( - `Restore result: success=${restore.success}, restored=${restore.restoredDirs.join(",")}; files=${restore.restoredFiles.join(",")}, failed=${restore.failedDirs.join(",")}; failedFiles=${restore.failedFiles.join(",")}`, - ); - restoreSucceeded = restore.success; - if (!restore.success) { - console.error(` Partial restore: ${restore.restoredDirs.join(", ") || "none"}`); - console.error(` Failed: ${restore.failedDirs.join(", ")}`); - if (restore.failedFiles.length > 0) { - console.error(` Failed files: ${restore.failedFiles.join(", ")}`); - } - console.error(` Manual restore available from: ${backupManifest.backupPath}`); - } else { - console.log( - ` ${G}\u2713${R} State restored (${restore.restoredDirs.length} directories, ${restore.restoredFiles.length} files)`, - ); - } - } - - // Step 5.5: Restore policy presets (#1952) - // Built-in policy presets live in the gateway policy engine, not the sandbox - // filesystem, so they are lost when the sandbox is destroyed and recreated. - // Re-apply the presets captured in the backup manifest. On stale-sandbox - // recovery there is no manifest, so fall back to the built-in preset names - // recorded on the registry entry (`sb.policies`) — the same source the backup - // manifest is built from — so the recovered sandbox keeps its built-in egress - // presets (#4497). Custom `policy-add --from-file/--from-dir` rules - // (`sb.customPolicies`) are not re-applied here; like a normal rebuild, they - // follow the recreate/onboard path and must be re-added if they were in use. - const registryPolicyPresets = Array.isArray(sb.policies) - ? sb.policies.filter((value: unknown): value is string => typeof value === "string") - : []; - const rebuildDisabledChannels = [...(rebuildMessagingPlan?.disabledChannels ?? [])]; - const rebuildEnabledChannelIds = (rebuildMessagingPlan?.channels ?? []) - .filter((ch) => !ch.disabled) - .map((ch) => ch.channelId); - const savedPresets = mergeRebuildMessagingPolicyPresets( - backupManifest?.policyPresets, - registryPolicyPresets, - rebuildEnabledChannelIds, - rebuildDisabledChannels, - ); - const restoredPresets: string[] = []; - const failedPresets: string[] = []; - if (savedPresets.length > 0) { - console.log(""); - console.log(" Restoring policy presets..."); - log(`Policy presets to restore: [${savedPresets.join(",")}]`); - for (const presetName of savedPresets) { - try { - log(`Applying preset: ${presetName}`); - const applied = policies.applyPreset(sandboxName, presetName); - if (applied) { - restoredPresets.push(presetName); - } else { - failedPresets.push(presetName); - } - } catch (err) { - const errorMessage = err instanceof Error ? err.message : String(err); - log(`Failed to apply preset '${presetName}': ${errorMessage}`); - failedPresets.push(presetName); - } - } - if (restoredPresets.length > 0) { - console.log(` ${G}\u2713${R} Policy presets restored: ${restoredPresets.join(", ")}`); - } - if (failedPresets.length > 0) { - console.error(` ${YW}\u26a0${R} Failed to restore presets: ${failedPresets.join(", ")}`); - console.error(` Re-apply manually with: ${CLI_NAME} ${sandboxName} policy-add`); - } - } - - // Step 6: Post-restore agent-specific migration - const rebuiltAgent = agentRuntime.getSessionAgent(sandboxName); - const rebuiltAgentName = agentRuntime.getAgentDisplayName(rebuiltAgent); - const agentDef = rebuiltAgent ? loadAgent(rebuiltAgent.name) : loadAgent("openclaw"); - // #4538: set when the post-upgrade mutable-config permission repair ran but - // could not verify the contract — the rebuilt sandbox may still EACCES on - // gateway-side config writes, so the final result is downgraded below. - let mutablePermsRepairUnverified = false; - let mutableConfigHashRefreshUnverified = false; - let messagingHostForwardUnverified = false; - const policyPresetRestoreIncomplete = failedPresets.length > 0; - if (agentDef.name === "openclaw") { - // openclaw doctor --fix validates and repairs directory structure. - // Idempotent and safe — catches structural changes between OpenClaw versions - // (new symlinks, new data dirs, etc.) that the restored state may be missing. - log("Running openclaw doctor --fix inside sandbox for post-upgrade structure repair"); - const doctorResult = executeSandboxCommand(sandboxName, "openclaw doctor --fix"); - log( - `doctor --fix: exit=${doctorResult?.status}, stdout=${(doctorResult?.stdout || "").substring(0, 200)}`, - ); - if (doctorResult && doctorResult.status === 0) { - console.log(` ${G}\u2713${R} Post-upgrade structure check passed`); - } else { - console.log( - ` ${D}Post-upgrade structure check skipped (doctor returned ${doctorResult?.status ?? "null"})${R}`, - ); - } - - // doctor --fix may rewrite openclaw.json after the image build applied - // manifest-owned messaging render and post-agent-install build-file outputs. - // Reapply the staged plan so channel config and WeChat account seed files - // remain paired with the restored OpenClaw extension state. - await reapplyMessagingManifestAfterOpenClawDoctor(sandboxName, rebuildMessagingPlan, log); - - // The post-restore structure repair and seed helper can rewrite - // openclaw.json after restoreStateFile has already refreshed - // .config-hash. Refresh the mutable hash here so the gateway token and - // channel seed changes are integrity-valid before the sandbox is handed - // back to the user. - log("Refreshing mutable OpenClaw config hash after post-restore config writes"); - if (!refreshMutableOpenClawConfigHashAfterPostRestoreWrites(sandboxName, log)) { - mutableConfigHashRefreshUnverified = true; - } - - // #4538: `openclaw doctor --fix` enforces a single-user 700/600 state - // layout, which silently tightens NemoClaw's mutable config contract - // (setgid + group-writable /sandbox/.openclaw and group-writable - // openclaw.json). Run this LAST in the OpenClaw post-restore sequence — - // after doctor --fix and messaging manifest reapply, both of which can - // rewrite openclaw.json — so the - // restored contract is not immediately undone. No-op for shields-up - // sandboxes (config is intentionally root-owned/locked). - log("Restoring mutable OpenClaw config permissions after post-restore config writes"); - // The shields wrapper can throw before it returns a structured result - // (validateName, or getShieldsPosture triggering inline auto-restore). A - // thrown error here must not abort the rest of the rebuild — treat it as an - // unverified repair and continue. - let permRepair: ReturnType | null = null; - try { - permRepair = shields.repairMutableConfigPerms(sandboxName); - } catch (err) { - mutablePermsRepairUnverified = true; - console.error( - ` ${YW}⚠${R} Mutable config permission repair errored: ${err instanceof Error ? err.message : String(err)}`, - ); - } - if (permRepair === null) { - // already handled above - } else if (!permRepair.applied) { - if (permRepair.skipReason === "unreadable") { - // Posture could not be determined, so the contract may still be broken. - // This is NOT a benign skip — surface it as incomplete. - mutablePermsRepairUnverified = true; - console.error( - ` ${YW}⚠${R} Mutable config permissions not restored: ${permRepair.reason}`, - ); - } else { - // "locked" (shields up — config is intentionally root-owned/locked) or - // "agent": a deliberate no-op, not a broken contract. Do not downgrade. - log(`Mutable config permission repair skipped: ${permRepair.reason}`); - } - } else if (permRepair.verified) { - console.log(` ${G}✓${R} Mutable config permissions restored`); - } else { - mutablePermsRepairUnverified = true; - console.error( - ` ${YW}⚠${R} Mutable config permission repair incomplete: ${permRepair.errors.join("; ")}`, - ); - } - } - // Hermes: no explicit post-restore step needed. Hermes's SessionDB._init_schema() - // auto-migrates state.db (SQLite) on first connection via sequential ALTER TABLE - // migrations (idempotent, schema_version tracked). ensure_hermes_home() repairs - // missing directories implicitly. The NemoClaw plugin's skill cache refreshes on - // on_session_start. Gateway startup is non-fatal if state.db migration fails. - - // Step 7: Update registry with new version - // - // Source-of-truth reconciliation for `policies`: - // - // - Invalid state: `registry.policies` retained a preset name after the - // reapply loop pruned it (disabled messaging channel) or skipped it - // (failed `applyPreset`), so `policy-list` showed a ● marker for a - // preset whose rules were absent from the gateway. - // - Source boundary: `policies.applyPreset` only appends to - // `registry.policies`; nothing else writes the canonical post-rebuild - // set. The reapply loop above is the only place that knows which - // presets were actually reapplied. - // - Source-fix constraint: must run after the reapply loop and use the - // successfully restored subset, not `savedPresets` (which still - // includes failures). - // - Regression test: - // `src/lib/actions/sandbox/rebuild-flow.test.ts` asserts - // `registry.updateSandbox` receives `policies: restoredPresets` for - // both the successful-rebuild and partial-restore harnesses. - // - Removal condition: drop this once `applyPreset` writes the - // canonical post-apply set itself (replacing its append-only - // contract), making the rebuild flow's reconciliation redundant. - registry.updateSandbox(sandboxName, { - agentVersion: agentDef.expectedVersion || null, - policies: restoredPresets, - }); - log( - `Registry updated: agentVersion=${agentDef.expectedVersion}, policies=[${restoredPresets.join(",")}]`, - ); - - if (!relockShieldsIfNeeded(true)) return bail("Failed to re-apply shields lockdown."); - if (!ensureMessagingHostForwardAfterRebuild(sandboxName, rebuildMessagingPlan)) { - messagingHostForwardUnverified = true; - } - - console.log(""); - const postRestoreComplete = - restoreSucceeded && - !mutablePermsRepairUnverified && - !mutableConfigHashRefreshUnverified && - !messagingHostForwardUnverified && - !policyPresetRestoreIncomplete; - if (postRestoreComplete) { - console.log(` ${G}\u2713${R} Sandbox '${sandboxName}' rebuilt successfully`); - if (staleRecovery && !backupManifest) { - console.log( - ` ${D}Recovered from a stale registry entry \u2014 no prior workspace state was available to restore.${R}`, - ); - } - if (versionCheck.expectedVersion) { - console.log(` Now running: ${rebuiltAgentName} v${versionCheck.expectedVersion}`); - } - } else { - // At least one post-restore step is incomplete. Surface every applicable - // failure (#4538: a failed state restore and an unverified permission - // repair are independent \u2014 report both so the operator does not miss the - // backup-restore recovery just because permissions also need attention). - console.log( - ` ${YW}\u26a0${R} Sandbox '${sandboxName}' rebuilt but some post-restore steps were incomplete`, - ); - if (!restoreSucceeded && backupManifest) { - console.log( - ` State restore was incomplete \u2014 backup available at: ${backupManifest.backupPath}`, - ); - } - if (mutablePermsRepairUnverified) { - console.log( - ` Mutable config permissions were not verified \u2014 run \`${CLI_NAME} ${sandboxName} doctor --fix\` to restore the OpenClaw config permission contract`, - ); - } - if (mutableConfigHashRefreshUnverified) { - console.log( - ` Mutable OpenClaw config hash was not refreshed \u2014 restart the sandbox or re-run \`${CLI_NAME} ${sandboxName} rebuild\` before relying on config integrity checks`, - ); - } - if (messagingHostForwardUnverified) { - console.log( - ` Messaging webhook forward was not verified \u2014 run \`${CLI_NAME} ${sandboxName} connect\` after resolving the port conflict`, - ); - } - if (policyPresetRestoreIncomplete) { - console.log( - ` Policy presets failed to reapply: ${failedPresets.join(", ")} \u2014 re-apply manually with \`${CLI_NAME} ${sandboxName} policy-add\``, - ); - } - } - // Stale recovery reset the shields state to mutable (the gone sandbox's lock - // seal could not carry over to the fresh image). If lockdown had been enabled, - // tell the operator to re-apply it on the recreated sandbox (#4497). - if (recoveryRecreate && staleSandboxWasLocked) { - console.log( - ` ${YW}\u26a0${R} Shields were previously enabled but the recreated sandbox starts unlocked \u2014 run \`${CLI_NAME} ${sandboxName} shields up\` to restore lockdown.`, - ); - } - if (preparedBackupRecovery && !postRestoreComplete) { - bail( - `Prepared backup recovery for '${sandboxName}' completed with unverified post-restore state.`, - ); - } - } finally { - try { - if (!rebuildShieldsWindow.relocked) { - relockShieldsIfNeeded(sandboxStillExists); - } - } finally { - dcodePreflight.cleanup(); - } - } -} +/** Public rebuild facade. Phase orchestration lives in focused rebuild modules. */ +export { + buildRefreshMutableOpenClawConfigHashCommand, + rebuildSandbox, + stageMessagingManifestPlanForRebuild, +} from "./rebuild-pipeline"; diff --git a/src/lib/actions/sandbox/snapshot.test.ts b/src/lib/actions/sandbox/snapshot.test.ts index 3dcd388ca22..66cb5123f69 100644 --- a/src/lib/actions/sandbox/snapshot.test.ts +++ b/src/lib/actions/sandbox/snapshot.test.ts @@ -693,6 +693,7 @@ describe("runSandboxSnapshot", () => { for (const processLine of [ "123 python3 -m deepagents_code --sandbox none --no-mcp -n work\n", "123 /opt/venv/bin/python3 -m deepagents_code --sandbox none --no-mcp -n work\n", + "123 /opt/venv/bin/python3 -I -m deepagents_code --sandbox none --no-mcp -n work\n", "124 /usr/local/bin/dcode task\n", "125 /opt/bin/deepagents_code task\n", "126 /opt/bin/deepagents-code task\n", diff --git a/src/lib/actions/sandbox/snapshot.ts b/src/lib/actions/sandbox/snapshot.ts index 47bb322aa77..3976c507672 100644 --- a/src/lib/actions/sandbox/snapshot.ts +++ b/src/lib/actions/sandbox/snapshot.ts @@ -24,6 +24,7 @@ import * as shields from "../../shields"; import { withTimerBoundShieldsMutationLock } from "../../shields/timer-bound-lock"; import { readTimerMarker } from "../../shields/timer-control"; import { isSandboxReady } from "../../state/gateway"; +import { withSandboxMutationLock } from "../../state/mcp-lifecycle-lock"; import type { SandboxEntry } from "../../state/registry"; import * as registry from "../../state/registry"; import * as sandboxState from "../../state/sandbox"; @@ -74,7 +75,7 @@ processes="$(ps -eo pid=,args= 2>/dev/null)" || { emit_dcode_probe_state no-runtime } printf '%s\n' "$processes" | awk ' -/^[[:space:]]*[0-9]+[[:space:]]+([^[:space:]]*\/)?python[0-9.]*[[:space:]]+-m[[:space:]]+deepagents[_]code([[:space:]]|$)/ { +/^[[:space:]]*[0-9]+[[:space:]]+([^[:space:]]*\/)?python[0-9.]*[[:space:]]+(-I[[:space:]]+)?-m[[:space:]]+deepagents[_]code([[:space:]]|$)/ { found = 1 } /^[[:space:]]*[0-9]+[[:space:]]+([^[:space:]]*\/)?[d]code([[:space:]]|$)/ { @@ -662,6 +663,16 @@ async function runSnapshotRestore( const target = request.to ?? sandboxName; const targetSandbox = target === sandboxName ? sandboxName : validateName(target, "target sandbox name"); + return withSandboxMutationLock(targetSandbox, () => + runSnapshotRestoreUnlocked(sandboxName, request, targetSandbox), + ); +} + +async function runSnapshotRestoreUnlocked( + sandboxName: string, + request: Extract, + targetSandbox: string, +): Promise { const sourceLiveNames = requireLiveSandboxesOnSandboxGateway( sandboxName, " Failed to query live sandbox state from OpenShell.", diff --git a/src/lib/actions/sandbox/status-text.ts b/src/lib/actions/sandbox/status-text.ts index c966c13bb81..d792251032a 100644 --- a/src/lib/actions/sandbox/status-text.ts +++ b/src/lib/actions/sandbox/status-text.ts @@ -185,7 +185,7 @@ function printActiveSessions(sandboxName: string): void { } function printShieldsPosture(sandboxName: string): void { - const posture = shields.getShieldsPosture(sandboxName, true); + const posture = shields.getShieldsPosture(sandboxName, false); if (posture.mode === "locked") return; const detail = posture.mode === "mutable_default" diff --git a/src/lib/adapters/dns/resolve.test.ts b/src/lib/adapters/dns/resolve.test.ts new file mode 100644 index 00000000000..adff8f29b81 --- /dev/null +++ b/src/lib/adapters/dns/resolve.test.ts @@ -0,0 +1,22 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { type DnsLookupAll, resolveHostAddresses } from "./resolve"; + +describe("DNS resolver adapter", () => { + it("requests all addresses in resolver order through the injected lookup", async () => { + const addresses = [ + { address: "203.0.113.10", family: 4 }, + { address: "2001:db8::10", family: 6 }, + ]; + const lookup = vi.fn().mockResolvedValue(addresses); + + await expect(resolveHostAddresses("mcp.example.test", lookup)).resolves.toEqual(addresses); + expect(lookup).toHaveBeenCalledWith("mcp.example.test", { + all: true, + verbatim: true, + }); + }); +}); diff --git a/src/lib/adapters/dns/resolve.ts b/src/lib/adapters/dns/resolve.ts new file mode 100644 index 00000000000..6eae89ead91 --- /dev/null +++ b/src/lib/adapters/dns/resolve.ts @@ -0,0 +1,17 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import dns from "node:dns/promises"; + +export type DnsLookupAddress = { address: string; family: number }; +export type DnsLookupAll = ( + hostname: string, + options: { all: true; verbatim: true }, +) => Promise; + +export async function resolveHostAddresses( + hostname: string, + lookup: DnsLookupAll = dns.lookup as DnsLookupAll, +): Promise { + return lookup(hostname, { all: true, verbatim: true }); +} diff --git a/src/lib/adapters/openshell/client.test.ts b/src/lib/adapters/openshell/client.test.ts index e781334de20..8a5fb2b5071 100644 --- a/src/lib/adapters/openshell/client.test.ts +++ b/src/lib/adapters/openshell/client.test.ts @@ -3,7 +3,7 @@ import type { SpawnSyncReturns } from "node:child_process"; -import { describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { captureOpenshellCommand, @@ -50,6 +50,10 @@ function exitWithCode(code: number): never { } describe("openshell helpers", () => { + afterEach(() => { + vi.unstubAllEnvs(); + }); + it("strips ANSI sequences", () => { expect(stripAnsi("\u001b[32mConnected\u001b[0m")).toBe("Connected"); }); @@ -119,6 +123,35 @@ describe("openshell helpers", () => { expect(result.status).toBe(0); }); + it("can replace the parent environment for credential-bearing OpenShell commands", () => { + vi.stubEnv("NEMOCLAW_TEST_UNRELATED_SECRET", "must-not-leak"); + let observedEnv: NodeJS.ProcessEnv | undefined; + runOpenshellCommand("openshell", ["provider", "create"], { + replaceEnv: true, + env: { PATH: "/safe/bin", MCP_TOKEN: "selected-secret" }, + spawnSyncImpl: (_command, _args, options) => { + observedEnv = options.env; + return makeSpawnResult({ status: 0, stdout: "ok\n", stderr: "" }); + }, + }); + + expect(observedEnv).toEqual({ PATH: "/safe/bin", MCP_TOKEN: "selected-secret" }); + }); + + it("filters unrelated parent secrets from ordinary OpenShell commands", () => { + vi.stubEnv("NEMOCLAW_TEST_UNRELATED_SECRET", "must-not-leak"); + let observedEnv: NodeJS.ProcessEnv | undefined; + runOpenshellCommand("openshell", ["status"], { + spawnSyncImpl: (_command, _args, options) => { + observedEnv = options.env; + return makeSpawnResult({ status: 0, stdout: "ok\n", stderr: "" }); + }, + }); + + expect(observedEnv?.NEMOCLAW_TEST_UNRELATED_SECRET).toBeUndefined(); + expect(observedEnv?.PATH).toBe(process.env.PATH); + }); + it("passes timeout and maxBuffer options through to OpenShell spawn calls", () => { const observedOptions: Array<{ timeout?: number; maxBuffer?: number }> = []; const spawnSyncImpl: OpenshellSpawnSync = (_command, _args, options) => { diff --git a/src/lib/adapters/openshell/client.ts b/src/lib/adapters/openshell/client.ts index cfbebb8a273..75536c0dd0f 100644 --- a/src/lib/adapters/openshell/client.ts +++ b/src/lib/adapters/openshell/client.ts @@ -10,6 +10,8 @@ import { spawnSync, } from "node:child_process"; +import { buildSubprocessEnv } from "../../subprocess-env"; + export type OpenshellSpawnSync = ( command: string, args: readonly string[], @@ -21,6 +23,7 @@ export type OpenshellSpawn = typeof spawn; interface OpenshellSpawnOptions { cwd?: string; env?: NodeJS.ProcessEnv; + replaceEnv?: boolean; timeout?: number; ignoreError?: boolean; spawnSyncImpl?: OpenshellSpawnSync; @@ -28,6 +31,15 @@ interface OpenshellSpawnOptions { exit?: (code: number) => never; } +function openshellSpawnEnv(opts: OpenshellSpawnOptions): NodeJS.ProcessEnv { + const explicitEnv = Object.fromEntries( + Object.entries(opts.env ?? {}).filter( + (entry): entry is [string, string] => entry[1] !== undefined, + ), + ); + return opts.replaceEnv ? explicitEnv : buildSubprocessEnv(explicitEnv); +} + export interface RunOpenshellOptions extends OpenshellSpawnOptions { stdio?: SpawnSyncOptions["stdio"]; input?: string; @@ -149,7 +161,7 @@ export function runOpenshellCommand( const spawnSyncImpl = opts.spawnSyncImpl ?? spawnSync; const result = spawnSyncImpl(binary, args, { cwd: opts.cwd, - env: { ...process.env, ...opts.env }, + env: openshellSpawnEnv(opts), encoding: "utf-8", stdio: opts.stdio ?? "inherit", input: opts.input, @@ -176,7 +188,7 @@ export function captureOpenshellCommand( const spawnSyncImpl = opts.spawnSyncImpl ?? spawnSync; const result = spawnSyncImpl(binary, args, { cwd: opts.cwd, - env: { ...process.env, ...opts.env }, + env: openshellSpawnEnv(opts), encoding: "utf-8", stdio: ["ignore", "pipe", "pipe"], timeout: opts.timeout, @@ -231,7 +243,7 @@ export function captureOpenshellCommandAsync( return new Promise((resolve) => { const child = spawnImpl(binary, args, { cwd: opts.cwd, - env: { ...process.env, ...opts.env }, + env: openshellSpawnEnv(opts), detached: process.platform !== "win32", stdio: ["ignore", "pipe", "pipe"], }) as ChildProcess; diff --git a/src/lib/adapters/openshell/resolve.ts b/src/lib/adapters/openshell/resolve.ts index fdb3833571d..fa74b4a45d7 100644 --- a/src/lib/adapters/openshell/resolve.ts +++ b/src/lib/adapters/openshell/resolve.ts @@ -4,6 +4,8 @@ import { execSync } from "node:child_process"; import { accessSync, constants } from "node:fs"; +import { buildSubprocessEnv } from "../../subprocess-env"; + export interface ResolveOpenshellOptions { /** Mock result for `command -v` (undefined = run real command). */ commandVResult?: string | null; @@ -40,7 +42,10 @@ export function resolveOpenshell(opts: ResolveOpenshellOptions = {}): string | n // Step 1: command -v if (opts.commandVResult === undefined) { try { - const found = execSync("command -v openshell", { encoding: "utf-8" }).trim(); + const found = execSync("command -v openshell", { + encoding: "utf-8", + env: buildSubprocessEnv(), + }).trim(); if (found.startsWith("/")) return found; } catch { /* ignored */ diff --git a/src/lib/adapters/openshell/runtime-capabilities.ts b/src/lib/adapters/openshell/runtime-capabilities.ts new file mode 100644 index 00000000000..b155058ac36 --- /dev/null +++ b/src/lib/adapters/openshell/runtime-capabilities.ts @@ -0,0 +1,10 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Present in OpenShell artifacts that include native Streamable HTTP MCP policy + * support. NemoClaw uses this implementation marker only as an installed-artifact + * compatibility gate during onboarding. The running supervisor is validated by + * applying the actual generated MCP policy through `openshell policy set --wait`. + */ +export const OPENSHELL_MCP_POLICY_CAPABILITY_MARKER = "allow_all_known_mcp_methods"; diff --git a/src/lib/adapters/openshell/runtime.ts b/src/lib/adapters/openshell/runtime.ts index 4fefa1b0c8d..178159b13f8 100644 --- a/src/lib/adapters/openshell/runtime.ts +++ b/src/lib/adapters/openshell/runtime.ts @@ -18,6 +18,7 @@ type CommandArgs = string[]; type RunnerOptions = { env?: NodeJS.ProcessEnv; + replaceEnv?: boolean; stdio?: StdioOptions; input?: string; ignoreError?: boolean; @@ -46,6 +47,7 @@ export function runOpenshell(args: CommandArgs, opts: RunnerOptions = {}) { return runOpenshellCommand(getOpenshellBinary(), args, { cwd: ROOT, env: opts.env, + replaceEnv: opts.replaceEnv, stdio: opts.stdio, input: opts.input, ignoreError: opts.ignoreError, @@ -64,6 +66,7 @@ export function captureOpenshell(args: CommandArgs, opts: RunnerOptions = {}) { return captureOpenshellCommand(getOpenshellBinary(), args, { cwd: ROOT, env: opts.env, + replaceEnv: opts.replaceEnv, ignoreError: opts.ignoreError, includeStderr: opts.includeStderr, includeStreams: opts.includeStreams, @@ -79,6 +82,7 @@ export function captureSandboxSshConfig(sandboxName: string, opts: RunnerOptions return captureSandboxSshConfigCommand(getOpenshellBinary(), sandboxName, { cwd: ROOT, env: opts.env, + replaceEnv: opts.replaceEnv, ignoreError: opts.ignoreError, includeStreams: opts.includeStreams, timeout: opts.timeout, @@ -99,6 +103,7 @@ export function captureOpenshellForStatus(args: CommandArgs, opts: RunnerOptions return captureOpenshellCommandAsync(getOpenshellBinary(), args, { cwd: ROOT, env: opts.env, + replaceEnv: opts.replaceEnv, ignoreError: opts.ignoreError, includeStreams: opts.includeStreams, timeout: opts.timeout ?? getStatusProbeTimeoutMs(), diff --git a/src/lib/agent/base-image-hermes.test.ts b/src/lib/agent/base-image-hermes.test.ts new file mode 100644 index 00000000000..3aa4411b6b5 --- /dev/null +++ b/src/lib/agent/base-image-hermes.test.ts @@ -0,0 +1,136 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import path from "node:path"; + +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { makeAgent, withMockedDocker } from "../../../test/helpers/base-image-test-harness"; + +describe("agent base image provisioning", () => { + beforeEach(() => { + vi.restoreAllMocks(); + }); + + it("probes resolved Hermes bases for the native MCP Streamable HTTP runtime", () => { + withMockedDocker(({ ensureAgentBaseImage, dockerCaptureMock, resolveSandboxBaseImageMock }) => { + ensureAgentBaseImage(makeAgent()); + const options = resolveSandboxBaseImageMock.mock.calls[0]?.[0] as { + validateImage?: (imageRef: string) => boolean; + }; + + expect(options.validateImage?.("hermes-base:test")).toBe(true); + expect(dockerCaptureMock).toHaveBeenCalledWith( + [ + "run", + "--rm", + "--entrypoint", + "/opt/hermes/.venv/bin/python", + "hermes-base:test", + "-c", + expect.stringContaining("_MCP_HTTP_AVAILABLE"), + ], + { ignoreError: true, timeout: 20_000 }, + ); + + dockerCaptureMock.mockReturnValue(""); + expect(options.validateImage?.("hermes-base:stale")).toBe(false); + }); + }); + + it("accepts only the tracked published Hermes base digest", () => { + const dockerfilePath = path.resolve(import.meta.dirname, "../../../agents/hermes/Dockerfile"); + const dockerfile = fs.readFileSync(dockerfilePath, "utf8"); + const trackedRef = dockerfile.match( + /^ARG BASE_IMAGE=(ghcr\.io\/nvidia\/nemoclaw\/hermes-sandbox-base@(sha256:[0-9a-f]{64}))$/m, + ); + expect(trackedRef).not.toBeNull(); + + withMockedDocker(({ ensureAgentBaseImage, resolveSandboxBaseImageMock }) => { + resolveSandboxBaseImageMock.mockReturnValue({ + ref: trackedRef?.[1], + digest: trackedRef?.[2], + source: "source-sha", + glibcVersion: "2.41", + }); + + expect(ensureAgentBaseImage(makeAgent({ dockerfilePath }))).toEqual({ + imageTag: trackedRef?.[1], + built: false, + }); + + const differentRef = `ghcr.io/nvidia/nemoclaw/hermes-sandbox-base@sha256:${"0".repeat(64)}`; + resolveSandboxBaseImageMock.mockReturnValue({ + ref: differentRef, + digest: `sha256:${"0".repeat(64)}`, + source: "source-sha", + glibcVersion: "2.41", + }); + expect(() => ensureAgentBaseImage(makeAgent({ dockerfilePath }))).toThrow( + "Hermes final image does not accept base image ref", + ); + }); + }); + + it("fails a forced rebuild before deletion when the built base fails validation", () => { + withMockedDocker(({ ensureAgentBaseImage, resolveSandboxBaseImageMock }) => { + resolveSandboxBaseImageMock.mockReturnValue(null); + + expect(() => ensureAgentBaseImage(makeAgent(), { forceBaseImageRebuild: true })).toThrow( + "failed the required runtime compatibility checks", + ); + }); + }); + + it("validates an explicit override strictly instead of falling back", () => { + const envVar = "NEMOCLAW_HERMES_SANDBOX_BASE_IMAGE_REF"; + const prior = process.env[envVar]; + process.env[envVar] = "localhost:5000/custom/hermes:latest"; + try { + withMockedDocker(({ ensureAgentBaseImage, resolveSandboxBaseImageMock }) => { + resolveSandboxBaseImageMock.mockReturnValue({ + ref: process.env[envVar], + digest: null, + source: "override", + glibcVersion: "2.41", + }); + + expect(() => ensureAgentBaseImage(makeAgent())).toThrow( + "Hermes final image does not accept base image ref", + ); + expect(resolveSandboxBaseImageMock).toHaveBeenCalledWith( + expect.objectContaining({ + localTag: "localhost:5000/custom/hermes:latest", + env: expect.objectContaining({ + [envVar]: "localhost:5000/custom/hermes:latest", + NEMOCLAW_SANDBOX_BASE_LOCAL_BUILD: "0", + }), + }), + ); + }); + } finally { + prior === undefined ? delete process.env[envVar] : (process.env[envVar] = prior); + } + }); + + it("fails closed when no MCP-capable Hermes base image can be resolved", () => { + withMockedDocker( + ({ + ensureAgentBaseImage, + dockerBuildMock, + dockerImageInspectMock, + resolveSandboxBaseImageMock, + }) => { + resolveSandboxBaseImageMock.mockReturnValue(null); + dockerImageInspectMock.mockReturnValue({ status: 1 }); + + expect(() => ensureAgentBaseImage(makeAgent())).toThrow( + "No compatible Hermes Agent sandbox base image found", + ); + expect(dockerBuildMock).not.toHaveBeenCalled(); + expect(dockerImageInspectMock).not.toHaveBeenCalled(); + }, + ); + }); +}); diff --git a/src/lib/agent/base-image.test.ts b/src/lib/agent/base-image.test.ts index fdf8b2af606..f9adddb2671 100644 --- a/src/lib/agent/base-image.test.ts +++ b/src/lib/agent/base-image.test.ts @@ -2,114 +2,8 @@ // SPDX-License-Identifier: Apache-2.0 import { beforeEach, describe, expect, it, vi } from "vitest"; -import type { AgentDefinition } from "./defs"; - -type AgentOnboardModule = typeof import("./onboard"); -type DockerImageModule = typeof import("../adapters/docker/image"); -type DockerInspectModule = typeof import("../adapters/docker/inspect"); -type SandboxBaseImageModule = typeof import("../sandbox-base-image"); - -/** - * Build a minimal Hermes agent manifest for base-image provisioning tests. - */ -function makeAgent(overrides: Partial = {}): AgentDefinition { - return { - name: "hermes", - displayName: "Hermes Agent", - healthProbe: { url: "http://127.0.0.1:8642/health", port: 8642, timeout_seconds: 90 }, - forwardPort: 8642, - dashboard: { - kind: "api", - label: "OpenAI-compatible API", - path: "/v1", - healthPath: "/health", - auth: "none", - }, - webAuth: { method: "bearer_token", env: "API_SERVER_KEY" }, - configPaths: { - dir: "/sandbox/.hermes", - configFile: "config.yaml", - envFile: ".env", - format: "yaml", - }, - inferenceProviderOptions: [], - stateDirs: [], - stateFiles: [], - userManagedFiles: [], - versionCommand: "hermes --version", - expectedVersion: "2026.4.30", - hasDevicePairing: false, - phoneHomeHosts: [], - dockerfileBasePath: "/test/root/agents/hermes/Dockerfile.base", - dockerfilePath: "/test/root/agents/hermes/Dockerfile", - startScriptPath: null, - policyAdditionsPath: null, - policyPermissivePath: null, - pluginDir: null, - legacyPaths: null, - agentDir: "/repo/root/agents/hermes", - manifestPath: "/repo/root/agents/hermes/manifest.yaml", - ...overrides, - }; -} - -/** - * Load `agent-onboard` with Docker helpers replaced by Vitest mocks. - */ -function withMockedDocker( - run: (deps: { - ensureAgentBaseImage: AgentOnboardModule["ensureAgentBaseImage"]; - dockerBuildMock: ReturnType; - dockerImageInspectMock: ReturnType; - resolveSandboxBaseImageMock: ReturnType; - root: string; - }) => T, -): T { - // eslint-disable-next-line @typescript-eslint/no-require-imports - const dockerImageModule = require("../adapters/docker/image") as DockerImageModule; - // eslint-disable-next-line @typescript-eslint/no-require-imports - const dockerInspectModule = require("../adapters/docker/inspect") as DockerInspectModule; - // eslint-disable-next-line @typescript-eslint/no-require-imports - const sandboxBaseImageModule = require("../sandbox-base-image") as SandboxBaseImageModule; - // eslint-disable-next-line @typescript-eslint/no-require-imports - const runnerModule = require("../runner") as { ROOT: string }; - const originalDockerBuild = dockerImageModule.dockerBuild; - const originalDockerImageInspect = dockerInspectModule.dockerImageInspect; - const originalResolveSandboxBaseImage = sandboxBaseImageModule.resolveSandboxBaseImage; - const agentOnboardModulePath = require.resolve("./onboard"); - delete require.cache[agentOnboardModulePath]; - - const dockerBuildMock = vi.fn().mockReturnValue({ status: 0 }); - const dockerImageInspectMock = vi.fn(); - const resolveSandboxBaseImageMock = vi.fn().mockReturnValue({ - ref: "ghcr.io/nvidia/nemoclaw/hermes-sandbox-base:compatible", - digest: null, - source: "source-sha", - glibcVersion: process.platform === "linux" ? "2.41" : null, - }); - dockerImageModule.dockerBuild = dockerBuildMock as DockerImageModule["dockerBuild"]; - dockerInspectModule.dockerImageInspect = - dockerImageInspectMock as DockerInspectModule["dockerImageInspect"]; - sandboxBaseImageModule.resolveSandboxBaseImage = - resolveSandboxBaseImageMock as SandboxBaseImageModule["resolveSandboxBaseImage"]; - - try { - // eslint-disable-next-line @typescript-eslint/no-require-imports - const agentOnboardModule = require("./onboard") as AgentOnboardModule; - return run({ - ensureAgentBaseImage: agentOnboardModule.ensureAgentBaseImage, - dockerBuildMock, - dockerImageInspectMock, - resolveSandboxBaseImageMock, - root: runnerModule.ROOT, - }); - } finally { - dockerImageModule.dockerBuild = originalDockerBuild; - dockerInspectModule.dockerImageInspect = originalDockerImageInspect; - sandboxBaseImageModule.resolveSandboxBaseImage = originalResolveSandboxBaseImage; - delete require.cache[agentOnboardModulePath]; - } -} + +import { makeAgent, withMockedDocker } from "../../../test/helpers/base-image-test-harness"; describe("agent base image provisioning", () => { beforeEach(() => { @@ -128,7 +22,7 @@ describe("agent base image provisioning", () => { const result = ensureAgentBaseImage(makeAgent()); expect(result).toEqual({ - imageTag: "ghcr.io/nvidia/nemoclaw/hermes-sandbox-base:compatible", + imageTag: "nemoclaw-hermes-sandbox-base-local:compatible", built: false, }); expect(resolveSandboxBaseImageMock).toHaveBeenCalledWith( @@ -139,6 +33,8 @@ describe("agent base image provisioning", () => { label: "Hermes Agent sandbox base image", requireOpenshellSandboxAbi: process.platform === "linux", rootDir: root, + validateImage: expect.any(Function), + validationDescription: "the required MCP Streamable HTTP runtime", }), ); expect(dockerImageInspectMock).not.toHaveBeenCalled(); @@ -152,7 +48,10 @@ describe("agent base image provisioning", () => { ({ ensureAgentBaseImage, dockerBuildMock, + dockerImageInspectFormatMock, dockerImageInspectMock, + dockerRmiMock, + dockerTagMock, resolveSandboxBaseImageMock, root, }) => { @@ -160,18 +59,38 @@ describe("agent base image provisioning", () => { const result = ensureAgentBaseImage(makeAgent(), { forceBaseImageRebuild: true }); - expect(result).toEqual({ - imageTag: "ghcr.io/nvidia/nemoclaw/hermes-sandbox-base:latest", - built: true, - }); - expect(resolveSandboxBaseImageMock).not.toHaveBeenCalled(); + expect(result.imageTag).toBe(`nemoclaw-hermes-sandbox-base-local:image-${"a".repeat(64)}`); + expect(result.built).toBe(true); + expect(resolveSandboxBaseImageMock).toHaveBeenCalledWith( + expect.objectContaining({ + localTag: result.imageTag, + env: expect.objectContaining({ + NEMOCLAW_HERMES_SANDBOX_BASE_IMAGE_REF: result.imageTag, + NEMOCLAW_SANDBOX_BASE_LOCAL_BUILD: "0", + }), + }), + ); expect(dockerImageInspectMock).not.toHaveBeenCalled(); expect(dockerBuildMock).toHaveBeenCalledWith( "/test/root/agents/hermes/Dockerfile.base", - "ghcr.io/nvidia/nemoclaw/hermes-sandbox-base:latest", + expect.stringMatching(/^nemoclaw-hermes-sandbox-base-local:build-\d+-[0-9a-f]{16}$/), root, { ignoreError: true, stdio: ["ignore", "inherit", "inherit"] }, ); + expect(dockerImageInspectFormatMock).toHaveBeenCalledWith( + "{{.Id}}", + expect.stringMatching(/^nemoclaw-hermes-sandbox-base-local:build-\d+-[0-9a-f]{16}$/), + { ignoreError: true }, + ); + expect(dockerTagMock).toHaveBeenCalledWith( + expect.stringMatching(/^nemoclaw-hermes-sandbox-base-local:build-\d+-[0-9a-f]{16}$/), + result.imageTag, + { ignoreError: true }, + ); + expect(dockerRmiMock).toHaveBeenCalledWith( + expect.stringMatching(/^nemoclaw-hermes-sandbox-base-local:build-\d+-[0-9a-f]{16}$/), + { ignoreError: true, suppressOutput: true }, + ); }, ); }); @@ -187,29 +106,58 @@ describe("agent base image provisioning", () => { }); }); - it("builds an agent base image when no resolved image or cached image exists on non-Linux hosts", () => { + it("pins different image IDs to different recreate refs at the same source revision", () => { withMockedDocker( - ({ - ensureAgentBaseImage, - dockerBuildMock, - dockerImageInspectMock, - resolveSandboxBaseImageMock, - }) => { - resolveSandboxBaseImageMock.mockReturnValue(null); - dockerImageInspectMock.mockReturnValue({ status: 1 }); + ({ ensureAgentBaseImage, dockerImageInspectFormatMock, resolveSandboxBaseImageMock }) => { + dockerImageInspectFormatMock + .mockReturnValueOnce(`sha256:${"a".repeat(64)}`) + .mockReturnValueOnce(`sha256:${"b".repeat(64)}`); + resolveSandboxBaseImageMock.mockImplementation((options) => ({ + ref: options.env?.[options.envVar], + digest: null, + source: "override", + glibcVersion: "2.41", + })); + + const first = ensureAgentBaseImage(makeAgent(), { forceBaseImageRebuild: true }); + const second = ensureAgentBaseImage(makeAgent(), { forceBaseImageRebuild: true }); + + expect(first.imageTag).toBe(`nemoclaw-hermes-sandbox-base-local:image-${"a".repeat(64)}`); + expect(second.imageTag).toBe(`nemoclaw-hermes-sandbox-base-local:image-${"b".repeat(64)}`); + }, + ); + }); - if (process.platform === "linux") { - expect(() => ensureAgentBaseImage(makeAgent())).toThrow( - "No compatible Hermes Agent sandbox base image found", - ); - expect(dockerBuildMock).not.toHaveBeenCalled(); - return; - } + it("canonicalizes a mutable local override to its full image-ID ref", () => { + withMockedDocker( + ({ pinAgentSandboxBaseImageRef, dockerImageInspectFormatMock, dockerTagMock }) => { + dockerImageInspectFormatMock.mockReturnValue(`sha256:${"c".repeat(64)}`); - const result = ensureAgentBaseImage(makeAgent()); + const pinned = pinAgentSandboxBaseImageRef( + "hermes", + "nemoclaw-hermes-sandbox-base-local:caller", + ); - expect(result.built).toBe(true); - expect(dockerBuildMock).toHaveBeenCalledOnce(); + expect(pinned).toBe(`nemoclaw-hermes-sandbox-base-local:image-${"c".repeat(64)}`); + expect(dockerTagMock).toHaveBeenCalledWith( + "nemoclaw-hermes-sandbox-base-local:caller", + pinned, + { ignoreError: true }, + ); + }, + ); + }); + + it("does not trust a moved image-ID-shaped tag without inspecting it", () => { + withMockedDocker( + ({ pinAgentSandboxBaseImageRef, dockerImageInspectFormatMock, dockerTagMock }) => { + const claimed = `nemoclaw-hermes-sandbox-base-local:image-${"a".repeat(64)}`; + dockerImageInspectFormatMock.mockReturnValue(`sha256:${"d".repeat(64)}`); + + const pinned = pinAgentSandboxBaseImageRef("hermes", claimed); + + expect(pinned).toBe(`nemoclaw-hermes-sandbox-base-local:image-${"d".repeat(64)}`); + expect(dockerTagMock).toHaveBeenCalledWith(claimed, pinned, { ignoreError: true }); }, ); }); diff --git a/src/lib/agent/base-image.ts b/src/lib/agent/base-image.ts new file mode 100644 index 00000000000..db7bc02a72a --- /dev/null +++ b/src/lib/agent/base-image.ts @@ -0,0 +1,263 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import crypto from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { + dockerBuild, + dockerCapture, + dockerImageInspect, + dockerImageInspectFormat, + dockerRmi, + dockerTag, +} from "../adapters/docker"; +import { ROOT } from "../runner"; +import { + buildLocalBaseTag, + resolveSandboxBaseImage, + SANDBOX_BASE_TAG, +} from "../sandbox-base-image"; +import type { AgentDefinition } from "./defs"; + +const HERMES_MCP_RUNTIME_PROBE_OK = "nemoclaw-hermes-mcp-runtime-ok"; + +export function getAgentSandboxBaseImageEnvVar(agentName: string): string { + return `NEMOCLAW_${agentName.toUpperCase().replace(/[^A-Z0-9]+/g, "_")}_SANDBOX_BASE_IMAGE_REF`; +} + +function immutableLocalBaseImageTag(agentName: string, imageId: string): string { + const match = imageId.trim().match(/^sha256:([0-9a-f]{64})$/i); + if (!match) { + throw new Error(`Docker returned an invalid image ID for ${agentName} base image`); + } + return `nemoclaw-${agentName}-sandbox-base-local:image-${match[1].toLowerCase()}`; +} + +export function pinAgentSandboxBaseImageRef(agentName: string, imageRef: string): string { + if (imageRef.includes("@sha256:")) return imageRef; + const imageId = dockerImageInspectFormat("{{.Id}}", imageRef, { ignoreError: true }); + const pinnedRef = immutableLocalBaseImageTag(agentName, imageId); + if (imageRef === pinnedRef) return pinnedRef; + const tagResult = dockerTag(imageRef, pinnedRef, { ignoreError: true }); + if (tagResult.error || tagResult.status !== 0) { + const detail = tagResult.error + ? `: ${tagResult.error.message}` + : ` (exit ${tagResult.status ?? "unknown"})`; + throw new Error(`Failed to pin ${agentName} base image${detail}`); + } + return pinnedRef; +} + +function hermesFinalDockerfileAcceptsBase(agent: AgentDefinition, imageRef: string): boolean { + if (agent.name !== "hermes") return true; + if ( + imageRef === "nemoclaw-hermes-base-local" || + /^nemoclaw-hermes-(?:root-entrypoint-base|sandbox-base-local|secret-boundary-base|stale-openclaw-dir-base|stale-openclaw-link-base):[^\s]+$/.test( + imageRef, + ) + ) { + return true; + } + if (!imageRef.startsWith("ghcr.io/nvidia/nemoclaw/hermes-sandbox-base@sha256:")) return false; + const finalDockerfile = agent.dockerfilePath; + if (!finalDockerfile) return false; + let dockerfile: string; + try { + dockerfile = fs.readFileSync(finalDockerfile, "utf8"); + } catch { + return false; + } + const declarations = [...dockerfile.matchAll(/^ARG BASE_IMAGE=(\S+)$/gm)].map( + (match) => match[1], + ); + return ( + declarations.length === 1 && + /^ghcr\.io\/nvidia\/nemoclaw\/hermes-sandbox-base@sha256:[0-9a-f]{64}$/.test( + declarations[0] ?? "", + ) && + imageRef === declarations[0] + ); +} + +/** + * Verify that a Hermes base contains both the MCP SDK and Hermes' native + * Streamable HTTP integration. Version output alone is insufficient because + * these dependencies are installed through an optional upstream extra. + */ +export function hermesBaseImageSupportsMcp(imageRef: string): boolean { + const output = dockerCapture( + [ + "run", + "--rm", + "--entrypoint", + "/opt/hermes/.venv/bin/python", + imageRef, + "-c", + `import mcp; from tools import mcp_tool; assert getattr(mcp_tool, "_MCP_AVAILABLE", False); assert getattr(mcp_tool, "_MCP_HTTP_AVAILABLE", False); print("${HERMES_MCP_RUNTIME_PROBE_OK}")`, + ], + { ignoreError: true, timeout: 20_000 }, + ); + return output.trim() === HERMES_MCP_RUNTIME_PROBE_OK; +} + +/** + * Ensure the agent-specific sandbox base image exists locally. + * Rebuild callers can force this so local Dockerfile.base edits are applied. + */ +export function ensureAgentBaseImage( + agent: AgentDefinition, + opts: { forceBaseImageRebuild?: boolean } = {}, +): { + imageTag: string | null; + built: boolean; +} { + const baseDockerfile = agent.dockerfileBasePath; + + if (!baseDockerfile) { + return { imageTag: null, built: false }; + } + + const baseImageName = `ghcr.io/nvidia/nemoclaw/${agent.name}-sandbox-base`; + const baseImageTag = `${baseImageName}:${SANDBOX_BASE_TAG}`; + const localBaseImageTag = buildLocalBaseTag(`nemoclaw-${agent.name}-sandbox-base-local`, ROOT); + const overrideEnvVar = getAgentSandboxBaseImageEnvVar(agent.name); + const validateImage = agent.name === "hermes" ? hermesBaseImageSupportsMcp : undefined; + const validationDescription = + agent.name === "hermes" ? "the required MCP Streamable HTTP runtime" : undefined; + const resolutionOptions = { + imageName: baseImageName, + dockerfilePath: baseDockerfile, + localTag: localBaseImageTag, + envVar: overrideEnvVar, + label: `${agent.displayName} sandbox base image`, + requireOpenshellSandboxAbi: process.platform === "linux", + rootDir: ROOT, + validateImage, + validationDescription, + }; + const resolveExactImage = (imageRef: string) => + resolveSandboxBaseImage({ + ...resolutionOptions, + localTag: imageRef, + env: { + ...process.env, + [overrideEnvVar]: imageRef, + NEMOCLAW_SANDBOX_BASE_LOCAL_BUILD: "0", + }, + }); + const forceBaseImageRebuild = opts.forceBaseImageRebuild === true; + if (forceBaseImageRebuild) { + const forceBuildTag = `nemoclaw-${agent.name}-sandbox-base-local:build-${process.pid}-${crypto.randomBytes(8).toString("hex")}`; + console.log(` Rebuilding ${agent.displayName} base image...`); + const buildResult = dockerBuild(baseDockerfile, forceBuildTag, ROOT, { + ignoreError: true, + stdio: ["ignore", "inherit", "inherit"], + }); + if (buildResult.error || buildResult.status !== 0) { + dockerRmi(forceBuildTag, { ignoreError: true, suppressOutput: true }); + const detail = buildResult.error + ? `: ${buildResult.error.message}` + : ` (exit ${buildResult.status ?? "unknown"})`; + throw new Error(`Failed to build ${agent.displayName} base image${detail}`); + } + try { + const pinnedBaseImageTag = pinAgentSandboxBaseImageRef(agent.name, forceBuildTag); + const resolved = resolveExactImage(pinnedBaseImageTag); + if (!resolved) { + throw new Error( + `Built ${agent.displayName} base image failed the required runtime compatibility checks`, + ); + } + if (!hermesFinalDockerfileAcceptsBase(agent, pinnedBaseImageTag)) { + throw new Error( + `Hermes final image does not accept base image ref '${pinnedBaseImageTag}'; use the tracked official digest or a repository-built local base`, + ); + } + console.log(` \u2713 Base image built: ${pinnedBaseImageTag}`); + return { imageTag: pinnedBaseImageTag, built: true }; + } finally { + dockerRmi(forceBuildTag, { ignoreError: true, suppressOutput: true }); + } + } + + const explicitOverride = process.env[overrideEnvVar]?.trim(); + const resolved = explicitOverride + ? resolveExactImage(explicitOverride) + : resolveSandboxBaseImage(resolutionOptions); + if (resolved && !forceBaseImageRebuild) { + if (!hermesFinalDockerfileAcceptsBase(agent, resolved.ref)) { + throw new Error( + `Hermes final image does not accept base image ref '${resolved.ref}'; use the tracked official digest or a repository-built local base`, + ); + } + console.log(` Using ${agent.displayName} base image: ${resolved.ref}`); + return { imageTag: resolved.ref, built: false }; + } + if (!resolved && (process.platform === "linux" || validateImage) && !forceBaseImageRebuild) { + throw new Error( + `No compatible ${agent.displayName} sandbox base image found for ${baseImageName}`, + ); + } + const inspectResult = dockerImageInspect(baseImageTag, { + ignoreError: true, + suppressOutput: true, + }); + if (inspectResult?.status !== 0) { + console.log(` Building ${agent.displayName} base image (first time only)...`); + const buildResult = dockerBuild(baseDockerfile, baseImageTag, ROOT, { + ignoreError: true, + stdio: ["ignore", "inherit", "inherit"], + }); + if (buildResult.error || buildResult.status !== 0) { + const detail = buildResult.error + ? `: ${buildResult.error.message}` + : ` (exit ${buildResult.status ?? "unknown"})`; + throw new Error(`Failed to build ${agent.displayName} base image${detail}`); + } + console.log(` \u2713 Base image built: ${baseImageTag}`); + return { imageTag: baseImageTag, built: true }; + } + + console.log(` Base image exists: ${baseImageTag}`); + return { imageTag: baseImageTag, built: false }; +} + +/** Stage build context for an agent-specific sandbox image. */ +export function createAgentSandbox( + agent: AgentDefinition, + opts: { forceBaseImageRebuild?: boolean } = {}, +): { + buildCtx: string; + stagedDockerfile: string; +} { + const agentDockerfile = agent.dockerfilePath; + + if (!agentDockerfile) { + throw new Error(`${agent.displayName} is missing a sandbox Dockerfile`); + } + + const { imageTag: baseImageRef } = ensureAgentBaseImage(agent, opts); + const buildCtx = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-build-")); + fs.cpSync(ROOT, buildCtx, { + recursive: true, + filter: (src) => { + const base = path.basename(src); + return !["node_modules", ".git", ".venv", "__pycache__", ".claude"].includes(base); + }, + }); + const stagedDockerfile = path.join(buildCtx, "Dockerfile"); + fs.copyFileSync(agentDockerfile, stagedDockerfile); + if (baseImageRef) { + const dockerfile = fs.readFileSync(stagedDockerfile, "utf8"); + fs.writeFileSync( + stagedDockerfile, + dockerfile.replace(/^ARG BASE_IMAGE(?:=.*)?$/m, `ARG BASE_IMAGE=${baseImageRef}`), + ); + } + console.log(` Using ${agent.displayName} Dockerfile: ${agentDockerfile}`); + + return { buildCtx, stagedDockerfile }; +} diff --git a/src/lib/agent/definition-types.ts b/src/lib/agent/definition-types.ts new file mode 100644 index 00000000000..6d80ec25430 --- /dev/null +++ b/src/lib/agent/definition-types.ts @@ -0,0 +1,120 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { AgentDashboardUi } from "./dashboard-ui"; +import type { AgentRuntime } from "./runtime-manifest"; +import type { AgentWebAuth } from "./web-auth"; + +export type ManifestScalar = string | number | boolean | null | Date; +export type ManifestValue = ManifestScalar | ManifestRecord | ManifestValue[]; +export type ManifestRecord = { [key: string]: ManifestValue }; +export type StringMap = { [key: string]: string }; + +export interface AgentHealthProbe { + url: string; + port: number; + timeout_seconds: number; +} + +export interface AgentConfigPaths { + dir: string; + configFile: string; + envFile: string | null; + format: string; +} + +export type AgentStateFileStrategy = "copy" | "sqlite_backup"; + +export interface AgentStateFile { + path: string; + strategy: AgentStateFileStrategy; +} + +export type AgentDashboardKind = "ui" | "api"; + +export interface AgentDashboard { + kind: AgentDashboardKind; + label: string; + path: string; + healthPath: string; + auth: "url_token" | "session" | "none"; +} + +export interface AgentInference { + provider_type?: string; + provider_options?: string[]; +} + +export type AgentMcpSupport = "bridge" | "disabled"; +export type AgentMcpAdapter = "mcporter" | "hermes-config" | "deepagents-config"; + +export interface AgentMcpCapability { + support: AgentMcpSupport; + adapter?: AgentMcpAdapter; + reason?: string; +} + +export interface AgentLegacyPaths { + dockerfileBase: string | null; + dockerfile: string | null; + startScript: string | null; + policy: string | null; + plugin: string | null; +} + +export type AgentVersionScheme = "semver" | "calendar"; + +export interface AgentDefinition { + name: string; + description?: string; + display_name?: string; + binary_path?: string; + version_command?: string; + expected_version?: string; + version_scheme?: AgentVersionScheme; + gateway_command?: string; + runtime?: AgentRuntime; + device_pairing?: boolean; + phone_home_hosts?: string[]; + forward_ports?: number[]; + health_probe?: AgentHealthProbe; + config?: ManifestRecord; + inference?: AgentInference; + mcp?: AgentMcpCapability; + state_dirs?: string[]; + state_files?: AgentStateFile[]; + user_managed_files?: string[]; + _legacy_paths?: StringMap; + agentDir: string; + manifestPath: string; + readonly displayName: string; + readonly healthProbe: AgentHealthProbe | null; + readonly forwardPort: number; + readonly dashboard: AgentDashboard; + readonly webAuth: AgentWebAuth; + readonly dashboardUi?: AgentDashboardUi | null; + readonly configPaths: AgentConfigPaths; + readonly inferenceProviderOptions: string[]; + readonly mcpCapability: AgentMcpCapability; + readonly stateDirs: string[]; + readonly stateFiles: AgentStateFile[]; + readonly userManagedFiles: string[]; + readonly versionCommand: string; + readonly expectedVersion: string | null; + readonly versionScheme?: AgentVersionScheme | null; + readonly hasDevicePairing: boolean; + readonly phoneHomeHosts: string[]; + readonly dockerfileBasePath: string | null; + readonly dockerfilePath: string | null; + readonly startScriptPath: string | null; + readonly policyAdditionsPath: string | null; + readonly policyPermissivePath: string | null; + readonly pluginDir: string | null; + readonly legacyPaths: AgentLegacyPaths | null; +} + +export interface AgentChoice { + name: string; + displayName: string; + description: string; +} diff --git a/src/lib/agent/defs.test.ts b/src/lib/agent/defs.test.ts index ac8e722a018..5066d9ab069 100644 --- a/src/lib/agent/defs.test.ts +++ b/src/lib/agent/defs.test.ts @@ -50,6 +50,7 @@ describe("agent definitions", () => { format: "json", }); expect(openclaw.inferenceProviderOptions).toEqual([]); + expect(openclaw.mcpCapability).toEqual({ support: "bridge", adapter: "mcporter" }); // OpenClaw uses device_pairing web auth — no fetchable bearer token. expect(openclaw.webAuth).toEqual({ method: "none", env: null }); // #5027: openclaw.json must be declared as a durable state file so @@ -73,6 +74,7 @@ describe("agent definitions", () => { format: "yaml", }); expect(hermes.inferenceProviderOptions).toEqual(["hermesProvider"]); + expect(hermes.mcpCapability).toEqual({ support: "bridge", adapter: "hermes-config" }); expect(hermes.healthProbe?.url).toBe("http://localhost:8642/health"); expect(hermes.forwardPort).toBe(18789); expect(hermes.forward_ports).toEqual([18789, 8642]); @@ -123,10 +125,14 @@ describe("agent definitions", () => { format: "toml", }); expect(deepAgentsCode.inference?.provider_type).toBe("openai_compatible"); + expect(deepAgentsCode.mcpCapability).toEqual({ + support: "bridge", + adapter: "deepagents-config", + }); expect(deepAgentsCode.stateDirs).toEqual([".state", "skills", "agent/skills"]); expect(deepAgentsCode.stateFiles).toEqual([{ path: "config.toml", strategy: "copy" }]); expect(deepAgentsCode.stateFiles.map((entry) => entry.path)).not.toContain(".env"); - expect(deepAgentsCode.userManagedFiles).toEqual([".env", ".mcp.json"]); + expect(deepAgentsCode.userManagedFiles).toEqual([".deepagents/.env", ".deepagents/.mcp.json"]); }); it("orders OpenClaw first in interactive choices", () => { @@ -291,6 +297,34 @@ describe("agent definitions", () => { expect(() => loadAgent(agentName)).toThrow(/inference\.provider_type/); }); + it("rejects invalid MCP bridge adapter declarations in manifests", () => { + const agentName = `invalid-mcp-adapter-${String(Date.now())}`; + writeTempAgentManifest( + agentName, + [ + `name: ${agentName}`, + "display_name: Broken MCP", + "mcp:", + " support: bridge", + " adapter: unsupported-adapter", + ].join("\n"), + ); + + expect(() => loadAgent(agentName)).toThrow(/mcp\.adapter/); + }); + + it("requires an MCP adapter when bridge support is declared", () => { + const agentName = `missing-mcp-adapter-${String(Date.now())}`; + writeTempAgentManifest( + agentName, + [`name: ${agentName}`, "display_name: Missing MCP Adapter", "mcp:", " support: bridge"].join( + "\n", + ), + ); + + expect(() => loadAgent(agentName)).toThrow(/mcp\.adapter/); + }); + it("loads terminal runtime manifests without OpenClaw gateway defaults", () => { const agentName = `terminal-agent-${String(Date.now())}`; writeTempAgentManifest( diff --git a/src/lib/agent/defs.ts b/src/lib/agent/defs.ts index 61a63c5b3ca..c8b7dc21c6c 100644 --- a/src/lib/agent/defs.ts +++ b/src/lib/agent/defs.ts @@ -1,8 +1,10 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 // -// Agent definition loader — reads agents/*/manifest.yaml and provides -// accessors for agent-specific configuration used during onboarding. +// Agent definition loader — each agent's definition already lives in its +// agents/*/manifest.yaml. This facade scans those per-agent files and builds +// the stable derived accessors used during onboarding; schema types and +// validation readers stay in focused sibling modules. import fs from "node:fs"; import path from "node:path"; @@ -13,120 +15,58 @@ import { resolveAgentNameAlias as resolveKnownAgentNameAlias, } from "./aliases"; import { type AgentDashboardUi, readDashboardUi } from "./dashboard-ui"; +import type { + AgentChoice, + AgentConfigPaths, + AgentDashboard, + AgentDefinition, + AgentHealthProbe, + AgentLegacyPaths, + AgentMcpCapability, + AgentStateFile, + AgentVersionScheme, +} from "./definition-types"; +import { + loadManifestRecord, + readBoolean, + readDashboard, + readHealthProbe, + readInference, + readMcpCapability, + readObject, + readPortArray, + readStateFiles, + readString, + readStringArray, + readStringMap, + readUserManagedFiles, + readVersionScheme, +} from "./manifest-readers"; import { type AgentRuntime, readAgentRuntime } from "./runtime-manifest"; import { type AgentWebAuth, readWebAuth } from "./web-auth"; +export type { + AgentChoice, + AgentConfigPaths, + AgentDashboard, + AgentDashboardKind, + AgentDefinition, + AgentHealthProbe, + AgentInference, + AgentLegacyPaths, + AgentMcpAdapter, + AgentMcpCapability, + AgentMcpSupport, + AgentStateFile, + AgentStateFileStrategy, + AgentVersionScheme, +} from "./definition-types"; export type { AgentRuntime, AgentRuntimeKind } from "./runtime-manifest"; export { getAgentRuntimeKind, isTerminalAgent } from "./runtime-manifest"; export type { AgentWebAuth, AgentWebAuthMethod } from "./web-auth"; export const AGENTS_DIR = path.join(ROOT, "agents"); -type ManifestScalar = string | number | boolean | null | Date; -type ManifestValue = ManifestScalar | ManifestRecord | ManifestValue[]; -type ManifestRecord = { [key: string]: ManifestValue }; -type StringMap = { [key: string]: string }; - -const yaml: { load(input: string): unknown } = require("js-yaml"); - -export interface AgentHealthProbe { - url: string; - port: number; - timeout_seconds: number; -} - -export interface AgentConfigPaths { - dir: string; - configFile: string; - envFile: string | null; - format: string; -} - -export type AgentStateFileStrategy = "copy" | "sqlite_backup"; - -export interface AgentStateFile { - path: string; - strategy: AgentStateFileStrategy; -} - -export type AgentDashboardKind = "ui" | "api"; - -export interface AgentDashboard { - kind: AgentDashboardKind; - label: string; - path: string; - healthPath: string; - auth: "url_token" | "session" | "none"; -} - -export interface AgentInference { - provider_type?: string; - provider_options?: string[]; -} - -export interface AgentLegacyPaths { - dockerfileBase: string | null; - dockerfile: string | null; - startScript: string | null; - policy: string | null; - plugin: string | null; -} - -export type AgentVersionScheme = "semver" | "calendar"; - -export interface AgentDefinition { - name: string; - description?: string; - display_name?: string; - binary_path?: string; - version_command?: string; - expected_version?: string; - version_scheme?: AgentVersionScheme; - gateway_command?: string; - runtime?: AgentRuntime; - device_pairing?: boolean; - phone_home_hosts?: string[]; - forward_ports?: number[]; - health_probe?: AgentHealthProbe; - config?: ManifestRecord; - inference?: AgentInference; - state_dirs?: string[]; - state_files?: AgentStateFile[]; - user_managed_files?: string[]; - _legacy_paths?: StringMap; - agentDir: string; - manifestPath: string; - readonly displayName: string; - readonly healthProbe: AgentHealthProbe | null; - readonly forwardPort: number; - readonly dashboard: AgentDashboard; - readonly webAuth: AgentWebAuth; - readonly dashboardUi?: AgentDashboardUi | null; - readonly configPaths: AgentConfigPaths; - readonly inferenceProviderOptions: string[]; - readonly stateDirs: string[]; - readonly stateFiles: AgentStateFile[]; - readonly userManagedFiles: string[]; - readonly versionCommand: string; - readonly expectedVersion: string | null; - readonly versionScheme?: AgentVersionScheme | null; - readonly hasDevicePairing: boolean; - readonly phoneHomeHosts: string[]; - readonly dockerfileBasePath: string | null; - readonly dockerfilePath: string | null; - readonly startScriptPath: string | null; - readonly policyAdditionsPath: string | null; - readonly policyPermissivePath: string | null; - readonly pluginDir: string | null; - readonly legacyPaths: AgentLegacyPaths | null; -} - -export interface AgentChoice { - name: string; - displayName: string; - description: string; -} - const _cache = new Map(); export { agentAliasSummary } from "./aliases"; @@ -148,268 +88,6 @@ function unknownAgentMessage( return `Unknown agent '${value}'${suffix}. Available: ${choices}${formatAgentAliasSuffix(available)}`; } -function isManifestValue(value: unknown): value is ManifestValue { - if (value === null || value instanceof Date) return true; - if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") { - return true; - } - if (Array.isArray(value)) { - return value.every((entry) => isManifestValue(entry)); - } - return isManifestRecord(value); -} - -function isManifestRecord(value: unknown): value is ManifestRecord { - if (typeof value !== "object" || value === null || Array.isArray(value)) { - return false; - } - - const prototype = Object.getPrototypeOf(value); - if (prototype !== Object.prototype && prototype !== null) { - return false; - } - - return Object.values(value).every((entry) => isManifestValue(entry)); -} - -function readString(record: ManifestRecord, key: string): string | undefined { - const value = record[key]; - return typeof value === "string" ? value : undefined; -} - -function readBoolean(record: ManifestRecord, key: string): boolean | undefined { - const value = record[key]; - return typeof value === "boolean" ? value : undefined; -} - -function readVersionScheme(record: ManifestRecord): AgentVersionScheme | undefined { - const value = record.version_scheme; - if (value === "semver" || value === "calendar") return value; - return undefined; -} - -function readObject(record: ManifestRecord, key: string): ManifestRecord | undefined { - const value = record[key]; - return isManifestRecord(value) ? value : undefined; -} - -function readStringArray(record: ManifestRecord, key: string): string[] | undefined { - const value = record[key]; - if (!Array.isArray(value)) return undefined; - return value.filter((entry): entry is string => typeof entry === "string"); -} - -const CONTROL_CHAR_RE = /[\x00-\x1f\x7f]/; - -function readUserManagedFiles(record: ManifestRecord): string[] | undefined { - const value = record.user_managed_files; - if (value === undefined) return undefined; - if (!Array.isArray(value)) { - throw new Error("Agent manifest field 'user_managed_files' must be an array"); - } - - return value.map((entry, index) => { - if (typeof entry !== "string") { - throw new Error( - `Agent manifest field 'user_managed_files[${String(index)}]' must be a string`, - ); - } - if (entry.length === 0) { - throw new Error( - `Agent manifest field 'user_managed_files[${String(index)}]' must not be empty`, - ); - } - if (CONTROL_CHAR_RE.test(entry)) { - throw new Error( - `Agent manifest field 'user_managed_files[${String(index)}]' must not contain control characters`, - ); - } - if (entry.startsWith("/")) { - throw new Error( - `Agent manifest field 'user_managed_files[${String(index)}]' must be a relative path, not absolute`, - ); - } - const segments = entry.split("/"); - if (segments.some((segment) => segment === "..")) { - throw new Error( - `Agent manifest field 'user_managed_files[${String(index)}]' must not contain '..' path components`, - ); - } - return entry; - }); -} - -function readStateFiles(record: ManifestRecord): AgentStateFile[] | undefined { - const value = record.state_files; - if (value === undefined) return undefined; - if (!Array.isArray(value)) { - throw new Error("Agent manifest field 'state_files' must be an array"); - } - - return value.map((entry, index) => { - if (typeof entry === "string") { - return { path: entry, strategy: "copy" }; - } - if (!isManifestRecord(entry)) { - throw new Error( - `Agent manifest field 'state_files[${String(index)}]' must be a string or object`, - ); - } - const statePath = readString(entry, "path"); - if (!statePath) { - throw new Error(`Agent manifest field 'state_files[${String(index)}].path' is required`); - } - const rawStrategy = readString(entry, "strategy") ?? "copy"; - if (rawStrategy !== "copy" && rawStrategy !== "sqlite_backup") { - throw new Error( - `Agent manifest field 'state_files[${String(index)}].strategy' must be copy or sqlite_backup`, - ); - } - return { path: statePath, strategy: rawStrategy }; - }); -} - -function isValidPort(value: unknown, min = 1): value is number { - return typeof value === "number" && Number.isInteger(value) && value >= min && value <= 65535; -} - -function readPortArray(record: ManifestRecord, key: string): number[] | undefined { - const value = record[key]; - if (value === undefined) return undefined; - if (!Array.isArray(value)) { - throw new Error(`Agent manifest field '${key}' must be an array of TCP ports`); - } - - const ports = value.map((entry, index) => { - if (!isValidPort(entry, 1024)) { - throw new Error( - `Agent manifest field '${key}[${String(index)}]' must be an integer TCP port between 1024 and 65535`, - ); - } - return entry; - }); - - return ports.length > 0 ? ports : undefined; -} - -function readStringMap(record: ManifestRecord, key: string): StringMap | undefined { - const value = readObject(record, key); - if (!value) return undefined; - - const result: StringMap = {}; - for (const [entryKey, entryValue] of Object.entries(value)) { - if (typeof entryValue === "string") { - result[entryKey] = entryValue; - } - } - return result; -} - -function readHealthProbe(record: ManifestRecord): AgentHealthProbe | undefined { - const healthProbe = readObject(record, "health_probe"); - if (!healthProbe) return undefined; - - const url = readString(healthProbe, "url"); - const port = healthProbe.port; - const timeoutSeconds = healthProbe.timeout_seconds; - - if (port !== undefined && !isValidPort(port)) { - throw new Error( - "Agent manifest field 'health_probe.port' must be an integer TCP port between 1 and 65535", - ); - } - - if ( - typeof url === "string" && - isValidPort(port) && - typeof timeoutSeconds === "number" && - Number.isFinite(timeoutSeconds) - ) { - return { - url, - port, - timeout_seconds: timeoutSeconds, - }; - } - - return undefined; -} - -function readDashboard(record: ManifestRecord): AgentDashboard { - const d = readObject(record, "dashboard") ?? {}; - const rawKind = d.kind; - if (rawKind !== undefined && rawKind !== "ui" && rawKind !== "api") { - throw new Error("Agent manifest field 'dashboard.kind' must be ui or api"); - } - const kind: AgentDashboardKind = rawKind === "api" ? "api" : "ui"; - const defaultLabel = kind === "api" ? "API" : "UI"; - const normalizedLabel = typeof d.label === "string" ? d.label.trim() : ""; - - const normalizePath = (key: "path" | "health_path", fallback: string): string => { - const value = d[key]; - if (value === undefined) return fallback; - if (typeof value !== "string" || !value.startsWith("/")) { - throw new Error(`Agent manifest field 'dashboard.${key}' must be an absolute path`); - } - return value.trim() || fallback; - }; - - const rawAuth = d.auth; - if ( - rawAuth !== undefined && - rawAuth !== "url_token" && - rawAuth !== "session" && - rawAuth !== "none" - ) { - throw new Error("Agent manifest field 'dashboard.auth' must be url_token, session, or none"); - } - - return { - kind, - label: normalizedLabel || defaultLabel, - path: normalizePath("path", "/"), - healthPath: normalizePath("health_path", "/health"), - auth: rawAuth ?? (kind === "api" ? "none" : "url_token"), - }; -} - -function readInference(record: ManifestRecord): AgentInference | undefined { - const inference = readObject(record, "inference"); - if (!inference) return undefined; - - const providerType = inference.provider_type; - if (providerType !== undefined && typeof providerType !== "string") { - throw new Error("Agent manifest field 'inference.provider_type' must be a string"); - } - - const providerOptions = inference.provider_options; - let providerOptionList: string[] | undefined; - if (providerOptions !== undefined) { - if ( - !Array.isArray(providerOptions) || - providerOptions.some((entry) => typeof entry !== "string") - ) { - throw new Error( - "Agent manifest field 'inference.provider_options' must be an array of strings", - ); - } - providerOptionList = providerOptions as string[]; - } - - return { - provider_type: providerType, - provider_options: providerOptionList, - }; -} - -function loadManifestRecord(manifestPath: string): ManifestRecord { - const parsed = yaml.load(fs.readFileSync(manifestPath, "utf8")); - if (!isManifestRecord(parsed)) { - throw new Error(`Agent manifest must be a YAML object: ${manifestPath}`); - } - return parsed; -} - /** * List available agent names by scanning agents/ for directories with * a manifest.yaml file. @@ -453,6 +131,7 @@ export function loadAgent(name: string): AgentDefinition { const healthProbe = readHealthProbe(raw); const config = readObject(raw, "config"); const inference = readInference(raw); + const mcp = readMcpCapability(raw); const stateDirs = readStringArray(raw, "state_dirs"); const stateFiles = readStateFiles(raw); const userManagedFiles = readUserManagedFiles(raw); @@ -477,6 +156,7 @@ export function loadAgent(name: string): AgentDefinition { health_probe: healthProbe, config, inference, + mcp, state_dirs: stateDirs, state_files: stateFiles, user_managed_files: userManagedFiles, @@ -533,6 +213,10 @@ export function loadAgent(name: string): AgentDefinition { return inference?.provider_options ?? []; }, + get mcpCapability(): AgentMcpCapability { + return mcp; + }, + get stateDirs(): string[] { return stateDirs ?? []; }, diff --git a/src/lib/agent/hermes-recovery-boundary-fixtures.ts b/src/lib/agent/hermes-recovery-boundary-fixtures.ts index cd5e581081b..d844558e53c 100644 --- a/src/lib/agent/hermes-recovery-boundary-fixtures.ts +++ b/src/lib/agent/hermes-recovery-boundary-fixtures.ts @@ -25,6 +25,10 @@ export function makeAgent(overrides: Partial = {}): AgentDefini format: "yaml", }, inferenceProviderOptions: [], + mcpCapability: { + support: "disabled", + reason: "test fixture", + }, stateDirs: [], stateFiles: [], userManagedFiles: [], diff --git a/src/lib/agent/manifest-readers.ts b/src/lib/agent/manifest-readers.ts new file mode 100644 index 00000000000..3961eff3954 --- /dev/null +++ b/src/lib/agent/manifest-readers.ts @@ -0,0 +1,310 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import type { + AgentDashboard, + AgentDashboardKind, + AgentHealthProbe, + AgentInference, + AgentMcpCapability, + AgentStateFile, + AgentVersionScheme, + ManifestRecord, + ManifestValue, + StringMap, +} from "./definition-types"; + +const yaml: { load(input: string): unknown } = require("js-yaml"); + +function isManifestValue(value: unknown): value is ManifestValue { + if (value === null || value instanceof Date) return true; + if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") { + return true; + } + if (Array.isArray(value)) { + return value.every((entry) => isManifestValue(entry)); + } + return isManifestRecord(value); +} + +function isManifestRecord(value: unknown): value is ManifestRecord { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + return false; + } + + const prototype = Object.getPrototypeOf(value); + if (prototype !== Object.prototype && prototype !== null) { + return false; + } + + return Object.values(value).every((entry) => isManifestValue(entry)); +} + +export function readString(record: ManifestRecord, key: string): string | undefined { + const value = record[key]; + return typeof value === "string" ? value : undefined; +} + +export function readBoolean(record: ManifestRecord, key: string): boolean | undefined { + const value = record[key]; + return typeof value === "boolean" ? value : undefined; +} + +export function readVersionScheme(record: ManifestRecord): AgentVersionScheme | undefined { + const value = record.version_scheme; + if (value === "semver" || value === "calendar") return value; + return undefined; +} + +export function readObject(record: ManifestRecord, key: string): ManifestRecord | undefined { + const value = record[key]; + return isManifestRecord(value) ? value : undefined; +} + +export function readStringArray(record: ManifestRecord, key: string): string[] | undefined { + const value = record[key]; + if (!Array.isArray(value)) return undefined; + return value.filter((entry): entry is string => typeof entry === "string"); +} + +const CONTROL_CHAR_RE = /[\x00-\x1f\x7f]/; + +export function readUserManagedFiles(record: ManifestRecord): string[] | undefined { + const value = record.user_managed_files; + if (value === undefined) return undefined; + if (!Array.isArray(value)) { + throw new Error("Agent manifest field 'user_managed_files' must be an array"); + } + + return value.map((entry, index) => { + if (typeof entry !== "string") { + throw new Error( + `Agent manifest field 'user_managed_files[${String(index)}]' must be a string`, + ); + } + if (entry.length === 0) { + throw new Error( + `Agent manifest field 'user_managed_files[${String(index)}]' must not be empty`, + ); + } + if (CONTROL_CHAR_RE.test(entry)) { + throw new Error( + `Agent manifest field 'user_managed_files[${String(index)}]' must not contain control characters`, + ); + } + if (entry.startsWith("/")) { + throw new Error( + `Agent manifest field 'user_managed_files[${String(index)}]' must be a relative path, not absolute`, + ); + } + const segments = entry.split("/"); + if (segments.some((segment) => segment === "..")) { + throw new Error( + `Agent manifest field 'user_managed_files[${String(index)}]' must not contain '..' path components`, + ); + } + return entry; + }); +} + +export function readStateFiles(record: ManifestRecord): AgentStateFile[] | undefined { + const value = record.state_files; + if (value === undefined) return undefined; + if (!Array.isArray(value)) { + throw new Error("Agent manifest field 'state_files' must be an array"); + } + + return value.map((entry, index) => { + if (typeof entry === "string") { + return { path: entry, strategy: "copy" }; + } + if (!isManifestRecord(entry)) { + throw new Error( + `Agent manifest field 'state_files[${String(index)}]' must be a string or object`, + ); + } + const statePath = readString(entry, "path"); + if (!statePath) { + throw new Error(`Agent manifest field 'state_files[${String(index)}].path' is required`); + } + const rawStrategy = readString(entry, "strategy") ?? "copy"; + if (rawStrategy !== "copy" && rawStrategy !== "sqlite_backup") { + throw new Error( + `Agent manifest field 'state_files[${String(index)}].strategy' must be copy or sqlite_backup`, + ); + } + return { path: statePath, strategy: rawStrategy }; + }); +} + +function isValidPort(value: unknown, min = 1): value is number { + return typeof value === "number" && Number.isInteger(value) && value >= min && value <= 65535; +} + +export function readPortArray(record: ManifestRecord, key: string): number[] | undefined { + const value = record[key]; + if (value === undefined) return undefined; + if (!Array.isArray(value)) { + throw new Error(`Agent manifest field '${key}' must be an array of TCP ports`); + } + + const ports = value.map((entry, index) => { + if (!isValidPort(entry, 1024)) { + throw new Error( + `Agent manifest field '${key}[${String(index)}]' must be an integer TCP port between 1024 and 65535`, + ); + } + return entry; + }); + + return ports.length > 0 ? ports : undefined; +} + +export function readStringMap(record: ManifestRecord, key: string): StringMap | undefined { + const value = readObject(record, key); + if (!value) return undefined; + + const result: StringMap = {}; + for (const [entryKey, entryValue] of Object.entries(value)) { + if (typeof entryValue === "string") { + result[entryKey] = entryValue; + } + } + return result; +} + +export function readHealthProbe(record: ManifestRecord): AgentHealthProbe | undefined { + const healthProbe = readObject(record, "health_probe"); + if (!healthProbe) return undefined; + + const url = readString(healthProbe, "url"); + const port = healthProbe.port; + const timeoutSeconds = healthProbe.timeout_seconds; + + if (port !== undefined && !isValidPort(port)) { + throw new Error( + "Agent manifest field 'health_probe.port' must be an integer TCP port between 1 and 65535", + ); + } + + if ( + typeof url === "string" && + isValidPort(port) && + typeof timeoutSeconds === "number" && + Number.isFinite(timeoutSeconds) + ) { + return { url, port, timeout_seconds: timeoutSeconds }; + } + + return undefined; +} + +export function readDashboard(record: ManifestRecord): AgentDashboard { + const dashboard = readObject(record, "dashboard") ?? {}; + const rawKind = dashboard.kind; + if (rawKind !== undefined && rawKind !== "ui" && rawKind !== "api") { + throw new Error("Agent manifest field 'dashboard.kind' must be ui or api"); + } + const kind: AgentDashboardKind = rawKind === "api" ? "api" : "ui"; + const defaultLabel = kind === "api" ? "API" : "UI"; + const normalizedLabel = typeof dashboard.label === "string" ? dashboard.label.trim() : ""; + + const normalizePath = (key: "path" | "health_path", fallback: string): string => { + const value = dashboard[key]; + if (value === undefined) return fallback; + if (typeof value !== "string" || !value.startsWith("/")) { + throw new Error(`Agent manifest field 'dashboard.${key}' must be an absolute path`); + } + return value.trim() || fallback; + }; + + const rawAuth = dashboard.auth; + if ( + rawAuth !== undefined && + rawAuth !== "url_token" && + rawAuth !== "session" && + rawAuth !== "none" + ) { + throw new Error("Agent manifest field 'dashboard.auth' must be url_token, session, or none"); + } + + return { + kind, + label: normalizedLabel || defaultLabel, + path: normalizePath("path", "/"), + healthPath: normalizePath("health_path", "/health"), + auth: rawAuth ?? (kind === "api" ? "none" : "url_token"), + }; +} + +export function readInference(record: ManifestRecord): AgentInference | undefined { + const inference = readObject(record, "inference"); + if (!inference) return undefined; + + const providerType = inference.provider_type; + if (providerType !== undefined && typeof providerType !== "string") { + throw new Error("Agent manifest field 'inference.provider_type' must be a string"); + } + + const providerOptions = inference.provider_options; + let providerOptionList: string[] | undefined; + if (providerOptions !== undefined) { + if ( + !Array.isArray(providerOptions) || + providerOptions.some((entry) => typeof entry !== "string") + ) { + throw new Error( + "Agent manifest field 'inference.provider_options' must be an array of strings", + ); + } + providerOptionList = providerOptions as string[]; + } + + return { provider_type: providerType, provider_options: providerOptionList }; +} + +export function readMcpCapability(record: ManifestRecord): AgentMcpCapability { + const mcp = readObject(record, "mcp"); + if (!mcp) { + return { support: "disabled", reason: "MCP support is not declared for this agent." }; + } + + const support = readString(mcp, "support"); + if (support !== "bridge" && support !== "disabled") { + throw new Error("Agent manifest field 'mcp.support' must be bridge or disabled"); + } + + const adapter = readString(mcp, "adapter"); + if ( + adapter !== undefined && + adapter !== "mcporter" && + adapter !== "hermes-config" && + adapter !== "deepagents-config" + ) { + throw new Error( + "Agent manifest field 'mcp.adapter' must be mcporter, hermes-config, or deepagents-config", + ); + } + if (support === "bridge" && !adapter) { + throw new Error("Agent manifest field 'mcp.adapter' is required when mcp.support is bridge"); + } + if (support === "disabled" && adapter) { + throw new Error("Agent manifest field 'mcp.adapter' is only valid when mcp.support is bridge"); + } + + const reason = readString(mcp, "reason")?.trim(); + return { + support, + ...(adapter ? { adapter } : {}), + ...(reason ? { reason } : {}), + }; +} + +export function loadManifestRecord(manifestPath: string): ManifestRecord { + const parsed = yaml.load(fs.readFileSync(manifestPath, "utf8")); + if (!isManifestRecord(parsed)) { + throw new Error(`Agent manifest must be a YAML object: ${manifestPath}`); + } + return parsed; +} diff --git a/src/lib/agent/onboard.test.ts b/src/lib/agent/onboard.test.ts index 8d11f8860ec..744c290136c 100644 --- a/src/lib/agent/onboard.test.ts +++ b/src/lib/agent/onboard.test.ts @@ -26,6 +26,10 @@ function makeAgent(overrides: Partial = {}): AgentDefinition { format: "yaml", }, inferenceProviderOptions: [], + mcpCapability: { + support: "disabled", + reason: "test fixture", + }, stateDirs: [], stateFiles: [], userManagedFiles: [], diff --git a/src/lib/agent/onboard.ts b/src/lib/agent/onboard.ts index ffb6d68cfa8..a0ebe050492 100644 --- a/src/lib/agent/onboard.ts +++ b/src/lib/agent/onboard.ts @@ -5,23 +5,14 @@ // non-default agent (e.g. Hermes) is selected via --agent flag or // NEMOCLAW_AGENT env var. The OpenClaw path never touches this module. -import fs from "fs"; -import os from "os"; -import path from "path"; - -import { dockerBuild, dockerImageInspect } from "../adapters/docker"; import { buildValidatedCurlCommandArgs } from "../adapters/http/curl-args"; import { getAgentBranding } from "../cli/branding"; import type { JsonObject as LooseObject } from "../core/json-types"; import { sleepSeconds } from "../core/wait"; import { getProviderSelectionConfig } from "../inference/config"; import { runSandboxConfigSync } from "../onboard/config-sync"; -import { ROOT, redact, run } from "../runner"; -import { - buildLocalBaseTag, - resolveSandboxBaseImage, - SANDBOX_BASE_TAG, -} from "../sandbox-base-image"; +import { redact, run } from "../runner"; +import * as baseImage from "./base-image"; import { describeAgentBinaryFailure, verifyAgentBinaryAvailable } from "./binary-availability"; import { printOptionalDashboardUi } from "./dashboard-ui"; import { type AgentDefinition, isTerminalAgent, loadAgent, resolveAgentName } from "./defs"; @@ -41,6 +32,35 @@ export interface OnboardContext { skippedStepMessage: (stepName: string, sandboxName: string) => void; } +// Keep these compatibility exports as ordinary writable functions. Focused +// onboarding and rebuild harnesses replace them at the facade boundary, while +// the implementation stays isolated in base-image.ts. +export function getAgentSandboxBaseImageEnvVar(agentName: string): string { + return baseImage.getAgentSandboxBaseImageEnvVar(agentName); +} + +export function pinAgentSandboxBaseImageRef(agentName: string, imageRef: string): string { + return baseImage.pinAgentSandboxBaseImageRef(agentName, imageRef); +} + +export function hermesBaseImageSupportsMcp(imageRef: string): boolean { + return baseImage.hermesBaseImageSupportsMcp(imageRef); +} + +export function ensureAgentBaseImage( + agent: AgentDefinition, + opts: { forceBaseImageRebuild?: boolean } = {}, +): { imageTag: string | null; built: boolean } { + return baseImage.ensureAgentBaseImage(agent, opts); +} + +export function createAgentSandbox( + agent: AgentDefinition, + opts: { forceBaseImageRebuild?: boolean } = {}, +): { buildCtx: string; stagedDockerfile: string } { + return baseImage.createAgentSandbox(agent, opts); +} + /** * Resolve the effective agent from CLI flags, env, or session. * Returns null for openclaw (default path), loaded agent object otherwise. @@ -57,125 +77,6 @@ export function resolveAgent({ return loadAgent(name); } -/** - * Ensure the agent-specific sandbox base image exists locally. - * Rebuild callers can force this so local Dockerfile.base edits are applied. - */ -export function ensureAgentBaseImage( - agent: AgentDefinition, - opts: { forceBaseImageRebuild?: boolean } = {}, -): { - imageTag: string | null; - built: boolean; -} { - const baseDockerfile = agent.dockerfileBasePath; - - if (!baseDockerfile) { - return { imageTag: null, built: false }; - } - - const baseImageName = `ghcr.io/nvidia/nemoclaw/${agent.name}-sandbox-base`; - const baseImageTag = `${baseImageName}:${SANDBOX_BASE_TAG}`; - const forceBaseImageRebuild = opts.forceBaseImageRebuild === true; - if (forceBaseImageRebuild) { - console.log(` Rebuilding ${agent.displayName} base image...`); - const buildResult = dockerBuild(baseDockerfile, baseImageTag, ROOT, { - ignoreError: true, - stdio: ["ignore", "inherit", "inherit"], - }); - if (buildResult.error || buildResult.status !== 0) { - const detail = buildResult.error - ? `: ${buildResult.error.message}` - : ` (exit ${buildResult.status ?? "unknown"})`; - throw new Error(`Failed to build ${agent.displayName} base image${detail}`); - } - console.log(` \u2713 Base image built: ${baseImageTag}`); - return { imageTag: baseImageTag, built: true }; - } - - const resolved = resolveSandboxBaseImage({ - imageName: baseImageName, - dockerfilePath: baseDockerfile, - localTag: buildLocalBaseTag(`nemoclaw-${agent.name}-sandbox-base-local`, ROOT), - envVar: `NEMOCLAW_${agent.name.toUpperCase().replace(/[^A-Z0-9]+/g, "_")}_SANDBOX_BASE_IMAGE_REF`, - label: `${agent.displayName} sandbox base image`, - requireOpenshellSandboxAbi: process.platform === "linux", - rootDir: ROOT, - }); - if (resolved && !forceBaseImageRebuild) { - console.log(` Using ${agent.displayName} base image: ${resolved.ref}`); - return { imageTag: resolved.ref, built: false }; - } - if (!resolved && process.platform === "linux" && !forceBaseImageRebuild) { - throw new Error( - `No compatible ${agent.displayName} sandbox base image found for ${baseImageName}`, - ); - } - const inspectResult = dockerImageInspect(baseImageTag, { - ignoreError: true, - suppressOutput: true, - }); - if (inspectResult?.status !== 0) { - console.log(` Building ${agent.displayName} base image (first time only)...`); - const buildResult = dockerBuild(baseDockerfile, baseImageTag, ROOT, { - ignoreError: true, - stdio: ["ignore", "inherit", "inherit"], - }); - if (buildResult.error || buildResult.status !== 0) { - const detail = buildResult.error - ? `: ${buildResult.error.message}` - : ` (exit ${buildResult.status ?? "unknown"})`; - throw new Error(`Failed to build ${agent.displayName} base image${detail}`); - } - console.log(` \u2713 Base image built: ${baseImageTag}`); - return { imageTag: baseImageTag, built: true }; - } - - console.log(` Base image exists: ${baseImageTag}`); - return { imageTag: baseImageTag, built: false }; -} - -/** - * Stage build context for an agent-specific sandbox image. - * Builds the base image if the agent defines one and it's not cached locally. - */ -export function createAgentSandbox( - agent: AgentDefinition, - opts: { forceBaseImageRebuild?: boolean } = {}, -): { - buildCtx: string; - stagedDockerfile: string; -} { - const agentDockerfile = agent.dockerfilePath; - - if (!agentDockerfile) { - throw new Error(`${agent.displayName} is missing a sandbox Dockerfile`); - } - - const { imageTag: baseImageRef } = ensureAgentBaseImage(agent, opts); - - const buildCtx = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-build-")); - fs.cpSync(ROOT, buildCtx, { - recursive: true, - filter: (src) => { - const base = path.basename(src); - return !["node_modules", ".git", ".venv", "__pycache__", ".claude"].includes(base); - }, - }); - const stagedDockerfile = path.join(buildCtx, "Dockerfile"); - fs.copyFileSync(agentDockerfile, stagedDockerfile); - if (baseImageRef) { - const dockerfile = fs.readFileSync(stagedDockerfile, "utf8"); - fs.writeFileSync( - stagedDockerfile, - dockerfile.replace(/^ARG BASE_IMAGE(?:=.*)?$/m, `ARG BASE_IMAGE=${baseImageRef}`), - ); - } - console.log(` Using ${agent.displayName} Dockerfile: ${agentDockerfile}`); - - return { buildCtx, stagedDockerfile }; -} - /** * Get the agent-specific network policy path, or null to use the default. */ diff --git a/src/lib/agent/runtime.test.ts b/src/lib/agent/runtime.test.ts index 9b591c941b6..4607d9f72cf 100644 --- a/src/lib/agent/runtime.test.ts +++ b/src/lib/agent/runtime.test.ts @@ -23,6 +23,7 @@ function makeAgent(overrides: Partial = {}): AgentDefinition { format: "yaml", }, inferenceProviderOptions: [], + mcpCapability: { support: "disabled", reason: "test fixture" }, stateDirs: [], stateFiles: [], userManagedFiles: [], diff --git a/src/lib/cli/command-display.ts b/src/lib/cli/command-display.ts index 0e53ea62914..f67ed774645 100644 --- a/src/lib/cli/command-display.ts +++ b/src/lib/cli/command-display.ts @@ -7,6 +7,7 @@ export type CommandGroup = | "Skills" | "Policy Presets" | "Messaging Channels" + | "MCP Servers" | "Compatibility Commands" | "Services" | "Troubleshooting" diff --git a/src/lib/cli/command-registry.ts b/src/lib/cli/command-registry.ts index 366dfeae5c8..519b266a867 100644 --- a/src/lib/cli/command-registry.ts +++ b/src/lib/cli/command-registry.ts @@ -44,6 +44,7 @@ export const GROUP_ORDER: readonly CommandGroup[] = [ "Skills", "Policy Presets", "Messaging Channels", + "MCP Servers", "Compatibility Commands", "Services", "Troubleshooting", diff --git a/src/lib/cli/public-display-defaults.ts b/src/lib/cli/public-display-defaults.ts index e3c5e080bc5..51d00d4becf 100644 --- a/src/lib/cli/public-display-defaults.ts +++ b/src/lib/cli/public-display-defaults.ts @@ -5,6 +5,7 @@ import type { PublicCommandDisplayEntry } from "./command-display"; import { getRegisteredOclifCommandMetadata } from "./oclif-metadata"; import { SANDBOX_AGENTS_DISPLAY_LAYOUT } from "./public-display-agents"; import type { PublicDisplayLayout } from "./public-display-layout"; +import { SANDBOX_MCP_DISPLAY_LAYOUT } from "./public-display-mcp"; import { SANDBOX_SESSIONS_DISPLAY_LAYOUT } from "./public-display-sessions"; import { globalRouteTokenVariants, sandboxRouteTokens } from "./public-route-metadata"; @@ -197,6 +198,7 @@ const PUBLIC_DISPLAY_LAYOUT: Record = { flags: "[--channel ] [--json]", }, ], + ...SANDBOX_MCP_DISPLAY_LAYOUT, "sandbox:config:get": [ { group: "Sandbox Management", diff --git a/src/lib/cli/public-display-mcp.test.ts b/src/lib/cli/public-display-mcp.test.ts new file mode 100644 index 00000000000..9d6eb9e6098 --- /dev/null +++ b/src/lib/cli/public-display-mcp.test.ts @@ -0,0 +1,28 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { PUBLIC_DISPLAY_ENTRIES } from "./public-display-defaults"; +import { SANDBOX_MCP_DISPLAY_LAYOUT } from "./public-display-mcp"; + +describe("sandbox MCP public display layout", () => { + it("owns the complete MCP lifecycle help surface and feeds the public registry", () => { + expect(Object.keys(SANDBOX_MCP_DISPLAY_LAYOUT)).toEqual(["sandbox:mcp"]); + expect(SANDBOX_MCP_DISPLAY_LAYOUT["sandbox:mcp"]?.map((entry) => entry.usage)).toEqual([ + "nemoclaw mcp list", + "nemoclaw mcp add", + "nemoclaw mcp status", + "nemoclaw mcp restart", + "nemoclaw mcp remove", + ]); + expect(PUBLIC_DISPLAY_ENTRIES["sandbox:mcp"]).toHaveLength(5); + expect(PUBLIC_DISPLAY_ENTRIES["sandbox:mcp"]?.map((entry) => entry.group)).toEqual([ + "MCP Servers", + "MCP Servers", + "MCP Servers", + "MCP Servers", + "MCP Servers", + ]); + }); +}); diff --git a/src/lib/cli/public-display-mcp.ts b/src/lib/cli/public-display-mcp.ts new file mode 100644 index 00000000000..3f425a454c5 --- /dev/null +++ b/src/lib/cli/public-display-mcp.ts @@ -0,0 +1,44 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { PublicDisplayLayout } from "./public-display-layout"; + +export const SANDBOX_MCP_DISPLAY_LAYOUT: Record = { + "sandbox:mcp": [ + { + group: "MCP Servers", + order: 25.1, + usage: "nemoclaw mcp list", + description: "List configured MCP servers", + flags: "[--json]", + }, + { + group: "MCP Servers", + order: 25.2, + usage: "nemoclaw mcp add", + description: "Add an OpenShell-enforced MCP HTTP server", + flags: " --url --env KEY", + }, + { + group: "MCP Servers", + order: 25.3, + usage: "nemoclaw mcp status", + description: "Inspect MCP server health", + flags: "[server] [--json]", + }, + { + group: "MCP Servers", + order: 25.4, + usage: "nemoclaw mcp restart", + description: "Refresh one or all MCP server registrations", + flags: "[server]", + }, + { + group: "MCP Servers", + order: 25.5, + usage: "nemoclaw mcp remove", + description: "Remove an MCP server, provider, and generated policy", + flags: " [--force]", + }, + ], +}; diff --git a/src/lib/gateway-runtime-action.ts b/src/lib/gateway-runtime-action.ts index bd34d6efcc7..b42744cd1a0 100644 --- a/src/lib/gateway-runtime-action.ts +++ b/src/lib/gateway-runtime-action.ts @@ -1,13 +1,6 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -const { startGatewayForRecovery } = require("./onboard") as { - startGatewayForRecovery: (options?: { - gatewayName?: string; - gatewayPort?: number; - }) => Promise; -}; - import { stripAnsi } from "./adapters/openshell/client"; import { captureOpenshell, runOpenshell } from "./adapters/openshell/runtime"; import { @@ -148,6 +141,15 @@ export async function recoverNamedGatewayRuntime(options: RecoverNamedGatewayRun ); if (shouldStartGateway) { + // Keep this lazy to avoid the deliberate onboard -> runner -> gateway + // recovery cycle at module-import time. Lifecycle helpers do not need to + // load the full onboarding graph until recovery actually starts. + const { startGatewayForRecovery } = (await import("./onboard")) as unknown as { + startGatewayForRecovery: (startOptions?: { + gatewayName?: string; + gatewayPort?: number; + }) => Promise; + }; try { await startGatewayForRecovery({ gatewayName, diff --git a/src/lib/hermes-provider-auth.test.ts b/src/lib/hermes-provider-auth.test.ts index 6f68ce255d7..f558b4e66c2 100644 --- a/src/lib/hermes-provider-auth.test.ts +++ b/src/lib/hermes-provider-auth.test.ts @@ -39,6 +39,23 @@ afterEach(() => { }); describe("Hermes provider OpenShell credential handoff", () => { + it("inspects exact OpenShell credential key bindings without exposing values", () => { + const auth = loadAuth(); + const binding = auth.inspectHermesProviderBinding(() => ({ + status: 0, + stdout: "Provider:\n\n Name: hermes-provider\n Credential keys: NOUS_API_KEY\n", + stderr: "", + })); + expect(binding).toEqual({ exists: true, credentialKeys: ["NOUS_API_KEY"] }); + }); + + it("fails closed when OpenShell provider details omit credential metadata", () => { + const auth = loadAuth(); + expect( + auth.inspectHermesProviderBinding(() => ({ status: 0, stdout: "Provider: exists" })), + ).toEqual({ exists: true, credentialKeys: null }); + }); + it("registers Nous API-key inference in OpenShell without host-side persistence", async () => { const originalHome = process.env.HOME; const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-hermes-api-key-")); diff --git a/src/lib/hermes-provider-auth.ts b/src/lib/hermes-provider-auth.ts index 172ce6a2fb2..2333a3feca9 100644 --- a/src/lib/hermes-provider-auth.ts +++ b/src/lib/hermes-provider-auth.ts @@ -47,6 +47,7 @@ export type HermesAuthMethod = "oauth" | "api_key"; type RunOpenshellResult = { status?: number | null; + output?: string | Buffer | null; stdout?: string | Buffer | null; stderr?: string | Buffer | null; }; @@ -86,6 +87,31 @@ export function isHermesProviderRegistered(runOpenshell: RunOpenshell): boolean return onboardProviders.providerExistsInGateway(HERMES_PROVIDER_NAME, runOpenshell); } +export type HermesProviderBinding = { + exists: boolean; + credentialKeys: string[] | null; +}; + +export function inspectHermesProviderBinding(runOpenshell: RunOpenshell): HermesProviderBinding { + const result = runOpenshell(["provider", "get", HERMES_PROVIDER_NAME], { + ignoreError: true, + stdio: ["ignore", "pipe", "pipe"], + }); + if (result.status !== 0) return { exists: false, credentialKeys: null }; + const output = `${result.stdout ?? ""}\n${result.stderr ?? ""}\n${result.output ?? ""}`; + const rawKeys = output.match(/Credential keys:\s*([^\r\n]+)/i)?.[1]?.trim(); + if (!rawKeys) return { exists: true, credentialKeys: null }; + if (rawKeys === "") return { exists: true, credentialKeys: [] }; + return { + exists: true, + credentialKeys: rawKeys + .split(",") + .map((value) => value.trim()) + .filter(Boolean) + .sort(), + }; +} + export function registerHermesInferenceProvider( apiKey: string, runOpenshell: RunOpenshell, @@ -206,6 +232,7 @@ module.exports = { HERMES_NOUS_API_KEY_CREDENTIAL_ENV, AGENT_KEY_MIN_TTL_SECONDS, isHermesProviderRegistered, + inspectHermesProviderBinding, registerHermesInferenceProvider, ensureHermesProviderOAuthCredentials, ensureHermesProviderApiKeyCredentials, diff --git a/src/lib/inference/selection.test.ts b/src/lib/inference/selection.test.ts new file mode 100644 index 00000000000..6db11d82b11 --- /dev/null +++ b/src/lib/inference/selection.test.ts @@ -0,0 +1,40 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; +import { normalizeInferenceSelection } from "./selection"; + +describe("normalizeInferenceSelection", () => { + it("persists canonical compatible-endpoint reasoning values", () => { + expect( + normalizeInferenceSelection({ + provider: "compatible-endpoint", + compatibleEndpointReasoning: " TRUE ", + }).compatibleEndpointReasoning, + ).toBe("true"); + expect( + normalizeInferenceSelection({ + provider: "compatible-endpoint", + compatibleEndpointReasoning: "false", + }).compatibleEndpointReasoning, + ).toBe("false"); + }); + + it("rejects malformed reasoning values", () => { + expect( + normalizeInferenceSelection({ + provider: "compatible-endpoint", + compatibleEndpointReasoning: "yes", + }).compatibleEndpointReasoning, + ).toBeNull(); + }); + + it("clears reasoning state for non-compatible providers", () => { + expect( + normalizeInferenceSelection({ + provider: "nvidia-prod", + compatibleEndpointReasoning: "true", + }).compatibleEndpointReasoning, + ).toBeNull(); + }); +}); diff --git a/src/lib/inference/selection.ts b/src/lib/inference/selection.ts index d47bf3a16de..9cddf240c1a 100644 --- a/src/lib/inference/selection.ts +++ b/src/lib/inference/selection.ts @@ -7,10 +7,16 @@ export interface InferenceSelection { endpointUrl: string | null; credentialEnv: string | null; preferredInferenceApi: string | null; + compatibleEndpointReasoning: "true" | "false" | null; nimContainer: string | null; } -export type InferenceSelectionInput = Partial | null | undefined; +export type InferenceSelectionInput = + | (Partial> & { + compatibleEndpointReasoning?: unknown; + }) + | null + | undefined; function nullableString(value: unknown): string | null { if (typeof value !== "string") return null; @@ -29,13 +35,27 @@ function nullableInferenceApi(value: unknown): string | null { return normalized && SUPPORTED_INFERENCE_APIS.has(normalized) ? normalized : null; } +function nullableCompatibleEndpointReasoning( + provider: string | null, + value: unknown, +): "true" | "false" | null { + if (provider !== "compatible-endpoint") return null; + const normalized = nullableString(value)?.toLowerCase(); + return normalized === "true" || normalized === "false" ? normalized : null; +} + export function normalizeInferenceSelection(input: InferenceSelectionInput): InferenceSelection { + const provider = nullableString(input?.provider); return { - provider: nullableString(input?.provider), + provider, model: nullableString(input?.model), endpointUrl: nullableString(input?.endpointUrl), credentialEnv: nullableString(input?.credentialEnv), preferredInferenceApi: nullableInferenceApi(input?.preferredInferenceApi), + compatibleEndpointReasoning: nullableCompatibleEndpointReasoning( + provider, + input?.compatibleEndpointReasoning, + ), nimContainer: nullableString(input?.nimContainer), }; } diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 39c7c381152..81bf251eca1 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -180,7 +180,10 @@ const { dockerStop, } = docker; const gatewayDrift: typeof import("./adapters/openshell/gateway-drift") = require("./adapters/openshell/gateway-drift"); -const { getGatewayClusterContainerName, getGatewayClusterImageDrift } = gatewayDrift; +const { + getGatewayClusterContainerName, + getGatewayClusterImageDrift: getGatewayClusterImageDriftForName, +} = gatewayDrift; const sandboxBaseImage: typeof import("./sandbox-base-image") = require("./sandbox-base-image"); const { OPENCLAW_SANDBOX_BASE_IMAGE: SANDBOX_BASE_IMAGE, SANDBOX_BASE_TAG } = sandboxBaseImage; const { @@ -201,7 +204,7 @@ type RunnerOptions = { const { DASHBOARD_PORT, - GATEWAY_PORT, + GATEWAY_PORT: DEFAULT_GATEWAY_PORT, VLLM_PORT, OLLAMA_PORT, OLLAMA_PROXY_PORT, @@ -295,7 +298,7 @@ const { OLLAMA_PROXY_CREDENTIAL_ENV: string; VLLM_LOCAL_CREDENTIAL_ENV: string; getProviderLabel: (key: string) => string; - getNonInteractiveProvider: () => string | null; + getNonInteractiveProvider: (allowHostedInferenceStaging?: boolean) => string | null; getNonInteractiveModel: (providerKey: string) => string | null; getSandboxInferenceConfig: ( model: string, @@ -340,6 +343,8 @@ const { cleanupStaleHostFiles, }: typeof import("./host-artifact-cleanup") = require("./host-artifact-cleanup"); const registry: typeof import("./state/registry") = require("./state/registry"); +const sandboxMutationLock: typeof import("./state/mcp-lifecycle-lock") = + require("./state/mcp-lifecycle-lock"); const { resolveSandboxImageTagFromCreateOutput } = require("./domain/sandbox/image-tag") as typeof import("./domain/sandbox/image-tag"); const nim: typeof import("./inference/nim") = require("./inference/nim"); @@ -350,7 +355,6 @@ const { const { getFutureShellPathHint, getPortConflictServiceHints, - printRemediationActions, }: typeof import("./onboard/remediation") = require("./onboard/remediation"); const resumeConfig: typeof import("./onboard/resume-config") = require("./onboard/resume-config"); const { @@ -388,11 +392,7 @@ const { createOpenshellCliHelpers, }: typeof import("./onboard/openshell-cli") = require("./onboard/openshell-cli"); const sandboxGpuPreflight: typeof import("./onboard/sandbox-gpu-preflight") = require("./onboard/sandbox-gpu-preflight"); -const { - exitOnSandboxGpuConfigErrors, - resolveSandboxGpuFlagFromOptions, - validateSandboxGpuPreflight, -} = sandboxGpuPreflight; +const { resolveSandboxGpuFlagFromOptions, validateSandboxGpuPreflight } = sandboxGpuPreflight; const openshellVersion: typeof import("./onboard/openshell-version") = require("./onboard/openshell-version"); const { getBlueprintMaxOpenshellVersion, @@ -477,9 +477,8 @@ const { const { advanceTo, }: typeof import("./onboard/machine/result") = require("./onboard/machine/result"); -const { - getOnboardProgressStep, -}: typeof import("./onboard/machine/progress") = require("./onboard/machine/progress"); +const { skippedStepMessage }: typeof import("./onboard/skipped-step-message") = + require("./onboard/skipped-step-message"); const policies: typeof import("./policy") = require("./policy"); const policyPresetCarry: typeof import("./onboard/policy-preset-persistence") = require("./onboard/policy-preset-persistence"); const tiers: typeof import("./policy/tiers") = require("./policy/tiers"); @@ -490,6 +489,8 @@ const { preflightDashboardPortRangeAvailability, resolveCreateSandboxDashboardPort, } = require("./onboard/dashboard-port") as typeof import("./onboard/dashboard-port"); +const authoritativeRebuildTarget: typeof import("./onboard/authoritative-rebuild-target") = + require("./onboard/authoritative-rebuild-target"); const { assertDashboardPortNotReserved, buildRequiredPreflightPorts } = require("./onboard/preflight-ports") as typeof import("./onboard/preflight-ports"); const { tryCleanupOrphanedDashboardForward } = @@ -510,12 +511,11 @@ const { reconcilePreflightGatewayReuseState } = require("./onboard/preflight-gateway-reuse") as typeof import("./onboard/preflight-gateway-reuse"); const { getGatewayReuseHealthWaitConfig, - isDockerDriverGatewayHttpReady, - isGatewayHttpReady, - waitForGatewayHttpReady, -} = - require("./onboard/gateway-http-readiness") as typeof import("./onboard/gateway-http-readiness"); -const { isGatewayTcpReady } = + isDockerDriverGatewayHttpReady: probeDockerDriverGatewayHttpReady, + isGatewayHttpReady: probeGatewayHttpReady, + waitForGatewayHttpReady: waitForGatewayHttpReadyBase, +} = require("./onboard/gateway-http-readiness") as typeof import("./onboard/gateway-http-readiness"); +const { isGatewayTcpReady: probeGatewayTcpReady } = require("./onboard/gateway-tcp-readiness") as typeof import("./onboard/gateway-tcp-readiness"); const { trackChildExit } = require("./onboard/child-exit-tracker") as typeof import("./onboard/child-exit-tracker"); @@ -525,14 +525,14 @@ const { printDockerDaemonRecovery, reportLegacyGatewayStartResultFailure } = require("./onboard/gateway-start-failure") as typeof import("./onboard/gateway-start-failure"); const dockerDriverGatewayEnv: typeof import("./onboard/docker-driver-gateway-env") = require("./onboard/docker-driver-gateway-env"); -const { getDockerDriverGatewayEndpoint } = dockerDriverGatewayEnv; const dockerDriverGatewayRuntimeMarker: typeof import("./onboard/docker-driver-gateway-runtime-marker") = require("./onboard/docker-driver-gateway-runtime-marker"); const gatewayBinding: typeof import("./onboard/gateway-binding") = require("./onboard/gateway-binding"); +const fatalRuntimePreflight: typeof import("./onboard/fatal-runtime-preflight") = + require("./onboard/fatal-runtime-preflight"); const preflightUtils: typeof import("./onboard/preflight") = require("./onboard/preflight"); const clusterImagePatch: typeof import("./cluster-image-patch") = require("./cluster-image-patch"); -const { assessHost, checkPortAvailable, ensureSwap, getMemoryInfo, planHostRemediation } = - preflightUtils; +const { assessHost, checkPortAvailable, ensureSwap, getMemoryInfo } = preflightUtils; const { assertDockerBridgeAndContainerDnsHealthy, }: typeof import("./onboard/bridge-dns-preflight") = require("./onboard/bridge-dns-preflight"); @@ -604,7 +604,7 @@ import { } from "./onboard/sandbox-gpu-mode"; import type { SelectionDrift } from "./onboard/selection-drift"; import { formatOnboardConfigSummary, formatSandboxBuildEstimateNote } from "./onboard/summary"; -import type { ModelValidationResult, ValidationFailureLike } from "./onboard/types"; +import type { ModelValidationResult, OnboardOptions, ValidationFailureLike } from "./onboard/types"; import type { ContainerRuntime } from "./platform"; import { listChannels } from "./sandbox/channels"; import type { GatewayReuseState } from "./state/gateway"; @@ -618,7 +618,8 @@ const USE_COLOR = !process.env.NO_COLOR && !!process.stdout.isTTY; const DIM = USE_COLOR ? "\x1b[2m" : ""; const RESET = USE_COLOR ? "\x1b[0m" : ""; let OPENSHELL_BIN: string | null = null; -const GATEWAY_NAME = gatewayBinding.resolveGatewayName(GATEWAY_PORT); +let GATEWAY_PORT = DEFAULT_GATEWAY_PORT; +let GATEWAY_NAME = gatewayBinding.resolveGatewayName(GATEWAY_PORT); const { clearDockerDriverGatewayRuntimeFiles, getDockerDriverGatewayEnv, @@ -636,7 +637,7 @@ const { resolveOpenShellSandboxBinary, shouldRequireDockerDriverEnv, } = dockerDriverGatewayRuntime.createDockerDriverGatewayRuntimeHelpers({ - gatewayPort: GATEWAY_PORT, + gatewayPort: () => GATEWAY_PORT, getCachedOpenshellBinary: () => OPENSHELL_BIN, getBlueprintMaxOpenshellVersion, getInstalledOpenshellVersion, @@ -648,23 +649,6 @@ const { import type { JsonObject as LooseObject } from "./core/json-types"; import type { PreparedSandboxBuildContext } from "./onboard/build-context-stage"; - -type OnboardOptions = import("./onboard/prepared-dcode-rebuild").PreparedDcodeRebuildOptions & { - nonInteractive?: boolean; - recreateSandbox?: boolean; - resume?: boolean; - fresh?: boolean; - fromDockerfile?: string | null; - sandboxName?: string | null; - sandboxGpu?: "enable" | "disable" | null; - sandboxGpuDevice?: string | null; - acceptThirdPartySoftware?: boolean; - agent?: string | null; - controlUiPort?: number | null; - gpu?: boolean; - noGpu?: boolean; - autoYes?: boolean; -}; // Non-interactive mode: set by --non-interactive flag or env var. // When active, all prompts use env var overrides or sensible defaults. let NON_INTERACTIVE = false; @@ -674,6 +658,10 @@ let AUTO_YES = false; // null means "use auto-allocation" (skip dashboard port check in preflight). let _preflightDashboardPort: number | null = null; +function getOnboardDashboardPort(): number { + return _preflightDashboardPort ?? DASHBOARD_PORT; +} + function isNonInteractive(): boolean { return NON_INTERACTIVE || process.env.NEMOCLAW_NON_INTERACTIVE === "1"; } @@ -715,6 +703,24 @@ async function promptYesNoOrDefault( // ── Helpers ────────────────────────────────────────────────────── +const { + getDockerDriverGatewayEndpoint, + getGatewayClusterImageDrift, + isGatewayHttpReady, + isDockerDriverGatewayHttpReady, + waitForGatewayHttpReady, + isGatewayTcpReady, +} = gatewayBinding.createDynamicGatewayRuntimeHelpers({ + getGatewayName: () => GATEWAY_NAME, + getGatewayPort: () => GATEWAY_PORT, + getDockerDriverGatewayEndpoint: dockerDriverGatewayEnv.getDockerDriverGatewayEndpoint, + getGatewayClusterImageDrift: getGatewayClusterImageDriftForName, + probeGatewayHttpReady, + probeDockerDriverGatewayHttpReady, + waitForGatewayHttpReadyBase, + probeGatewayTcpReady, +}); + const { getOpenshellBinary, openshellShellCommand, @@ -735,11 +741,11 @@ const { // Gateway state functions — delegated to src/lib/state/gateway.ts const { isSandboxReady, parseSandboxStatus, getSandboxStateFromOutputs } = gatewayState; const { hasStaleGateway, isSelectedGateway, isGatewayHealthy, getGatewayReuseState } = - gatewayBinding.createGatewayNameBoundClassifiers(gatewayState, GATEWAY_NAME); + gatewayBinding.createGatewayNameBoundClassifiers(gatewayState, () => GATEWAY_NAME); const { getGatewayReuseSnapshot, selectNamedGatewayForReuseIfNeeded } = gatewayReuse.createGatewayReuseHelpers({ - gatewayName: GATEWAY_NAME, + gatewayName: () => GATEWAY_NAME, runCaptureOpenshell, runOpenshell, cliDisplayName, @@ -994,15 +1000,19 @@ function isInferenceRouteReady(provider: string, model: string): boolean { return Boolean(live && live.provider === provider && live.model === model); } -const { pruneStaleSandboxEntry, confirmRecreateForSelectionDrift, isOpenclawReady } = - sandboxLifecycle.createSandboxLifecycleHelpers({ - runCaptureOpenshell, - fetchGatewayAuthTokenFromSandbox: (sandboxName: string) => - fetchGatewayAuthTokenFromSandbox(sandboxName), - agentProductName, - prompt, - isAffirmativeAnswer, - }); +const { + reconcileSandboxForCreate, + pruneStaleSandboxEntry, + confirmRecreateForSelectionDrift, + isOpenclawReady, +} = sandboxLifecycle.createSandboxLifecycleHelpers({ + runCaptureOpenshell, + fetchGatewayAuthTokenFromSandbox: (sandboxName: string) => + fetchGatewayAuthTokenFromSandbox(sandboxName), + agentProductName, + prompt, + isAffirmativeAnswer, +}); // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. const { ensureValidatedWebSearchCredential, ensureValidatedBraveSearchCredential, configureWebSearch, verifyWebSearchInsideSandbox } = createWebSearchFlowHelpers({ prompt, note, isNonInteractive, cliName, runCaptureOpenshell }); @@ -1125,15 +1135,15 @@ function areRequiredDockerDriverBinariesPresent( ); } -function ensureOpenshellForOnboard(): { - installed?: boolean; - localBin: string | null; - futureShellPathHint: string | null; -} { - return openshellInstallFlow.ensureOpenshellForOnboard(getOpenShellInstallDeps()); +function ensureOpenshellForOnboard( + exitProcess: (code: number) => never = (code) => process.exit(code), +): OpenShellInstallResult { + return openshellInstallFlow.ensureOpenshellForOnboard(getOpenShellInstallDeps(exitProcess)); } -function getOpenShellInstallDeps(): OpenShellInstallDeps { +function getOpenShellInstallDeps( + exitProcess: (code: number) => never = (code) => process.exit(code), +): OpenShellInstallDeps { return { isLinuxDockerDriverGatewayEnabled, resolveOpenShellGatewayBinary, @@ -1147,11 +1157,14 @@ function getOpenShellInstallDeps(): OpenShellInstallDeps { shouldUseOpenshellDevChannel, isOpenshellDevVersion, versionGte, + hasRequiredOpenshellMessagingFeatures: () => + // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. + (require("./onboard/openshell-feature-gate") as typeof import("./onboard/openshell-feature-gate")).hasRequiredOpenshellMessagingFeatures({ openshellBin: resolveOpenshell(), gatewayBin: resolveOpenShellGatewayBinary(), sandboxBin: resolveOpenShellSandboxBinary(), allowExternalGatewayBin: Boolean(process.env.NEMOCLAW_OPENSHELL_GATEWAY_BIN?.trim()), allowExternalSandboxBin: Boolean(process.env.NEMOCLAW_OPENSHELL_SANDBOX_BIN?.trim()), requireSandboxBin: process.platform !== "darwin" || Boolean(process.env.NEMOCLAW_OPENSHELL_SANDBOX_BIN?.trim()) }), shouldAllowOpenshellAboveBlueprintMax, cliDisplayName, log: console.log, error: console.error, - exit: process.exit, + exit: exitProcess, }; } @@ -1209,7 +1222,7 @@ function stopDockerDriverGatewayProcess(): boolean { } function stopLegacyGatewayClusterContainer(): boolean { - const containerName = getGatewayClusterContainerName(); + const containerName = getGatewayClusterContainerName(GATEWAY_NAME); const inspectResult = dockerInspect(["--type", "container", containerName], { ignoreError: true, suppressOutput: true, @@ -1233,7 +1246,7 @@ function stopLegacyGatewayClusterContainer(): boolean { } function retireLegacyGatewayForDockerDriverUpgrade(): void { - runOpenshell(["forward", "stop", String(DASHBOARD_PORT)], { ignoreError: true }); + runOpenshell(["forward", "stop", String(getOnboardDashboardPort())], { ignoreError: true }); stopDockerDriverGatewayProcess(); const stoppedLegacyContainer = stopLegacyGatewayClusterContainer(); removeDockerDriverGatewayRegistration(); @@ -1264,6 +1277,7 @@ async function refreshDockerDriverGatewayReuseState( gatewayEnv: baseDesiredEnv, stateDir: getDockerDriverGatewayStateDir(), sandboxBin: resolveOpenShellSandboxBinary(), + gatewayName: GATEWAY_NAME, compatContainerName: gatewayBinding.resolveGatewayCompatContainerName(GATEWAY_PORT), }) : null; @@ -1398,7 +1412,7 @@ function handleFinalGatewayStartFailure({ } function getGatewayClusterContainerState(): string { - const containerName = getGatewayClusterContainerName(); + const containerName = getGatewayClusterContainerName(GATEWAY_NAME); const state = dockerContainerInspectFormat( "{{.State.Status}}{{if .State.Health}} {{.State.Health.Status}}{{end}}", containerName, @@ -1430,7 +1444,7 @@ function getGatewayHealthWaitConfig(_startStatus = 0, containerState = "") { } function buildGatewayClusterExecArgv(script: string): string[] { - return dockerExecArgv(getGatewayClusterContainerName(), ["sh", "-lc", script]); + return dockerExecArgv(getGatewayClusterContainerName(GATEWAY_NAME), ["sh", "-lc", script]); } function captureProcessArgs(pid: number): string { @@ -1444,7 +1458,7 @@ function checkGatewayPortAvailable() { } function getGatewayLocalEndpoint(): string { - return dockerDriverGatewayEnv.getGatewayHttpsEndpoint(); + return dockerDriverGatewayEnv.getGatewayHttpsEndpoint(GATEWAY_PORT); } const { gatewayClusterHealthcheckPassed, repairGatewayBootstrapSecrets } = @@ -1575,117 +1589,25 @@ function waitForSandboxReady(sandboxName: string, attempts = 10, delaySeconds = // ── Step 1: Preflight ──────────────────────────────────────────── -type PreflightOptions = Pick< - OnboardOptions, - "sandboxGpu" | "sandboxGpuDevice" | "gpu" | "noGpu" -> & { - optedOutGpuPassthrough?: boolean; -}; - -// Reject unsupported container runtimes (currently only Podman with the -// Linux Docker-driver gateway) before any Docker-specific probes. Both -// the fresh preflight and `--resume` backstop call this — if `docker` -// resolves to Podman, surface the unsupported-runtime message instead of -// running bridge/DNS diagnostics that would be misleading. -function rejectUnsupportedContainerRuntime(host: ReturnType): void { - if (isLinuxDockerDriverGatewayEnabled() && host.runtime === "podman") { - console.error(` ✗ ${cliDisplayName()} onboarding now uses OpenShell's Docker driver.`); - console.error(` Podman is not supported for this ${cliDisplayName()} integration path.`); - console.error(" Switch to Docker Engine and rerun onboarding."); - process.exit(1); - } -} +type PreflightOptions = import("./onboard/fatal-runtime-preflight").FatalRuntimePreflightOptions; async function preflight( preflightOpts: PreflightOptions = {}, ): Promise> { step(1, 8, "Preflight checks"); - const host = assessHost(); - - // Docker / runtime - if (!host.dockerReachable) { - console.error(" Docker is not reachable. Please fix Docker and try again."); - printRemediationActions(planHostRemediation(host)); - process.exit(1); - } - // Reject unsupported runtimes (Podman) BEFORE the success log so - // Podman users do not see a misleading `✓ Docker is running` line - // immediately followed by a fatal unsupported-runtime exit. - rejectUnsupportedContainerRuntime(host); - console.log(" ✓ Docker is running"); - require("./onboard/http-proxy-preflight").warnIfHostProxyMissesLoopback(); - const gpu = nim.detectGpu(); - const sandboxGpuConfig = resolveSandboxGpuConfig(gpu, { - flag: resolveSandboxGpuFlagFromOptions(preflightOpts), - device: preflightOpts.sandboxGpuDevice ?? null, - }); - exitOnSandboxGpuConfigErrors(sandboxGpuConfig); - const explicitlyOptedOutGpuPassthrough = - preflightOpts.optedOutGpuPassthrough === true || preflightOpts.noGpu === true; - preflightUtils.assertCdiNvidiaGpuSpecPresent( - host, - explicitlyOptedOutGpuPassthrough, - sandboxGpuConfig.hostGpuPlatform, + const { gpu, host, sandboxGpuConfig } = fatalRuntimePreflight.runFatalOnboardRuntimePreflight( + preflightOpts, + { + nonInteractive: isNonInteractive(), + }, ); - assertDockerBridgeAndContainerDnsHealthy(host, isNonInteractive()); - - if (host.runtime !== "unknown") { - console.log(` ✓ Container runtime: ${host.runtime}`); - } - if (host.notes.includes("Running under WSL")) { - console.log(" ⓘ Running under WSL"); - } - - if ( - host.isContainerRuntimeUnderProvisioned && - process.env.NEMOCLAW_IGNORE_RUNTIME_RESOURCES !== "1" - ) { - const detected: string[] = []; - if (typeof host.dockerCpus === "number") detected.push(`${host.dockerCpus} vCPU`); - if (typeof host.dockerMemTotalBytes === "number") { - const gib = host.dockerMemTotalBytes / 1024 ** 3; - detected.push(`${gib.toFixed(1)} GiB`); - } - const detectedStr = detected.length > 0 ? detected.join(" / ") : "unknown"; - console.warn( - ` ⚠ Container runtime under-provisioned: ${detectedStr} detected ` + - `(recommended: ${preflightUtils.MIN_RECOMMENDED_DOCKER_CPUS} vCPU / ${preflightUtils.MIN_RECOMMENDED_DOCKER_MEM_GIB} GiB).`, - ); - console.warn(" The sandbox build will be slow and may stall on default Colima settings."); - if (host.runtime === "colima") { - console.warn( - ` Suggested: colima stop && colima start --cpu ${preflightUtils.MIN_RECOMMENDED_DOCKER_CPUS} --memory ${preflightUtils.MIN_RECOMMENDED_DOCKER_MEM_GIB}`, - ); - } else if (host.runtime === "docker-desktop") { - console.warn(" Suggested: Docker Desktop → Settings → Resources, raise CPU/memory."); - } - console.warn(" Set NEMOCLAW_IGNORE_RUNTIME_RESOURCES=1 to silence this check."); - if (isNonInteractive()) { - console.warn( - " WARNING: Non-interactive mode is continuing despite under-provisioned runtime.", - ); - } else { - const proceed = await promptYesNoOrDefault(" Continue with onboarding?", null, false); - if (!proceed) { - console.error( - " Aborted by user. Resize your container runtime and rerun `nemoclaw onboard`.", - ); - process.exit(1); - } - } - } else if (host.dockerReachable) { - const detected: string[] = []; - if (typeof host.dockerCpus === "number") detected.push(`${host.dockerCpus} vCPU`); - if (typeof host.dockerMemTotalBytes === "number") { - const gib = host.dockerMemTotalBytes / 1024 ** 3; - detected.push(`${gib.toFixed(1)} GiB`); - } - if (detected.length > 0) { - console.log(` ✓ Container runtime resources: ${detected.join(" / ")}`); - } - } + await preflightUtils.checkContainerRuntimeResources(host, { + ignored: process.env.NEMOCLAW_IGNORE_RUNTIME_RESOURCES === "1", + nonInteractive: isNonInteractive(), + confirm: () => promptYesNoOrDefault(" Continue with onboarding?", null, false), + }); ensureOpenshellForOnboard(); @@ -1711,7 +1633,9 @@ async function preflight( waitForGatewayHttpReady, getGatewayLocalEndpoint, stopDashboardForward: () => - runOpenshell(["forward", "stop", String(DASHBOARD_PORT)], { ignoreError: true }), + runOpenshell(["forward", "stop", String(getOnboardDashboardPort())], { + ignoreError: true, + }), stopAllDashboardForwards, destroyGateway, destroyGatewayForReuse, @@ -1723,7 +1647,7 @@ async function preflight( gatewayReuseState, isDockerDriverGatewayEnabled: isLinuxDockerDriverGatewayEnabled(), cliDisplayName: cliDisplayName(), - dashboardPort: DASHBOARD_PORT, + dashboardPort: getOnboardDashboardPort(), log: console.log, runOpenshell, destroyGateway, @@ -1791,7 +1715,7 @@ async function preflight( const reuse = await applyHealthyPortReuse({ port, gatewayPort: GATEWAY_PORT, - dashboardPort: DASHBOARD_PORT, + dashboardPort: getOnboardDashboardPort(), label, runtimeDisplayName: cliDisplayName(), gatewayName: GATEWAY_NAME, @@ -1822,7 +1746,7 @@ async function preflight( // (e.g. dashboard forward left behind after destroy). Only kill the process // if its command line contains "openshell" to avoid killing unrelated SSH // tunnels the user may have set up on the same port. (#1950) - if (port === DASHBOARD_PORT && portCheck.process === "ssh" && portCheck.pid) { + if (port === getOnboardDashboardPort() && portCheck.process === "ssh" && portCheck.pid) { const outcome = await tryCleanupOrphanedDashboardForward({ port, pid: portCheck.pid, @@ -1890,7 +1814,6 @@ async function preflight( console.log(" ⓘ Local NIM unavailable — no GPU detected"); } - validateSandboxGpuPreflight(sandboxGpuConfig); if (sandboxGpuConfig.sandboxGpuEnabled) { console.log( ` ✓ Sandbox GPU: enabled (${sandboxGpuConfig.mode}${sandboxGpuConfig.sandboxGpuDevice ? `, device ${sandboxGpuConfig.sandboxGpuDevice}` : ""})`, @@ -2153,6 +2076,7 @@ async function startDockerDriverGateway({ gatewayEnv, stateDir, sandboxBin: resolveOpenShellSandboxBinary(), + gatewayName: GATEWAY_NAME, compatContainerName: gatewayBinding.resolveGatewayCompatContainerName(GATEWAY_PORT), ensureLocalTlsBundle: true, }) @@ -2170,12 +2094,17 @@ async function startDockerDriverGateway({ await dockerDriverGatewayEnv.startPackageManagedDockerDriverGatewayWithEnvOverride({ clearDockerDriverGatewayRuntimeFiles, exitOnFailure, - gatewayEnv, + gatewayEnv: driftGatewayEnv, gatewayName: GATEWAY_NAME, + isDockerDriverGatewayReady: () => isDockerDriverGatewayHttpReady(), registerDockerDriverGatewayEndpoint, runCaptureOpenshell, skipSandboxBridgeReachability, - verifySandboxBridgeGatewayReachableOrExit, + verifySandboxBridgeGatewayReachableOrExit: (fail, options) => + verifySandboxBridgeGatewayReachableOrExit(fail, { + ...options, + port: GATEWAY_PORT, + }), }) ) return; @@ -2201,6 +2130,7 @@ async function startDockerDriverGateway({ } else if (registerDockerDriverGatewayEndpoint() && (await isDockerDriverGatewayHttpReady())) { await verifySandboxBridgeGatewayReachableOrExit(exitOnFailure, { skip: skipSandboxBridgeReachability, + port: GATEWAY_PORT, }); console.log(" ✓ Reusing existing Docker-driver gateway"); return; @@ -2241,6 +2171,7 @@ async function startDockerDriverGateway({ ) { await verifySandboxBridgeGatewayReachableOrExit(exitOnFailure, { skip: skipSandboxBridgeReachability, + port: GATEWAY_PORT, }); console.log(` ✓ Reusing existing Docker-driver gateway process (PID ${portListenerPid})`); return; @@ -2330,6 +2261,7 @@ async function startDockerDriverGateway({ ) { await verifySandboxBridgeGatewayReachableOrExit(exitOnFailure, { skip: skipSandboxBridgeReachability, + port: GATEWAY_PORT, }); console.log(" ✓ Docker-driver gateway is healthy"); return; @@ -2359,7 +2291,7 @@ async function startGatewayForRecovery(options = {}): Promise { } function getGatewayStartEnv(): Record { - const gatewayEnv = dockerDriverGatewayEnv.getGatewayStartNetworkEnv(); + const gatewayEnv = dockerDriverGatewayEnv.getGatewayStartNetworkEnv(GATEWAY_PORT); const openshellVersion = getInstalledOpenshellVersion(); const stableGatewayImage = openshellVersion ? `ghcr.io/nvidia/openshell/cluster:${openshellVersion}` @@ -2523,8 +2455,6 @@ async function recoverGatewayRuntime() { return false; } -// ── Step 3: Sandbox ────────────────────────────────────────────── - const { getSandboxRuntimeRegistryFields, hasSandboxGpuDrift, updateReusedSandboxMetadata } = sandboxRegistryMetadata.createSandboxRegistryMetadataHelpers({ isLinuxDockerDriverGatewayEnabled, @@ -2532,8 +2462,6 @@ const { getSandboxRuntimeRegistryFields, hasSandboxGpuDrift, updateReusedSandbox runCaptureOpenshell, }); -// ── Step 5: Sandbox ────────────────────────────────────────────── - async function createSandbox( gpu: ReturnType, model: string, @@ -2548,6 +2476,7 @@ async function createSandbox( sandboxGpuConfig: SandboxGpuConfig | null = null, resourceProfile: import("./resources-cmd").ResourceProfile | null = null, hermesToolGateways: string[] = [], + hermesAuthMethod: HermesAuthMethod | null = null, preparedBuildContext: PreparedSandboxBuildContext | null = null, ) { step(6, 8, "Creating sandbox"); @@ -2622,10 +2551,7 @@ async function createSandbox( }, ); - const existingRegistryEntryBeforePrune = registry.getSandbox(sandboxName); - - // Reconcile local registry state with the live OpenShell gateway state. - const liveExists = pruneStaleSandboxEntry(sandboxName); + const { existingEntry, preservedMcpState, liveExists } = reconcileSandboxForCreate(sandboxName); // #4614: capture default AFTER prune so a stale registry row isn't read as a live sandbox. const sandboxWasLiveDefault = liveExists && wasSandboxDefault(registry.getDefault(), sandboxName); @@ -2637,7 +2563,7 @@ async function createSandbox( pendingStateRestoreBackupPath = notReadyRecreate.selectPreUpgradeBackupForCreate({ liveExists, - hasExistingRegistryEntry: existingRegistryEntryBeforePrune !== null, + hasExistingRegistryEntry: existingEntry !== null, sandboxName, note, }); @@ -2875,6 +2801,16 @@ async function createSandbox( note(` Sandbox '${sandboxName}' exists but is not ready — recreating it.`); } + if (preservedMcpState) { + console.error( + ` Sandbox '${sandboxName}' has managed MCP servers. Refusing the generic onboard recreation path.`, + ); + console.error( + ` Run \`${cliName()} ${sandboxName} rebuild --yes\` so MCP providers and adapter state are preserved transactionally.`, + ); + process.exit(1); + } + const previousEntry: SandboxEntry | null = registry.getSandbox(sandboxName); policyPresetCarry.applyRecreatePolicyCarryForward(sandboxName, isNonInteractive(), note); @@ -3004,6 +2940,7 @@ async function createSandbox( webSearchConfig, hermesToolGateways, sandboxGpuConfig: effectiveSandboxGpuConfig, + gatewayPort: GATEWAY_PORT, }); const sandboxReadyTimeoutSecs = getSandboxReadyTimeoutSecs(effectiveSandboxGpuConfig); const { createCommand, effectiveDashboardPort, sandboxEnv, sandboxStartupCommand } = @@ -3054,6 +2991,9 @@ async function createSandbox( dockerGpuCreatePatch.exitOnPatchError(); + const restoreBackupPath = + pendingStateRestore?.manifest?.backupPath ?? pendingStateRestoreBackupPath; + if (createResult.status !== 0) { const failure = classifySandboxCreateFailure(createResult.output); if (failure.kind === "sandbox_create_incomplete") { @@ -3072,6 +3012,9 @@ async function createSandbox( console.error(""); console.error(createResult.output); } + sandboxCreateFailureDiagnostics.printSandboxCreateFailureDiagnostics(sandboxName, { + backupPath: restoreBackupPath, + }); console.error(" Try: openshell sandbox list # check gateway state"); printSandboxCreateRecoveryHints(createResult.output, { createArgs }); process.exit(createResult.status || 1); @@ -3095,28 +3038,12 @@ async function createSandbox( sleep: sleepSeconds, }); - const restoreBackupPath = - pendingStateRestore?.manifest?.backupPath ?? pendingStateRestoreBackupPath; - if (!readiness.ready) { - const diagnostics = sandboxCreateFailureDiagnostics.collectSandboxCreateFailureDiagnostics( - sandboxName, - { backupPath: restoreBackupPath }, - ); console.error(""); sandboxReadinessTracing.printReadinessFailure(readiness, sandboxName, sandboxReadyTimeoutSecs); - if (diagnostics) { - console.error(` Diagnostics saved: ${diagnostics.dir}`); - if (diagnostics.summaryLines.length > 0) { - console.error(" Recent OpenShell gateway failure:"); - for (const line of diagnostics.summaryLines) { - console.error(` ${line}`); - } - } - if (diagnostics.backupPath) { - console.error(` State backup retained: ${diagnostics.backupPath}`); - } - } + sandboxCreateFailureDiagnostics.printSandboxCreateFailureDiagnostics(sandboxName, { + backupPath: restoreBackupPath, + }); if (useDockerGpuPatch) { dockerGpuCreatePatch.printReadinessFailureIfEnabled(); } else { @@ -3173,8 +3100,7 @@ async function createSandbox( hermesDashboardForwarding.ensureForState(finalHermesDashboardState, sandboxName, true); } - // Register only after confirmed ready — prevents phantom entries - // openshell tags images with seconds; buildId is ms. Parse actual tag from output. Fixes #2672. + // Register only after ready; OpenShell tags in seconds, so parse the tag instead of using buildId. const resolvedImageTag = resolveSandboxImageTagFromCreateOutput(createResult.output, buildId); const sandboxRuntimeFields = getSandboxRuntimeRegistryFields(effectiveSandboxGpuConfig); @@ -3187,7 +3113,10 @@ async function createSandbox( agentVersionKnown: !fromDockerfile, imageTag: resolvedImageTag, appliedPolicies: initialSandboxPolicy.appliedPresets, + // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. + ...sandboxRegistration.creationFidelity(webSearchConfig, fromDockerfile, normalizeHermesAuthMethod(hermesAuthMethod)), plannedMessagingState, + preservedMcpState, hermesToolGateways, hermesDashboardState: finalHermesDashboardState, dashboardPort: actualDashboardPort, @@ -4547,25 +4476,27 @@ async function setupPoliciesWithSelection( sandboxName: string, options: SetupPolicySelectionOptions = {}, ) { - return setupPoliciesWithSelectionImpl( - { - policies, - tiers, - localInferenceProviders: LOCAL_INFERENCE_PROVIDERS, - step, - note, - isNonInteractive, - waitForSandboxReady, - syncPresetSelection, - selectPolicyTier, - setPolicyTier: (s, t) => registry.updateSandbox(s, { policyTier: t }), - getRecordedPolicyTier: (s) => registry.getSandbox(s)?.policyTier ?? null, - selectTierPresetsAndAccess, - parsePolicyPresetEnv, - env: process.env, - }, - sandboxName, - options, + return sandboxMutationLock.withSandboxMutationLock(sandboxName, () => + setupPoliciesWithSelectionImpl( + { + policies, + tiers, + localInferenceProviders: LOCAL_INFERENCE_PROVIDERS, + step, + note, + isNonInteractive, + waitForSandboxReady, + syncPresetSelection, + selectPolicyTier, + setPolicyTier: (s, t) => registry.updateSandbox(s, { policyTier: t }), + getRecordedPolicyTier: (s) => registry.getSandbox(s)?.policyTier ?? null, + selectTierPresetsAndAccess, + parsePolicyPresetEnv, + env: process.env, + }, + sandboxName, + options, + ), ); } @@ -4621,28 +4552,69 @@ const recordCompatibleStateResult = const recordPostVerifyStarted = onboardRuntimeBoundary.recordPostVerifyStarted.bind(onboardRuntimeBoundary); -function skippedStepMessage( - stepName: string, - detail?: string | null, - reason: "resume" | "reuse" = "resume", -): void { - const progressStep = getOnboardProgressStep(stepName); - const stepInfo = - progressStep && stepName === "openclaw" - ? { ...progressStep, title: `Setting up ${agentProductName()} inside sandbox` } - : progressStep; - if (stepInfo) { - step(stepInfo.number, stepInfo.total, stepInfo.title); +/** Run only non-mutating fatal onboard gates while the rebuild target is still intact. */ +async function preflightAuthoritativeRebuildTarget( + opts: import("./onboard/authoritative-rebuild-target").AuthoritativeRebuildPreflightOptions, +): Promise { + const authoritativeGateway = + authoritativeRebuildTarget.resolveAuthoritativeOnboardGatewayBinding(opts); + if (!authoritativeGateway) throw new Error("Authoritative rebuild preflight has no gateway"); + const previous = { + dashboardPort: _preflightDashboardPort, + gatewayName: GATEWAY_NAME, + gatewayPort: GATEWAY_PORT, + nonInteractive: NON_INTERACTIVE, + }; + GATEWAY_NAME = authoritativeGateway.name; + GATEWAY_PORT = authoritativeGateway.port; + NON_INTERACTIVE = true; + _preflightDashboardPort = opts.controlUiPort ?? null; + const fail = (message: string): never => { + throw new Error(message); + }; + try { + await authoritativeRebuildTarget.preflightAuthoritativeRebuildTarget( + { ...opts, controlUiPort: opts.controlUiPort ?? null }, + { + runFatalRuntimePreflight: () => + fatalRuntimePreflight.runFatalOnboardRuntimePreflight( + { + sandboxGpu: opts.sandboxGpu, + sandboxGpuDevice: opts.sandboxGpuDevice, + noGpu: opts.noGpu, + }, + { + nonInteractive: true, + exitProcess: (code) => + fail(`onboard runtime preflight exited with code ${String(code)}`), + }, + ), + ensureOpenshell: () => + ensureOpenshellForOnboard((code) => + fail(`OpenShell component preflight exited with code ${String(code)}`), + ), + inferenceRouteReady: isInferenceRouteReady, + captureForwardList: () => runCaptureOpenshell(["forward", "list"], { ignoreError: true }), + checkPort: (port) => checkPortAvailable(port), + }, + ); + } finally { + GATEWAY_NAME = previous.gatewayName; + GATEWAY_PORT = previous.gatewayPort; + NON_INTERACTIVE = previous.nonInteractive; + _preflightDashboardPort = previous.dashboardPort; } - const prefix = reason === "reuse" ? "[reuse]" : "[resume]"; - console.log(` ${prefix} Skipping ${stepName}${detail ? ` (${detail})` : ""}`); } // ── Main ───────────────────────────────────────────────────────── async function onboard(opts: OnboardOptions = {}): Promise { + const authoritativeGateway = + authoritativeRebuildTarget.resolveAuthoritativeOnboardGatewayBinding(opts); + const previousGatewayBinding = { name: GATEWAY_NAME, port: GATEWAY_PORT }; + const previousOpenshellGateway = process.env.OPENSHELL_GATEWAY; const preparedDcodeRuntime = preparedDcodeRebuild.createPreparedDcodeRebuildRuntime( opts, - GATEWAY_NAME, + authoritativeGateway?.name ?? GATEWAY_NAME, ); setOnboardBrandingAgent(opts.agent || process.env.NEMOCLAW_AGENT || null); NON_INTERACTIVE = opts.nonInteractive || process.env.NEMOCLAW_NON_INTERACTIVE === "1"; @@ -4651,6 +4623,7 @@ async function onboard(opts: OnboardOptions = {}): Promise { _preflightDashboardPort = opts.controlUiPort ?? (process.env.NEMOCLAW_DASHBOARD_PORT != null ? DASHBOARD_PORT : null); onboardRuntimeBoundary.reset(); + if (!authoritativeGateway) delete process.env.OPENSHELL_GATEWAY; preparedDcodeRuntime.applyGatewayEnv(process.env); const { resume, fresh, requestedFromDockerfile, requestedSandboxName, cannotPrompt } = onboardEntryOptions.resolveOnboardEntryOptions( @@ -4681,14 +4654,15 @@ async function onboard(opts: OnboardOptions = {}): Promise { if (!noticeAccepted) { process.exit(1); } - // Validate NEMOCLAW_PROVIDER and NEMOCLAW_VLLM_MODEL early so invalid values - // fail before preflight (Docker/OpenShell checks). Without this, users see a - // misleading 'Docker is not reachable' error instead of the real - // problem: an unsupported provider value or unrecognised vLLM model slug. - resumeConfig.preflightEarlyOnboardEnv(); - const lockResult = onboardSession.acquireOnboardLock( - `nemoclaw onboard${resume ? " --resume" : ""}${fresh ? " --fresh" : ""}${isNonInteractive() ? " --non-interactive" : ""}${requestedFromDockerfile ? ` --from ${requestedFromDockerfile}` : ""}`, - ); + // Validate provider/model hints before preflight so configuration errors are not reported as Docker failures. + // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. + resumeConfig.preflightEarlyOnboardEnvForResume(isNonInteractive(), opts.authoritativeResumeConfig === true); + const ownsOnboardLock = opts.onboardLockAlreadyHeld !== true; + const lockResult = ownsOnboardLock + ? onboardSession.acquireOnboardLock( + `nemoclaw onboard${resume ? " --resume" : ""}${fresh ? " --fresh" : ""}${isNonInteractive() ? " --non-interactive" : ""}${requestedFromDockerfile ? ` --from ${requestedFromDockerfile}` : ""}`, + ) + : { acquired: true as const }; if (!lockResult.acquired) { console.error(` Another ${cliDisplayName()} onboarding run is already in progress.`); if (lockResult.holderPid) { @@ -4745,11 +4719,17 @@ async function onboard(opts: OnboardOptions = {}): Promise { let lockReleased = false; const releaseOnboardLock = () => { - if (lockReleased) return; + if (lockReleased || !ownsOnboardLock) return; lockReleased = true; onboardSession.releaseOnboardLock(); }; - process.once("exit", releaseOnboardLock); + if (ownsOnboardLock) process.once("exit", releaseOnboardLock); + + if (authoritativeGateway) { + GATEWAY_NAME = authoritativeGateway.name; + GATEWAY_PORT = authoritativeGateway.port; + process.env.OPENSHELL_GATEWAY = authoritativeGateway.name; + } let onboardTrace: ReturnType = { collector: null, @@ -4767,6 +4747,7 @@ async function onboard(opts: OnboardOptions = {}): Promise { requestedSandboxName, cannotPrompt, nonInteractive: isNonInteractive(), + authoritativeResumeConfig: opts.authoritativeResumeConfig === true, agentFlag: opts.agent || null, envAgent: process.env.NEMOCLAW_AGENT || null, }, @@ -4907,7 +4888,7 @@ async function onboard(opts: OnboardOptions = {}): Promise { runPreflight: (preflightOptions) => preflight({ ...opts, ...preflightOptions }), assessHost, assertCdiNvidiaGpuSpecPresent: preflightUtils.assertCdiNvidiaGpuSpecPresent, - rejectUnsupportedContainerRuntime, + rejectUnsupportedContainerRuntime: fatalRuntimePreflight.rejectUnsupportedContainerRuntime, assertDockerBridgeAndContainerDnsHealthy, resolveSandboxGpuConfig, validateSandboxGpuPreflight, @@ -4929,7 +4910,7 @@ async function onboard(opts: OnboardOptions = {}): Promise { waitForGatewayHttpReady, recoverGatewayRuntime, getGatewayLocalEndpoint, - stopDashboardForward: () => bestEffortForwardStop(runOpenshell, DASHBOARD_PORT), + stopDashboardForward: () => bestEffortForwardStop(runOpenshell, getOnboardDashboardPort()), destroyGateway, destroyGatewayForReuse, getGatewayClusterImageDrift, @@ -4997,6 +4978,7 @@ async function onboard(opts: OnboardOptions = {}): Promise { const [providerInferencePhase, sandboxPhase] = createCoreOnboardFlowPhases({ forceProviderSelection: forceProviderSelectionForAgentChange, + authoritativeResumeConfig: opts.authoritativeResumeConfig === true, env: process.env, constants: { hermesProviderName: hermesProviderAuth.HERMES_PROVIDER_NAME, @@ -5070,6 +5052,7 @@ async function onboard(opts: OnboardOptions = {}): Promise { getSandboxReuseState, hasSandboxGpuDrift, getSandboxHermesToolGateways: (name) => registry.getSandbox(name)?.hermesToolGateways, + getSandboxRegistryEntry: registry.getSandbox, normalizeHermesToolGatewaySelections, stringSetsEqual, removeSandboxFromRegistry: registry.removeSandbox.bind(registry), @@ -5098,6 +5081,7 @@ async function onboard(opts: OnboardOptions = {}): Promise { skippedStepMessage, recordStateSkipped, recordRepairEvent, + withSandboxMutationLock: sandboxMutationLock.withSandboxMutationLock, error: (message) => console.error(message), exitProcess: (code) => process.exit(code), }, @@ -5274,6 +5258,12 @@ async function onboard(opts: OnboardOptions = {}): Promise { releaseOnboardLock(); onboardRuntimeBoundary.clear(); onboardTracing.finishOnboardTrace(onboardTrace, traceCompleted); + if (authoritativeGateway) { + GATEWAY_NAME = previousGatewayBinding.name; + GATEWAY_PORT = previousGatewayBinding.port; + if (previousOpenshellGateway === undefined) delete process.env.OPENSHELL_GATEWAY; + else process.env.OPENSHELL_GATEWAY = previousOpenshellGateway; + } } } @@ -5352,6 +5342,7 @@ module.exports = { providerExistsInGateway, parsePolicyPresetEnv, parseSandboxStatus, + preflightAuthoritativeRebuildTarget, pruneStaleSandboxEntry, repairRecordedSandbox, recoverGatewayRuntime, diff --git a/src/lib/onboard/authoritative-rebuild-target.test.ts b/src/lib/onboard/authoritative-rebuild-target.test.ts new file mode 100644 index 00000000000..a078d7dfc7c --- /dev/null +++ b/src/lib/onboard/authoritative-rebuild-target.test.ts @@ -0,0 +1,159 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { + type AuthoritativeRebuildTargetDeps, + preflightAuthoritativeRebuildTarget, + resolveAuthoritativeOnboardGatewayBinding, +} from "./authoritative-rebuild-target"; + +const target = { + sandboxName: "alpha", + provider: "nvidia-prod", + model: "nvidia/nemotron", + targetGatewayName: "nemoclaw-12345", + controlUiPort: 18789, +}; +const originalGateway = process.env.OPENSHELL_GATEWAY; + +function deps(overrides: Partial = {}) { + return { + runFatalRuntimePreflight: vi.fn(), + ensureOpenshell: vi.fn(), + inferenceRouteReady: vi.fn(() => true), + captureForwardList: vi.fn(() => "alpha 127.0.0.1 18789 42 active"), + checkPort: vi.fn(async () => ({ ok: true })), + ...overrides, + } satisfies AuthoritativeRebuildTargetDeps; +} + +afterEach(() => { + switch (originalGateway) { + case undefined: + delete process.env.OPENSHELL_GATEWAY; + break; + default: + process.env.OPENSHELL_GATEWAY = originalGateway; + } +}); + +describe("authoritative rebuild gateway binding", () => { + const resolve = resolveAuthoritativeOnboardGatewayBinding; + + it("accepts only a paired canonical gateway name and port", () => { + expect( + resolve({ + authoritativeResumeConfig: true, + targetGatewayName: " nemoclaw-8081 ", + targetGatewayPort: 8081, + }), + ).toEqual({ name: "nemoclaw-8081", port: 8081 }); + expect(resolve({})).toBeNull(); + }); + + it.each([ + { authoritativeResumeConfig: true, targetGatewayName: "nemoclaw-8081" }, + { authoritativeResumeConfig: true, targetGatewayPort: 8081 }, + { targetGatewayName: "nemoclaw-8081", targetGatewayPort: 8081 }, + ])("rejects partial or non-authoritative target options", (options) => { + expect(() => resolve(options)).toThrow(/only together for an authoritative rebuild resume/); + }); + + it("rejects a non-canonical name or invalid target port", () => { + expect(() => + resolve({ + authoritativeResumeConfig: true, + targetGatewayName: "nemoclaw-9090", + targetGatewayPort: 8081, + }), + ).toThrow(/does not match port 8081/); + for (const port of [0, 65536, 8081.5]) { + expect(() => + resolve({ + authoritativeResumeConfig: true, + targetGatewayName: "nemoclaw-8081", + targetGatewayPort: port, + }), + ).toThrow(/Invalid authoritative rebuild gateway port/); + } + }); + + it("requires a complete authoritative target when the outer lifecycle owns the lock", () => { + expect(() => resolve({ onboardLockAlreadyHeld: true })).toThrow( + /lock handoff requires an authoritative rebuild resume/, + ); + }); +}); + +describe("authoritative rebuild target preflight", () => { + it("pins the requested gateway for route and forward checks, then restores it", async () => { + process.env.OPENSHELL_GATEWAY = "before"; + const seen: string[] = []; + const checkPort = vi.fn(); + await preflightAuthoritativeRebuildTarget( + target, + deps({ + inferenceRouteReady: vi.fn(() => { + seen.push(`route:${process.env.OPENSHELL_GATEWAY}`); + return true; + }), + captureForwardList: vi.fn(() => { + seen.push(`forward:${process.env.OPENSHELL_GATEWAY}`); + return "alpha 127.0.0.1 18789 42 active"; + }), + checkPort, + }), + ); + + expect(seen).toEqual(["route:nemoclaw-12345", "forward:nemoclaw-12345"]); + expect(checkPort).not.toHaveBeenCalled(); + expect(process.env.OPENSHELL_GATEWAY).toBe("before"); + }); + + it("rejects an exact provider/model route mismatch", async () => { + await expect( + preflightAuthoritativeRebuildTarget( + target, + deps({ inferenceRouteReady: vi.fn(() => false) }), + ), + ).rejects.toThrow("inference route does not match"); + }); + + it("rejects a dashboard forward owned by another sandbox", async () => { + await expect( + preflightAuthoritativeRebuildTarget( + target, + deps({ captureForwardList: vi.fn(() => "beta 127.0.0.1 18789 42 active") }), + ), + ).rejects.toThrow("belongs to sandbox 'beta'"); + }); + + it("rejects an occupied dashboard port with no OpenShell owner", async () => { + await expect( + preflightAuthoritativeRebuildTarget( + target, + deps({ + captureForwardList: vi.fn(() => ""), + checkPort: vi.fn(async () => ({ ok: false, process: "node", pid: 99, reason: "" })), + }), + ), + ).rejects.toThrow("occupied by node (PID 99)"); + }); + + it("restores gateway scope when a fatal runtime check throws", async () => { + process.env.OPENSHELL_GATEWAY = "before"; + await expect( + preflightAuthoritativeRebuildTarget( + target, + deps({ + runFatalRuntimePreflight: vi.fn(() => { + throw new Error("fatal runtime gate"); + }), + }), + ), + ).rejects.toThrow("fatal runtime gate"); + expect(process.env.OPENSHELL_GATEWAY).toBe("before"); + }); +}); diff --git a/src/lib/onboard/authoritative-rebuild-target.ts b/src/lib/onboard/authoritative-rebuild-target.ts new file mode 100644 index 00000000000..b8b01f37bbf --- /dev/null +++ b/src/lib/onboard/authoritative-rebuild-target.ts @@ -0,0 +1,119 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { findDashboardForwardOwner } from "./dashboard-port"; +import { resolveGatewayName } from "./gateway-binding"; +import type { PortProbeResult } from "./preflight"; +import { assertDashboardPortNotReserved } from "./preflight-ports"; +import type { OnboardOptions } from "./types"; + +export type AuthoritativeOnboardGatewayBinding = { name: string; port: number }; + +export type AuthoritativeGatewayOptions = Pick< + OnboardOptions, + "authoritativeResumeConfig" | "targetGatewayName" | "targetGatewayPort" | "onboardLockAlreadyHeld" +>; + +export type AuthoritativeRebuildPreflightOptions = Pick< + OnboardOptions, + "sandboxGpu" | "sandboxGpuDevice" | "noGpu" | "controlUiPort" +> & { + authoritativeResumeConfig: true; + model: string; + provider: string; + sandboxName: string; + targetGatewayName: string; + targetGatewayPort: number; +}; + +export function resolveAuthoritativeOnboardGatewayBinding( + opts: AuthoritativeGatewayOptions, +): AuthoritativeOnboardGatewayBinding | null { + const hasName = + typeof opts.targetGatewayName === "string" && opts.targetGatewayName.trim() !== ""; + const hasPort = opts.targetGatewayPort !== undefined && opts.targetGatewayPort !== null; + if ( + opts.onboardLockAlreadyHeld === true && + (!opts.authoritativeResumeConfig || !hasName || !hasPort) + ) { + throw new Error( + "The internal onboard lock handoff requires an authoritative rebuild resume with a target gateway.", + ); + } + if (!hasName && !hasPort) return null; + if (!opts.authoritativeResumeConfig || !hasName || !hasPort) { + throw new Error( + "An internal target gateway name and port may be supplied only together for an authoritative rebuild resume.", + ); + } + const name = opts.targetGatewayName?.trim() ?? ""; + const port = Number(opts.targetGatewayPort); + if (!Number.isInteger(port) || port < 1 || port > 65535) { + throw new Error( + `Invalid authoritative rebuild gateway port '${String(opts.targetGatewayPort)}'.`, + ); + } + if (resolveGatewayName(port) !== name) { + throw new Error(`Authoritative rebuild gateway '${name}' does not match port ${port}.`); + } + return { name, port }; +} + +export type AuthoritativeRebuildTarget = { + sandboxName: string; + provider: string; + model: string; + targetGatewayName: string; + controlUiPort: number | null; +}; + +export type AuthoritativeRebuildTargetDeps = { + runFatalRuntimePreflight(): unknown; + ensureOpenshell(): unknown; + inferenceRouteReady(provider: string, model: string): boolean; + captureForwardList(): string | null; + checkPort(port: number): Promise; + env?: NodeJS.ProcessEnv; +}; + +/** Run non-mutating target checks under an exact process-local gateway scope. */ +export async function preflightAuthoritativeRebuildTarget( + target: AuthoritativeRebuildTarget, + deps: AuthoritativeRebuildTargetDeps, +): Promise { + const env = deps.env ?? process.env; + const previousGateway = env.OPENSHELL_GATEWAY; + const fail = (message: string): never => { + throw new Error(message); + }; + env.OPENSHELL_GATEWAY = target.targetGatewayName; + try { + deps.runFatalRuntimePreflight(); + deps.ensureOpenshell(); + if (!deps.inferenceRouteReady(target.provider, target.model)) { + fail( + `OpenShell inference route does not match provider '${target.provider}' and model '${target.model}'.`, + ); + } + if (target.controlUiPort === null) return; + assertDashboardPortNotReserved(target.controlUiPort, fail); + const owner = findDashboardForwardOwner( + deps.captureForwardList(), + String(target.controlUiPort), + ); + if (owner && owner !== target.sandboxName) { + fail(`Dashboard port ${target.controlUiPort} belongs to sandbox '${owner}'.`); + } + if (owner) return; + const portCheck = await deps.checkPort(target.controlUiPort); + if (!portCheck.ok) { + const blocker = portCheck.process + ? `${portCheck.process}${portCheck.pid ? ` (PID ${portCheck.pid})` : ""}` + : portCheck.reason; + fail(`Dashboard port ${target.controlUiPort} is occupied by ${blocker}.`); + } + } finally { + if (previousGateway === undefined) delete env.OPENSHELL_GATEWAY; + else env.OPENSHELL_GATEWAY = previousGateway; + } +} diff --git a/src/lib/onboard/bridge-dns-preflight.ts b/src/lib/onboard/bridge-dns-preflight.ts index 97dc410df5f..f5b8f47d5d9 100644 --- a/src/lib/onboard/bridge-dns-preflight.ts +++ b/src/lib/onboard/bridge-dns-preflight.ts @@ -149,7 +149,11 @@ export function printDockerBridgeContainerStartFailure( * wall (mirroring the [[assertCdiNvidiaGpuSpecPresent]] resume backstop * pattern at #3152). */ -export function assertDockerBridgeAndContainerDnsHealthy(host: Host, nonInteractive = false): void { +export function assertDockerBridgeAndContainerDnsHealthy( + host: Host, + nonInteractive = false, + exitProcess: (code: number) => never = (code) => process.exit(code), +): void { // A minimal bridge-backed container start catches Docker/kernel failures // (notably Jetson veth "operation not supported") before longer gateway or // sandbox build work starts. Only veth/timeout/killed/daemon-unreachable @@ -167,7 +171,7 @@ export function assertDockerBridgeAndContainerDnsHealthy(host: Host, nonInteract bridgeStart.reason === "docker_daemon_unreachable" ) { printDockerBridgeContainerStartFailure(bridgeStart, host); - process.exit(1); + exitProcess(1); } else { console.warn( ` ⚠ Bridge container start probe inconclusive (reason: ${bridgeStart.reason ?? "unknown"}).`, @@ -233,7 +237,7 @@ export function assertDockerBridgeAndContainerDnsHealthy(host: Host, nonInteract }, host, ); - process.exit(1); + exitProcess(1); } if (dns.reason === "docker_daemon_unreachable") { printDockerBridgeContainerStartFailure( @@ -247,7 +251,7 @@ export function assertDockerBridgeAndContainerDnsHealthy(host: Host, nonInteract }, host, ); - process.exit(1); + exitProcess(1); } if (dns.reason === "timeout" || dns.reason === "killed") { console.error(" ✗ Container DNS probe did not complete."); @@ -276,7 +280,7 @@ export function assertDockerBridgeAndContainerDnsHealthy(host: Host, nonInteract } else { printContainerDnsRemediation(host); } - process.exit(1); + exitProcess(1); } /** diff --git a/src/lib/onboard/docker-driver-gateway-env.test.ts b/src/lib/onboard/docker-driver-gateway-env.test.ts index e2e234ed16d..c264c530c2c 100644 --- a/src/lib/onboard/docker-driver-gateway-env.test.ts +++ b/src/lib/onboard/docker-driver-gateway-env.test.ts @@ -1,9 +1,18 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, expect, it } from "vitest"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; -import { buildDockerDriverGatewayEnv } from "./docker-driver-gateway-env"; +import { describe, expect, it, vi } from "vitest"; + +import { + buildDockerDriverGatewayEnv, + buildDockerGatewayDebEnvFile, + startPackageManagedDockerDriverGatewayWithEnvOverride, + writeDockerGatewayDebEnvOverride, +} from "./docker-driver-gateway-env"; describe("buildDockerDriverGatewayEnv", () => { it("sets Docker-driver gateway networking from NemoClaw configuration", () => { @@ -53,3 +62,178 @@ describe("buildDockerDriverGatewayEnv", () => { expect(env.OPENSHELL_DRIVER_DIR).toBeUndefined(); }); }); + +describe("buildDockerGatewayDebEnvFile", () => { + it("replaces all managed gateway env keys and preserves unrelated values", () => { + const next = buildDockerGatewayDebEnvFile( + [ + "KEEP_ME=1", + "OPENSHELL_BIND_ADDRESS=127.0.0.1", + "OPENSHELL_SERVER_PORT=8080", + "OPENSHELL_DOCKER_SUPERVISOR_IMAGE=old", + "OPENSHELL_GATEWAY_CONFIG=/tmp/old.toml", + ].join("\n"), + { + OPENSHELL_DRIVERS: "docker", + OPENSHELL_BIND_ADDRESS: "0.0.0.0", + OPENSHELL_SERVER_PORT: "8990", + OPENSHELL_DISABLE_TLS: "true", + OPENSHELL_DISABLE_GATEWAY_AUTH: "true", + OPENSHELL_DB_URL: "sqlite:/tmp/openshell.db", + OPENSHELL_GRPC_ENDPOINT: "http://127.0.0.1:8990", + OPENSHELL_SSH_GATEWAY_HOST: "127.0.0.1", + OPENSHELL_SSH_GATEWAY_PORT: "8990", + OPENSHELL_DOCKER_NETWORK_NAME: "openshell-docker", + OPENSHELL_DOCKER_SUPERVISOR_IMAGE: "new", + OPENSHELL_GATEWAY_CONFIG: "/tmp/openshell-gateway.toml", + OPENSHELL_VM_DRIVER_STATE_DIR: "/tmp/old-vm-driver", + }, + ); + + expect(next).toContain("KEEP_ME=1\n"); + expect(next).toContain("OPENSHELL_BIND_ADDRESS=0.0.0.0\n"); + expect(next).toContain("OPENSHELL_SERVER_PORT=8990\n"); + expect(next).toContain("OPENSHELL_DOCKER_SUPERVISOR_IMAGE=new\n"); + expect(next).toContain("OPENSHELL_GATEWAY_CONFIG=/tmp/openshell-gateway.toml\n"); + expect(next).toContain("OPENSHELL_VM_DRIVER_STATE_DIR=/tmp/old-vm-driver\n"); + expect(next).not.toContain("OPENSHELL_BIND_ADDRESS=127.0.0.1"); + expect(next).not.toContain("OPENSHELL_DOCKER_SUPERVISOR_IMAGE=old"); + expect(next).not.toContain("OPENSHELL_GATEWAY_CONFIG=/tmp/old.toml"); + }); + + it("removes stale VM driver env keys when writing a Docker-driver env file", () => { + const next = buildDockerGatewayDebEnvFile( + [ + "OPENSHELL_DRIVERS=vm", + "OPENSHELL_VM_DRIVER_STATE_DIR=/tmp/old-vm-driver", + "OPENSHELL_DRIVER_DIR=/tmp/old-driver-dir", + ].join("\n"), + { + OPENSHELL_DRIVERS: "docker", + }, + ); + + expect(next).toBe("OPENSHELL_DRIVERS=docker\n"); + }); + + it("rejects multiline managed values", () => { + expect(() => + buildDockerGatewayDebEnvFile("", { + OPENSHELL_BIND_ADDRESS: "127.0.0.1\nINJECTED=1", + }), + ).toThrow("line break"); + }); +}); + +describe("writeDockerGatewayDebEnvOverride", () => { + it("enforces restrictive permissions on an existing env directory and file", () => { + const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-gateway-env-")); + const envDir = path.join(tempHome, ".config", "openshell"); + const envFile = path.join(envDir, "gateway.env"); + fs.mkdirSync(envDir, { recursive: true, mode: 0o755 }); + fs.chmodSync(envDir, 0o755); + fs.writeFileSync(envFile, "KEEP_ME=1\n", { mode: 0o644 }); + fs.chmodSync(envFile, 0o644); + + const existsSpy = vi + .spyOn(fs, "existsSync") + .mockImplementation( + (candidate) => candidate === "/usr/lib/systemd/user/openshell-gateway.service", + ); + const homedirSpy = vi.spyOn(os, "homedir").mockReturnValue(tempHome); + + try { + const wrote = writeDockerGatewayDebEnvOverride( + () => ({ + OPENSHELL_BIND_ADDRESS: "127.0.0.1", + }), + { platform: "linux" }, + ); + + const envFileContent = fs.readFileSync(envFile, "utf-8"); + expect(wrote).toBe(true); + expect(fs.statSync(envDir).mode & 0o777).toBe(0o700); + expect(fs.statSync(envFile).mode & 0o777).toBe(0o600); + expect(envFileContent).toContain("KEEP_ME=1\n"); + expect(envFileContent).toContain("OPENSHELL_BIND_ADDRESS=127.0.0.1\n"); + } finally { + existsSpy.mockRestore(); + homedirSpy.mockRestore(); + fs.rmSync(tempHome, { recursive: true, force: true }); + } + }); + + it("does not write service env for standalone gateway binaries", () => { + const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-gateway-env-")); + const existsSpy = vi + .spyOn(fs, "existsSync") + .mockImplementation((candidate) => candidate === "/usr/bin/openshell-gateway"); + const homedirSpy = vi.spyOn(os, "homedir").mockReturnValue(tempHome); + + try { + const wrote = writeDockerGatewayDebEnvOverride( + () => ({ + OPENSHELL_BIND_ADDRESS: "127.0.0.1", + }), + { platform: "linux" }, + ); + + expect(wrote).toBe(false); + expect(fs.existsSync(path.join(tempHome, ".config", "openshell", "gateway.env"))).toBe(false); + } finally { + existsSpy.mockRestore(); + homedirSpy.mockRestore(); + fs.rmSync(tempHome, { recursive: true, force: true }); + } + }); + + it("writes the service env only when package-managed startup prepares the service", async () => { + const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-gateway-env-")); + const envFile = path.join(tempHome, ".config", "openshell", "gateway.env"); + const gatewayEnv = buildDockerDriverGatewayEnv({ + platform: "darwin", + stateDir: path.join(tempHome, "state"), + getDockerSupervisorImage: () => "ghcr.io/nvidia/openshell/supervisor:0.0.72", + resolveSandboxBin: () => null, + }); + const existsSpy = vi + .spyOn(fs, "existsSync") + .mockImplementation( + (candidate) => candidate === "/usr/lib/systemd/user/openshell-gateway.service", + ); + const homedirSpy = vi.spyOn(os, "homedir").mockReturnValue(tempHome); + + try { + await expect( + startPackageManagedDockerDriverGatewayWithEnvOverride({ + clearDockerDriverGatewayRuntimeFiles: vi.fn(), + exitOnFailure: false, + gatewayEnv, + gatewayName: "nemoclaw", + hasOpenShellGatewayUserService: () => true, + isDockerDriverGatewayReady: async () => true, + registerDockerDriverGatewayEndpoint: () => true, + runCaptureOpenshell: (args) => + args[0] === "status" + ? "Gateway: nemoclaw\nConnected" + : "Gateway: nemoclaw\nGateway endpoint: https://127.0.0.1:8080/", + skipSandboxBridgeReachability: false, + startOpenShellGatewayUserService: (opts) => { + opts?.prepareServiceEnv?.(); + return { attempted: true, fallbackAllowed: false, started: true }; + }, + verifySandboxBridgeGatewayReachableOrExit: async () => undefined, + }), + ).resolves.toBe(true); + + expect(fs.readFileSync(envFile, "utf-8")).toContain("OPENSHELL_BIND_ADDRESS=127.0.0.1\n"); + expect(fs.readFileSync(envFile, "utf-8")).toContain( + `OPENSHELL_GATEWAY_CONFIG=${gatewayEnv.OPENSHELL_GATEWAY_CONFIG}\n`, + ); + } finally { + existsSpy.mockRestore(); + homedirSpy.mockRestore(); + fs.rmSync(tempHome, { recursive: true, force: true }); + } + }); +}); diff --git a/src/lib/onboard/docker-driver-gateway-env.ts b/src/lib/onboard/docker-driver-gateway-env.ts index 22c7b88ca0f..69d71f4d863 100644 --- a/src/lib/onboard/docker-driver-gateway-env.ts +++ b/src/lib/onboard/docker-driver-gateway-env.ts @@ -46,6 +46,7 @@ export const DOCKER_DRIVER_GATEWAY_RUNTIME_ENV_KEYS = [ export interface BuildDockerDriverGatewayEnvOptions { platform?: NodeJS.Platform; + gatewayPort?: number; stateDir: string; dockerNetworkName?: string; getDockerSupervisorImage: () => string; @@ -63,12 +64,14 @@ export function getGatewayPortCheckOptions(): { host: string } { return { host: GATEWAY_BIND_ADDRESS }; } -export function getGatewayStartNetworkEnv(): Record { +export function getGatewayStartNetworkEnv( + gatewayPort: number = GATEWAY_PORT, +): Record { return { OPENSHELL_BIND_ADDRESS: GATEWAY_BIND_ADDRESS, - OPENSHELL_SERVER_PORT: String(GATEWAY_PORT), + OPENSHELL_SERVER_PORT: String(gatewayPort), OPENSHELL_SSH_GATEWAY_HOST: getGatewayConnectHost(), - OPENSHELL_SSH_GATEWAY_PORT: String(GATEWAY_PORT), + OPENSHELL_SSH_GATEWAY_PORT: String(gatewayPort), }; } @@ -181,8 +184,8 @@ export function assertDockerDriverGatewayAuthConfigSafe(gatewayEnv: Record { const env: Record = { OPENSHELL_DRIVERS: "docker", - ...getGatewayStartNetworkEnv(), + ...getGatewayStartNetworkEnv(gatewayPort), ...buildDockerDriverGatewayLocalTlsEnv(stateDir), OPENSHELL_DB_URL: `sqlite:${path.join(stateDir, "openshell.db")}`, - OPENSHELL_GRPC_ENDPOINT: getDockerDriverGatewayEndpoint(), + OPENSHELL_GRPC_ENDPOINT: getDockerDriverGatewayEndpoint(gatewayPort), OPENSHELL_DOCKER_NETWORK_NAME: dockerNetworkName, OPENSHELL_DOCKER_SUPERVISOR_IMAGE: getDockerSupervisorImage(), }; diff --git a/src/lib/onboard/docker-driver-gateway-launch.test.ts b/src/lib/onboard/docker-driver-gateway-launch.test.ts index 6198b78d6d5..2c5eec75327 100644 --- a/src/lib/onboard/docker-driver-gateway-launch.test.ts +++ b/src/lib/onboard/docker-driver-gateway-launch.test.ts @@ -127,9 +127,10 @@ describe("docker-driver-gateway-launch", () => { }); it("uses the host binary as the drift binary outside compatibility mode", () => { - withTempBinaries(({ dir, gatewayBin }) => { + withTempBinaries(({ dir, gatewayBin, sandboxBin }) => { const identity = buildDockerDriverGatewayRuntimeIdentity({ gatewayBin, + sandboxBin, stateDir: dir, platform: "linux", env: {}, @@ -140,6 +141,7 @@ describe("docker-driver-gateway-launch", () => { expect(identity.launch?.mode).toBe("host"); expect(identity.driftGatewayBin).toBe(gatewayBin); + expect(identity.desiredEnv.OPENSHELL_DOCKER_SUPERVISOR_BIN).toBe(sandboxBin); expect(identity.desiredEnv.OPENSHELL_GATEWAY_CONFIG).toBe( path.join(dir, "openshell-gateway.toml"), ); diff --git a/src/lib/onboard/docker-driver-gateway-launch.ts b/src/lib/onboard/docker-driver-gateway-launch.ts index aeb859889d9..1564d1ec982 100644 --- a/src/lib/onboard/docker-driver-gateway-launch.ts +++ b/src/lib/onboard/docker-driver-gateway-launch.ts @@ -93,6 +93,9 @@ type BuildGatewayLaunchOptions = { hostGlibcVersion?: string | null; requiredGlibcVersions?: string[]; ensureLocalTlsBundle?: boolean; + // Multi-gateway callers pass the selected name. The hardened config derives + // its JWT gateway identity from the already gateway-scoped state directory. + gatewayName?: string; // Default compatibility container name when NEMOCLAW_OPENSHELL_GATEWAY_COMPAT_CONTAINER_NAME // is unset. Callers pass a per-gateway-port name so a second sandbox's compat // container (and its pre-launch `docker rm`) cannot tear down the first @@ -166,25 +169,16 @@ export function buildDockerDriverGatewayRuntimeIdentity( options: BuildGatewayLaunchOptions, ): DockerDriverGatewayRuntimeIdentity { const launch = buildDockerDriverGatewayLaunch(options); - const desiredEnv = - launch.mode === "container" - ? { - ...options.gatewayEnv, - ...Object.fromEntries( - Object.entries(launch.env).filter( - ([key, val]) => key in options.gatewayEnv && typeof val === "string", - ) as [string, string][], - ), - ...(typeof launch.env.OPENSHELL_GATEWAY_CONFIG === "string" - ? { OPENSHELL_GATEWAY_CONFIG: launch.env.OPENSHELL_GATEWAY_CONFIG } - : {}), - } - : { - ...options.gatewayEnv, - ...(typeof launch.env.OPENSHELL_GATEWAY_CONFIG === "string" - ? { OPENSHELL_GATEWAY_CONFIG: launch.env.OPENSHELL_GATEWAY_CONFIG } - : {}), - }; + const desiredKeys = new Set([ + ...Object.keys(options.gatewayEnv), + "OPENSHELL_DOCKER_SUPERVISOR_BIN", + "OPENSHELL_GATEWAY_CONFIG", + ]); + const desiredEnv = Object.fromEntries( + Object.entries(launch.env).filter( + ([key, value]) => desiredKeys.has(key) && typeof value === "string", + ) as [string, string][], + ); return { launch, desiredEnv, diff --git a/src/lib/onboard/docker-driver-gateway-runtime.test.ts b/src/lib/onboard/docker-driver-gateway-runtime.test.ts index 8e5682fc14b..4388bf3630d 100644 --- a/src/lib/onboard/docker-driver-gateway-runtime.test.ts +++ b/src/lib/onboard/docker-driver-gateway-runtime.test.ts @@ -114,6 +114,22 @@ describe("docker-driver gateway runtime helpers", () => { } }); + it("uses the moving dev supervisor image for an explicit or detected dev runtime", () => { + const explicit = makeHelpers({ shouldUseOpenshellDevChannel: () => true }); + expect( + explicit.helpers.getDockerDriverGatewayEnv("openshell 0.0.72", "linux") + .OPENSHELL_DOCKER_SUPERVISOR_IMAGE, + ).toBe("ghcr.io/nvidia/openshell/supervisor:dev"); + + const detected = makeHelpers({ + isOpenshellDevVersion: (versionOutput) => String(versionOutput).includes("-dev."), + }); + expect( + detected.helpers.getDockerDriverGatewayEnv("openshell 0.0.72-dev.8+g7bce1223", "linux") + .OPENSHELL_DOCKER_SUPERVISOR_IMAGE, + ).toBe("ghcr.io/nvidia/openshell/supervisor:dev"); + }); + it("pins the stable 0.0.72 supervisor default while preserving an explicit override", () => { const image = (fallback: string) => makeHelpers({ diff --git a/src/lib/onboard/docker-driver-gateway-runtime.ts b/src/lib/onboard/docker-driver-gateway-runtime.ts index a2476c0d7dd..0036f6a1b27 100644 --- a/src/lib/onboard/docker-driver-gateway-runtime.ts +++ b/src/lib/onboard/docker-driver-gateway-runtime.ts @@ -9,11 +9,11 @@ import { resolveOpenshell } from "../adapters/openshell/resolve"; import { isErrnoException } from "../core/errno"; import * as dockerDriverGatewayRuntimeMarker from "./docker-driver-gateway-runtime-marker"; import { isLinuxDockerDriverGatewayEnabled } from "./docker-driver-platform"; +import * as gatewayBinding from "./gateway-binding"; import { gatewayProcessCmdlineMatches, OPENSHELL_GATEWAY_PROCESS_NAMES, } from "./gateway-process-identity"; -import * as gatewayBinding from "./gateway-binding"; import type { PortProbeResult } from "./preflight"; import * as vmDriverProcess from "./vm-driver-process"; @@ -35,7 +35,7 @@ type DockerDriverGatewayEnvModule = typeof import("./docker-driver-gateway-env") // attached to a Docker-driver gateway. These heuristics can be retired when // OpenShell owns and reports the same runtime identity fields directly. export interface DockerDriverGatewayRuntimeDeps { - gatewayPort: number; + gatewayPort: number | (() => number); getCachedOpenshellBinary(): string | null; getBlueprintMaxOpenshellVersion(): string | null; getInstalledOpenshellVersion(versionOutput?: string | null): string | null; @@ -100,10 +100,13 @@ export function createDockerDriverGatewayRuntimeHelpers(deps: DockerDriverGatewa const dockerDriverGatewayEnv: DockerDriverGatewayEnvModule = deps.loadDockerDriverGatewayEnv?.() ?? require("./docker-driver-gateway-env"); + const currentGatewayPort = () => + typeof deps.gatewayPort === "function" ? deps.gatewayPort() : deps.gatewayPort; + function getDockerDriverGatewayStateDir(): string { const configured = process.env.NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR; if (configured && configured.trim()) return path.resolve(configured.trim()); - const dir = gatewayBinding.resolveGatewayStateDirName(deps.gatewayPort); + const dir = gatewayBinding.resolveGatewayStateDirName(currentGatewayPort()); return path.join(os.homedir(), ".local", "state", "nemoclaw", dir); } @@ -179,6 +182,7 @@ export function createDockerDriverGatewayRuntimeHelpers(deps: DockerDriverGatewa ): Record { const gatewayEnv = dockerDriverGatewayEnv.buildDockerDriverGatewayEnv({ platform, + gatewayPort: currentGatewayPort(), stateDir: getDockerDriverGatewayStateDir(), dockerNetworkName: process.env.OPENSHELL_DOCKER_NETWORK_NAME || "openshell-docker", getDockerSupervisorImage: () => getOpenShellDockerSupervisorImage(versionOutput), @@ -233,7 +237,8 @@ export function createDockerDriverGatewayRuntimeHelpers(deps: DockerDriverGatewa return ( env.OPENSHELL_DRIVERS === "docker" || Boolean(env.OPENSHELL_DOCKER_SUPERVISOR_IMAGE) || - env.OPENSHELL_GRPC_ENDPOINT === dockerDriverGatewayEnv.getDockerDriverGatewayEndpoint() + env.OPENSHELL_GRPC_ENDPOINT === + dockerDriverGatewayEnv.getDockerDriverGatewayEndpoint(currentGatewayPort()) ); } @@ -321,7 +326,7 @@ export function createDockerDriverGatewayRuntimeHelpers(deps: DockerDriverGatewa { pid, desiredEnv, - endpoint: dockerDriverGatewayEnv.getDockerDriverGatewayEndpoint(), + endpoint: dockerDriverGatewayEnv.getDockerDriverGatewayEndpoint(currentGatewayPort()), gatewayBin, dockerHost: process.env.DOCKER_HOST || null, platform, diff --git a/src/lib/onboard/docker-gpu-local-inference.ts b/src/lib/onboard/docker-gpu-local-inference.ts index bbe7934a0ea..5f4600a856d 100644 --- a/src/lib/onboard/docker-gpu-local-inference.ts +++ b/src/lib/onboard/docker-gpu-local-inference.ts @@ -41,6 +41,7 @@ type DockerGpuLocalInferenceConfig = { type DockerGpuLocalInferenceOptions = { dockerDriverGateway: boolean; + gatewayPort?: number; dockerDesktopWsl?: boolean; env?: NodeJS.ProcessEnv; platform?: NodeJS.Platform; @@ -137,15 +138,21 @@ export async function enforceDockerGpuPatchPreserveNetwork( "loopback is not reachable from the sandbox network namespace, so OpenClaw routes through " + "the OpenShell-managed inference path (host networking is not needed for GPU device access).", ); - await (options.reverifyBridgeReachability ?? defaultReverifyBridgeReachability)(); + await ( + options.reverifyBridgeReachability ?? + (() => defaultReverifyBridgeReachability(options.gatewayPort)) + )(); return true; } /** Re-run the sandbox→gateway bridge reachability probe (with UFW auto-fix). */ -function defaultReverifyBridgeReachability(): Promise { +function defaultReverifyBridgeReachability(gatewayPort?: number): Promise { const { verifySandboxBridgeGatewayReachableOrExit } = require("./gateway-sandbox-reachability") as typeof import("./gateway-sandbox-reachability"); - return verifySandboxBridgeGatewayReachableOrExit(true, { skip: false }); + return verifySandboxBridgeGatewayReachableOrExit(true, { + skip: false, + ...(gatewayPort === undefined ? {} : { port: gatewayPort }), + }); } export type SandboxExecResult = { status: number; stdout: string; stderr: string } | null; diff --git a/src/lib/onboard/fatal-runtime-preflight.ts b/src/lib/onboard/fatal-runtime-preflight.ts new file mode 100644 index 00000000000..e5d8b34ddce --- /dev/null +++ b/src/lib/onboard/fatal-runtime-preflight.ts @@ -0,0 +1,89 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { detectGpu, type GpuDetection } from "../inference/nim"; +import { cliDisplayName } from "./branding"; +import { assertDockerBridgeAndContainerDnsHealthy } from "./bridge-dns-preflight"; +import { isLinuxDockerDriverGatewayEnabled } from "./docker-driver-platform"; +import { warnIfHostProxyMissesLoopback } from "./http-proxy-preflight"; +import { + assertCdiNvidiaGpuSpecPresent, + assessHost, + type HostAssessment, + planHostRemediation, +} from "./preflight"; +import { printRemediationActions } from "./remediation"; +import { resolveSandboxGpuConfig, type SandboxGpuConfig } from "./sandbox-gpu-mode"; +import { + resolveSandboxGpuFlagFromOptions, + validateSandboxGpuPreflight, +} from "./sandbox-gpu-preflight"; +import type { OnboardOptions } from "./types"; + +export type FatalRuntimePreflightOptions = Pick< + OnboardOptions, + "sandboxGpu" | "sandboxGpuDevice" | "gpu" | "noGpu" +> & { + optedOutGpuPassthrough?: boolean; +}; + +export interface FatalRuntimePreflightContext { + nonInteractive: boolean; + exitProcess?: (code: number) => never; +} + +export interface FatalRuntimePreflightResult { + gpu: GpuDetection | null; + host: HostAssessment; + sandboxGpuConfig: SandboxGpuConfig; +} + +const exitProcessByDefault = (code: number): never => process.exit(code); + +/** Reject runtimes that cannot support the OpenShell Docker-driver integration. */ +export function rejectUnsupportedContainerRuntime( + host: HostAssessment, + exitProcess: (code: number) => never = exitProcessByDefault, +): void { + if (isLinuxDockerDriverGatewayEnabled() && host.runtime === "podman") { + console.error(` ✗ ${cliDisplayName()} onboarding now uses OpenShell's Docker driver.`); + console.error(` Podman is not supported for this ${cliDisplayName()} integration path.`); + console.error(" Switch to Docker Engine and rerun onboarding."); + exitProcess(1); + } +} + +/** Run the non-mutating runtime gates shared by fresh, resume, and rebuild onboarding. */ +export function runFatalOnboardRuntimePreflight( + options: FatalRuntimePreflightOptions, + context: FatalRuntimePreflightContext, +): FatalRuntimePreflightResult { + const exitProcess = context.exitProcess ?? exitProcessByDefault; + const host = assessHost(); + if (!host.dockerReachable) { + console.error(" Docker is not reachable. Please fix Docker and try again."); + printRemediationActions(planHostRemediation(host)); + exitProcess(1); + } + rejectUnsupportedContainerRuntime(host, exitProcess); + console.log(" ✓ Docker is running"); + warnIfHostProxyMissesLoopback(); + const gpu = detectGpu(); + const sandboxGpuConfig = resolveSandboxGpuConfig(gpu, { + flag: resolveSandboxGpuFlagFromOptions(options), + device: options.sandboxGpuDevice ?? null, + }); + const explicitlyOptedOutGpuPassthrough = + options.optedOutGpuPassthrough === true || options.noGpu === true; + assertCdiNvidiaGpuSpecPresent( + host, + explicitlyOptedOutGpuPassthrough, + sandboxGpuConfig.hostGpuPlatform, + exitProcess, + ); + assertDockerBridgeAndContainerDnsHealthy(host, context.nonInteractive, exitProcess); + validateSandboxGpuPreflight(sandboxGpuConfig, {}, exitProcess); + if (host.runtime !== "unknown") console.log(` ✓ Container runtime: ${host.runtime}`); + if (host.notes.includes("Running under WSL")) console.log(" ⓘ Running under WSL"); + return { gpu, host, sandboxGpuConfig }; +} diff --git a/src/lib/onboard/gateway-binding.test.ts b/src/lib/onboard/gateway-binding.test.ts index f34627411b6..97aae82bb80 100644 --- a/src/lib/onboard/gateway-binding.test.ts +++ b/src/lib/onboard/gateway-binding.test.ts @@ -5,7 +5,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { DEFAULT_GATEWAY_PORT } from "../core/ports"; import { buildDockerDriverGatewayLaunch } from "./docker-driver-gateway-launch"; import { @@ -18,6 +18,7 @@ import { BASE_GATEWAY_COMPAT_CONTAINER_NAME, BASE_GATEWAY_NAME, BASE_GATEWAY_STATE_DIR_NAME, + createDynamicGatewayRuntimeHelpers, resolveGatewayCompatContainerName, resolveGatewayName, resolveGatewayPortFromName, @@ -25,6 +26,89 @@ import { resolveSandboxGatewayName, } from "./gateway-binding"; +describe("dynamic gateway runtime helpers", () => { + it("resolves every default probe from the current process-local gateway binding", async () => { + let gatewayName = "nemoclaw"; + let gatewayPort = 8080; + const probeGatewayHttpReady = vi.fn(async () => true); + const probeDockerDriverGatewayHttpReady = vi.fn(async () => true); + const probeGatewayTcpReady = vi.fn(async () => true); + const getGatewayClusterImageDrift = vi.fn(() => null); + const helpers = createDynamicGatewayRuntimeHelpers({ + getGatewayName: () => gatewayName, + getGatewayPort: () => gatewayPort, + getDockerDriverGatewayEndpoint: (port) => `http://127.0.0.1:${port}`, + getGatewayClusterImageDrift, + probeGatewayHttpReady, + probeDockerDriverGatewayHttpReady, + waitForGatewayHttpReadyBase: vi.fn(async () => true), + probeGatewayTcpReady, + }); + + expect(helpers.getDockerDriverGatewayEndpoint()).toBe("http://127.0.0.1:8080"); + await helpers.isGatewayHttpReady(); + await helpers.isDockerDriverGatewayHttpReady(); + await helpers.isGatewayTcpReady(250); + helpers.getGatewayClusterImageDrift(); + expect(probeGatewayHttpReady).toHaveBeenLastCalledWith( + undefined, + "http://127.0.0.1:8080/", + undefined, + ); + expect(probeDockerDriverGatewayHttpReady).toHaveBeenLastCalledWith( + undefined, + "http://127.0.0.1:8080/openshell.v1.OpenShell/Health", + ); + expect(probeGatewayTcpReady).toHaveBeenLastCalledWith(8080, 250); + expect(getGatewayClusterImageDrift).toHaveBeenLastCalledWith({ gatewayName: "nemoclaw" }); + + gatewayName = "nemoclaw-8081"; + gatewayPort = 8081; + expect(helpers.getDockerDriverGatewayEndpoint()).toBe("http://127.0.0.1:8081"); + await helpers.isGatewayHttpReady(); + helpers.getGatewayClusterImageDrift(); + expect(probeGatewayHttpReady).toHaveBeenLastCalledWith( + undefined, + "http://127.0.0.1:8081/", + undefined, + ); + expect(getGatewayClusterImageDrift).toHaveBeenLastCalledWith({ + gatewayName: "nemoclaw-8081", + }); + }); + + it("preserves explicit probe URLs and injects the bound default wait probe", async () => { + const probeGatewayHttpReady = vi.fn(async () => true); + const waitForGatewayHttpReadyBase = vi.fn(async (options) => { + expect(options.probe).toBeTypeOf("function"); + return options.probe?.(); + }); + const helpers = createDynamicGatewayRuntimeHelpers({ + getGatewayName: () => "nemoclaw-9090", + getGatewayPort: () => 9090, + getDockerDriverGatewayEndpoint: (port) => `http://127.0.0.1:${port}`, + getGatewayClusterImageDrift: vi.fn(() => null), + probeGatewayHttpReady, + probeDockerDriverGatewayHttpReady: vi.fn(async () => true), + waitForGatewayHttpReadyBase, + probeGatewayTcpReady: vi.fn(async () => true), + }); + + await helpers.isGatewayHttpReady(25, "https://probe.example/health", "POST"); + expect(probeGatewayHttpReady).toHaveBeenLastCalledWith( + 25, + "https://probe.example/health", + "POST", + ); + await expect(helpers.waitForGatewayHttpReady()).resolves.toBe(true); + expect(probeGatewayHttpReady).toHaveBeenLastCalledWith( + undefined, + "http://127.0.0.1:9090/", + undefined, + ); + }); +}); + describe("gateway-binding resolver (#4422)", () => { it("keeps the bare nemoclaw names for the default gateway port", () => { expect(resolveGatewayName(DEFAULT_GATEWAY_PORT)).toBe(BASE_GATEWAY_NAME); diff --git a/src/lib/onboard/gateway-binding.ts b/src/lib/onboard/gateway-binding.ts index 016fc29d11c..9d247985bff 100644 --- a/src/lib/onboard/gateway-binding.ts +++ b/src/lib/onboard/gateway-binding.ts @@ -189,14 +189,76 @@ export interface GatewayNameBoundClassifiers { */ export function createGatewayNameBoundClassifiers( state: typeof import("../state/gateway"), - gatewayName: string, + gatewayName: string | (() => string), ): GatewayNameBoundClassifiers { + const currentGatewayName = () => + typeof gatewayName === "function" ? gatewayName() : gatewayName; return { - hasStaleGateway: (gwInfoOutput = "") => state.hasStaleGateway(gwInfoOutput, gatewayName), - isSelectedGateway: (statusOutput = "") => state.isSelectedGateway(statusOutput, gatewayName), + hasStaleGateway: (gwInfoOutput = "") => + state.hasStaleGateway(gwInfoOutput, currentGatewayName()), + isSelectedGateway: (statusOutput = "") => + state.isSelectedGateway(statusOutput, currentGatewayName()), isGatewayHealthy: (statusOutput = "", gwInfoOutput = "", activeGatewayInfoOutput = "") => - state.isGatewayHealthy(statusOutput, gwInfoOutput, activeGatewayInfoOutput, gatewayName), + state.isGatewayHealthy( + statusOutput, + gwInfoOutput, + activeGatewayInfoOutput, + currentGatewayName(), + ), getGatewayReuseState: (statusOutput = "", gwInfoOutput = "", activeGatewayInfoOutput = "") => - state.getGatewayReuseState(statusOutput, gwInfoOutput, activeGatewayInfoOutput, gatewayName), + state.getGatewayReuseState( + statusOutput, + gwInfoOutput, + activeGatewayInfoOutput, + currentGatewayName(), + ), + }; +} + +export interface DynamicGatewayRuntimeDeps { + getGatewayName(): string; + getGatewayPort(): number; + getDockerDriverGatewayEndpoint: typeof import("./docker-driver-gateway-env").getDockerDriverGatewayEndpoint; + getGatewayClusterImageDrift: typeof import("../adapters/openshell/gateway-drift").getGatewayClusterImageDrift; + probeGatewayHttpReady: typeof import("./gateway-http-readiness").isGatewayHttpReady; + probeDockerDriverGatewayHttpReady: typeof import("./gateway-http-readiness").isDockerDriverGatewayHttpReady; + waitForGatewayHttpReadyBase: typeof import("./gateway-http-readiness").waitForGatewayHttpReady; + probeGatewayTcpReady: typeof import("./gateway-tcp-readiness").isGatewayTcpReady; +} + +/** Bind gateway probes and drift checks to the process-local dynamic gateway target. */ +export function createDynamicGatewayRuntimeHelpers(deps: DynamicGatewayRuntimeDeps) { + const getDockerDriverGatewayEndpoint = () => + deps.getDockerDriverGatewayEndpoint(deps.getGatewayPort()); + const getGatewayClusterImageDrift = () => + deps.getGatewayClusterImageDrift({ gatewayName: deps.getGatewayName() }); + const isGatewayHttpReady = (timeoutMs?: number, url?: string, method?: "GET" | "POST") => + deps.probeGatewayHttpReady( + timeoutMs, + url ?? `${deps.getDockerDriverGatewayEndpoint(deps.getGatewayPort())}/`, + method, + ); + const isDockerDriverGatewayHttpReady = (timeoutMs?: number, url?: string) => + deps.probeDockerDriverGatewayHttpReady( + timeoutMs, + url ?? + `${deps.getDockerDriverGatewayEndpoint(deps.getGatewayPort())}/openshell.v1.OpenShell/Health`, + ); + const waitForGatewayHttpReady = ( + opts: import("./gateway-http-readiness").WaitForGatewayHttpReadyOpts = {}, + ) => + deps.waitForGatewayHttpReadyBase({ + ...opts, + probe: opts.probe ?? (() => isGatewayHttpReady()), + }); + const isGatewayTcpReady = (timeoutMs?: number) => + deps.probeGatewayTcpReady(deps.getGatewayPort(), timeoutMs); + return { + getDockerDriverGatewayEndpoint, + getGatewayClusterImageDrift, + isGatewayHttpReady, + isDockerDriverGatewayHttpReady, + waitForGatewayHttpReady, + isGatewayTcpReady, }; } diff --git a/src/lib/onboard/gateway-reuse.ts b/src/lib/onboard/gateway-reuse.ts index 5008cb52177..8dd41c8522f 100644 --- a/src/lib/onboard/gateway-reuse.ts +++ b/src/lib/onboard/gateway-reuse.ts @@ -11,7 +11,7 @@ export type GatewayReuseSnapshot = { }; export interface GatewayReuseDeps { - gatewayName: string; + gatewayName: string | (() => string); runCaptureOpenshell(args: string[], opts?: Record): string; runOpenshell(args: string[], opts?: Record): { status: number | null }; cliDisplayName(): string; @@ -23,9 +23,13 @@ export interface GatewayReuseHelpers { } export function createGatewayReuseHelpers(deps: GatewayReuseDeps): GatewayReuseHelpers { + const currentGatewayName = () => + typeof deps.gatewayName === "function" ? deps.gatewayName() : deps.gatewayName; + function getGatewayReuseSnapshot(): GatewayReuseSnapshot { + const gatewayName = currentGatewayName(); const gatewayStatus = deps.runCaptureOpenshell(["status"], { ignoreError: true }); - const gwInfo = deps.runCaptureOpenshell(["gateway", "info", "-g", deps.gatewayName], { + const gwInfo = deps.runCaptureOpenshell(["gateway", "info", "-g", gatewayName], { ignoreError: true, }); const activeGatewayInfo = deps.runCaptureOpenshell(["gateway", "info"], { ignoreError: true }); @@ -37,7 +41,7 @@ export function createGatewayReuseHelpers(deps: GatewayReuseDeps): GatewayReuseH gatewayStatus, gwInfo, activeGatewayInfo, - deps.gatewayName, + gatewayName, ), }; } @@ -45,18 +49,19 @@ export function createGatewayReuseHelpers(deps: GatewayReuseDeps): GatewayReuseH function selectNamedGatewayForReuseIfNeeded( snapshot: GatewayReuseSnapshot, ): GatewayReuseSnapshot { + const gatewayName = currentGatewayName(); if ( !shouldSelectNamedGatewayForReuse( snapshot.gatewayStatus, snapshot.gwInfo, snapshot.activeGatewayInfo, - deps.gatewayName, + gatewayName, ) ) { return snapshot; } - const selectResult = deps.runOpenshell(["gateway", "select", deps.gatewayName], { + const selectResult = deps.runOpenshell(["gateway", "select", gatewayName], { ignoreError: true, suppressOutput: true, }); @@ -66,7 +71,7 @@ export function createGatewayReuseHelpers(deps: GatewayReuseDeps): GatewayReuseH const refreshed = getGatewayReuseSnapshot(); if (refreshed.gatewayReuseState === "healthy") { - process.env.OPENSHELL_GATEWAY = deps.gatewayName; + process.env.OPENSHELL_GATEWAY = gatewayName; console.log(` ✓ Selected existing ${deps.cliDisplayName()} gateway`); } return refreshed; diff --git a/src/lib/onboard/gateway-sandbox-reachability.test.ts b/src/lib/onboard/gateway-sandbox-reachability.test.ts index 4b5f246c998..f2b08c09f45 100644 --- a/src/lib/onboard/gateway-sandbox-reachability.test.ts +++ b/src/lib/onboard/gateway-sandbox-reachability.test.ts @@ -626,11 +626,13 @@ describe("verifySandboxBridgeGatewayReachableOrExit host-gateway retry", () => { const log = vi.spyOn(console, "log").mockImplementation(() => undefined); try { await verifySandboxBridgeGatewayReachableOrExit(true, { + port: 19080, reachabilityImpl, retryAttempts: 3, retryDelayMs: 25, sleepMsImpl, }); + expect(reachabilityImpl).toHaveBeenCalledWith({ port: 19080 }); expect(reachabilityImpl).toHaveBeenCalledTimes(2); expect(sleepMsImpl).toHaveBeenCalledTimes(1); expect(sleepMsImpl).toHaveBeenCalledWith(25); diff --git a/src/lib/onboard/gateway-sandbox-reachability.ts b/src/lib/onboard/gateway-sandbox-reachability.ts index 86c7f489a09..0afbfdb4bfd 100644 --- a/src/lib/onboard/gateway-sandbox-reachability.ts +++ b/src/lib/onboard/gateway-sandbox-reachability.ts @@ -479,9 +479,9 @@ export function formatSandboxBridgeUnreachableMessage( interface SandboxBridgeVerifierOptions { skip?: boolean; port?: number; - reachabilityImpl?: () => - | Promise - | SandboxBridgeReachabilityResult; + reachabilityImpl?: (options?: { + port: number; + }) => Promise | SandboxBridgeReachabilityResult; autoApplyImpl?: ( reach: SandboxBridgeReachabilityResult, ) => Promise | UfwAutoApplyResult; @@ -523,7 +523,7 @@ export async function verifySandboxBridgeGatewayReachableOrExit( ((result: SandboxBridgeReachabilityResult) => tryAutoApplyUfwRule(result, { optedIn: true, port })); - let reach = await reachability(); + let reach = await reachability({ port }); if (reach.ok) return; const retryAttempts = options.retryAttempts ?? DEFAULT_HOST_GATEWAY_RETRY_ATTEMPTS; const retryDelayMs = options.retryDelayMs ?? DEFAULT_HOST_GATEWAY_RETRY_DELAY_MS; @@ -537,7 +537,7 @@ export async function verifySandboxBridgeGatewayReachableOrExit( ` Docker-driver sandbox bridge probe attempt ${attempt - 1}/${retryAttempts} failed (${reach.reason}); retrying in ${retryDelayMs} ms...`, ); await sleep(retryDelayMs); - reach = await reachability(); + reach = await reachability({ port }); if (reach.ok) { console.log( ` ✓ Docker-driver sandbox bridge reachable on attempt ${attempt}/${retryAttempts}`, @@ -558,7 +558,7 @@ export async function verifySandboxBridgeGatewayReachableOrExit( ? `allow from ${reach.subnet} to ${reach.gatewayIp}:${port}/tcp` : `allow sandbox bridge traffic to port ${port}/tcp`; console.log(` ✓ Applied UFW rule (NEMOCLAW_AUTO_FIX_FIREWALL=1): ${ruleDescription}`); - reach = await reachability(); + reach = await reachability({ port }); if (reach.ok) return; } else if (!SILENT_UFW_AUTO_APPLY_REASONS.has(autoApplyResult.reason)) { console.warn( diff --git a/src/lib/onboard/machine/core-flow-phases.test.ts b/src/lib/onboard/machine/core-flow-phases.test.ts index 2ee75f4d952..3d7972604f9 100644 --- a/src/lib/onboard/machine/core-flow-phases.test.ts +++ b/src/lib/onboard/machine/core-flow-phases.test.ts @@ -79,11 +79,12 @@ function createPhases( env: {}, constants: { hermesProviderName: "hermes", - hermesApiKeyAuthMethod: "api-key", + hermesApiKeyAuthMethod: "api_key", hermesApiKeyCredentialEnv: "HERMES_API_KEY", }, providerDeps: { - normalizeHermesAuthMethod: (value) => value ?? null, + normalizeHermesAuthMethod: (value) => + value === "oauth" || value === "api_key" ? value : null, setupNim: vi.fn(async () => ({ model: "nvidia/test", provider: "nim", @@ -150,6 +151,7 @@ function createPhases( getSandboxReuseState: () => "missing", hasSandboxGpuDrift: () => false, getSandboxHermesToolGateways: () => [], + getSandboxRegistryEntry: () => null, normalizeHermesToolGatewaySelections: (value) => (Array.isArray(value) ? value : []), stringSetsEqual: (left, right) => left.length === right.length && left.every((item) => right.includes(item)), diff --git a/src/lib/onboard/machine/core-flow-phases.ts b/src/lib/onboard/machine/core-flow-phases.ts index 9f8f1f16268..688a1c5228c 100644 --- a/src/lib/onboard/machine/core-flow-phases.ts +++ b/src/lib/onboard/machine/core-flow-phases.ts @@ -26,6 +26,7 @@ export interface CoreOnboardFlowPhaseOptions< ResourceProfile = unknown, > { forceProviderSelection: boolean; + authoritativeResumeConfig?: boolean; env: NodeJS.ProcessEnv; constants: ProviderInferenceStateOptions["constants"]; providerDeps: ProviderInferenceStateOptions["deps"]; @@ -61,6 +62,7 @@ export function createCoreOnboardFlowPhases< sandboxName: context.sandboxName, agent: context.agent, forceProviderSelection: options.forceProviderSelection, + authoritativeResumeConfig: options.authoritativeResumeConfig, initial: { model: context.model, provider: context.provider, @@ -102,6 +104,7 @@ export function createCoreOnboardFlowPhases< const sandboxStateResult = await handleSandboxState({ resume: context.resume, fresh: context.fresh, + authoritativeResumeConfig: options.authoritativeResumeConfig, resumeAgentChanged: options.sandbox.resumeAgentChanged, session: context.session, sandboxName: context.sandboxName, @@ -116,6 +119,7 @@ export function createCoreOnboardFlowPhases< preferredInferenceApi: context.preferredInferenceApi, sandboxGpuConfig: context.sandboxGpuConfig, hermesToolGateways: context.hermesToolGateways, + hermesAuthMethod: context.hermesAuthMethod, controlUiPort: options.sandbox.controlUiPort, rootDir: options.sandbox.rootDir, env: options.env, diff --git a/src/lib/onboard/machine/flow-context.ts b/src/lib/onboard/machine/flow-context.ts index b3a7cf9e9eb..060ad49759b 100644 --- a/src/lib/onboard/machine/flow-context.ts +++ b/src/lib/onboard/machine/flow-context.ts @@ -18,7 +18,7 @@ export interface OnboardFlowContext value ?? null, + normalizeHermesAuthMethod: (value: string | null | undefined) => + value === "oauth" || value === "api_key" ? value : null, setupNim: calls.setupNim, setupInference: calls.setupInference, startRecordedStep: calls.startStep, @@ -296,6 +297,41 @@ describe("handleProviderInferenceState", () => { expect(calls.setupInference).toHaveBeenCalled(); }); + it("uses a preflighted authoritative rebuild selection despite an incomplete old step marker", async () => { + const session = createSession({ + provider: "compatible-endpoint", + model: "mock/mcp-bridge", + endpointUrl: "https://compatible.example.test/v1", + credentialEnv: "COMPATIBLE_API_KEY", + preferredInferenceApi: "openai-completions", + }); + const { deps, calls } = createDeps({ isInferenceRouteReady: vi.fn(() => true) }); + + const result = await handleProviderInferenceState({ + ...baseOptions(deps, session), + resume: true, + authoritativeResumeConfig: true, + sandboxName: "mcp-rebuild", + }); + + expect(calls.setupNim).not.toHaveBeenCalled(); + expect(calls.recoverProvider).toHaveBeenCalledWith("compatible-endpoint", "COMPATIBLE_API_KEY"); + expect(calls.complete).toHaveBeenCalledWith( + "provider_selection", + expect.objectContaining({ + provider: "compatible-endpoint", + model: "mock/mcp-bridge", + endpointUrl: "https://compatible.example.test/v1", + }), + ); + expect(result).toMatchObject({ + provider: "compatible-endpoint", + model: "mock/mcp-bridge", + endpointUrl: "https://compatible.example.test/v1", + preferredInferenceApi: "openai-completions", + }); + }); + it("clears non-NVIDIA provider credentials when inference setup fails", async () => { const setupNim = vi.fn(async () => ({ ...baseSelection, diff --git a/src/lib/onboard/machine/handlers/provider-inference.ts b/src/lib/onboard/machine/handlers/provider-inference.ts index a28d01dd1b9..5e4e75fd841 100644 --- a/src/lib/onboard/machine/handlers/provider-inference.ts +++ b/src/lib/onboard/machine/handlers/provider-inference.ts @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import type { WebSearchConfig } from "../../../inference/web-search"; -import type { Session, SessionUpdates } from "../../../state/onboard-session"; +import type { HermesAuthMethod, Session, SessionUpdates } from "../../../state/onboard-session"; import { withInferenceTrace, withProviderSelectionTrace } from "../../tracing"; import { advanceTo, type OnboardStateTransitionResult, retryTo } from "../result"; @@ -13,7 +13,7 @@ export interface ProviderSelectionResult { provider: string; endpointUrl: string | null; credentialEnv: string | null; - hermesAuthMethod: string | null; + hermesAuthMethod: HermesAuthMethod | null; hermesToolGateways: string[]; preferredInferenceApi: string | null; compatibleEndpointReasoning: string | null; @@ -30,12 +30,14 @@ export interface ProviderInferenceStateOptions { sandboxName: string | null; agent: Agent; forceProviderSelection?: boolean; + /** Trust the rebuild-preflighted session selection even if its old step marker is incomplete. */ + authoritativeResumeConfig?: boolean; initial: { model: string | null; provider: string | null; endpointUrl: string | null; credentialEnv: string | null; - hermesAuthMethod: string | null; + hermesAuthMethod: HermesAuthMethod | null; hermesToolGateways: string[]; preferredInferenceApi: string | null; compatibleEndpointReasoning: string | null; @@ -46,11 +48,11 @@ export interface ProviderInferenceStateOptions { env: NodeJS.ProcessEnv; constants: { hermesProviderName: string; - hermesApiKeyAuthMethod: string; + hermesApiKeyAuthMethod: HermesAuthMethod; hermesApiKeyCredentialEnv: string; }; deps: { - normalizeHermesAuthMethod(value: string | null | undefined): string | null; + normalizeHermesAuthMethod(value: string | null | undefined): HermesAuthMethod | null; setupNim( gpu: Gpu, sandboxName: string | null, @@ -63,7 +65,7 @@ export interface ProviderInferenceStateOptions { provider: string, endpointUrl: string | null, credentialEnv: string | null, - hermesAuthMethod: string | null, + hermesAuthMethod: HermesAuthMethod | null, hermesToolGateways: string[], options?: { allowToolsIncompatible?: boolean; skipHostInferenceSmoke?: boolean }, ): Promise; @@ -142,7 +144,7 @@ export interface ProviderInferenceStateResult { provider: string; endpointUrl: string | null; credentialEnv: string | null; - hermesAuthMethod: string | null; + hermesAuthMethod: HermesAuthMethod | null; hermesToolGateways: string[]; preferredInferenceApi: string | null; compatibleEndpointReasoning: string | null; @@ -214,6 +216,7 @@ export async function handleProviderInferenceState({ sandboxName, agent, forceProviderSelection: initialForceProviderSelection = false, + authoritativeResumeConfig = false, initial, selectedMessagingChannels, env, @@ -247,7 +250,7 @@ export async function handleProviderInferenceState({ const resumeProviderSelection = !forceProviderSelection && effectiveResume && - session?.steps?.provider_selection?.status === "complete" && + (authoritativeResumeConfig || session?.steps?.provider_selection?.status === "complete") && typeof provider === "string" && typeof model === "string"; let shouldRecordProviderSelection = false; @@ -261,6 +264,12 @@ export async function handleProviderInferenceState({ provider, model, }); + // Rebuild may be resuming a legacy session whose step marker was never + // completed even though the pre-delete registry selection was validated + // and rewritten into the session. Persist that trusted selection so a + // later plain `onboard --resume` recovery cannot fall back to ambient or + // default provider selection if the recreate fails after this point. + shouldRecordProviderSelection = authoritativeResumeConfig; const hydratedCredential = deps.hydrateCredentialEnv(credentialEnv); // A rebuild recreate may leave `openshell inference get` reporting the // same provider/model while the newly created messaging sandbox's diff --git a/src/lib/onboard/machine/handlers/sandbox.test.ts b/src/lib/onboard/machine/handlers/sandbox.test.ts index 653f095bc76..6b6bbd32c99 100644 --- a/src/lib/onboard/machine/handlers/sandbox.test.ts +++ b/src/lib/onboard/machine/handlers/sandbox.test.ts @@ -149,6 +149,12 @@ function createDeps( getSandboxReuseState: () => "missing", hasSandboxGpuDrift: () => false, getSandboxHermesToolGateways: () => [], + getSandboxRegistryEntry: (name: string) => ({ + name, + webSearchEnabled: false, + fromDockerfile: null, + hermesAuthMethod: null, + }), normalizeHermesToolGatewaySelections: (value: unknown) => Array.isArray(value) ? (value as string[]) : [], stringSetsEqual: (left: string[], right: string[]) => @@ -220,6 +226,7 @@ function baseOptions( preferredInferenceApi: "openai-completions", sandboxGpuConfig: { sandboxGpuEnabled: false, mode: "0" }, hermesToolGateways: [], + hermesAuthMethod: null, controlUiPort: null, rootDir: "/repo", env: {}, @@ -260,6 +267,7 @@ describe("handleSandboxState", () => { { sandboxGpuEnabled: false, mode: "0" }, null, [], + null, ); expect(calls.updateSandbox).toHaveBeenCalledWith( "my-assistant", @@ -286,6 +294,21 @@ describe("handleSandboxState", () => { }); }); + it("does not auto-enable web search from ambient credentials during authoritative rebuild", async () => { + const configureWebSearch = vi.fn(async () => ({ fetchEnabled: true as const })); + const { deps, calls } = createDeps({ configureWebSearch }); + + const result = await handleSandboxState({ + ...baseOptions(deps), + authoritativeResumeConfig: true, + env: { NEMOCLAW_WEB_SEARCH_PROVIDER: "tavily" }, + }); + + expect(configureWebSearch).not.toHaveBeenCalled(); + expect((calls.createSandbox.mock.calls[0] as unknown[] | undefined)?.[5]).toBeNull(); + expect(result.webSearchConfig).toBeNull(); + }); + it("removes the conflicting Hermes nous-web gateway when Tavily is selected", async () => { const { deps, calls } = createDeps(); @@ -310,6 +333,7 @@ describe("handleSandboxState", () => { expect.anything(), null, ["nous-audio"], + null, ); expect(result.hermesToolGateways).toEqual(["nous-audio"]); expect(calls.note).toHaveBeenCalledWith( @@ -351,6 +375,34 @@ describe("handleSandboxState", () => { expect(result.session).toBe(skippedSession); }); + it("backfills absent rebuild fidelity after validated sandbox reuse", async () => { + const session = createSession({ + sandboxName: "saved", + webSearchConfig: { fetchEnabled: true }, + hermesAuthMethod: "api_key", + }); + session.steps.sandbox.status = "complete"; + const { deps, calls } = createDeps({ + getSandboxReuseState: () => "ready", + getSandboxRegistryEntry: (name) => ({ name, nemoclawVersion: "0.1.0" }), + }); + + await handleSandboxState({ + ...baseOptions(deps, session), + resume: true, + sandboxName: "saved", + webSearchConfig: { fetchEnabled: true }, + hermesAuthMethod: "api_key", + }); + + expect(calls.updateSandbox).toHaveBeenCalledWith("saved", { + webSearchEnabled: true, + webSearchProvider: "brave", + fromDockerfile: null, + hermesAuthMethod: "api_key", + }); + }); + it("marks web search changed when recreate implicitly enables Tavily", async () => { const session = createSession({ sandboxName: "saved" }); session.steps.sandbox.status = "complete"; @@ -513,6 +565,7 @@ describe("handleSandboxState", () => { { sandboxGpuEnabled: false, mode: "0" }, null, [], + null, ); expect(result.webSearchConfigChanged).toBe(true); }); @@ -546,6 +599,50 @@ describe("handleSandboxState", () => { expect(calls.createSandbox).not.toHaveBeenCalled(); }); + it("fails before credential or registry mutation when Tavily collides with managed MCP", async () => { + const session = createSession({ + sandboxName: "saved", + webSearchConfig: { fetchEnabled: true, provider: "brave" }, + }); + session.steps.sandbox.status = "complete"; + const { deps, calls } = createDeps({ + getSandboxReuseState: () => "ready", + agentSupportsWebSearchProvider: () => true, + getSandboxRegistryEntry: (name: string) => ({ + name, + mcp: { + bridges: { + search: { + server: "search", + agent: "openclaw", + url: "https://mcp.example.com/mcp", + env: ["TAVILY_API_KEY"], + policyName: "saved-mcp-search", + addedAt: "2026-07-03T00:00:00.000Z", + }, + }, + }, + }), + }); + + await expect( + handleSandboxState({ + ...baseOptions(deps, session), + resume: true, + sandboxName: "saved", + webSearchConfig: { fetchEnabled: true, provider: "brave" }, + env: { NEMOCLAW_WEB_SEARCH_PROVIDER: "tavily" }, + }), + ).rejects.toThrow("exit 1"); + + expect(calls.error).toHaveBeenCalledWith( + expect.stringContaining("already owns TAVILY_API_KEY"), + ); + expect(calls.validateBrave).not.toHaveBeenCalled(); + expect(calls.removeSandbox).not.toHaveBeenCalled(); + expect(calls.createSandbox).not.toHaveBeenCalled(); + }); + it("drops saved web search config when credential revalidation returns to provider selection", async () => { const session = createSession({ sandboxName: "saved", @@ -581,6 +678,7 @@ describe("handleSandboxState", () => { { sandboxGpuEnabled: false, mode: "0" }, null, [], + null, ); expect(result.webSearchConfig).toBeNull(); }); diff --git a/src/lib/onboard/machine/handlers/sandbox.ts b/src/lib/onboard/machine/handlers/sandbox.ts index 74d0ae5fffc..1e435360236 100644 --- a/src/lib/onboard/machine/handlers/sandbox.ts +++ b/src/lib/onboard/machine/handlers/sandbox.ts @@ -6,11 +6,13 @@ import { type WebSearchConfig as SharedWebSearchConfig, WEB_SEARCH_PROVIDER_ENV, webSearchConfigsEqual, + webSearchEnvFor, webSearchLabelFor, webSearchProviderForConfig, } from "../../../inference/web-search"; import type { SandboxMessagingPlan } from "../../../messaging/manifest"; -import type { Session, SessionUpdates } from "../../../state/onboard-session"; +import type { HermesAuthMethod, Session, SessionUpdates } from "../../../state/onboard-session"; +import type { SandboxEntry } from "../../../state/registry"; import { withSandboxPhaseTrace } from "../../tracing"; import { branchTo, type OnboardStateTransitionResult } from "../result"; import { reconcileReusedSandboxMessaging, reconcileSandboxMessaging } from "./sandbox-messaging"; @@ -30,6 +32,8 @@ export interface SandboxStateOptions< > { resume: boolean; fresh: boolean; + /** Internal rebuild mode: null web-search state is an authoritative disable, not a prompt. */ + authoritativeResumeConfig?: boolean; resumeAgentChanged: boolean; session: Session | null; sandboxName: string | null; @@ -44,6 +48,7 @@ export interface SandboxStateOptions< preferredInferenceApi: string | null; sandboxGpuConfig: SandboxGpuConfig; hermesToolGateways: string[]; + hermesAuthMethod: HermesAuthMethod | null; controlUiPort: number | null; rootDir: string; env: NodeJS.ProcessEnv; @@ -76,6 +81,7 @@ export interface SandboxStateOptions< getSandboxReuseState(sandboxName: string | null): string; hasSandboxGpuDrift(sandboxName: string, config: SandboxGpuConfig): boolean; getSandboxHermesToolGateways(sandboxName: string): unknown; + getSandboxRegistryEntry(sandboxName: string): SandboxEntry | null; normalizeHermesToolGatewaySelections(value: unknown): string[]; stringSetsEqual(left: string[], right: string[]): boolean; removeSandboxFromRegistry(sandboxName: string): void; @@ -123,6 +129,7 @@ export interface SandboxStateOptions< sandboxGpuConfig: SandboxGpuConfig, resourceProfile: ResourceProfile | null, hermesToolGateways: string[], + hermesAuthMethod: HermesAuthMethod | null, ): Promise; updateSandboxRegistry(sandboxName: string, updates: Record): void; getSandboxAgentRegistryFields( @@ -144,6 +151,7 @@ export interface SandboxStateOptions< metadata?: Record | null; }, ): Promise; + withSandboxMutationLock?(sandboxName: string, action: () => Promise): Promise; error(message?: string): void; exitProcess(code: number): never; }; @@ -174,13 +182,31 @@ interface SandboxStepState { function resolveRequestedWebSearchConfig( current: WebSearchConfig | null, env: NodeJS.ProcessEnv, + authoritative: boolean, ): WebSearchConfig | null { + if (authoritative) return current; const explicit = parseExplicitWebSearchProvider(env[WEB_SEARCH_PROVIDER_ENV]); if (!explicit.specified) return current; if (!explicit.provider) return null; return { fetchEnabled: true, provider: explicit.provider } as WebSearchConfig; } +function missingWebSearchFidelity( + existing: SandboxEntry | null, + webSearchConfig: SharedWebSearchConfig | null, +): Partial { + const fidelity: Partial = {}; + if (existing?.webSearchEnabled === undefined) { + fidelity.webSearchEnabled = Boolean(webSearchConfig); + } + if (existing?.webSearchProvider === undefined) { + fidelity.webSearchProvider = webSearchConfig + ? webSearchProviderForConfig(webSearchConfig) + : null; + } + return fidelity; +} + function knownAgentSupportsWebSearchProvider( agent: { name?: string } | null, provider: "brave" | "tavily", @@ -203,6 +229,32 @@ function effectiveHermesToolGatewaysForWebSearch( type SandboxCreationDecision = Exclude; +function mcpRegistryRemovalBlockReason( + decision: SandboxCreationDecision, + sandboxName: string | null, + webSearchConfig: SharedWebSearchConfig | null, + getSandboxRegistryEntry: (sandboxName: string) => SandboxEntry | null, +): string | null { + if (decision.kind !== "recreate") return null; + if (!decision.removeRegistryEntry) return null; + if (!sandboxName) return null; + const mcpState = getSandboxRegistryEntry(sandboxName)?.mcp; + if (!mcpState) return null; + + const selectedProvider = webSearchConfig ? webSearchProviderForConfig(webSearchConfig) : null; + if (selectedProvider) { + const credentialEnv = webSearchEnvFor(selectedProvider); + const collidingBridge = Object.values(mcpState.bridges).find((entry) => + entry.env.includes(credentialEnv), + ); + if (collidingBridge) { + return ` Cannot enable ${webSearchLabelFor(selectedProvider)}: MCP server '${collidingBridge.server}' already owns ${credentialEnv}. Use a distinct credential name.`; + } + } + + return ` Sandbox '${sandboxName}' has managed MCP state. Use the transactional rebuild command before changing settings that recreate the sandbox.`; +} + class SandboxStateFlow< Gpu, Agent, @@ -245,6 +297,7 @@ class SandboxStateFlow< const requestedWebSearchConfig = resolveRequestedWebSearchConfig( this.options.webSearchConfig, this.options.env, + this.options.authoritativeResumeConfig === true, ); const webSearchConfigChanged = !webSearchConfigsEqual( this.options.session?.webSearchConfig, @@ -357,6 +410,7 @@ class SandboxStateFlow< return current; }); } + this.backfillReusedSandboxFidelity(state); this.deps.skippedStepMessage("sandbox", state.sandboxName); const skippedSession = await this.deps.recordStateSkipped("sandbox", { reason: "resume", @@ -369,10 +423,32 @@ class SandboxStateFlow< }; } + private backfillReusedSandboxFidelity(state: SandboxStepState): void { + if (!state.sandboxName) return; + const existing = this.deps.getSandboxRegistryEntry(state.sandboxName); + const fidelity = missingWebSearchFidelity( + existing, + state.webSearchConfig as unknown as SharedWebSearchConfig | null, + ); + if ( + existing?.fromDockerfile === undefined && + (this.options.fromDockerfile || existing?.nemoclawVersion) + ) { + fidelity.fromDockerfile = this.options.fromDockerfile; + } + if (existing?.hermesAuthMethod === undefined && this.options.hermesAuthMethod) { + fidelity.hermesAuthMethod = this.options.hermesAuthMethod; + } + if (Object.keys(fidelity).length > 0) { + this.deps.updateSandboxRegistry(state.sandboxName, fidelity); + } + } + private async resolveWebSearchForCreation( state: SandboxStepState, ): Promise { if (!state.webSearchConfig) { + if (this.options.authoritativeResumeConfig) return null; return this.deps.configureWebSearch( null, this.options.agent, @@ -427,6 +503,7 @@ class SandboxStateFlow< this.options.sandboxGpuConfig, resourceProfile, effectiveHermesToolGateways, + this.options.hermesAuthMethod, ), ); // createSandbox() owns the build fingerprint. In particular, reusing an @@ -461,6 +538,16 @@ class SandboxStateFlow< state: SandboxStepState, decision: SandboxCreationDecision, ): Promise> { + const mcpBlockReason = mcpRegistryRemovalBlockReason( + decision, + state.sandboxName, + state.webSearchConfig as unknown as SharedWebSearchConfig | null, + this.deps.getSandboxRegistryEntry, + ); + if (mcpBlockReason) { + this.deps.error(mcpBlockReason); + return this.deps.exitProcess(1); + } const webSearchConfig = await this.resolveWebSearchForCreation(state); const webSearchConfigChanged = state.webSearchConfigChanged || @@ -567,5 +654,8 @@ export async function handleSandboxState< ResourceProfile >, ): Promise> { - return new SandboxStateFlow(options).run(); + const run = () => new SandboxStateFlow(options).run(); + return options.sandboxName && options.deps.withSandboxMutationLock + ? options.deps.withSandboxMutationLock(options.sandboxName, run) + : run(); } diff --git a/src/lib/onboard/openshell-feature-gate.test.ts b/src/lib/onboard/openshell-feature-gate.test.ts new file mode 100644 index 00000000000..dc42fc5e866 --- /dev/null +++ b/src/lib/onboard/openshell-feature-gate.test.ts @@ -0,0 +1,365 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; + +import { + hasRequiredOpenshellMessagingFeatures, + pinnedOpenShellSandboxBuildVersion, + REQUIRED_OPENSHELL_MCP_FEATURES, + REQUIRED_OPENSHELL_SANDBOX_MCP_FEATURE, + resolveOpenShellComponentBuildVersion, +} from "./openshell-feature-gate"; + +function writeExecutable(target: string, contents: string, version = "0.0.72") { + fs.writeFileSync( + target, + `#!/bin/sh +if [ "\${1:-}" = "--version" ]; then echo "${path.basename(target)} ${version}"; exit 0; fi +# ${contents} +exit 0 +`, + { mode: 0o755 }, + ); +} + +describe("OpenShell MCP feature gate", () => { + it("identifies the pinned v0.0.72 sandbox artifacts without executing them", () => { + const sandbox = path.join( + fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-openshell-features-")), + "openshell-sandbox", + ); + try { + writeExecutable(sandbox, "non-host-runnable sandbox"); + fs.writeFileSync( + sandbox, + `#!/bin/sh\nexit 127\n# ${REQUIRED_OPENSHELL_SANDBOX_MCP_FEATURE}\n`, + { mode: 0o755 }, + ); + const digest = "f9f991a24d10772ad5d24ae27a8ea6baad8cac671695bd90fcd0355e0e0ad198"; + const arm64Digest = "32ca44fe7d9e6d332f2a753c6b8a1a6117b7388281dad9b5274d23ffc67e216f"; + + expect(pinnedOpenShellSandboxBuildVersion(digest)).toBe("0.0.72"); + expect(pinnedOpenShellSandboxBuildVersion(arm64Digest)).toBe("0.0.72"); + expect(pinnedOpenShellSandboxBuildVersion("0".repeat(64))).toBeNull(); + expect(resolveOpenShellComponentBuildVersion(sandbox, "sandbox", () => digest)).toBe( + "0.0.72", + ); + expect(resolveOpenShellComponentBuildVersion(sandbox, "gateway", () => digest)).toBeNull(); + expect( + resolveOpenShellComponentBuildVersion(sandbox, "sandbox", () => "0".repeat(64)), + ).toBeNull(); + } finally { + fs.rmSync(path.dirname(sandbox), { recursive: true, force: true }); + } + }); + + it("finds provider rewrite and MCP L7 markers across OpenShell binaries", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-openshell-features-")); + try { + const openshell = path.join(dir, "openshell"); + const gateway = path.join(dir, "openshell-gateway"); + const sandbox = path.join(dir, "openshell-sandbox"); + writeExecutable(openshell, `binary ${REQUIRED_OPENSHELL_MCP_FEATURES[0]}`); + writeExecutable(gateway, `binary ${REQUIRED_OPENSHELL_MCP_FEATURES[1]}`); + writeExecutable( + sandbox, + `binary ${REQUIRED_OPENSHELL_MCP_FEATURES.slice(2).join(" ")} ${REQUIRED_OPENSHELL_SANDBOX_MCP_FEATURE}`, + ); + + expect( + hasRequiredOpenshellMessagingFeatures({ + openshellBin: openshell, + gatewayBin: null, + sandboxBin: null, + }), + ).toBe(true); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + + it("rejects mixed install roots unless the component paths are explicit overrides", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-openshell-features-")); + try { + const cliDir = path.join(root, "cli"); + const runtimeDir = path.join(root, "runtime"); + fs.mkdirSync(cliDir); + fs.mkdirSync(runtimeDir); + const openshell = path.join(cliDir, "openshell"); + const gateway = path.join(runtimeDir, "openshell-gateway"); + const sandbox = path.join(runtimeDir, "openshell-sandbox"); + writeExecutable(openshell, `binary ${REQUIRED_OPENSHELL_MCP_FEATURES.join(" ")}`); + writeExecutable(gateway, "selected external gateway"); + writeExecutable(sandbox, `binary ${REQUIRED_OPENSHELL_SANDBOX_MCP_FEATURE}`); + + const selected = { openshellBin: openshell, gatewayBin: gateway, sandboxBin: sandbox }; + expect(hasRequiredOpenshellMessagingFeatures(selected)).toBe(false); + expect( + hasRequiredOpenshellMessagingFeatures({ + ...selected, + gatewayBin: path.join(runtimeDir, "missing-gateway"), + allowExternalGatewayBin: true, + allowExternalSandboxBin: true, + }), + ).toBe(false); + expect( + hasRequiredOpenshellMessagingFeatures({ + ...selected, + allowExternalGatewayBin: true, + allowExternalSandboxBin: true, + }), + ).toBe(true); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); + + it("compares canonical roots so a symlink farm cannot combine releases", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-openshell-features-")); + try { + const linksDir = path.join(root, "links"); + const cliDir = path.join(root, "cli"); + const runtimeDir = path.join(root, "runtime"); + fs.mkdirSync(linksDir); + fs.mkdirSync(cliDir); + fs.mkdirSync(runtimeDir); + const realOpenshell = path.join(cliDir, "openshell"); + const realGateway = path.join(runtimeDir, "openshell-gateway"); + const realSandbox = path.join(runtimeDir, "openshell-sandbox"); + writeExecutable(realOpenshell, `binary ${REQUIRED_OPENSHELL_MCP_FEATURES.join(" ")}`); + writeExecutable(realGateway, "stale gateway"); + writeExecutable(realSandbox, `binary ${REQUIRED_OPENSHELL_SANDBOX_MCP_FEATURE}`); + const openshell = path.join(linksDir, "openshell"); + fs.symlinkSync(realOpenshell, openshell); + fs.symlinkSync(realGateway, path.join(linksDir, "openshell-gateway")); + fs.symlinkSync(realSandbox, path.join(linksDir, "openshell-sandbox")); + + expect( + hasRequiredOpenshellMessagingFeatures({ + openshellBin: openshell, + gatewayBin: null, + sandboxBin: null, + }), + ).toBe(false); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); + + it("rejects a selected component that is not executable", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-openshell-features-")); + try { + const openshell = path.join(root, "openshell"); + const gateway = path.join(root, "openshell-gateway"); + writeExecutable(openshell, `binary ${REQUIRED_OPENSHELL_MCP_FEATURES.join(" ")}`); + fs.writeFileSync(gateway, "non-executable gateway", { mode: 0o644 }); + + expect( + hasRequiredOpenshellMessagingFeatures({ + openshellBin: openshell, + gatewayBin: gateway, + sandboxBin: null, + }), + ).toBe(false); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); + + it("rejects a stale component copied into the active install root", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-openshell-features-")); + try { + const openshell = path.join(root, "openshell"); + const gateway = path.join(root, "openshell-gateway"); + writeExecutable(openshell, `binary ${REQUIRED_OPENSHELL_MCP_FEATURES.join(" ")}`); + writeExecutable(gateway, "stale gateway", "0.0.71"); + + expect( + hasRequiredOpenshellMessagingFeatures({ + openshellBin: openshell, + gatewayBin: gateway, + sandboxBin: null, + }), + ).toBe(false); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); + + it("accepts equivalent dev build identities with different git-prefix lengths", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-openshell-features-")); + try { + const openshell = path.join(root, "openshell"); + const gateway = path.join(root, "openshell-gateway"); + writeExecutable( + openshell, + `binary ${REQUIRED_OPENSHELL_MCP_FEATURES.join(" ")}`, + "0.0.72-dev.8+g7bce1223d", + ); + writeExecutable(gateway, "current gateway", "0.0.72-dev.8+g7bce1223"); + + expect( + hasRequiredOpenshellMessagingFeatures({ + openshellBin: openshell, + gatewayBin: gateway, + sandboxBin: null, + }), + ).toBe(true); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); + + it("rejects a selected sandbox runtime that cannot be read", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-openshell-features-")); + try { + const openshell = path.join(root, "openshell"); + const sandbox = path.join(root, "openshell-sandbox"); + writeExecutable(openshell, `binary ${REQUIRED_OPENSHELL_MCP_FEATURES.join(" ")}`); + fs.writeFileSync(sandbox, `binary ${REQUIRED_OPENSHELL_SANDBOX_MCP_FEATURE}`, { + mode: 0o111, + }); + + expect( + hasRequiredOpenshellMessagingFeatures({ + openshellBin: openshell, + gatewayBin: null, + sandboxBin: sandbox, + }), + ).toBe(false); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); + + it("fails closed when any required marker is absent", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-openshell-features-")); + try { + const openshell = path.join(dir, "openshell"); + writeExecutable(openshell, `binary ${REQUIRED_OPENSHELL_MCP_FEATURES[0]}`); + + expect( + hasRequiredOpenshellMessagingFeatures({ + openshellBin: openshell, + gatewayBin: null, + sandboxBin: null, + }), + ).toBe(false); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + + it("requires native MCP policy support from the exact sandbox runtime binary", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-openshell-features-")); + try { + const openshell = path.join(dir, "openshell"); + const sandbox = path.join(dir, "openshell-sandbox"); + writeExecutable( + openshell, + `binary ${REQUIRED_OPENSHELL_MCP_FEATURES.join(" ")} ${REQUIRED_OPENSHELL_SANDBOX_MCP_FEATURE}`, + ); + writeExecutable(sandbox, "binary without the transport boundary"); + + expect( + hasRequiredOpenshellMessagingFeatures({ + openshellBin: openshell, + gatewayBin: null, + sandboxBin: sandbox, + }), + ).toBe(false); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + + it("does not let a capable sibling rescue an explicit stale sandbox runtime", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-openshell-features-")); + try { + const cliDir = path.join(root, "cli"); + const runtimeDir = path.join(root, "runtime"); + fs.mkdirSync(cliDir); + fs.mkdirSync(runtimeDir); + const openshell = path.join(cliDir, "openshell"); + const siblingSandbox = path.join(cliDir, "openshell-sandbox"); + const selectedSandbox = path.join(runtimeDir, "openshell-sandbox"); + writeExecutable(openshell, `binary ${REQUIRED_OPENSHELL_MCP_FEATURES.join(" ")}`); + writeExecutable(siblingSandbox, `binary ${REQUIRED_OPENSHELL_SANDBOX_MCP_FEATURE}`); + writeExecutable(selectedSandbox, "stale sandbox without the MCP policy marker"); + + expect( + hasRequiredOpenshellMessagingFeatures({ + openshellBin: openshell, + gatewayBin: null, + sandboxBin: selectedSandbox, + allowExternalSandboxBin: true, + }), + ).toBe(false); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); + + it("defers a compressed VM supervisor check to the in-sandbox runtime probe", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-openshell-features-")); + try { + const openshell = path.join(dir, "openshell"); + const vmDriver = path.join(dir, "openshell-driver-vm"); + writeExecutable(openshell, `binary ${REQUIRED_OPENSHELL_MCP_FEATURES.join(" ")}`); + writeExecutable(vmDriver, "compressed supervisor payload without inspectable markers"); + + expect( + hasRequiredOpenshellMessagingFeatures({ + openshellBin: openshell, + gatewayBin: null, + sandboxBin: null, + }), + ).toBe(true); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + + it("ignores stale sibling and fallback sandbox artifacts for a macOS VM-driver install", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-openshell-features-")); + try { + const cliDir = path.join(root, "cli"); + const fallbackDir = path.join(root, "fallback"); + fs.mkdirSync(cliDir); + fs.mkdirSync(fallbackDir); + const openshell = path.join(cliDir, "openshell"); + const gateway = path.join(cliDir, "openshell-gateway"); + const siblingSandbox = path.join(cliDir, "openshell-sandbox"); + const fallbackSandbox = path.join(fallbackDir, "openshell-sandbox"); + writeExecutable(openshell, `binary ${REQUIRED_OPENSHELL_MCP_FEATURES.join(" ")}`); + writeExecutable(gateway, "current gateway"); + writeExecutable(siblingSandbox, "stale sibling sandbox", "0.0.44"); + writeExecutable(fallbackSandbox, "stale fallback sandbox", "0.0.44"); + + for (const sandboxBin of [siblingSandbox, fallbackSandbox]) { + expect( + hasRequiredOpenshellMessagingFeatures({ + openshellBin: openshell, + gatewayBin: gateway, + sandboxBin, + requireSandboxBin: false, + }), + ).toBe(true); + } + expect( + hasRequiredOpenshellMessagingFeatures({ + openshellBin: openshell, + gatewayBin: gateway, + sandboxBin: siblingSandbox, + requireSandboxBin: true, + }), + ).toBe(false); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/lib/onboard/openshell-feature-gate.ts b/src/lib/onboard/openshell-feature-gate.ts new file mode 100644 index 00000000000..90fc37ddc7c --- /dev/null +++ b/src/lib/onboard/openshell-feature-gate.ts @@ -0,0 +1,217 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; + +import { OPENSHELL_MCP_POLICY_CAPABILITY_MARKER } from "../adapters/openshell/runtime-capabilities"; + +/** + * Installation-integrity preflight shared by onboarding and install repair. + * + * This stays separate from either caller because it validates the selected + * host-visible OpenShell component set, rejects mixed or stale installations, + * and is also the single migration point for a future native capability + * command. Supervisor artifacts that are not host-visible remain subject to + * authoritative runtime policy verification. This gate does not authorize MCP + * mutations. + */ + +export const REQUIRED_OPENSHELL_MCP_FEATURES = [ + "request-body-credential-rewrite", + "websocket-credential-rewrite", + OPENSHELL_MCP_POLICY_CAPABILITY_MARKER, +] as const; + +export const REQUIRED_OPENSHELL_SANDBOX_MCP_FEATURE = OPENSHELL_MCP_POLICY_CAPABILITY_MARKER; + +function canonicalExecutableFile(candidate: string): string | null { + try { + const canonical = fs.realpathSync(candidate); + if (!fs.statSync(canonical).isFile()) return null; + fs.accessSync(canonical, fs.constants.R_OK | fs.constants.X_OK); + return canonical; + } catch { + return null; + } +} + +function pathEntryExists(candidate: string): boolean { + try { + fs.lstatSync(candidate); + return true; + } catch { + return false; + } +} + +const PINNED_SANDBOX_BUILD_VERSIONS = new Map([ + // OpenShell v0.0.72 standalone sandbox binaries. The Docker driver only + // bind-mounts these into the supervisor container, so the host may be too + // old to execute `--version` (the release requires GLIBC_2.39). + ["f9f991a24d10772ad5d24ae27a8ea6baad8cac671695bd90fcd0355e0e0ad198", "0.0.72"], + ["32ca44fe7d9e6d332f2a753c6b8a1a6117b7388281dad9b5274d23ffc67e216f", "0.0.72"], +]); + +export function pinnedOpenShellSandboxBuildVersion(sha256: string): string | null { + return PINNED_SANDBOX_BUILD_VERSIONS.get(sha256.toLowerCase()) ?? null; +} + +function executableSha256(candidate: string): string | null { + try { + return createHash("sha256").update(fs.readFileSync(candidate)).digest("hex"); + } catch { + return null; + } +} + +export function resolveOpenShellComponentBuildVersion( + candidate: string, + componentRole: "cli" | "gateway" | "sandbox", + digestFile: (path: string) => string | null = executableSha256, +): string | null { + const result = spawnSync(candidate, ["--version"], { + encoding: "utf8", + timeout: 5_000, + }); + if (result.status === 0 && !result.error) { + const version = `${result.stdout}${result.stderr}`.match(/\d+\.\d+\.\d+\S*/)?.[0]; + if (version) return version; + } + + // Never synthesize coherence from arbitrary version-like strings embedded + // in a binary. The fallback is sandbox-only and exact-digest pinned. + if (componentRole !== "sandbox") return null; + const digest = digestFile(candidate); + return digest ? pinnedOpenShellSandboxBuildVersion(digest) : null; +} + +function componentBuildVersionsMatch(left: string, right: string): boolean { + if (left === right) return true; + const leftGit = left.match(/^(.*\+g)([0-9a-f]{7,})$/i); + const rightGit = right.match(/^(.*\+g)([0-9a-f]{7,})$/i); + return Boolean( + leftGit && + rightGit && + leftGit[1] === rightGit[1] && + (leftGit[2].startsWith(rightGit[2]) || rightGit[2].startsWith(leftGit[2])), + ); +} + +// invalidState: a mixed or stale OpenShell installation appears feature-ready +// from version text alone. sourceBoundary: OpenShell owns component identity +// and the future native capability response; this scanner is an artifact and +// install-repair preflight only and never authorizes an MCP mutation. +// whyNotSourceFix: v0.0.72 has no structured installed-feature response. +// regressionTest: openshell-feature-gate.test.ts covers mixed roots, symlink +// farms, stale components, unreadable binaries, and the pinned sandbox digest. +// removalCondition: replace this scan when OpenShell exposes a versioned native +// capability command. Until then the running supervisor remains authoritative: +// MCP applies and exact-matches the generated policy with `policy set --wait` +// before provider credentials are created or updated. + +export function hasRequiredOpenshellMessagingFeatures(options: { + openshellBin: string | null; + gatewayBin: string | null; + sandboxBin: string | null; + allowExternalGatewayBin?: boolean; + allowExternalSandboxBin?: boolean; + requireSandboxBin?: boolean; +}): boolean { + if (!options.openshellBin) return false; + const selectedOpenshellBin = path.resolve(options.openshellBin); + const openshellBin = canonicalExecutableFile(selectedOpenshellBin); + if (!openshellBin) return false; + const openshellDir = path.dirname(openshellBin); + const selectedGatewayBin = options.gatewayBin + ? path.resolve(options.gatewayBin) + : path.join(path.dirname(selectedOpenshellBin), "openshell-gateway"); + const requireSandboxBin = options.requireSandboxBin ?? true; + const selectedSandboxBin = requireSandboxBin + ? options.sandboxBin + ? path.resolve(options.sandboxBin) + : path.join(path.dirname(selectedOpenshellBin), "openshell-sandbox") + : null; + const gatewayBin = canonicalExecutableFile(selectedGatewayBin); + const sandboxBin = selectedSandboxBin ? canonicalExecutableFile(selectedSandboxBin) : null; + if ((options.gatewayBin || pathEntryExists(selectedGatewayBin)) && !gatewayBin) return false; + if ( + selectedSandboxBin && + (options.sandboxBin || pathEntryExists(selectedSandboxBin)) && + !sandboxBin + ) { + return false; + } + if (gatewayBin && path.dirname(gatewayBin) !== openshellDir && !options.allowExternalGatewayBin) { + return false; + } + if (sandboxBin && path.dirname(sandboxBin) !== openshellDir && !options.allowExternalSandboxBin) { + return false; + } + const openshellVersion = resolveOpenShellComponentBuildVersion(openshellBin, "cli"); + if (!openshellVersion) return false; + for (const [componentBin, componentRole] of [ + [gatewayBin, "gateway"], + [sandboxBin, "sandbox"], + ] as const) { + if (!componentBin) continue; + const componentVersion = resolveOpenShellComponentBuildVersion(componentBin, componentRole); + if (!componentVersion || !componentBuildVersionsMatch(openshellVersion, componentVersion)) { + return false; + } + } + + // Scan one selected component set. Do not union arbitrary PATH fallbacks or + // let an explicit external component be rescued by a different sibling. + const candidates = [openshellBin, gatewayBin, sandboxBin].filter( + (candidate): candidate is string => candidate !== null, + ); + + const requiredMarkers = REQUIRED_OPENSHELL_MCP_FEATURES.map((marker) => Buffer.from(marker)); + const foundMarkers = new Set(); + const seen = new Set(); + for (const candidate of candidates) { + if (seen.has(candidate)) continue; + seen.add(candidate); + let content: Buffer; + let fd: number | null = null; + try { + fd = fs.openSync(candidate, "r"); + if (!fs.fstatSync(fd).isFile()) continue; + content = fs.readFileSync(fd); + } catch { + return false; + } finally { + if (fd !== null) fs.closeSync(fd); + } + for (let index = 0; index < requiredMarkers.length; index += 1) { + if (content.includes(requiredMarkers[index])) { + foundMarkers.add(REQUIRED_OPENSHELL_MCP_FEATURES[index]); + } + } + if (REQUIRED_OPENSHELL_MCP_FEATURES.every((marker) => foundMarkers.has(marker))) break; + } + if (!REQUIRED_OPENSHELL_MCP_FEATURES.every((marker) => foundMarkers.has(marker))) return false; + + // MCP policy enforcement and credential replacement execute in the sandbox + // supervisor. When that exact host artifact is available, require its native + // MCP marker rather than accepting a union of unrelated binaries. + const sandboxMarker = Buffer.from(REQUIRED_OPENSHELL_SANDBOX_MCP_FEATURE); + if (sandboxBin) { + try { + return fs.readFileSync(sandboxBin).includes(sandboxMarker); + } catch { + return false; + } + } + // VM drivers embed a compressed supervisor, so scanning their host binary is + // neither sufficient nor reliable. Some VM/Docker installations expose no + // supervisor host file at all. + // Returning true here means only that no install repair can be justified + // from host artifacts. The MCP command's authoritative runtime check loads + // the exact generated protocol:mcp policy with --wait and exact-matches the + // effective state before any credential or provider side effect. + return true; +} diff --git a/src/lib/onboard/openshell-install.test.ts b/src/lib/onboard/openshell-install.test.ts new file mode 100644 index 00000000000..95220e033a6 --- /dev/null +++ b/src/lib/onboard/openshell-install.test.ts @@ -0,0 +1,76 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { + ensureOpenshellForOnboard, + type OpenShellInstallDeps, + type OpenShellInstallResult, +} from "./openshell-install"; + +function makeDeps(overrides: Partial = {}) { + const installResult: OpenShellInstallResult = { + installed: true, + localBin: "/tmp/openshell", + futureShellPathHint: null, + }; + const deps: OpenShellInstallDeps = { + isLinuxDockerDriverGatewayEnabled: () => false, + resolveOpenShellGatewayBinary: () => "/tmp/openshell-gateway", + resolveOpenShellSandboxBinary: () => "/tmp/openshell-sandbox", + isOpenshellInstalled: () => true, + installOpenshell: vi.fn(() => installResult), + getInstalledOpenshellVersion: () => "0.0.72", + getBlueprintMinOpenshellVersion: () => "0.0.72", + getBlueprintMaxOpenshellVersion: () => "0.0.72", + runCaptureOpenshell: () => "openshell 0.0.72", + shouldUseOpenshellDevChannel: () => false, + isOpenshellDevVersion: () => false, + versionGte: (a, b) => + a.localeCompare(b, undefined, { + numeric: true, + sensitivity: "base", + }) >= 0, + hasRequiredOpenshellMessagingFeatures: () => true, + shouldAllowOpenshellAboveBlueprintMax: () => false, + cliDisplayName: () => "nemoclaw", + log: vi.fn(), + error: vi.fn(), + exit: (code: number): never => { + throw new Error(`exit ${code}`); + }, + platform: "linux", + arch: "x64", + ...overrides, + }; + return deps; +} + +describe("ensureOpenshellForOnboard", () => { + it("reinstalls when the installed OpenShell lacks messaging rewrite or MCP L7 support", () => { + const hasFeatures = vi.fn().mockReturnValueOnce(false).mockReturnValue(true); + const deps = makeDeps({ + hasRequiredOpenshellMessagingFeatures: hasFeatures, + }); + + ensureOpenshellForOnboard(deps); + + expect(deps.installOpenshell).toHaveBeenCalledTimes(1); + expect(deps.log).toHaveBeenCalledWith( + " OpenShell is missing provider credential rewrite or MCP L7 policy support. Reinstalling...", + ); + }); + + it("fails closed after reinstall if OpenShell still lacks messaging rewrite or MCP L7 support", () => { + const deps = makeDeps({ + hasRequiredOpenshellMessagingFeatures: () => false, + }); + + expect(() => ensureOpenshellForOnboard(deps)).toThrow("exit 1"); + expect(deps.installOpenshell).toHaveBeenCalledTimes(1); + expect(deps.error).toHaveBeenCalledWith( + " \u2717 openshell is missing provider credential rewrite or MCP L7 policy support.", + ); + }); +}); diff --git a/src/lib/onboard/openshell-install.ts b/src/lib/onboard/openshell-install.ts index 1a1e4c8133c..4c13d70ff6a 100644 --- a/src/lib/onboard/openshell-install.ts +++ b/src/lib/onboard/openshell-install.ts @@ -8,9 +8,20 @@ export type OpenShellInstallResult = { }; export type OpenshellInstallVersionResolution = - | { kind: "pin"; version: string; latest: string | null; reason: "latest" | "max-cap" } + | { + kind: "pin"; + version: string; + latest: string | null; + reason: "latest" | "max-cap"; + } | { kind: "no-max"; latest: string | null } - | { kind: "incompatible"; latest: string | null; max: string; message: string }; + | { + kind: "incompatible"; + latest: string | null; + min: string | null; + max: string; + message: string; + }; const SEMVER_TRIPLE = /^[0-9]+\.[0-9]+\.[0-9]+$/; @@ -35,9 +46,9 @@ export function parseOpenshellReleaseTag(tag: unknown): string | null { * * - If `options.max` is null, returns `kind: "no-max"` so callers leave the * install path alone (legacy behaviour — script picks its own pin / latest). - * - Otherwise, picks the highest entry of `available` that is `<= max`. - * Returns `kind: "incompatible"` with a message naming both latest and max - * when no such release exists. + * - Otherwise, picks the highest entry of `available` inside the inclusive + * `[min, max]` range. A missing or malformed min leaves the lower bound open. + * Returns `kind: "incompatible"` when no release is in range. * * Malformed entries in `available` (empty string, leading `-`, non-semver) are * silently dropped. The shipped blueprint guarantees `max` is valid before it @@ -45,7 +56,7 @@ export function parseOpenshellReleaseTag(tag: unknown): string | null { */ export function resolveOpenshellInstallVersion( available: readonly string[], - options: { max: string | null }, + options: { min?: string | null; max: string | null }, helpers: { versionGte: (a: string, b: string) => boolean }, ): OpenshellInstallVersionResolution { const sanitized = (available ?? []) @@ -59,22 +70,28 @@ export function resolveOpenshellInstallVersion( return { kind: "no-max", latest }; } - if (latest && helpers.versionGte(max, latest)) { - return { kind: "pin", version: latest, latest, reason: "latest" }; - } - - const capped = sanitized.find((entry) => helpers.versionGte(max, entry)); - if (capped) { - return { kind: "pin", version: capped, latest, reason: "max-cap" }; + const min = parseOpenshellReleaseTag(options.min ?? null); + const selected = sanitized.find( + (entry) => helpers.versionGte(max, entry) && (min === null || helpers.versionGte(entry, min)), + ); + if (selected) { + return { + kind: "pin", + version: selected, + latest, + reason: selected === latest ? "latest" : "max-cap", + }; } return { kind: "incompatible", latest, + min, max, message: - `No OpenShell release ≤ ${max} is available (latest published: ${latest ?? "unknown"}). ` + - "Upgrade NemoClaw or raise max_openshell_version in nemoclaw-blueprint/blueprint.yaml.", + `No OpenShell release in the supported range ${min ?? "0.0.0"} through ${max} is available ` + + `(latest published: ${latest ?? "unknown"}). Use an OpenShell build in that range or update ` + + "min_openshell_version and max_openshell_version in nemoclaw-blueprint/blueprint.yaml.", }; } @@ -100,6 +117,7 @@ export type OpenShellInstallDeps = { shouldUseOpenshellDevChannel: () => boolean; isOpenshellDevVersion: (versionOutput: string | null) => boolean; versionGte: (a: string, b: string) => boolean; + hasRequiredOpenshellMessagingFeatures: () => boolean; shouldAllowOpenshellAboveBlueprintMax: (versionOutput: string | null) => boolean; cliDisplayName: () => string; log: (message: string) => void; @@ -160,7 +178,9 @@ export function ensureOpenshellForOnboard(deps: OpenShellInstallDeps): OpenShell } } else { const minOpenshellVersion = deps.getBlueprintMinOpenshellVersion() ?? "0.0.72"; - const currentVersionOutput = deps.runCaptureOpenshell(["--version"], { ignoreError: true }); + const currentVersionOutput = deps.runCaptureOpenshell(["--version"], { + ignoreError: true, + }); const needsDevChannel = deps.isLinuxDockerDriverGatewayEnabled(platform, arch) && deps.shouldUseOpenshellDevChannel() && @@ -168,10 +188,12 @@ export function ensureOpenshellForOnboard(deps: OpenShellInstallDeps): OpenShell const needsDockerDriverBinaries = deps.isLinuxDockerDriverGatewayEnabled(platform, arch) && !areRequiredDockerDriverBinariesPresent(deps, platform, {}, arch); + const needsMessagingFeatures = !deps.hasRequiredOpenshellMessagingFeatures(); const needsUpgrade = !deps.versionGte(currentVersion, minOpenshellVersion) || needsDevChannel || - needsDockerDriverBinaries; + needsDockerDriverBinaries || + needsMessagingFeatures; if (needsUpgrade) { if (needsDevChannel) { deps.log(" OpenShell Docker-driver onboarding requires the dev channel. Upgrading..."); @@ -180,6 +202,10 @@ export function ensureOpenshellForOnboard(deps: OpenShellInstallDeps): OpenShell deps.log( ` OpenShell standalone gateway onboarding requires the ${required} binaries. Reinstalling...`, ); + } else if (needsMessagingFeatures) { + deps.log( + " OpenShell is missing provider credential rewrite or MCP L7 policy support. Reinstalling...", + ); } else { deps.log(` openshell ${currentVersion} is below minimum required version. Upgrading...`); } @@ -193,7 +219,9 @@ export function ensureOpenshellForOnboard(deps: OpenShellInstallDeps): OpenShell } } - const openshellVersionOutput = deps.runCaptureOpenshell(["--version"], { ignoreError: true }); + const openshellVersionOutput = deps.runCaptureOpenshell(["--version"], { + ignoreError: true, + }); deps.log(` \u2713 openshell CLI: ${openshellVersionOutput || "unknown"}`); const installedOpenshellVersion = deps.getInstalledOpenshellVersion(openshellVersionOutput); const minOpenshellVersion = deps.getBlueprintMinOpenshellVersion(); @@ -216,6 +244,18 @@ export function ensureOpenshellForOnboard(deps: OpenShellInstallDeps): OpenShell deps.exit(1); } + if (!deps.hasRequiredOpenshellMessagingFeatures()) { + deps.error(""); + deps.error( + " \u2717 openshell is missing provider credential rewrite or MCP L7 policy support.", + ); + deps.error(""); + deps.error(" Install a supported OpenShell build and retry:"); + deps.error(" https://github.com/NVIDIA/OpenShell/releases"); + deps.error(""); + deps.exit(1); + } + const maxOpenshellVersion = deps.getBlueprintMaxOpenshellVersion(); if ( installedOpenshellVersion && diff --git a/src/lib/onboard/openshell-pin.ts b/src/lib/onboard/openshell-pin.ts index 16e16d683d7..71a60d65b83 100644 --- a/src/lib/onboard/openshell-pin.ts +++ b/src/lib/onboard/openshell-pin.ts @@ -131,13 +131,14 @@ function listOpenshellReleaseTagsViaCurl(): string[] | null { export function resolveOpenshellInstallPin( deps: OpenshellInstallPinDeps, ): OpenshellInstallPinResult { + const minVersion = deps.getBlueprintMinOpenshellVersion?.() ?? null; const maxVersion = deps.getBlueprintMaxOpenshellVersion(); if (!maxVersion) return { kind: "no-max" }; const releases = (deps.listReleases ?? listOpenshellReleaseTags)(); if (releases === null || releases.length === 0) return { kind: "no-max" }; const resolution: OpenshellInstallVersionResolution = resolveOpenshellInstallVersion( releases, - { max: maxVersion }, + { min: minVersion, max: maxVersion }, { versionGte: deps.versionGte }, ); if (resolution.kind === "pin") { @@ -168,7 +169,12 @@ export function computeOpenshellInstallEnv( baseEnv: NodeJS.ProcessEnv, deps: OpenshellInstallPinDeps, ): OpenshellInstallEnvDirective { - const pin = resolveOpenshellInstallPin(deps); + const channel = (baseEnv.NEMOCLAW_OPENSHELL_CHANNEL ?? "auto").trim(); + // Dev installs already identify a non-stable build source. Stable release + // discovery must not block that current-main proof path merely because the + // next semver release has not been published yet. + const pin: OpenshellInstallPinResult = + channel === "dev" ? { kind: "no-max" } : resolveOpenshellInstallPin(deps); if (pin.kind === "incompatible") { const error = deps.error ?? ((m: string) => console.error(m)); error(""); @@ -182,6 +188,11 @@ export function computeOpenshellInstallEnv( if (blueprintMin) overlay.NEMOCLAW_OPENSHELL_MIN_VERSION = blueprintMin; if (blueprintMax) overlay.NEMOCLAW_OPENSHELL_MAX_VERSION = blueprintMax; if (pin.kind === "pin") overlay.NEMOCLAW_OPENSHELL_PIN_VERSION = pin.version; + if (channel === "dev") { + const env = { ...baseEnv, ...overlay }; + delete env.NEMOCLAW_OPENSHELL_PIN_VERSION; + return { env }; + } return Object.keys(overlay).length === 0 ? { env: baseEnv } : { env: { ...baseEnv, ...overlay } }; } @@ -202,6 +213,12 @@ export type RunOpenshellInstallDeps = OpenshellInstallPinDeps & { export function runOpenshellInstall(deps: RunOpenshellInstallDeps): OpenShellInstallResult { const { env } = computeOpenshellInstallEnv(process.env, deps); if (env === null) return { installed: false, localBin: null, futureShellPathHint: null }; + const installEnv = { ...env }; + for (const key of ["NEMOCLAW_OPENSHELL_GATEWAY_BIN", "NEMOCLAW_OPENSHELL_SANDBOX_BIN"] as const) { + const configured = installEnv[key]?.trim(); + if (configured) installEnv[key] = path.resolve(configured); + else delete installEnv[key]; + } // Stream install-openshell.sh output live (info() progress + curl progress bar) // so the in-onboard OpenShell upgrade shows progress instead of sitting silent // for the whole download/verify (#4431). `inherit` keeps this call synchronous @@ -209,7 +226,7 @@ export function runOpenshellInstall(deps: RunOpenshellInstallDeps): OpenShellIns // to the terminal in real time. const result = spawnSync("bash", [path.join(deps.scriptsDir, "install-openshell.sh")], { cwd: deps.cwd, - env, + env: installEnv, stdio: ["ignore", "inherit", "inherit"], timeout: 300_000, }); diff --git a/src/lib/onboard/preflight-runtime-resources.test.ts b/src/lib/onboard/preflight-runtime-resources.test.ts new file mode 100644 index 00000000000..59a4b4e4239 --- /dev/null +++ b/src/lib/onboard/preflight-runtime-resources.test.ts @@ -0,0 +1,79 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { assessHost, checkContainerRuntimeResources } from "./preflight"; + +function colimaHost(cpus = 2, memoryGiB = 2) { + return assessHost({ + platform: "darwin", + env: {}, + dockerInfoOutput: JSON.stringify({ + ServerVersion: "27.4.0", + OperatingSystem: "Colima", + NCPU: cpus, + MemTotal: memoryGiB * 1024 ** 3, + }), + commandExistsImpl: (name: string) => name === "docker", + }); +} + +describe("checkContainerRuntimeResources", () => { + it("aborts an interactive run when the user declines an undersized runtime", async () => { + const confirm = vi.fn(async () => false); + const exit = vi.fn((code: number): never => { + throw new Error(`exit:${code}`); + }); + const warn = vi.fn(); + const error = vi.fn(); + + await expect( + checkContainerRuntimeResources(colimaHost(), { + ignored: false, + nonInteractive: false, + confirm, + warn, + error, + exit, + }), + ).rejects.toThrow("exit:1"); + + expect(confirm).toHaveBeenCalledOnce(); + expect(warn.mock.calls.flat().join("\n")).toContain("2 vCPU / 2.0 GiB"); + expect(error).toHaveBeenCalledWith(expect.stringContaining("Aborted by user")); + expect(exit).toHaveBeenCalledWith(1); + }); + + it("warns but does not prompt in non-interactive mode", async () => { + const confirm = vi.fn(async () => false); + const warn = vi.fn(); + + await checkContainerRuntimeResources(colimaHost(), { + ignored: false, + nonInteractive: true, + confirm, + warn, + }); + + expect(confirm).not.toHaveBeenCalled(); + expect(warn.mock.calls.flat().join("\n")).toContain( + "Non-interactive mode is continuing despite under-provisioned runtime", + ); + }); + + it("honors the ignore override while still reporting detected capacity", async () => { + const confirm = vi.fn(async () => false); + const log = vi.fn(); + + await checkContainerRuntimeResources(colimaHost(), { + ignored: true, + nonInteractive: false, + confirm, + log, + }); + + expect(confirm).not.toHaveBeenCalled(); + expect(log).toHaveBeenCalledWith(" ✓ Container runtime resources: 2 vCPU / 2.0 GiB"); + }); +}); diff --git a/src/lib/onboard/preflight.ts b/src/lib/onboard/preflight.ts index c477a73fa6f..b3b7f299b3e 100644 --- a/src/lib/onboard/preflight.ts +++ b/src/lib/onboard/preflight.ts @@ -16,7 +16,6 @@ import os from "node:os"; import path from "node:path"; import { DASHBOARD_PORT } from "../core/ports"; -import { printRemediationActions } from "./remediation"; import { assessNvidiaCdiHost, buildNvidiaCdiRefreshCommands, @@ -28,10 +27,12 @@ import { extractCdiMismatchFilePath, getNvidiaCdiSpecPath, } from "./docker-cdi"; +import { printRemediationActions } from "./remediation"; import { isWslDockerDesktopRuntime, wslDockerDesktopGpuCompatibilityAction, } from "./wsl-docker-desktop-gpu"; + export { getNvidiaCdiSpecPath, parseDockerCdiSpecDirs } from "./docker-cdi"; export { isWslDockerDesktopRuntime } from "./wsl-docker-desktop-gpu"; @@ -363,6 +364,60 @@ export function isDockerUnderProvisioned( return cpuLow || memLow; } +export interface CheckContainerRuntimeResourcesOptions { + ignored: boolean; + nonInteractive: boolean; + confirm(): Promise; + log?: (message: string) => void; + warn?: (message: string) => void; + error?: (message: string) => void; + exit?: (code: number) => never; +} + +/** Report container capacity and gate interactive continuation when it is undersized. */ +export async function checkContainerRuntimeResources( + host: HostAssessment, + options: CheckContainerRuntimeResourcesOptions, +): Promise { + const log = options.log ?? console.log; + const warn = options.warn ?? console.warn; + const error = options.error ?? console.error; + const exit = options.exit ?? ((code: number): never => process.exit(code)); + const detected: string[] = []; + if (typeof host.dockerCpus === "number") detected.push(`${host.dockerCpus} vCPU`); + if (typeof host.dockerMemTotalBytes === "number") { + detected.push(`${(host.dockerMemTotalBytes / 1024 ** 3).toFixed(1)} GiB`); + } + if (!host.isContainerRuntimeUnderProvisioned || options.ignored) { + if (host.dockerReachable && detected.length > 0) { + log(` ✓ Container runtime resources: ${detected.join(" / ")}`); + } + return; + } + + warn( + ` ⚠ Container runtime under-provisioned: ${detected.join(" / ") || "unknown"} detected ` + + `(recommended: ${MIN_RECOMMENDED_DOCKER_CPUS} vCPU / ${MIN_RECOMMENDED_DOCKER_MEM_GIB} GiB).`, + ); + warn(" The sandbox build will be slow and may stall on default Colima settings."); + if (host.runtime === "colima") { + warn( + ` Suggested: colima stop && colima start --cpu ${MIN_RECOMMENDED_DOCKER_CPUS} --memory ${MIN_RECOMMENDED_DOCKER_MEM_GIB}`, + ); + } else if (host.runtime === "docker-desktop") { + warn(" Suggested: Docker Desktop → Settings → Resources, raise CPU/memory."); + } + warn(" Set NEMOCLAW_IGNORE_RUNTIME_RESOURCES=1 to silence this check."); + if (options.nonInteractive) { + warn(" WARNING: Non-interactive mode is continuing despite under-provisioned runtime."); + return; + } + if (!(await options.confirm())) { + error(" Aborted by user. Resize your container runtime and rerun `nemoclaw onboard`."); + exit(1); + } +} + function readDockerDefaultCgroupnsMode( readFileImpl: (filePath: string, encoding: BufferEncoding) => string, ): "host" | "private" | "unknown" { @@ -664,6 +719,7 @@ export function assertCdiNvidiaGpuSpecPresent( host: HostAssessment, explicitlyOptedOutGpuPassthrough: boolean, hostGpuPlatform: string | null | undefined = null, + exitProcess: (code: number) => never = (code) => process.exit(code), ): void { if (hostGpuPlatform === "jetson" || isWslDockerDesktopRuntime(host)) return; if ( @@ -678,7 +734,7 @@ export function assertCdiNvidiaGpuSpecPresent( " Docker is configured for CDI device injection (CDISpecDirs is set), but the NVIDIA GPU CDI spec is missing or stale. OpenShell GPU startup can fail until the CDI spec is refreshed.", ); printRemediationActions(planHostRemediation(host)); - process.exit(1); + exitProcess(1); } export function planHostRemediation(assessment: HostAssessment): RemediationAction[] { diff --git a/src/lib/onboard/providers.test.ts b/src/lib/onboard/providers.test.ts index 7edb51f0649..e9bd5905780 100644 --- a/src/lib/onboard/providers.test.ts +++ b/src/lib/onboard/providers.test.ts @@ -32,8 +32,14 @@ const { credentialEnv: string, baseUrl: string | null, ) => string[]; - getRequestedModelHint: (nonInteractive: boolean) => string | null; - getRequestedProviderHint: (nonInteractive: boolean) => string | null; + getRequestedModelHint: ( + nonInteractive: boolean, + allowHostedInferenceStaging?: boolean, + ) => string | null; + getRequestedProviderHint: ( + nonInteractive: boolean, + allowHostedInferenceStaging?: boolean, + ) => string | null; isProviderKeyCredentialCandidate: (value: string | null | undefined) => boolean; providerExistsInGateway: (name: string, runOpenshell: RunOpenshell) => boolean; stageHostedInferenceSourceSecretEnv: () => boolean; @@ -318,6 +324,21 @@ describe("onboard provider helpers", () => { ); }); + it("does not synthesize hosted selection when authoritative resume disables staging", () => { + withProviderEnv( + { + NVIDIA_INFERENCE_API_KEY: "repo-hosted-key", + }, + () => { + expect(getRequestedProviderHint(true, false)).toBeNull(); + expect(getRequestedModelHint(true, false)).toBeNull(); + expect(process.env.NEMOCLAW_PROVIDER).toBeUndefined(); + expect(process.env.NEMOCLAW_MODEL).toBeUndefined(); + expect(process.env.COMPATIBLE_API_KEY).toBeUndefined(); + }, + ); + }); + it("stages Deep Agents NEMOCLAW_PROVIDER_KEY as hosted custom inference", () => { withProviderEnv( { diff --git a/src/lib/onboard/providers.ts b/src/lib/onboard/providers.ts index 77a5eaceed0..99b3b665e62 100644 --- a/src/lib/onboard/providers.ts +++ b/src/lib/onboard/providers.ts @@ -212,8 +212,8 @@ function getEffectiveProviderName(providerKey) { // ── Non-interactive helpers ────────────────────────────────────── -function getNonInteractiveProvider() { - stageHostedInferenceSourceSecretEnv(); +function getNonInteractiveProvider(allowHostedInferenceStaging = true) { + if (allowHostedInferenceStaging) stageHostedInferenceSourceSecretEnv(); const providerKey = (process.env.NEMOCLAW_PROVIDER || "").trim().toLowerCase(); if (!providerKey) return null; const normalized = NON_INTERACTIVE_PROVIDER_ALIASES[providerKey] || providerKey; @@ -303,13 +303,14 @@ function getNonInteractiveModel(providerKey) { } // No default for nonInteractive — onboard.ts wrapper supplies isNonInteractive(). -function getRequestedProviderHint(nonInteractive) { - return nonInteractive ? getNonInteractiveProvider() : null; +function getRequestedProviderHint(nonInteractive, allowHostedInferenceStaging = true) { + return nonInteractive ? getNonInteractiveProvider(allowHostedInferenceStaging) : null; } -function getRequestedModelHint(nonInteractive) { +function getRequestedModelHint(nonInteractive, allowHostedInferenceStaging = true) { if (!nonInteractive) return null; - const providerKey = getRequestedProviderHint(nonInteractive) || "cloud"; + const providerKey = + getRequestedProviderHint(nonInteractive, allowHostedInferenceStaging) || "cloud"; return getNonInteractiveModel(providerKey); } diff --git a/src/lib/onboard/resume-config.test.ts b/src/lib/onboard/resume-config.test.ts new file mode 100644 index 00000000000..bf0e73f6af1 --- /dev/null +++ b/src/lib/onboard/resume-config.test.ts @@ -0,0 +1,33 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { getResumeConfigConflicts } from "./resume-config"; + +afterEach(() => { + vi.unstubAllEnvs(); +}); + +describe("authoritative rebuild resume config", () => { + it("ignores a hosted credential alias rehydrated after ambient env isolation", () => { + vi.stubEnv("NVIDIA_INFERENCE_API_KEY", "legacy-hosted-source-key"); + vi.stubEnv("NEMOCLAW_PROVIDER", ""); + vi.stubEnv("NEMOCLAW_MODEL", ""); + vi.stubEnv("COMPATIBLE_API_KEY", ""); + + expect( + getResumeConfigConflicts( + { + sandboxName: "mcp-rebuild", + provider: "compatible-endpoint", + model: "mock/mcp-bridge", + }, + { nonInteractive: true, authoritativeResumeConfig: true }, + ), + ).toEqual([]); + expect(process.env.NEMOCLAW_PROVIDER).toBe(""); + expect(process.env.NEMOCLAW_MODEL).toBe(""); + expect(process.env.COMPATIBLE_API_KEY).toBe(""); + }); +}); diff --git a/src/lib/onboard/resume-config.ts b/src/lib/onboard/resume-config.ts index dba45b93768..d39d4718e21 100644 --- a/src/lib/onboard/resume-config.ts +++ b/src/lib/onboard/resume-config.ts @@ -59,8 +59,11 @@ export function getResumeSandboxConflict( : null; } -export function getRequestedProviderHint(nonInteractive = false): string | null { - return onboardProviders.getRequestedProviderHint(nonInteractive); +export function getRequestedProviderHint( + nonInteractive = false, + allowHostedInferenceStaging = true, +): string | null { + return onboardProviders.getRequestedProviderHint(nonInteractive, allowHostedInferenceStaging); } /** @@ -69,14 +72,30 @@ export function getRequestedProviderHint(nonInteractive = false): string | null * preflight (#5207). Either may exit the process with a non-zero code on an * invalid value. */ -export function preflightEarlyOnboardEnv(nonInteractive = false): string | null { - const providerHint = getRequestedProviderHint(nonInteractive); +export function preflightEarlyOnboardEnv( + nonInteractive = false, + allowHostedInferenceStaging = true, +): string | null { + const providerHint = getRequestedProviderHint(nonInteractive, allowHostedInferenceStaging); preflightVllmModelEnvOrExit(); return providerHint; } -export function getRequestedModelHint(nonInteractive = false): string | null { - return onboardProviders.getRequestedModelHint(nonInteractive); +export function preflightEarlyOnboardEnvForResume( + nonInteractive: boolean, + authoritativeResumeConfig: boolean, +): string | null { + return preflightEarlyOnboardEnv( + authoritativeResumeConfig ? nonInteractive : false, + !authoritativeResumeConfig, + ); +} + +export function getRequestedModelHint( + nonInteractive = false, + allowHostedInferenceStaging = true, +): string | null { + return onboardProviders.getRequestedModelHint(nonInteractive, allowHostedInferenceStaging); } export function getResumeConfigConflicts( @@ -86,10 +105,17 @@ export function getResumeConfigConflicts( fromDockerfile?: string | null; sandboxName?: string | null; agent?: string | null; + /** + * Internal rebuild-resume mode: the caller already rewrote the session from + * validated registry state, so credential aliases must not synthesize a new + * provider/model request while checking that session for conflicts. + */ + authoritativeResumeConfig?: boolean; } = {}, ): ResumeConfigConflict[] { const conflicts: ResumeConfigConflict[] = []; const nonInteractive = opts.nonInteractive ?? false; + const allowHostedInferenceStaging = opts.authoritativeResumeConfig !== true; const sandboxConflict = getResumeSandboxConflict(session, { sandboxName: opts.sandboxName }); if (sandboxConflict) { @@ -100,7 +126,7 @@ export function getResumeConfigConflicts( }); } - const requestedProvider = getRequestedProviderHint(nonInteractive); + const requestedProvider = getRequestedProviderHint(nonInteractive, allowHostedInferenceStaging); const effectiveRequestedProvider = onboardProviders.getEffectiveProviderName(requestedProvider); if ( effectiveRequestedProvider && @@ -114,7 +140,7 @@ export function getResumeConfigConflicts( }); } - const requestedModel = getRequestedModelHint(nonInteractive); + const requestedModel = getRequestedModelHint(nonInteractive, allowHostedInferenceStaging); if (requestedModel && session?.model && requestedModel !== session.model) { conflicts.push({ field: "model", diff --git a/src/lib/onboard/sandbox-create-failure.ts b/src/lib/onboard/sandbox-create-failure.ts index d2838a23190..c05df67492b 100644 --- a/src/lib/onboard/sandbox-create-failure.ts +++ b/src/lib/onboard/sandbox-create-failure.ts @@ -8,6 +8,7 @@ import path from "node:path"; const ANSI_RE = /\x1B(?:\[[0-?]*[ -/]*[@-~]|\][^\x07]*(?:\x07|\x1B\\)|[@-_])/g; const UUID_RE = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/i; const MAX_RELEVANT_LOG_LINES = 120; +const MAX_GATEWAY_TAIL_LINES = 240; export type SandboxCreateFailureDiagnostics = { dir: string; @@ -16,6 +17,7 @@ export type SandboxCreateFailureDiagnostics = { stateDir: string | null; consoleOutput: string | null; copiedConsoleOutput: string | null; + gatewayTailPath: string | null; backupPath: string | null; summaryLines: string[]; }; @@ -171,6 +173,10 @@ export function collectSandboxCreateFailureDiagnostics( const block = rawLines ? findLatestSandboxBlock(rawLines, sandboxName) : []; const sandboxId = getLatestSandboxId(block, sandboxName); const relevantLines = filterRelevantLines(block, sandboxName, sandboxId); + const gatewayTailLines = + rawLines && relevantLines.length === 0 + ? rawLines.filter((line) => line.trim()).slice(-MAX_GATEWAY_TAIL_LINES) + : []; const stateDir = latestFieldValue(relevantLines, "state_dir"); const consoleOutput = latestFieldValue(relevantLines, "console_output") ?? @@ -191,11 +197,17 @@ export function collectSandboxCreateFailureDiagnostics( }, ); } + const gatewayTailPath = + gatewayTailLines.length > 0 ? path.join(dir, "openshell-gateway-tail.log") : null; + if (gatewayTailPath) { + fs.writeFileSync(gatewayTailPath, `${gatewayTailLines.join("\n")}\n`, { mode: 0o600 }); + } const summaryLines = [ `created_at=${now.toISOString()}`, `sandbox_name=${sandboxName}`, `sandbox_id=${sandboxId ?? "unknown"}`, `gateway_log=${gatewayLogPath ?? "not-found"}`, + `gateway_tail=${gatewayTailPath ?? "not-written"}`, `state_dir=${stateDir ?? "unknown"}`, `console_output=${consoleOutput ?? "unknown"}`, `copied_console_output=${copiedConsoleOutput ?? "not-copied"}`, @@ -216,7 +228,28 @@ export function collectSandboxCreateFailureDiagnostics( stateDir, consoleOutput, copiedConsoleOutput, + gatewayTailPath, backupPath, - summaryLines: relevantLines.slice(-8), + summaryLines: relevantLines.length > 0 ? relevantLines.slice(-8) : gatewayTailLines.slice(-8), }; } + +export function printSandboxCreateFailureDiagnostics( + sandboxName: string, + options: SandboxCreateFailureDiagnosticOptions = {}, +): SandboxCreateFailureDiagnostics | null { + const diagnostics = collectSandboxCreateFailureDiagnostics(sandboxName, options); + if (!diagnostics) return null; + + console.error(` Diagnostics saved: ${diagnostics.dir}`); + if (diagnostics.summaryLines.length > 0) { + console.error(" Recent OpenShell gateway failure:"); + for (const line of diagnostics.summaryLines) { + console.error(` ${line}`); + } + } + if (diagnostics.backupPath) { + console.error(` State backup retained: ${diagnostics.backupPath}`); + } + return diagnostics; +} diff --git a/src/lib/onboard/sandbox-dockerfile-patch-flow.ts b/src/lib/onboard/sandbox-dockerfile-patch-flow.ts index 2b1050c44d7..5a9837fd4bf 100644 --- a/src/lib/onboard/sandbox-dockerfile-patch-flow.ts +++ b/src/lib/onboard/sandbox-dockerfile-patch-flow.ts @@ -36,6 +36,7 @@ export type PrepareSandboxDockerfilePatchInput = { webSearchConfig: WebSearchConfig | null; hermesToolGateways: string[]; sandboxGpuConfig: SandboxGpuConfig; + gatewayPort?: number; log?: (message: string) => void; warn?: (message: string) => void; deps?: SandboxDockerfilePatchDeps; @@ -95,6 +96,7 @@ export async function prepareSandboxDockerfilePatch({ webSearchConfig, hermesToolGateways, sandboxGpuConfig, + gatewayPort, log = console.log, warn = console.warn, deps = {}, @@ -135,6 +137,7 @@ export async function prepareSandboxDockerfilePatch({ sandboxGpuConfig, { dockerDriverGateway: getDockerDriverGateway(), + gatewayPort, log, }, ); diff --git a/src/lib/onboard/sandbox-gpu-preflight.ts b/src/lib/onboard/sandbox-gpu-preflight.ts index 9778b9ab3cc..ee3a695f199 100644 --- a/src/lib/onboard/sandbox-gpu-preflight.ts +++ b/src/lib/onboard/sandbox-gpu-preflight.ts @@ -80,11 +80,14 @@ export function sandboxGpuRemediationLines( ]; } -export function exitOnSandboxGpuConfigErrors(config: SandboxGpuConfig): void { +export function exitOnSandboxGpuConfigErrors( + config: SandboxGpuConfig, + exitProcess: (code: number) => never = (code) => process.exit(code), +): void { if (config.errors.length > 0) { console.error(""); for (const error of config.errors) console.error(` ✗ ${error}`); - process.exit(1); + exitProcess(1); } } @@ -123,7 +126,10 @@ export function dockerNvidiaRuntimeAvailable(deps: SandboxGpuPreflightDeps = {}) } } -function validateJetsonSandboxGpuPreflight(deps: SandboxGpuPreflightDeps): void { +function validateJetsonSandboxGpuPreflight( + deps: SandboxGpuPreflightDeps, + exitProcess: (code: number) => never, +): void { if (!dockerNvidiaRuntimeAvailable(deps)) { console.error(""); console.error(" ✗ Docker NVIDIA runtime was not detected for Jetson/Tegra sandbox GPU."); @@ -134,7 +140,7 @@ function validateJetsonSandboxGpuPreflight(deps: SandboxGpuPreflightDeps): void console.error(" sudo nvidia-ctk runtime configure --runtime=docker"); console.error(" sudo systemctl restart docker"); console.error(" Or force CPU sandbox behavior with NEMOCLAW_SANDBOX_GPU=0."); - process.exit(1); + exitProcess(1); } console.log(" ✓ Docker NVIDIA runtime detected for Jetson/Tegra sandbox GPU"); } @@ -283,14 +289,15 @@ export function createDirectSandboxGpuVerifier( export function validateSandboxGpuPreflight( config: SandboxGpuConfig, deps: SandboxGpuPreflightDeps = {}, + exitProcess: (code: number) => never = (code) => process.exit(code), ): void { - exitOnSandboxGpuConfigErrors(config); + exitOnSandboxGpuConfigErrors(config, exitProcess); if (!config.sandboxGpuEnabled) return; const platform = deps.platform ?? process.platform; if (platform !== "linux") return; if (config.hostGpuPlatform === "jetson") { - validateJetsonSandboxGpuPreflight(deps); + validateJetsonSandboxGpuPreflight(deps, exitProcess); return; } @@ -314,7 +321,7 @@ export function validateSandboxGpuPreflight( })) { console.error(` ${line}`); } - process.exit(1); + exitProcess(1); } console.log(` ✓ Docker CDI GPU support detected (${cdiSpecFiles.join(", ")})`); } diff --git a/src/lib/onboard/sandbox-lifecycle.test.ts b/src/lib/onboard/sandbox-lifecycle.test.ts new file mode 100644 index 00000000000..fd43a260424 --- /dev/null +++ b/src/lib/onboard/sandbox-lifecycle.test.ts @@ -0,0 +1,70 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import type { SandboxEntry } from "../state/registry"; + +const registryState = vi.hoisted(() => ({ + removeSandbox: vi.fn(), + sandbox: null as SandboxEntry | null, +})); + +vi.mock("../state/registry", () => ({ + getSandbox: () => registryState.sandbox, + removeSandbox: registryState.removeSandbox, +})); + +import { createSandboxLifecycleHelpers } from "./sandbox-lifecycle"; + +describe("sandbox lifecycle MCP destroy boundaries", () => { + beforeEach(() => { + registryState.removeSandbox.mockReset(); + registryState.sandbox = null; + }); + + for (const marker of ["destroyPreparedAt", "destroyPendingAt"] as const) { + for (const withBridge of [false, true]) { + it(`preserves ${marker} and blocks absent-sandbox recreation${withBridge ? " with bridges" : " without bridges"}`, () => { + const runCaptureOpenshell = vi.fn(() => null); + registryState.sandbox = { + name: "alpha", + agent: "openclaw", + mcp: { + bridges: withBridge + ? { + github: { + server: "github", + agent: "openclaw", + adapter: "mcporter", + url: "https://mcp.example.test/mcp", + env: ["GITHUB_TOKEN"], + providerName: "alpha-mcp-github", + providerId: "provider-123", + policyName: "mcp-github", + addedAt: "2026-07-02T22:49:42.000Z", + }, + } + : {}, + [marker]: "2026-07-02T22:49:42.000Z", + }, + }; + const before = JSON.stringify(registryState.sandbox); + const helpers = createSandboxLifecycleHelpers({ + runCaptureOpenshell, + fetchGatewayAuthTokenFromSandbox: () => null, + agentProductName: () => "OpenClaw", + prompt: async () => "no", + isAffirmativeAnswer: () => false, + }); + + expect(() => helpers.reconcileSandboxForCreate("alpha")).toThrow( + /incomplete MCP destroy transaction.*finish cleanup before recreating/i, + ); + expect(runCaptureOpenshell).not.toHaveBeenCalled(); + expect(registryState.removeSandbox).not.toHaveBeenCalled(); + expect(JSON.stringify(registryState.sandbox)).toBe(before); + }); + } + } +}); diff --git a/src/lib/onboard/sandbox-lifecycle.ts b/src/lib/onboard/sandbox-lifecycle.ts index 74fed0814b1..1520d847ef1 100644 --- a/src/lib/onboard/sandbox-lifecycle.ts +++ b/src/lib/onboard/sandbox-lifecycle.ts @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import type { SandboxEntry, SandboxMcpState } from "../state/registry"; import * as registry from "../state/registry"; import type { SelectionDrift } from "./selection-drift"; @@ -13,7 +14,11 @@ export interface SandboxLifecycleDeps { } export interface SandboxLifecycleHelpers { - sandboxExistsInGateway(sandboxName: string): boolean; + reconcileSandboxForCreate(sandboxName: string): { + existingEntry: SandboxEntry | null; + preservedMcpState: SandboxMcpState | undefined; + liveExists: boolean; + }; pruneStaleSandboxEntry(sandboxName: string): boolean; shouldRestoreLatestBackupOnRecreate(): boolean; confirmRecreateForSelectionDrift( @@ -40,6 +45,25 @@ export function createSandboxLifecycleHelpers(deps: SandboxLifecycleDeps): Sandb return liveExists; } + function reconcileSandboxForCreate(sandboxName: string) { + const existingEntry = registry.getSandbox(sandboxName); + if (existingEntry?.mcp?.destroyPreparedAt || existingEntry?.mcp?.destroyPendingAt) { + throw new Error( + `Sandbox '${sandboxName}' has an incomplete MCP destroy transaction. Re-run the sandbox destroy command to finish cleanup before recreating it.`, + ); + } + const preservedMcpState = + existingEntry?.mcp && Object.keys(existingEntry.mcp.bridges).length > 0 + ? existingEntry.mcp + : undefined; + // MCP state is the rebuild transaction manifest. Preserve it while the + // sandbox is absent; registration carries the validated state forward. + const liveExists = preservedMcpState + ? sandboxExistsInGateway(sandboxName) + : pruneStaleSandboxEntry(sandboxName); + return { existingEntry, preservedMcpState, liveExists }; + } + function shouldRestoreLatestBackupOnRecreate(): boolean { return process.env.NEMOCLAW_RESTORE_LATEST_BACKUP_ON_RECREATE === "1"; } @@ -71,7 +95,7 @@ export function createSandboxLifecycleHelpers(deps: SandboxLifecycleDeps): Sandb } return { - sandboxExistsInGateway, + reconcileSandboxForCreate, pruneStaleSandboxEntry, shouldRestoreLatestBackupOnRecreate, confirmRecreateForSelectionDrift, diff --git a/src/lib/onboard/sandbox-registration.test.ts b/src/lib/onboard/sandbox-registration.test.ts index 0cedf413923..c06e8554d50 100644 --- a/src/lib/onboard/sandbox-registration.test.ts +++ b/src/lib/onboard/sandbox-registration.test.ts @@ -35,6 +35,7 @@ describe("buildCreatedSandboxRegistryEntry", () => { endpointUrl: "https://example.test/v1", credentialEnv: "COMPATIBLE_API_KEY", preferredInferenceApi: "openai-completions", + compatibleEndpointReasoning: null, nimContainer: null, }, runtimeFields, @@ -42,6 +43,9 @@ describe("buildCreatedSandboxRegistryEntry", () => { agentVersionKnown: true, imageTag: "nemoclaw-demo:123", appliedPolicies: ["discord", "slack"], + webSearchEnabled: true, + fromDockerfile: "/tmp/Dockerfile.custom", + hermesAuthMethod: "api_key", plannedMessagingState: plannedMessagingState as any, hermesToolGateways: ["filesystem"], hermesDashboardState: { @@ -62,6 +66,9 @@ describe("buildCreatedSandboxRegistryEntry", () => { preferredInferenceApi: "openai-completions", imageTag: "nemoclaw-demo:123", policies: ["discord", "slack"], + webSearchEnabled: true, + fromDockerfile: "/tmp/Dockerfile.custom", + hermesAuthMethod: "api_key", hermesToolGateways: ["filesystem"], hermesDashboardEnabled: true, hermesDashboardPort: 18790, @@ -93,6 +100,7 @@ describe("buildCreatedSandboxRegistryEntry", () => { endpointUrl: "", credentialEnv: "", preferredInferenceApi: "", + compatibleEndpointReasoning: null, nimContainer: "", }, runtimeFields, @@ -129,6 +137,54 @@ describe("buildCreatedSandboxRegistryEntry", () => { expect(entry.hermesDashboardPort).toBeUndefined(); expect(entry.hermesDashboardInternalPort).toBeUndefined(); expect(entry.hermesDashboardTui).toBeUndefined(); + expect(entry.webSearchEnabled).toBe(false); + expect(entry.fromDockerfile).toBeNull(); + expect(entry.hermesAuthMethod).toBeNull(); + }); + + it("carries a durable MCP rebuild manifest into the replacement registry entry", () => { + const preservedMcpState = { + bridges: { + github: { + server: "github", + agent: "openclaw", + adapter: "mcporter", + url: "https://mcp.example.test/mcp", + env: ["GITHUB_TOKEN"], + providerName: "demo-mcp-github", + policyName: "mcp-bridge-github", + addedAt: "2026-06-27T00:00:00.000Z", + }, + }, + }; + const entry = buildCreatedSandboxRegistryEntry({ + sandboxName: "demo", + inferenceSelection: { + model: "llama", + provider: "compatible-endpoint", + endpointUrl: null, + credentialEnv: null, + preferredInferenceApi: null, + compatibleEndpointReasoning: "true", + nimContainer: null, + }, + runtimeFields, + agent: null, + agentVersionKnown: true, + imageTag: "nemoclaw-demo:replacement", + appliedPolicies: [], + plannedMessagingState: undefined, + preservedMcpState, + hermesToolGateways: [], + hermesDashboardState: { enabled: false, config: null }, + dashboardPort: 18789, + gatewayName: "nemoclaw", + gatewayPort: 8080, + }); + + expect(entry.mcp).toBe(preservedMcpState); + expect(entry.mcp?.bridges.github?.providerName).toBe("demo-mcp-github"); + expect(entry.compatibleEndpointReasoning).toBe("true"); }); it("normalizes invalid preferred inference API values", () => { @@ -140,6 +196,7 @@ describe("buildCreatedSandboxRegistryEntry", () => { endpointUrl: "https://example.test/v1", credentialEnv: "COMPATIBLE_API_KEY", preferredInferenceApi: "chat", + compatibleEndpointReasoning: null, nimContainer: null, }, runtimeFields, @@ -171,6 +228,7 @@ describe("selection", () => { model: "llama", endpointUrl: "https://wrong.test/v1", credentialEnv: "WRONG_KEY", + compatibleEndpointReasoning: "true", nimContainer: "wrong", }); @@ -180,6 +238,7 @@ describe("selection", () => { endpointUrl: null, credentialEnv: null, preferredInferenceApi: "openai-completions", + compatibleEndpointReasoning: null, nimContainer: null, }); }); @@ -191,6 +250,7 @@ describe("selection", () => { model: "llama", endpointUrl: "https://right.test/v1", credentialEnv: "COMPATIBLE_API_KEY", + compatibleEndpointReasoning: "true", nimContainer: "nim-right", }); @@ -200,6 +260,7 @@ describe("selection", () => { endpointUrl: "https://right.test/v1", credentialEnv: "COMPATIBLE_API_KEY", preferredInferenceApi: "openai-completions", + compatibleEndpointReasoning: "true", nimContainer: "nim-right", }); }); @@ -217,6 +278,7 @@ describe("registerCreatedSandbox", () => { endpointUrl: null, credentialEnv: null, preferredInferenceApi: null, + compatibleEndpointReasoning: null, nimContainer: null, }, runtimeFields, diff --git a/src/lib/onboard/sandbox-registration.ts b/src/lib/onboard/sandbox-registration.ts index 49fc399a5f5..618ee3a30ea 100644 --- a/src/lib/onboard/sandbox-registration.ts +++ b/src/lib/onboard/sandbox-registration.ts @@ -4,8 +4,9 @@ import type { AgentDefinition } from "../agent/defs"; import type { InferenceSelection } from "../inference/selection"; import { inferenceSelectionRegistryFields } from "../inference/selection"; +import { type WebSearchConfig, webSearchProviderForConfig } from "../inference/web-search"; import * as onboardSession from "../state/onboard-session"; -import type { SandboxEntry, SandboxMessagingState } from "../state/registry"; +import type { SandboxEntry, SandboxMcpState, SandboxMessagingState } from "../state/registry"; import * as registry from "../state/registry"; import { getHermesDashboardRegistryFields, @@ -33,7 +34,16 @@ export interface CreatedSandboxRegistryEntryInput { agentVersionKnown: boolean; imageTag: string | null; appliedPolicies: string[]; + webSearchEnabled?: boolean; + webSearchProvider?: SandboxEntry["webSearchProvider"]; + fromDockerfile?: string | null; + hermesAuthMethod?: "oauth" | "api_key" | null; plannedMessagingState: SandboxMessagingState | undefined; + /** + * Durable MCP rebuild manifest carried across an already-absent sandbox. + * The caller must only supply state captured from the same sandbox name. + */ + preservedMcpState?: SandboxMcpState; hermesToolGateways: string[]; hermesDashboardState: HermesDashboardOnboardState; dashboardPort: number; @@ -45,6 +55,22 @@ export interface CreatedSandboxRegistrationInput extends CreatedSandboxRegistryE registerSandbox?(entry: SandboxEntry): void; } +export function creationFidelity( + webSearchConfig: WebSearchConfig | null, + fromDockerfile: string | null, + hermesAuthMethod: "oauth" | "api_key" | null, +): Pick< + SandboxEntry, + "webSearchEnabled" | "webSearchProvider" | "fromDockerfile" | "hermesAuthMethod" +> { + return { + webSearchEnabled: webSearchConfig?.fetchEnabled === true, + webSearchProvider: webSearchConfig ? webSearchProviderForConfig(webSearchConfig) : null, + fromDockerfile, + hermesAuthMethod, + }; +} + export function selection( sandboxName: string, provider: string, @@ -62,6 +88,9 @@ export function selection( endpointUrl: sessionMatches ? (session.endpointUrl ?? null) : null, credentialEnv: sessionMatches ? (session.credentialEnv ?? null) : null, preferredInferenceApi, + compatibleEndpointReasoning: sessionMatches + ? (session.compatibleEndpointReasoning ?? null) + : null, nimContainer: sessionMatches ? (session.nimContainer ?? null) : null, }); } @@ -81,7 +110,13 @@ export function buildCreatedSandboxRegistryEntry( ...getSandboxAgentRegistryFields(input.agent, input.agentVersionKnown), imageTag: input.imageTag, policies: input.appliedPolicies, + webSearchEnabled: input.webSearchEnabled === true, + webSearchProvider: + input.webSearchEnabled === true ? (input.webSearchProvider ?? "brave") : null, + fromDockerfile: input.fromDockerfile ?? null, + hermesAuthMethod: input.hermesAuthMethod ?? null, messaging: messagingState, + mcp: input.preservedMcpState, hermesToolGateways: input.hermesToolGateways.length > 0 ? [...input.hermesToolGateways] : undefined, ...getHermesDashboardRegistryFields(input.hermesDashboardState), diff --git a/src/lib/onboard/session-bootstrap.ts b/src/lib/onboard/session-bootstrap.ts index 1936e9da4ad..e51f43947a3 100644 --- a/src/lib/onboard/session-bootstrap.ts +++ b/src/lib/onboard/session-bootstrap.ts @@ -11,6 +11,7 @@ export interface OnboardSessionBootstrapInput { requestedSandboxName: string | null; cannotPrompt: boolean; nonInteractive: boolean; + authoritativeResumeConfig?: boolean; agentFlag?: string | null; envAgent?: string | null; } @@ -30,6 +31,7 @@ export interface OnboardSessionBootstrapDeps { fromDockerfile?: string | null; sandboxName?: string | null; agent?: string | null; + authoritativeResumeConfig?: boolean; }, ): ResumeConfigConflict[]; recordResumeConflict(conflict: ResumeConfigConflict): Promise; @@ -152,6 +154,7 @@ async function prepareResumeSession( fromDockerfile: input.requestedFromDockerfile, sandboxName: input.requestedSandboxName, agent: input.agentFlag || null, + authoritativeResumeConfig: input.authoritativeResumeConfig, }); if (resumeConflicts.length > 0) { await exitForResumeConflicts(resumeConflicts, deps); diff --git a/src/lib/onboard/skipped-step-message.ts b/src/lib/onboard/skipped-step-message.ts new file mode 100644 index 00000000000..2cc1f4e8679 --- /dev/null +++ b/src/lib/onboard/skipped-step-message.ts @@ -0,0 +1,21 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { agentProductName } from "./branding"; +import { getOnboardProgressStep } from "./machine/progress"; +import { step } from "./prompt-helpers"; + +export function skippedStepMessage( + stepName: string, + detail?: string | null, + reason: "resume" | "reuse" = "resume", +): void { + const progressStep = getOnboardProgressStep(stepName); + const stepInfo = + progressStep && stepName === "openclaw" + ? { ...progressStep, title: `Setting up ${agentProductName()} inside sandbox` } + : progressStep; + if (stepInfo) step(stepInfo.number, stepInfo.total, stepInfo.title); + const prefix = reason === "reuse" ? "[reuse]" : "[resume]"; + console.log(` ${prefix} Skipping ${stepName}${detail ? ` (${detail})` : ""}`); +} diff --git a/src/lib/onboard/types.ts b/src/lib/onboard/types.ts index 80782199230..3d4e74f4b2a 100644 --- a/src/lib/onboard/types.ts +++ b/src/lib/onboard/types.ts @@ -52,3 +52,29 @@ export interface ModelValidationFailure extends ValidationFailureLike { } export type ModelValidationResult = ModelValidationSuccess | ModelValidationFailure; + +export type OnboardOptions = { + nonInteractive?: boolean; + recreateSandbox?: boolean; + authoritativeResumeConfig?: boolean; + /** Internal authoritative rebuild target; never exposed as a public CLI option. */ + targetGatewayName?: string | null; + /** Internal authoritative rebuild target; must match targetGatewayName. */ + targetGatewayPort?: number | null; + /** Internal rebuild handoff: the outer destructive lifecycle owns the onboard lock. */ + onboardLockAlreadyHeld?: boolean; + /** Internal one-shot handoff for a prevalidated managed DCode replacement. */ + preparedDcodeRebuild?: import("./prepared-dcode-rebuild").PreparedDcodeRebuildHandoff; + resume?: boolean; + fresh?: boolean; + fromDockerfile?: string | null; + sandboxName?: string | null; + sandboxGpu?: "enable" | "disable" | null; + sandboxGpuDevice?: string | null; + acceptThirdPartySoftware?: boolean; + agent?: string | null; + controlUiPort?: number | null; + gpu?: boolean; + noGpu?: boolean; + autoYes?: boolean; +}; diff --git a/src/lib/policy/gateway-state.ts b/src/lib/policy/gateway-state.ts new file mode 100644 index 00000000000..e56b2df9f77 --- /dev/null +++ b/src/lib/policy/gateway-state.ts @@ -0,0 +1,99 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { isDeepStrictEqual } from "node:util"; + +import YAML from "yaml"; + +export type PresetContentSource = { name: string; content: string | null }; +export type PresetContentGatewayState = "match" | "absent" | "drift" | null; + +type GatewayInspectionOptions = { + readPolicy: () => string; + parseCurrentPolicy: (raw: string | null | undefined) => string; + extractPresetEntries: (content: string | null | undefined) => string | null; +}; + +function readParsedPolicy(options: GatewayInspectionOptions): Record | null { + let rawPolicy: string; + try { + rawPolicy = options.readPolicy(); + } catch { + return null; + } + const currentPolicy = options.parseCurrentPolicy(rawPolicy); + if (!currentPolicy) return null; + try { + const parsed = YAML.parse(currentPolicy); + return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : null; + } catch { + return null; + } +} + +function presetPolicyKeys( + content: string | null, + extractPresetEntries: GatewayInspectionOptions["extractPresetEntries"], +): string[] | null { + const entries = extractPresetEntries(content); + if (!entries) return null; + try { + const policies = YAML.parse(`network_policies:\n${entries}`)?.network_policies; + if (!policies || typeof policies !== "object" || Array.isArray(policies)) return null; + const keys = Object.keys(policies); + return keys.length > 0 ? keys : null; + } catch { + return null; + } +} + +export function inspectGatewayPresetNames( + options: GatewayInspectionOptions & { sources: () => readonly PresetContentSource[] }, +): string[] | null { + const parsed = readParsedPolicy(options); + if (!parsed) return null; + const policies = parsed.network_policies; + if (!policies || typeof policies !== "object" || Array.isArray(policies)) return []; + const gatewayKeys = new Set(Object.keys(policies)); + return options.sources().flatMap((source) => { + const keys = presetPolicyKeys(source.content, options.extractPresetEntries); + return keys?.every((key) => gatewayKeys.has(key)) ? [source.name] : []; + }); +} + +export function inspectPresetContentGatewayState( + options: GatewayInspectionOptions & { presetContent: string }, +): PresetContentGatewayState { + const parsed = readParsedPolicy(options); + if (!parsed) return null; + const current = parsed.network_policies; + const entries = options.extractPresetEntries(options.presetContent); + if (!entries) return "drift"; + try { + const expected = YAML.parse(`network_policies:\n${entries}`)?.network_policies; + if ( + !current || + typeof current !== "object" || + Array.isArray(current) || + !expected || + typeof expected !== "object" || + Array.isArray(expected) + ) { + return "drift"; + } + const currentPolicies = current as Record; + const expectedPolicies = expected as Record; + const expectedKeys = Object.keys(expectedPolicies); + if (expectedKeys.length === 0) return "drift"; + const presentKeys = expectedKeys.filter((key) => Object.hasOwn(currentPolicies, key)); + if (presentKeys.length === 0) return "absent"; + if (presentKeys.length !== expectedKeys.length) return "drift"; + return expectedKeys.every((key) => + isDeepStrictEqual(currentPolicies[key], expectedPolicies[key]), + ) + ? "match" + : "drift"; + } catch { + return "drift"; + } +} diff --git a/src/lib/policy/index.ts b/src/lib/policy/index.ts index 84082a1c0f3..90b8938f313 100644 --- a/src/lib/policy/index.ts +++ b/src/lib/policy/index.ts @@ -3,7 +3,15 @@ // // Policy preset management — list, load, merge, and apply presets. -import type { JsonObject, JsonValue } from "../core/json-types"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import readline from "node:readline"; +import YAML from "yaml"; + +// Namespace access keeps resolveOpenshell spyable in focused policy tests. +import * as openshellResolveModule from "../adapters/openshell/resolve"; +import { loadAgent } from "../agent/defs"; import { getMessagingPolicyKeyAliases, getMessagingPolicyPresetValidationWarnings, @@ -13,28 +21,29 @@ import { listMessagingPolicyPresetMetadata, loadMessagingChannelPolicyPreset, } from "../messaging/channels"; +import { ROOT, run, runCapture } from "../runner"; +import * as registry from "../state/registry"; import { buildPolicyGetCommand, buildPolicyGetFullCommand, buildPolicySetCommand, } from "./commands"; +import { inspectGatewayPresetNames, inspectPresetContentGatewayState } from "./gateway-state"; import { parseOpenShellPolicy, stripProviderComposedPolicies, withoutProviderComposedPolicies, } from "./merge"; - -const fs = require("fs"); -const path = require("path"); -const os = require("os"); -const readline = require("readline"); -const YAML = require("yaml"); -const { ROOT, run, runCapture } = require("../runner"); -const registry = require("../state/registry"); -const { loadAgent } = require("../agent/defs"); -// Late-binding access via the module exports so tests can spy on -// resolveOpenshell without rewiring requires. -const openshellResolveModule = require("../adapters/openshell/resolve"); +import { findUnexpectedExistingPolicyKey } from "./preset-ownership"; +import { + isPolicyDocument, + isPolicyObject, + isPresetPolicyMap, + type PolicyDocument, + type PolicyObject, + type PolicyValue, + parseNetworkPolicies, +} from "./preset-parsing"; const PRESETS_DIR = path.join(ROOT, "nemoclaw-blueprint", "policies", "presets"); @@ -46,15 +55,6 @@ type PresetInfo = { description: string; }; -// Re-use shared JSON types under policy-domain names. -type PolicyValue = JsonValue; -type PolicyObject = JsonObject; - -type PolicyDocument = PolicyObject & { - version?: number; - network_policies?: PolicyObject; -}; - type SelectionOptions = { applied?: string[]; }; @@ -76,10 +76,6 @@ type SetupPolicyPresetSupportOptions = { agent?: string | null; }; -function isPolicyDocument(value: PolicyValue): value is PolicyDocument { - return typeof value === "object" && value !== null && !Array.isArray(value); -} - /** * Enumerate every built-in preset and return `{ file, name, description }` * triples parsed from each file's `preset:` header. Non-messaging presets live @@ -141,30 +137,6 @@ function loadPresetForAgent(name: string, options: PresetLoadOptions = {}): stri function loadPreset(name: string): string | null { return loadPresetForAgent(name, { agent: "openclaw" }); } - -function isPolicyObject(value: PolicyValue): value is PolicyObject { - return typeof value === "object" && value !== null && !Array.isArray(value); -} - -function isPresetPolicyMap(value: PolicyValue): value is PolicyObject { - return ( - isPolicyObject(value) && - Object.keys(value).length > 0 && - Object.values(value).every(isPolicyObject) - ); -} - -function parseNetworkPolicies(content: string | null | undefined): PolicyObject | null { - if (!content) return null; - try { - const parsed = YAML.parse(content); - const networkPolicies = isPolicyDocument(parsed) ? parsed.network_policies : null; - return isPolicyObject(networkPolicies) ? networkPolicies : null; - } catch { - return null; - } -} - // The single sandbox->host bridge hostname OpenShell provisions. An endpoint // that pins `allowed_ips` for THIS host is the legitimate host-gateway flow // (e.g. web_fetch to host.openshell.internal); `allowed_ips` on any other host @@ -441,12 +413,13 @@ function parseCurrentPolicyOrEmpty(raw: string | null | undefined): string { /** * Pre-spawn check used at command entry points before any * `run(buildPolicy*Command(...))`. If the binary cannot be resolved, prints - * every location checked and an install hint, then exits nonzero — instead - * of letting the spawn surface as the opaque `spawnSync openshell ENOENT` - * (issue #4224). + * every location checked and an install hint. Normal command entry points + * exit nonzero; transactional lifecycle callers can request `nonFatal` and + * retain control for rollback instead of surfacing the opaque + * `spawnSync openshell ENOENT` (issue #4224). */ -function assertOpenshellResolvable(): void { - if (openshellResolveModule.resolveOpenshell()) return; +function assertOpenshellResolvable(options: { nonFatal?: boolean } = {}): boolean { + if (openshellResolveModule.resolveOpenshell()) return true; const home = process.env.HOME; const override = process.env.NEMOCLAW_OPENSHELL_BIN; @@ -471,9 +444,31 @@ function assertOpenshellResolvable(): void { console.error( " Install OpenShell (https://github.com/NVIDIA/OpenShell) or set NEMOCLAW_OPENSHELL_BIN to an absolute, executable path.", ); + if (options.nonFatal) return false; process.exit(1); } +/** + * Apply a policy file while optionally keeping control in the caller on + * failure. Lifecycle code that owns compensating actions must use nonFatal so + * a failed OpenShell mutation cannot bypass its rollback through process.exit. + */ +function setPolicyFile( + policyFile: string, + sandboxName: string, + options: { nonFatal?: boolean } = {}, +): boolean { + const result = run(buildPolicySetCommand(policyFile, sandboxName), { + ignoreError: options.nonFatal === true, + }); + if (!options.nonFatal) return true; + if (!result.error && result.status === 0) return true; + + const detail = result.error?.message ?? `exit ${result.status ?? "unknown"}`; + console.error(` Failed to update policy for sandbox '${sandboxName}' (${detail}).`); + return false; +} + /** * Merge preset entries into existing policy YAML using structured YAML * parsing. Invalid input fails closed instead of falling back to text @@ -668,7 +663,11 @@ function removePresetFromPolicy( * Returns `false` if the preset is unknown or has no `network_policies` * section. */ -function removePreset(sandboxName: string, presetName: string): boolean { +function removePreset( + sandboxName: string, + presetName: string, + options: { nonFatal?: boolean; skipRegistryUpdate?: boolean } = {}, +): boolean { // Guard against truncated sandbox names — WSL can truncate hyphenated // names during argument parsing, e.g. "my-assistant" → "m" const isRfc1123Label = /^[a-z0-9]([a-z0-9-]*[a-z0-9])?$/.test(sandboxName); @@ -733,14 +732,14 @@ function removePreset(sandboxName: string, presetName: string): boolean { // Run before creating temp resources so a missing-binary exit doesn't // orphan files in $TMPDIR (the finally cleanup doesn't run on process.exit). - assertOpenshellResolvable(); + if (!assertOpenshellResolvable(options)) return false; const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-policy-")); const tmpFile = path.join(tmpDir, "policy.yaml"); fs.writeFileSync(tmpFile, updated, { encoding: "utf-8", mode: 0o600 }); try { - run(buildPolicySetCommand(tmpFile, sandboxName)); + if (!setPolicyFile(tmpFile, sandboxName, options)) return false; console.log(` Removed preset: ${presetName}`); } finally { try { @@ -755,7 +754,7 @@ function removePreset(sandboxName: string, presetName: string): boolean { } } - const sandbox = registry.getSandbox(sandboxName); + const sandbox = options.skipRegistryUpdate ? undefined : registry.getSandbox(sandboxName); if (sandbox) { if (isCustom) { registry.removeCustomPolicyByName(sandboxName, presetName); @@ -839,7 +838,12 @@ function applyPresetContent( sandboxName: string, presetName: string, presetContent: string, - options: { custom?: { sourcePath?: string } } = {}, + options: { + custom?: { sourcePath?: string }; + expectedExistingNetworkPolicyContent?: string | null; + nonFatal?: boolean; + skipRegistryUpdate?: boolean; + } = {}, ): boolean { // Guard against truncated sandbox names — WSL can truncate hyphenated // names during argument parsing, e.g. "my-assistant" → "m" @@ -885,6 +889,27 @@ function applyPresetContent( ); return false; } + if (Object.prototype.hasOwnProperty.call(options, "expectedExistingNetworkPolicyContent")) { + let collision: string | null = null; + try { + collision = findUnexpectedExistingPolicyKey( + currentPolicy, + presetEntries, + options.expectedExistingNetworkPolicyContent ?? null, + ); + } catch { + console.error( + ` Could not validate network policy key ownership for '${presetName}'; refusing to apply it.`, + ); + return false; + } + if (collision) { + console.error( + ` Network policy key '${collision}' does not match the exact state owned by '${presetName}'; refusing to replace it.`, + ); + return false; + } + } const merged = mergePresetIntoPolicy(currentPolicy, presetEntries); const endpoints = getPresetEndpoints(presetContent); @@ -894,14 +919,14 @@ function applyPresetContent( // Run before creating temp resources so a missing-binary exit doesn't // orphan files in $TMPDIR (the finally cleanup doesn't run on process.exit). - assertOpenshellResolvable(); + if (!assertOpenshellResolvable(options)) return false; const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-policy-")); const tmpFile = path.join(tmpDir, "policy.yaml"); fs.writeFileSync(tmpFile, merged, { encoding: "utf-8", mode: 0o600 }); try { - run(buildPolicySetCommand(tmpFile, sandboxName)); + if (!setPolicyFile(tmpFile, sandboxName, options)) return false; console.log(` Applied preset: ${presetName}`); } finally { @@ -917,6 +942,12 @@ function applyPresetContent( } } + // Some multi-resource lifecycle callers reserve ownership in the registry + // before mutating the live gateway. That ordering prevents a successful + // policy set followed by a registry-write failure from leaving an unowned + // live key. They explicitly request no second registry write here. + if (options.skipRegistryUpdate) return true; + const sandbox = registry.getSandbox(sandboxName); if (sandbox) { if (options.custom) { @@ -1222,34 +1253,6 @@ function listCustomPresets(sandboxName: string): PresetInfo[] { })); } -/** - * True when every `network_policies` key declared in `content` is present in - * `gatewayPolicyNames`. Works for both built-in preset YAML and the custom - * preset YAML stored under a sandbox's registry entry — keeping a single - * matching rule means `policy-list` and `status` stay consistent for either - * preset source. (#3590) - */ -function presetMatchesGateway( - content: string | null, - gatewayPolicyNames: ReadonlySet, -): boolean { - const entries = extractPresetEntries(content); - if (!entries) return false; - - let presetPolicies; - try { - const presetParsed = YAML.parse("network_policies:\n" + entries); - presetPolicies = presetParsed?.network_policies; - } catch { - return false; - } - - if (!presetPolicies || typeof presetPolicies !== "object") return false; - - const presetKeys = Object.keys(presetPolicies); - return presetKeys.length > 0 && presetKeys.every((k) => gatewayPolicyNames.has(k)); -} - /** * Query the gateway for the currently loaded policy and determine which * presets are actually enforced by matching network_policies entries @@ -1263,54 +1266,48 @@ function presetMatchesGateway( * matching presets" (`[]`). */ function getGatewayPresets(sandboxName: string): string[] | null { - let rawPolicy = ""; - try { - rawPolicy = runCapture(buildPolicyGetFullCommand(sandboxName), { ignoreError: true }); - } catch { - return null; - } - - const currentPolicy = parseCurrentPolicyOrEmpty(rawPolicy); - if (!currentPolicy) return null; - - let parsed; - try { - parsed = YAML.parse(currentPolicy); - } catch { - return null; - } - - if (!parsed || typeof parsed !== "object") return null; - - // Gateway returned valid YAML but has no network_policies section — - // this is a reachable gateway with an empty/default policy. - const gatewayPolicies = parsed.network_policies; - if (!gatewayPolicies || typeof gatewayPolicies !== "object" || Array.isArray(gatewayPolicies)) { - return []; - } - - const gatewayPolicyNames = new Set(Object.keys(gatewayPolicies)); - const matched: string[] = []; let sandboxAgent: string | null = null; try { sandboxAgent = registry.getSandbox(sandboxName)?.agent ?? null; } catch { sandboxAgent = null; } + return inspectGatewayPresetNames({ + readPolicy: () => runCapture(buildPolicyGetFullCommand(sandboxName), { ignoreError: true }), + parseCurrentPolicy: parseCurrentPolicyOrEmpty, + extractPresetEntries, + sources: () => [ + ...listPresets({ agent: sandboxAgent }).map((preset) => ({ + name: preset.name, + content: loadPresetForSandbox(sandboxName, preset.name), + })), + ...registry.getCustomPolicies(sandboxName).map((entry) => ({ + name: entry.name, + content: entry.content, + })), + ], + }); +} - for (const preset of listPresets({ agent: sandboxAgent })) { - if (presetMatchesGateway(loadPresetForSandbox(sandboxName, preset.name), gatewayPolicyNames)) { - matched.push(preset.name); - } - } - - for (const entry of registry.getCustomPolicies(sandboxName)) { - if (presetMatchesGateway(entry.content, gatewayPolicyNames)) { - matched.push(entry.name); - } - } +/** + * Compare the full network-policy entries in a preset with the live gateway + * policy. Unlike getGatewayPresets(), this detects same-key policy drift. + */ +function getPresetContentGatewayState( + sandboxName: string, + presetContent: string, +): "match" | "absent" | "drift" | null { + return inspectPresetContentGatewayState({ + readPolicy: () => runCapture(buildPolicyGetCommand(sandboxName)), + parseCurrentPolicy: parseCurrentPolicyOrEmpty, + extractPresetEntries, + presetContent, + }); +} - return matched; +function presetContentMatchesGateway(sandboxName: string, presetContent: string): boolean | null { + const state = getPresetContentGatewayState(sandboxName, presetContent); + return state === null ? null : state === "match"; } /** @@ -1435,6 +1432,7 @@ export { filterSetupPolicyPresets, getAppliedPresets, getGatewayPresets, + getPresetContentGatewayState, getPresetEndpoints, getPresetValidationWarning, isMessagingChannelPolicyPreset, @@ -1451,6 +1449,7 @@ export { PRESETS_DIR, parseCurrentPolicyOrEmpty as parseCurrentPolicy, parsePresetPolicyKeys, + presetContentMatchesGateway, removePreset, removePresetFromPolicy, resolvePermissivePolicyPath, diff --git a/src/lib/policy/preset-ownership.ts b/src/lib/policy/preset-ownership.ts new file mode 100644 index 00000000000..1dea5169dd7 --- /dev/null +++ b/src/lib/policy/preset-ownership.ts @@ -0,0 +1,36 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { isDeepStrictEqual } from "node:util"; + +import YAML from "yaml"; + +function policyMap(content: string): Record { + const policies = YAML.parse(content)?.network_policies; + return policies && typeof policies === "object" && !Array.isArray(policies) ? policies : {}; +} + +/** + * Return the first incoming key whose live value is not exactly the value the + * caller previously proved it owned. A null expected document owns no keys. + */ +export function findUnexpectedExistingPolicyKey( + currentPolicy: string, + presetEntries: string, + expectedPolicyContent: string | null, +): string | null { + const current = policyMap(currentPolicy); + const incoming = policyMap(`network_policies:\n${presetEntries}`); + const expected = expectedPolicyContent === null ? {} : policyMap(expectedPolicyContent); + return ( + Object.keys(incoming).find((key) => { + const currentHasKey = Object.prototype.hasOwnProperty.call(current, key); + if (expectedPolicyContent === null) return currentHasKey; + return ( + !currentHasKey || + !Object.prototype.hasOwnProperty.call(expected, key) || + !isDeepStrictEqual(current[key], expected[key]) + ); + }) ?? null + ); +} diff --git a/src/lib/policy/preset-parsing.ts b/src/lib/policy/preset-parsing.ts new file mode 100644 index 00000000000..b0eb5840d3d --- /dev/null +++ b/src/lib/policy/preset-parsing.ts @@ -0,0 +1,40 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import YAML from "yaml"; + +import type { JsonObject, JsonValue } from "../core/json-types"; + +export type PolicyValue = JsonValue; +export type PolicyObject = JsonObject; +export type PolicyDocument = PolicyObject & { + version?: number; + network_policies?: PolicyObject; +}; + +export function isPolicyDocument(value: PolicyValue): value is PolicyDocument { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +export function isPolicyObject(value: PolicyValue): value is PolicyObject { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +export function isPresetPolicyMap(value: PolicyValue): value is PolicyObject { + return ( + isPolicyObject(value) && + Object.keys(value).length > 0 && + Object.values(value).every(isPolicyObject) + ); +} + +export function parseNetworkPolicies(content: string | null | undefined): PolicyObject | null { + if (!content) return null; + try { + const parsed = YAML.parse(content); + const networkPolicies = isPolicyDocument(parsed) ? parsed.network_policies : null; + return isPolicyObject(networkPolicies) ? networkPolicies : null; + } catch { + return null; + } +} diff --git a/src/lib/runner.ts b/src/lib/runner.ts index ba96f4cf8ad..c1d1b2d406a 100644 --- a/src/lib/runner.ts +++ b/src/lib/runner.ts @@ -6,13 +6,14 @@ import type { SpawnSyncOptionsWithStringEncoding, SpawnSyncReturns, } from "node:child_process"; -import { NAME_ALLOWED_FORMAT, NAME_MAX_LENGTH, NAME_VALID_PATTERN } from "./name-validation"; +import { spawnSync } from "node:child_process"; +import path from "node:path"; -const { spawnSync } = require("child_process"); -const path = require("path"); -const { detectDockerHost } = require("./platform"); -const { shellQuote } = require("./core/shell-quote") as typeof import("./core/shell-quote"); -const { buildSubprocessEnv } = require("./subprocess-env") as typeof import("./subprocess-env"); +import { shellQuote } from "./core/shell-quote"; +import { NAME_ALLOWED_FORMAT, NAME_MAX_LENGTH, NAME_VALID_PATTERN } from "./name-validation"; +import { detectDockerHost } from "./platform"; +import { redact, redactError, writeRedactedResult } from "./security/redact"; +import { buildSubprocessEnv } from "./subprocess-env"; const ROOT = path.resolve(__dirname, "..", ".."); const SCRIPTS = path.join(ROOT, "scripts"); @@ -284,17 +285,13 @@ function runCapture(cmd: readonly string[], opts: CaptureOptions = {}): string { throw new Error(`Command failed with status ${result.status}`); } - const stdout = result.stdout || ""; - return (typeof stdout === "string" ? stdout : stdout.toString("utf-8")).trim(); + return (result.stdout || "").trim(); } catch (err) { if (ignoreError) return ""; throw redactError(err); } } -// Unified redaction — see redact.ts (#2381). -const { redact, redactError, writeRedactedResult } = require("./security/redact"); - /** Structured result returned by runCaptureEx. */ export interface CaptureResult { stdout: string; @@ -342,11 +339,9 @@ function runCaptureEx( const timedOut = (result.error != null && (result.error as NodeJS.ErrnoException).code === "ETIMEDOUT") || result.status === 28; - const stdout = result.stdout || ""; - const stderr = result.stderr || ""; return { - stdout: (typeof stdout === "string" ? stdout : stdout.toString("utf-8")).trim(), - stderr: (typeof stderr === "string" ? stderr : stderr.toString("utf-8")).trim(), + stdout: (result.stdout || "").trim(), + stderr: (result.stderr || "").trim(), exitCode: result.status, timedOut, }; diff --git a/src/lib/sandbox-base-image.ts b/src/lib/sandbox-base-image.ts index 3c41a741258..3a3171c5fd4 100644 --- a/src/lib/sandbox-base-image.ts +++ b/src/lib/sandbox-base-image.ts @@ -28,6 +28,8 @@ type ResolveBaseImageOptions = { minGlibcVersion?: string; rootDir?: string; env?: NodeJS.ProcessEnv; + validateImage?: (imageRef: string) => boolean; + validationDescription?: string; }; export type SandboxBaseImageResolution = { @@ -363,6 +365,14 @@ function resolvePulledCandidate( } } + if (options.validateImage && !options.validateImage(imageRef)) { + console.warn( + ` Warning: ${options.label || "sandbox base image"} ${imageRef} lacks ` + + `${options.validationDescription || "a required runtime capability"}.`, + ); + return null; + } + const repoDigest = getRepoDigest(imageName, imageRef); return { ref: repoDigest?.ref || imageRef, @@ -381,7 +391,7 @@ function resolveLocalCandidate( const check = options.requireOpenshellSandboxAbi ? imageMeetsMinimumGlibc(imageRef, options.minGlibcVersion || OPENSHELL_SANDBOX_MIN_GLIBC) : { ok: true, version: null }; - if (check.ok) { + if (check.ok && (!options.validateImage || options.validateImage(imageRef))) { return { ref: imageRef, digest: null, source: "local", glibcVersion: check.version }; } } @@ -424,6 +434,14 @@ function resolveLocalCandidate( return null; } + if (options.validateImage && !options.validateImage(imageRef)) { + console.error( + ` Local ${label} ${imageRef} lacks ` + + `${options.validationDescription || "a required runtime capability"}.`, + ); + return null; + } + return { ref: imageRef, digest: null, source: "local", glibcVersion: check.version }; } @@ -436,7 +454,7 @@ export function resolveSandboxBaseImage( if (override) { const resolved = resolvePulledCandidate(options.imageName, override, "override", options); if (resolved) return resolved; - if (!options.requireOpenshellSandboxAbi) return null; + if (!options.requireOpenshellSandboxAbi && !options.validateImage) return null; } else { for (const tag of getVersionedBaseImageTags(options.rootDir || ROOT, env)) { const imageRef = `${options.imageName}:${tag}`; @@ -467,7 +485,7 @@ export function resolveSandboxBaseImage( if (resolved) return resolved; } - if (options.requireOpenshellSandboxAbi) { + if (options.requireOpenshellSandboxAbi || options.validateImage) { return resolveLocalCandidate(options); } return null; diff --git a/src/lib/sandbox/build-context.ts b/src/lib/sandbox/build-context.ts index 73c564398e2..4103c91ba65 100644 --- a/src/lib/sandbox/build-context.ts +++ b/src/lib/sandbox/build-context.ts @@ -37,6 +37,16 @@ function normalizeReadModesForDockerCopy(rootDir: string): void { } } +function stageMcporterRuntime(rootDir: string, buildCtx: string): void { + const sourceDir = path.join(rootDir, "agents", "openclaw", "mcporter-runtime"); + const stagedDir = path.join(buildCtx, "agents", "openclaw", "mcporter-runtime"); + fs.mkdirSync(stagedDir, { recursive: true }); + for (const fileName of ["package.json", "package-lock.json"]) { + fs.copyFileSync(path.join(sourceDir, fileName), path.join(stagedDir, fileName)); + } + normalizeReadModesForDockerCopy(path.join(buildCtx, "agents")); +} + function stageLegacySandboxBuildContext( rootDir: string, tmpDir: string = os.tmpdir(), @@ -47,19 +57,27 @@ function stageLegacySandboxBuildContext( path.join(rootDir, "tsconfig.runtime-preloads.json"), path.join(buildCtx, "tsconfig.runtime-preloads.json"), ); - fs.cpSync(path.join(rootDir, "nemoclaw"), path.join(buildCtx, "nemoclaw"), { recursive: true }); + stageMcporterRuntime(rootDir, buildCtx); + fs.cpSync(path.join(rootDir, "nemoclaw"), path.join(buildCtx, "nemoclaw"), { + recursive: true, + }); fs.cpSync(path.join(rootDir, "nemoclaw-blueprint"), path.join(buildCtx, "nemoclaw-blueprint"), { recursive: true, }); normalizeReadModesForDockerCopy(path.join(buildCtx, "nemoclaw-blueprint")); - fs.cpSync(path.join(rootDir, "scripts"), path.join(buildCtx, "scripts"), { recursive: true }); + fs.cpSync(path.join(rootDir, "scripts"), path.join(buildCtx, "scripts"), { + recursive: true, + }); fs.cpSync( path.join(rootDir, "src", "lib", "messaging"), path.join(buildCtx, "src", "lib", "messaging"), { recursive: true }, ); normalizeReadModesForDockerCopy(path.join(buildCtx, "src")); - fs.rmSync(path.join(buildCtx, "nemoclaw", "node_modules"), { recursive: true, force: true }); + fs.rmSync(path.join(buildCtx, "nemoclaw", "node_modules"), { + recursive: true, + force: true, + }); normalizeReadModesForDockerCopy(path.join(buildCtx, "nemoclaw")); return { @@ -85,6 +103,7 @@ function stageOptimizedSandboxBuildContext( path.join(rootDir, "tsconfig.runtime-preloads.json"), path.join(buildCtx, "tsconfig.runtime-preloads.json"), ); + stageMcporterRuntime(rootDir, buildCtx); fs.mkdirSync(stagedNemoclawDir, { recursive: true }); for (const fileName of [ diff --git a/src/lib/sandbox/config.ts b/src/lib/sandbox/config.ts index f36ad55140f..d5965658643 100644 --- a/src/lib/sandbox/config.ts +++ b/src/lib/sandbox/config.ts @@ -28,6 +28,9 @@ const { appendAuditEntry } = require("../shields/audit"); const { withTimerBoundShieldsMutationLock, }: typeof import("../shields/timer-bound-lock") = require("../shields/timer-bound-lock"); +const { + withSandboxMutationLock, +}: typeof import("../state/mcp-lifecycle-lock") = require("../state/mcp-lifecycle-lock"); const { runOpenClawConfigGuard, }: typeof import("../shields/openclaw-config-lock") = require("../shields/openclaw-config-lock"); @@ -994,42 +997,44 @@ async function configSet(sandboxName: string, opts: ConfigSetOpts = {}): Promise configFail(` URL validation failed${suffix}: ${message}`); } - // Serialize only the authoritative re-read/CAS write. Interactive approval - // and DNS validation above must not hold the shields transition lock across - // the auto-restore deadline. If anything changed while the user was - // deciding, fail closed and ask them to retry against the new baseline. - withTimerBoundShieldsMutationLock(sandboxName, "config set write", () => { - const { isShieldsDown }: typeof import("../shields") = require("../shields"); - if ( - (target.agentName === "openclaw" || target.agentName === "hermes") && - !isShieldsDown(sandboxName, true) - ) { - configFail( - ` ${target.agentName} config changes are unavailable while shields are up for '${sandboxName}'. Run 'nemoclaw ${sandboxName} shields down' first.`, - ); - } - const currentConfig = readSandboxConfig(sandboxName, target); - const currentConfigSha256 = ( - currentConfig as ConfigObject & { [CONFIG_SOURCE_SHA256]?: string } - )[CONFIG_SOURCE_SHA256]; - if (currentConfigSha256 !== initialConfigSha256) { - configFail( - ` ${target.agentName} config changed while this update was being validated. Re-run config set against the current value.`, - ); - } - setDotpath(currentConfig, opts.key!, safeValue); - - console.log(` Writing config to sandbox (${target.configPath})...`); - writeSandboxConfig(sandboxName, target, currentConfig); - recomputeSandboxConfigHash(sandboxName, target); - - appendAuditEntry({ - action: "config_set", - sandbox: sandboxName, - timestamp: new Date().toISOString(), - reason: `config set ${target.agentName}:${opts.key}`, - }); - }); + // Serialize only the authoritative re-read/CAS write under the shared + // sandbox lock and then the shields transition lock. Interactive approval + // and DNS validation above must not hold either lock across the auto-restore + // deadline. If anything changed while the user was deciding, fail closed. + await withSandboxMutationLock(sandboxName, () => + withTimerBoundShieldsMutationLock(sandboxName, "config set write", () => { + const { isShieldsDown }: typeof import("../shields") = require("../shields"); + if ( + (target.agentName === "openclaw" || target.agentName === "hermes") && + !isShieldsDown(sandboxName, true) + ) { + configFail( + ` ${target.agentName} config changes are unavailable while shields are up for '${sandboxName}'. Run 'nemoclaw ${sandboxName} shields down' first.`, + ); + } + const currentConfig = readSandboxConfig(sandboxName, target); + const currentConfigSha256 = ( + currentConfig as ConfigObject & { [CONFIG_SOURCE_SHA256]?: string } + )[CONFIG_SOURCE_SHA256]; + if (currentConfigSha256 !== initialConfigSha256) { + configFail( + ` ${target.agentName} config changed while this update was being validated. Re-run config set against the current value.`, + ); + } + setDotpath(currentConfig, opts.key!, safeValue); + + console.log(` Writing config to sandbox (${target.configPath})...`); + writeSandboxConfig(sandboxName, target, currentConfig); + recomputeSandboxConfigHash(sandboxName, target); + + appendAuditEntry({ + action: "config_set", + sandbox: sandboxName, + timestamp: new Date().toISOString(), + reason: `config set ${target.agentName}:${opts.key}`, + }); + }), + ); console.log(` ${target.agentName} config updated.`); diff --git a/src/lib/sandbox/privileged-exec.test.ts b/src/lib/sandbox/privileged-exec.test.ts index e225ce318e9..eda0e5cecb2 100644 --- a/src/lib/sandbox/privileged-exec.test.ts +++ b/src/lib/sandbox/privileged-exec.test.ts @@ -66,54 +66,41 @@ describe("privileged sandbox exec routing", () => { expect(containerNameMatchesSandbox("openshell-gateway-nemoclaw", "demo")).toBe(false); }); - it("prefers the exact direct sandbox container when present", () => { - const selected = selectDirectSandboxContainer( - "demo", - "openshell-demo-helper\nopenshell-demo\n", - ["demo"], + it("selects the immutable id of one labeled direct sandbox container", () => { + expect(selectDirectSandboxContainer("demo", "abc123\topenshell-demo-2026\n", ["demo"])).toBe( + "abc123", ); + }); - expect(selected).toBe("openshell-demo"); + it("rejects ambiguous labeled running containers", () => { + expect(() => + selectDirectSandboxContainer( + "demo", + "abc123\topenshell-demo-one\ndef456\topenshell-demo-two\n", + ["demo"], + ), + ).toThrow(/Multiple running OpenShell containers.*refusing ambiguous/); }); - it("falls back to a generated direct sandbox container suffix", () => { - const selected = selectDirectSandboxContainer( - "demo", - "openshell-other\nopenshell-demo-abc123\n", - ["demo"], + it("rejects malformed Docker metadata", () => { + expect(() => selectDirectSandboxContainer("demo", "openshell-demo\n", ["demo"])).toThrow( + /malformed OpenShell sandbox container metadata/, ); - - expect(selected).toBe("openshell-demo-abc123"); }); - it("fails closed when multiple suffix containers match without an exact identity", () => { + it("rejects an authoritative label and container-name mismatch", () => { expect(() => - selectDirectSandboxContainer("demo", "openshell-demo-old\nopenshell-demo-new\n", ["demo"]), - ).toThrow(/Multiple running direct OpenShell containers.*demo.*old.*new/); + selectDirectSandboxContainer("alpha", "gateway-id\topenshell-gateway-nemoclaw\n", ["alpha"]), + ).toThrow(/labels and names disagree.*refusing lifecycle execution/); }); - it("uses the longest registered sandbox-name match to avoid prefix collisions", () => { - const containerNames = [ - "openshell-alpha-child", - "openshell-alpha-child-2026", - "openshell-alpha-abc123", - ].join("\n"); - - expect(selectDirectSandboxContainer("alpha", containerNames, ["alpha", "alpha-child"])).toBe( - "openshell-alpha-abc123", - ); - expect( - selectDirectSandboxContainer("alpha-child", containerNames, ["alpha", "alpha-child"]), - ).toBe("openshell-alpha-child"); - }); - - it("does not consider unrelated OpenShell containers direct sandbox matches", () => { - expect( - selectDirectSandboxContainer("alpha", "openshell-gateway-nemoclaw\nopenshell-alpha-child\n", [ + it("uses the longest registered sandbox-name match to reject prefix collisions", () => { + expect(() => + selectDirectSandboxContainer("alpha", "child-id\topenshell-alpha-child\n", [ "alpha", "alpha-child", ]), - ).toBeNull(); + ).toThrow(/labels and names disagree.*refusing lifecycle execution/); }); it("builds privileged docker exec argv through the registered direct sandbox container", () => { @@ -124,7 +111,7 @@ describe("privileged sandbox exec routing", () => { sandboxes: [{ name: "alpha" }, { name: "alpha-child" }], defaultSandbox: "alpha", }), - dockerCapture: () => "openshell-alpha-child\nopenshell-alpha-abc123\n", + dockerCapture: () => "immutable-alpha-id\topenshell-alpha-abc123\n", }, ({ privilegedSandboxExecArgv }) => { expect(privilegedSandboxExecArgv("alpha", ["id"], true)).toEqual([ @@ -132,7 +119,7 @@ describe("privileged sandbox exec routing", () => { "-i", "--user", "root", - "openshell-alpha-abc123", + "immutable-alpha-id", "id", ]); }, @@ -151,7 +138,7 @@ describe("privileged sandbox exec routing", () => { listSandboxes: () => ({ sandboxes: [{ name: "alpha" }], defaultSandbox: "alpha" }), dockerCapture: (args, options) => { discoveryCalls.push({ args, timeout: options?.timeout }); - return "openshell-alpha\n"; + return "immutable-alpha-id\topenshell-alpha\n"; }, }, ({ privilegedSandboxExecArgv }) => { @@ -159,7 +146,7 @@ describe("privileged sandbox exec routing", () => { "exec", "--user", "root", - "openshell-alpha", + "immutable-alpha-id", "id", ]); }, @@ -167,7 +154,16 @@ describe("privileged sandbox exec routing", () => { expect(discoveryCalls).toEqual([ { - args: ["ps", "--format", "{{.Names}}"], + args: [ + "ps", + "--no-trunc", + "--filter", + "label=openshell.ai/managed-by=openshell", + "--filter", + "label=openshell.ai/sandbox-name=alpha", + "--format", + "{{.ID}}\t{{.Names}}", + ], timeout: 5000, }, ]); @@ -178,7 +174,7 @@ describe("privileged sandbox exec routing", () => { { getSandbox: () => ({ name: "alpha", openshellDriver: "docker" }), listSandboxes: () => ({ sandboxes: [{ name: "alpha" }], defaultSandbox: "alpha" }), - dockerCapture: () => "openshell-alpha\n", + dockerCapture: () => "immutable-alpha-id\topenshell-alpha\n", }, ({ privilegedSandboxExecArgv }) => { const argv = privilegedSandboxExecArgv("alpha", ["/trusted/control"], false, true); @@ -190,7 +186,12 @@ describe("privileged sandbox exec routing", () => { expect(argv).toContain("PYTHONUSERBASE="); expect(argv).toContain("PYTHONNOUSERSITE=1"); expect(argv).toContain("BASH_ENV="); - expect(argv.slice(-4)).toEqual(["--user", "root", "openshell-alpha", "/trusted/control"]); + expect(argv.slice(-4)).toEqual([ + "--user", + "root", + "immutable-alpha-id", + "/trusted/control", + ]); }, ); }); @@ -205,7 +206,7 @@ describe("privileged sandbox exec routing", () => { listSandboxes: () => ({ sandboxes: [], defaultSandbox: null }), dockerCapture: () => { dockerPsCalls += 1; - return "openshell-alpha-child\n"; + return "child-id\topenshell-alpha-child\n"; }, }, ({ privilegedSandboxExecArgv }) => { @@ -223,7 +224,7 @@ describe("privileged sandbox exec routing", () => { listSandboxes: () => ({ sandboxes: [{ name: "alpha" }], defaultSandbox: "alpha" }), dockerCapture: () => { dockerPsCalls += 1; - return "openshell-alpha-stale\n"; + return "stale-id\topenshell-alpha-stale\n"; }, }, ({ privilegedSandboxExecArgv }) => { @@ -245,7 +246,7 @@ describe("privileged sandbox exec routing", () => { }, dockerCapture: () => { dockerPsCalls += 1; - return "openshell-alpha-child\n"; + return "child-id\topenshell-alpha-child\n"; }, }, ({ privilegedSandboxExecArgv }) => { @@ -282,7 +283,7 @@ describe("privileged sandbox exec routing", () => { sandboxes: [{ name: "alpha" }, { name: "alpha-child" }], defaultSandbox: "alpha", }), - dockerCapture: () => "openshell-alpha-child\n", + dockerCapture: () => "", }, ({ isDirectSandboxFallbackUnavailableError, privilegedSandboxExecArgv }) => { let refusal: unknown; diff --git a/src/lib/sandbox/privileged-exec.ts b/src/lib/sandbox/privileged-exec.ts index 9003bd644a1..02a4bb9a6a2 100644 --- a/src/lib/sandbox/privileged-exec.ts +++ b/src/lib/sandbox/privileged-exec.ts @@ -4,11 +4,20 @@ import { dockerCapture } from "../adapters/docker/run"; import * as registry from "../state/registry"; +const OPENSHELL_MANAGED_BY_LABEL = "openshell.ai/managed-by"; +const OPENSHELL_MANAGED_BY_VALUE = "openshell"; +const OPENSHELL_SANDBOX_NAME_LABEL = "openshell.ai/sandbox-name"; + type SandboxEntry = { name?: string; openshellDriver?: string | null; }; +type LabeledSandboxContainer = { + id: string; + name: string; +}; + const DIRECT_SANDBOX_DISCOVERY_TIMEOUT_MS = 5000; const SANITIZED_PRIVILEGED_ENV = [ "BASH_ENV=", @@ -81,38 +90,48 @@ function owningRegisteredSandboxName( return registeredNames.find((name) => containerNameMatchesSandbox(containerName, name)) ?? null; } +function parseLabeledSandboxContainers(output: string): LabeledSandboxContainer[] { + return output + .split(/\r?\n/) + .map((line) => line.trim()) + .filter(Boolean) + .map((line) => { + const [id, name, ...unexpected] = line.split("\t"); + if (!id || !name || unexpected.length > 0 || /\s/.test(id)) { + throw new Error("Docker returned malformed OpenShell sandbox container metadata."); + } + return { id, name }; + }); +} + function selectDirectSandboxContainer( sandboxName: string, - containerNames: string, + labeledContainerRows: string, registeredNames: readonly string[] = [sandboxName], ): string | null { const names = Array.from(new Set([...registeredNames, sandboxName])).sort( (a, b) => b.length - a.length || a.localeCompare(b), ); - const candidates = Array.from( - new Set( - containerNames - .split("\n") - .map((line: string) => line.trim()) - .filter(Boolean) - .filter((containerName: string) => { - if (!containerNameMatchesSandbox(containerName, sandboxName)) return false; - return owningRegisteredSandboxName(containerName, names) === sandboxName; - }), - ), - ); - - const exact = candidates.find( - (containerName: string) => containerName === `openshell-${sandboxName}`, - ); - if (exact) return exact; - if (candidates.length === 1) return candidates[0]; + const candidates = parseLabeledSandboxContainers(labeledContainerRows); + if ( + candidates.some( + ({ name }) => + !containerNameMatchesSandbox(name, sandboxName) || + owningRegisteredSandboxName(name, names) !== sandboxName, + ) + ) { + throw new Error( + `OpenShell container labels and names disagree for sandbox '${sandboxName}'; ` + + "refusing lifecycle execution.", + ); + } if (candidates.length > 1) { throw new Error( - `Multiple running direct OpenShell containers match registered sandbox '${sandboxName}': ${candidates.join(", ")}. Refusing privileged exec without an exact container identity.`, + `Multiple running OpenShell containers are labeled for sandbox '${sandboxName}'; ` + + "refusing ambiguous lifecycle execution.", ); } - return null; + return candidates[0]?.id ?? null; } function expectedDirectContainerPattern(sandboxName: string): string { @@ -123,9 +142,19 @@ function findDirectSandboxContainer(sandboxName: string): string | null { const names = registeredSandboxNames(sandboxName); let output: string; try { - output = dockerCapture(["ps", "--format", "{{.Names}}"], { - timeout: DIRECT_SANDBOX_DISCOVERY_TIMEOUT_MS, - }); + output = dockerCapture( + [ + "ps", + "--no-trunc", + "--filter", + `label=${OPENSHELL_MANAGED_BY_LABEL}=${OPENSHELL_MANAGED_BY_VALUE}`, + "--filter", + `label=${OPENSHELL_SANDBOX_NAME_LABEL}=${sandboxName}`, + "--format", + "{{.ID}}\t{{.Names}}", + ], + { timeout: DIRECT_SANDBOX_DISCOVERY_TIMEOUT_MS }, + ); } catch (error) { const detail = error instanceof Error ? error.message : String(error); throw new DirectSandboxFallbackUnavailableError( @@ -140,7 +169,8 @@ function missingDirectContainerError(sandboxName: string, driver: string | null) const driverLabel = driver ?? "unspecified"; return new DirectSandboxFallbackUnavailableError( `No running direct OpenShell sandbox container found for '${sandboxName}' ` + - `(driver: ${driverLabel}). Expected a running container named ` + + `(driver: ${driverLabel}). Expected one OpenShell-managed container labeled ` + + `'${OPENSHELL_SANDBOX_NAME_LABEL}=${sandboxName}' and named ` + `${expectedDirectContainerPattern(sandboxName)}. Is the sandbox running?`, ); } diff --git a/src/lib/security/mcp-url-target.ts b/src/lib/security/mcp-url-target.ts new file mode 100644 index 00000000000..8f11b9854df --- /dev/null +++ b/src/lib/security/mcp-url-target.ts @@ -0,0 +1,95 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { BlockList, isIP } from "node:net"; + +export const MCP_SERVER_URL_MAX_LENGTH = 2_048; + +const OPENSHELL_HOST_ALIASES = new Set([ + "host.openshell.internal", + "host.docker.internal", + "host.containers.internal", +]); + +const RESERVED_HOST_NAMES = new Set(["localhost", "local", "internal", "metadata"]); + +const blockedMcpTargets = new BlockList(); +for (const [address, prefix] of [ + ["0.0.0.0", 8], + ["10.0.0.0", 8], + ["100.64.0.0", 10], + ["127.0.0.0", 8], + ["169.254.0.0", 16], + ["172.16.0.0", 12], + ["192.0.0.0", 24], + ["192.0.2.0", 24], + ["192.31.196.0", 24], + ["192.52.193.0", 24], + ["192.88.99.0", 24], + ["192.168.0.0", 16], + ["192.175.48.0", 24], + ["198.18.0.0", 15], + ["198.51.100.0", 24], + ["203.0.113.0", 24], + ["224.0.0.0", 4], + ["240.0.0.0", 4], +] as const) { + blockedMcpTargets.addSubnet(address, prefix, "ipv4"); +} +for (const [address, prefix] of [ + ["::", 128], + ["::1", 128], + // Deprecated IPv4-compatible encodings (for example ::7f00:1) can hide + // loopback/private IPv4 targets from a naive IPv6-only check. + ["::", 96], + ["64:ff9b::", 96], + ["64:ff9b:1::", 48], + ["100::", 64], + // IETF protocol assignments including Teredo, benchmarking, ORCHID, and + // other non-global special-purpose destinations. + ["2001::", 23], + ["2001:db8::", 32], + ["2002::", 16], + ["2620:4f:8000::", 48], + ["3fff::", 20], + ["5f00::", 16], + ["fc00::", 7], + ["fe80::", 10], + ["fec0::", 10], + ["ff00::", 8], +] as const) { + blockedMcpTargets.addSubnet(address, prefix, "ipv6"); +} + +export function normalizeMcpHostname(hostname: string): string { + return hostname + .toLowerCase() + .replace(/^\[|\]$/g, "") + .replace(/\.$/, ""); +} + +export function isOpenShellMcpHostAlias(hostname: string): boolean { + return OPENSHELL_HOST_ALIASES.has(normalizeMcpHostname(hostname)); +} + +function isReservedMcpName(hostname: string): boolean { + if (RESERVED_HOST_NAMES.has(hostname)) return true; + for (const reserved of RESERVED_HOST_NAMES) { + if (hostname.endsWith(`.${reserved}`)) return true; + } + return false; +} + +export function isBlockedMcpUrlTargetHost(hostname: string): boolean { + const normalized = normalizeMcpHostname(hostname); + if (isOpenShellMcpHostAlias(normalized)) return false; + if (isReservedMcpName(normalized)) return true; + // Node's URL parser canonicalizes mapped literals such as + // ::ffff:10.0.0.1 to ::ffff:a00:1. Reject the mapped class explicitly; + // putting ::ffff/96 in the shared BlockList also matches every ordinary + // IPv4 check because Node internally maps IPv4 addresses. + if (normalized.startsWith("::ffff:")) return true; + const family = isIP(normalized); + if (family === 0) return false; + return blockedMcpTargets.check(normalized, family === 6 ? "ipv6" : "ipv4"); +} diff --git a/src/lib/security/redact.ts b/src/lib/security/redact.ts index a12cc2592e2..65136359207 100644 --- a/src/lib/security/redact.ts +++ b/src/lib/security/redact.ts @@ -1,6 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import type { StdioOptions } from "node:child_process"; + /** * Unified secret redaction — single module for all consumers. * @@ -91,7 +93,7 @@ export function redactError(err: unknown): unknown { export function writeRedactedResult( result: { stdout?: Buffer | string | null; stderr?: Buffer | string | null } | null, - stdio: string | string[], + stdio: StdioOptions | undefined, ): void { if (!result || stdio === "inherit" || !Array.isArray(stdio)) return; if (stdio[1] === "pipe" && result.stdout) { @@ -134,6 +136,16 @@ export function redactFull(text: string): string { return result; } +/** Redact self-identifying tokens and secret blocks without rewriting surrounding structure. */ +export function redactStandaloneSecretsFull(text: string): string { + let result = text; + for (const pattern of [...TOKEN_PREFIX_PATTERNS, ...SECRET_BLOCK_PATTERNS]) { + pattern.lastIndex = 0; + result = result.replace(pattern, ""); + } + return result.replace(/\/bot[^/\s]+\//g, "/bot/"); +} + // ── Sensitive text redaction (onboard-session.ts style) ───────── export function redactSensitiveText(value: unknown): string | null { diff --git a/src/lib/shields/flow.test.ts b/src/lib/shields/flow.test.ts index 410116d66ab..63212fbaaf4 100644 --- a/src/lib/shields/flow.test.ts +++ b/src/lib/shields/flow.test.ts @@ -256,7 +256,9 @@ describe("shields command flow", () => { delete require.cache[requireDist.resolve("../cli/branding.js")]; }); - it("shieldsDown captures policy, unlocks config, saves state, and skips timer on request", () => { + it("shieldsDown captures policy, unlocks config, saves state, and skips timer on request", { + timeout: 15_000, + }, () => { const harness = createHarness(); harness.shieldsDown("openclaw", { @@ -279,7 +281,7 @@ describe("shields command flow", () => { expect(harness.logSpy.mock.calls.flat().join("\n")).toContain( "Config unlocked for openclaw (no auto-lockdown timer", ); - }, 15_000); + }); it("binds manual shields-up to the active auto-restore timer generation", () => { const stateDir = path.join(tmpDir, ".nemoclaw", "state"); @@ -881,7 +883,7 @@ describe("shields command flow", () => { ).toBe(true); }); - it("shieldsStatus restores an expired dead timer through the same lock path as shields up", () => { + it("shieldsStatus restores an expired dead timer under the shared sandbox lock", async () => { const configPath = "/sandbox/.openclaw/openclaw.json"; const configDir = "/sandbox/.openclaw"; const hashPath = `${configDir}/.config-hash`; @@ -899,7 +901,14 @@ describe("shields command flow", () => { [` sha256sum ${hashPath}`, `${hashHash} ${hashPath}\n`], [` sha256sum ${configPath}`, `${configHash} ${configPath}\n`], ]); + const lifecycleLock = requireDist("../state/mcp-lifecycle-lock.js"); + const sandboxMutationLockPath = lifecycleLock.getMcpLifecycleLockPath("openclaw"); + let policySetSawSandboxLock = false; const harness = createHarness({ + run: () => { + policySetSawSandboxLock = fs.existsSync(sandboxMutationLockPath); + return { status: 0 }; + }, dockerExecFileSync: (argv: unknown) => { const args = Array.isArray(argv) ? argv.map(String) : []; const cmd = args.join(" "); @@ -956,7 +965,9 @@ describe("shields command flow", () => { return true; }); - harness.shieldsStatus("openclaw"); + await lifecycleLock.withSandboxMutationLock("openclaw", () => + harness.shieldsStatus("openclaw"), + ); const state = JSON.parse( fs.readFileSync(path.join(stateDir, "shields-openclaw.json"), "utf-8"), @@ -969,6 +980,8 @@ describe("shields command flow", () => { }); expect(fs.existsSync(path.join(stateDir, "shields-timer-openclaw.json"))).toBe(false); expect(fs.existsSync(lockPath)).toBe(false); + expect(policySetSawSandboxLock).toBe(true); + expect(fs.existsSync(sandboxMutationLockPath)).toBe(false); expect(harness.auditSpy).toHaveBeenCalledWith( expect.objectContaining({ action: "shields_auto_restore", diff --git a/src/lib/shields/timer.test.ts b/src/lib/shields/timer.test.ts index 8dd80effec4..418ee94a4cf 100644 --- a/src/lib/shields/timer.test.ts +++ b/src/lib/shields/timer.test.ts @@ -6,6 +6,7 @@ import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { getMcpLifecycleLockPath } from "../state/mcp-lifecycle-lock"; const shieldsIndexMock = vi.hoisted(() => ({ lockAgentConfig: vi.fn() as unknown, @@ -63,13 +64,16 @@ describe("shields timer authorization", () => { fs.rmSync(tmpHome, { recursive: true, force: true }); }); - function invokeTimerAndCaptureExit(runRestoreTimer: (args: any) => void, args: unknown): number { + async function invokeTimerAndCaptureExit( + runRestoreTimer: (args: any) => Promise, + args: unknown, + ): Promise { const exitSpy = vi.spyOn(process, "exit").mockImplementation((code?: any) => { throw new Error(`process.exit:${String(code ?? 0)}`); }); try { - runRestoreTimer(args); + await runRestoreTimer(args); throw new Error("Expected runRestoreTimer to exit"); } catch (error) { const message = error instanceof Error ? error.message : String(error); @@ -81,13 +85,16 @@ describe("shields timer authorization", () => { } } - function invokeTimerAndExpectRetry(runRestoreTimer: (args: any) => void, args: unknown): void { + async function invokeTimerAndExpectRetry( + runRestoreTimer: (args: any) => Promise, + args: unknown, + ): Promise { vi.useFakeTimers(); const exitSpy = vi .spyOn(process, "exit") .mockImplementation((() => undefined) as typeof process.exit); try { - runRestoreTimer(args); + await runRestoreTimer(args); expect(exitSpy).not.toHaveBeenCalled(); expect(vi.getTimerCount()).toBe(1); } finally { @@ -113,7 +120,7 @@ describe("shields timer authorization", () => { const args = timer.parseTimerArgs([sandboxName, snapshotPath, restoreAtIso, "", "", "tok"]); expect(args).not.toBeNull(); - const exitCode = invokeTimerAndCaptureExit(timer.runRestoreTimer, args); + const exitCode = await invokeTimerAndCaptureExit(timer.runRestoreTimer, args); expect(exitCode).toBe(0); expect(runMock).not.toHaveBeenCalled(); @@ -155,7 +162,7 @@ describe("shields timer authorization", () => { ]); expect(args).not.toBeNull(); - const exitCode = invokeTimerAndCaptureExit(timer.runRestoreTimer, args); + const exitCode = await invokeTimerAndCaptureExit(timer.runRestoreTimer, args); expect(exitCode).toBe(0); expect(runMock).not.toHaveBeenCalled(); @@ -253,7 +260,7 @@ describe("shields timer authorization", () => { ]); expect(args).not.toBeNull(); - timer.runRestoreTimer(args!); + await timer.runRestoreTimer(args!); expect(runMock).not.toHaveBeenCalled(); expect(exitSpy).not.toHaveBeenCalled(); @@ -278,6 +285,7 @@ describe("shields timer authorization", () => { const snapshotPath = path.join(stateDir, "snapshot.yaml"); const restoreAtIso = new Date().toISOString(); const markerPath = path.join(stateDir, `shields-timer-${sandboxName}.json`); + const sandboxMutationLockPath = getMcpLifecycleLockPath(sandboxName, stateDir); fs.writeFileSync(snapshotPath, "version: 1\nnetwork_policies: {}\n"); fs.writeFileSync( markerPath, @@ -291,7 +299,10 @@ describe("shields timer authorization", () => { leaseOwnerStartIdentity: "proc:dead-owner", }), ); - runMock.mockReturnValueOnce({ status: 17 }); + runMock.mockImplementationOnce(() => { + expect(fs.existsSync(sandboxMutationLockPath)).toBe(true); + return { status: 17 }; + }); const args = timer.parseTimerArgs([ sandboxName, snapshotPath, @@ -305,12 +316,13 @@ describe("shields timer authorization", () => { ]); expect(args).not.toBeNull(); - timer.runRestoreTimer(args!); + await timer.runRestoreTimer(args!); expect(runMock).toHaveBeenCalledTimes(1); expect(exitSpy).not.toHaveBeenCalled(); expect(vi.getTimerCount()).toBe(1); expect(fs.existsSync(markerPath)).toBe(true); + expect(fs.existsSync(sandboxMutationLockPath)).toBe(false); } finally { exitSpy.mockRestore(); vi.useRealTimers(); @@ -352,7 +364,7 @@ describe("shields timer authorization", () => { ]); expect(args).not.toBeNull(); - const exitCode = invokeTimerAndCaptureExit(timer.runRestoreTimer, args); + const exitCode = await invokeTimerAndCaptureExit(timer.runRestoreTimer, args); expect(exitCode).toBe(0); expect(runMock).not.toHaveBeenCalled(); @@ -415,6 +427,7 @@ describe("shields timer authorization", () => { const snapshotPath = path.join(stateDir, "snapshot.yaml"); const restoreAtIso = new Date(Date.now() + 60_000).toISOString(); const markerPath = path.join(stateDir, `shields-timer-${sandboxName}.json`); + const sandboxMutationLockPath = getMcpLifecycleLockPath(sandboxName, stateDir); fs.writeFileSync(snapshotPath, "version: 1\nnetwork_policies:\n default: {}\n"); fs.writeFileSync( @@ -440,6 +453,7 @@ describe("shields timer authorization", () => { const lockPath = path.join(stateDir, `shields-transition-lock-${sandboxName}.json`); runMock.mockImplementationOnce(() => { + expect(fs.existsSync(sandboxMutationLockPath)).toBe(true); expect(JSON.parse(fs.readFileSync(lockPath, "utf-8"))).toMatchObject({ sandboxName, command: "shields auto-restore", @@ -448,7 +462,7 @@ describe("shields timer authorization", () => { return { status: 0 }; }); - const exitCode = invokeTimerAndCaptureExit(timer.runRestoreTimer, args); + const exitCode = await invokeTimerAndCaptureExit(timer.runRestoreTimer, args); const stateFile = path.join(stateDir, `shields-${sandboxName}.json`); const updatedState = JSON.parse(fs.readFileSync(stateFile, "utf-8")); @@ -457,6 +471,7 @@ describe("shields timer authorization", () => { expect(updatedState.shieldsDown).toBe(false); expect(updatedState.shieldsDownAt).toBeNull(); expect(fs.existsSync(markerPath)).toBe(false); + expect(fs.existsSync(sandboxMutationLockPath)).toBe(false); }); it("retains recovery authority when the locked-state commit cannot be persisted", async () => { @@ -503,7 +518,7 @@ describe("shields timer authorization", () => { PROCESS_TOKEN, ]); expect(args).not.toBeNull(); - invokeTimerAndExpectRetry(timer.runRestoreTimer, args); + await invokeTimerAndExpectRetry(timer.runRestoreTimer, args); } finally { renameSpy.mockRestore(); } @@ -567,7 +582,7 @@ describe("shields timer authorization", () => { ]); expect(args).not.toBeNull(); - const exitCode = invokeTimerAndCaptureExit(timer.runRestoreTimer, args); + const exitCode = await invokeTimerAndCaptureExit(timer.runRestoreTimer, args); const updatedState = JSON.parse(fs.readFileSync(stateFile, "utf-8")); expect(exitCode).toBe(0); @@ -629,7 +644,7 @@ describe("shields timer authorization", () => { ]); expect(args).not.toBeNull(); - invokeTimerAndExpectRetry(timer.runRestoreTimer, args); + await invokeTimerAndExpectRetry(timer.runRestoreTimer, args); const updatedState = JSON.parse(fs.readFileSync(stateFile, "utf-8")); const auditEntries = fs .readFileSync(auditFile, "utf-8") @@ -728,7 +743,7 @@ describe("shields timer authorization", () => { ]); expect(args).not.toBeNull(); - const exitCode = invokeTimerAndCaptureExit(timer.runRestoreTimer, args); + const exitCode = await invokeTimerAndCaptureExit(timer.runRestoreTimer, args); // A single instantaneous lock+verify cannot prove the gateway didn't // re-permission .config-hash afterward. The fix must re-confirm the lock @@ -801,7 +816,7 @@ describe("shields timer authorization", () => { ]); expect(args).not.toBeNull(); - invokeTimerAndExpectRetry(timer.runRestoreTimer, args); + await invokeTimerAndExpectRetry(timer.runRestoreTimer, args); const updatedState = JSON.parse(fs.readFileSync(stateFile, "utf-8")); const auditEntries = fs .readFileSync(auditFile, "utf-8") diff --git a/src/lib/shields/timer.ts b/src/lib/shields/timer.ts index a0c25602535..3c729508995 100644 --- a/src/lib/shields/timer.ts +++ b/src/lib/shields/timer.ts @@ -14,6 +14,7 @@ import { isRecord, type UnknownRecord } from "../core/json-types"; import { buildPolicySetCommand } from "../policy"; import { run } from "../runner"; import { resolveAgentConfig } from "../sandbox/config"; +import { withSandboxMutationLock } from "../state/mcp-lifecycle-lock"; import { resolveNemoclawStateDir } from "../state/paths"; import { appendAuditEntry, type ShieldsAuditEntry } from "./audit"; import * as shields from "./index"; @@ -226,14 +227,16 @@ function rebuildLeaseOwnerIsCurrent(args: TimerArgs): boolean { ); } -function runRestoreTimer(args: TimerArgs): void { +async function runRestoreTimer(args: TimerArgs): Promise { const now = new Date().toISOString(); let exitCode = 0; let retryScheduled = false; const scheduleRetry = (): boolean => { if (!markerMatchesCurrentTimer(args)) return false; retryScheduled = true; - setTimeout(() => runRestoreTimer(args), AUTO_RESTORE_RETRY_MS); + setTimeout(() => { + void runRestoreTimer(args); + }, AUTO_RESTORE_RETRY_MS); return true; }; @@ -264,120 +267,88 @@ function runRestoreTimer(args: TimerArgs): void { args.snapshotPath, ); - withShieldsTransitionLock( - args.sandboxName, - "shields auto-restore", - () => { - // A manual hardening command may have completed while this timer waited - // for the host mutation lock. The marker is the timer's authority, so - // re-check it only after serialization is established. - if (!markerMatchesCurrentTimer(args)) return; - - if (!fs.existsSync(args.snapshotPath)) { - appendAudit({ - action: "shields_up_failed", - sandbox: args.sandboxName, - timestamp: now, - restored_by: "auto_timer", - error: "Policy snapshot file missing", - }); - exitCode = 1; - scheduleRetry(); - return; - } - - // Restore policy (slow — openshell policy set --wait blocks) - const result = run(buildPolicySetCommand(args.snapshotPath, args.sandboxName), { - ignoreError: true, - }); - const status = typeof result.status === "number" ? result.status : 1; + await withSandboxMutationLock(args.sandboxName, () => + withShieldsTransitionLock( + args.sandboxName, + "shields auto-restore", + () => { + // A manual hardening command may have completed while this timer waited + // for the host mutation lock. The marker is the timer's authority, so + // re-check it only after serialization is established. + if (!markerMatchesCurrentTimer(args)) return; + + if (!fs.existsSync(args.snapshotPath)) { + appendAudit({ + action: "shields_up_failed", + sandbox: args.sandboxName, + timestamp: now, + restored_by: "auto_timer", + error: "Policy snapshot file missing", + }); + exitCode = 1; + scheduleRetry(); + return; + } - if (status !== 0) { - appendAudit({ - action: "shields_up_failed", - sandbox: args.sandboxName, - timestamp: now, - restored_by: "auto_timer", - error: `Policy restore exited with status ${String(status)}`, + // Restore policy (slow — openshell policy set --wait blocks) + const result = run(buildPolicySetCommand(args.snapshotPath, args.sandboxName), { + ignoreError: true, }); - exitCode = 1; - scheduleRetry(); - return; - } - - // Destroy and force-restore can revoke this marker while a slow - // policy restore is already in flight. Stop before the next sandbox - // mutation if this timer generation no longer owns recovery. - if (!markerMatchesCurrentTimer(args)) return; - - // Re-lock config file using the shared lockAgentConfig from shields.ts. - // lockAgentConfig runs each operation independently and verifies the - // on-disk state — it throws if verification fails. - // - // NC-2227-03: Resolve the full agent config target (including sensitive - // files like .config-hash, .env) so the timer re-locks the same scope - // that interactive `shields up` uses. Fall back to the bare configPath/ - // configDir from argv if resolution fails (e.g., registry unavailable). - let lockVerified = true; - let lockedChattr: boolean | null = null; - let lockedHashes: { [path: string]: string } | null = null; - if (args.configPath) { - let lockTarget: { - agentName?: string; - configPath: string; - configDir: string; - sensitiveFiles?: string[]; - } | null = null; - try { - // Always prefer the resolved target — even DEFAULT_AGENT_CONFIG - // carries the OpenClaw sensitiveFiles (.config-hash) that - // shields-up locks and that the content seal hashes. Dropping - // them here would persist a partial fileHashes map and the next - // `shields status` would flag the missing entries as drift. - lockTarget = resolveAgentConfig(args.sandboxName); - } catch { - // Resolver itself threw (registry unavailable). Fall back to - // argv-supplied paths, but still infer sensitiveFiles from - // configDir so the locked set matches what shields-up uses. - if (args.configDir) { - lockTarget = { - configPath: args.configPath, - configDir: args.configDir, - sensitiveFiles: [`${args.configDir}/.config-hash`], - }; - } else { - lockVerified = false; - appendAudit({ - action: "shields_auto_restore_lock_warning", - sandbox: args.sandboxName, - timestamp: now, - restored_by: "auto_timer", - warning: "Missing config directory for auto-restore re-lock verification", - lock_verified: false, - }); - } + const status = typeof result.status === "number" ? result.status : 1; + + if (status !== 0) { + appendAudit({ + action: "shields_up_failed", + sandbox: args.sandboxName, + timestamp: now, + restored_by: "auto_timer", + error: `Policy restore exited with status ${String(status)}`, + }); + exitCode = 1; + scheduleRetry(); + return; } - if (lockTarget) { + + // Destroy and force-restore can revoke this marker while a slow + // policy restore is already in flight. Stop before the next sandbox + // mutation if this timer generation no longer owns recovery. + if (!markerMatchesCurrentTimer(args)) return; + + // Re-lock config file using the shared lockAgentConfig from shields.ts. + // lockAgentConfig runs each operation independently and verifies the + // on-disk state — it throws if verification fails. + // + // NC-2227-03: Resolve the full agent config target (including sensitive + // files like .config-hash, .env) so the timer re-locks the same scope + // that interactive `shields up` uses. Fall back to the bare configPath/ + // configDir from argv if resolution fails (e.g., registry unavailable). + let lockVerified = true; + let lockedChattr: boolean | null = null; + let lockedHashes: { [path: string]: string } | null = null; + if (args.configPath) { + let lockTarget: { + agentName?: string; + configPath: string; + configDir: string; + sensitiveFiles?: string[]; + } | null = null; try { - if (!markerMatchesCurrentTimer(args)) return; - const lockAgentConfig = resolveLockAgentConfig(); - // #4663: a single instantaneous lock+verify cannot prove an - // in-sandbox reconciler didn't re-permission .config-hash after the - // verified lock returned. Re-confirm the lock held once the gateway - // has settled, re-applying if it drifted. This narrows (does not - // close) the revert window; fail closed (leave shields DOWN + audit) - // when the lock will not re-confirm within the retry budget. - const relock = relockAndReconfirm(() => - lockAgentConfig( - args.sandboxName, - lockTarget, - false, - args.allowLegacyHermesProtocol, - ), - ); - if (relock.ok && relock.lastResult) { - lockedChattr = relock.lastResult.chattrApplied; - lockedHashes = relock.lastResult.fileHashes; + // Always prefer the resolved target — even DEFAULT_AGENT_CONFIG + // carries the OpenClaw sensitiveFiles (.config-hash) that + // shields-up locks and that the content seal hashes. Dropping + // them here would persist a partial fileHashes map and the next + // `shields status` would flag the missing entries as drift. + lockTarget = resolveAgentConfig(args.sandboxName); + } catch { + // Resolver itself threw (registry unavailable). Fall back to + // argv-supplied paths, but still infer sensitiveFiles from + // configDir so the locked set matches what shields-up uses. + if (args.configDir) { + lockTarget = { + configPath: args.configPath, + configDir: args.configDir, + sensitiveFiles: [`${args.configDir}/.config-hash`], + }; } else { lockVerified = false; appendAudit({ @@ -385,68 +356,103 @@ function runRestoreTimer(args: TimerArgs): void { sandbox: args.sandboxName, timestamp: now, restored_by: "auto_timer", - warning: relock.error ?? "Config re-lock did not re-confirm after settle window", + warning: "Missing config directory for auto-restore re-lock verification", lock_verified: false, }); } - } catch (error: unknown) { - lockVerified = false; - appendAudit({ - action: "shields_auto_restore_lock_warning", - sandbox: args.sandboxName, - timestamp: now, - restored_by: "auto_timer", - warning: error instanceof Error ? error.message : String(error), - lock_verified: false, - }); } + if (lockTarget) { + try { + if (!markerMatchesCurrentTimer(args)) return; + const lockAgentConfig = resolveLockAgentConfig(); + // #4663: a single instantaneous lock+verify cannot prove an + // in-sandbox reconciler didn't re-permission .config-hash after the + // verified lock returned. Re-confirm the lock held once the gateway + // has settled, re-applying if it drifted. This narrows (does not + // close) the revert window; fail closed (leave shields DOWN + audit) + // when the lock will not re-confirm within the retry budget. + const relock = relockAndReconfirm(() => + lockAgentConfig( + args.sandboxName, + lockTarget, + false, + args.allowLegacyHermesProtocol, + ), + ); + if (relock.ok && relock.lastResult) { + lockedChattr = relock.lastResult.chattrApplied; + lockedHashes = relock.lastResult.fileHashes; + } else { + lockVerified = false; + appendAudit({ + action: "shields_auto_restore_lock_warning", + sandbox: args.sandboxName, + timestamp: now, + restored_by: "auto_timer", + warning: + relock.error ?? "Config re-lock did not re-confirm after settle window", + lock_verified: false, + }); + } + } catch (error: unknown) { + lockVerified = false; + appendAudit({ + action: "shields_auto_restore_lock_warning", + sandbox: args.sandboxName, + timestamp: now, + restored_by: "auto_timer", + warning: error instanceof Error ? error.message : String(error), + lock_verified: false, + }); + } + } + } + + // Re-lock verification includes a settle window. Do not rewrite state + // or remove a replacement marker if authority changed while it ran. + if (!markerMatchesCurrentTimer(args)) return; + + // Only mark shields as UP if the lock was verified (or no config path). + if (lockVerified) { + const patch: ShieldsStatePatch = { + shieldsDown: false, + shieldsDownAt: null, + shieldsDownTimeout: null, + shieldsDownReason: null, + shieldsDownPolicy: null, + }; + if (lockedChattr !== null) patch.chattrApplied = lockedChattr; + if (lockedHashes !== null) patch.fileHashes = lockedHashes; + updateState(args.stateFile, patch); + + appendAudit({ + action: "shields_auto_restore", + sandbox: args.sandboxName, + timestamp: now, + restored_by: "auto_timer", + policy_snapshot: args.snapshotPath, + scheduled_restore_at: args.restoreAtIso, + }); + cleanupOwnedTimerMarker(args); + return; } - } - - // Re-lock verification includes a settle window. Do not rewrite state - // or remove a replacement marker if authority changed while it ran. - if (!markerMatchesCurrentTimer(args)) return; - - // Only mark shields as UP if the lock was verified (or no config path). - if (lockVerified) { - const patch: ShieldsStatePatch = { - shieldsDown: false, - shieldsDownAt: null, - shieldsDownTimeout: null, - shieldsDownReason: null, - shieldsDownPolicy: null, - }; - if (lockedChattr !== null) patch.chattrApplied = lockedChattr; - if (lockedHashes !== null) patch.fileHashes = lockedHashes; - updateState(args.stateFile, patch); + // Explicitly ensure state reflects shields are still DOWN. + // shieldsDown() already wrote shieldsDown: true, but be explicit rather + // than relying on the absence of an update. + updateState(args.stateFile, { shieldsDown: true }); appendAudit({ - action: "shields_auto_restore", + action: "shields_up_failed", sandbox: args.sandboxName, timestamp: now, restored_by: "auto_timer", - policy_snapshot: args.snapshotPath, - scheduled_restore_at: args.restoreAtIso, + error: "Config re-lock verification failed — shields remain DOWN", }); - cleanupOwnedTimerMarker(args); - return; - } - - // Explicitly ensure state reflects shields are still DOWN. - // shieldsDown() already wrote shieldsDown: true, but be explicit rather - // than relying on the absence of an update. - updateState(args.stateFile, { shieldsDown: true }); - appendAudit({ - action: "shields_up_failed", - sandbox: args.sandboxName, - timestamp: now, - restored_by: "auto_timer", - error: "Config re-lock verification failed — shields remain DOWN", - }); - exitCode = 1; - scheduleRetry(); - }, - { takeoverToken: args.processToken }, + exitCode = 1; + scheduleRetry(); + }, + { takeoverToken: args.processToken }, + ), ); } catch (error: unknown) { appendAudit({ @@ -475,7 +481,7 @@ function main(): void { scheduled = true; setTimeout( () => { - runRestoreTimer(args); + void runRestoreTimer(args); }, Math.max(0, args.restoreAtMs - Date.now()), ); diff --git a/src/lib/state/mcp-lifecycle-lock-acquisition.ts b/src/lib/state/mcp-lifecycle-lock-acquisition.ts new file mode 100644 index 00000000000..b1606c16667 --- /dev/null +++ b/src/lib/state/mcp-lifecycle-lock-acquisition.ts @@ -0,0 +1,247 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { AsyncLocalStorage } from "node:async_hooks"; +import crypto from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import { performance } from "node:perf_hooks"; + +import { + classifyMcpLifecycleLock, + createMcpLifecycleLockOwner, + type LockObservation, + type McpLifecycleLockDisposition, +} from "./mcp-lifecycle-lock-identity"; +import { + getMcpLifecycleLockPath, + mcpLifecycleLockPathExists, + readMcpLifecycleLockObservation, + reclaimStaleMcpLifecycleLockGeneration, + safelyReleaseMcpLifecycleLock, + writeMcpLifecycleLockCandidateAndLink, +} from "./mcp-lifecycle-lock-storage"; +import { resolveNemoclawStateDir } from "./paths"; + +const DEFAULT_POLL_INTERVAL_MS = 100; +const DEFAULT_TIMEOUT_MS = 30 * 60_000; +const DEFAULT_CORRUPT_LOCK_GRACE_MS = 30_000; + +interface CorruptGenerationTracker { + generation: string | null; + firstSeenAt: number; +} + +interface AcquiredMcpLifecycleLock { + lockPath: string; + token: string; +} + +export interface McpLifecycleLockOptions { + /** Override used by focused tests. Production callers use ~/.nemoclaw/state. */ + stateDir?: string; + pollIntervalMs?: number; + timeoutMs?: number; + corruptLockGraceMs?: number; +} + +interface HeldLockLease { + active: boolean; +} + +type HeldLockContext = ReadonlyMap; + +const heldLocks = new AsyncLocalStorage(); + +function positiveInteger(value: number | undefined, fallback: number): number { + return Number.isFinite(value) && (value ?? 0) > 0 ? Math.floor(value as number) : fallback; +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function resetCorruptGenerationTracker(tracker: CorruptGenerationTracker): void { + tracker.generation = null; + tracker.firstSeenAt = 0; +} + +/** Age one continuously observed corrupt inode with a monotonic clock. */ +function classifyObservedMcpLifecycleLock( + observation: LockObservation, + sandboxName: string, + corruptLockGraceMs: number, + corruptTracker: CorruptGenerationTracker, +): McpLifecycleLockDisposition { + if (!observation.owner || observation.owner.sandboxName !== sandboxName) { + const generation = `${observation.dev}:${observation.ino}:${observation.mtimeMs}`; + const now = performance.now(); + if (corruptTracker.generation !== generation) { + corruptTracker.generation = generation; + corruptTracker.firstSeenAt = now; + return "wait"; + } + return now - corruptTracker.firstSeenAt >= corruptLockGraceMs ? "stale" : "wait"; + } + resetCorruptGenerationTracker(corruptTracker); + // The wall-clock arguments are irrelevant for a structurally valid owner. + return classifyMcpLifecycleLock( + observation, + sandboxName, + observation.mtimeMs, + corruptLockGraceMs, + ); +} + +async function tryReapStaleLock( + lockPath: string, + sandboxName: string, + corruptLockGraceMs: number, + corruptTracker: CorruptGenerationTracker, +): Promise { + const reaperPath = `${lockPath}.reaper`; + const reaperToken = crypto.randomUUID(); + const reaperOwner = createMcpLifecycleLockOwner(sandboxName, reaperToken); + if (!(await writeMcpLifecycleLockCandidateAndLink(reaperPath, reaperOwner))) return false; + + try { + const latest = await readMcpLifecycleLockObservation(lockPath); + if (!latest) return true; + if ( + classifyObservedMcpLifecycleLock(latest, sandboxName, corruptLockGraceMs, corruptTracker) !== + "stale" + ) { + return false; + } + + return reclaimStaleMcpLifecycleLockGeneration(lockPath, latest); + } finally { + await safelyReleaseMcpLifecycleLock(reaperPath, reaperToken); + } +} + +async function acquireMcpLifecycleLock( + sandboxName: string, + options: McpLifecycleLockOptions, +): Promise { + const pollIntervalMs = positiveInteger(options.pollIntervalMs, DEFAULT_POLL_INTERVAL_MS); + const timeoutMs = positiveInteger(options.timeoutMs, DEFAULT_TIMEOUT_MS); + const corruptLockGraceMs = positiveInteger( + options.corruptLockGraceMs, + DEFAULT_CORRUPT_LOCK_GRACE_MS, + ); + const lockPath = getMcpLifecycleLockPath(sandboxName, options.stateDir); + await fs.promises.mkdir(path.dirname(lockPath), { + recursive: true, + mode: 0o700, + }); + + const startedAt = performance.now(); + const corruptMainTracker: CorruptGenerationTracker = { generation: null, firstSeenAt: 0 }; + const corruptReaperTracker: CorruptGenerationTracker = { generation: null, firstSeenAt: 0 }; + let lastOwnerPid: number | null = null; + for (;;) { + if (performance.now() - startedAt >= timeoutMs) { + const ownerSuffix = lastOwnerPid ? ` (owner pid ${lastOwnerPid})` : ""; + throw new Error( + `Timed out waiting for the sandbox mutation lock for '${sandboxName}'${ownerSuffix}. Another lifecycle, policy, channel, shields, or snapshot operation is still running.`, + ); + } + + const reaperPath = `${lockPath}.reaper`; + const reaperObservation = await readMcpLifecycleLockObservation(reaperPath); + if (reaperObservation) { + const reaperDisposition = classifyObservedMcpLifecycleLock( + reaperObservation, + sandboxName, + corruptLockGraceMs, + corruptReaperTracker, + ); + if (reaperDisposition === "stale") { + // The reaper has the same atomic, PID-identified owner format as the + // main lock. A SIGKILL at any point in stale-lock cleanup is therefore + // recoverable without age-expiring a legitimate long operation. + await reclaimStaleMcpLifecycleLockGeneration(reaperPath, reaperObservation); + continue; + } + await sleep(pollIntervalMs); + continue; + } + resetCorruptGenerationTracker(corruptReaperTracker); + + if (!(await mcpLifecycleLockPathExists(reaperPath))) { + const token = crypto.randomUUID(); + const owner = createMcpLifecycleLockOwner(sandboxName, token); + if (await writeMcpLifecycleLockCandidateAndLink(lockPath, owner)) { + // A stale-lock reaper may have appeared between our pre-check and the + // atomic link. Do not enter the critical section until that generation + // gate has gone away. + if (!(await mcpLifecycleLockPathExists(reaperPath))) return { lockPath, token }; + await safelyReleaseMcpLifecycleLock(lockPath, token); + } + } + + const observation = await readMcpLifecycleLockObservation(lockPath); + if (observation) { + lastOwnerPid = observation.owner?.pid ?? null; + if ( + classifyObservedMcpLifecycleLock( + observation, + sandboxName, + corruptLockGraceMs, + corruptMainTracker, + ) === "stale" + ) { + if (await tryReapStaleLock(lockPath, sandboxName, corruptLockGraceMs, corruptMainTracker)) { + continue; + } + } + } else { + resetCorruptGenerationTracker(corruptMainTracker); + } + await sleep(pollIntervalMs); + } +} + +/** + * Serializes the complete MCP lifecycle for one sandbox across processes. + * AsyncLocalStorage makes nested calls in the same lifecycle operation + * reentrant (rebuild recovery -> MCP restart), while separate top-level + * promises in one Node process still contend on the filesystem lock. + * + * The lease is host-local. If a state directory is shared across machines or + * PID namespaces, foreign owners fail closed and require operator/distributed + * lease resolution; local PID probing is never used to reap them. + * + * This is a CLI state lock only. It is not an MCP bridge, proxy, listener, or + * credential process and never participates in sandbox network traffic. + */ +export async function withMcpLifecycleLock( + sandboxName: string, + operation: () => Promise | T, + options: McpLifecycleLockOptions = {}, +): Promise { + const stateDir = options.stateDir ?? resolveNemoclawStateDir(); + const lockKey = getMcpLifecycleLockPath(sandboxName, stateDir); + const inherited = heldLocks.getStore(); + if (inherited?.get(lockKey)?.active) return await operation(); + + const acquired = await acquireMcpLifecycleLock(sandboxName, { + ...options, + stateDir, + }); + const lease: HeldLockLease = { active: true }; + const context = new Map(inherited ?? []); + context.set(lockKey, lease); + return heldLocks.run(context, async () => { + try { + return await operation(); + } finally { + // Async resources created by the callback retain their ALS store. Mark + // the lease inactive before releasing so a detached/later promise cannot + // mistake an ended parent operation for a still-held reentrant lock. + lease.active = false; + await safelyReleaseMcpLifecycleLock(acquired.lockPath, acquired.token); + } + }); +} diff --git a/src/lib/state/mcp-lifecycle-lock-identity.test.ts b/src/lib/state/mcp-lifecycle-lock-identity.test.ts new file mode 100644 index 00000000000..79f359a2bab --- /dev/null +++ b/src/lib/state/mcp-lifecycle-lock-identity.test.ts @@ -0,0 +1,392 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import fc from "fast-check"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { + classifyMcpLifecycleLock, + isMcpLifecycleLockOwner, + type LockObservation, + type McpLifecycleLockIdentityProbes, + type McpLifecycleLockOwner, +} from "./mcp-lifecycle-lock-identity"; +import { + getMcpLifecycleLockPath, + readMcpLifecycleLockObservation, +} from "./mcp-lifecycle-lock-storage"; + +const PROPERTY_RUNS = 250; +const PROPERTY_IO_TIMEOUT_MS = 15_000; +const SANDBOX_NAME = "property-sandbox"; +const LOCAL_HOST = "host:local"; +const LOCAL_NAMESPACE = "pid:[4026531836]"; + +const boundaryPidArbitrary = fc.oneof( + fc.integer({ min: 1, max: Number.MAX_SAFE_INTEGER }), + fc.constantFrom( + 1, + 32_767, + 4_194_303, + 4_194_304, + 2_147_483_647, + 2_147_483_648, + 4_294_967_295, + Number.MAX_SAFE_INTEGER, + ), +); +const clockArbitrary = fc.integer({ min: Number.MIN_SAFE_INTEGER, max: Number.MAX_SAFE_INTEGER }); +const tickValueArbitrary = fc.oneof( + fc.bigInt({ min: 0n, max: (1n << 128n) - 1n }), + fc.constantFrom(0n, 1n, (1n << 64n) - 1n, 1n << 64n, (1n << 128n) - 1n), +); +const tickArbitrary = tickValueArbitrary.map(String); +const bootArbitrary = fc.uuid(); +const distinctBootPairArbitrary = fc + .tuple(bootArbitrary, bootArbitrary) + .filter(([ownerBoot, currentBoot]) => ownerBoot !== currentBoot); +const processIdentityArbitrary = fc + .tuple(bootArbitrary, tickArbitrary) + .map(([boot, ticks]) => `linux:${boot}:${ticks}`); +const nonEmptyStringArbitrary = fc.string({ minLength: 1, maxLength: 80 }); + +function owner( + pid: number, + processIdentity: string | null, + overrides: Partial = {}, +): McpLifecycleLockOwner { + return { + version: 1, + sandboxName: SANDBOX_NAME, + pid, + processIdentity, + hostIdentity: LOCAL_HOST, + pidNamespaceIdentity: LOCAL_NAMESPACE, + token: "owner-token", + acquiredAt: "2026-06-30T00:00:00.000Z", + ...overrides, + }; +} + +function observation(lockOwner: McpLifecycleLockOwner | null, mtimeMs = 0): LockObservation { + return { owner: lockOwner, mtimeMs, dev: 10, ino: 20 }; +} + +function probes( + overrides: Partial = {}, +): McpLifecycleLockIdentityProbes { + return { + localHostIdentity: LOCAL_HOST, + localPidNamespaceIdentity: LOCAL_NAMESPACE, + processIsAlive: () => true, + readProcessIdentity: () => null, + ...overrides, + }; +} + +describe("MCP lifecycle lock identity properties", () => { + it("keeps a matching live owner active across PID, start-tick, and clock boundaries", () => { + fc.assert( + fc.property( + boundaryPidArbitrary, + processIdentityArbitrary, + clockArbitrary, + clockArbitrary, + (pid, identity, nowMs, mtimeMs) => { + const result = classifyMcpLifecycleLock( + observation(owner(pid, identity), mtimeMs), + SANDBOX_NAME, + nowMs, + 30_000, + probes({ readProcessIdentity: () => identity }), + ); + + expect(result).toBe("active"); + }, + ), + { numRuns: PROPERTY_RUNS }, + ); + }); + + it("reclaims a dead local owner independently of wall-clock skew", () => { + fc.assert( + fc.property( + boundaryPidArbitrary, + processIdentityArbitrary, + clockArbitrary, + clockArbitrary, + (pid, identity, nowMs, mtimeMs) => { + const result = classifyMcpLifecycleLock( + observation(owner(pid, identity), mtimeMs), + SANDBOX_NAME, + nowMs, + 30_000, + probes({ processIsAlive: () => false }), + ); + + expect(result).toBe("stale"); + }, + ), + { numRuns: PROPERTY_RUNS }, + ); + }); + + it("reclaims PID reuse only after a fresh start-tick mismatch", () => { + fc.assert( + fc.property(boundaryPidArbitrary, bootArbitrary, tickArbitrary, (pid, boot, ticks) => { + const ownerIdentity = `linux:${boot}:${ticks}`; + const replacementIdentity = `linux:${boot}:${BigInt(ticks) + 1n}`; + const readProcessIdentity = vi.fn(() => replacementIdentity); + const result = classifyMcpLifecycleLock( + observation(owner(pid, ownerIdentity)), + SANDBOX_NAME, + 0, + 30_000, + probes({ readProcessIdentity }), + ); + + expect(result).toBe("stale"); + expect(readProcessIdentity).toHaveBeenNthCalledWith(1, pid); + expect(readProcessIdentity).toHaveBeenNthCalledWith(2, pid, true); + }), + { numRuns: PROPERTY_RUNS }, + ); + }); + + it("reclaims a live PID whose boot identity changed", () => { + fc.assert( + fc.property( + boundaryPidArbitrary, + distinctBootPairArbitrary, + tickArbitrary, + (pid, [ownerBoot, currentBoot], ticks) => { + const ownerIdentity = `linux:${ownerBoot}:${ticks}`; + const replacementIdentity = `linux:${currentBoot}:${ticks}`; + const result = classifyMcpLifecycleLock( + observation(owner(pid, ownerIdentity)), + SANDBOX_NAME, + 0, + 30_000, + probes({ readProcessIdentity: () => replacementIdentity }), + ); + + expect(result).toBe("stale"); + }, + ), + { numRuns: PROPERTY_RUNS }, + ); + }); + + it("keeps ownership when a cached start mismatch disappears on refresh", () => { + fc.assert( + fc.property(boundaryPidArbitrary, processIdentityArbitrary, (pid, identity) => { + const readProcessIdentity = vi + .fn() + .mockReturnValueOnce(`${identity}:cached-other-process`) + .mockReturnValueOnce(identity); + const result = classifyMcpLifecycleLock( + observation(owner(pid, identity)), + SANDBOX_NAME, + 0, + 30_000, + probes({ readProcessIdentity }), + ); + + expect(result).toBe("active"); + expect(readProcessIdentity).toHaveBeenNthCalledWith(2, pid, true); + }), + { numRuns: PROPERTY_RUNS }, + ); + }); + + it("never probes or reaps an owner from a different host", () => { + fc.assert( + fc.property( + boundaryPidArbitrary, + nonEmptyStringArbitrary, + processIdentityArbitrary, + (pid, localHost, identity) => { + const probe = probes({ + localHostIdentity: localHost, + processIsAlive: () => { + throw new Error("foreign host reached local PID probe"); + }, + readProcessIdentity: () => { + throw new Error("foreign host reached local process-identity probe"); + }, + }); + const result = classifyMcpLifecycleLock( + observation(owner(pid, identity, { hostIdentity: `${localHost}:foreign` })), + SANDBOX_NAME, + 0, + 30_000, + probe, + ); + + expect(result).toBe("active"); + }, + ), + { numRuns: PROPERTY_RUNS }, + ); + }); + + it("never probes or reaps an owner from a different PID namespace", () => { + fc.assert( + fc.property( + boundaryPidArbitrary, + nonEmptyStringArbitrary, + processIdentityArbitrary, + (pid, localNamespace, identity) => { + const probe = probes({ + localPidNamespaceIdentity: localNamespace, + processIsAlive: () => { + throw new Error("foreign namespace reached local PID probe"); + }, + readProcessIdentity: () => { + throw new Error("foreign namespace reached local process-identity probe"); + }, + }); + const result = classifyMcpLifecycleLock( + observation( + owner(pid, identity, { pidNamespaceIdentity: `${localNamespace}:foreign` }), + ), + SANDBOX_NAME, + 0, + 30_000, + probe, + ); + + expect(result).toBe("active"); + }, + ), + { numRuns: PROPERTY_RUNS }, + ); + }); + + it("never probes or reaps an owner with missing namespace provenance", () => { + fc.assert( + fc.property( + boundaryPidArbitrary, + processIdentityArbitrary, + fc.constantFrom(null, undefined), + (pid, identity, ownerNamespace) => { + const probe = probes({ + processIsAlive: () => { + throw new Error("unknown namespace reached local PID probe"); + }, + readProcessIdentity: () => { + throw new Error("unknown namespace reached local process-identity probe"); + }, + }); + const result = classifyMcpLifecycleLock( + observation(owner(pid, identity, { pidNamespaceIdentity: ownerNamespace })), + SANDBOX_NAME, + 0, + 30_000, + probe, + ); + + expect(result).toBe("active"); + }, + ), + { numRuns: PROPERTY_RUNS }, + ); + }); + + it("rejects every positive PID beyond the safe-integer wire boundary", () => { + fc.assert( + fc.property( + fc.bigInt({ + min: BigInt(Number.MAX_SAFE_INTEGER) + 1n, + max: (1n << 128n) - 1n, + }), + (unsafePid) => { + expect(isMcpLifecycleLockOwner(owner(Number(unsafePid), "process"))).toBe(false); + }, + ), + { numRuns: PROPERTY_RUNS }, + ); + }); +}); + +describe("MCP lifecycle lock storage properties", () => { + let stateDir: string; + + beforeEach(() => { + stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-lock-property-")); + }); + + afterEach(() => { + fs.rmSync(stateDir, { force: true, recursive: true }); + }); + + it("round-trips valid owner records without changing their wire shape", { + timeout: PROPERTY_IO_TIMEOUT_MS, + }, async () => { + await fc.assert( + fc.asyncProperty( + nonEmptyStringArbitrary, + boundaryPidArbitrary, + fc.option(processIdentityArbitrary, { nil: null }), + fc.option(nonEmptyStringArbitrary, { nil: null }), + fc.option(nonEmptyStringArbitrary, { nil: null }), + nonEmptyStringArbitrary, + async (sandboxName, pid, processIdentity, hostIdentity, pidNamespaceIdentity, token) => { + const lockOwner: McpLifecycleLockOwner = { + version: 1, + sandboxName, + pid, + processIdentity, + hostIdentity, + pidNamespaceIdentity, + token, + acquiredAt: "9999-12-31T23:59:59.999Z", + }; + const lockPath = getMcpLifecycleLockPath(sandboxName, stateDir); + fs.mkdirSync(path.dirname(lockPath), { recursive: true }); + fs.writeFileSync(lockPath, `${JSON.stringify(lockOwner)}\n`); + + const observed = await readMcpLifecycleLockObservation(lockPath); + + expect(observed?.owner).toEqual(lockOwner); + }, + ), + { numRuns: PROPERTY_RUNS }, + ); + }); + + it("classifies arbitrary non-JSON lock content as corrupt ownership", { + timeout: PROPERTY_IO_TIMEOUT_MS, + }, async () => { + await fc.assert( + fc.asyncProperty(fc.string({ maxLength: 1_024 }), async (content) => { + const lockPath = getMcpLifecycleLockPath(SANDBOX_NAME, stateDir); + fs.mkdirSync(path.dirname(lockPath), { recursive: true }); + fs.writeFileSync(lockPath, `not-json:${content}`); + + const observed = await readMcpLifecycleLockObservation(lockPath); + + expect(observed?.owner).toBeNull(); + expect(observed?.ino).toBeGreaterThan(0); + }), + { numRuns: PROPERTY_RUNS }, + ); + }); + + it("returns no observation for arbitrary missing lock paths", { + timeout: PROPERTY_IO_TIMEOUT_MS, + }, async () => { + await fc.assert( + fc.asyncProperty(fc.string({ maxLength: 1_024 }), async (sandboxName) => { + const lockPath = getMcpLifecycleLockPath(sandboxName, stateDir); + + await expect(readMcpLifecycleLockObservation(lockPath)).resolves.toBeNull(); + }), + { numRuns: PROPERTY_RUNS }, + ); + }); +}); diff --git a/src/lib/state/mcp-lifecycle-lock-identity.ts b/src/lib/state/mcp-lifecycle-lock-identity.ts new file mode 100644 index 00000000000..b8e864ea4aa --- /dev/null +++ b/src/lib/state/mcp-lifecycle-lock-identity.ts @@ -0,0 +1,236 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import { performance } from "node:perf_hooks"; + +import { isErrnoException } from "../core/errno"; +import { buildSubprocessEnv } from "../subprocess-env"; + +const LOCK_SCHEMA_VERSION = 1; +const OWNER_IDENTITY_CACHE_MS = 1_000; + +export interface McpLifecycleLockOwner { + version: typeof LOCK_SCHEMA_VERSION; + sandboxName: string; + pid: number; + processIdentity: string | null; + /** Stable machine identity. A foreign owner is never reaped by local PID checks. */ + hostIdentity?: string | null; + /** Linux PID namespace identity. Cross-namespace owners fail closed. */ + pidNamespaceIdentity?: string | null; + token: string; + acquiredAt: string; +} + +export interface LockObservation { + owner: McpLifecycleLockOwner | null; + mtimeMs: number; + dev: number; + ino: number; +} + +export type McpLifecycleLockDisposition = "active" | "stale" | "wait"; + +/** Injectable OS evidence keeps ownership classification deterministic under test. */ +export interface McpLifecycleLockIdentityProbes { + localHostIdentity: string; + localPidNamespaceIdentity: string | null; + processIsAlive(pid: number): boolean; + readProcessIdentity(pid: number, fresh?: boolean): string | null; +} + +const processIdentityCache = new Map(); + +export function isMcpLifecycleLockOwner(value: unknown): value is McpLifecycleLockOwner { + if (!value || typeof value !== "object") return false; + const candidate = value as Record; + return ( + candidate.version === LOCK_SCHEMA_VERSION && + typeof candidate.sandboxName === "string" && + Number.isSafeInteger(candidate.pid) && + (candidate.pid as number) > 0 && + (candidate.processIdentity === null || typeof candidate.processIdentity === "string") && + (candidate.hostIdentity === undefined || + candidate.hostIdentity === null || + typeof candidate.hostIdentity === "string") && + (candidate.pidNamespaceIdentity === undefined || + candidate.pidNamespaceIdentity === null || + typeof candidate.pidNamespaceIdentity === "string") && + typeof candidate.token === "string" && + candidate.token.length > 0 && + typeof candidate.acquiredAt === "string" + ); +} + +function processIsAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (error) { + return isErrnoException(error) && error.code === "EPERM"; + } +} + +/** + * Returns an OS process-start identity rather than only a PID. A stale lock + * whose PID has been recycled must not be mistaken for its now-unrelated live + * process. Linux exposes the kernel boot id plus /proc start ticks; macOS and + * other supported POSIX hosts fall back to ps(1)'s process start timestamp. + */ +export function readMcpLockProcessIdentity(pid: number, fresh = false): string | null { + const cached = processIdentityCache.get(pid); + const now = performance.now(); + if ( + !fresh && + cached && + now >= cached.checkedAt && + now - cached.checkedAt < OWNER_IDENTITY_CACHE_MS + ) { + return cached.identity; + } + + let identity: string | null = null; + if (process.platform === "linux") { + try { + const statText = fs.readFileSync(`/proc/${pid}/stat`, "utf8"); + const closeParen = statText.lastIndexOf(")"); + if (closeParen >= 0) { + const fieldsAfterComm = statText + .slice(closeParen + 2) + .trim() + .split(/\s+/); + // The first value after comm is field 3; index 19 is field 22, + // process start time in clock ticks since boot. + const startTicks = fieldsAfterComm[19]; + if (startTicks && /^\d+$/.test(startTicks)) { + let bootIdentity = "unknown-boot"; + try { + bootIdentity = fs.readFileSync("/proc/sys/kernel/random/boot_id", "utf8").trim(); + } catch { + const bootTime = fs + .readFileSync("/proc/stat", "utf8") + .split("\n") + .find((line) => line.startsWith("btime ")); + if (bootTime) bootIdentity = bootTime.trim(); + } + identity = `linux:${bootIdentity}:${startTicks}`; + } + } + } catch { + identity = null; + } + } else { + const result = spawnSync("ps", ["-o", "lstart=", "-p", String(pid)], { + encoding: "utf8", + env: buildSubprocessEnv(), + stdio: ["ignore", "pipe", "ignore"], + timeout: 1_000, + }); + const startedAt = result.status === 0 ? result.stdout.trim() : ""; + if (startedAt) identity = `${process.platform}:${startedAt}`; + } + + processIdentityCache.set(pid, { checkedAt: now, identity }); + return identity; +} + +/** Stable enough to distinguish independent hosts sharing a state directory. */ +export function readMcpLockHostIdentity(): string { + if (process.platform === "linux") { + for (const candidate of ["/etc/machine-id", "/var/lib/dbus/machine-id"]) { + try { + const machineId = fs.readFileSync(candidate, "utf8").trim(); + if (machineId) return `linux:${machineId}`; + } catch { + // Fall through to the hostname identity. + } + } + } + return `${process.platform}:${os.hostname() || "unknown-host"}`; +} + +/** A shared state directory does not make local PID checks safe across namespaces. */ +export function readMcpLockPidNamespaceIdentity(): string | null { + if (process.platform !== "linux") return null; + try { + return fs.readlinkSync("/proc/self/ns/pid"); + } catch { + return null; + } +} + +const LOCAL_HOST_IDENTITY = readMcpLockHostIdentity(); +const LOCAL_PID_NAMESPACE_IDENTITY = readMcpLockPidNamespaceIdentity(); + +const LOCAL_IDENTITY_PROBES: McpLifecycleLockIdentityProbes = { + localHostIdentity: LOCAL_HOST_IDENTITY, + localPidNamespaceIdentity: LOCAL_PID_NAMESPACE_IDENTITY, + processIsAlive, + readProcessIdentity: readMcpLockProcessIdentity, +}; + +export function createMcpLifecycleLockOwner( + sandboxName: string, + token: string, +): McpLifecycleLockOwner { + return { + version: LOCK_SCHEMA_VERSION, + sandboxName, + pid: process.pid, + processIdentity: readMcpLockProcessIdentity(process.pid), + hostIdentity: LOCAL_HOST_IDENTITY, + pidNamespaceIdentity: LOCAL_PID_NAMESPACE_IDENTITY, + token, + acquiredAt: new Date().toISOString(), + }; +} + +/** Exported for deterministic stale-owner/PID-recycle tests. */ +export function classifyMcpLifecycleLock( + observation: LockObservation, + sandboxName: string, + nowMs: number, + corruptLockGraceMs: number, + probes: McpLifecycleLockIdentityProbes = LOCAL_IDENTITY_PROBES, +): McpLifecycleLockDisposition { + const { owner } = observation; + if (!owner || owner.sandboxName !== sandboxName) { + return nowMs - observation.mtimeMs >= corruptLockGraceMs ? "stale" : "wait"; + } + // The lock coordinates local CLI processes, not independent hosts or PID + // namespaces. Never use this process's PID table to reap a foreign owner; + // wait for operator/distributed-lease resolution instead of risking overlap. + // Legacy or incomplete records have unknown provenance. Treat them as + // foreign instead of using this host's PID table to reap them. + if (!owner.hostIdentity || owner.hostIdentity !== probes.localHostIdentity) return "active"; + if ( + (probes.localPidNamespaceIdentity !== null && !owner.pidNamespaceIdentity) || + (owner.pidNamespaceIdentity !== null && + owner.pidNamespaceIdentity !== undefined && + owner.pidNamespaceIdentity !== probes.localPidNamespaceIdentity) + ) { + return "active"; + } + if (!probes.processIsAlive(owner.pid)) return "stale"; + + const observedIdentity = probes.readProcessIdentity(owner.pid); + if ( + owner.processIdentity !== null && + observedIdentity !== null && + owner.processIdentity !== observedIdentity + ) { + // PID identities are cached briefly. Confirm a mismatch without the cache + // before reaping so rapid PID reuse cannot evict a newly live owner. + const refreshedIdentity = probes.readProcessIdentity(owner.pid, true); + if (refreshedIdentity !== null && owner.processIdentity !== refreshedIdentity) { + return "stale"; + } + } + // If this OS cannot recover process-start identity, a live PID is treated as + // active. Failing closed may require waiting for that process to exit, but it + // never breaks mutual exclusion for a legitimate long rebuild/destroy. + return "active"; +} diff --git a/src/lib/state/mcp-lifecycle-lock-storage.ts b/src/lib/state/mcp-lifecycle-lock-storage.ts new file mode 100644 index 00000000000..ad5d7c0b5c7 --- /dev/null +++ b/src/lib/state/mcp-lifecycle-lock-storage.ts @@ -0,0 +1,182 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import crypto from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; + +import { isErrnoException } from "../core/errno"; +import { + isMcpLifecycleLockOwner, + type LockObservation, + type McpLifecycleLockOwner, +} from "./mcp-lifecycle-lock-identity"; +import { resolveNemoclawStateDir } from "./paths"; + +export const MCP_LIFECYCLE_LOCK_DIRNAME = "mcp-lifecycle-locks"; + +function lockFileStem(sandboxName: string): string { + // Hashing makes the filesystem key traversal-safe even if a caller reaches + // the lock before the command's normal sandbox-name validation. + return crypto.createHash("sha256").update(sandboxName).digest("hex"); +} + +export function getMcpLifecycleLockPath( + sandboxName: string, + stateDir = resolveNemoclawStateDir(), +): string { + return path.join(stateDir, MCP_LIFECYCLE_LOCK_DIRNAME, `${lockFileStem(sandboxName)}.lock`); +} + +function ownerFileContent(owner: McpLifecycleLockOwner): string { + return `${JSON.stringify(owner)}\n`; +} + +export async function readMcpLifecycleLockObservation( + lockPath: string, +): Promise { + let handle: fs.promises.FileHandle; + try { + handle = await fs.promises.open( + lockPath, + fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW | fs.constants.O_NONBLOCK, + ); + } catch (error) { + if (isErrnoException(error) && error.code === "ENOENT") return null; + try { + const stat = await fs.promises.lstat(lockPath); + if (!stat.isFile() || stat.isSymbolicLink()) { + return { owner: null, mtimeMs: stat.mtimeMs, dev: stat.dev, ino: stat.ino }; + } + } catch (statError) { + if (isErrnoException(statError) && statError.code === "ENOENT") return null; + throw statError; + } + throw error; + } + + try { + const stat = await handle.stat(); + if (!stat.isFile()) { + return { owner: null, mtimeMs: stat.mtimeMs, dev: stat.dev, ino: stat.ino }; + } + try { + const parsed: unknown = JSON.parse(await handle.readFile("utf8")); + return { + owner: isMcpLifecycleLockOwner(parsed) ? parsed : null, + mtimeMs: stat.mtimeMs, + dev: stat.dev, + ino: stat.ino, + }; + } catch { + return { owner: null, mtimeMs: stat.mtimeMs, dev: stat.dev, ino: stat.ino }; + } + } finally { + await handle.close(); + } +} + +export async function mcpLifecycleLockPathExists(targetPath: string): Promise { + try { + await fs.promises.lstat(targetPath); + return true; + } catch (error) { + if (isErrnoException(error) && error.code === "ENOENT") return false; + throw error; + } +} + +export async function safelyReleaseMcpLifecycleLock( + lockPath: string, + token: string, +): Promise { + const observation = await readMcpLifecycleLockObservation(lockPath); + if (!observation || observation.owner?.token !== token) return; + // Claim and verify the generation before deletion. A replacement appearing + // after the token read is restored rather than unlinked. + await reclaimStaleMcpLifecycleLockGeneration(lockPath, observation); +} + +export async function reclaimStaleMcpLifecycleLockGeneration( + targetPath: string, + expected: LockObservation, +): Promise { + const quarantinePath = `${targetPath}.reclaim-${process.pid}-${crypto.randomUUID()}`; + try { + // Rename is the atomic claim. Another waiter may have already removed the + // stale generation and published a replacement after our earlier read, so + // the moved file must be verified before it is ever deleted. + await fs.promises.rename(targetPath, quarantinePath); + } catch (error) { + if (isErrnoException(error) && error.code === "ENOENT") return false; + throw error; + } + + const claimed = await readMcpLifecycleLockObservation(quarantinePath); + const expectedToken = expected.owner?.token ?? null; + const claimedExpectedGeneration = + expectedToken === null + ? claimed !== null && + claimed.owner === null && + claimed.dev === expected.dev && + claimed.ino === expected.ino + : claimed?.owner?.token === expectedToken; + if (claimedExpectedGeneration) { + await fs.promises.rm(quarantinePath, { force: true, recursive: true }); + return true; + } + + // We raced a replacement owner. Restore the exact moved inode with a hard + // link (which cannot overwrite a newer generation), then drop only our + // quarantine name. If another generation already occupies the canonical + // path, preserve the displaced owner record for diagnosis rather than ever + // deleting an owner we did not claim. + try { + await fs.promises.link(quarantinePath, targetPath); + await fs.promises.rm(quarantinePath, { force: true }); + } catch (error) { + if (!isErrnoException(error) || error.code !== "EEXIST") throw error; + } + return false; +} + +export async function writeMcpLifecycleLockCandidateAndLink( + lockPath: string, + owner: McpLifecycleLockOwner, +): Promise { + const candidatePath = `${lockPath}.candidate-${process.pid}-${owner.token}`; + try { + const handle = await fs.promises.open(candidatePath, "wx", 0o600); + try { + await handle.writeFile(ownerFileContent(owner), "utf8"); + await handle.sync(); + } finally { + await handle.close(); + } + try { + // The hard link is the atomic publication point: waiters can never see a + // partially written owner record. + await fs.promises.link(candidatePath, lockPath); + return true; + } catch (error) { + // NFS may execute LINK but lose/replay its reply. Reconcile the result + // from the unique candidate's link count plus our unguessable owner token + // before treating EEXIST (or another transport error) as a failed claim. + const candidateStat = await fs.promises.stat(candidatePath); + const published = await readMcpLifecycleLockObservation(lockPath); + if (candidateStat.nlink >= 2 && published?.owner?.token === owner.token) { + return true; + } + if (isErrnoException(error) && error.code === "EEXIST") return false; + throw error; + } + } finally { + try { + await fs.promises.rm(candidatePath, { force: true }); + } catch { + // Publication is decided only by LINK plus owner-token reconciliation. + // A unique candidate cleanup failure must not strand a live canonical + // self-lock before the caller enters its protected operation. + } + } +} diff --git a/src/lib/state/mcp-lifecycle-lock.ts b/src/lib/state/mcp-lifecycle-lock.ts new file mode 100644 index 00000000000..4d6bc2934c6 --- /dev/null +++ b/src/lib/state/mcp-lifecycle-lock.ts @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export { + type McpLifecycleLockOptions, + withMcpLifecycleLock, + withMcpLifecycleLock as withSandboxMutationLock, +} from "./mcp-lifecycle-lock-acquisition"; +export { + classifyMcpLifecycleLock, + type McpLifecycleLockDisposition, + readMcpLockHostIdentity, + readMcpLockPidNamespaceIdentity, + readMcpLockProcessIdentity, +} from "./mcp-lifecycle-lock-identity"; +export { + getMcpLifecycleLockPath, + MCP_LIFECYCLE_LOCK_DIRNAME, +} from "./mcp-lifecycle-lock-storage"; diff --git a/src/lib/state/registry-mcp.ts b/src/lib/state/registry-mcp.ts new file mode 100644 index 00000000000..2fdf7fc2d02 --- /dev/null +++ b/src/lib/state/registry-mcp.ts @@ -0,0 +1,149 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { isBlockedMcpUrlTargetHost, MCP_SERVER_URL_MAX_LENGTH } from "../security/mcp-url-target"; + +export interface McpBridgeEntry { + server: string; + agent: string; + adapter?: string; + url: string; + env: string[]; + providerName?: string; + /** Immutable OpenShell ObjectMeta.id captured after provider creation. */ + providerId?: string; + policyName: string; + addedAt: string; + updatedAt?: string; + /** + * Durable add-transaction marker. `prepared` owns no OpenShell/adapter + * resources yet; `preflighted` proves the derived names were absent before + * side effects began. Exact retry/cleanup additionally requires `providerId` + * once provider creation succeeds. Omitted entries are fully committed + * bridges (including legacy records, which fail closed without providerId). + */ + addState?: "prepared" | "preflighted"; +} + +export interface SandboxMcpState { + bridges: Record; + /** Set after in-sandbox adapter scrub/provider detach and before delete. */ + destroyPreparedAt?: string; + /** + * Set only after OpenShell has confirmed the sandbox was deleted (or was + * already absent) and global MCP provider cleanup is still in progress. + * The bridge entries remain the durable cleanup manifest until every exact + * matching provider has been deleted. + */ + destroyPendingAt?: string; +} + +const MCP_SERVER_RE = /^[A-Za-z][A-Za-z0-9_-]{0,63}$/; +const MCP_ENV_RE = /^[A-Za-z_][A-Za-z0-9_]{0,127}$/; +const MCP_SAFE_NAME_RE = /^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$/; +const MCP_PROVIDER_ID_RE = /^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$/; +const MCP_ADAPTERS = new Set(["mcporter", "hermes-config", "deepagents-config"]); + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +export function serializeSandboxMcpStateForDisk(value: unknown): SandboxMcpState | undefined { + const state = normalizeSandboxMcpState(value); + if (!state) return undefined; + return state; +} + +export function normalizeSandboxMcpState(value: unknown): SandboxMcpState | undefined { + if (!isRecord(value)) return undefined; + const bridgesValue = value.bridges; + if (!isRecord(bridgesValue)) return undefined; + const bridges: Record = {}; + for (const [name, rawEntry] of Object.entries(bridgesValue)) { + const entry = normalizeMcpBridgeEntry(name, rawEntry); + if (entry) bridges[entry.server] = entry; + } + const destroyPendingAt = + typeof value.destroyPendingAt === "string" && value.destroyPendingAt + ? value.destroyPendingAt + : undefined; + const destroyPreparedAt = + typeof value.destroyPreparedAt === "string" && value.destroyPreparedAt + ? value.destroyPreparedAt + : undefined; + if (Object.keys(bridges).length === 0 && !destroyPreparedAt && !destroyPendingAt) { + return undefined; + } + return { + bridges, + ...(destroyPreparedAt ? { destroyPreparedAt } : {}), + ...(destroyPendingAt ? { destroyPendingAt } : {}), + }; +} + +function normalizeMcpUrl(value: string): string | null { + if (value.length > MCP_SERVER_URL_MAX_LENGTH) return null; + let parsed: URL; + try { + parsed = new URL(value); + } catch { + return null; + } + if (parsed.protocol !== "http:" && parsed.protocol !== "https:") return null; + if (!parsed.hostname || parsed.username || parsed.password) return null; + if (isBlockedMcpUrlTargetHost(parsed.hostname)) return null; + if (parsed.hash) parsed.hash = ""; + if (!parsed.pathname) parsed.pathname = "/"; + const normalized = parsed.toString(); + return normalized.length <= MCP_SERVER_URL_MAX_LENGTH ? normalized : null; +} + +function normalizeMcpBridgeEntry(server: string, value: unknown): McpBridgeEntry | null { + if (!isRecord(value)) return null; + const serverName = typeof value.server === "string" && value.server ? value.server : server; + if (!MCP_SERVER_RE.test(serverName)) return null; + const url = typeof value.url === "string" ? normalizeMcpUrl(value.url) : null; + const policyName = typeof value.policyName === "string" ? value.policyName : ""; + if (!url || !MCP_SAFE_NAME_RE.test(policyName)) return null; + const rawEnv = value.env; + const env = + Array.isArray(rawEnv) && + rawEnv.every((entry): entry is string => typeof entry === "string" && MCP_ENV_RE.test(entry)) + ? [...new Set(rawEnv)] + : null; + if (!env) return null; + const adapter = typeof value.adapter === "string" && value.adapter ? value.adapter : undefined; + if (adapter && !MCP_ADAPTERS.has(adapter)) return null; + const providerName = + typeof value.providerName === "string" && value.providerName ? value.providerName : undefined; + if (providerName && !MCP_SAFE_NAME_RE.test(providerName)) return null; + const providerId = + typeof value.providerId === "string" && value.providerId ? value.providerId : undefined; + if (value.providerId !== undefined && (!providerId || !MCP_PROVIDER_ID_RE.test(providerId))) { + return null; + } + if (providerId && !providerName) return null; + const rawAddState = value.addState; + const addState = + rawAddState === undefined + ? undefined + : rawAddState === "prepared" || rawAddState === "preflighted" + ? rawAddState + : "preflighted"; + return { + server: serverName, + agent: typeof value.agent === "string" && value.agent ? value.agent : "openclaw", + ...(adapter ? { adapter } : {}), + url, + env, + ...(providerName ? { providerName } : {}), + ...(providerId ? { providerId } : {}), + policyName, + addedAt: + typeof value.addedAt === "string" && value.addedAt + ? value.addedAt + : new Date(0).toISOString(), + ...(typeof value.updatedAt === "string" ? { updatedAt: value.updatedAt } : {}), + ...(addState ? { addState } : {}), + }; +} diff --git a/src/lib/state/registry.ts b/src/lib/state/registry.ts index 4289850b93d..c8fe3a9a6fa 100644 --- a/src/lib/state/registry.ts +++ b/src/lib/state/registry.ts @@ -4,8 +4,8 @@ import fs from "node:fs"; import path from "node:path"; import { isErrnoException } from "../core/errno"; -import { inferenceSelectionRegistryFields } from "../inference/selection"; import type { InferenceSelection } from "../inference/selection"; +import { inferenceSelectionRegistryFields } from "../inference/selection"; import { ensureConfigDir, readConfigFile, writeConfigFile } from "./config-io"; import { applyAddExtraProvider, @@ -14,6 +14,11 @@ import { normalizeExtraProviders, readExtraProviders, } from "./extra-providers"; +import { + normalizeSandboxMcpState, + type SandboxMcpState, + serializeSandboxMcpStateForDisk, +} from "./registry-mcp"; import type { SandboxMessagingState } from "./registry-messaging"; export { @@ -30,6 +35,9 @@ import { serializeSandboxMessagingStateForDisk, setChannelDisabled as setRegistryChannelDisabled, } from "./registry-messaging"; +import type { WebSearchProvider } from "../inference/web-search"; + +export type { McpBridgeEntry, SandboxMcpState } from "./registry-mcp"; export { getConfiguredMessagingChannelsFromEntry, @@ -42,6 +50,8 @@ export { export interface CustomPolicyEntry { name: string; content: string; + /** Desired content reserved before a crash-safe generated-policy transition. */ + pendingContent?: string; sourcePath?: string; appliedAt?: string; } @@ -85,6 +95,9 @@ export interface SandboxEntry extends Partial { // policy step never finished — so re-onboard knows whether `policies` // represents a final selection it can carry forward. See #4621. policyPresetsFinalized?: boolean; + webSearchEnabled?: boolean; + /** Durable provider identity for enabled managed web search. */ + webSearchProvider?: WebSearchProvider | null; agent?: string | null; agentVersion?: string | null; // NemoClaw build fingerprint (the NemoClaw CLI/build version) stamped only on @@ -94,8 +107,11 @@ export interface SandboxEntry extends Partial { // (`--from`) sandboxes are intentionally left without a fingerprint so they // are never auto-rebuilt onto the default image (#5026). nemoclawVersion?: string | null; + fromDockerfile?: string | null; + hermesAuthMethod?: "oauth" | "api_key" | null; imageTag?: string | null; messaging?: SandboxMessagingState; + mcp?: SandboxMcpState; hermesToolGateways?: string[]; hermesDashboardEnabled?: boolean; hermesDashboardPort?: number | null; @@ -122,7 +138,6 @@ export const LOCK_OWNER = path.join(LOCK_DIR, "owner"); export const LOCK_STALE_MS = 10_000; export const LOCK_RETRY_MS = 100; export const LOCK_MAX_RETRIES = 120; - /** kill(pid, 0) liveness probe. EPERM means the pid exists but is owned by * another user, which still counts as alive. */ function isProcessAlive(pid: number): boolean { @@ -377,11 +392,13 @@ function isRecord(value: unknown): value is Record { function normalizeSandboxEntryForRuntime(entry: SandboxEntry): SandboxEntry { const messaging = cloneSandboxMessagingState(entry.messaging); - if (!messaging) { - const { messaging: _messaging, ...rest } = entry; - return rest; - } - return { ...entry, messaging }; + const mcp = normalizeSandboxMcpState(entry.mcp); + const { messaging: _messaging, mcp: _mcp, ...rest } = entry; + return { + ...rest, + ...(messaging ? { messaging } : {}), + ...(mcp ? { mcp } : {}), + }; } /** @@ -403,11 +420,13 @@ function serializeSandboxEntryForDisk(entry: SandboxEntry): SandboxEntry { livePhase?: string | null; }; const messaging = serializeSandboxMessagingStateForDisk(durable.messaging); - if (!messaging) { - const { messaging: _messaging, ...rest } = durable; - return rest; - } - return { ...durable, messaging }; + const mcp = serializeSandboxMcpStateForDisk(durable.mcp); + const { messaging: _messaging, mcp: _mcp, ...rest } = durable; + return { + ...rest, + ...(messaging ? { messaging } : {}), + ...(mcp ? { mcp } : {}), + }; } export function getSandbox(name: string): SandboxEntry | null { @@ -441,6 +460,13 @@ export function registerSandbox(entry: SandboxEntry): void { openshellVersion: entry.openshellVersion || null, policies: entry.policies || [], policyTier: entry.policyTier || null, + webSearchEnabled: + typeof entry.webSearchEnabled === "boolean" ? entry.webSearchEnabled : undefined, + webSearchProvider: + entry.webSearchEnabled === true && + (entry.webSearchProvider === "brave" || entry.webSearchProvider === "tavily") + ? entry.webSearchProvider + : null, // policyPresetsFinalized is intentionally not set here: registration means // the policy step has not completed for this entry. It is stamped only by // the post-policy registry write (see policy-preset-persistence), so a @@ -449,8 +475,14 @@ export function registerSandbox(entry: SandboxEntry): void { agent: entry.agent || null, agentVersion: entry.agentVersion || null, nemoclawVersion: entry.nemoclawVersion || null, + fromDockerfile: entry.fromDockerfile || null, + hermesAuthMethod: + entry.hermesAuthMethod === "oauth" || entry.hermesAuthMethod === "api_key" + ? entry.hermesAuthMethod + : null, imageTag: entry.imageTag || null, messaging: cloneSandboxMessagingState(entry.messaging), + mcp: normalizeSandboxMcpState(entry.mcp), hermesToolGateways: Array.isArray(entry.hermesToolGateways) && entry.hermesToolGateways.length > 0 ? [...entry.hermesToolGateways] diff --git a/src/lib/state/sandbox.ts b/src/lib/state/sandbox.ts index 38b10f91fcd..6ba8054025d 100644 --- a/src/lib/state/sandbox.ts +++ b/src/lib/state/sandbox.ts @@ -178,7 +178,9 @@ function isCustomPolicyEntryArray(value: unknown): value is CustomPolicyEntry[] typeof entry === "object" && entry !== null && typeof (entry as { name?: unknown }).name === "string" && - typeof (entry as { content?: unknown }).content === "string", + typeof (entry as { content?: unknown }).content === "string" && + ((entry as { pendingContent?: unknown }).pendingContent === undefined || + typeof (entry as { pendingContent?: unknown }).pendingContent === "string"), ) ); } diff --git a/src/lib/subprocess-env.ts b/src/lib/subprocess-env.ts index 9387aa5e616..54067365810 100644 --- a/src/lib/subprocess-env.ts +++ b/src/lib/subprocess-env.ts @@ -40,14 +40,34 @@ const TLS = [ const TOOLCHAIN = ["DOCKER_HOST", "KUBECONFIG", "SSH_AUTH_SOCK", "RUST_LOG", "RUST_BACKTRACE"]; -const ALLOWED_ENV_NAMES = new Set([...SYSTEM, ...TEMP, ...LOCALE, ...PROXY, ...TLS, ...TOOLCHAIN]); +export const SUBPROCESS_ENV_ALLOWED_NAMES: readonly string[] = Object.freeze([ + ...SYSTEM, + ...TEMP, + ...LOCALE, + ...PROXY, + ...TLS, + ...TOOLCHAIN, +]); +const ALLOWED_ENV_NAMES = new Set(SUBPROCESS_ENV_ALLOWED_NAMES); // ── Allowed prefixes ─────────────────────────────────────────── -const ALLOWED_ENV_PREFIXES = ["LC_", "XDG_", "OPENSHELL_", "GRPC_"]; +export const SUBPROCESS_ENV_ALLOWED_PREFIXES: readonly string[] = Object.freeze([ + "LC_", + "XDG_", + "OPENSHELL_", + "GRPC_", +]); // ── Public API ───────────────────────────────────────────────── +export function isSubprocessEnvNameAllowed(name: string): boolean { + return ( + ALLOWED_ENV_NAMES.has(name) || + SUBPROCESS_ENV_ALLOWED_PREFIXES.some((prefix) => name.startsWith(prefix)) + ); +} + /** * When any HTTP proxy is forwarded, augment NO_PROXY so the host proxy is * never asked to forward traffic destined for the host loopback, the @@ -102,7 +122,7 @@ export function buildSubprocessEnv(extra?: Record): Record = {}; for (const [key, value] of Object.entries(process.env)) { if (value === undefined) continue; - if (ALLOWED_ENV_NAMES.has(key) || ALLOWED_ENV_PREFIXES.some((p) => key.startsWith(p))) { + if (isSubprocessEnvNameAllowed(key)) { env[key] = value; } } diff --git a/test/brev-launchable-ci-cpu-checksum.test.ts b/test/brev-launchable-ci-cpu-checksum.test.ts index c4d13cbc543..6f0c3d5db94 100644 --- a/test/brev-launchable-ci-cpu-checksum.test.ts +++ b/test/brev-launchable-ci-cpu-checksum.test.ts @@ -9,6 +9,7 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; const SCRIPT = path.join(import.meta.dirname, "..", "scripts", "brev-launchable-ci-cpu.sh"); +const BREV_LIFECYCLE_SCRIPT_MAX_BYTES = 16 * 1024; const ASSET = "openshell-x86_64-unknown-linux-musl.tar.gz"; const PINNED_ASSET_SHA256 = "37836c3b50383e03249c5e16512c1806e591fba8451408a84fb2f628ddb318c4"; @@ -253,6 +254,10 @@ function combinedLaunchableOutput(result: ReturnType, launchLo } describe("brev-launchable-ci-cpu.sh OpenShell checksum gate", { timeout: 30_000 }, () => { + it("fits within Brev's lifecycle setup-script limit", () => { + expect(fs.statSync(SCRIPT).size).toBeLessThanOrEqual(BREV_LIFECYCLE_SCRIPT_MAX_BYTES); + }); + it("rejects malformed OPENSHELL_VERSION before downloads or privileged setup", () => { const { fake, result } = runLaunchable({ checksum: "match", diff --git a/test/brev-nightly-workflow.test.ts b/test/brev-nightly-workflow.test.ts index 9afda37d5db..7cc06931786 100644 --- a/test/brev-nightly-workflow.test.ts +++ b/test/brev-nightly-workflow.test.ts @@ -3,15 +3,36 @@ import { describe, expect, it } from "vitest"; +import { BREV_WORKFLOW_OWNERSHIP_ENV } from "../tools/e2e/brev-remote-vitest.mts"; import { readYaml } from "./helpers/e2e-workflow-contract"; type ReusableCallerJob = { + env?: Record; + if?: string; + outputs?: Record; + permissions?: Record; + "timeout-minutes"?: number; + steps?: Array<{ + env?: Record; + if?: string; + name?: string; + run?: string; + uses?: string; + with?: Record; + }>; uses?: string; with?: Record; secrets?: Record; + strategy?: { + matrix?: { + test_suite?: string[]; + }; + }; }; type Workflow = { + concurrency?: { group?: string }; + permissions?: Record; on?: { workflow_call?: { inputs?: Record; @@ -47,14 +68,161 @@ describe("Brev nightly workflow contract", () => { } }); + it("grants the reusable workflow permission ceiling so GitHub can start the run", () => { + expect(nightly.permissions).toEqual(branchValidation.permissions); + expect(nightly.permissions).toEqual({ + contents: "read", + checks: "write", + "pull-requests": "write", + }); + }); + + it("keeps write permissions out of the secret-bearing target-branch job", () => { + const caller = nightly.jobs?.["brev-nightly-e2e"]; + const validation = branchValidation.jobs?.["e2e-branch-validation"]; + const reporter = branchValidation.jobs?.["report-pr"]; + const checkout = validation?.steps?.find((step) => step.name === "Checkout target branch"); + const resolveBranch = validation?.steps?.find( + (step) => step.name === "Resolve branch from PR number", + ); + const recordRevision = validation?.steps?.find( + (step) => step.name === "Record exact tested revision", + ); + + expect(nightly.on?.workflow_dispatch?.inputs).not.toHaveProperty("branch"); + expect(caller?.with?.branch).toBe("${{ github.ref_name }}"); + expect(validation?.permissions).toEqual({ + contents: "read", + "pull-requests": "read", + }); + expect(checkout?.with?.["persist-credentials"]).toBe(false); + expect(resolveBranch?.env?.PR_NUMBER).toBe("${{ inputs.pr_number }}"); + expect(resolveBranch?.run).not.toContain("gh pr view ${{"); + expect(validation?.outputs?.tested_sha).toBe("${{ steps.tested-ref.outputs.sha }}"); + expect(recordRevision?.run).toContain("git rev-parse HEAD"); + expect(validation?.env?.BREV_E2E_INSTANCE_NAME).toContain("inputs.test_suite"); + expect(reporter?.permissions).toEqual({ + contents: "read", + checks: "write", + "pull-requests": "write", + }); + expect(reporter?.if).toContain("inputs.pr_number != ''"); + expect(reporter?.steps?.[0]?.env?.TESTED_SHA).toBe( + "${{ needs.e2e-branch-validation.outputs.tested_sha }}", + ); + expect(reporter?.steps?.[0]?.env?.INSTANCE_NAME).toContain("inputs.test_suite"); + expect(reporter?.steps?.[0]?.run).toContain( + "PR head moved after Brev validation; refusing to report stale evidence", + ); + expect(reporter?.steps?.some((step) => step.uses?.includes("checkout"))).toBe(false); + expect(JSON.stringify(reporter)).not.toMatch(/BREV_|NVIDIA_INFERENCE_API_KEY/); + }); + + it("keeps every suite in the nightly matrix in a distinct concurrency group", () => { + expect(branchValidation.concurrency?.group).toContain("inputs.test_suite"); + }); + + it("fails closed on unsupported reusable test-suite values before checkout", () => { + const steps = branchValidation.jobs?.["e2e-branch-validation"]?.steps ?? []; + const validation = steps.find((step) => step.name === "Validate test suite"); + const checkout = steps.find((step) => step.name === "Checkout target branch"); + + expect(validation?.env?.TEST_SUITE).toBe("${{ inputs.test_suite }}"); + expect(validation?.run).toContain( + "full|credential-sanitization|telegram-injection|messaging-providers|messaging-compatible-endpoint|dashboard-remote-bind|gpu|all", + ); + expect(validation?.run).toContain("exit 1"); + expect(steps.indexOf(validation as NonNullable)).toBeLessThan( + steps.indexOf(checkout as NonNullable), + ); + }); + + it("runs stateful messaging targets on separate fresh instances", () => { + expect(nightly.jobs?.["brev-nightly-e2e"]?.strategy?.matrix?.test_suite).toEqual([ + "all", + "messaging-providers", + "messaging-compatible-endpoint", + "full", + ]); + expect(branchValidation.jobs?.["e2e-branch-validation"]?.["timeout-minutes"]).toBe(130); + }); + + it("keeps failure diagnostics ahead of workflow-owned instance deletion", () => { + const steps = branchValidation.jobs?.["e2e-branch-validation"]?.steps ?? []; + const run = steps.find((step) => step.name === "Run ephemeral Brev E2E"); + const collect = steps.find((step) => step.name === "Collect Brev debug bundle on failure"); + const uploadDebug = steps.find((step) => step.name === "Upload Brev debug bundle on failure"); + const uploadLogs = steps.find((step) => step.name === "Upload test logs"); + const cleanup = steps.find((step) => step.name === "Delete Brev instance"); + + expect(branchValidation.on?.workflow_call?.inputs?.keep_alive).toMatchObject({ + default: false, + }); + expect(run?.env?.[BREV_WORKFLOW_OWNERSHIP_ENV]).toBe("1"); + expect(cleanup?.if).toBe("always() && !inputs.keep_alive"); + expect(cleanup?.env?.INSTANCE).toBe("${{ env.BREV_E2E_INSTANCE_NAME }}"); + expect(uploadDebug?.with?.name).toBe( + "brev-debug-bundle-${{ inputs.test_suite }}-${{ github.run_attempt }}", + ); + expect(uploadLogs?.with?.name).toBe( + "e2e-branch-validation-logs-${{ inputs.test_suite }}-${{ github.run_attempt }}", + ); + expect(cleanup?.run).toContain("for attempt in 1 2 3"); + expect(cleanup?.run).toContain('timeout 30s brev delete "$INSTANCE"'); + expect(cleanup?.run).toContain("timeout 30s brev ls --json"); + expect(cleanup?.run).toContain("timeout 30s brev refresh"); + expect(cleanup?.run).not.toMatch(/grep.*not found/); + expect(steps.indexOf(cleanup as NonNullable)).toBeGreaterThan( + steps.indexOf(collect as NonNullable), + ); + expect(steps.indexOf(cleanup as NonNullable)).toBeGreaterThan( + steps.indexOf(uploadDebug as NonNullable), + ); + expect(steps.indexOf(cleanup as NonNullable)).toBeGreaterThan( + steps.indexOf(uploadLogs as NonNullable), + ); + }); + + it("keeps manual dispatch inputs out of the Brev credential boundary", () => { + const validation = branchValidation.jobs?.["e2e-branch-validation"]; + const install = validation?.steps?.find((step) => step.name === "Install Brev CLI"); + const run = validation?.steps?.find((step) => step.name === "Run ephemeral Brev E2E"); + + expect(branchValidation.on?.workflow_dispatch?.inputs).not.toHaveProperty("brev_token"); + expect(install?.env?.BREV_API_TOKEN).toBe("${{ secrets.BREV_API_TOKEN }}"); + expect(run?.env?.BREV_API_TOKEN).toBe("${{ secrets.BREV_API_TOKEN }}"); + expect(JSON.stringify(validation)).not.toContain("inputs.brev_token"); + }); + + it("verifies the pinned Brev CLI digest before extracting it", () => { + const validation = branchValidation.jobs?.["e2e-branch-validation"]; + const install = validation?.steps?.find((step) => step.name === "Install Brev CLI"); + const script = install?.run ?? ""; + + expect(install?.env?.BREV_CLI_VERSION).toBe("0.6.324"); + expect(install?.env?.BREV_CLI_SHA256).toBe( + "c7056c17d4810134e3fe7194c233619b1b888a640df1929ea7c6f69c0425e58c", + ); + expect(script).toContain("releases/download/v${BREV_CLI_VERSION}"); + expect(script).toContain("brev-cli_${BREV_CLI_VERSION}_linux_amd64.tar.gz"); + expect(script).toContain("sha256sum -c -"); + expect(script.indexOf("sha256sum -c -")).toBeGreaterThan(script.indexOf("curl -fsSL")); + expect(script.indexOf("tar -xzf")).toBeGreaterThan(script.indexOf("sha256sum -c -")); + }); + it("does not expose stale published-launchable controls", () => { const dispatchInputs = Object.keys(nightly.on?.workflow_dispatch?.inputs ?? {}); + const reusableInputs = Object.keys(branchValidation.on?.workflow_call?.inputs ?? {}); const callerInputs = Object.values(nightly.jobs ?? {}).flatMap((job) => Object.keys(job.with ?? {}), ); + const validation = branchValidation.jobs?.["e2e-branch-validation"]; + const run = validation?.steps?.find((step) => step.name === "Run ephemeral Brev E2E"); expect(dispatchInputs).not.toContain("launchable_id"); + expect(reusableInputs).not.toContain("setup_script_url"); expect(callerInputs).not.toContain("launchable_id"); expect(callerInputs).not.toContain("use_published_launchable"); + expect(run?.env).not.toHaveProperty("LAUNCHABLE_SETUP_SCRIPT"); }); }); diff --git a/test/brev-remote-vitest.test.ts b/test/brev-remote-vitest.test.ts new file mode 100644 index 00000000000..988429d8a69 --- /dev/null +++ b/test/brev-remote-vitest.test.ts @@ -0,0 +1,161 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { + BREV_MESSAGING_COMPAT_TIMEOUT_MS, + BREV_MESSAGING_PROVIDER_TIMEOUT_MS, + BREV_REMOTE_WRAPPER_GRACE_MS, + BREV_SECURITY_SUITE_TIMEOUT_MS, + BREV_WORKFLOW_OWNERSHIP_ENV, + brevSuiteHarnessSandboxName, + brevSuiteNeedsHarnessSandbox, + brevWorkflowOwnsInstance, + buildBrevRemoteVitestCommand, +} from "../tools/e2e/brev-remote-vitest.mts"; + +const TARGET = "test/e2e/live/credential-sanitization.test.ts"; + +type Fixture = { + fakeBin: string; + fixtureVitest: string; + npmLog: string; + root: string; + vitestLog: string; +}; + +function writeExecutable(target: string, source: string): void { + fs.writeFileSync(target, source, { mode: 0o755 }); +} + +function createFixture(): Fixture { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-brev-vitest-")); + const fakeBin = path.join(root, "fake-bin"); + const fixtureVitest = path.join(root, "fixture-vitest"); + const npmLog = path.join(root, "npm.log"); + const vitestLog = path.join(root, "vitest.log"); + fs.mkdirSync(fakeBin, { recursive: true }); + writeExecutable( + fixtureVitest, + [ + "#!/usr/bin/env bash", + "set -euo pipefail", + `printf 'env=%s\\n' "\${NEMOCLAW_RUN_LIVE_E2E:-}" >> "$VITEST_LOG"`, + `printf 'arg=%s\\n' "$@" >> "$VITEST_LOG"`, + "", + ].join("\n"), + ); + writeExecutable( + path.join(fakeBin, "npm"), + [ + "#!/usr/bin/env bash", + "set -euo pipefail", + `printf '%s\\n' "$*" >> "$NPM_LOG"`, + "mkdir -p node_modules/.bin", + `cp "$FIXTURE_VITEST" node_modules/.bin/vitest`, + "chmod +x node_modules/.bin/vitest", + "", + ].join("\n"), + ); + return { fakeBin, fixtureVitest, npmLog, root, vitestLog }; +} + +function runRemoteCommand(fixture: Fixture) { + return spawnSync("bash", ["-c", buildBrevRemoteVitestCommand("e2e-live", TARGET)], { + cwd: fixture.root, + encoding: "utf8", + env: { + ...process.env, + FIXTURE_VITEST: fixture.fixtureVitest, + NPM_LOG: fixture.npmLog, + PATH: `${fixture.fakeBin}:${process.env.PATH ?? ""}`, + VITEST_LOG: fixture.vitestLog, + }, + }); +} + +function expectedVitestLog(): string { + return [ + "env=1", + "arg=run", + "arg=--project", + "arg=e2e-live", + `arg=${TARGET}`, + "arg=--silent=false", + "arg=--reporter=default", + "", + ].join("\n"); +} + +describe("Brev remote Vitest command", () => { + it("leaves each messaging target inside the fresh-instance job budget", () => { + expect(BREV_SECURITY_SUITE_TIMEOUT_MS).toBe(20 * 60_000); + expect(BREV_MESSAGING_PROVIDER_TIMEOUT_MS).toBe(70 * 60_000); + expect(BREV_MESSAGING_COMPAT_TIMEOUT_MS).toBe(40 * 60_000); + expect(BREV_REMOTE_WRAPPER_GRACE_MS).toBe(120_000); + }); + + it("recognizes workflow ownership only from the explicit sentinel", () => { + expect(BREV_WORKFLOW_OWNERSHIP_ENV).toBe("NEMOCLAW_BREV_WORKFLOW_OWNS_INSTANCE"); + expect(brevWorkflowOwnsInstance({ NEMOCLAW_BREV_WORKFLOW_OWNS_INSTANCE: "1" })).toBe(true); + expect(brevWorkflowOwnsInstance({ NEMOCLAW_BREV_WORKFLOW_OWNS_INSTANCE: "0" })).toBe(false); + expect(brevWorkflowOwnsInstance({})).toBe(false); + }); + + it("does not seed shared harness state for suites that own their sandbox lifecycle", () => { + expect(brevSuiteNeedsHarnessSandbox("all")).toBe(false); + expect(brevSuiteNeedsHarnessSandbox("full")).toBe(false); + expect(brevSuiteNeedsHarnessSandbox("gpu")).toBe(false); + expect(brevSuiteNeedsHarnessSandbox("messaging-compatible-endpoint")).toBe(false); + expect(brevSuiteNeedsHarnessSandbox("messaging-providers")).toBe(false); + expect(brevSuiteHarnessSandboxName("all")).toBeUndefined(); + expect(brevSuiteHarnessSandboxName("messaging-compatible-endpoint")).toBeUndefined(); + expect(brevSuiteHarnessSandboxName("messaging-providers")).toBeUndefined(); + }); + + it("preserves harness onboarding for single-target suites", () => { + expect(brevSuiteNeedsHarnessSandbox("credential-sanitization")).toBe(true); + expect(brevSuiteNeedsHarnessSandbox("telegram-injection")).toBe(true); + expect(brevSuiteNeedsHarnessSandbox("dashboard-remote-bind")).toBe(true); + expect(brevSuiteHarnessSandboxName("dashboard-remote-bind")).toBe("e2e-test"); + }); + + it("uses the repository-local Vitest binary without invoking a package runner", () => { + const fixture = createFixture(); + try { + const localVitest = path.join(fixture.root, "node_modules/.bin/vitest"); + fs.mkdirSync(path.dirname(localVitest), { recursive: true }); + fs.copyFileSync(fixture.fixtureVitest, localVitest); + fs.chmodSync(localVitest, 0o755); + + const result = runRemoteCommand(fixture); + + expect(result.status, result.stderr).toBe(0); + expect(fs.existsSync(fixture.npmLog)).toBe(false); + expect(fs.readFileSync(fixture.vitestLog, "utf8")).toBe(expectedVitestLog()); + } finally { + fs.rmSync(fixture.root, { recursive: true, force: true }); + } + }); + + it("restores the lockfile graph when a prior suite prunes Vitest", () => { + const fixture = createFixture(); + try { + const result = runRemoteCommand(fixture); + + expect(result.status, result.stderr).toBe(0); + expect(fs.readFileSync(fixture.npmLog, "utf8")).toBe( + "ci --ignore-scripts --no-audit --no-fund\n", + ); + expect(fs.readFileSync(fixture.vitestLog, "utf8")).toBe(expectedVitestLog()); + } finally { + fs.rmSync(fixture.root, { recursive: true, force: true }); + } + }); +}); diff --git a/test/channels-add-preset.test.ts b/test/channels-add-preset.test.ts index 05e0e57910e..3adf90a785b 100644 --- a/test/channels-add-preset.test.ts +++ b/test/channels-add-preset.test.ts @@ -1603,7 +1603,7 @@ processRecovery.executeSandboxExecCommand = (name, command) => { }; processRecovery.executeSandboxCommand = () => null; -const rebuild = require(${j("actions/sandbox/rebuild.js")}); +const rebuild = require(${j("actions/sandbox/rebuild-pipeline.js")}); let rebuildCount = 0; rebuild.rebuildSandbox = async () => { rebuildCount += 1; }; diff --git a/test/cli/connect-recovery-settle.test.ts b/test/cli/connect-recovery-settle.test.ts index c3125a4edc1..baebe94f1dc 100644 --- a/test/cli/connect-recovery-settle.test.ts +++ b/test/cli/connect-recovery-settle.test.ts @@ -91,11 +91,11 @@ describe("CLI dispatch", () => { `state_file=${JSON.stringify(stateFile)}`, 'printf \'docker %s\\n\' "$*" >> "$marker_file"', 'if [ "$1" = "info" ]; then echo "24.0.0"; exit 0; fi', - 'if [ "$1" = "ps" ] && [ "$2" = "--format" ]; then', - " echo openshell-alpha", + 'if [ "$1" = "ps" ]; then', + " printf 'container-id\\topenshell-alpha\\n'", " exit 0", "fi", - 'if [[ "$*" == *"--env LD_PRELOAD="* ]] && [[ "$*" == *"--env PYTHONPATH="* ]] && [[ "$*" == *"--user root openshell-alpha /usr/local/bin/nemoclaw-gateway-control recover "* ]]; then', + 'if [[ "$*" == *"--env LD_PRELOAD="* ]] && [[ "$*" == *"--env PYTHONPATH="* ]] && [[ "$*" == *"--user root container-id /usr/local/bin/nemoclaw-gateway-control recover "* ]]; then', ' nonce="${!#}"', ' case "$nonce" in *[!0-9a-f]*|"") exit 64 ;; esac', ' [ "${#nonce}" -eq 64 ] || exit 64', @@ -103,7 +103,7 @@ describe("CLI dispatch", () => { " echo 'GATEWAY_PID=123'", " exit 0", "fi", - 'if [[ "$*" == *"--env LD_PRELOAD="* ]] && [[ "$*" == *"--env PYTHONPATH="* ]] && [[ "$*" == *"--user root openshell-alpha /usr/local/bin/nemoclaw-gateway-control probe "* ]]; then', + 'if [[ "$*" == *"--env LD_PRELOAD="* ]] && [[ "$*" == *"--env PYTHONPATH="* ]] && [[ "$*" == *"--user root container-id /usr/local/bin/nemoclaw-gateway-control probe "* ]]; then', ' nonce="${!#}"', ' case "$nonce" in *[!0-9a-f]*|"") exit 64 ;; esac', ' [ "${#nonce}" -eq 64 ] || exit 64', @@ -131,10 +131,10 @@ describe("CLI dispatch", () => { expect(r.out).toContain("config change requires gateway restart (plugins.installs)"); const calls = fs.readFileSync(markerFile, "utf8"); expect(calls).toMatch( - /^docker exec (?:--env [A-Z0-9_]+=[^ ]* )+--user root openshell-alpha \/usr\/local\/bin\/nemoclaw-gateway-control recover [0-9a-f]{64}$/m, + /^docker exec (?:--env [A-Z0-9_]+=[^ ]* )+--user root container-id \/usr\/local\/bin\/nemoclaw-gateway-control recover [0-9a-f]{64}$/m, ); expect(calls).toMatch( - /^docker exec (?:--env [A-Z0-9_]+=[^ ]* )+--user root openshell-alpha \/usr\/local\/bin\/nemoclaw-gateway-control probe [0-9a-f]{64}$/m, + /^docker exec (?:--env [A-Z0-9_]+=[^ ]* )+--user root container-id \/usr\/local\/bin\/nemoclaw-gateway-control probe [0-9a-f]{64}$/m, ); expect(calls).toContain("--env LD_PRELOAD="); expect(calls).toContain("--env PYTHONPATH="); diff --git a/test/cli/connect-recovery.test.ts b/test/cli/connect-recovery.test.ts index 83af5d33041..3b946de8cd2 100644 --- a/test/cli/connect-recovery.test.ts +++ b/test/cli/connect-recovery.test.ts @@ -68,7 +68,10 @@ function writeGatewayControlDockerStub( function expectGatewayControlRecovery(callsFile: string): void { const calls = fs.readFileSync(callsFile, "utf8"); - expect(calls).toContain("ps --format {{.Names}}"); + expect(calls).toContain( + "ps --no-trunc --filter label=openshell.ai/managed-by=openshell " + + "--filter label=openshell.ai/sandbox-name=alpha --format {{.ID}}\t{{.Names}}", + ); const recoveryCall = calls .split("\n") .find((line) => line.includes("/usr/local/bin/nemoclaw-gateway-control recover")); @@ -80,7 +83,7 @@ function expectGatewayControlRecovery(callsFile: string): void { expect(recoveryCall).toContain("--env PYTHONUSERBASE="); expect(recoveryCall).toContain("--env PYTHONNOUSERSITE=1"); expect(recoveryCall).toMatch( - /^exec (?:--env [A-Z0-9_]+=[^ ]* )+--user root openshell-alpha \/usr\/local\/bin\/nemoclaw-gateway-control recover [0-9a-f]{64}$/, + /^exec (?:--env [A-Z0-9_]+=[^ ]* )+--user root container-id \/usr\/local\/bin\/nemoclaw-gateway-control recover [0-9a-f]{64}$/, ); expect(calls).not.toContain("OPENCLAW="); expect(calls).not.toContain("base64 -d | sh"); diff --git a/test/cli/destroy-gateway-unreachable.test.ts b/test/cli/destroy-gateway-unreachable.test.ts index 6636640294c..8e111364093 100644 --- a/test/cli/destroy-gateway-unreachable.test.ts +++ b/test/cli/destroy-gateway-unreachable.test.ts @@ -14,7 +14,7 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; -import { runWithEnv, testTimeoutOptions } from "./helpers"; +import { execTimeout, runWithEnv, testTimeoutOptions } from "./helpers"; // Fake openshell whose `sandbox delete` fails as if the gateway is down; every // other call succeeds so the destroy flow reaches the delete. @@ -63,13 +63,17 @@ function registryHasAlpha(registryPath: string): boolean { } describe("CLI destroy when the gateway is unreachable (#6046)", () => { - it("removes the local sandbox record with --force", testTimeoutOptions(30_000), () => { + it("removes the local sandbox record with --force", testTimeoutOptions(40_000), () => { const { home, registryPath, localBin } = fixture(); try { - const r = runWithEnv("alpha destroy --force", { - HOME: home, - PATH: `${localBin}:${process.env.PATH || ""}`, - }); + const r = runWithEnv( + "alpha destroy --force", + { + HOME: home, + PATH: `${localBin}:${process.env.PATH || ""}`, + }, + execTimeout(30_000), + ); // --force succeeds (exit 0); the gateway-unreachable warning goes to // stderr (not captured on success), so assert the behavioral outcome: @@ -82,13 +86,17 @@ describe("CLI destroy when the gateway is unreachable (#6046)", () => { } }); - it("fails with a recovery hint when --force is absent", testTimeoutOptions(30_000), () => { + it("fails with a recovery hint when --force is absent", testTimeoutOptions(40_000), () => { const { home, registryPath, localBin } = fixture(); try { - const r = runWithEnv("alpha destroy -y", { - HOME: home, - PATH: `${localBin}:${process.env.PATH || ""}`, - }); + const r = runWithEnv( + "alpha destroy -y", + { + HOME: home, + PATH: `${localBin}:${process.env.PATH || ""}`, + }, + execTimeout(30_000), + ); expect(r.code).not.toBe(0); expect(r.out).toContain("The OpenShell gateway is unreachable"); diff --git a/test/cloudflared-update-check-workflow.test.ts b/test/cloudflared-update-check-workflow.test.ts new file mode 100644 index 00000000000..99ee32e3463 --- /dev/null +++ b/test/cloudflared-update-check-workflow.test.ts @@ -0,0 +1,196 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import crypto from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { readYaml, type WorkflowStep } from "./helpers/e2e-workflow-contract"; + +const ROOT = path.resolve(import.meta.dirname, ".."); +const E2E_WORKFLOW = path.join(ROOT, ".github", "workflows", "e2e.yaml"); +const CHECK_SCRIPT = path.join(ROOT, "scripts", "checks", "check-cloudflared-update.sh"); +const FULL_SHA_ACTION = /@[0-9a-f]{40}$/iu; + +type CloudflaredUpdateWorkflow = { + on?: { + schedule?: Array<{ cron?: string }>; + workflow_dispatch?: Record; + }; + permissions?: Record; + jobs?: Record< + string, + { + permissions?: Record; + steps?: WorkflowStep[]; + } + >; +}; + +function pinValues(source: string, name: string): string[] { + return [...source.matchAll(new RegExp(`^\\s*${name}:\\s*"([^"]+)"`, "gmu"))].map( + (match) => match[1], + ); +} + +function writePinFixture(file: string, version: string, sha256: string): void { + fs.writeFileSync( + file, + ["one", "two", "three"] + .map( + (job) => + ` ${job}:\n env:\n CLOUDFLARED_VERSION: "${version}"\n CLOUDFLARED_DEB_SHA256: "${sha256}"`, + ) + .join("\n"), + ); +} + +function runFixtureCheck(options: { pinnedVersion: string; latestVersion: string }) { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cloudflared-update-")); + const workflowPath = path.join(tempDir, "e2e.yaml"); + const releasePath = path.join(tempDir, "release.json"); + const assetPath = path.join(tempDir, "cloudflared-linux-amd64.deb"); + const curlPath = path.join(tempDir, "curl"); + const callLogPath = path.join(tempDir, "curl-calls.txt"); + const asset = Buffer.from("fixture cloudflared linux-amd64 package\n", "utf8"); + const latestSha = crypto.createHash("sha256").update(asset).digest("hex"); + const pinnedSha = options.pinnedVersion === options.latestVersion ? latestSha : "0".repeat(64); + const apiUrl = "https://api.example.invalid/cloudflared/latest"; + const downloadBase = "https://downloads.example.invalid/cloudflared"; + const assetUrl = `${downloadBase}/${options.latestVersion}/cloudflared-linux-amd64.deb`; + + writePinFixture(workflowPath, options.pinnedVersion, pinnedSha); + fs.writeFileSync(assetPath, asset); + fs.writeFileSync( + releasePath, + JSON.stringify({ + tag_name: options.latestVersion, + assets: [{ name: "cloudflared-linux-amd64.deb", browser_download_url: assetUrl }], + }), + ); + fs.writeFileSync( + curlPath, + `#!/usr/bin/env bash +set -euo pipefail +output="" +url="" +while (( $# > 0 )); do + case "$1" in + --output|-o) output="$2"; shift 2 ;; + --header) shift 2 ;; + --retry|--retry-delay) shift 2 ;; + --fail|--silent|--show-error|--location|--retry-all-errors) shift ;; + *) url="$1"; shift ;; + esac +done +printf '%s\n' "$url" >> "$FAKE_CALL_LOG" +case "$url" in + "$FAKE_API_URL") cp "$FAKE_RELEASE_JSON" "$output" ;; + "$FAKE_ASSET_URL") cp "$FAKE_ASSET" "$output" ;; + *) printf 'unexpected URL: %s\\n' "$url" >&2; exit 2 ;; +esac +`, + { mode: 0o755 }, + ); + + const result = spawnSync("bash", [CHECK_SCRIPT], { + cwd: ROOT, + encoding: "utf8", + env: { + ...process.env, + CLOUDFLARED_CURL_BIN: curlPath, + CLOUDFLARED_DOWNLOAD_BASE_URL: downloadBase, + CLOUDFLARED_E2E_WORKFLOW: workflowPath, + CLOUDFLARED_RELEASE_API_URL: apiUrl, + FAKE_API_URL: apiUrl, + FAKE_ASSET: assetPath, + FAKE_ASSET_URL: assetUrl, + FAKE_CALL_LOG: callLogPath, + FAKE_RELEASE_JSON: releasePath, + RUNNER_TEMP: tempDir, + }, + }); + + return { apiUrl, assetUrl, callLogPath, result, latestSha, tempDir }; +} + +describe("cloudflared update-check workflow contract", () => { + const workflow = readYaml( + ".github/workflows/cloudflared-update-check.yaml", + ); + const e2e = fs.readFileSync(E2E_WORKFLOW, "utf8"); + + it("runs weekly and manually with read-only permissions and a credential-free checkout", () => { + expect(workflow.on?.schedule).toEqual([{ cron: "23 13 * * 1" }]); + expect(workflow.on?.workflow_dispatch).toEqual({}); + expect(workflow.permissions).toEqual({ contents: "read" }); + + const job = workflow.jobs?.["check-cloudflared"]; + const checkout = job?.steps?.find((step) => step.uses?.startsWith("actions/checkout@")); + const check = job?.steps?.find( + (step) => step.name === "Compare reviewed pin with the latest upstream release", + ); + expect(job?.permissions).toBeUndefined(); + expect(checkout?.uses).toMatch(FULL_SHA_ACTION); + expect(checkout?.with?.["persist-credentials"]).toBe(false); + expect(check?.run).toBe("bash scripts/checks/check-cloudflared-update.sh"); + }); + + it("extracts exactly three identical reviewed version and SHA256 pins", () => { + const versions = pinValues(e2e, "CLOUDFLARED_VERSION"); + const hashes = pinValues(e2e, "CLOUDFLARED_DEB_SHA256"); + expect(versions).toHaveLength(3); + expect(hashes).toHaveLength(3); + expect(new Set(versions).size).toBe(1); + expect(new Set(hashes).size).toBe(1); + expect(versions[0]).toMatch(/^[0-9]{4}\.[0-9]{1,2}\.[0-9]+$/u); + expect(hashes[0]).toMatch(/^[0-9a-f]{64}$/u); + }); + + it("queries the upstream latest release and verifies its exact linux-amd64 asset", () => { + const fixture = runFixtureCheck({ pinnedVersion: "2026.7.1", latestVersion: "2026.7.1" }); + try { + expect(fixture.result.status, fixture.result.stderr).toBe(0); + expect(fs.readFileSync(fixture.callLogPath, "utf8").trim().split("\n")).toEqual([ + fixture.apiUrl, + fixture.assetUrl, + ]); + } finally { + fs.rmSync(fixture.tempDir, { recursive: true, force: true }); + } + }); + + it("passes only when the latest release asset matches the reviewed SHA256", () => { + const fixture = runFixtureCheck({ pinnedVersion: "2026.7.1", latestVersion: "2026.7.1" }); + try { + expect(fixture.result.status, fixture.result.stderr).toBe(0); + expect(fixture.result.stdout).toContain("cloudflared pin is current"); + expect(fixture.result.stdout).toContain(fixture.latestSha); + expect(fixture.result.stdout).toContain("OK"); + } finally { + fs.rmSync(fixture.tempDir, { recursive: true, force: true }); + } + }); + + it("fails an outdated pin with the latest version, hash, and all update locations", () => { + const fixture = runFixtureCheck({ pinnedVersion: "2026.6.1", latestVersion: "2026.7.1" }); + try { + expect(fixture.result.status).toBe(1); + expect(fixture.result.stderr).toContain("cloudflared update required"); + expect(fixture.result.stderr).toContain("Pinned version: 2026.6.1"); + expect(fixture.result.stderr).toContain("Latest version: 2026.7.1"); + expect(fixture.result.stderr).toContain( + `Latest linux-amd64.deb SHA256: ${fixture.latestSha}`, + ); + expect(fixture.result.stderr).toContain("CLOUDFLARED_VERSION lines:"); + expect(fixture.result.stderr).toContain("CLOUDFLARED_DEB_SHA256 lines:"); + expect(fixture.result.stderr).toContain("Set all three version/SHA256 pairs"); + } finally { + fs.rmSync(fixture.tempDir, { recursive: true, force: true }); + } + }); +}); diff --git a/test/config-set.test.ts b/test/config-set.test.ts index f4bf928b8e5..b695af20f12 100644 --- a/test/config-set.test.ts +++ b/test/config-set.test.ts @@ -76,30 +76,27 @@ describe("buildRecomputeSandboxConfigHashScript", () => { }); describe("selectDirectSandboxContainer", () => { - it("returns the exact direct sandbox container when present", () => { - const selected = selectDirectSandboxContainer( - "demo", - "openshell-demo\nopenshell-demo-helper\n", - ["demo"], - ); + it("returns the immutable id for an exact direct sandbox container", () => { + const selected = selectDirectSandboxContainer("demo", "exact-id\topenshell-demo\n", ["demo"]); - expect(selected).toBe("openshell-demo"); + expect(selected).toBe("exact-id"); }); - it("falls back to the generated direct sandbox container prefix", () => { - const selected = selectDirectSandboxContainer( + it("returns the immutable id for a generated direct sandbox container", () => { + const selected = selectDirectSandboxContainer("demo", "generated-id\topenshell-demo-abc123\n", [ "demo", - "openshell-other\nopenshell-demo-abc123\n", - ["demo"], - ); + ]); - expect(selected).toBe("openshell-demo-abc123"); + expect(selected).toBe("generated-id"); }); - it("does not select a prefix-collision container owned by a longer sandbox name", () => { - expect( - selectDirectSandboxContainer("demo", "openshell-demo-child\n", ["demo", "demo-child"]), - ).toBeNull(); + it("rejects a prefix-collision container owned by a longer sandbox name", () => { + expect(() => + selectDirectSandboxContainer("demo", "child-id\topenshell-demo-child\n", [ + "demo", + "demo-child", + ]), + ).toThrow(/labels and names disagree.*refusing lifecycle execution/); }); }); diff --git a/test/dcode-wrapper-empty-prompt.test.ts b/test/dcode-wrapper-empty-prompt.test.ts index 94cf086d611..acfccbe8d92 100644 --- a/test/dcode-wrapper-empty-prompt.test.ts +++ b/test/dcode-wrapper-empty-prompt.test.ts @@ -56,7 +56,7 @@ function runWrapper(args: string[]): WrapperRun { /export PATH="([^"]*)"/, (_match, managedPath: string) => `export PATH=${JSON.stringify(`${bin}:${managedPath}`)}`, ) - .replace("/opt/venv/bin/python3 -I", "python3 -I"); + .replaceAll("/opt/venv/bin/python3 -I", "python3 -I"); expect(wrapperFixture).not.toBe(wrapperSource); fs.writeFileSync(path.join(dir, "dcode"), wrapperFixture, { mode: 0o755 }); diff --git a/test/dcode-wrapper-identity.test.ts b/test/dcode-wrapper-identity.test.ts index e392e990b04..61691862d0b 100644 --- a/test/dcode-wrapper-identity.test.ts +++ b/test/dcode-wrapper-identity.test.ts @@ -40,6 +40,12 @@ const SAMPLE_CONFIG = [ const OPAQUE = "Zx3Qw9Lp7Rt2Vn5Bd8Kf1Mh6Cg4Js0Ay"; const CANONICAL_TLS_KEY_PATH = "/etc/openshell/tls/client/tls.key"; +const MANAGED_MCP_VALIDATOR_INVOCATION = [ + 'managed_mcp_config="$(', + " /opt/venv/bin/python3 -I -c \\", + " 'from deepagents_code._nemoclaw_managed import managed_mcp_config_path; print(managed_mcp_config_path() or \"\")'", + ')"', +].join("\n"); function fakePrivateKeyBlock(type = "", newline = "\\n"): string { const label = type ? `${type} PRIVATE KEY-----` : "PRIVATE KEY-----"; @@ -61,6 +67,7 @@ function buildFixture(tempDir: string, configContent: string): Fixture { const configFile = path.join(tempDir, "config.toml"); const fixture = fs .readFileSync(WRAPPER, "utf8") + .replace(MANAGED_MCP_VALIDATOR_INVOCATION, 'managed_mcp_config=""') .replace( 'readonly DEEPAGENTS_ENV_FILE="/sandbox/.deepagents/.env"', `readonly DEEPAGENTS_ENV_FILE="${envFile}"`, diff --git a/test/deepagents-mcp-legacy-lifecycle.test.ts b/test/deepagents-mcp-legacy-lifecycle.test.ts new file mode 100644 index 00000000000..fec72474152 --- /dev/null +++ b/test/deepagents-mcp-legacy-lifecycle.test.ts @@ -0,0 +1,284 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +function runLegacyLifecycle(body: string) { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-deepagents-mcp-legacy-")); + const script = String.raw` +process.env.HOME = ${JSON.stringify(home)}; +const registry = require("./src/lib/state/registry.js"); +const globalActions = require("./src/lib/actions/global.js"); +const gatewayRuntime = require("./src/lib/gateway-runtime-action.js"); +const policies = require("./src/lib/policy/index.js"); +const processRecovery = require("./src/lib/actions/sandbox/process-recovery.js"); + +const providerId = "11111111-2222-4333-8444-555555555555"; +let providerExists = true; +let attached = true; +let adapterRegistered = true; +let deepAgentsCapability = false; +let policyApplyCalls = 0; +let policyState = "match"; +const adapterCalls = []; + +gatewayRuntime.recoverNamedGatewayRuntime = async () => ({ + recovered: true, + attempted: false, + before: { state: "healthy_named" }, + after: { state: "healthy_named" }, +}); +globalActions.runOpenshellProviderCommand = (args) => { + const command = args.join(" "); + if (command === "status --output json") { + return { status: 0, stdout: "ready", stderr: "" }; + } + if (args[0] === "provider" && args[1] === "get") { + return providerExists + ? { + status: 0, + stdout: "Id: " + providerId + "\nType: generic\nResource version: 1\nCredential keys: GITHUB_TOKEN\n", + stderr: "", + } + : { status: 1, stdout: "", stderr: "Provider not found" }; + } + if (args[0] === "sandbox" && args[1] === "provider" && args[2] === "list") { + return { + status: 0, + stdout: attached + ? "NAME TYPE CREDENTIAL_KEYS CONFIG_KEYS\nalpha-mcp-github generic 1 0\n" + : "No providers attached to sandbox alpha.\n", + stderr: "", + }; + } + if (args[0] === "sandbox" && args[1] === "provider" && args[2] === "detach") { + attached = false; + return { status: 0, stdout: "Detached provider", stderr: "" }; + } + if (args[0] === "sandbox" && args[1] === "provider" && args[2] === "attach") { + attached = true; + return { status: 0, stdout: "Attached provider", stderr: "" }; + } + if (args[0] === "provider" && args[1] === "delete") { + providerExists = false; + attached = false; + return { status: 0, stdout: "Deleted provider", stderr: "" }; + } + throw new Error("Unexpected OpenShell call: " + command); +}; +policies.getPresetContentGatewayState = () => policyState; +policies.applyPresetContent = () => { + policyApplyCalls += 1; + policyState = "match"; + return true; +}; +policies.removePreset = () => { + policyState = "absent"; + return true; +}; +processRecovery.executeSandboxCommand = (_sandbox, command) => { + adapterCalls.push(command); + if (command === "/usr/local/bin/deepagents-code --nemoclaw-mcp-capability") { + return deepAgentsCapability + ? { status: 0, stdout: "NEMOCLAW_DEEPAGENTS_MCP_CAPABILITY=1\n", stderr: "" } + : { status: 2, stdout: "", stderr: "unknown option" }; + } + if (command.includes("servers.pop(payload['server'], None)")) { + adapterRegistered = false; + return { status: 0, stdout: "", stderr: "" }; + } + if (command.includes("data = {'mcpServers': payload['expectedServers']}")) { + adapterRegistered = true; + return { status: 0, stdout: "", stderr: "" }; + } + if (command.includes("print('registered' if ok else ('mismatch' if present else 'absent'))")) { + return { + status: 0, + stdout: adapterRegistered ? "registered\n" : "absent\n", + stderr: "", + }; + } + return { status: 0, stdout: "", stderr: "" }; +}; +processRecovery.executeSandboxExecCommand = (_sandbox, command) => { + const encoded = command.match(/printf '%s' '([A-Za-z0-9+/=]+)' \| base64 -d/)?.[1] || ""; + const proof = encoded ? Buffer.from(encoded, "base64").toString("utf8") : command; + const isRevisionObservation = proof.includes("valid_placeholder()"); + const isDetachedProof = + !isRevisionObservation && proof.includes('[ -z "\${GITHUB_TOKEN+x}" ]'); + return { + status: isDetachedProof && attached ? 1 : 0, + stdout: attached ? "canonical" : "absent", + stderr: "", + }; +}; + +const entry = { + server: "github", + agent: "langchain-deepagents-code", + adapter: "deepagents-config", + url: "https://8.8.8.8/github", + env: ["GITHUB_TOKEN"], + providerName: "alpha-mcp-github", + providerId, + policyName: "mcp-bridge-github", + addedAt: "2026-06-27T00:00:00.000Z", +}; +registry.registerSandbox({ + name: "alpha", + agent: "langchain-deepagents-code", + gatewayName: "nemoclaw", + mcp: { bridges: { github: entry } }, +}); +registry.addCustomPolicy("alpha", { + name: entry.policyName, + content: "network_policies: {}\n", + sourcePath: "generated:nemoclaw-mcp-bridge", +}); +const bridge = require("./src/lib/actions/sandbox/mcp-bridge.js"); +${body} +`; + const result = spawnSync(process.execPath, ["-e", script], { + cwd: process.cwd(), + encoding: "utf8", + env: { ...process.env, HOME: home }, + }); + fs.rmSync(home, { recursive: true, force: true }); + return result; +} + +function parseResult(result: ReturnType) { + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + return JSON.parse(result.stdout.slice(result.stdout.indexOf("{"))) as { + error?: string; + entryCount?: number; + attached: boolean; + adapterRegistered: boolean; + providerExists: boolean; + policyApplyCalls: number; + markerCalls: number; + }; +} + +const resultExpression = `JSON.stringify({ + attached, + adapterRegistered, + providerExists, + policyApplyCalls, + markerCalls: adapterCalls.filter((call) => + call.includes("deepagents-code --nemoclaw-mcp-capability") + ).length, +})`; + +describe("legacy Deep Agents managed MCP lifecycle", () => { + it("removes an existing entry without requiring the new launcher marker", () => { + const result = runLegacyLifecycle(` +(async () => { + await bridge.removeMcpBridge("alpha", "github"); + process.stdout.write(${resultExpression}); +})().catch((error) => { console.error(error); process.exit(1); }); +`); + expect(parseResult(result)).toMatchObject({ + attached: false, + adapterRegistered: false, + providerExists: false, + markerCalls: 0, + }); + }); + + for (const [label, method] of [ + ["destroy", "prepareMcpBridgesForDestroy"], + ["rebuild", "prepareMcpBridgesForRebuild"], + ] as const) { + it(`${label} teardown does not require the marker from the old image`, () => { + const result = runLegacyLifecycle(` +(async () => { + const preparation = await bridge.${method}("alpha"); + process.stdout.write(JSON.stringify({ + entryCount: preparation.entries.length, + attached, + adapterRegistered, + providerExists, + policyApplyCalls, + markerCalls: adapterCalls.filter((call) => + call.includes("deepagents-code --nemoclaw-mcp-capability") + ).length, + })); +})().catch((error) => { console.error(error); process.exit(1); }); +`); + expect(parseResult(result)).toMatchObject({ + entryCount: 1, + attached: false, + adapterRegistered: false, + providerExists: true, + markerCalls: 0, + }); + }); + } + + it("proves the replacement image marker before post-rebuild reattachment", () => { + const result = runLegacyLifecycle(` +(async () => { + const preparation = await bridge.prepareMcpBridgesForRebuild("alpha"); + let error = ""; + try { + await bridge.restoreMcpBridgesAfterRebuild("alpha", preparation.entries); + } catch (caught) { + error = caught instanceof Error ? caught.message : String(caught); + } + process.stdout.write(JSON.stringify({ + error, + attached, + adapterRegistered, + providerExists, + policyApplyCalls, + markerCalls: adapterCalls.filter((call) => + call.includes("deepagents-code --nemoclaw-mcp-capability") + ).length, + })); +})().catch((error) => { console.error(error); process.exit(1); }); +`); + expect(parseResult(result)).toMatchObject({ + error: expect.stringMatching(/does not contain the managed MCP-aware launcher/i), + attached: false, + adapterRegistered: false, + providerExists: true, + policyApplyCalls: 0, + markerCalls: 1, + }); + }); + + for (const [label, prepare, restore] of [ + [ + "destroy", + "prepareMcpBridgesForDestroy", + "restoreMcpBridgesAfterDestroyAbort('alpha', preparation)", + ], + [ + "rebuild", + "prepareMcpBridgesForRebuild", + "reattachMcpProvidersAfterRebuildAbort('alpha', preparation.detachedProviderEntries, preparation.scrubbedAdapterEntries)", + ], + ] as const) { + it(`restores the old image when ${label} deletion aborts`, () => { + const result = runLegacyLifecycle(` +(async () => { + const preparation = await bridge.${prepare}("alpha"); + await bridge.${restore}; + process.stdout.write(${resultExpression}); +})().catch((error) => { console.error(error); process.exit(1); }); +`); + expect(parseResult(result)).toMatchObject({ + attached: true, + adapterRegistered: true, + providerExists: true, + markerCalls: 0, + }); + }); + } +}); diff --git a/test/deepagents-mcp-runtime-capability.test.ts b/test/deepagents-mcp-runtime-capability.test.ts new file mode 100644 index 00000000000..fd6f203aef7 --- /dev/null +++ b/test/deepagents-mcp-runtime-capability.test.ts @@ -0,0 +1,71 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; + +import { describe, expect, it } from "vitest"; + +type ProbeResult = { status: number; stdout: string; stderr: string } | null; + +function runDeepAgentsProbe(result: ProbeResult) { + const script = String.raw` +const processRecovery = require("./src/lib/actions/sandbox/process-recovery.js"); +const calls = []; +processRecovery.executeSandboxCommand = (sandboxName, command) => { + calls.push({ sandboxName, command }); + return ${JSON.stringify(result)}; +}; +const adapters = require("./src/lib/actions/sandbox/mcp-bridge-adapters.js"); +let message = ""; +try { + adapters.assertAgentMcpMutationRuntimeCapability("deepagents-box", "deepagents-config"); +} catch (error) { + message = error instanceof Error ? error.message : String(error); +} +process.stdout.write(JSON.stringify({ calls, message })); +`; + const child = spawnSync(process.execPath, ["-e", script], { + cwd: process.cwd(), + encoding: "utf8", + env: process.env, + }); + expect(child.status, `${child.stdout}\n${child.stderr}`).toBe(0); + return JSON.parse(child.stdout) as { + calls: Array<{ sandboxName: string; command: string }>; + message: string; + }; +} + +describe("Deep Agents managed MCP runtime capability", () => { + it("accepts only the exact managed launcher capability marker", () => { + expect( + runDeepAgentsProbe({ + status: 0, + stdout: "NEMOCLAW_DEEPAGENTS_MCP_CAPABILITY=1\n", + stderr: "", + }), + ).toEqual({ + calls: [ + { + sandboxName: "deepagents-box", + command: "/usr/local/bin/deepagents-code --nemoclaw-mcp-capability", + }, + ], + message: "", + }); + }); + + it("requires a rebuild before MCP side effects on stale or unreachable images", () => { + for (const result of [ + null, + { status: 2, stdout: "", stderr: "unknown option" }, + { status: 0, stdout: "deepagents-code 0.1.12\n", stderr: "" }, + ]) { + const probe = runDeepAgentsProbe(result); + expect(probe.calls).toHaveLength(1); + expect(probe.message).toMatch(/does not contain the managed MCP-aware launcher/i); + expect(probe.message).toMatch(/rebuild the sandbox before changing authenticated MCP state/i); + expect(probe.message).not.toContain("unknown option"); + } + }); +}); diff --git a/test/e2e-advisor-targets.test.ts b/test/e2e-advisor-targets.test.ts index 6bf9011787e..eb429e2a553 100644 --- a/test/e2e-advisor-targets.test.ts +++ b/test/e2e-advisor-targets.test.ts @@ -6,14 +6,14 @@ import { describe, expect, it } from "vitest"; import { buildTargetComment } from "../tools/e2e-advisor/target-comment.mts"; import { buildPrompt, - buildTargetPromptTurn, buildSystemPrompt, + buildTargetPromptTurn, canonicalDispatchCommand, + E2E_TARGET_ADVISOR_WORKFLOWS, + type E2eTargetAdvisorResult, extractFreeStandingE2eJobs, normalizeE2eTargetAdvisorResult, renderTargetSummary, - E2E_TARGET_ADVISOR_WORKFLOWS, - type E2eTargetAdvisorResult, } from "../tools/e2e-advisor/targets.mts"; // Tests target observable behavior of the target advisor pipeline: diff --git a/test/e2e/brev-e2e.test.ts b/test/e2e/brev-e2e.test.ts index 0edd66d9df5..b40f3c72474 100644 --- a/test/e2e/brev-e2e.test.ts +++ b/test/e2e/brev-e2e.test.ts @@ -29,7 +29,6 @@ * TEST_SUITE — which test to run: full (default), deploy-cli, gpu, * credential-sanitization, telegram-injection, messaging-providers, * messaging-compatible-endpoint, dashboard-remote-bind, all - * LAUNCHABLE_SETUP_SCRIPT — URL to setup script for launchable path (default: brev-launchable-ci-cpu.sh on main) * BREV_MIN_VCPU — Minimum vCPUs for CPU instance (default: 4) * BREV_MIN_RAM — Minimum RAM in GB for CPU instance (default: 16) * BREV_PROVIDER — Cloud provider filter for brev search (default: gcp for CPU, any for GPU) @@ -54,6 +53,16 @@ import { execFileSync, execSync, type StdioOptions, spawnSync } from "node:child import path from "node:path"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { shellQuote } from "../../src/lib/core/shell-quote"; +import { + BREV_MESSAGING_COMPAT_TIMEOUT_MS, + BREV_MESSAGING_PROVIDER_TIMEOUT_MS, + BREV_REMOTE_WRAPPER_GRACE_MS, + BREV_SECURITY_SUITE_TIMEOUT_MS, + brevSuiteHarnessSandboxName, + brevSuiteNeedsHarnessSandbox, + brevWorkflowOwnsInstance, + buildBrevRemoteVitestCommand, +} from "../../tools/e2e/brev-remote-vitest.mts"; // Instance configuration const BREV_MIN_VCPU = parseInt(process.env.BREV_MIN_VCPU || "4", 10); @@ -102,11 +111,9 @@ function requireInstanceName(): string { // Launchable configuration // CI-Ready CPU setup script: pre-bakes Docker, Node.js, OpenShell CLI, and npm deps. // The Brev CLI (v0.6.322+) uses `brev search cpu | brev create --startup-script @file`. -// Default: use the repo-local script (hermetic — always matches the checked-out branch). -// Override via LAUNCHABLE_SETUP_SCRIPT env var to test a remote URL instead. -const DEFAULT_SETUP_SCRIPT_PATH = - process.env.LAUNCHABLE_SETUP_SCRIPT || - path.join(REPO_DIR, "scripts", "brev-launchable-ci-cpu.sh"); +// Use the repo-local script so secret-bearing branch validation cannot execute +// mutable setup code selected outside the reviewed checkout. +const SETUP_SCRIPT_PATH = path.join(REPO_DIR, "scripts", "brev-launchable-ci-cpu.sh"); // Sentinel file written by brev-launchable-ci-cpu.sh when setup is complete. // More reliable than grepping log files. const LAUNCHABLE_SENTINEL = "/var/run/nemoclaw-launchable-ready"; @@ -264,12 +271,15 @@ function sshEnv( { timeout = 600_000, stream = false }: { timeout?: number; stream?: boolean } = {}, ): string { const gpuE2eModel = process.env.NEMOCLAW_GPU_E2E_MODEL || "qwen3.5:9b"; + const harnessSandboxName = brevSuiteHarnessSandboxName(TEST_SUITE); const envParts = [ `export NVIDIA_INFERENCE_API_KEY='${shellEscape(process.env.NVIDIA_INFERENCE_API_KEY)}'`, `export GITHUB_TOKEN='${shellEscape(process.env.GITHUB_TOKEN)}'`, `export NEMOCLAW_NON_INTERACTIVE=1`, `export NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1`, - `export NEMOCLAW_SANDBOX_NAME=e2e-test`, + ...(harnessSandboxName + ? [`export NEMOCLAW_SANDBOX_NAME='${shellEscape(harnessSandboxName)}'`] + : []), `export NEMOCLAW_TRACE_DIR=/tmp/nemoclaw-traces`, ]; if (GPU_TEST_SUITE) { @@ -425,10 +435,8 @@ function runRemoteCommand( return ssh("cat /tmp/test-output.log", { timeout: 30_000 }); } -function runRemoteVitest(project: "cli" | "e2e-live", target: string): string { - return runRemoteCommand( - `NEMOCLAW_RUN_LIVE_E2E=1 npx vitest run --project ${project} ${target} --silent=false --reporter=default`, - ); +function runRemoteVitest(project: "cli" | "e2e-live", target: string, timeoutMs?: number): string { + return runRemoteCommand(buildBrevRemoteVitestCommand(project, target), timeoutMs); } function expectVitestPassed(output: string): void { @@ -591,7 +599,7 @@ function summarizeBrevCandidates(output: string, maxLines = 10): string { function createBrevInstance(elapsed: () => string): void { const instanceKind = GPU_TEST_SUITE ? "gpu" : "cpu"; console.log(`[${elapsed()}] Creating ${instanceKind} instance via launchable...`); - console.log(`[${elapsed()}] setup-script: ${DEFAULT_SETUP_SCRIPT_PATH}`); + console.log(`[${elapsed()}] setup-script: ${SETUP_SCRIPT_PATH}`); console.log(`[${elapsed()}] create timeout: ${Math.round(BREV_CREATE_TIMEOUT_MS / 1000)}s`); if (GPU_TEST_SUITE) { if (BREV_GPU_TYPE) { @@ -607,21 +615,8 @@ function createBrevInstance(elapsed: () => string): void { ); } - // Resolve the setup script to a local file path. - // Default: repo-local scripts/brev-launchable-ci-cpu.sh (hermetic). - // Override: set LAUNCHABLE_SETUP_SCRIPT to a URL and it gets downloaded. - let setupScriptPath: string; - if (DEFAULT_SETUP_SCRIPT_PATH.startsWith("http")) { - setupScriptPath = "/tmp/brev-ci-setup.sh"; - execFileSync("curl", ["-fsSL", "-o", setupScriptPath, DEFAULT_SETUP_SCRIPT_PATH], { - encoding: "utf-8", - timeout: 30_000, - }); - console.log(`[${elapsed()}] Setup script downloaded to ${setupScriptPath}`); - } else { - setupScriptPath = DEFAULT_SETUP_SCRIPT_PATH; - console.log(`[${elapsed()}] Using repo-local setup script`); - } + const setupScriptPath = SETUP_SCRIPT_PATH; + console.log(`[${elapsed()}] Using repo-local setup script`); try { if (GPU_TEST_SUITE) { @@ -874,7 +869,13 @@ function bootstrapLaunchable(elapsed: () => string): { remoteDir: string; needsO ); console.log(`[${elapsed()}] nemoclaw CLI linked`); - return { remoteDir: resolvedRemoteDir, needsOnboard: true }; + return { + remoteDir: resolvedRemoteDir, + // The composite security suite provisions and tears down its own sandbox + // in each live target. Seeding a second harness-owned registry here leaves + // stale state after the first target destroys the shared gateway. + needsOnboard: brevSuiteNeedsHarnessSandbox(TEST_SUITE), + }; } /** @@ -1061,7 +1062,7 @@ describe("Brev deploy input validation", () => { NEMOCLAW_DEPLOY_NO_CONNECT: "1", NEMOCLAW_DEPLOY_NO_START_SERVICES: "1", }, - timeout: 30_000, + timeout: 60_000, }); const output = `${result.stdout}${result.stderr}`; @@ -1076,7 +1077,7 @@ describe("Brev deploy input validation", () => { expect(output).not.toContain("Waiting for Brev instance readiness"); expect(output).not.toContain("Waiting for SSH"); expect(output).not.toContain("bash scripts/install.sh"); - }); + }, 65_000); }); describe("Brev GPU runtime setup", () => { @@ -1131,7 +1132,7 @@ describe.runIf(hasRequiredVars && hasAuthenticatedBrev)("Brev E2E", () => { } // Verify sandbox registry (only when beforeAll created a sandbox) - if (TEST_SUITE !== "full" && !GPU_TEST_SUITE) { + if (brevSuiteNeedsHarnessSandbox(TEST_SUITE) && !GPU_TEST_SUITE) { console.log(`[${elapsed()}] Verifying sandbox registry...`); const registry = JSON.parse(ssh(`cat ~/.nemoclaw/sandboxes.json`, { timeout: 10_000 })); expect(registry.defaultSandbox).toBe("e2e-test"); @@ -1150,19 +1151,25 @@ describe.runIf(hasRequiredVars && hasAuthenticatedBrev)("Brev E2E", () => { afterAll(() => { if (!instanceCreated) return; - if (process.env.KEEP_ALIVE === "true") { - console.log(`\n Instance "${INSTANCE_NAME}" kept alive for debugging.`); - console.log(` To connect: brev refresh && ssh ${INSTANCE_NAME}`); - console.log(` To delete: brev delete ${INSTANCE_NAME}\n`); + const keepAlive = process.env.KEEP_ALIVE === "true"; + const workflowOwnsInstance = brevWorkflowOwnsInstance(); + if (keepAlive || workflowOwnsInstance) { + const lines = keepAlive + ? [ + `\n Instance "${INSTANCE_NAME}" kept alive for debugging.`, + ` To connect: brev refresh && ssh ${INSTANCE_NAME}`, + ` To delete: brev delete ${INSTANCE_NAME}\n`, + ] + : [`Instance "${INSTANCE_NAME}" deletion deferred to workflow-owned cleanup.`]; + console.log(lines.join("\n")); return; } deleteBrevInstance(requireInstanceName()); }, 120_000); // 2 min for cleanup - // NOTE: The full E2E test runs install.sh --non-interactive which destroys and - // rebuilds the sandbox from scratch. It cannot run alongside the security tests - // (credential-sanitization, telegram-injection) which depend on the sandbox - // that beforeAll already created. Run it only when TEST_SUITE=full. + // NOTE: The full E2E test runs install.sh --non-interactive and owns the + // complete sandbox lifecycle. The composite security suite also lets each + // remote target own that lifecycle, without a shared harness registry. it.runIf(TEST_SUITE === "full")( "full E2E suite passes on remote VM", () => { @@ -1184,19 +1191,27 @@ describe.runIf(hasRequiredVars && hasAuthenticatedBrev)("Brev E2E", () => { it.runIf(TEST_SUITE === "credential-sanitization" || TEST_SUITE === "all")( "credential sanitization suite passes on remote VM", () => { - const output = runRemoteVitest("e2e-live", "test/e2e/live/credential-sanitization.test.ts"); + const output = runRemoteVitest( + "e2e-live", + "test/e2e/live/credential-sanitization.test.ts", + BREV_SECURITY_SUITE_TIMEOUT_MS, + ); expectVitestPassed(output); }, - 600_000, + BREV_SECURITY_SUITE_TIMEOUT_MS + BREV_REMOTE_WRAPPER_GRACE_MS, ); it.runIf(TEST_SUITE === "telegram-injection" || TEST_SUITE === "all")( "telegram bridge injection suite passes on remote VM", () => { - const output = runRemoteVitest("e2e-live", "test/e2e/live/telegram-injection.test.ts"); + const output = runRemoteVitest( + "e2e-live", + "test/e2e/live/telegram-injection.test.ts", + BREV_SECURITY_SUITE_TIMEOUT_MS, + ); expectVitestPassed(output); }, - 600_000, + BREV_SECURITY_SUITE_TIMEOUT_MS + BREV_REMOTE_WRAPPER_GRACE_MS, ); it.runIf(TEST_SUITE === "deploy-cli")( @@ -1216,31 +1231,35 @@ describe.runIf(hasRequiredVars && hasAuthenticatedBrev)("Brev E2E", () => { 120_000, ); - // NOTE: The messaging-providers test creates its own sandbox (e2e-msg-provider) - // with messaging tokens attached. It does not conflict with the e2e-test sandbox - // used by other tests, but it may recreate the gateway. - it.runIf(TEST_SUITE === "messaging-providers" || TEST_SUITE === "all")( + // This stateful target owns its sandbox and gateway lifecycle. Brev runs it + // single-shot on a dedicated instance; a retry means a new workflow run and + // therefore a new VM, never a second installer behind a live onboard lock. + it.runIf(TEST_SUITE === "messaging-providers")( "messaging credential provider suite passes on remote VM", () => { - const output = runRemoteVitest("e2e-live", "test/e2e/live/messaging-providers.test.ts"); + const output = runRemoteVitest( + "e2e-live", + "test/e2e/live/messaging-providers.test.ts", + BREV_MESSAGING_PROVIDER_TIMEOUT_MS, + ); expectVitestPassed(output); }, - 900_000, // 15 min — creates a new sandbox with messaging providers + BREV_MESSAGING_PROVIDER_TIMEOUT_MS + BREV_REMOTE_WRAPPER_GRACE_MS, ); - // NOTE: The compatible-endpoint messaging test creates its own sandbox - // (e2e-msg-compat) with Telegram attached and a local OpenAI-compatible - // mock endpoint. It covers the inference.local path used by Telegram turns. - it.runIf(TEST_SUITE === "messaging-compatible-endpoint" || TEST_SUITE === "all")( + // The compatible-endpoint target also owns its sandbox lifecycle and runs + // on a separate Brev instance so provider cleanup cannot leak across it. + it.runIf(TEST_SUITE === "messaging-compatible-endpoint")( "messaging compatible endpoint suite passes on remote VM", () => { const output = runRemoteVitest( "e2e-live", "test/e2e/live/messaging-compatible-endpoint.test.ts", + BREV_MESSAGING_COMPAT_TIMEOUT_MS, ); expectVitestPassed(output); }, - 900_000, // 15 min — creates a new sandbox with Telegram + compatible endpoint + BREV_MESSAGING_COMPAT_TIMEOUT_MS + BREV_REMOTE_WRAPPER_GRACE_MS, ); it.runIf(TEST_SUITE === "dashboard-remote-bind")( diff --git a/test/e2e/docs/README.md b/test/e2e/docs/README.md index 49be8c2aae1..6372757199c 100644 --- a/test/e2e/docs/README.md +++ b/test/e2e/docs/README.md @@ -74,16 +74,13 @@ npx vitest run --project e2e-support --silent=false --reporter=default # Opt-in live E2E targets npm run build:cli NEMOCLAW_RUN_LIVE_E2E=1 npx vitest run --project e2e-live --silent=false --reporter=default - -# Force two retries locally (three total attempts) for external-service flakes -NEMOCLAW_RUN_LIVE_E2E=1 NEMOCLAW_E2E_RETRIES=2 npx vitest run --project e2e-live ``` -Live E2E projects retry failed tests automatically in CI. The default is -2 retries after the first failure (3 total attempts). Local opt-in runs default -to no full-test retry; set `NEMOCLAW_E2E_RETRIES=` to override either -local or CI behavior. Overrides are capped at 5 retries so a typo cannot create -unbounded credentialed live infrastructure attempts. +Live E2E projects do not retry an entire failed test. These tests mutate host, +Docker, gateway, and sandbox state, so re-entering one on the same runner can +replace the original failure with stale-lock, storage-exhaustion, or ownership +noise. A target may retry a transient operation only inside its own cleanup +boundary. Retry a full target by starting a fresh workflow run and runner. The retired `--emit-matrix` and `--plan-only` paths must not be reintroduced. diff --git a/test/e2e/fixtures/clients/sandbox.ts b/test/e2e/fixtures/clients/sandbox.ts index df89a78160d..0100eca7637 100644 --- a/test/e2e/fixtures/clients/sandbox.ts +++ b/test/e2e/fixtures/clients/sandbox.ts @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { Buffer } from "node:buffer"; import { buildAvailabilityProbeEnv } from "../availability-env.ts"; import type { ShellProbeResult, ShellProbeRunOptions } from "../shell-probe.ts"; import { trustedShellCommand } from "../shell-probe.ts"; @@ -42,16 +43,22 @@ export type TrustedSandboxShellScript = string & { }; export function trustedSandboxShellScript(script: string): TrustedSandboxShellScript { - if (script.length === 0 || script.includes("\0")) { - throw new Error("sandbox shell script must be non-empty and contain no NUL bytes"); + if (script.length === 0) { + throw new Error("sandbox shell script must not be empty"); + } + if (script.includes("\0")) { + throw new Error("sandbox shell script must contain no NUL bytes"); } return script as TrustedSandboxShellScript; } function sandboxShellArgument(script: TrustedSandboxShellScript): string { - if (!/[\r\n]/u.test(script)) return script; - const encoded = Buffer.from(script, "utf8").toString("base64"); - return `eval "$(printf '%s' '${encoded}' | base64 -d)"`; + const encodedScript = Buffer.from(script, "utf8").toString("base64"); + return [ + "command -v base64 >/dev/null 2>&1 || { echo NEMOCLAW_BASE64_MISSING >&2; exit 127; }", + `_NEMOCLAW_E2E_SCRIPT="$(printf '%s' '${encodedScript}' | base64 -d)" || exit $?`, + `eval "$_NEMOCLAW_E2E_SCRIPT"`, + ].join("; "); } export class SandboxClient { diff --git a/test/e2e/fixtures/mcp-bridge-credentials.ts b/test/e2e/fixtures/mcp-bridge-credentials.ts new file mode 100644 index 00000000000..b2b7f057a14 --- /dev/null +++ b/test/e2e/fixtures/mcp-bridge-credentials.ts @@ -0,0 +1,9 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export const MCP_BRIDGE_TEST_CREDENTIALS = { + host: "fake-host-mcp-secret-value", + rotatedHost: "fake-rotated-mcp-secret-value", + rebindHost: "fake-rebind-mcp-secret-value", + compatibleEndpoint: "fake-compatible-mcp-bridge-key", +} as const; diff --git a/test/e2e/fixtures/redaction.ts b/test/e2e/fixtures/redaction.ts index 0c45e601ac4..fad529357c9 100644 --- a/test/e2e/fixtures/redaction.ts +++ b/test/e2e/fixtures/redaction.ts @@ -146,6 +146,7 @@ const FIXTURE_ENV_ALLOWLIST: ReadonlySet = new Set([ "NEMOCLAW_NON_INTERACTIVE", "NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE", "NEMOCLAW_E2E_USE_HOSTED_INFERENCE", + "NEMOCLAW_OPENSHELL_CHANNEL", "NEMOCLAW_TRACE_DIR", ]); diff --git a/test/e2e/live/dns-rebinding-hosts-fixture.ts b/test/e2e/live/dns-rebinding-hosts-fixture.ts new file mode 100644 index 00000000000..8f5c70c15c3 --- /dev/null +++ b/test/e2e/live/dns-rebinding-hosts-fixture.ts @@ -0,0 +1,170 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import os from "node:os"; +import path from "node:path"; + +import { shellQuote } from "../../../src/lib/core/shell-quote"; +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import type { HostCliClient } from "../fixtures/clients/host.ts"; +import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; + +export interface DnsRebindingHostsFixture { + hostname: string; + hostBackupPath: string; + sandboxBackupPath: string; +} + +function assertHostFixtureProbeSucceeded(result: ShellProbeResult, label: string): void { + if (result.exitCode === 0) return; + throw new Error(`${label}\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`); +} + +export async function setupDnsRebindingHostsFixture( + host: HostCliClient, + sandboxName: string, + hostname: string, +): Promise { + const tempDir = process.env.RUNNER_TEMP ?? os.tmpdir(); + const suffix = `${process.pid}-${sandboxName}`; + const fixture = { + hostname, + hostBackupPath: path.join(tempDir, `nemoclaw-rebind-hosts-host-${suffix}`), + sandboxBackupPath: path.join(tempDir, `nemoclaw-rebind-hosts-sandbox-${suffix}`), + }; + const result = await host.command( + "bash", + [ + "-lc", + [ + "set -euo pipefail", + `sandbox_name=${shellQuote(sandboxName)}`, + `hostname=${shellQuote(hostname)}`, + `host_backup=${shellQuote(fixture.hostBackupPath)}`, + `sandbox_backup=${shellQuote(fixture.sandboxBackupPath)}`, + 'container_id="$(docker ps --filter "label=openshell.ai/sandbox-name=${sandbox_name}" --format \'{{.ID}}\' | head -n 1)"', + '[ -n "$container_id" ] || { echo "OpenShell sandbox container not found" >&2; exit 1; }', + "sudo -n true", + 'rm -f "$host_backup" "$sandbox_backup"', + 'sudo -n cat /etc/hosts > "$host_backup"', + 'docker exec "$container_id" cat /etc/hosts > "$sandbox_backup"', + 'if grep -Fq "$hostname" "$host_backup" || grep -Fq "$hostname" "$sandbox_backup"; then rm -f "$host_backup" "$sandbox_backup"; echo "DNS rebinding fixture hostname already exists in /etc/hosts" >&2; exit 1; fi', + ].join("\n"), + ], + { + artifactName: "dns-rebinding-backup-hosts", + env: buildAvailabilityProbeEnv(), + timeoutMs: 60_000, + }, + ); + assertHostFixtureProbeSucceeded( + result, + "back up host and sandbox hosts files for DNS rebinding proof", + ); + return fixture; +} + +export async function remapDnsRebindingHostname( + host: HostCliClient, + sandboxName: string, + fixture: DnsRebindingHostsFixture, + address: string, + artifactName: string, +): Promise { + const resolverCheck = [ + 'const dns = require("node:dns");', + "const [hostname, expected] = process.argv.slice(1);", + "dns.lookup(hostname, { all: true, verbatim: true }, (error, results) => {", + " if (error) throw error;", + " const addresses = [...new Set(results.map((result) => result.address))];", + " console.log(JSON.stringify({ hostname, addresses }));", + " process.exit(addresses.length === 1 && addresses[0] === expected ? 0 : 1);", + "});", + ].join(" "); + const result = await host.command( + "bash", + [ + "-lc", + [ + "set -euo pipefail", + `sandbox_name=${shellQuote(sandboxName)}`, + `hostname=${shellQuote(fixture.hostname)}`, + `expected_ip=${shellQuote(address)}`, + `host_backup=${shellQuote(fixture.hostBackupPath)}`, + `sandbox_backup=${shellQuote(fixture.sandboxBackupPath)}`, + '[ -s "$host_backup" ] && [ -s "$sandbox_backup" ] || { echo "DNS rebinding hosts backups are missing" >&2; exit 1; }', + 'container_id="$(docker ps --filter "label=openshell.ai/sandbox-name=${sandbox_name}" --format \'{{.ID}}\' | head -n 1)"', + '[ -n "$container_id" ] || { echo "OpenShell sandbox container not found" >&2; exit 1; }', + 'sudo -n tee /etc/hosts < "$host_backup" >/dev/null', + 'printf "\\n%s %s\\n" "$expected_ip" "$hostname" | sudo -n tee -a /etc/hosts >/dev/null', + 'docker exec --user 0 -i "$container_id" sh -c \'cat > /etc/hosts\' < "$sandbox_backup"', + 'printf "\\n%s %s\\n" "$expected_ip" "$hostname" | docker exec --user 0 -i "$container_id" tee -a /etc/hosts >/dev/null', + `node -e ${shellQuote(resolverCheck)} "$hostname" "$expected_ip"`, + 'docker exec "$container_id" grep -F "$expected_ip $hostname" /etc/hosts >/dev/null', + ].join("\n"), + ], + { + artifactName, + env: buildAvailabilityProbeEnv(), + timeoutMs: 60_000, + }, + ); + assertHostFixtureProbeSucceeded(result, `map DNS rebinding fixture hostname to ${address}`); +} + +export async function restoreDnsRebindingHostsFixture( + host: HostCliClient, + sandboxName: string, + fixture: DnsRebindingHostsFixture, +): Promise { + const result = await host.command( + "bash", + [ + "-lc", + [ + // Cleanup must report the exact failed operation. An implicit `errexit` + // here can turn a transient file/container race into an unexplained + // exit 1 with empty stdout/stderr, which defeats the cleanup artifact. + "set -uo pipefail", + `sandbox_name=${shellQuote(sandboxName)}`, + `host_backup=${shellQuote(fixture.hostBackupPath)}`, + `sandbox_backup=${shellQuote(fixture.sandboxBackupPath)}`, + 'if [ ! -f "$host_backup" ] && [ ! -f "$sandbox_backup" ]; then echo "DNS rebinding hosts backups already absent"; exit 0; fi', + "host_restore_failed=0", + 'if [ -f "$host_backup" ]; then', + ' if ! sudo -n tee /etc/hosts < "$host_backup" >/dev/null; then', + ' echo "failed to restore host /etc/hosts" >&2; host_restore_failed=1', + ' elif ! cmp -s "$host_backup" /etc/hosts; then', + ' echo "host /etc/hosts differs after restoration" >&2; host_restore_failed=1', + " else", + ' echo "restored host /etc/hosts"', + " fi", + "else", + ' echo "host /etc/hosts backup is missing while sandbox backup remains" >&2; host_restore_failed=1', + "fi", + 'if [ -f "$sandbox_backup" ]; then', + " sandbox_restored=0", + " for attempt in 1 2 3; do", + ' container_id="$(docker ps --filter "label=openshell.ai/sandbox-name=${sandbox_name}" --format \'{{.ID}}\' 2>/dev/null | head -n 1 || true)"', + ' if [ -n "$container_id" ] && docker exec --user 0 -i "$container_id" sh -c \'cat > /etc/hosts\' < "$sandbox_backup"; then sandbox_restored=1; break; fi', + ' [ "$attempt" -eq 3 ] || sleep 1', + " done", + ' if [ "$sandbox_restored" -eq 1 ]; then echo "restored sandbox /etc/hosts"; else echo "::warning::could not restore ephemeral sandbox /etc/hosts; cleanup will destroy the sandbox" >&2; fi', + "fi", + 'if [ "$host_restore_failed" -ne 0 ]; then exit 1; fi', + 'if ! rm -f "$host_backup" "$sandbox_backup"; then echo "failed to remove DNS rebinding hosts backups" >&2; exit 1; fi', + 'echo "removed DNS rebinding hosts backups"', + "exit 0", + ].join("\n"), + ], + { + artifactName: "dns-rebinding-restore-hosts", + env: buildAvailabilityProbeEnv(), + timeoutMs: 60_000, + }, + ); + assertHostFixtureProbeSucceeded( + result, + "restore host and sandbox hosts files after DNS rebinding proof", + ); +} diff --git a/test/e2e/live/hermes-discord.test.ts b/test/e2e/live/hermes-discord.test.ts index ab8ab84b629..09d71ae9118 100644 --- a/test/e2e/live/hermes-discord.test.ts +++ b/test/e2e/live/hermes-discord.test.ts @@ -661,7 +661,7 @@ current_pid="$$" for p in /proc/[0-9]*; do pid=$(basename "$p") [ "$pid" = "$current_pid" ] && continue - cmd=$(tr "\000" " " < "$p/cmdline" 2>/dev/null || true) + cmd=$( { tr "\000" " " < "$p/cmdline"; } 2>/dev/null || true) case "$cmd" in *"name_needle="*|*"for p in /proc/"*) continue ;; esac case "$cmd" in *"$name_needle"*) echo PROCESS_FACADE ;; esac case "$cmd" in *"$decode_needle"*) echo PROCESS_DECODE_PROXY ;; esac diff --git a/test/e2e/live/hermes-slack-e2e-helpers.ts b/test/e2e/live/hermes-slack-e2e-helpers.ts index eee7c7a5f93..b857747d5d7 100644 --- a/test/e2e/live/hermes-slack-e2e-helpers.ts +++ b/test/e2e/live/hermes-slack-e2e-helpers.ts @@ -488,7 +488,7 @@ current_pid="$$" for p in /proc/[0-9]*; do pid=$(basename "$p") [ "$pid" = "$current_pid" ] && continue - cmd=$(tr "\000" " " < "$p/cmdline" 2>/dev/null || true) + cmd=$( { tr "\000" " " < "$p/cmdline"; } 2>/dev/null || true) case "$cmd" in *"$decode_needle"*) echo PROCESS_DECODE_PROXY ;; esac done`, [], diff --git a/test/e2e/live/launchable-smoke.test.ts b/test/e2e/live/launchable-smoke.test.ts index 1f719ec7732..885dc7db89f 100644 --- a/test/e2e/live/launchable-smoke.test.ts +++ b/test/e2e/live/launchable-smoke.test.ts @@ -311,6 +311,12 @@ runLaunchableSmokeTest( timeoutMs: 30_000, }); expectExitZero(openshellVersion, "openshell is on PATH and --version works"); + const openshellVersionText = `${openshellVersion.stdout}\n${openshellVersion.stderr}`; + expect( + process.env.NEMOCLAW_OPENSHELL_CHANNEL !== "dev" || + /\d+\.\d+\.\d+[.-]dev\d*(?:[.+-][0-9A-Za-z]+)*/i.test(openshellVersionText), + "the dev integration target must install a dev-channel OpenShell build", + ).toBe(true); const nodeVersion = await host.command( "node", diff --git a/test/e2e/live/mcp-bridge-sandbox.ts b/test/e2e/live/mcp-bridge-sandbox.ts new file mode 100644 index 00000000000..bb9da5af88e --- /dev/null +++ b/test/e2e/live/mcp-bridge-sandbox.ts @@ -0,0 +1,136 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { shellQuote } from "../../../src/lib/core/shell-quote"; +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import type { HostCliClient } from "../fixtures/clients/host.ts"; +import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; + +const MCP_CURL_HTTP_CODE_MARKER = "NEMOCLAW_MCP_CURL_HTTP_CODE="; + +export type McpDnsRebindingAdapter = "mcporter" | "hermes-config" | "deepagents-config"; + +export async function hostAddressForSandbox(host: HostCliClient): Promise { + const probe = await host.command( + "bash", + [ + "-lc", + [ + 'ip_addr="$(ip route get 1.1.1.1 2>/dev/null | awk \'{for (i=1;i<=NF;i++) if ($i=="src") {print $(i+1); exit}}\')"', + 'if [ -n "$ip_addr" ]; then echo "$ip_addr"; exit 0; fi', + "ip_addr=\"$(hostname -I 2>/dev/null | awk '{print $1}')\"", + 'if [ -n "$ip_addr" ]; then echo "$ip_addr"; exit 0; fi', + "echo 127.0.0.1", + ].join("\n"), + ], + { + artifactName: "host-ip-for-mcp-compatible-endpoint", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }, + ); + return probe.stdout.trim().split(/\s+/)[0] || "127.0.0.1"; +} + +export { + type DnsRebindingHostsFixture, + remapDnsRebindingHostname, + restoreDnsRebindingHostsFixture, + setupDnsRebindingHostsFixture, +} from "./dns-rebinding-hosts-fixture.ts"; + +/** + * Accept the two fail-closed shapes OpenShell can expose for a denied HTTPS + * request: an L7 HTTP 403, or curl's exit 56 for a CONNECT-level proxy 403. + */ +export function isExpectedMcpCurlPolicyDenial( + result: Pick, +): boolean { + if (result.timedOut) return false; + + const httpCode = result.stdout.match( + new RegExp(`^${MCP_CURL_HTTP_CODE_MARKER}([0-9]{3})$`, "m"), + )?.[1]; + if (result.exitCode === 0) return httpCode === "403"; + + return ( + result.exitCode === 56 && + /curl:\s*\(56\)\s*CONNECT tunnel failed,\s*response 403/i.test(result.stderr) + ); +} + +/** + * Build an MCP request whose curl child retains the selected adapter runtime + * as an ancestor. OpenShell v0.0.72 attributes policy to /proc//exe and + * ancestors, so this exercises the same unavoidable Node/Python identity used + * by the corresponding adapter instead of an unrelated curl-only identity. + * + * Pinned upstream source contract: + * NVIDIA/OpenShell@8cb16de9eae4c44d7d31e1493747d8c10abb5963, + * crates/openshell-supervisor-network/src/proxy.rs:2476-2502 resolves once, + * :2527-2567 validates that address list, :2622-2630 returns it unchanged, + * and :822-832 passes that same list directly to TcpStream::connect. + */ +export function buildMcpDnsRebindingProbeScript( + adapter: McpDnsRebindingAdapter, + targetUrl: string, + credentialKey: string, +): string { + const fileStem = `/tmp/nemoclaw-mcp-rebinding-${adapter}`; + const responsePath = `${fileStem}.body`; + const stdoutPath = `${fileStem}.stdout`; + const stderrPath = `${fileStem}.stderr`; + const body = JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/list" }); + const curlArgs = [ + "curl", + "-sS", + "--max-time", + "30", + "-o", + responsePath, + "-w", + `${MCP_CURL_HTTP_CODE_MARKER}%{http_code}\n`, + "-X", + "POST", + targetUrl, + "-H", + "content-type: application/json", + "-H", + `authorization: Bearer openshell:resolve:env:${credentialKey}`, + "--data-binary", + body, + ]; + const quotedCurl = curlArgs.map(shellQuote).join(" "); + const runtimeCommand = (() => { + switch (adapter) { + case "mcporter": { + const runner = + 'const { spawnSync } = require("node:child_process"); const result = spawnSync(process.argv[1], process.argv.slice(2), { stdio: "inherit" }); process.exit(result.status ?? 1);'; + return `nemoclaw-start node -e ${shellQuote(runner)} ${quotedCurl}`; + } + case "hermes-config": { + const runner = + "import subprocess, sys; raise SystemExit(subprocess.run(sys.argv[1:], check=False).returncode)"; + return `/opt/hermes/.venv/bin/python -c ${shellQuote(runner)} ${quotedCurl}`; + } + case "deepagents-config": { + const runner = + "import subprocess, sys; raise SystemExit(subprocess.run(sys.argv[1:], check=False).returncode)"; + return `/opt/venv/bin/python3 -c ${shellQuote(runner)} ${quotedCurl}`; + } + } + })(); + + return [ + "set -u", + `rm -f ${shellQuote(responsePath)} ${shellQuote(stdoutPath)} ${shellQuote(stderrPath)}`, + "set +e", + `${runtimeCommand} >${shellQuote(stdoutPath)} 2>${shellQuote(stderrPath)}`, + "probe_rc=$?", + "set -e", + `cat ${shellQuote(responsePath)} 2>/dev/null || true`, + `cat ${shellQuote(stdoutPath)} 2>/dev/null || true`, + `cat ${shellQuote(stderrPath)} >&2 2>/dev/null || true`, + 'exit "$probe_rc"', + ].join("\n"); +} diff --git a/test/e2e/live/mcp-bridge-servers.ts b/test/e2e/live/mcp-bridge-servers.ts new file mode 100644 index 00000000000..c7cf2e7a170 --- /dev/null +++ b/test/e2e/live/mcp-bridge-servers.ts @@ -0,0 +1,630 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { type ChildProcess, spawn } from "node:child_process"; +import fs from "node:fs"; +import http from "node:http"; +import https from "node:https"; +import type { AddressInfo } from "node:net"; +import os from "node:os"; + +import type { CleanupRegistry } from "../fixtures/cleanup.ts"; + +type TestServer = http.Server | https.Server; + +export interface StartedHttpServer { + port: number; + close(): Promise; +} + +export interface FakeMcpHttpsServer extends StartedHttpServer { + setSecret(secret: string): void; + requests: Array<{ + method: string; + path: string; + auth: string; + body: string; + rpcMethod?: string; + }>; +} + +export interface StartedPublicMcpTunnel { + origin: string; + url: string; + close(): Promise; +} + +type TunnelCleanupRegistry = Pick; + +interface McpRequestPayload { + id?: unknown; + method?: unknown; + params?: { name?: unknown; arguments?: { challenge?: unknown } }; +} + +const MCP_NOTIFICATION_METHODS = new Set([ + "notifications/initialized", + "notifications/cancelled", + "notifications/progress", + "notifications/roots/list_changed", + "notifications/elicitation/complete", +]); + +const TRYCLOUDFLARE_ORIGIN_PATTERN = /https:\/\/[a-z0-9-]+\.trycloudflare\.com(?=$|[\s"'\\/])/i; +const QUICK_TUNNEL_ATTEMPTS = 3; +const QUICK_TUNNEL_ATTEMPT_TIMEOUT_MS = 45_000; +const QUICK_TUNNEL_LOG_LIMIT = 32 * 1024; +const CLOUDFLARED_ENV_NAMES = new Set([ + "PATH", + "TMPDIR", + "TMP", + "TEMP", + "LANG", + "HTTP_PROXY", + "HTTPS_PROXY", + "NO_PROXY", + "http_proxy", + "https_proxy", + "no_proxy", + "SSL_CERT_FILE", + "SSL_CERT_DIR", +]); + +const EMPTY_TASK = { + taskId: "fake-task", + status: "completed", + createdAt: "2026-01-01T00:00:00.000Z", + lastUpdatedAt: "2026-01-01T00:00:00.000Z", + ttl: null, +}; + +const MCP_EMPTY_RESULT_BY_METHOD: Record = { + ping: {}, + "resources/list": { resources: [] }, + "resources/read": { contents: [] }, + "resources/templates/list": { resourceTemplates: [] }, + "resources/subscribe": {}, + "resources/unsubscribe": {}, + "prompts/list": { prompts: [] }, + "prompts/get": { messages: [] }, + "tasks/list": { tasks: [] }, + "tasks/get": EMPTY_TASK, + "tasks/update": {}, + "tasks/result": { content: [], isError: false }, + "tasks/cancel": EMPTY_TASK, + "completion/complete": { completion: { values: [] } }, + "logging/setLevel": {}, + "server/discover": { + supportedVersions: ["2025-11-25", "2025-03-26"], + capabilities: { tools: {} }, + serverInfo: { name: "fake", version: "1.0.0" }, + }, + "messages/listen": {}, +}; + +function jsonResponse(res: http.ServerResponse, status: number, payload: unknown): void { + const body = JSON.stringify(payload); + res.writeHead(status, { + "Content-Type": "application/json", + "Content-Length": Buffer.byteLength(body), + }); + res.end(body); +} + +async function readRequestBody(req: http.IncomingMessage): Promise { + return await new Promise((resolve) => { + let body = ""; + req.setEncoding("utf8"); + req.on("data", (chunk: string) => { + body += chunk; + }); + req.on("end", () => resolve(body)); + }); +} + +function requireTcpPort(server: TestServer, label: string): number { + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error(`${label} did not bind to a TCP port`); + } + return (address as AddressInfo).port; +} + +function closeServer(server: TestServer): Promise { + return new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }); +} + +async function listenOnRandomPort(server: TestServer): Promise { + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "0.0.0.0", () => { + server.off("error", reject); + resolve(); + }); + }); +} + +function delay(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function buildCloudflaredSubprocessEnv(): Record { + const env: Record = { + // Do not let quick-tunnel discovery consume a developer's named-tunnel + // credentials or config. The CI runner temp directory is job-isolated. + HOME: process.env.RUNNER_TEMP ?? os.tmpdir(), + XDG_CONFIG_HOME: process.env.RUNNER_TEMP ?? os.tmpdir(), + }; + for (const [name, value] of Object.entries(process.env)) { + if (value === undefined) continue; + if (CLOUDFLARED_ENV_NAMES.has(name) || name.startsWith("LC_")) env[name] = value; + } + return env; +} + +function waitForExit(child: ChildProcess): Promise { + if (child.exitCode !== null || child.signalCode !== null) return Promise.resolve(); + return new Promise((resolve) => { + child.once("close", () => resolve()); + child.once("error", () => resolve()); + }); +} + +function signalProcessGroup(child: ChildProcess, signal: NodeJS.Signals): void { + if (typeof child.pid === "number") { + try { + process.kill(-child.pid, signal); + return; + } catch { + // Fall through to signalling the process leader when no group exists. + } + } + try { + child.kill(signal); + } catch { + // The process already exited. + } +} + +async function stopCloudflared(child: ChildProcess, exited: Promise): Promise { + if (child.exitCode !== null || child.signalCode !== null) return; + signalProcessGroup(child, "SIGTERM"); + const graceful = await Promise.race([exited.then(() => true), delay(5_000).then(() => false)]); + if (graceful) return; + signalProcessGroup(child, "SIGKILL"); + await exited; +} + +export function parseTryCloudflareOrigin(log: string): string | null { + return log.match(TRYCLOUDFLARE_ORIGIN_PATTERN)?.[0] ?? null; +} + +export function buildCloudflaredQuickTunnelArgs(port: number): string[] { + if (!Number.isInteger(port) || port < 1 || port > 65_535) { + throw new Error(`invalid local MCP HTTPS port: ${port}`); + } + return [ + "tunnel", + "--no-autoupdate", + "--protocol", + "http2", + "--url", + `https://127.0.0.1:${port}`, + "--no-tls-verify", + "--loglevel", + "info", + ]; +} + +async function probePublicTunnel(origin: string): Promise<{ + ready: boolean; + diagnostic: string; +}> { + try { + const response = await fetch(`${origin}/mcp`, { + method: "HEAD", + redirect: "manual", + signal: AbortSignal.timeout(5_000), + }); + await response.body?.cancel(); + return { + ready: response.status === 405, + diagnostic: `public HEAD /mcp returned HTTP ${response.status}`, + }; + } catch (error) { + return { + ready: false, + // Avoid reflecting request URLs or child output here. The error class is + // enough to distinguish DNS/transport failure without risking headers. + diagnostic: `public HEAD /mcp failed (${error instanceof Error ? error.name : "unknown error"})`, + }; + } +} + +export async function startPublicMcpHttpsTunnel(options: { + cleanup: TunnelCleanupRegistry; + label: string; + server: StartedHttpServer; + cloudflaredBin?: string; +}): Promise { + const args = buildCloudflaredQuickTunnelArgs(options.server.port); + let lastFailure = "cloudflared did not publish a quick-tunnel URL"; + + for (let attempt = 1; attempt <= QUICK_TUNNEL_ATTEMPTS; attempt += 1) { + let output = ""; + let spawnError: Error | undefined; + const appendOutput = (chunk: string): void => { + output = `${output}${chunk}`.slice(-QUICK_TUNNEL_LOG_LIMIT); + }; + const child = spawn(options.cloudflaredBin ?? "cloudflared", args, { + detached: true, + env: buildCloudflaredSubprocessEnv(), + stdio: ["ignore", "pipe", "pipe"], + }); + const exited = waitForExit(child); + child.stdout?.setEncoding("utf8"); + child.stdout?.on("data", appendOutput); + child.stderr?.setEncoding("utf8"); + child.stderr?.on("data", appendOutput); + child.once("error", (error) => { + spawnError = error; + }); + + let closePromise: Promise | undefined; + const close = (): Promise => { + closePromise ??= stopCloudflared(child, exited); + return closePromise; + }; + const deadline = Date.now() + QUICK_TUNNEL_ATTEMPT_TIMEOUT_MS; + let origin: string | null = null; + + while (Date.now() < deadline) { + if (spawnError) { + lastFailure = spawnError.message; + break; + } + if (child.exitCode !== null || child.signalCode !== null) { + lastFailure = `cloudflared exited before readiness (code=${String(child.exitCode)}, signal=${String(child.signalCode)})`; + break; + } + origin ??= parseTryCloudflareOrigin(output); + if (origin) { + const probe = await probePublicTunnel(origin); + if (probe.ready) { + const tunnel = { + origin, + url: `${origin}/mcp`, + close, + }; + options.cleanup.add(`stop ${options.label} cloudflared quick tunnel`, tunnel.close); + return tunnel; + } + lastFailure = `cloudflared published a quick-tunnel URL but ${probe.diagnostic}`; + } + await delay(500); + } + + await close(); + const diagnostic = output.trim().split("\n").slice(-12).join("\n"); + if (diagnostic) lastFailure = `${lastFailure}\n${diagnostic}`; + if (attempt < QUICK_TUNNEL_ATTEMPTS) await delay(attempt * 1_000); + } + + throw new Error( + `${options.label} public MCP HTTPS tunnel failed after ${QUICK_TUNNEL_ATTEMPTS} attempts: ${lastFailure}`, + ); +} + +export async function startCompatibleMock(options: { + apiKey: string; + model: string; + toolChallenge?: string; + toolResultToken?: string; + toolNames?: string[]; + deferredToolName?: string; +}): Promise { + const server = http.createServer(async (req, res) => { + const requestPath = new URL(req.url ?? "/", "http://compatible.mock").pathname; + const auth = req.headers.authorization === `Bearer ${options.apiKey}`; + if (!auth) { + jsonResponse(res, 401, { error: { message: "missing bearer credential" } }); + return; + } + + if (req.method === "GET" && ["/models", "/v1/models"].includes(requestPath)) { + jsonResponse(res, 200, { + object: "list", + data: [{ id: options.model, object: "model" }], + }); + return; + } + + if ( + req.method === "POST" && + ["/chat/completions", "/v1/chat/completions"].includes(requestPath) + ) { + const body = JSON.parse(await readRequestBody(req)) as { + stream?: boolean; + messages?: Array<{ role?: string; content?: unknown }>; + tools?: Array<{ function?: { name?: string } }>; + }; + const directToolName = body.tools + ?.map((tool) => tool.function?.name) + .find( + (name): name is string => + typeof name === "string" && (options.toolNames ?? []).includes(name), + ); + const deferredToolWrapper = + !directToolName && + options.deferredToolName && + body.tools?.some((tool) => tool.function?.name === "tool_call") + ? "tool_call" + : undefined; + const toolName = directToolName ?? deferredToolWrapper; + const toolArguments = directToolName + ? { challenge: options.toolChallenge } + : { + name: options.deferredToolName, + arguments: { challenge: options.toolChallenge }, + }; + const sawAuthenticatedToolResult = (body.messages ?? []).some( + (message) => + message.role === "tool" && + JSON.stringify(message.content).includes(options.toolResultToken ?? "__never__"), + ); + const responseMessage = sawAuthenticatedToolResult + ? { + role: "assistant", + content: options.toolResultToken, + } + : toolName && options.toolChallenge + ? { + role: "assistant", + content: null, + tool_calls: [ + { + index: 0, + id: "call_mcp_bridge_proof", + type: "function", + function: { + name: toolName, + arguments: JSON.stringify(toolArguments), + }, + }, + ], + } + : { role: "assistant", content: "ok" }; + const finishReason = "tool_calls" in responseMessage ? "tool_calls" : "stop"; + if (body.stream) { + res.writeHead(200, { + "Content-Type": "text/event-stream", + "Cache-Control": "no-cache", + Connection: "keep-alive", + }); + res.write( + `data: ${JSON.stringify({ + id: "chatcmpl-mcp-bridge", + object: "chat.completion.chunk", + created: 0, + model: options.model, + choices: [ + { + index: 0, + delta: responseMessage, + finish_reason: null, + }, + ], + })}\n\n`, + ); + res.write( + `data: ${JSON.stringify({ + id: "chatcmpl-mcp-bridge", + object: "chat.completion.chunk", + created: 0, + model: options.model, + choices: [{ index: 0, delta: {}, finish_reason: finishReason }], + })}\n\n`, + ); + res.end("data: [DONE]\n\n"); + } else { + jsonResponse(res, 200, { + id: "chatcmpl-mcp-bridge", + object: "chat.completion", + created: 0, + model: options.model, + choices: [ + { + index: 0, + message: responseMessage, + finish_reason: finishReason, + }, + ], + }); + } + return; + } + + if (req.method === "POST" && ["/responses", "/v1/responses"].includes(requestPath)) { + await readRequestBody(req); + jsonResponse(res, 200, { + id: "resp-mcp-bridge", + object: "response", + output: [ + { + type: "message", + role: "assistant", + content: [{ type: "output_text", text: "ok" }], + }, + ], + }); + return; + } + + jsonResponse(res, 404, { error: { message: "not found" } }); + }); + + await listenOnRandomPort(server); + return { + port: requireTcpPort(server, "compatible endpoint mock"), + close: () => closeServer(server), + }; +} + +export async function startFakeMcpHttpsServer(options: { + secret: string; + challenge?: string; + resultToken?: string; + tls?: { cert: Buffer; key: Buffer }; +}): Promise { + let expectedSecret = options.secret; + const tls = + options.tls ?? + (() => { + const certPath = process.env.NEMOCLAW_MCP_TLS_CERT; + const keyPath = process.env.NEMOCLAW_MCP_TLS_KEY; + if (!certPath || !keyPath) { + throw new Error( + "NEMOCLAW_MCP_TLS_CERT and NEMOCLAW_MCP_TLS_KEY are required for the HTTPS MCP fixture", + ); + } + return { cert: fs.readFileSync(certPath), key: fs.readFileSync(keyPath) }; + })(); + const requests: Array<{ + method: string; + path: string; + auth: string; + body: string; + }> = []; + const server = https.createServer(tls, async (req, res) => { + const requestPath = new URL(req.url ?? "/", "https://fake-mcp.local").pathname; + const body = await readRequestBody(req); + const auth = Array.isArray(req.headers.authorization) + ? req.headers.authorization.join(",") + : (req.headers.authorization ?? ""); + let parsedPayload: McpRequestPayload | null = null; + try { + parsedPayload = JSON.parse(body) as McpRequestPayload; + } catch { + // The protocol error below handles malformed JSON after recording it. + } + // The public quick-tunnel readiness probe uses HEAD /mcp. Keep it out of + // the protocol request ledger so zero-upstream decoy and policy-denial + // assertions continue to measure only attempted MCP traffic. + if (req.method !== "HEAD") { + requests.push({ + method: req.method ?? "", + path: requestPath, + auth, + body, + ...(typeof parsedPayload?.method === "string" ? { rpcMethod: parsedPayload.method } : {}), + }); + } + if (requestPath !== "/mcp") { + jsonResponse(res, 404, { error: { message: "not found" } }); + return; + } + if (req.method === "HEAD" || req.method === "GET") { + res.writeHead(405, { Allow: "POST" }); + res.end(); + return; + } + if (req.method !== "POST") { + jsonResponse(res, 405, { error: { message: "method not allowed" } }); + return; + } + if (auth !== `Bearer ${expectedSecret}`) { + jsonResponse(res, 401, { error: { message: "missing rewritten bearer credential" } }); + return; + } + + if (!parsedPayload) { + jsonResponse(res, 400, { error: { message: "invalid json" } }); + return; + } + if ( + typeof parsedPayload.method === "string" && + MCP_NOTIFICATION_METHODS.has(parsedPayload.method) + ) { + res.writeHead(202); + res.end(); + return; + } + let result: unknown; + if (parsedPayload.method === "initialize") { + const request = JSON.parse(body) as { + params?: { protocolVersion?: string }; + }; + result = { + protocolVersion: request.params?.protocolVersion ?? "2025-03-26", + capabilities: { tools: {} }, + serverInfo: { name: "fake", version: "1.0.0" }, + }; + } else if (parsedPayload.method === "tools/list") { + result = { + tools: [ + { + name: "fake_echo", + description: "Returns an authenticated MCP proof token", + inputSchema: { + type: "object", + properties: { challenge: { type: "string" } }, + required: ["challenge"], + additionalProperties: false, + }, + }, + ], + }; + } else if (parsedPayload.method === "tools/call") { + const challenge = parsedPayload.params?.arguments?.challenge; + if ( + parsedPayload.params?.name !== "fake_echo" || + (options.challenge !== undefined && challenge !== options.challenge) + ) { + jsonResponse(res, 200, { + jsonrpc: "2.0", + id: parsedPayload.id ?? 1, + error: { code: -32602, message: "invalid fake_echo challenge" }, + }); + return; + } + result = { + content: [ + { + type: "text", + text: options.resultToken ?? `MCP_AUTH_REWRITE_OK::${String(challenge ?? "")}`, + }, + ], + isError: false, + }; + } else if ( + typeof parsedPayload.method === "string" && + Object.prototype.hasOwnProperty.call(MCP_EMPTY_RESULT_BY_METHOD, parsedPayload.method) + ) { + result = MCP_EMPTY_RESULT_BY_METHOD[parsedPayload.method]; + } else { + jsonResponse(res, 200, { + jsonrpc: "2.0", + id: parsedPayload.id ?? 1, + error: { code: -32601, message: "method not found" }, + }); + return; + } + jsonResponse(res, 200, { + jsonrpc: "2.0", + id: parsedPayload.id ?? 1, + result, + }); + }); + + await listenOnRandomPort(server); + return { + port: requireTcpPort(server, "fake MCP endpoint"), + requests, + setSecret: (secret: string) => { + expectedSecret = secret; + }, + close: () => closeServer(server), + }; +} diff --git a/test/e2e/live/mcp-bridge.test.ts b/test/e2e/live/mcp-bridge.test.ts new file mode 100644 index 00000000000..0c2403793c6 --- /dev/null +++ b/test/e2e/live/mcp-bridge.test.ts @@ -0,0 +1,1500 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import YAML from "yaml"; + +import { + buildDeepAgentsMcpStatusCommand, + buildHermesMcpStatusCommand, + buildOpenClawMcporterInspectCommand, +} from "../../../src/lib/actions/sandbox/mcp-bridge-adapter-status"; +import { shellQuote } from "../../../src/lib/core/shell-quote"; +import { parseOpenShellPolicy } from "../../../src/lib/policy/merge"; +import type { McpBridgeEntry } from "../../../src/lib/state/registry"; +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import type { CleanupRegistry } from "../fixtures/cleanup.ts"; +import type { HostCliClient } from "../fixtures/clients/host.ts"; +import { type SandboxClient, trustedSandboxShellScript } from "../fixtures/clients/sandbox.ts"; +import { expect, test } from "../fixtures/e2e-test.ts"; +import { MCP_BRIDGE_TEST_CREDENTIALS } from "../fixtures/mcp-bridge-credentials.ts"; +import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; +import { + buildMcpDnsRebindingProbeScript, + hostAddressForSandbox, + isExpectedMcpCurlPolicyDenial, + type McpDnsRebindingAdapter, + remapDnsRebindingHostname, + restoreDnsRebindingHostsFixture, + setupDnsRebindingHostsFixture, +} from "./mcp-bridge-sandbox.ts"; +import { + startCompatibleMock, + startFakeMcpHttpsServer, + startPublicMcpHttpsTunnel, +} from "./mcp-bridge-servers.ts"; +import { assertRawOpenShellAllowedIpsRebindingDenied } from "./openshell-allowed-ips-rebinding.ts"; + +const OPENCLAW_SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-mcp-bridge"; +const HERMES_SANDBOX_NAME = process.env.NEMOCLAW_MCP_HERMES_SANDBOX_NAME ?? "e2e-mcp-hermes"; +const DEEPAGENTS_SANDBOX_NAME = process.env.NEMOCLAW_MCP_DEEPAGENTS_SANDBOX_NAME ?? "e2e-mcp-dcode"; +const SERVER_NAME = "fake"; +const SERVER_POLICY_KEY = "mcp_bridge_fake"; +const CONCURRENT_SERVER_NAME = "concurrent"; +const REBIND_SERVER_NAME = "rebind"; +const REBIND_POLICY_KEY = "mcp_bridge_rebind"; +const REBIND_HOSTNAME = "mcp-rebind.example.test"; +const REBIND_PUBLIC_IP = "1.1.1.1"; +const REBIND_CREDENTIAL_KEY = "REBIND_MCP_SECRET"; +const HOST_SECRET = MCP_BRIDGE_TEST_CREDENTIALS.host; +const ROTATED_HOST_SECRET = MCP_BRIDGE_TEST_CREDENTIALS.rotatedHost; +const REBIND_HOST_SECRET = MCP_BRIDGE_TEST_CREDENTIALS.rebindHost; +const COMPATIBLE_KEY = MCP_BRIDGE_TEST_CREDENTIALS.compatibleEndpoint; +const COMPATIBLE_MODEL = "mock/mcp-bridge"; +const TOOL_CHALLENGE = "nemoclaw-authenticated-mcp-proof"; +const REGISTRY_FILE = path.join(process.env.HOME ?? os.homedir(), ".nemoclaw", "sandboxes.json"); +const liveTest = process.env.NEMOCLAW_RUN_LIVE_E2E === "1" ? test : test.skip; +const liveAgentMatrixTest = + process.env.NEMOCLAW_RUN_LIVE_E2E === "1" && process.env.NEMOCLAW_MCP_BRIDGE_AGENT_MATRIX === "1" + ? test + : test.skip; + +type McpAgent = "openclaw" | "hermes" | "langchain-deepagents-code"; +type McpAdapter = "mcporter" | "hermes-config" | "deepagents-config"; +const MCP_MUTATION_TIMEOUT_MS: Record = { + "deepagents-config": 3 * 60_000, + "hermes-config": 12 * 60_000, + mcporter: 3 * 60_000, +}; + +function resultText(result: ShellProbeResult): string { + return [result.stdout, result.stderr].filter(Boolean).join("\n"); +} + +function expectExitZero(result: ShellProbeResult, label: string): void { + expect(result.exitCode, `${label}\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`).toBe(0); +} + +function expectExitNonZero(result: ShellProbeResult, label: string, pattern: RegExp): void { + expect( + result.exitCode, + `${label}\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`, + ).not.toBe(0); + expect(resultText(result)).toMatch(pattern); +} + +function parseCurrentPolicy(raw: string): string { + return parseOpenShellPolicy(raw).yamlBody; +} + +async function bestEffortRemoveBridge( + host: HostCliClient, + sandboxName: string, + server: string, + adapter: McpAdapter, +): Promise { + await host.nemoclaw([sandboxName, "mcp", "remove", server, "--force"], { + artifactName: `cleanup-mcp-remove-${server}`, + env: buildAvailabilityProbeEnv(), + timeoutMs: MCP_MUTATION_TIMEOUT_MS[adapter], + }); +} + +async function cleanupSandbox(host: HostCliClient, sandboxName: string): Promise { + await host.bestEffortCleanupSandbox(sandboxName, { + artifactName: "cleanup-destroy-sandbox", + timeoutMs: 15 * 60_000, + }); +} + +async function onboardAgent( + host: HostCliClient, + cleanup: CleanupRegistry, + endpointUrl: string, + options: { agent: McpAgent; sandboxName: string; artifactName: string }, +): Promise { + cleanup.add(`destroy MCP bridge ${options.agent} sandbox`, () => + cleanupSandbox(host, options.sandboxName), + ); + await host.cleanupSandbox(options.sandboxName, { + artifactName: "precleanup-destroy-sandbox", + timeoutMs: 15 * 60_000, + }); + const result = await host.nemoclaw( + ["onboard", "--non-interactive", "--yes", "--yes-i-accept-third-party-software"], + { + artifactName: options.artifactName, + env: { + ...buildAvailabilityProbeEnv(), + COMPATIBLE_API_KEY: COMPATIBLE_KEY, + NVIDIA_INFERENCE_API_KEY: COMPATIBLE_KEY, + NEMOCLAW_AGENT: options.agent, + NEMOCLAW_ENDPOINT_URL: endpointUrl, + NEMOCLAW_MODEL: COMPATIBLE_MODEL, + NEMOCLAW_COMPAT_MODEL: COMPATIBLE_MODEL, + NEMOCLAW_PREFERRED_API: "openai-completions", + NEMOCLAW_PROVIDER: "custom", + NEMOCLAW_SANDBOX_NAME: options.sandboxName, + NEMOCLAW_RECREATE_SANDBOX: "1", + }, + redactionValues: [COMPATIBLE_KEY], + timeoutMs: 20 * 60_000, + }, + ); + expectExitZero(result, `onboard ${options.agent} sandbox for MCP bridge`); +} + +async function assertSecretAbsentFromSandbox( + sandbox: SandboxClient, + sandboxName: string, + paths: string[], + secrets: string[] = [HOST_SECRET], + artifactName = "assert-secret-absent-from-sandbox", +): Promise { + const script = [ + "set -eu", + ...secrets.map( + (secret) => `! grep -R ${JSON.stringify(secret)} ${paths.join(" ")} 2>/dev/null`, + ), + ].join("\n"); + const result = await sandbox.execShell(sandboxName, trustedSandboxShellScript(script), { + artifactName, + env: buildAvailabilityProbeEnv(), + redactionValues: [...secrets, Buffer.from(script, "utf8").toString("base64")], + timeoutMs: 60_000, + }); + expectExitZero(result, "host MCP secret must not appear in sandbox files"); +} + +async function assertAdapterDnsRebindingDenied( + host: HostCliClient, + sandbox: SandboxClient, + cleanup: CleanupRegistry, + options: { + adapter: McpDnsRebindingAdapter; + artifactPrefix: string; + hostAddress: string; + sandboxName: string; + secretPaths: string[]; + }, +): Promise { + const rebindMcp = await startFakeMcpHttpsServer({ + secret: REBIND_HOST_SECRET, + }); + cleanup.add(`stop ${options.artifactPrefix} DNS rebinding fake MCP HTTPS server`, () => + rebindMcp.close(), + ); + cleanup.add(`remove ${options.artifactPrefix} DNS rebinding MCP bridge`, () => + bestEffortRemoveBridge(host, options.sandboxName, REBIND_SERVER_NAME, options.adapter), + ); + const rebindMcpUrl = `https://${REBIND_HOSTNAME}:${rebindMcp.port}/mcp`; + const hostsFixture = await setupDnsRebindingHostsFixture( + host, + options.sandboxName, + REBIND_HOSTNAME, + ); + cleanup.add(`restore ${options.artifactPrefix} DNS rebinding hosts fixture`, () => + restoreDnsRebindingHostsFixture(host, options.sandboxName, hostsFixture), + ); + + await remapDnsRebindingHostname( + host, + options.sandboxName, + hostsFixture, + REBIND_PUBLIC_IP, + `${options.artifactPrefix}-mcp-dns-rebinding-map-public-before-add`, + ); + const add = await host.nemoclaw( + [ + options.sandboxName, + "mcp", + "add", + REBIND_SERVER_NAME, + "--url", + rebindMcpUrl, + "--env", + REBIND_CREDENTIAL_KEY, + ], + { + artifactName: `${options.artifactPrefix}-mcp-dns-rebinding-add-with-public-resolution`, + env: { + ...buildAvailabilityProbeEnv(), + [REBIND_CREDENTIAL_KEY]: REBIND_HOST_SECRET, + }, + redactionValues: [REBIND_HOST_SECRET], + timeoutMs: MCP_MUTATION_TIMEOUT_MS[options.adapter], + }, + ); + expectExitZero( + add, + `${options.artifactPrefix} registers MCP route while its dedicated hostname resolves publicly`, + ); + + const status = await host.nemoclaw( + [options.sandboxName, "mcp", "status", REBIND_SERVER_NAME, "--json"], + { + artifactName: `${options.artifactPrefix}-mcp-dns-rebinding-status-after-add`, + env: { + ...buildAvailabilityProbeEnv(), + [REBIND_CREDENTIAL_KEY]: REBIND_HOST_SECRET, + }, + redactionValues: [REBIND_HOST_SECRET], + timeoutMs: 60_000, + }, + ); + expectExitZero(status, `${options.artifactPrefix} inspects DNS rebinding route after add`); + expect(JSON.parse(status.stdout)).toMatchObject({ + support: { supported: true, adapter: options.adapter }, + server: REBIND_SERVER_NAME, + url: rebindMcpUrl, + env: { names: [REBIND_CREDENTIAL_KEY], ready: true, missing: [] }, + provider: { attached: true, credentialReady: true }, + policy: { gatewayPresent: true }, + adapter: { registered: true }, + }); + + const policy = await sandbox.openshell(["policy", "get", "--full", options.sandboxName], { + artifactName: `${options.artifactPrefix}-mcp-dns-rebinding-policy-pinned-public-ip`, + env: buildAvailabilityProbeEnv(), + timeoutMs: 60_000, + }); + expectExitZero(policy, `${options.artifactPrefix} inspects add-time DNS pin`); + const policyJson = YAML.parse(parseCurrentPolicy(resultText(policy))) as { + network_policies?: Record< + string, + { endpoints?: Array<{ host?: string; allowed_ips?: string[] }> } + >; + }; + expect(policyJson.network_policies?.[REBIND_POLICY_KEY]?.endpoints?.[0]).toMatchObject({ + host: REBIND_HOSTNAME, + allowed_ips: [REBIND_PUBLIC_IP], + }); + await assertSecretAbsentFromSandbox( + sandbox, + options.sandboxName, + options.secretPaths, + [REBIND_HOST_SECRET], + `${options.artifactPrefix}-dns-rebinding-secret-absent-from-sandbox`, + ); + + // If OpenShell resolved a second time after validating allowed_ips, this + // reachable runner address would receive the request. The pinned v0.0.72 + // implementation instead returns the one resolved-and-validated SocketAddr + // list directly to connect; see the exact proxy.rs citation in the helper. + expect(options.hostAddress).not.toBe(REBIND_PUBLIC_IP); + await remapDnsRebindingHostname( + host, + options.sandboxName, + hostsFixture, + options.hostAddress, + `${options.artifactPrefix}-mcp-dns-rebinding-map-private-unpinned-after-add`, + ); + const denial = await sandbox.execShell( + options.sandboxName, + trustedSandboxShellScript( + buildMcpDnsRebindingProbeScript(options.adapter, rebindMcpUrl, REBIND_CREDENTIAL_KEY), + ), + { + artifactName: `${options.artifactPrefix}-mcp-dns-rebinding-adapter-denied`, + env: buildAvailabilityProbeEnv(), + redactionValues: [REBIND_HOST_SECRET], + timeoutMs: 90_000, + }, + ); + expect( + isExpectedMcpCurlPolicyDenial(denial), + `${options.artifactPrefix} adapter identity must receive an OpenShell policy denial after rebinding\nstdout:\n${denial.stdout}\nstderr:\n${denial.stderr}`, + ).toBe(true); + expect( + rebindMcp.requests, + `${options.artifactPrefix} rebound request must not reach the upstream MCP server`, + ).toHaveLength(0); + + // Restore while the current sandbox container is stable. Removing the MCP + // route reloads policy and can restart the container first; the registered + // cleanup remains an idempotent fallback. + await restoreDnsRebindingHostsFixture(host, options.sandboxName, hostsFixture); + const remove = await host.nemoclaw([options.sandboxName, "mcp", "remove", REBIND_SERVER_NAME], { + artifactName: `${options.artifactPrefix}-mcp-dns-rebinding-remove`, + env: buildAvailabilityProbeEnv(), + timeoutMs: MCP_MUTATION_TIMEOUT_MS[options.adapter], + }); + expectExitZero(remove, `${options.artifactPrefix} removes DNS rebinding route after proof`); +} + +async function addBridgeAndReadStatus( + host: HostCliClient, + options: { + sandboxName: string; + mcpUrl: string; + expectedAdapter: McpAdapter; + artifactPrefix: string; + }, +): Promise { + const add = await host.nemoclaw( + [ + options.sandboxName, + "mcp", + "add", + SERVER_NAME, + "--url", + options.mcpUrl, + "--env", + "FAKE_MCP_SECRET", + ], + { + artifactName: `${options.artifactPrefix}-mcp-add-fake-server`, + env: { + ...buildAvailabilityProbeEnv(), + FAKE_MCP_SECRET: HOST_SECRET, + }, + redactionValues: [HOST_SECRET], + timeoutMs: MCP_MUTATION_TIMEOUT_MS[options.expectedAdapter], + }, + ); + expectExitZero(add, `${options.artifactPrefix} mcp add fake server`); + + const status = await host.nemoclaw( + [options.sandboxName, "mcp", "status", SERVER_NAME, "--json"], + { + artifactName: `${options.artifactPrefix}-mcp-status-json`, + env: { + ...buildAvailabilityProbeEnv(), + FAKE_MCP_SECRET: HOST_SECRET, + }, + redactionValues: [HOST_SECRET], + timeoutMs: 60_000, + }, + ); + expectExitZero(status, `${options.artifactPrefix} mcp status --json`); + const statusJson = JSON.parse(status.stdout) as { + support: { supported: boolean; adapter: string }; + server: string; + url: string; + warnings: string[]; + env: { names: string[]; ready: boolean; missing: string[] }; + provider: { + name: string; + gatewayPresent: boolean | null; + attached: boolean | null; + }; + policy: { gatewayPresent: boolean | null }; + adapter: { registered: boolean | null }; + }; + expect(statusJson.support).toMatchObject({ + supported: true, + adapter: options.expectedAdapter, + }); + expect(statusJson).toMatchObject({ + server: SERVER_NAME, + url: options.mcpUrl, + env: { names: ["FAKE_MCP_SECRET"], ready: true, missing: [] }, + provider: { gatewayPresent: true, attached: true }, + policy: { gatewayPresent: true }, + adapter: { registered: true }, + }); + expect(statusJson.warnings).toEqual([ + expect.stringMatching(/provider at sandbox scope.*endpoint-exclusive credential binding/i), + ]); + expect(status.stdout).not.toContain(HOST_SECRET); + expect(statusJson.provider.name).toMatch( + new RegExp(`^${options.sandboxName}-mcp-${SERVER_NAME}-[a-f0-9]{16}$`), + ); + return statusJson.provider.name; +} + +async function assertConcurrentAddSerialized( + host: HostCliClient, + cleanup: CleanupRegistry, + options: { + sandboxName: string; + mcpUrl: string; + expectedAdapter: McpAdapter; + artifactPrefix: string; + }, +): Promise { + cleanup.add(`remove ${options.artifactPrefix} concurrent MCP bridge`, () => + bestEffortRemoveBridge( + host, + options.sandboxName, + CONCURRENT_SERVER_NAME, + options.expectedAdapter, + ), + ); + const args = [ + options.sandboxName, + "mcp", + "add", + CONCURRENT_SERVER_NAME, + "--url", + options.mcpUrl, + "--env", + "FAKE_MCP_SECRET", + ]; + const env = { + ...buildAvailabilityProbeEnv(), + FAKE_MCP_SECRET: HOST_SECRET, + }; + const attempts = await Promise.all( + ["first", "second"].map((attempt) => + host.nemoclaw(args, { + artifactName: `${options.artifactPrefix}-mcp-concurrent-add-${attempt}`, + env, + redactionValues: [HOST_SECRET], + // Hermes may need one host-authenticated managed restart (210s), a + // fresh helper-readiness window (90s), and its acknowledged config + // reload (300s). Keep both concurrent clients alive through that + // bounded recovery; the loser then acquires the lifecycle lock and + // rejects the committed duplicate. + timeoutMs: MCP_MUTATION_TIMEOUT_MS[options.expectedAdapter], + }), + ), + ); + const successful = attempts.filter((result) => result.exitCode === 0); + const rejected = attempts.filter((result) => result.exitCode !== 0); + expect(successful).toHaveLength(1); + expect(rejected).toHaveLength(1); + expectExitNonZero( + rejected[0]!, + `${options.artifactPrefix} concurrent MCP add rejects the serialized duplicate`, + /already exists/, + ); + const status = await host.nemoclaw( + [options.sandboxName, "mcp", "status", CONCURRENT_SERVER_NAME, "--json"], + { + artifactName: `${options.artifactPrefix}-mcp-concurrent-add-coherent-status`, + env, + redactionValues: [HOST_SECRET], + timeoutMs: 60_000, + }, + ); + expectExitZero(status, `${options.artifactPrefix} concurrent add leaves one coherent bridge`); + expect(JSON.parse(status.stdout)).toMatchObject({ + server: CONCURRENT_SERVER_NAME, + url: options.mcpUrl, + support: { adapter: options.expectedAdapter }, + env: { names: ["FAKE_MCP_SECRET"], ready: true, missing: [] }, + provider: { + registryPresent: true, + gatewayPresent: true, + attached: true, + credentialReady: true, + }, + policy: { registryPresent: true, gatewayPresent: true }, + adapter: { registered: true }, + }); + const remove = await host.nemoclaw( + [options.sandboxName, "mcp", "remove", CONCURRENT_SERVER_NAME], + { + artifactName: `${options.artifactPrefix}-mcp-concurrent-add-remove`, + env: buildAvailabilityProbeEnv(), + // Adapter removal performs the same acknowledged config reload as add. + timeoutMs: MCP_MUTATION_TIMEOUT_MS[options.expectedAdapter], + }, + ); + expectExitZero(remove, `${options.artifactPrefix} removes concurrent MCP bridge`); + const list = await host.nemoclaw([options.sandboxName, "mcp", "list", "--json"], { + artifactName: `${options.artifactPrefix}-mcp-concurrent-add-list-after-remove`, + env: buildAvailabilityProbeEnv(), + timeoutMs: 60_000, + }); + expectExitZero(list, `${options.artifactPrefix} lists after concurrent bridge removal`); + expect(JSON.parse(list.stdout).bridges).toEqual([]); +} + +async function expectMcpCliFailure( + host: HostCliClient, + sandboxName: string, + args: string[], + pattern: RegExp, + artifactName: string, + env: NodeJS.ProcessEnv = buildAvailabilityProbeEnv(), +): Promise { + const result = await host.nemoclaw([sandboxName, "mcp", ...args], { + artifactName, + env, + redactionValues: [HOST_SECRET], + timeoutMs: 60_000, + }); + expectExitNonZero(result, artifactName, pattern); +} + +async function assertBridgeInfrastructure( + host: HostCliClient, + sandbox: SandboxClient, + options: { + sandboxName: string; + artifactPrefix: string; + providerName: string; + mcpUrl: string; + }, +): Promise { + const policy = await sandbox.openshell(["policy", "get", "--full", options.sandboxName], { + artifactName: `${options.artifactPrefix}-openshell-policy-get-mcp`, + env: buildAvailabilityProbeEnv(), + timeoutMs: 60_000, + }); + expectExitZero(policy, `${options.artifactPrefix} openshell policy get --full`); + expect(resultText(policy)).toContain(SERVER_POLICY_KEY); + expect(resultText(policy)).toContain("protocol: mcp"); + expect(resultText(policy)).not.toContain("tls: require"); + expect(resultText(policy)).not.toContain("credential_keys"); + expect(resultText(policy)).not.toContain("FAKE_MCP_SECRET"); + expect(resultText(policy)).toContain("strict_tool_names"); + expect(resultText(policy)).toContain("method: tools/list"); + expect(resultText(policy)).toContain("method: tools/call"); + expect(resultText(policy)).toContain(new URL(options.mcpUrl).hostname); + const provider = await host.command("openshell", ["provider", "get", options.providerName], { + artifactName: `${options.artifactPrefix}-openshell-provider-get-mcp`, + env: buildAvailabilityProbeEnv(), + timeoutMs: 60_000, + }); + expectExitZero(provider, `${options.artifactPrefix} openshell provider get mcp provider`); + expect(resultText(provider)).toContain("FAKE_MCP_SECRET"); + expect(resultText(provider)).not.toContain(HOST_SECRET); +} + +async function removeBridgeAndAssertEmpty( + host: HostCliClient, + sandbox: SandboxClient, + options: { + agent: McpAgent; + adapter: McpAdapter; + sandboxName: string; + artifactPrefix: string; + providerName: string; + mcpUrl: string; + }, +): Promise { + const remove = await host.nemoclaw([options.sandboxName, "mcp", "remove", SERVER_NAME], { + artifactName: `${options.artifactPrefix}-mcp-remove-fake-server`, + env: buildAvailabilityProbeEnv(), + timeoutMs: MCP_MUTATION_TIMEOUT_MS[options.adapter], + }); + expectExitZero(remove, `${options.artifactPrefix} mcp remove fake server`); + const list = await host.nemoclaw([options.sandboxName, "mcp", "list", "--json"], { + artifactName: `${options.artifactPrefix}-mcp-list-after-remove`, + env: buildAvailabilityProbeEnv(), + timeoutMs: 60_000, + }); + expectExitZero(list, `${options.artifactPrefix} mcp list after remove`); + expect(JSON.parse(list.stdout).bridges).toEqual([]); + const provider = await host.command("openshell", ["provider", "get", options.providerName], { + artifactName: `${options.artifactPrefix}-provider-absent-after-mcp-remove`, + env: buildAvailabilityProbeEnv(), + timeoutMs: 60_000, + }); + expectExitNonZero( + provider, + `${options.artifactPrefix} provider absent after remove`, + /not found/i, + ); + const attachments = await host.command( + "openshell", + ["sandbox", "provider", "list", options.sandboxName], + { + artifactName: `${options.artifactPrefix}-provider-detached-after-mcp-remove`, + env: buildAvailabilityProbeEnv(), + timeoutMs: 60_000, + }, + ); + expectExitZero(attachments, `${options.artifactPrefix} provider list after remove`); + expect(resultText(attachments)).not.toContain(options.providerName); + const policy = await sandbox.openshell(["policy", "get", "--full", options.sandboxName], { + artifactName: `${options.artifactPrefix}-policy-absent-after-mcp-remove`, + env: buildAvailabilityProbeEnv(), + timeoutMs: 60_000, + }); + expectExitZero(policy, `${options.artifactPrefix} policy after remove`); + expect(resultText(policy)).not.toMatch(/mcp[-_]bridge[-_]fake/); + const entry: McpBridgeEntry = { + server: SERVER_NAME, + agent: options.agent, + adapter: options.adapter, + url: options.mcpUrl, + env: ["FAKE_MCP_SECRET"], + providerName: options.providerName, + policyName: "mcp-bridge-fake", + addedAt: "2026-06-01T00:00:00.000Z", + }; + const adapterStatusCommand = + options.adapter === "mcporter" + ? buildOpenClawMcporterInspectCommand(entry, true) + : options.adapter === "hermes-config" + ? buildHermesMcpStatusCommand(entry) + : buildDeepAgentsMcpStatusCommand(entry); + const adapterStatus = await sandbox.execShell( + options.sandboxName, + trustedSandboxShellScript(["set -eu", adapterStatusCommand].join("\n")), + { + artifactName: `${options.artifactPrefix}-adapter-absent-after-mcp-remove`, + env: buildAvailabilityProbeEnv(), + timeoutMs: 60_000, + }, + ); + expectExitZero(adapterStatus, `${options.artifactPrefix} adapter status after remove`); + expect(resultText(adapterStatus)).toMatch(/(?:^|\n)absent(?:\n|$)/); +} +async function assertAdapterRequestDeniedAfterRemove( + sandbox: SandboxClient, + fakeMcp: Awaited>, + options: { + adapter: McpDnsRebindingAdapter; + sandboxName: string; + mcpUrl: string; + artifactPrefix: string; + }, +): Promise { + const requestCount = fakeMcp.requests.length; + const denial = await sandbox.execShell( + options.sandboxName, + trustedSandboxShellScript( + buildMcpDnsRebindingProbeScript(options.adapter, options.mcpUrl, "FAKE_MCP_SECRET"), + ), + { + artifactName: `${options.artifactPrefix}-mcp-adapter-request-denied-after-remove`, + env: buildAvailabilityProbeEnv(), + redactionValues: [HOST_SECRET, ROTATED_HOST_SECRET], + timeoutMs: 90_000, + }, + ); + expect( + isExpectedMcpCurlPolicyDenial(denial), + `${options.artifactPrefix} adapter identity must receive an OpenShell policy denial after remove\nstdout:\n${denial.stdout}\nstderr:\n${denial.stderr}`, + ).toBe(true); + expect(fakeMcp.requests).toHaveLength(requestCount); +} +async function assertHermesConfig( + sandbox: SandboxClient, + sandboxName: string, + mcpUrl: string, +): Promise { + const script = [ + "set -eu", + "/opt/hermes/.venv/bin/python - <<'PY'", + "import pathlib, yaml", + "path = pathlib.Path('/sandbox/.hermes/config.yaml')", + "text = path.read_text(encoding='utf-8')", + "data = yaml.safe_load(text) or {}", + `entry = data['mcp_servers'][${JSON.stringify(SERVER_NAME)}]`, + `assert entry['url'] == ${JSON.stringify(mcpUrl)}`, + "assert entry['headers']['Authorization'] == 'Bearer openshell:resolve:env:FAKE_MCP_SECRET'", + `assert ${JSON.stringify(HOST_SECRET)} not in text`, + "PY", + ].join("\n"); + const result = await sandbox.execShell(sandboxName, trustedSandboxShellScript(script), { + artifactName: "hermes-mcp-config-assertions", + env: buildAvailabilityProbeEnv(), + redactionValues: [HOST_SECRET, Buffer.from(script, "utf8").toString("base64")], + timeoutMs: 60_000, + }); + expectExitZero(result, "Hermes MCP config contains placeholder and no raw host secret"); +} +async function assertDeepAgentsConfig( + sandbox: SandboxClient, + sandboxName: string, + mcpUrl: string, +): Promise { + const script = [ + "set -eu", + "python3 - <<'PY'", + "import json, pathlib", + "path = pathlib.Path('/sandbox/.deepagents/.mcp.json')", + "text = path.read_text(encoding='utf-8')", + "data = json.loads(text)", + `entry = data['mcpServers'][${JSON.stringify(SERVER_NAME)}]`, + "assert entry['type'] == 'http'", + `assert entry['url'] == ${JSON.stringify(mcpUrl)}`, + "assert entry['headers']['Authorization'] == 'Bearer openshell:resolve:env:FAKE_MCP_SECRET'", + `assert ${JSON.stringify(HOST_SECRET)} not in text`, + "PY", + ].join("\n"); + const result = await sandbox.execShell(sandboxName, trustedSandboxShellScript(script), { + artifactName: "deepagents-mcp-config-assertions", + env: buildAvailabilityProbeEnv(), + redactionValues: [HOST_SECRET, Buffer.from(script, "utf8").toString("base64")], + timeoutMs: 60_000, + }); + expectExitZero(result, "Deep Agents MCP config contains placeholder and no raw host secret"); +} + +async function assertAuthenticatedMcpDiscovery( + fakeMcp: Awaited>, + options: { + requestOffset: number; + expectedSecret: string; + label: string; + }, +): Promise { + await expect + .poll( + () => { + const requests = fakeMcp.requests.slice(options.requestOffset); + const observed = (rpcMethod: "initialize" | "tools/list") => + requests.some( + (request) => + request.method === "POST" && + request.path === "/mcp" && + request.rpcMethod === rpcMethod && + request.auth === `Bearer ${options.expectedSecret}`, + ); + return { + initialized: observed("initialize"), + toolsListed: observed("tools/list"), + requests: requests.map((request) => ({ + method: request.method, + path: request.path, + rpcMethod: request.rpcMethod, + credentialRewritten: request.auth === `Bearer ${options.expectedSecret}`, + })), + }; + }, + { interval: 500, timeout: 90_000, message: options.label }, + ) + .toMatchObject({ initialized: true, toolsListed: true }); +} + +async function assertRealAdapterToolCall( + sandbox: SandboxClient, + fakeMcp: Awaited>, + options: { + agent: McpAgent; + sandboxName: string; + resultToken: string; + artifactName: string; + expectedSecret?: string; + }, +): Promise { + const before = fakeMcp.requests.filter((request) => request.rpcMethod === "tools/call").length; + const prompt = `Call the fake MCP tool exactly once with challenge ${TOOL_CHALLENGE} and return its result verbatim.`; + const hermesPayload = JSON.stringify({ + model: COMPATIBLE_MODEL, + messages: [{ role: "user", content: prompt }], + max_tokens: 256, + }); + const command = + options.agent === "openclaw" + ? `nemoclaw-start mcporter call fake.fake_echo --args ${JSON.stringify(JSON.stringify({ challenge: TOOL_CHALLENGE }))} --output json` + : options.agent === "hermes" + ? [ + "set -a", + "[ ! -f /sandbox/.hermes/.env ] || . /sandbox/.hermes/.env", + "set +a", + `if [ -n "\${API_SERVER_KEY:-}" ]; then curl -fsS --max-time 180 http://localhost:8642/v1/chat/completions -H 'Content-Type: application/json' -H "Authorization: Bearer \${API_SERVER_KEY}" --data-binary ${shellQuote(hermesPayload)}; else curl -fsS --max-time 180 http://localhost:8642/v1/chat/completions -H 'Content-Type: application/json' --data-binary ${shellQuote(hermesPayload)}; fi`, + ].join("\n") + : `nemoclaw-start dcode -n ${JSON.stringify(prompt)}`; + const result = await sandbox.execShell( + options.sandboxName, + trustedSandboxShellScript(["set -eu", command].join("\n")), + { + artifactName: options.artifactName, + env: buildAvailabilityProbeEnv(), + timeoutMs: 5 * 60_000, + }, + ); + expectExitZero(result, `${options.agent} real MCP tool call`); + expect(resultText(result)).toContain(options.resultToken); + const calls = fakeMcp.requests.filter((request) => request.rpcMethod === "tools/call"); + expect(calls).toHaveLength(before + 1); + expect(calls.at(-1)).toMatchObject({ + auth: `Bearer ${options.expectedSecret ?? HOST_SECRET}`, + path: "/mcp", + }); + expect(calls.at(-1)?.auth).not.toContain("openshell:resolve:env"); +} + +async function rotateBridgeCredential( + host: HostCliClient, + sandboxName: string, + artifactPrefix: string, +): Promise { + const restart = await host.nemoclaw([sandboxName, "mcp", "restart", SERVER_NAME], { + artifactName: `${artifactPrefix}-mcp-rotate-provider-credential`, + env: { + ...buildAvailabilityProbeEnv(), + FAKE_MCP_SECRET: ROTATED_HOST_SECRET, + }, + redactionValues: [HOST_SECRET, ROTATED_HOST_SECRET], + timeoutMs: 12 * 60_000, + }); + expectExitZero(restart, `${artifactPrefix} mcp credential rotation`); +} + +async function restartBridgeWithoutHostSecret( + host: HostCliClient, + sandboxName: string, + artifactPrefix: string, +): Promise { + const restart = await host.nemoclaw([sandboxName, "mcp", "restart", SERVER_NAME], { + artifactName: `${artifactPrefix}-mcp-restart-provider-reuse`, + env: buildAvailabilityProbeEnv(), + timeoutMs: 12 * 60_000, + }); + expectExitZero(restart, `${artifactPrefix} mcp restart without host secret`); +} + +async function rebuildWithoutMcpHostSecret( + host: HostCliClient, + sandboxName: string, + artifactPrefix: string, +): Promise { + const rebuild = await host.nemoclaw([sandboxName, "rebuild", "--yes"], { + artifactName: `${artifactPrefix}-rebuild-with-provider-backed-mcp`, + env: { + ...buildAvailabilityProbeEnv(), + COMPATIBLE_API_KEY: COMPATIBLE_KEY, + NVIDIA_INFERENCE_API_KEY: COMPATIBLE_KEY, + }, + redactionValues: [COMPATIBLE_KEY, HOST_SECRET, ROTATED_HOST_SECRET], + timeoutMs: 25 * 60_000, + }); + expectExitZero(rebuild, `${artifactPrefix} rebuild without MCP host secret`); +} + +liveTest("mcp-bridge", { timeout: 45 * 60_000 }, async ({ artifacts, cleanup, host, sandbox }) => { + await artifacts.writeJson("scenario.json", { + id: "mcp-bridge", + sandbox: OPENCLAW_SANDBOX_NAME, + server: SERVER_NAME, + }); + const compatibleMock = await startCompatibleMock({ + apiKey: COMPATIBLE_KEY, + model: COMPATIBLE_MODEL, + }); + cleanup.add("stop MCP bridge compatible endpoint mock", () => compatibleMock.close()); + const fakeMcp = await startFakeMcpHttpsServer({ secret: HOST_SECRET }); + cleanup.add("stop fake MCP HTTPS server", () => fakeMcp.close()); + const fakeMcpTunnel = await startPublicMcpHttpsTunnel({ + cleanup, + label: "fake MCP HTTPS server", + server: fakeMcp, + }); + const decoyMcp = await startFakeMcpHttpsServer({ secret: HOST_SECRET }); + cleanup.add("stop unconfigured decoy MCP HTTPS server", () => decoyMcp.close()); + const decoyMcpTunnel = await startPublicMcpHttpsTunnel({ + cleanup, + label: "unconfigured decoy MCP HTTPS server", + server: decoyMcp, + }); + const hostAddress = await hostAddressForSandbox(host); + const endpointUrl = `http://${hostAddress}:${compatibleMock.port}/v1`; + const mcpUrl = fakeMcpTunnel.url; + const decoyMcpUrl = decoyMcpTunnel.url; + await onboardAgent(host, cleanup, endpointUrl, { + agent: "openclaw", + sandboxName: OPENCLAW_SANDBOX_NAME, + artifactName: "onboard-openclaw-mcp-bridge", + }); + // Exercise the raw OpenShell `allowed_ips` boundary before any NemoClaw MCP + // mutation. The helper uses a direct curl request with a /** binary grant, + // then restores this sandbox's exact base policy before returning, so this + // proof is independent of both the CLI implementation and adapter identity. + await assertRawOpenShellAllowedIpsRebindingDenied({ + artifacts, + env: buildAvailabilityProbeEnv(), + host, + policySettleMs: 5_000, + sandbox, + sandboxName: OPENCLAW_SANDBOX_NAME, + timeoutMs: 120_000, + }); + + cleanup.add("remove MCP bridge", () => + bestEffortRemoveBridge(host, OPENCLAW_SANDBOX_NAME, SERVER_NAME, "mcporter"), + ); + cleanup.add("remove unexpected missing-secret MCP state", () => + bestEffortRemoveBridge(host, OPENCLAW_SANDBOX_NAME, "missingsecret", "mcporter"), + ); + + await expectMcpCliFailure( + host, + OPENCLAW_SANDBOX_NAME, + ["add", "missingurl"], + /MCP server URL is required/, + "mcp-negative-missing-url", + ); + await expectMcpCliFailure( + host, + OPENCLAW_SANDBOX_NAME, + ["add", "badurl", "--url", "stdio://local"], + /must use https:\/\//, + "mcp-negative-invalid-url", + ); + await expectMcpCliFailure( + host, + OPENCLAW_SANDBOX_NAME, + ["add", "ssrf", "--url", "https://169.254.169.254/latest"], + /private, local, or special-use/, + "mcp-negative-ssrf-url", + ); + await expectMcpCliFailure( + host, + OPENCLAW_SANDBOX_NAME, + ["add", "noauth", "--url", mcpUrl], + /Authenticated MCP requires exactly one --env KEY/, + "mcp-negative-missing-credential-reference", + ); + await expectMcpCliFailure( + host, + OPENCLAW_SANDBOX_NAME, + ["add", "missingsecret", "--url", mcpUrl, "--env", "MISSING_MCP_SECRET"], + /Host environment variable 'MISSING_MCP_SECRET' is required/, + "mcp-negative-missing-secret", + ); + + await assertConcurrentAddSerialized(host, cleanup, { + sandboxName: OPENCLAW_SANDBOX_NAME, + mcpUrl, + expectedAdapter: "mcporter", + artifactPrefix: "openclaw", + }); + + const providerName = await addBridgeAndReadStatus(host, { + sandboxName: OPENCLAW_SANDBOX_NAME, + mcpUrl, + expectedAdapter: "mcporter", + artifactPrefix: "openclaw", + }); + await assertBridgeInfrastructure(host, sandbox, { + sandboxName: OPENCLAW_SANDBOX_NAME, + artifactPrefix: "openclaw", + providerName, + mcpUrl, + }); + await expectMcpCliFailure( + host, + OPENCLAW_SANDBOX_NAME, + ["add", SERVER_NAME, "--url", mcpUrl, "--env", "FAKE_MCP_SECRET"], + /already exists/, + "mcp-negative-duplicate-server", + { + ...buildAvailabilityProbeEnv(), + FAKE_MCP_SECRET: HOST_SECRET, + }, + ); + + const mcporterList = await sandbox.execShell( + OPENCLAW_SANDBOX_NAME, + trustedSandboxShellScript( + ["set -eu", `nemoclaw-start mcporter list ${SERVER_NAME} --json`].join("\n"), + ), + { + artifactName: "mcp-mcporter-list-tools", + env: buildAvailabilityProbeEnv(), + timeoutMs: 90_000, + }, + ); + expectExitZero(mcporterList, "mcporter lists tools through OpenShell MCP policy"); + expect(resultText(mcporterList)).toContain("fake_echo"); + expect(fakeMcp.requests.some((request) => request.auth === `Bearer ${HOST_SECRET}`)).toBe(true); + expect(fakeMcp.requests.every((request) => !request.auth.includes("openshell:resolve:env"))).toBe( + true, + ); + + const mcpCallScript = `const https = require("node:https"); +const url = new URL(process.argv[2]); +const method = process.argv[3]; +const expectation = process.argv[4]; +const credentialKey = process.argv[5] || "FAKE_MCP_SECRET"; +const body = JSON.stringify({ jsonrpc: "2.0", id: 1, method }); +const req = https.request({ + hostname: url.hostname, + port: url.port, + path: url.pathname, + method: "POST", + headers: { + "content-type": "application/json", + "content-length": Buffer.byteLength(body), + "authorization": "Bearer openshell:resolve:env:" + credentialKey + } +}, (res) => { + let data = ""; + res.setEncoding("utf8"); + res.on("data", (chunk) => { data += chunk; }); + res.on("end", () => { + console.log(JSON.stringify({ status: res.statusCode, body: data })); + const allowed = res.statusCode === 200 && data.includes("fake_echo"); + const denied = res.statusCode === 403; + process.exit(expectation === "allow" ? (allowed ? 0 : 1) : (denied ? 0 : 1)); + }); +}); +req.on("error", (error) => { + console.error(error.message); + const strictDenied = expectation === "deny-strict" && /HTTP\\/1\\.[01] 403 Forbidden/.test(error.message); + strictDenied && console.log(JSON.stringify({ status: 403, error: error.message })); + process.exit(expectation === "deny" || strictDenied ? 0 : 1); +}); +req.end(body); +`; + await artifacts.writeText("mcp-provider-rewrite-proof.cjs", mcpCallScript); + const mcpCallScriptB64 = Buffer.from(mcpCallScript, "utf8").toString("base64"); + const runNodeMcpProbe = async ( + targetUrl: string, + method: string, + expectation: "allow" | "deny" | "deny-strict", + artifactName: string, + credentialKey = "FAKE_MCP_SECRET", + ): Promise => + sandbox.execShell( + OPENCLAW_SANDBOX_NAME, + trustedSandboxShellScript( + [ + "set -eu", + `printf '%s' ${JSON.stringify(mcpCallScriptB64)} | base64 -d > /tmp/nemoclaw-mcp-provider-rewrite-proof.cjs`, + `nemoclaw-start node /tmp/nemoclaw-mcp-provider-rewrite-proof.cjs ${JSON.stringify(targetUrl)} ${JSON.stringify(method)} ${expectation} ${JSON.stringify(credentialKey)}`, + ].join("\n"), + ), + { + artifactName, + env: buildAvailabilityProbeEnv(), + timeoutMs: 90_000, + }, + ); + + await assertAdapterDnsRebindingDenied(host, sandbox, cleanup, { + adapter: "mcporter", + artifactPrefix: "openclaw", + hostAddress, + sandboxName: OPENCLAW_SANDBOX_NAME, + secretPaths: ["/sandbox/.openclaw", "/sandbox/.mcp.json"], + }); + + const requestCountBeforeAllowedNodeProof = fakeMcp.requests.length; + const allowedNodeCall = await runNodeMcpProbe( + mcpUrl, + "tools/list", + "allow", + "mcp-provider-rewrite-tools-list", + ); + expectExitZero(allowedNodeCall, "Node runtime identity can use an explicitly allowed MCP method"); + const allowedNodeRequests = fakeMcp.requests.slice(requestCountBeforeAllowedNodeProof); + expect(allowedNodeRequests).toHaveLength(1); + expect(allowedNodeRequests[0]).toMatchObject({ + method: "POST", + path: "/mcp", + auth: `Bearer ${HOST_SECRET}`, + }); + expect(JSON.parse(allowedNodeRequests[0].body)).toMatchObject({ + jsonrpc: "2.0", + method: "tools/list", + }); + expect(fakeMcp.requests.every((request) => !request.auth.includes("openshell:resolve:env"))).toBe( + true, + ); + + const requestCountAfterAllowedNodeProof = fakeMcp.requests.length; + const deniedNodeCall = await runNodeMcpProbe( + mcpUrl, + "admin/delete", + "deny", + "mcp-provider-rewrite-extension-method-denied", + ); + expectExitZero(deniedNodeCall, "Node runtime identity cannot use a non-allowlisted MCP method"); + expect(fakeMcp.requests.length).toBe(requestCountAfterAllowedNodeProof); + + const deniedWrongPathCall = await runNodeMcpProbe( + `${new URL(mcpUrl).origin}/not-the-configured-mcp-path`, + "tools/list", + "deny", + "mcp-provider-rewrite-unconfigured-path-denied", + ); + expectExitZero( + deniedWrongPathCall, + "allowed Node runtime cannot replay the placeholder to another path", + ); + expect(fakeMcp.requests.length).toBe(requestCountAfterAllowedNodeProof); + + const deniedDecoyCall = await runNodeMcpProbe( + decoyMcpUrl, + "tools/list", + "deny", + "mcp-provider-rewrite-unconfigured-endpoint-denied", + ); + expectExitZero( + deniedDecoyCall, + "allowed Node runtime cannot replay the placeholder to another endpoint", + ); + expect(decoyMcp.requests).toHaveLength(0); + expect(fakeMcp.requests.length).toBe(requestCountAfterAllowedNodeProof); + + const deniedCurl = await sandbox.execShell( + OPENCLAW_SANDBOX_NAME, + trustedSandboxShellScript( + [ + "set -eu", + `body='{"jsonrpc":"2.0","id":1,"method":"tools/list"}'`, + "rm -f /tmp/nemoclaw-mcp-denied.out /tmp/nemoclaw-mcp-denied.err", + "set +e", + `code="$(curl -sS -o /tmp/nemoclaw-mcp-denied.out -w '%{http_code}' -X POST ${JSON.stringify(mcpUrl)} -H 'content-type: application/json' -H 'authorization: Bearer openshell:resolve:env:FAKE_MCP_SECRET' --data "$body" 2>/tmp/nemoclaw-mcp-denied.err)"`, + "curl_rc=$?", + "set -e", + "cat /tmp/nemoclaw-mcp-denied.out 2>/dev/null || true", + "cat /tmp/nemoclaw-mcp-denied.err >&2", + 'printf "NEMOCLAW_MCP_CURL_HTTP_CODE=%s\\n" "$code"', + 'exit "$curl_rc"', + ].join("\n"), + ), + { + artifactName: "mcp-non-allowlisted-binary-curl-denied", + env: buildAvailabilityProbeEnv(), + timeoutMs: 60_000, + }, + ); + expect( + isExpectedMcpCurlPolicyDenial(deniedCurl), + `non-allowlisted curl must receive an OpenShell policy denial\nstdout:\n${deniedCurl.stdout}\nstderr:\n${deniedCurl.stderr}`, + ).toBe(true); + expect(fakeMcp.requests.length).toBe(requestCountAfterAllowedNodeProof); + + const registryRaw = fs.existsSync(REGISTRY_FILE) ? fs.readFileSync(REGISTRY_FILE, "utf8") : ""; + expect(registryRaw).toContain(mcpUrl); + expect(registryRaw).toContain(providerName); + expect(registryRaw).not.toContain("enc:v1:"); + expect(registryRaw).not.toContain("proxy.pid"); + expect(registryRaw).not.toContain(HOST_SECRET); + await assertSecretAbsentFromSandbox(sandbox, OPENCLAW_SANDBOX_NAME, [ + "/sandbox/.openclaw", + "/sandbox/.mcp.json", + ]); + + const openClawResult = `MCP_AUTH_REWRITE_OK::${TOOL_CHALLENGE}`; + await assertRealAdapterToolCall(sandbox, fakeMcp, { + agent: "openclaw", + sandboxName: OPENCLAW_SANDBOX_NAME, + resultToken: openClawResult, + artifactName: "openclaw-real-mcp-tool-call-initial", + }); + await restartBridgeWithoutHostSecret(host, OPENCLAW_SANDBOX_NAME, "openclaw"); + await assertRealAdapterToolCall(sandbox, fakeMcp, { + agent: "openclaw", + sandboxName: OPENCLAW_SANDBOX_NAME, + resultToken: openClawResult, + artifactName: "openclaw-real-mcp-tool-call-after-restart", + }); + fakeMcp.setSecret(ROTATED_HOST_SECRET); + await rotateBridgeCredential(host, OPENCLAW_SANDBOX_NAME, "openclaw"); + await assertRealAdapterToolCall(sandbox, fakeMcp, { + agent: "openclaw", + sandboxName: OPENCLAW_SANDBOX_NAME, + resultToken: openClawResult, + artifactName: "openclaw-real-mcp-tool-call-after-credential-rotation", + expectedSecret: ROTATED_HOST_SECRET, + }); + await assertSecretAbsentFromSandbox( + sandbox, + OPENCLAW_SANDBOX_NAME, + ["/sandbox/.openclaw", "/sandbox/.mcp.json"], + [HOST_SECRET, ROTATED_HOST_SECRET], + "openclaw-assert-secrets-absent-after-rotation", + ); + await rebuildWithoutMcpHostSecret(host, OPENCLAW_SANDBOX_NAME, "openclaw"); + await assertSecretAbsentFromSandbox( + sandbox, + OPENCLAW_SANDBOX_NAME, + ["/sandbox/.openclaw", "/sandbox/.mcp.json"], + [HOST_SECRET, ROTATED_HOST_SECRET], + "openclaw-assert-secrets-absent-after-rebuild", + ); + await assertRealAdapterToolCall(sandbox, fakeMcp, { + agent: "openclaw", + sandboxName: OPENCLAW_SANDBOX_NAME, + resultToken: openClawResult, + artifactName: "openclaw-real-mcp-tool-call-after-rebuild", + expectedSecret: ROTATED_HOST_SECRET, + }); + + await removeBridgeAndAssertEmpty(host, sandbox, { + agent: "openclaw", + adapter: "mcporter", + sandboxName: OPENCLAW_SANDBOX_NAME, + artifactPrefix: "openclaw", + providerName, + mcpUrl, + }); + await assertAdapterRequestDeniedAfterRemove(sandbox, fakeMcp, { + adapter: "mcporter", + sandboxName: OPENCLAW_SANDBOX_NAME, + mcpUrl, + artifactPrefix: "openclaw", + }); +}); + +liveAgentMatrixTest( + "mcp-bridge-hermes", + { timeout: 45 * 60_000 }, + async ({ artifacts, cleanup, host, sandbox }) => { + await artifacts.writeJson("scenario.json", { + id: "mcp-bridge-hermes", + sandbox: HERMES_SANDBOX_NAME, + server: SERVER_NAME, + }); + const hermesResult = `MCP_AUTH_REWRITE_OK::${TOOL_CHALLENGE}`; + const compatibleMock = await startCompatibleMock({ + apiKey: COMPATIBLE_KEY, + model: COMPATIBLE_MODEL, + toolChallenge: TOOL_CHALLENGE, + toolResultToken: hermesResult, + toolNames: ["mcp_fake_fake_echo"], + deferredToolName: "mcp_fake_fake_echo", + }); + cleanup.add("stop Hermes MCP bridge compatible endpoint mock", () => compatibleMock.close()); + const fakeMcp = await startFakeMcpHttpsServer({ + secret: HOST_SECRET, + challenge: TOOL_CHALLENGE, + resultToken: hermesResult, + }); + cleanup.add("stop fake Hermes MCP HTTPS server", () => fakeMcp.close()); + const fakeMcpTunnel = await startPublicMcpHttpsTunnel({ + cleanup, + label: "fake Hermes MCP HTTPS server", + server: fakeMcp, + }); + const hostAddress = await hostAddressForSandbox(host); + const endpointUrl = `http://${hostAddress}:${compatibleMock.port}/v1`; + const mcpUrl = fakeMcpTunnel.url; + await onboardAgent(host, cleanup, endpointUrl, { + agent: "hermes", + sandboxName: HERMES_SANDBOX_NAME, + artifactName: "onboard-hermes-mcp-bridge", + }); + cleanup.add("remove Hermes MCP bridge", () => + bestEffortRemoveBridge(host, HERMES_SANDBOX_NAME, SERVER_NAME, "hermes-config"), + ); + + await assertConcurrentAddSerialized(host, cleanup, { + sandboxName: HERMES_SANDBOX_NAME, + mcpUrl, + expectedAdapter: "hermes-config", + artifactPrefix: "hermes", + }); + + const initialDiscoveryOffset = fakeMcp.requests.length; + const providerName = await addBridgeAndReadStatus(host, { + sandboxName: HERMES_SANDBOX_NAME, + mcpUrl, + expectedAdapter: "hermes-config", + artifactPrefix: "hermes", + }); + await assertAuthenticatedMcpDiscovery(fakeMcp, { + requestOffset: initialDiscoveryOffset, + expectedSecret: HOST_SECRET, + label: "Hermes initial MCP discovery", + }); + await assertBridgeInfrastructure(host, sandbox, { + sandboxName: HERMES_SANDBOX_NAME, + artifactPrefix: "hermes", + providerName, + mcpUrl, + }); + await assertHermesConfig(sandbox, HERMES_SANDBOX_NAME, mcpUrl); + await assertSecretAbsentFromSandbox(sandbox, HERMES_SANDBOX_NAME, ["/sandbox/.hermes"]); + await assertAdapterDnsRebindingDenied(host, sandbox, cleanup, { + adapter: "hermes-config", + artifactPrefix: "hermes", + hostAddress, + sandboxName: HERMES_SANDBOX_NAME, + secretPaths: ["/sandbox/.hermes"], + }); + await assertRealAdapterToolCall(sandbox, fakeMcp, { + agent: "hermes", + sandboxName: HERMES_SANDBOX_NAME, + resultToken: hermesResult, + artifactName: "hermes-real-mcp-tool-call-initial", + }); + await restartBridgeWithoutHostSecret(host, HERMES_SANDBOX_NAME, "hermes"); + await assertRealAdapterToolCall(sandbox, fakeMcp, { + agent: "hermes", + sandboxName: HERMES_SANDBOX_NAME, + resultToken: hermesResult, + artifactName: "hermes-real-mcp-tool-call-after-restart", + }); + fakeMcp.setSecret(ROTATED_HOST_SECRET); + await rotateBridgeCredential(host, HERMES_SANDBOX_NAME, "hermes"); + await assertRealAdapterToolCall(sandbox, fakeMcp, { + agent: "hermes", + sandboxName: HERMES_SANDBOX_NAME, + resultToken: hermesResult, + artifactName: "hermes-real-mcp-tool-call-after-credential-rotation", + expectedSecret: ROTATED_HOST_SECRET, + }); + await assertSecretAbsentFromSandbox( + sandbox, + HERMES_SANDBOX_NAME, + ["/sandbox/.hermes"], + [HOST_SECRET, ROTATED_HOST_SECRET], + "hermes-assert-secrets-absent-after-rotation", + ); + const rebuildDiscoveryOffset = fakeMcp.requests.length; + await rebuildWithoutMcpHostSecret(host, HERMES_SANDBOX_NAME, "hermes"); + await assertAuthenticatedMcpDiscovery(fakeMcp, { + requestOffset: rebuildDiscoveryOffset, + expectedSecret: ROTATED_HOST_SECRET, + label: "Hermes post-rebuild MCP discovery", + }); + await assertHermesConfig(sandbox, HERMES_SANDBOX_NAME, mcpUrl); + await assertSecretAbsentFromSandbox( + sandbox, + HERMES_SANDBOX_NAME, + ["/sandbox/.hermes"], + [HOST_SECRET, ROTATED_HOST_SECRET], + "hermes-assert-secrets-absent-after-rebuild", + ); + await assertRealAdapterToolCall(sandbox, fakeMcp, { + agent: "hermes", + sandboxName: HERMES_SANDBOX_NAME, + resultToken: hermesResult, + artifactName: "hermes-real-mcp-tool-call-after-rebuild", + expectedSecret: ROTATED_HOST_SECRET, + }); + await removeBridgeAndAssertEmpty(host, sandbox, { + agent: "hermes", + adapter: "hermes-config", + sandboxName: HERMES_SANDBOX_NAME, + artifactPrefix: "hermes", + providerName, + mcpUrl, + }); + await assertAdapterRequestDeniedAfterRemove(sandbox, fakeMcp, { + adapter: "hermes-config", + sandboxName: HERMES_SANDBOX_NAME, + mcpUrl, + artifactPrefix: "hermes", + }); + }, +); + +liveAgentMatrixTest( + "mcp-bridge-deepagents", + { timeout: 45 * 60_000 }, + async ({ artifacts, cleanup, host, sandbox }) => { + await artifacts.writeJson("scenario.json", { + id: "mcp-bridge-deepagents", + sandbox: DEEPAGENTS_SANDBOX_NAME, + server: SERVER_NAME, + }); + const deepAgentsResult = `MCP_AUTH_REWRITE_OK::${TOOL_CHALLENGE}`; + const compatibleMock = await startCompatibleMock({ + apiKey: COMPATIBLE_KEY, + model: COMPATIBLE_MODEL, + toolChallenge: TOOL_CHALLENGE, + toolResultToken: deepAgentsResult, + toolNames: ["fake_fake_echo"], + }); + cleanup.add("stop Deep Agents MCP bridge compatible endpoint mock", () => + compatibleMock.close(), + ); + const fakeMcp = await startFakeMcpHttpsServer({ + secret: HOST_SECRET, + challenge: TOOL_CHALLENGE, + resultToken: deepAgentsResult, + }); + cleanup.add("stop fake Deep Agents MCP HTTPS server", () => fakeMcp.close()); + const fakeMcpTunnel = await startPublicMcpHttpsTunnel({ + cleanup, + label: "fake Deep Agents MCP HTTPS server", + server: fakeMcp, + }); + const hostAddress = await hostAddressForSandbox(host); + const endpointUrl = `http://${hostAddress}:${compatibleMock.port}/v1`; + const mcpUrl = fakeMcpTunnel.url; + await onboardAgent(host, cleanup, endpointUrl, { + agent: "langchain-deepagents-code", + sandboxName: DEEPAGENTS_SANDBOX_NAME, + artifactName: "onboard-deepagents-mcp-bridge", + }); + cleanup.add("remove Deep Agents MCP bridge", () => + bestEffortRemoveBridge(host, DEEPAGENTS_SANDBOX_NAME, SERVER_NAME, "deepagents-config"), + ); + + await assertConcurrentAddSerialized(host, cleanup, { + sandboxName: DEEPAGENTS_SANDBOX_NAME, + mcpUrl, + expectedAdapter: "deepagents-config", + artifactPrefix: "deepagents", + }); + + const providerName = await addBridgeAndReadStatus(host, { + sandboxName: DEEPAGENTS_SANDBOX_NAME, + mcpUrl, + expectedAdapter: "deepagents-config", + artifactPrefix: "deepagents", + }); + await assertBridgeInfrastructure(host, sandbox, { + sandboxName: DEEPAGENTS_SANDBOX_NAME, + artifactPrefix: "deepagents", + providerName, + mcpUrl, + }); + await assertDeepAgentsConfig(sandbox, DEEPAGENTS_SANDBOX_NAME, mcpUrl); + await assertSecretAbsentFromSandbox(sandbox, DEEPAGENTS_SANDBOX_NAME, ["/sandbox/.deepagents"]); + await assertAdapterDnsRebindingDenied(host, sandbox, cleanup, { + adapter: "deepagents-config", + artifactPrefix: "deepagents", + hostAddress, + sandboxName: DEEPAGENTS_SANDBOX_NAME, + secretPaths: ["/sandbox/.deepagents"], + }); + await assertRealAdapterToolCall(sandbox, fakeMcp, { + agent: "langchain-deepagents-code", + sandboxName: DEEPAGENTS_SANDBOX_NAME, + resultToken: deepAgentsResult, + artifactName: "deepagents-real-mcp-tool-call-initial", + }); + await restartBridgeWithoutHostSecret(host, DEEPAGENTS_SANDBOX_NAME, "deepagents"); + await assertRealAdapterToolCall(sandbox, fakeMcp, { + agent: "langchain-deepagents-code", + sandboxName: DEEPAGENTS_SANDBOX_NAME, + resultToken: deepAgentsResult, + artifactName: "deepagents-real-mcp-tool-call-after-restart", + }); + fakeMcp.setSecret(ROTATED_HOST_SECRET); + await rotateBridgeCredential(host, DEEPAGENTS_SANDBOX_NAME, "deepagents"); + await assertRealAdapterToolCall(sandbox, fakeMcp, { + agent: "langchain-deepagents-code", + sandboxName: DEEPAGENTS_SANDBOX_NAME, + resultToken: deepAgentsResult, + artifactName: "deepagents-real-mcp-tool-call-after-credential-rotation", + expectedSecret: ROTATED_HOST_SECRET, + }); + await assertSecretAbsentFromSandbox( + sandbox, + DEEPAGENTS_SANDBOX_NAME, + ["/sandbox/.deepagents"], + [HOST_SECRET, ROTATED_HOST_SECRET], + "deepagents-assert-secrets-absent-after-rotation", + ); + await rebuildWithoutMcpHostSecret(host, DEEPAGENTS_SANDBOX_NAME, "deepagents"); + await assertDeepAgentsConfig(sandbox, DEEPAGENTS_SANDBOX_NAME, mcpUrl); + await assertSecretAbsentFromSandbox( + sandbox, + DEEPAGENTS_SANDBOX_NAME, + ["/sandbox/.deepagents"], + [HOST_SECRET, ROTATED_HOST_SECRET], + "deepagents-assert-secrets-absent-after-rebuild", + ); + await assertRealAdapterToolCall(sandbox, fakeMcp, { + agent: "langchain-deepagents-code", + sandboxName: DEEPAGENTS_SANDBOX_NAME, + resultToken: deepAgentsResult, + artifactName: "deepagents-real-mcp-tool-call-after-rebuild", + expectedSecret: ROTATED_HOST_SECRET, + }); + await removeBridgeAndAssertEmpty(host, sandbox, { + agent: "langchain-deepagents-code", + adapter: "deepagents-config", + sandboxName: DEEPAGENTS_SANDBOX_NAME, + artifactPrefix: "deepagents", + providerName, + mcpUrl, + }); + await assertAdapterRequestDeniedAfterRemove(sandbox, fakeMcp, { + adapter: "deepagents-config", + sandboxName: DEEPAGENTS_SANDBOX_NAME, + mcpUrl, + artifactPrefix: "deepagents", + }); + }, +); diff --git a/test/e2e/live/openshell-allowed-ips-rebinding.ts b/test/e2e/live/openshell-allowed-ips-rebinding.ts new file mode 100644 index 00000000000..c3cf7e398e3 --- /dev/null +++ b/test/e2e/live/openshell-allowed-ips-rebinding.ts @@ -0,0 +1,351 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import { createServer, type Server } from "node:http"; +import path from "node:path"; + +import YAML from "yaml"; + +import { isPrivateIp } from "../../../nemoclaw/src/blueprint/private-networks.ts"; +import { shellQuote } from "../../../src/lib/core/shell-quote"; +import { parseOpenShellPolicy } from "../../../src/lib/policy/merge"; +import type { ArtifactSink } from "../fixtures/artifacts.ts"; +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import type { HostCliClient } from "../fixtures/clients/host.ts"; +import { type SandboxClient, trustedSandboxShellScript } from "../fixtures/clients/sandbox.ts"; +import { expect } from "../fixtures/e2e-test.ts"; +import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; +import { + type DnsRebindingHostsFixture, + remapDnsRebindingHostname, + restoreDnsRebindingHostsFixture, + setupDnsRebindingHostsFixture, +} from "./dns-rebinding-hosts-fixture.ts"; + +export const RAW_OPENSHELL_REBIND_HOSTNAME = "openshell-rebind.example.test"; +export const RAW_OPENSHELL_REBIND_PINNED_IP = "1.1.1.1"; +export const RAW_OPENSHELL_REBIND_POLICY_KEY = "raw_openshell_allowed_ips_rebinding"; +export const RAW_OPENSHELL_REBIND_HTTP_CODE_MARKER = "NEMOCLAW_RAW_OPENSHELL_REBIND_HTTP_CODE="; + +type RawOpenShellPolicy = Record & { + network_policies?: Record; +}; + +type RawOpenShellEndpoint = Record & { + allowed_ips?: unknown; + host?: unknown; + port?: unknown; + protocol?: unknown; +}; + +function isMapping(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function resultText(result: Pick): string { + return [result.stdout, result.stderr].filter(Boolean).join("\n"); +} + +function parseRawPolicy(yaml: string): RawOpenShellPolicy { + const parsed: unknown = YAML.parse(yaml); + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + throw new Error("OpenShell base policy must be a YAML mapping"); + } + return parsed as RawOpenShellPolicy; +} + +export function parseRawOpenShellAllowedIpsRebindingEndpoint( + effectivePolicyOutput: string, +): RawOpenShellEndpoint { + const policy = parseOpenShellPolicy(effectivePolicyOutput).policy; + const networkPolicies = policy.network_policies; + if (!isMapping(networkPolicies)) { + throw new Error("effective OpenShell policy must contain network_policies"); + } + const rawPolicy = networkPolicies[RAW_OPENSHELL_REBIND_POLICY_KEY]; + if (!isMapping(rawPolicy) || !Array.isArray(rawPolicy.endpoints)) { + throw new Error( + `effective OpenShell policy must contain ${RAW_OPENSHELL_REBIND_POLICY_KEY} endpoints`, + ); + } + const endpoint = rawPolicy.endpoints.find( + (candidate): candidate is RawOpenShellEndpoint => + isMapping(candidate) && candidate.host === RAW_OPENSHELL_REBIND_HOSTNAME, + ); + if (!endpoint) { + throw new Error( + `effective OpenShell policy must contain the ${RAW_OPENSHELL_REBIND_HOSTNAME} endpoint`, + ); + } + return endpoint; +} + +export function buildRawOpenShellAllowedIpsRebindingPolicy( + basePolicyYaml: string, + port: number, +): string { + const policy = parseRawPolicy(basePolicyYaml); + policy.network_policies = { + ...(policy.network_policies ?? {}), + [RAW_OPENSHELL_REBIND_POLICY_KEY]: { + name: RAW_OPENSHELL_REBIND_POLICY_KEY, + endpoints: [ + { + host: RAW_OPENSHELL_REBIND_HOSTNAME, + port, + path: "/mcp", + protocol: "mcp", + enforcement: "enforce", + allowed_ips: [RAW_OPENSHELL_REBIND_PINNED_IP], + mcp: { + max_body_bytes: 4096, + strict_tool_names: true, + allow_all_known_mcp_methods: false, + }, + rules: [{ allow: { method: "tools/list" } }], + }, + ], + // Deliberately remove adapter attribution from this contract. The only + // reason the raw request may be denied is OpenShell's destination policy. + binaries: [{ path: "/**" }], + }, + }; + return YAML.stringify(policy); +} + +/** + * Exercise OpenShell directly with a raw MCP request and require an exact 403. + * This intentionally bypasses every NemoClaw MCP command and agent adapter. + * + * Pinned resolve-validate-connect implementation: + * https://github.com/NVIDIA/OpenShell/blob/8cb16de9eae4c44d7d31e1493747d8c10abb5963/crates/openshell-supervisor-network/src/proxy.rs#L2476-L2502 + * resolves once, #L2527-L2567 validates that address list, #L2622-L2630 + * returns it unchanged, and #L3885-L3893 plus #L4123-L4125 carry that same + * list through the explicit HTTP-forward connection path used by this probe. + */ +export function buildRawOpenShellAllowedIpsRebindingProbeScript(targetUrl: string): string { + const body = JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/list" }); + const responsePath = "/tmp/nemoclaw-raw-openshell-rebinding.body"; + const stderrPath = "/tmp/nemoclaw-raw-openshell-rebinding.stderr"; + return [ + "set -u", + `rm -f ${shellQuote(responsePath)} ${shellQuote(stderrPath)}`, + `body=${shellQuote(body)}`, + "set +e", + `status="$(curl -sS --max-time 30 -o ${shellQuote(responsePath)} -w '%{http_code}' -X POST ${shellQuote(targetUrl)} -H 'content-type: application/json' -H 'accept: application/json, text/event-stream' --data-binary "$body" 2>${shellQuote(stderrPath)})"`, + "curl_rc=$?", + "set -e", + `cat ${shellQuote(responsePath)} 2>/dev/null || true`, + `cat ${shellQuote(stderrPath)} >&2 2>/dev/null || true`, + `printf '${RAW_OPENSHELL_REBIND_HTTP_CODE_MARKER}%s\\n' "$status"`, + 'if [ "$curl_rc" -eq 0 ] && [ "$status" = "403" ]; then exit 0; fi', + 'if [ "$curl_rc" -ne 0 ]; then exit "$curl_rc"; fi', + "exit 1", + ].join("\n"); +} + +async function hostAddressForSandbox(host: HostCliClient): Promise { + const probe = await host.command( + "bash", + [ + "-lc", + [ + 'ip_addr="$(ip route get 1.1.1.1 2>/dev/null | awk \'{for (i=1;i<=NF;i++) if ($i=="src") {print $(i+1); exit}}\')"', + 'if [ -n "$ip_addr" ]; then echo "$ip_addr"; exit 0; fi', + "ip_addr=\"$(hostname -I 2>/dev/null | awk '{print $1}')\"", + 'if [ -n "$ip_addr" ]; then echo "$ip_addr"; exit 0; fi', + "echo 127.0.0.1", + ].join("\n"), + ], + { + artifactName: "raw-openshell-rebinding-host-address", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }, + ); + expect(probe.exitCode, resultText(probe)).toBe(0); + return probe.stdout.trim(); +} + +async function closeServer(server: Server): Promise { + if (!server.listening) return; + await new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }); +} + +async function startCountingMcpServer(): Promise<{ + close: () => Promise; + port: number; + requestCount: () => number; +}> { + let requestCount = 0; + const server = createServer((_request, response) => { + requestCount += 1; + response.writeHead(200, { "content-type": "application/json" }); + response.end('{"jsonrpc":"2.0","id":1,"result":{"tools":[]}}\n'); + }); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "0.0.0.0", resolve); + }); + const address = server.address(); + if (!address || typeof address === "string") { + await closeServer(server); + throw new Error("raw OpenShell rebinding server did not expose a TCP port"); + } + return { + close: () => closeServer(server), + port: address.port, + requestCount: () => requestCount, + }; +} + +export async function assertRawOpenShellAllowedIpsRebindingDenied(options: { + artifacts: ArtifactSink; + env?: NodeJS.ProcessEnv; + host: HostCliClient; + policySettleMs: number; + sandbox: SandboxClient; + sandboxName: string; + timeoutMs: number; +}): Promise { + const env = options.env ?? buildAvailabilityProbeEnv(); + const server = await startCountingMcpServer(); + let basePolicyPath: string | undefined; + let hostsFixture: DnsRebindingHostsFixture | undefined; + let policyMutationAttempted = false; + try { + const reboundAddress = await hostAddressForSandbox(options.host); + expect(reboundAddress).not.toBe(RAW_OPENSHELL_REBIND_PINNED_IP); + expect( + isPrivateIp(reboundAddress), + `${reboundAddress} must be a private rebinding target`, + ).toBe(true); + + hostsFixture = await setupDnsRebindingHostsFixture( + options.host, + options.sandboxName, + RAW_OPENSHELL_REBIND_HOSTNAME, + ); + await remapDnsRebindingHostname( + options.host, + options.sandboxName, + hostsFixture, + RAW_OPENSHELL_REBIND_PINNED_IP, + "raw-openshell-rebinding-map-public-pin", + ); + + const basePolicy = await options.sandbox.openshell( + ["policy", "get", "--base", options.sandboxName], + { + artifactName: "raw-openshell-rebinding-policy-get-base", + env, + timeoutMs: options.timeoutMs, + }, + ); + expect(basePolicy.exitCode, resultText(basePolicy)).toBe(0); + const basePolicyYaml = parseOpenShellPolicy(basePolicy.stdout).yamlBody; + basePolicyPath = options.artifacts.pathFor( + "policies/raw-openshell-allowed-ips-rebinding.base.yaml", + ); + const policyPath = options.artifacts.pathFor( + "policies/raw-openshell-allowed-ips-rebinding.yaml", + ); + fs.mkdirSync(path.dirname(policyPath), { recursive: true }); + fs.writeFileSync(basePolicyPath, basePolicyYaml, "utf8"); + fs.writeFileSync( + policyPath, + buildRawOpenShellAllowedIpsRebindingPolicy(basePolicyYaml, server.port), + "utf8", + ); + + policyMutationAttempted = true; + const applyPolicy = await options.sandbox.openshell( + ["policy", "set", "--policy", policyPath, "--wait", options.sandboxName], + { + artifactName: "raw-openshell-rebinding-policy-set", + env, + timeoutMs: options.timeoutMs, + }, + ); + expect(applyPolicy.exitCode, resultText(applyPolicy)).toBe(0); + await new Promise((resolve) => setTimeout(resolve, options.policySettleMs)); + + const effectivePolicy = await options.sandbox.openshell( + ["policy", "get", "--full", options.sandboxName], + { + artifactName: "raw-openshell-rebinding-policy-get-full", + env, + timeoutMs: options.timeoutMs, + }, + ); + expect(effectivePolicy.exitCode, resultText(effectivePolicy)).toBe(0); + const effectiveEndpoint = parseRawOpenShellAllowedIpsRebindingEndpoint(effectivePolicy.stdout); + expect(effectiveEndpoint).toMatchObject({ + allowed_ips: [RAW_OPENSHELL_REBIND_PINNED_IP], + host: RAW_OPENSHELL_REBIND_HOSTNAME, + port: server.port, + protocol: "mcp", + }); + + await remapDnsRebindingHostname( + options.host, + options.sandboxName, + hostsFixture, + reboundAddress, + "raw-openshell-rebinding-map-private-unpinned", + ); + + const targetUrl = `http://${RAW_OPENSHELL_REBIND_HOSTNAME}:${server.port}/mcp`; + const denial = await options.sandbox.execShell( + options.sandboxName, + trustedSandboxShellScript(buildRawOpenShellAllowedIpsRebindingProbeScript(targetUrl)), + { + artifactName: "raw-openshell-rebinding-exact-403", + env, + timeoutMs: 60_000, + }, + ); + expect(denial.exitCode, resultText(denial)).toBe(0); + expect(denial.stdout).toContain(`${RAW_OPENSHELL_REBIND_HTTP_CODE_MARKER}403`); + expect( + server.requestCount(), + "raw OpenShell allowed_ips denial must record zero upstream requests", + ).toBe(0); + } finally { + try { + if (policyMutationAttempted && basePolicyPath) { + const restorePolicy = await options.sandbox.openshell( + ["policy", "set", "--policy", basePolicyPath, "--wait", options.sandboxName], + { + artifactName: "raw-openshell-rebinding-policy-restore", + env, + timeoutMs: options.timeoutMs, + }, + ); + expect(restorePolicy.exitCode, resultText(restorePolicy)).toBe(0); + await new Promise((resolve) => setTimeout(resolve, options.policySettleMs)); + const restoredPolicy = await options.sandbox.openshell( + ["policy", "get", "--base", options.sandboxName], + { + artifactName: "raw-openshell-rebinding-policy-verify-restored", + env, + timeoutMs: options.timeoutMs, + }, + ); + expect(restoredPolicy.exitCode, resultText(restoredPolicy)).toBe(0); + expect(restoredPolicy.stdout).not.toContain(RAW_OPENSHELL_REBIND_POLICY_KEY); + } + } finally { + try { + if (hostsFixture) { + await restoreDnsRebindingHostsFixture(options.host, options.sandboxName, hostsFixture); + } + } finally { + await server.close(); + } + } + } +} diff --git a/test/e2e/live/openshell-gateway-upgrade.test.ts b/test/e2e/live/openshell-gateway-upgrade.test.ts index 590ed247029..5c8f05ba8b0 100644 --- a/test/e2e/live/openshell-gateway-upgrade.test.ts +++ b/test/e2e/live/openshell-gateway-upgrade.test.ts @@ -758,7 +758,17 @@ runOpenShellGatewayUpgrade( fs.mkdirSync(path.dirname(signLog), { recursive: true }); writeFakeDarwinUname(fakeBin); writeFakeCurrentOpenshell(fakeBin); - writeExecutable(path.join(fakeBin, "openshell-gateway"), "#!/usr/bin/env bash\nexit 0\n"); + writeExecutable( + path.join(fakeBin, "openshell-gateway"), + `#!/usr/bin/env bash +if [ "\${1:-}" = "--version" ]; then + printf 'openshell-gateway ${CURRENT_OPENSHELL_VERSION}\n' + exit 0 +fi +# allow_all_known_mcp_methods +exit 0 +`, + ); writeExecutable(path.join(fakeBin, "openshell-driver-vm"), "#!/usr/bin/env bash\nexit 0\n"); writeExecutable( path.join(fakeBin, "codesign"), diff --git a/test/e2e/live/openshell-version-pin.test.ts b/test/e2e/live/openshell-version-pin.test.ts index eae659c9039..c627a9f1a4d 100644 --- a/test/e2e/live/openshell-version-pin.test.ts +++ b/test/e2e/live/openshell-version-pin.test.ts @@ -250,8 +250,8 @@ esac`, } // tar stub: write the corresponding binary into the -C outdir. Each binary -// reports the replacement version + carries the messaging-rewrite capability -// marker so the post-install feature probe passes. +// reports the replacement version + carries the messaging-rewrite and MCP-L7 +// capability markers so the post-install feature probes pass. function createFakeTar(binDir: string, replacementVersion: string): void { writeExecutable( path.join(binDir, "tar"), @@ -275,7 +275,7 @@ esac cat > "$outdir/$name" <<'EOS' #!/usr/bin/env bash if [ "\${1:-}" = "--version" ]; then echo "openshell ${replacementVersion}"; exit 0; fi -# request-body-credential-rewrite websocket-credential-rewrite +# request-body-credential-rewrite websocket-credential-rewrite allow_all_known_mcp_methods exit 0 EOS chmod 755 "$outdir/$name"`, diff --git a/test/e2e/live/rebuild-hermes-env.ts b/test/e2e/live/rebuild-hermes-env.ts new file mode 100644 index 00000000000..10cbdaa0c97 --- /dev/null +++ b/test/e2e/live/rebuild-hermes-env.ts @@ -0,0 +1,26 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; + +/** + * Build the explicit child environment used by the Hermes rebuild scenario. + * The fixture-wide allowlist intentionally remains narrow; the selected + * OpenShell channel and its explicit dev-artifact opt-in are non-secret + * integration inputs needed by install.sh. + */ +export function buildRebuildHermesChildEnv( + base: NodeJS.ProcessEnv, + overlay: NodeJS.ProcessEnv, +): NodeJS.ProcessEnv { + const openshellChannel = base.NEMOCLAW_OPENSHELL_CHANNEL; + const acceptDevUnverifiedInstall = base.NEMOCLAW_ACCEPT_DEV_UNVERIFIED_INSTALL; + return { + ...buildAvailabilityProbeEnv(base), + ...(acceptDevUnverifiedInstall === undefined + ? {} + : { NEMOCLAW_ACCEPT_DEV_UNVERIFIED_INSTALL: acceptDevUnverifiedInstall }), + ...(openshellChannel === undefined ? {} : { NEMOCLAW_OPENSHELL_CHANNEL: openshellChannel }), + ...overlay, + }; +} diff --git a/test/e2e/live/rebuild-hermes.test.ts b/test/e2e/live/rebuild-hermes.test.ts index e3d0d778423..8d71bfc5704 100644 --- a/test/e2e/live/rebuild-hermes.test.ts +++ b/test/e2e/live/rebuild-hermes.test.ts @@ -14,6 +14,7 @@ import { expect, test } from "../fixtures/e2e-test.ts"; import { shouldRunLiveE2E } from "../fixtures/live-project-gate.ts"; import { listCredentialLeakPaths } from "../fixtures/phases/state-validation.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; +import { buildRebuildHermesChildEnv } from "./rebuild-hermes-env.ts"; // The migrated scope is the legacy non-interactive shell regression: install.sh, // Docker base-image builds, OpenShell provider/sandbox commands, direct Hermes @@ -98,8 +99,7 @@ interface SessionArtifactSummary { } function testEnv(apiKey?: string, extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { - return { - ...buildAvailabilityProbeEnv(), + return buildRebuildHermesChildEnv(process.env, { NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", NEMOCLAW_AGENT: "hermes", NEMOCLAW_COMPAT_MODEL: HOSTED_MODEL, @@ -118,7 +118,7 @@ function testEnv(apiKey?: string, extra: NodeJS.ProcessEnv = {}): NodeJS.Process } : {}), ...extra, - }; + }); } function snapshotFile(file: string): FileSnapshot { @@ -250,7 +250,7 @@ async function waitForSandboxReady(host: HostCliClient, apiKey: string): Promise throw new Error(`sandbox ${SANDBOX_NAME} did not become Ready`); } -function seedRegistryAndSession(): SessionArtifactSummary { +function seedRegistryAndSession(dashboardPort: number): SessionArtifactSummary { const registry = readJsonFile(REGISTRY_FILE, {}); registry.sandboxes = registry.sandboxes ?? {}; @@ -306,6 +306,11 @@ function seedRegistryAndSession(): SessionArtifactSummary { policyTier: null, agent: "hermes", agentVersion: OLD_HERMES_REGISTRY_VERSION, + dashboardPort, + // This curated old-version fixture is still a NemoClaw-managed image. + // Preserve that provenance explicitly; an absent value must remain + // fail-closed because it could represent a custom `--from` image. + fromDockerfile: null, messaging: { schemaVersion: 1, plan: messagingPlan }, }; expect( @@ -352,7 +357,13 @@ function seedRegistryAndSession(): SessionArtifactSummary { } function registryVersion(): unknown { - return readJsonFile(REGISTRY_FILE, {}).sandboxes?.[SANDBOX_NAME]?.agentVersion; + return registrySandbox().agentVersion; +} + +function registrySandbox(): Record { + const sandbox = readJsonFile(REGISTRY_FILE, {}).sandboxes?.[SANDBOX_NAME]; + expect(sandbox, `registry entry missing for ${SANDBOX_NAME}`).toBeDefined(); + return sandbox as Record; } test.skipIf(!shouldRunLiveE2E())( @@ -424,8 +435,7 @@ test.skipIf(!shouldRunLiveE2E())( redactionValues, timeoutMs: INSTALL_TIMEOUT_MS, }); - install.exitCode === 0 || - (await artifacts.writeText("phase-1-install-nonzero-note.txt", resultText(install))); + expectExitZero(install, "NemoClaw install.sh"); const cliProbe = await host.command( "bash", @@ -439,6 +449,23 @@ test.skipIf(!shouldRunLiveE2E())( ); expectExitZero(cliProbe, "NemoClaw/OpenShell installed by install.sh"); + const gatewayProbe = await host.command("openshell", ["gateway", "info", "-g", "nemoclaw"], { + artifactName: "phase-1-gateway-probe", + env: testEnv(apiKey), + redactionValues, + timeoutMs: 30_000, + }); + expectExitZero(gatewayProbe, "NemoClaw install must leave a reusable 'nemoclaw' gateway"); + + const phase1DashboardPort = registrySandbox().dashboardPort; + expect( + typeof phase1DashboardPort === "number" && + Number.isInteger(phase1DashboardPort) && + phase1DashboardPort > 0 && + phase1DashboardPort <= 65535, + "initial Hermes onboard must persist the dashboard port used by authoritative rebuild", + ).toBe(true); + const deleteCurrentSandbox = await host.command( "openshell", ["sandbox", "delete", SANDBOX_NAME], @@ -474,7 +501,7 @@ test.skipIf(!shouldRunLiveE2E())( "--build-arg", `HERMES_NPM_INTEGRITY=${OLD_HERMES_NPM_INTEGRITY}`, "--build-arg", - "HERMES_UV_EXTRAS=messaging", + "HERMES_UV_EXTRAS=messaging mcp", "-f", path.join(REPO_ROOT, "agents", "hermes", "Dockerfile.base"), "-t", @@ -610,18 +637,16 @@ test.skipIf(!shouldRunLiveE2E())( expectExitZero(preConfig, "read pre-rebuild Hermes config.yaml"); expect(preConfig.stdout).toContain("discord:"); - const sessionSummary = seedRegistryAndSession(); + const sessionSummary = seedRegistryAndSession(phase1DashboardPort as number); + const seededRegistry = registrySandbox(); await artifacts.writeJson("phase-4-registry-session-summary.json", { - registryVersion: registryVersion(), + registryVersion: seededRegistry.agentVersion, + dashboardPort: seededRegistry.dashboardPort, registryInference: { - provider: readJsonFile(REGISTRY_FILE, {}).sandboxes?.[SANDBOX_NAME]?.provider, - endpointUrl: readJsonFile(REGISTRY_FILE, {}).sandboxes?.[SANDBOX_NAME] - ?.endpointUrl, - credentialEnv: readJsonFile(REGISTRY_FILE, {}).sandboxes?.[SANDBOX_NAME] - ?.credentialEnv, - preferredInferenceApi: readJsonFile(REGISTRY_FILE, {}).sandboxes?.[ - SANDBOX_NAME - ]?.preferredInferenceApi, + provider: seededRegistry.provider, + endpointUrl: seededRegistry.endpointUrl, + credentialEnv: seededRegistry.credentialEnv, + preferredInferenceApi: seededRegistry.preferredInferenceApi, }, session: sessionSummary, }); diff --git a/test/e2e/live/rebuild-openclaw.test.ts b/test/e2e/live/rebuild-openclaw.test.ts index 7660370793f..6ff5dc4bd97 100644 --- a/test/e2e/live/rebuild-openclaw.test.ts +++ b/test/e2e/live/rebuild-openclaw.test.ts @@ -5,14 +5,13 @@ import { Buffer } from "node:buffer"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; - +import { shellQuote } from "../../../src/lib/core/shell-quote"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; import type { HostCliClient } from "../fixtures/clients/host.ts"; import { validateSandboxName } from "../fixtures/clients/sandbox.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; import { shouldRunLiveE2E } from "../fixtures/live-project-gate.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; -import { shellQuote } from "../../../src/lib/core/shell-quote"; import { createOldBaseBuildContext } from "./rebuild-openclaw-old-base-context.ts"; // The contract stays intentionally local to this live test: build an older @@ -218,7 +217,7 @@ async function configureGatewayInferenceRoute( ); } -function seedRegistryAndSession(): void { +function seedRegistryAndSession(dashboardPort: number): void { // The legacy rebuild regression requires an intentionally old OpenClaw sandbox // that NemoClaw cannot create through the normal onboard path because current // blueprints reject versions below min_openclaw_version. Create that sandbox @@ -241,6 +240,11 @@ function seedRegistryAndSession(): void { policyTier: null, agent: null, agentVersion: OLD_OPENCLAW_VERSION, + dashboardPort, + // This test creates an old NemoClaw-managed runtime directly through + // OpenShell. Record the managed-image provenance explicitly so rebuild + // does not have to guess whether an omitted legacy value meant `--from`. + fromDockerfile: null, }; registry.defaultSandbox = SANDBOX_NAME; writeJsonFile(REGISTRY_FILE, registry); @@ -465,6 +469,15 @@ test.skipIf(!shouldRunLiveE2E())( }); } + const phase1DashboardPort = registrySandbox().dashboardPort; + expect( + typeof phase1DashboardPort === "number" && + Number.isInteger(phase1DashboardPort) && + phase1DashboardPort > 0 && + phase1DashboardPort <= 65535, + "initial onboard must persist the dashboard port used by authoritative rebuild", + ).toBe(true); + await openshellBestEffort( host, ["sandbox", "delete", SANDBOX_NAME], @@ -612,7 +625,7 @@ print(json.dumps({'seeded': saved == os.environ['PRE_REBUILD_GATEWAY_TOKEN'], 'h const preRebuildConfigHash = preHashResult.stdout.trim(); expect(preRebuildConfigHash).toContain("openclaw.json"); - seedRegistryAndSession(); + seedRegistryAndSession(phase1DashboardPort as number); const sessionAfterSeed = readJsonFile>(SESSION_FILE, {}); const seededSteps = sessionAfterSeed.steps as Record | undefined; const seededSandbox = registrySandbox(); @@ -621,6 +634,7 @@ print(json.dumps({'seeded': saved == os.environ['PRE_REBUILD_GATEWAY_TOKEN'], 'h name: seededSandbox.name, provider: seededSandbox.provider, agentVersion: seededSandbox.agentVersion, + dashboardPort: seededSandbox.dashboardPort, policyCount: Array.isArray(seededSandbox.policies) ? seededSandbox.policies.length : 0, }, session: { diff --git a/test/e2e/live/upgrade-stale-sandbox-helpers.ts b/test/e2e/live/upgrade-stale-sandbox-helpers.ts index 911d014afd6..129ad413781 100644 --- a/test/e2e/live/upgrade-stale-sandbox-helpers.ts +++ b/test/e2e/live/upgrade-stale-sandbox-helpers.ts @@ -17,6 +17,10 @@ export const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); const BLUEPRINT_RELPATH = path.join("nemoclaw-blueprint", "blueprint.yaml"); const BLUEPRINT = path.join(REPO_ROOT, BLUEPRINT_RELPATH); const BASE_CONTEXT_SCRIPT_RELPATH = path.join("scripts", "lib", "sandbox-rlimits.sh"); +const MCPORTER_RUNTIME_RELPATHS = [ + path.join("agents", "openclaw", "mcporter-runtime", "package.json"), + path.join("agents", "openclaw", "mcporter-runtime", "package-lock.json"), +]; const TEST_SANDBOX_PREFIX = "e2e-upgrade-stale"; export const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? @@ -112,6 +116,11 @@ function createOldBaseBuildContext(): string { path.join(REPO_ROOT, BASE_CONTEXT_SCRIPT_RELPATH), path.join(buildContext, BASE_CONTEXT_SCRIPT_RELPATH), ); + for (const relativePath of MCPORTER_RUNTIME_RELPATHS) { + const target = path.join(buildContext, relativePath); + fs.mkdirSync(path.dirname(target), { recursive: true }); + fs.copyFileSync(path.join(REPO_ROOT, relativePath), target); + } return buildContext; } @@ -134,6 +143,14 @@ export function writeStaleRegistryEntry(): void { sandboxes?: Record>; defaultSandbox?: string; }>(REGISTRY_FILE, {}); + const dashboardPort = registry.sandboxes?.[SANDBOX_NAME]?.dashboardPort; + expect( + typeof dashboardPort === "number" && + Number.isInteger(dashboardPort) && + dashboardPort > 0 && + dashboardPort <= 65535, + "initial onboard must persist the dashboard port used by authoritative rebuild", + ).toBe(true); registry.sandboxes = registry.sandboxes ?? {}; registry.sandboxes[SANDBOX_NAME] = { name: SANDBOX_NAME, @@ -143,6 +160,8 @@ export function writeStaleRegistryEntry(): void { gpuEnabled: false, policies: [], policyTier: null, + fromDockerfile: null, + dashboardPort, agent: null, agentVersion: OLD_OPENCLAW_VERSION, }; diff --git a/test/e2e/setup-mcp-test-tls.sh b/test/e2e/setup-mcp-test-tls.sh new file mode 100755 index 00000000000..8c3c5c50d4e --- /dev/null +++ b/test/e2e/setup-mcp-test-tls.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +: "${RUNNER_TEMP:?RUNNER_TEMP is required}" +: "${GITHUB_ENV:?GITHUB_ENV is required}" + +tls_dir="${RUNNER_TEMP}/nemoclaw-mcp-tls" +install -d -m 700 "${tls_dir}" + +openssl req \ + -x509 \ + -newkey rsa:2048 \ + -sha256 \ + -nodes \ + -days 1 \ + -subj "/CN=NemoClaw MCP E2E Root CA" \ + -addext "basicConstraints=critical,CA:TRUE" \ + -addext "keyUsage=critical,keyCertSign,cRLSign" \ + -keyout "${tls_dir}/ca.key" \ + -out "${tls_dir}/ca.crt" + +openssl req \ + -newkey rsa:2048 \ + -sha256 \ + -nodes \ + -subj "/CN=host.openshell.internal" \ + -addext "subjectAltName=DNS:host.openshell.internal,DNS:mcp-rebind.example.test" \ + -keyout "${tls_dir}/server.key" \ + -out "${tls_dir}/server.csr" + +openssl x509 \ + -req \ + -sha256 \ + -days 1 \ + -in "${tls_dir}/server.csr" \ + -CA "${tls_dir}/ca.crt" \ + -CAkey "${tls_dir}/ca.key" \ + -CAcreateserial \ + -extfile <(printf '%s\n' \ + "basicConstraints=critical,CA:FALSE" \ + "keyUsage=critical,digitalSignature,keyEncipherment" \ + "extendedKeyUsage=serverAuth" \ + "subjectAltName=DNS:host.openshell.internal,DNS:mcp-rebind.example.test") \ + -out "${tls_dir}/server.crt" + +# The self-signed certificate secures only the loopback origin hop from +# cloudflared, which is launched with --no-tls-verify for that local fixture. +# Successful sandbox MCP connections use the public trycloudflare URL and its +# publicly trusted edge certificate. The direct DNS-rebinding fixture is denied +# by policy before TLS, so sandboxes never install or trust this private test CA. +{ + echo "NEMOCLAW_MCP_TLS_CERT=${tls_dir}/server.crt" + echo "NEMOCLAW_MCP_TLS_KEY=${tls_dir}/server.key" +} >>"${GITHUB_ENV}" diff --git a/test/e2e/support/e2e-clients.test.ts b/test/e2e/support/e2e-clients.test.ts index ee207e55886..ce9ba6b40c0 100644 --- a/test/e2e/support/e2e-clients.test.ts +++ b/test/e2e/support/e2e-clients.test.ts @@ -1,6 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { Buffer } from "node:buffer"; +import { spawnSync } from "node:child_process"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; @@ -379,6 +381,7 @@ describe("E2E fixture clients", () => { const runner = new FakeRunner(); const sandbox = new SandboxClient(runner, { openshellPath: "openshell" }); const script = trustedSandboxShellScript("echo ready"); + const encodedScript = Buffer.from(script, "utf8").toString("base64"); expectTypeOf< Parameters[1] @@ -391,7 +394,20 @@ describe("E2E fixture clients", () => { expect(runner.calls[0]).toEqual({ command: "openshell", - args: ["sandbox", "exec", "-n", "assistant", "--", "sh", "-lc", "echo ready"], + args: [ + "sandbox", + "exec", + "-n", + "assistant", + "--", + "sh", + "-lc", + [ + "command -v base64 >/dev/null 2>&1 || { echo NEMOCLAW_BASE64_MISSING >&2; exit 127; }", + `_NEMOCLAW_E2E_SCRIPT="$(printf '%s' '${encodedScript}' | base64 -d)" || exit $?`, + `eval "$_NEMOCLAW_E2E_SCRIPT"`, + ].join("; "), + ], options: { artifactName: "custom-exec-shell", timeoutMs: 123, @@ -399,22 +415,37 @@ describe("E2E fixture clients", () => { }); }); - it("encodes multiline shell scripts into an OpenShell-safe single argument", async () => { + it("sandbox client keeps multiline shell scripts out of OpenShell argv", async () => { const runner = new FakeRunner(); const sandbox = new SandboxClient(runner, { openshellPath: "openshell" }); - const source = "set -e\nprintf 'ready\\n'\n"; + const script = trustedSandboxShellScript("set -eu\nprintf '%s\\n' ready\r\n"); - await sandbox.execShell("assistant", trustedSandboxShellScript(source)); + await sandbox.execShell("assistant", script); - const argument = runner.calls[0]?.args.at(-1) ?? ""; - expect(argument).not.toMatch(/[\r\n]/u); - const encoded = argument.match(/'([A-Za-z0-9+/=]+)' \| base64 -d/u)?.[1]; - expect(encoded).toBeTruthy(); - expect(Buffer.from(encoded ?? "", "base64").toString("utf8")).toBe(source); + const payload = runner.calls[0]?.args.at(-1) ?? ""; + expect(payload).not.toMatch(/[\r\n]/); + const encodedScript = payload.match(/'([A-Za-z0-9+/=]+)'/)?.[1] ?? ""; + expect(Buffer.from(encodedScript, "base64").toString("utf8")).toBe(script); + }); + + it("sandbox client fails closed when the sandbox has no base64 decoder", async () => { + const runner = new FakeRunner(); + const sandbox = new SandboxClient(runner, { openshellPath: "openshell" }); + + await sandbox.execShell("assistant", trustedSandboxShellScript("echo should-not-run")); + + const payload = runner.calls[0]?.args.at(-1) ?? ""; + const result = spawnSync("/bin/sh", ["-c", payload], { + encoding: "utf8", + env: { PATH: "" }, + }); + expect(result.status).toBe(127); + expect(result.stderr).toContain("NEMOCLAW_BASE64_MISSING"); + expect(result.stdout).not.toContain("should-not-run"); }); it("sandbox client requires trusted non-empty shell scripts", () => { - expect(() => trustedSandboxShellScript("")).toThrow(/must be non-empty/); + expect(() => trustedSandboxShellScript("")).toThrow(/must not be empty/); expect(() => trustedSandboxShellScript("echo ready\0ignored")).toThrow(/no NUL bytes/); expectTypeOf[1]>().not.toEqualTypeOf(); }); diff --git a/test/e2e/support/e2e-live-project-config.test.ts b/test/e2e/support/e2e-live-project-config.test.ts index f2b44c1b311..f9de3bc564f 100644 --- a/test/e2e/support/e2e-live-project-config.test.ts +++ b/test/e2e/support/e2e-live-project-config.test.ts @@ -3,7 +3,6 @@ import { describe, expect, it } from "vitest"; import config from "../../../vitest.config.ts"; -import { resolveE2ERetryCount } from "../../helpers/e2e-retries.ts"; import { readYaml, type WorkflowStep } from "../../helpers/e2e-workflow-contract.ts"; import { shouldRunBranchValidationE2E, @@ -27,6 +26,7 @@ interface RootConfig { const INSTALLER_INTEGRATION_TESTS = [ "test/install-express-prompt.test.ts", + "test/install-build-dependency-preflight.test.ts", "test/install-preflight.test.ts", "test/install-preflight-docker-bootstrap.test.ts", "test/install-openshell-version-check.test.ts", @@ -90,29 +90,11 @@ describe("gated E2E Vitest projects", () => { expect(shouldRunBranchValidationE2E({ NEMOCLAW_RUN_BRANCH_VALIDATION_E2E: "1" })).toBe(true); }); - it("configures automatic retries only for live E2E Vitest projects", () => { - const expectedRetries = resolveE2ERetryCount(); - + it("keeps both stateful E2E projects single-shot", () => { expect(projectConfig("cli").test?.retry).toBeUndefined(); expect(projectConfig("e2e-support").test?.retry).toBeUndefined(); - expect(projectConfig("e2e-live").test?.retry).toBe(expectedRetries); - expect(projectConfig("e2e-branch-validation").test?.retry).toBe(expectedRetries); - }); - - it("defaults live E2E retries to CI only and supports explicit overrides", () => { - expect(resolveE2ERetryCount({})).toBe(0); - expect(resolveE2ERetryCount({ CI: "0" })).toBe(0); - expect(resolveE2ERetryCount({ CI: "1" })).toBe(2); - expect(resolveE2ERetryCount({ CI: "true" })).toBe(2); - expect(resolveE2ERetryCount({ GITHUB_ACTIONS: "true" })).toBe(2); - expect(resolveE2ERetryCount({ CI: "1", NEMOCLAW_E2E_RETRIES: "0" })).toBe(0); - expect(resolveE2ERetryCount({ NEMOCLAW_E2E_RETRIES: "3" })).toBe(3); - expect(resolveE2ERetryCount({ NEMOCLAW_E2E_RETRIES: "5" })).toBe(5); - expect(resolveE2ERetryCount({ NEMOCLAW_E2E_RETRIES: "6" })).toBe(5); - expect(resolveE2ERetryCount({ NEMOCLAW_E2E_RETRIES: "999999" })).toBe(5); - expect(resolveE2ERetryCount({ CI: "1", NEMOCLAW_E2E_RETRIES: "-1" })).toBe(2); - expect(resolveE2ERetryCount({ CI: "1", NEMOCLAW_E2E_RETRIES: "1.5" })).toBe(2); - expect(resolveE2ERetryCount({ CI: "1", NEMOCLAW_E2E_RETRIES: "invalid" })).toBe(2); + expect(projectConfig("e2e-live").test?.retry).toBe(0); + expect(projectConfig("e2e-branch-validation").test?.retry).toBe(0); }); it("sets the branch-validation sentinel in the reusable workflow live E2E step", () => { diff --git a/test/e2e/support/hosted-inference.test.ts b/test/e2e/support/hosted-inference.test.ts index 1cb5d67f691..dc3c9b08e52 100644 --- a/test/e2e/support/hosted-inference.test.ts +++ b/test/e2e/support/hosted-inference.test.ts @@ -195,11 +195,13 @@ describe("hosted inference E2E config", () => { HOME: "/tmp/home", PATH: "/usr/bin", NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1", + NEMOCLAW_OPENSHELL_CHANNEL: "dev", NVIDIA_INFERENCE_API_KEY: "repo-hosted-key", RANDOM_NON_SECRET: "not-allowlisted", }); expect(env.NEMOCLAW_E2E_USE_HOSTED_INFERENCE).toBe("1"); + expect(env.NEMOCLAW_OPENSHELL_CHANNEL).toBe("dev"); expect(env).not.toHaveProperty("NVIDIA_INFERENCE_API_KEY"); expect(env).not.toHaveProperty("RANDOM_NON_SECRET"); }); diff --git a/test/e2e/support/jetson-workflow-boundary.test.ts b/test/e2e/support/jetson-workflow-boundary.test.ts index 906e90bbe6f..0bd93539385 100644 --- a/test/e2e/support/jetson-workflow-boundary.test.ts +++ b/test/e2e/support/jetson-workflow-boundary.test.ts @@ -23,7 +23,7 @@ describe("Jetson nvmap GPU E2E workflow boundary", () => { expect(inventory.allowedJobs).toContain("jetson-nvmap-gpu"); expect(inventory.explicitOnlyJobs).toContain("jetson-nvmap-gpu"); expect(formatFreeStandingJobsInventoryForShell(inventory)).toContain( - "explicit_only_jobs_csv=openshell-gateway-auth-contract,hermes-gpu-startup,sandbox-rlimits-connect,jetson-nvmap-gpu", + "explicit_only_jobs_csv=openshell-gateway-auth-contract,mcp-bridge-dev,hermes-gpu-startup,sandbox-rlimits-connect,jetson-nvmap-gpu", ); expect(inventory.targetToJob.get("jetson-nvmap-gpu")).toBe("jetson-nvmap-gpu"); expect(evaluateE2eWorkflowDispatchSelectors({}).selectedFreeStandingJobs).not.toContain( diff --git a/test/e2e/support/mcp-bridge-sandbox.test.ts b/test/e2e/support/mcp-bridge-sandbox.test.ts new file mode 100644 index 00000000000..4e46c90c50e --- /dev/null +++ b/test/e2e/support/mcp-bridge-sandbox.test.ts @@ -0,0 +1,399 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, describe, expect, it } from "vitest"; +import YAML from "yaml"; + +import { testTimeout } from "../../helpers/timeouts"; +import type { HostCliClient } from "../fixtures/clients/host.ts"; +import { + buildMcpDnsRebindingProbeScript, + isExpectedMcpCurlPolicyDenial, + restoreDnsRebindingHostsFixture, +} from "../live/mcp-bridge-sandbox.ts"; +import { + buildRawOpenShellAllowedIpsRebindingPolicy, + buildRawOpenShellAllowedIpsRebindingProbeScript, + parseRawOpenShellAllowedIpsRebindingEndpoint, + RAW_OPENSHELL_REBIND_HOSTNAME, + RAW_OPENSHELL_REBIND_HTTP_CODE_MARKER, + RAW_OPENSHELL_REBIND_PINNED_IP, + RAW_OPENSHELL_REBIND_POLICY_KEY, +} from "../live/openshell-allowed-ips-rebinding.ts"; + +const SUITE_OPTIONS = { timeout: testTimeout(15_000) }; +const tempDirs: string[] = []; + +afterEach(() => { + for (const tempDir of tempDirs.splice(0)) { + fs.rmSync(tempDir, { force: true, recursive: true }); + } +}); + +function fakeCurlPath(): string { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-raw-rebind-")); + tempDirs.push(tempDir); + const curl = path.join(tempDir, "curl"); + fs.writeFileSync( + curl, + '#!/bin/sh\nprintf %s "${FAKE_HTTP_STATUS:-000}"\nexit "${FAKE_CURL_RC:-0}"\n', + { mode: 0o755 }, + ); + return tempDir; +} + +function denialResult( + overrides: { + exitCode?: number | null; + stderr?: string; + stdout?: string; + timedOut?: boolean; + } = {}, +) { + return { + exitCode: overrides.exitCode ?? 0, + stderr: overrides.stderr ?? "", + stdout: overrides.stdout ?? "", + timedOut: overrides.timedOut ?? false, + }; +} + +async function captureRestoreScript(hostBackupPath: string, sandboxBackupPath: string) { + let restoreScript = ""; + const host = { + command: async (_command: string, args: string[]) => { + restoreScript = args[1] ?? ""; + return denialResult(); + }, + } as unknown as HostCliClient; + + await restoreDnsRebindingHostsFixture(host, "test-sandbox", { + hostname: "mcp-rebind.example.test", + hostBackupPath, + sandboxBackupPath, + }); + return restoreScript; +} + +describe("MCP curl policy denial classification", SUITE_OPTIONS, () => { + it("accepts an L7 HTTP 403 denial", () => { + expect( + isExpectedMcpCurlPolicyDenial(denialResult({ stdout: "NEMOCLAW_MCP_CURL_HTTP_CODE=403\n" })), + ).toBe(true); + }); + + it("accepts curl exit 56 only for a CONNECT proxy 403", () => { + expect( + isExpectedMcpCurlPolicyDenial( + denialResult({ + exitCode: 56, + stderr: "curl: (56) CONNECT tunnel failed, response 403\n", + stdout: "NEMOCLAW_MCP_CURL_HTTP_CODE=\n", + }), + ), + ).toBe(true); + + expect( + isExpectedMcpCurlPolicyDenial( + denialResult({ exitCode: 56, stderr: "curl: (56) Failure when receiving data" }), + ), + ).toBe(false); + }); + + it("rejects allowed, unrelated, and timed-out results", () => { + expect( + isExpectedMcpCurlPolicyDenial(denialResult({ stdout: "NEMOCLAW_MCP_CURL_HTTP_CODE=200\n" })), + ).toBe(false); + expect( + isExpectedMcpCurlPolicyDenial( + denialResult({ exitCode: 7, stderr: "curl: (7) Connection refused" }), + ), + ).toBe(false); + expect( + isExpectedMcpCurlPolicyDenial( + denialResult({ + exitCode: 56, + stderr: "curl: (56) CONNECT tunnel failed, response 403", + timedOut: true, + }), + ), + ).toBe(false); + }); + + it("runs the rebinding request beneath each adapter runtime identity", () => { + const runtimes = { + mcporter: "nemoclaw-start node -e", + "hermes-config": "/opt/hermes/.venv/bin/python -c", + "deepagents-config": "/opt/venv/bin/python3 -c", + } as const; + + for (const [adapter, runtime] of Object.entries(runtimes)) { + const script = buildMcpDnsRebindingProbeScript( + adapter as keyof typeof runtimes, + "https://mcp-rebind.example.test:31337/mcp", + "REBIND_MCP_SECRET", + ); + expect(script, adapter).toContain(runtime); + expect(script, adapter).toMatch(/spawnSync|subprocess\.run/); + expect(script, adapter).toContain("'curl'"); + expect(script, adapter).toContain("NEMOCLAW_MCP_CURL_HTTP_CODE=%{http_code}"); + expect(script, adapter).toContain( + "authorization: Bearer openshell:resolve:env:REBIND_MCP_SECRET", + ); + expect(script, adapter).not.toContain("fake-rebind-mcp-secret-value"); + const syntax = spawnSync("/bin/bash", ["-n"], { input: script, encoding: "utf8" }); + expect(syntax.status, `${adapter}: ${syntax.stderr}`).toBe(0); + } + }); + + it("pins the resolve-validate-connect source contract to OpenShell v0.0.72", () => { + const commit = "8cb16de9eae4c44d7d31e1493747d8c10abb5963"; + const sourcePath = "crates/openshell-supervisor-network/src/proxy.rs"; + const citations = [ + `${sourcePath}:2476-2502`, + `${sourcePath}:2527-2567`, + `${sourcePath}:2622-2630`, + `${sourcePath}:822-832`, + `${sourcePath}:3885-3893`, + `${sourcePath}:4123-4125`, + ]; + + for (const docsPath of [ + "docs/deployment/set-up-mcp-bridge.mdx", + "docs/security/openshell-0.0.72-compatibility-review.mdx", + ]) { + const docs = fs.readFileSync(docsPath, "utf8"); + expect(docs, docsPath).toContain(commit); + for (const citation of citations) expect(docs, docsPath).toContain(citation); + } + }); + + it("adds one raw MCP policy with an exact public IP pin and no adapter identity", () => { + const rendered = buildRawOpenShellAllowedIpsRebindingPolicy( + `version: 1 +filesystem_policy: + include_workdir: true +network_policies: + existing: + name: existing + endpoints: [] + binaries: [] +`, + 31337, + ); + const parsed = YAML.parse(rendered) as { + network_policies: Record< + string, + { + binaries: Array<{ path: string }>; + endpoints: Array>; + } + >; + }; + + expect(parsed.network_policies.existing).toBeDefined(); + const raw = parsed.network_policies[RAW_OPENSHELL_REBIND_POLICY_KEY]; + expect(raw.binaries).toEqual([{ path: "/**" }]); + expect(raw.endpoints).toEqual([ + expect.objectContaining({ + allowed_ips: [RAW_OPENSHELL_REBIND_PINNED_IP], + host: RAW_OPENSHELL_REBIND_HOSTNAME, + path: "/mcp", + port: 31337, + protocol: "mcp", + rules: [{ allow: { method: "tools/list" } }], + }), + ]); + }); + + it("reads the effective raw policy semantically when OpenShell quotes allowed IPs", () => { + const endpoint = parseRawOpenShellAllowedIpsRebindingEndpoint(`Version: 1 +--- +version: 1 +network_policies: + ${RAW_OPENSHELL_REBIND_POLICY_KEY}: + endpoints: + - host: ${RAW_OPENSHELL_REBIND_HOSTNAME} + port: 31337 + protocol: mcp + allowed_ips: + - '${RAW_OPENSHELL_REBIND_PINNED_IP}' +`); + + expect(endpoint).toMatchObject({ + allowed_ips: [RAW_OPENSHELL_REBIND_PINNED_IP], + host: RAW_OPENSHELL_REBIND_HOSTNAME, + port: 31337, + protocol: "mcp", + }); + }); + + it("passes only an exact HTTP 403 and rejects an allowed response", () => { + const binDir = fakeCurlPath(); + const script = buildRawOpenShellAllowedIpsRebindingProbeScript( + `http://${RAW_OPENSHELL_REBIND_HOSTNAME}:31337/mcp`, + ); + const run = (status: string, curlRc = "0") => + spawnSync("/bin/bash", ["-c", script], { + encoding: "utf8", + env: { + ...process.env, + FAKE_CURL_RC: curlRc, + FAKE_HTTP_STATUS: status, + PATH: `${binDir}:${process.env.PATH ?? ""}`, + }, + }); + + const denied = run("403"); + expect(denied.status, denied.stderr).toBe(0); + expect(denied.stdout).toContain(`${RAW_OPENSHELL_REBIND_HTTP_CODE_MARKER}403`); + + const allowed = run("200"); + expect(allowed.status).toBe(1); + expect(allowed.stdout).toContain(`${RAW_OPENSHELL_REBIND_HTTP_CODE_MARKER}200`); + + const transportFailure = run("000", "7"); + expect(transportFailure.status).toBe(7); + }); + + it("runs the raw proof in both MCP lanes without calling an adapter and restores policy", () => { + const mcpBridgeSource = fs.readFileSync("test/e2e/live/mcp-bridge.test.ts", "utf8"); + const networkPolicySource = fs.readFileSync("test/e2e/live/network-policy.test.ts", "utf8"); + const contractSource = fs.readFileSync( + "test/e2e/live/openshell-allowed-ips-rebinding.ts", + "utf8", + ); + expect( + mcpBridgeSource.match(/await assertRawOpenShellAllowedIpsRebindingDenied/g), + ).toHaveLength(1); + expect(networkPolicySource).not.toContain("assertRawOpenShellAllowedIpsRebindingDenied"); + expect(contractSource).toContain('["policy", "set", "--policy"'); + expect(contractSource).toContain("server.requestCount()"); + expect(contractSource).toContain("raw-openshell-rebinding-policy-restore"); + expect(contractSource).toContain("raw-openshell-rebinding-policy-verify-restored"); + expect(contractSource.indexOf("raw-openshell-rebinding-policy-restore")).toBeGreaterThan( + contractSource.indexOf("} finally {"), + ); + expect(contractSource).toContain( + "https://github.com/NVIDIA/OpenShell/blob/8cb16de9eae4c44d7d31e1493747d8c10abb5963/", + ); + expect(contractSource).not.toContain("host.nemoclaw"); + expect(contractSource).not.toContain("assertAdapterDnsRebindingDenied"); + }); + + it("runs the zero-upstream rebinding proof for all three adapters", () => { + const source = fs.readFileSync("test/e2e/live/mcp-bridge.test.ts", "utf8"); + + expect(source.match(/await assertAdapterDnsRebindingDenied/g)).toHaveLength(3); + for (const adapter of [ + 'adapter: "mcporter"', + 'adapter: "hermes-config"', + 'adapter: "deepagents-config"', + ]) { + expect(source).toContain(adapter); + } + expect(source).toContain("rebound request must not reach the upstream MCP server"); + expect(source).toContain(").toHaveLength(0);"); + }); + + it("restores the DNS fixture before MCP removal can restart the sandbox", () => { + const source = fs.readFileSync("test/e2e/live/mcp-bridge.test.ts", "utf8"); + const denialProof = source.indexOf("rebound request must not reach the upstream MCP server"); + const restore = source.indexOf("await restoreDnsRebindingHostsFixture", denialProof); + const remove = source.indexOf("const remove = await host.nemoclaw", denialProof); + + expect(denialProof).toBeGreaterThanOrEqual(0); + expect(restore).toBeGreaterThan(denialProof); + expect(remove).toBeGreaterThan(restore); + }); + + it("restores host DNS strictly while treating the ephemeral sandbox as best effort", async () => { + const restoreScript = await captureRestoreScript("/tmp/host-backup", "/tmp/sandbox-backup"); + + expect(restoreScript).toContain("set -uo pipefail"); + expect(restoreScript).not.toContain("set -euo pipefail"); + expect(restoreScript).toContain('if ! sudo -n tee /etc/hosts < "$host_backup"'); + expect(restoreScript).toContain('if ! cmp -s "$host_backup" /etc/hosts'); + expect(restoreScript).toContain("host_restore_failed=1"); + expect(restoreScript).toContain('if [ "$host_restore_failed" -ne 0 ]; then exit 1; fi'); + expect(restoreScript).toContain("for attempt in 1 2 3; do"); + expect(restoreScript).toContain('docker exec --user 0 -i "$container_id"'); + expect(restoreScript).toContain( + "::warning::could not restore ephemeral sandbox /etc/hosts; cleanup will destroy the sandbox", + ); + expect(restoreScript).toContain("failed to remove DNS rebinding hosts backups"); + }); + + it("executes every restore outcome without an unlabeled errexit", async () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mcp-restore-")); + const binDir = path.join(tempDir, "bin"); + const hostBackupPath = path.join(tempDir, "host-backup"); + const sandboxBackupPath = path.join(tempDir, "sandbox-backup"); + const fakeHostsPath = path.join(tempDir, "hosts"); + fs.mkdirSync(binDir); + const writeExecutable = (name: string, source: string) => { + const target = path.join(binDir, name); + fs.writeFileSync(target, source, { mode: 0o755 }); + }; + writeExecutable( + "sudo", + '#!/bin/sh\n[ "${FAKE_SUDO_STATUS:-0}" -eq 0 ] || exit "$FAKE_SUDO_STATUS"\ncat > "$FAKE_HOSTS_PATH"\n', + ); + writeExecutable("cmp", '#!/bin/sh\nexit "${FAKE_CMP_STATUS:-0}"\n'); + writeExecutable( + "docker", + '#!/bin/sh\nif [ "$1" = ps ]; then echo fake-container; exit 0; fi\nif [ "$1" = exec ]; then cat >/dev/null; exit "${FAKE_DOCKER_EXEC_STATUS:-0}"; fi\nexit 64\n', + ); + writeExecutable("sleep", "#!/bin/sh\nexit 0\n"); + + try { + const restoreScript = await captureRestoreScript(hostBackupPath, sandboxBackupPath); + const runRestore = (extraEnv: Record = {}) => + spawnSync("/bin/bash", ["-c", restoreScript], { + encoding: "utf8", + env: { + ...process.env, + PATH: `${binDir}:${process.env.PATH ?? ""}`, + FAKE_HOSTS_PATH: fakeHostsPath, + ...extraEnv, + }, + }); + const resetBackups = () => { + fs.writeFileSync(hostBackupPath, "original host entries\n"); + fs.writeFileSync(sandboxBackupPath, "original sandbox entries\n"); + }; + + resetBackups(); + const success = runRestore(); + expect(success.status, success.stderr).toBe(0); + expect(success.stdout).toContain("restored host /etc/hosts"); + expect(success.stdout).toContain("restored sandbox /etc/hosts"); + expect(success.stdout).toContain("removed DNS rebinding hosts backups"); + expect(fs.existsSync(hostBackupPath)).toBe(false); + expect(fs.existsSync(sandboxBackupPath)).toBe(false); + + resetBackups(); + const hostFailure = runRestore({ FAKE_SUDO_STATUS: "1" }); + expect(hostFailure.status).toBe(1); + expect(hostFailure.stderr).toContain("failed to restore host /etc/hosts"); + expect(fs.existsSync(hostBackupPath)).toBe(true); + expect(fs.existsSync(sandboxBackupPath)).toBe(true); + + resetBackups(); + const sandboxFailure = runRestore({ FAKE_DOCKER_EXEC_STATUS: "1" }); + expect(sandboxFailure.status, sandboxFailure.stderr).toBe(0); + expect(sandboxFailure.stderr).toContain( + "::warning::could not restore ephemeral sandbox /etc/hosts; cleanup will destroy the sandbox", + ); + expect(fs.existsSync(hostBackupPath)).toBe(false); + expect(fs.existsSync(sandboxBackupPath)).toBe(false); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } + }); +}); diff --git a/test/e2e/support/mcp-workflow-boundary.test.ts b/test/e2e/support/mcp-workflow-boundary.test.ts new file mode 100644 index 00000000000..51eabd2671b --- /dev/null +++ b/test/e2e/support/mcp-workflow-boundary.test.ts @@ -0,0 +1,145 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; +import YAML from "yaml"; + +import { validateMcpOpenShellWorkflowBoundary } from "../../../tools/e2e/mcp-workflow-boundary.mts"; + +describe("MCP workflow artifact boundary", () => { + it("rejects upload action or path drift from the reviewed shared boundary", () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mcp-workflow-")); + const workflowPath = path.join(directory, "e2e.yaml"); + try { + const workflow = YAML.parse(fs.readFileSync(".github/workflows/e2e.yaml", "utf8")) as { + jobs: Record< + string, + { steps: Array<{ name?: string; uses?: string; with?: Record }> } + >; + }; + const upload = workflow.jobs["mcp-bridge"].steps.find( + (step) => step.name === "Upload MCP server artifacts", + ); + assert(upload?.with, "MCP artifact upload fixture is missing"); + upload.uses = "NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@main"; + upload.with.path = "e2e-artifacts/live/unscanned/"; + fs.writeFileSync(workflowPath, YAML.stringify(workflow)); + + expect(validateMcpOpenShellWorkflowBoundary(workflowPath)).toEqual( + expect.arrayContaining([ + "mcp-bridge artifact upload must use the reviewed shared uploader", + "mcp-bridge artifact upload must use exactly the scanned directory", + ]), + ); + } finally { + fs.rmSync(directory, { force: true, recursive: true }); + } + }); + + it("rejects an unverified or mutable cloudflared installer in either MCP lane", () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mcp-workflow-")); + const workflowPath = path.join(directory, "e2e.yaml"); + try { + const workflow = YAML.parse(fs.readFileSync(".github/workflows/e2e.yaml", "utf8")) as { + jobs: Record< + string, + { + steps: Array<{ + env?: Record; + name?: string; + run?: string; + }>; + } + >; + }; + const cloudflared = workflow.jobs["mcp-bridge-dev"].steps.find( + (step) => step.name === "Install and verify cloudflared prerequisite", + ); + assert(cloudflared?.env, "MCP cloudflared installer fixture is missing"); + cloudflared.env.CLOUDFLARED_DEB_SHA256 = "mutable"; + cloudflared.run = "sudo apt-get install -y cloudflared"; + fs.writeFileSync(workflowPath, YAML.stringify(workflow)); + + expect(validateMcpOpenShellWorkflowBoundary(workflowPath)).toEqual( + expect.arrayContaining([ + "mcp-bridge-dev must pin the reviewed cloudflared package checksum", + "mcp-bridge-dev cloudflared installation must not use mutable package repositories", + ]), + ); + } finally { + fs.rmSync(directory, { force: true, recursive: true }); + } + }); + + it("rejects any additional credential-persisting checkout", () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mcp-workflow-")); + const workflowPath = path.join(directory, "e2e.yaml"); + try { + const workflow = YAML.parse(fs.readFileSync(".github/workflows/e2e.yaml", "utf8")) as { + jobs: Record> }>; + }; + workflow.jobs["mcp-bridge"].steps.push({ + uses: "actions/checkout@v6", + with: { "persist-credentials": true }, + }); + fs.writeFileSync(workflowPath, YAML.stringify(workflow)); + + expect(validateMcpOpenShellWorkflowBoundary(workflowPath)).toEqual( + expect.arrayContaining([ + "mcp-bridge must use exactly one checkout step", + "mcp-bridge must use a SHA-pinned checkout", + "mcp-bridge checkout must set persist-credentials:false", + ]), + ); + } finally { + fs.rmSync(directory, { force: true, recursive: true }); + } + }); + + it("revokes Docker credentials before executing unverified dev artifacts", () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mcp-workflow-")); + const workflowPath = path.join(directory, "e2e.yaml"); + try { + const workflow = YAML.parse(fs.readFileSync(".github/workflows/e2e.yaml", "utf8")) as { + jobs: Record> }>; + }; + workflow.jobs["mcp-bridge-dev"].steps = workflow.jobs["mcp-bridge-dev"].steps.filter( + (step) => step.name !== "Revoke Docker auth before unverified dev tooling", + ); + fs.writeFileSync(workflowPath, YAML.stringify(workflow)); + + expect(validateMcpOpenShellWorkflowBoundary(workflowPath)).toContain( + "mcp-bridge-dev must revoke Docker auth before unverified dev tooling", + ); + } finally { + fs.rmSync(directory, { force: true, recursive: true }); + } + }); + + it("rejects any additional artifact upload outside the scanned directory", () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mcp-workflow-")); + const workflowPath = path.join(directory, "e2e.yaml"); + try { + const workflow = YAML.parse(fs.readFileSync(".github/workflows/e2e.yaml", "utf8")) as { + jobs: Record> }>; + }; + workflow.jobs["mcp-bridge-dev"].steps.push({ + name: "Upload unscanned output", + uses: "actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a", + with: { name: "unscanned", path: "e2e-artifacts/live/unscanned/" }, + }); + fs.writeFileSync(workflowPath, YAML.stringify(workflow)); + + expect(validateMcpOpenShellWorkflowBoundary(workflowPath)).toContain( + "mcp-bridge-dev must use exactly one reviewed MCP artifact upload step", + ); + } finally { + fs.rmSync(directory, { force: true, recursive: true }); + } + }); +}); diff --git a/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts b/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts index 24fbfa8882f..3c67b3d8782 100644 --- a/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts +++ b/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts @@ -77,7 +77,7 @@ function validateActionMutation(mutate: (action: MutableAction) => void): string } describe("upload-e2e-artifacts workflow boundary", () => { - it("binds one canonical uploader to all 71 E2E execution jobs", () => { + it("binds one canonical uploader to all 73 E2E execution jobs", () => { expect(validateUploadE2eArtifactsAction()).toEqual([]); expect(validateUploadE2eArtifactsInvocations(readWorkflow())).toEqual([]); }); @@ -146,6 +146,7 @@ describe("upload-e2e-artifacts workflow boundary", () => { uploadStep(workflow.jobs["hermes-slack"]).with!.path = "e2e-artifacts/live/hermes-slack/"; uploadStep(workflow.jobs["gpu-e2e"]).if = "success()"; + uploadStep(workflow.jobs["mcp-bridge"]).if = "always()"; uploadStep(workflow.jobs["docs-validation"]).env = { UNEXPECTED: "1" }; const orderedJob = workflow.jobs["network-policy"]; const orderedUpload = uploadStep(orderedJob); @@ -159,6 +160,7 @@ describe("upload-e2e-artifacts workflow boundary", () => { "credential-migration default upload caller must declare a valid E2E_TARGET_ID", "hermes-slack upload-e2e-artifacts must preserve its explicit name/path contract", "gpu-e2e upload-e2e-artifacts invocation must run with always()", + "mcp-bridge upload-e2e-artifacts invocation must remain gated by its reviewed pre-upload checks", "docs-validation upload-e2e-artifacts invocation must not override its contract", "network-policy upload-e2e-artifacts invocation must follow artifact producers and precede only Docker auth cleanup", ]), @@ -175,7 +177,7 @@ describe("upload-e2e-artifacts workflow boundary", () => { expect(validateUploadE2eArtifactsInvocations(workflow)).toEqual( expect.arrayContaining([ - "upload-e2e-artifacts must cover exactly 71 live and E2E_JOB execution jobs", + "upload-e2e-artifacts must cover exactly 73 live and E2E_JOB execution jobs", "upload-e2e-artifacts must keep exactly 62 default callers", ]), ); diff --git a/test/fetch-guard-patch-regression.test.ts b/test/fetch-guard-patch-regression.test.ts index 9f3398b4adf..26eaf5052dd 100644 --- a/test/fetch-guard-patch-regression.test.ts +++ b/test/fetch-guard-patch-regression.test.ts @@ -132,6 +132,28 @@ function readDockerfileOpenClawVersion(): string { ); } +function readDockerfileMcporterVersions(): { runtime: string; base: string } { + const pattern = /^ARG MCPORTER_VERSION=([^\s]+)/m; + return { + runtime: readRequiredMatch(DOCKERFILE, pattern, "mcporter runtime version"), + base: readRequiredMatch(DOCKERFILE_BASE, pattern, "mcporter base image version"), + }; +} + +function readDockerfileMcporterVersion(): string { + const versions = readDockerfileMcporterVersions(); + expect(versions.base, "mcporter base image version").toBe(versions.runtime); + return versions.runtime; +} + +function readDockerfileMcporterIntegrity(): string { + const pattern = /^ARG MCPORTER_0_7_3_INTEGRITY=([^\s]+)/m; + const runtime = readRequiredMatch(DOCKERFILE, pattern, "mcporter runtime integrity"); + const base = readRequiredMatch(DOCKERFILE_BASE, pattern, "mcporter base image integrity"); + expect(base, "mcporter base image integrity").toBe(runtime); + return runtime; +} + function readDockerfileBaseOpenClawIntegrity(): string { return readRequiredMatch( DOCKERFILE_BASE, @@ -177,29 +199,43 @@ function runOpenClawUpgradeBlock(currentVersion: string) { const log = path.join(tmp, "calls.log"); const openclawInstall = path.join(tmp, "openclaw-global"); const openclawShim = path.join(tmp, "openclaw-bin"); + const mcporterInstall = path.join(tmp, "mcporter-runtime"); + const mcporterShim = path.join(tmp, "mcporter-bin"); const openclawVersion = readDockerfileOpenClawVersion(); + const expectedMcporterVersion = readDockerfileMcporterVersion(); const openclawIntegrity = readDockerfileOpenClawIntegrity(); + const mcporterIntegrity = readDockerfileMcporterIntegrity(); fs.writeFileSync(blueprint, `min_openclaw_version: "${readBlueprintMinOpenClawVersion()}"\n`); fs.mkdirSync(openclawInstall, { recursive: true }); + fs.mkdirSync(mcporterInstall, { recursive: true }); fs.writeFileSync(openclawShim, ""); + fs.writeFileSync(mcporterShim, ""); const command = dockerRunCommandBetween( "# OPENCLAW_VERSION is the NemoClaw runtime build target", "# Patch OpenClaw media fetch", ) .replaceAll("/opt/nemoclaw-blueprint/blueprint.yaml", blueprint) .replaceAll("/usr/local/lib/node_modules/openclaw", openclawInstall) - .replaceAll("/usr/local/bin/openclaw", openclawShim); + .replaceAll("/usr/local/bin/openclaw", openclawShim) + .replaceAll("/usr/local/lib/node_modules/mcporter", mcporterInstall) + .replaceAll("/usr/local/lib/nemoclaw/mcporter-runtime", mcporterInstall) + .replaceAll("/usr/local/bin/mcporter", mcporterShim); const script = [ "#!/usr/bin/env bash", "set -euo pipefail", `call_log=${JSON.stringify(log)}`, `OPENCLAW_VERSION=${JSON.stringify(openclawVersion)}`, + `MCPORTER_VERSION=${JSON.stringify(expectedMcporterVersion)}`, `OPENCLAW_2026_5_27_INTEGRITY=${JSON.stringify(openclawIntegrity)}`, + `MCPORTER_0_7_3_INTEGRITY=${JSON.stringify(mcporterIntegrity)}`, `openclaw() { if [ "\${1:-}" = "--version" ]; then printf 'openclaw ${currentVersion}\\n'; else return 127; fi; }`, + `mcporter() { if [ "\${1:-}" = "--version" ]; then printf '${expectedMcporterVersion}\\n'; else return 127; fi; }`, "npm() {", ' printf "npm %s\\n" "$*" >> "$call_log";', ' if [ "${1:-}" = "view" ] && [ "${2:-}" = "openclaw@${OPENCLAW_VERSION}" ] && [ "${3:-}" = "dist.integrity" ]; then', ' printf "%s\\n" "$OPENCLAW_2026_5_27_INTEGRITY";', + ' elif [ "${1:-}" = "view" ] && [ "${2:-}" = "mcporter@${MCPORTER_VERSION}" ] && [ "${3:-}" = "dist.integrity" ]; then', + ' printf "%s\\n" "$MCPORTER_0_7_3_INTEGRITY";', " fi", "}", 'command() { if [ "${1:-}" = "-v" ] && [ "${2:-}" = "codex-acp" ]; then return 0; fi; builtin command "$@"; }', @@ -375,6 +411,26 @@ describe("fetch-guard patch regression guard", () => { ].join("\n"); const result = spawnSync("bash", ["-c", script], { encoding: "utf-8", timeout: 5000 }); expect(result.status).toBe(42); + + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-plugin-install-")); + const inspectMarker = path.join(tmp, "inspected"); + const successScript = [ + "openclaw() {", + ' case "${1:-} ${2:-} ${3:-}" in', + ' "plugins install /opt/nemoclaw") echo "installed" ;;', + ` "plugins inspect nemoclaw") : > ${JSON.stringify(inspectMarker)} ;;`, + ' "plugins enable nemoclaw") return 43 ;;', + " esac", + " return 0", + "}", + command, + ].join("\n"); + const success = spawnSync("bash", ["-c", successScript], { + encoding: "utf-8", + timeout: 5000, + }); + expect(success.status).toBe(0); + expect(fs.existsSync(inspectMarker)).toBe(true); }); it("upgrades stale OpenClaw to the runtime build target and leaves current installs alone", () => { @@ -397,6 +453,30 @@ describe("fetch-guard patch regression guard", () => { ); }); + it("reinstalls mcporter from the committed graph when the inherited version matches", () => { + const invocation = runOpenClawUpgradeBlock(CURRENT_REVIEWED_OPENCLAW_PATCH_CLASSIFIER_VERSION); + const expectedMcporterVersion = readDockerfileMcporterVersion(); + + expect(invocation.result.status).toBe(0); + expect(invocation.result.stdout).toContain( + `Installing locked mcporter ${expectedMcporterVersion} dependency graph`, + ); + expect(invocation.calls).toMatch( + /npm --prefix \S+ ci --ignore-scripts --omit=dev --no-audit --no-fund --no-progress/, + ); + readRequiredMatch( + DOCKERFILE_BASE, + /(npm --prefix \/usr\/local\/lib\/nemoclaw\/mcporter-runtime ci\s*\\\s*--ignore-scripts --omit=dev --no-audit --no-fund --no-progress)/, + "mcporter base lockfile install with lifecycle scripts disabled", + ); + expect( + dockerRunCommandBetween( + "# OPENCLAW_VERSION is the NemoClaw runtime build target", + "# Patch OpenClaw media fetch", + ), + ).toContain("rm -rf /usr/local/lib/node_modules/mcporter /usr/local/bin/mcporter"); + }); + it("requires classifier review and integrity evidence when the OpenClaw build pin changes", () => { const reviewMessage = "Update fetch-guard classifier expectations before changing the OpenClaw build version."; diff --git a/test/gateway-drift-preflight.test.ts b/test/gateway-drift-preflight.test.ts index 7e86522a11f..6f60aa5f3ef 100644 --- a/test/gateway-drift-preflight.test.ts +++ b/test/gateway-drift-preflight.test.ts @@ -76,8 +76,8 @@ function writeFakeOpenshell(binDir: string): void { path.join(binDir, "openshell"), `#!/usr/bin/env bash set -uo pipefail -: "\${NEMOCLAW_FAKE_CASE_DIR:?}" -printf '%s\n' "$*" >> "$NEMOCLAW_FAKE_CASE_DIR/openshell-calls.log" +case_dir="\${NEMOCLAW_FAKE_CASE_DIR:-\${TMPDIR:-/tmp}}" +printf '%s\n' "$*" >> "$case_dir/openshell-calls.log" case "\${1:-}" in --version|-V) printf 'openshell 0.0.37\n' @@ -122,7 +122,7 @@ function writeFakeDocker( path.join(binDir, "docker"), `#!/usr/bin/env bash set -uo pipefail -case_dir="\${NEMOCLAW_FAKE_CASE_DIR:-\${TMPDIR:-/tmp}/nemoclaw-gateway-drift-preflight-current}" +case_dir="\${NEMOCLAW_FAKE_CASE_DIR:-\${TMPDIR:-/tmp}}" printf '%s\n' "$*" >> "$case_dir/docker-calls.log" format="" if [ "\${1:-}" = "inspect" ] || { [ "\${1:-}" = "container" ] && [ "\${2:-}" = "inspect" ]; }; then @@ -160,7 +160,8 @@ function writeFakeDockerNoCluster(binDir: string): void { path.join(binDir, "docker"), `#!/usr/bin/env bash set -uo pipefail -printf '%s\n' "$*" >> "$NEMOCLAW_FAKE_CASE_DIR/docker-calls.log" +case_dir="\${NEMOCLAW_FAKE_CASE_DIR:-\${TMPDIR:-/tmp}}" +printf '%s\n' "$*" >> "$case_dir/docker-calls.log" if [ "\${1:-}" = "inspect" ] || { [ "\${1:-}" = "container" ] && [ "\${2:-}" = "inspect" ]; }; then printf 'Error: No such object\n' >&2 exit 1 diff --git a/test/gateway-state-reconcile-2276.test.ts b/test/gateway-state-reconcile-2276.test.ts index ea92feba053..689b67e7bb6 100644 --- a/test/gateway-state-reconcile-2276.test.ts +++ b/test/gateway-state-reconcile-2276.test.ts @@ -69,6 +69,8 @@ interface ScenarioScript { gatewaySelect: { output: string; exit: number }; // whether `gateway select nemoclaw` flips the active gateway to nemoclaw selectFlipsActive: boolean; + // `sandbox list` output; defaults to the live sandbox for scenarios 1-12. + sandboxList?: string; } interface HarnessResult { @@ -101,6 +103,11 @@ function writeDefaultRegistry() { model: "nvidia/nemotron-3-super-120b-a12b", provider: "nvidia-prod", gpuEnabled: false, + sandboxGpuMode: "0", + gatewayName: "nemoclaw", + gatewayPort: 8080, + dashboardPort: 28790, + fromDockerfile: null, policies: [], }, }, @@ -136,6 +143,7 @@ const callLogPath = ${JSON.stringify(callLogFile)}; const script = JSON.parse(fs.readFileSync(scriptPath, "utf8")); const state = JSON.parse(fs.readFileSync(statePath, "utf8") || "{}"); const args = process.argv.slice(2); +const requiredFeatures = "request-body-credential-rewrite websocket-credential-rewrite allow_all_known_mcp_methods"; fs.appendFileSync(callLogPath, JSON.stringify(args) + "\\n"); @@ -151,8 +159,8 @@ function emit(r) { process.exit(r.exit || 0); } -if (args[0] === "--version") { - process.stdout.write("openshell 0.0.25\\n"); +if (args[0] === "-V" || args[0] === "--version") { + process.stdout.write("openshell 0.0.72\\n"); process.exit(0); } @@ -188,20 +196,32 @@ if (args[0] === "policy" && args[1] === "get") { } if (args[0] === "sandbox" && args[1] === "list") { - // Return the sandbox as live to avoid the list-based destroy path. - process.stdout.write("Sandboxes:\\n - ${SANDBOX_NAME}\\n"); + process.stdout.write(script.sandboxList === undefined ? "Sandboxes:\\n - ${SANDBOX_NAME}\\n" : script.sandboxList); process.exit(0); } if (args[0] === "inference" && args[1] === "get") { - process.stdout.write("Provider: nvidia-prod\\nModel: nvidia/nemotron-3-super-120b-a12b\\n"); + process.stdout.write("Gateway inference:\\n Provider: nvidia-prod\\n Model: nvidia/nemotron-3-super-120b-a12b\\n"); process.exit(0); } +if (args[0] === "provider" && args[1] === "get") process.exit(0); + // forward stop/start, provider delete, logs, etc. — no-op success process.exit(0); `; fs.writeFileSync(openshellPath, stub, { mode: 0o755 }); + for (const component of ["openshell-gateway", "openshell-sandbox"]) { + fs.writeFileSync( + path.join(homeLocalBin, component), + `#!${process.execPath} +const requiredFeatures = "request-body-credential-rewrite websocket-credential-rewrite allow_all_known_mcp_methods"; +if (process.argv[2] === "-V" || process.argv[2] === "--version") process.stdout.write("${component} 0.0.72\\n"); +process.exit(0); +`, + { mode: 0o755 }, + ); + } } function runCli(action: string, extraEnv: Record = {}): HarnessResult { @@ -286,6 +306,32 @@ beforeEach(() => { fs.mkdirSync(registryDir, { recursive: true }); writeDefaultRegistry(); writeDefaultSession(); + fs.writeFileSync( + path.join(homeLocalBin, "docker"), + `#!${process.execPath} +const a = process.argv.slice(2); +if (a[0] === "info") { + process.stdout.write(JSON.stringify({ServerVersion:"27.0.0", OperatingSystem:"Docker Engine", NCPU:8, MemTotal:17179869184}) + "\\n"); + process.exit(0); +} +if (a[0] === "build") process.exit(0); +if (a[0] === "image" && a[1] === "inspect") { + const formatIndex = a.indexOf("--format"); + const format = formatIndex >= 0 ? a[formatIndex + 1] : ""; + if (format === "{{.Id}}") process.stdout.write("sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\\n"); + if (format === "{{json .RepoDigests}}") process.stdout.write("[]\\n"); + process.exit(0); +} +if (a[0] === "tag" || a[0] === "rmi") process.exit(0); +if (a[0] === "run") { + if (a.includes("nslookup")) process.stdout.write("Server: 127.0.0.11\\n** server can't find nemoclaw.invalid: NXDOMAIN\\n"); + else if (a.includes("/usr/bin/ldd")) process.stdout.write("ldd (GNU libc) 2.41\\n"); + process.exit(0); +} +process.exit(0); +`, + { mode: 0o755 }, + ); }); afterEach(() => { @@ -720,6 +766,7 @@ describe("connect preserves the registry so rebuild can recover in scenario 14 ( gatewayInfo: [{ output: GATEWAY_INFO_NEMOCLAW, exit: 0 }], gatewaySelect: { output: "", exit: 0 }, selectFlipsActive: false, + sandboxList: "", }); // Step 3: routine connect must preserve the registry entry. @@ -751,12 +798,14 @@ describe("connect preserves the registry so rebuild can recover in scenario 14 ( HOME: tmpDir, PATH: `${homeLocalBin}:/usr/bin:/bin`, NO_COLOR: "1", + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", + NEMOCLAW_SKIP_HOST_DNS_PREFLIGHT: "1", NEMOCLAW_NON_INTERACTIVE: "1", // The recreate handoff (onboard --resume) fails fast in this stubbed // HOME — fine: the assertions below target the recovery markers that // are emitted BEFORE the recreate, proving rebuild crossed the // backup gate that previously blocked it. - NVIDIA_INFERENCE_API_KEY: "", + NVIDIA_INFERENCE_API_KEY: "nvapi-test-key-for-rebuild", NEMOCLAW_PROVIDER_KEY: "", }, }, diff --git a/test/generate-openclaw-config.test.ts b/test/generate-openclaw-config.test.ts index de65d0fea17..3d1da6e55b2 100644 --- a/test/generate-openclaw-config.test.ts +++ b/test/generate-openclaw-config.test.ts @@ -1783,37 +1783,27 @@ describe("generate-openclaw-config.mts: config generation", () => { expect(config.gateway.auth.token).toBe(""); }); - it("disables bundled acpx runtime staging by default", () => { + it("disables bundled bonjour in sandbox config by default", () => { const config = runConfigScript(); - expect(config.plugins.entries.acpx.enabled).toBe(false); - expect(config.plugins.entries.acpx.config).toBeUndefined(); + expect(config.plugins.entries.bonjour.enabled).toBe(false); + expect(config.plugins.entries.bonjour.config).toBeUndefined(); }); - it("disables unused bundled provider plugins with staged runtime deps", () => { + it("omits stale disabled entries for optional bundled plugins", () => { const config = runConfigScript({ NEMOCLAW_PROVIDER_KEY: "inference" }); - expect(config.plugins.entries["amazon-bedrock"].enabled).toBe(false); - expect(config.plugins.entries["amazon-bedrock-mantle"].enabled).toBe(false); - expect(config.plugins.entries.anthropic.enabled).toBe(false); - expect(config.plugins.entries["anthropic-vertex"].enabled).toBe(false); - expect(config.plugins.entries.fireworks.enabled).toBe(false); - expect(config.plugins.entries.google.enabled).toBe(false); - expect(config.plugins.entries.kimi.enabled).toBe(false); - expect(config.plugins.entries.lmstudio.enabled).toBe(false); - expect(config.plugins.entries.ollama.enabled).toBe(false); - expect(config.plugins.entries.openai.enabled).toBe(false); - expect(config.plugins.entries.xai.enabled).toBe(false); + expect(Object.keys(config.plugins.entries)).toEqual(["bonjour"]); }); it("keeps the selected bundled provider plugin available", () => { const config = runConfigScript({ NEMOCLAW_PROVIDER_KEY: "anthropic" }); expect(config.plugins.entries.anthropic).toBeUndefined(); - expect(config.plugins.entries.google.enabled).toBe(false); + expect(config.plugins.entries.google).toBeUndefined(); }); it("keeps the selected OpenAI bundled provider plugin available", () => { const config = runConfigScript({ NEMOCLAW_PROVIDER_KEY: "openai" }); expect(config.plugins.entries.openai).toBeUndefined(); - expect(config.plugins.entries.xai.enabled).toBe(false); + expect(config.plugins.entries.xai).toBeUndefined(); }); it("enables the discord plugin entry when Discord is configured (#4246)", () => { diff --git a/test/helpers/base-image-test-harness.ts b/test/helpers/base-image-test-harness.ts new file mode 100644 index 00000000000..9ce0faf7e13 --- /dev/null +++ b/test/helpers/base-image-test-harness.ts @@ -0,0 +1,148 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createRequire } from "node:module"; + +import { vi } from "vitest"; + +import type { AgentDefinition } from "../../src/lib/agent/defs"; + +type AgentOnboardModule = typeof import("../../src/lib/agent/onboard"); +type DockerRunModule = typeof import("../../src/lib/adapters/docker/run"); +type DockerImageModule = typeof import("../../src/lib/adapters/docker/image"); +type DockerInspectModule = typeof import("../../src/lib/adapters/docker/inspect"); +type SandboxBaseImageModule = typeof import("../../src/lib/sandbox-base-image"); + +const requireSource = createRequire( + new URL("../../src/lib/agent/base-image.test.ts", import.meta.url), +); + +/** Build a minimal Hermes manifest for base-image provisioning tests. */ +export function makeAgent(overrides: Partial = {}): AgentDefinition { + return { + name: "hermes", + displayName: "Hermes Agent", + healthProbe: { url: "http://127.0.0.1:8642/health", port: 8642, timeout_seconds: 90 }, + forwardPort: 8642, + dashboard: { + kind: "api", + label: "OpenAI-compatible API", + path: "/v1", + healthPath: "/health", + auth: "none", + }, + webAuth: { method: "bearer_token", env: "API_SERVER_KEY" }, + configPaths: { + dir: "/sandbox/.hermes", + configFile: "config.yaml", + envFile: ".env", + format: "yaml", + }, + inferenceProviderOptions: [], + mcpCapability: { + support: "disabled", + reason: "test fixture", + }, + stateDirs: [], + stateFiles: [], + userManagedFiles: [], + versionCommand: "hermes --version", + expectedVersion: "2026.4.30", + hasDevicePairing: false, + phoneHomeHosts: [], + dockerfileBasePath: "/test/root/agents/hermes/Dockerfile.base", + dockerfilePath: "/test/root/agents/hermes/Dockerfile", + startScriptPath: null, + policyAdditionsPath: null, + policyPermissivePath: null, + pluginDir: null, + legacyPaths: null, + agentDir: "/repo/root/agents/hermes", + manifestPath: "/repo/root/agents/hermes/manifest.yaml", + ...overrides, + }; +} + +/** Load agent onboarding with source-backed Docker helpers replaced by mocks. */ +export function withMockedDocker( + run: (deps: { + ensureAgentBaseImage: AgentOnboardModule["ensureAgentBaseImage"]; + pinAgentSandboxBaseImageRef: AgentOnboardModule["pinAgentSandboxBaseImageRef"]; + dockerBuildMock: ReturnType; + dockerCaptureMock: ReturnType; + dockerImageInspectMock: ReturnType; + dockerImageInspectFormatMock: ReturnType; + dockerRmiMock: ReturnType; + dockerTagMock: ReturnType; + resolveSandboxBaseImageMock: ReturnType; + root: string; + }) => T, +): T { + const dockerRunModule = requireSource("../adapters/docker/run.js") as DockerRunModule; + const dockerImageModule = requireSource("../adapters/docker/image.js") as DockerImageModule; + const dockerInspectModule = requireSource("../adapters/docker/inspect.js") as DockerInspectModule; + const sandboxBaseImageModule = requireSource( + "../sandbox-base-image.js", + ) as SandboxBaseImageModule; + const runnerModule = requireSource("../runner.js") as { ROOT: string }; + const originalDockerCapture = dockerRunModule.dockerCapture; + const originalDockerBuild = dockerImageModule.dockerBuild; + const originalDockerRmi = dockerImageModule.dockerRmi; + const originalDockerTag = dockerImageModule.dockerTag; + const originalDockerImageInspect = dockerInspectModule.dockerImageInspect; + const originalDockerImageInspectFormat = dockerInspectModule.dockerImageInspectFormat; + const originalResolveSandboxBaseImage = sandboxBaseImageModule.resolveSandboxBaseImage; + const agentOnboardModulePath = requireSource.resolve("./onboard.js"); + delete require.cache[agentOnboardModulePath]; + + const dockerCaptureMock = vi.fn().mockReturnValue("nemoclaw-hermes-mcp-runtime-ok"); + const dockerBuildMock = vi.fn().mockReturnValue({ status: 0 }); + const dockerRmiMock = vi.fn().mockReturnValue({ status: 0 }); + const dockerTagMock = vi.fn().mockReturnValue({ status: 0 }); + const dockerImageInspectMock = vi.fn(); + const dockerImageInspectFormatMock = vi.fn().mockReturnValue(`sha256:${"a".repeat(64)}`); + const resolveSandboxBaseImageMock = vi.fn().mockImplementation((options) => { + const override = options.env?.[options.envVar]; + return { + ref: override ?? "nemoclaw-hermes-sandbox-base-local:compatible", + digest: null, + source: override ? "override" : "local", + glibcVersion: process.platform === "linux" ? "2.41" : null, + }; + }); + dockerRunModule.dockerCapture = dockerCaptureMock as DockerRunModule["dockerCapture"]; + dockerImageModule.dockerBuild = dockerBuildMock as DockerImageModule["dockerBuild"]; + dockerImageModule.dockerRmi = dockerRmiMock as DockerImageModule["dockerRmi"]; + dockerImageModule.dockerTag = dockerTagMock as DockerImageModule["dockerTag"]; + dockerInspectModule.dockerImageInspect = + dockerImageInspectMock as DockerInspectModule["dockerImageInspect"]; + dockerInspectModule.dockerImageInspectFormat = + dockerImageInspectFormatMock as DockerInspectModule["dockerImageInspectFormat"]; + sandboxBaseImageModule.resolveSandboxBaseImage = + resolveSandboxBaseImageMock as SandboxBaseImageModule["resolveSandboxBaseImage"]; + + try { + const agentOnboardModule = requireSource("./onboard.js") as AgentOnboardModule; + return run({ + ensureAgentBaseImage: agentOnboardModule.ensureAgentBaseImage, + pinAgentSandboxBaseImageRef: agentOnboardModule.pinAgentSandboxBaseImageRef, + dockerBuildMock, + dockerCaptureMock, + dockerImageInspectMock, + dockerImageInspectFormatMock, + dockerRmiMock, + dockerTagMock, + resolveSandboxBaseImageMock, + root: runnerModule.ROOT, + }); + } finally { + dockerRunModule.dockerCapture = originalDockerCapture; + dockerImageModule.dockerBuild = originalDockerBuild; + dockerImageModule.dockerRmi = originalDockerRmi; + dockerImageModule.dockerTag = originalDockerTag; + dockerInspectModule.dockerImageInspect = originalDockerImageInspect; + dockerInspectModule.dockerImageInspectFormat = originalDockerImageInspectFormat; + sandboxBaseImageModule.resolveSandboxBaseImage = originalResolveSandboxBaseImage; + delete require.cache[agentOnboardModulePath]; + } +} diff --git a/test/helpers/destroy-flow-test-assertions.ts b/test/helpers/destroy-flow-test-assertions.ts new file mode 100644 index 00000000000..ac9b1bb5508 --- /dev/null +++ b/test/helpers/destroy-flow-test-assertions.ts @@ -0,0 +1,197 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { expect, type MockInstance } from "vitest"; + +import { + type DestroyHarness, + loadDestroySandboxPresenceClassifier, + sandboxListJson, +} from "./destroy-flow-test-harness"; + +export function expectStrictSandboxPresenceClassification(): void { + const classifyDestroySandboxPresence = loadDestroySandboxPresenceClassifier(); + expect( + classifyDestroySandboxPresence("alpha", { + status: 0, + stdout: sandboxListJson(["alpha"]), + }), + ).toBe("present"); + expect( + classifyDestroySandboxPresence("alpha", { + status: 0, + stdout: sandboxListJson(["beta"]), + }), + ).toBe("absent"); + expect( + classifyDestroySandboxPresence("alpha", { + status: 1, + stderr: "gateway unavailable", + }), + ).toBe("unknown"); + expect( + classifyDestroySandboxPresence("alpha", { + status: 0, + stdout: "arbitrary warning text", + }), + ).toBe("unknown"); + expect( + classifyDestroySandboxPresence("alpha", { + status: 0, + stdout: JSON.stringify([{ name: "beta" }]), + }), + ).toBe("unknown"); + expect( + classifyDestroySandboxPresence("alpha", { + status: 0, + stdout: "", + }), + ).toBe("unknown"); +} + +export function expectSuccessfulLiveDestroy(harness: DestroyHarness, exitSpy: MockInstance): void { + expect(harness.selectGatewaySpy).toHaveBeenCalledWith( + "alpha", + "nemoclaw-19080", + harness.runOpenshellSpy, + ); + expect(harness.gatewayPinsAtSandboxList).toEqual(["nemoclaw-19080"]); + expect(harness.runOpenshellSpy).toHaveBeenCalledWith( + ["sandbox", "list", "-o", "json"], + expect.objectContaining({ ignoreError: true }), + ); + expect(harness.stopNimByNameSpy).toHaveBeenCalledWith("alpha-nim"); + expect(harness.killStaleProxySpy).toHaveBeenCalledTimes(1); + expect(harness.runOpenshellSpy).toHaveBeenCalledWith( + ["sandbox", "delete", "alpha"], + expect.objectContaining({ ignoreError: true }), + ); + expect(harness.unloadOllamaModelsSpy).toHaveBeenCalledTimes(1); + expect(harness.removeSandboxSpy).toHaveBeenCalledWith("alpha"); + expect(harness.cleanupGatewaySpy).toHaveBeenCalledWith("nemoclaw-19080", harness.runOpenshellSpy); + expect(harness.logSpy.mock.calls.map((call) => String(call[0])).join("\n")).toContain( + "Sandbox 'alpha' destroyed", + ); + expect(exitSpy).not.toHaveBeenCalled(); +} + +export function expectFailedDeletePreservesHostState( + harness: DestroyHarness, + exitSpy: MockInstance, +): void { + expect(harness.runOpenshellSpy).toHaveBeenCalledWith( + ["sandbox", "delete", "alpha"], + expect.objectContaining({ ignoreError: true }), + ); + expect(harness.removeSandboxSpy).not.toHaveBeenCalled(); + expect(harness.cleanupGatewaySpy).not.toHaveBeenCalled(); + expect(exitSpy).toHaveBeenCalledWith(7); +} + +export function expectShieldsUpRefusalBeforeMutation(harness: DestroyHarness): void { + expect(harness.stopNimByNameSpy).not.toHaveBeenCalled(); + expect(harness.killStaleProxySpy).not.toHaveBeenCalled(); + expect(harness.selectGatewaySpy).toHaveBeenCalledWith( + "alpha", + "nemoclaw-19080", + harness.runOpenshellSpy, + ); + expect(harness.prepareMcpBridgesForDestroySpy).not.toHaveBeenCalled(); + expect(harness.runOpenshellSpy).toHaveBeenCalledWith( + ["sandbox", "list", "-o", "json"], + expect.objectContaining({ ignoreError: true }), + ); +} + +export function expectActiveTimerDestroyOrder(harness: DestroyHarness): void { + expect(harness.events).toEqual( + expect.arrayContaining(["wipe", "harden", "detach", "delete", "timer-cleanup"]), + ); + expect(harness.events.indexOf("wipe")).toBeLessThan(harness.events.indexOf("harden")); + expect(harness.events.indexOf("harden")).toBeLessThan(harness.events.indexOf("delete")); + expect(harness.events.indexOf("delete")).toBeLessThan(harness.events.indexOf("timer-cleanup")); +} + +export function expectFailedHardeningStopsDelete(harness: DestroyHarness): void { + expect(harness.events).toContain("wipe"); + expect(harness.events).toContain("harden"); + expect(harness.events).not.toContain("delete"); + expect(harness.killTimerSpy).not.toHaveBeenCalled(); +} + +export function expectMcpFinalizeAfterDelete(harness: DestroyHarness): void { + expect(harness.prepareMcpBridgesForDestroySpy).toHaveBeenCalledWith("alpha"); + expect(harness.gatewayPinsAtMcpPrepare).toEqual(["nemoclaw-19080"]); + const deleteCall = harness.runOpenshellSpy.mock.calls.findIndex( + (call) => Array.isArray(call[0]) && call[0].join(" ") === "sandbox delete alpha", + ); + expect(deleteCall).toBeGreaterThanOrEqual(0); + expect(harness.prepareMcpBridgesForDestroySpy.mock.invocationCallOrder.at(-1)).toBeLessThan( + harness.runOpenshellSpy.mock.invocationCallOrder[deleteCall], + ); + expect( + harness.finalizeMcpBridgesAfterSandboxDeleteSpy.mock.invocationCallOrder.at(-1), + ).toBeGreaterThan(harness.runOpenshellSpy.mock.invocationCallOrder[deleteCall]); + expect(harness.finalizeMcpBridgesAfterSandboxDeleteSpy).toHaveBeenCalledWith( + "alpha", + expect.objectContaining({ + entries: [{ server: "github" }, { server: "slack" }], + }), + { force: false }, + ); + expect(harness.restoreMcpBridgesAfterDestroyAbortSpy).not.toHaveBeenCalled(); +} + +export function expectMcpRestoreAfterDeleteFailure(harness: DestroyHarness): void { + expect(harness.restoreMcpBridgesAfterDestroyAbortSpy).toHaveBeenCalledWith( + "alpha", + expect.objectContaining({ entries: [{ server: "github" }] }), + ); + expect(harness.finalizeMcpBridgesAfterSandboxDeleteSpy).not.toHaveBeenCalled(); + expect(harness.removeSandboxSpy).not.toHaveBeenCalled(); + expect(harness.events.filter((event) => event === "harden")).toHaveLength(2); + expect(harness.events.indexOf("delete")).toBeLessThan(harness.events.indexOf("unlock")); + expect(harness.events.indexOf("unlock")).toBeLessThan(harness.events.indexOf("mcp-restore")); + expect(harness.events.indexOf("mcp-restore")).toBeLessThan(harness.events.lastIndexOf("harden")); + expect(harness.shieldsDownSpy).toHaveBeenCalledWith( + "alpha", + expect.objectContaining({ + timeout: "15m", + deferAutoRestoreWhileOwnerAlive: true, + processToken: "a".repeat(32), + throwOnError: true, + }), + ); + expect(harness.shieldsDownSpy.mock.calls[0]?.[1]).not.toHaveProperty("skipTimer"); +} + +export function expectFailedMcpRestorePreservesDestroyFailure(harness: DestroyHarness): void { + expect(harness.events.filter((event) => event === "harden")).toHaveLength(2); + expect(harness.events.indexOf("mcp-restore")).toBeLessThan(harness.events.lastIndexOf("harden")); + expect(harness.removeSandboxSpy).not.toHaveBeenCalled(); +} + +export function expectFailedMcpFinalizePreservesRegistry(harness: DestroyHarness): void { + expect(harness.finalizeMcpBridgesAfterSandboxDeleteSpy).toHaveBeenCalledWith( + "alpha", + expect.any(Object), + { force: true }, + ); + expect(harness.removeSandboxSpy).not.toHaveBeenCalled(); + expect(harness.cleanupGatewaySpy).not.toHaveBeenCalled(); +} + +export function expectAbsentSandboxMcpFinalize(harness: DestroyHarness): void { + expect(harness.prepareMcpBridgesForDestroySpy).not.toHaveBeenCalled(); + expect(harness.prepareMcpBridgesForAbsentSandboxDestroySpy).toHaveBeenCalledWith("alpha", { + force: false, + }); + expect(harness.gatewayPinsAtMcpPrepare).toEqual(["nemoclaw-19080"]); + expect(harness.restoreMcpBridgesAfterDestroyAbortSpy).not.toHaveBeenCalled(); + expect(harness.finalizeMcpBridgesAfterSandboxDeleteSpy).toHaveBeenCalledWith( + "alpha", + expect.objectContaining({ entries: [{ server: "github" }] }), + { force: false }, + ); + expect(harness.removeSandboxSpy).toHaveBeenCalledWith("alpha"); +} diff --git a/test/helpers/destroy-flow-test-harness.ts b/test/helpers/destroy-flow-test-harness.ts new file mode 100644 index 00000000000..0f107ea7e0e --- /dev/null +++ b/test/helpers/destroy-flow-test-harness.ts @@ -0,0 +1,301 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createRequire } from "node:module"; + +import { expect, type MockInstance, vi } from "vitest"; + +type DestroySandbox = typeof import("../../src/lib/actions/sandbox/destroy")["destroySandbox"]; + +const requireDist = createRequire( + new URL("../../src/lib/actions/sandbox/destroy-flow.test.ts", import.meta.url), +); +const destroyModulePath = "./destroy.js"; + +export type DestroyHarness = { + cleanupGatewaySpy: MockInstance; + destroySandbox: DestroySandbox; + errorSpy: MockInstance; + events: string[]; + finalizeMcpBridgesAfterSandboxDeleteSpy: MockInstance; + gatewayPinsAtMcpPrepare: Array; + gatewayPinsAtSandboxList: Array; + killTimerSpy: MockInstance; + killStaleProxySpy: MockInstance; + logSpy: MockInstance; + prepareMcpBridgesForAbsentSandboxDestroySpy: MockInstance; + prepareMcpBridgesForDestroySpy: MockInstance; + removeSandboxSpy: MockInstance; + restoreMcpBridgesAfterDestroyAbortSpy: MockInstance; + runOpenshellSpy: MockInstance; + selectGatewaySpy: MockInstance; + shieldsDownSpy: MockInstance; + stopAllSpy: MockInstance; + stopNimByNameSpy: MockInstance; + unloadOllamaModelsSpy: MockInstance; +}; + +type DestroyHarnessOptions = { + activeTimer?: boolean; + agent?: "openclaw" | "hermes"; + deleteOutput?: string; + deleteStatus?: number; + finalizeMcpError?: string; + mcpAddState?: "prepared"; + mcpServers?: string[]; + registeredSandboxCount?: number; + restoreMcpError?: string; + sandboxPresent?: boolean; + shieldsDown?: boolean; + shieldsUpError?: Error; +}; + +const sandboxEntry = { + name: "alpha", + agent: "openclaw", + provider: "ollama-local", + model: "nvidia/nemotron", + imageTag: null, + nimContainer: "alpha-nim", + gatewayName: "nemoclaw-19080", + gatewayPort: 19080, +}; + +export function sandboxListJson(names: string[]): string { + return JSON.stringify( + names.map((name) => ({ + id: `sandbox-${name}`, + name, + labels: {}, + resource_version: 1, + created_at: "2026-06-27 00:00:00", + phase: "Ready", + current_policy_version: 1, + })), + ); +} + +export function resetDestroyModuleCache(): void { + delete require.cache[requireDist.resolve(destroyModulePath)]; +} + +type DestroySandboxPresenceClassifier = ( + sandboxName: string, + result: { status: number | null; stdout?: string; stderr?: string }, +) => string; + +export function loadDestroySandboxPresenceClassifier(): DestroySandboxPresenceClassifier { + resetDestroyModuleCache(); + const destroyModule = requireDist(destroyModulePath) as { + classifyDestroySandboxPresence: DestroySandboxPresenceClassifier; + }; + return destroyModule.classifyDestroySandboxPresence; +} + +export function createDestroyHarness(options: DestroyHarnessOptions = {}): DestroyHarness { + resetDestroyModuleCache(); + const events: string[] = []; + + const logSpy = vi.spyOn(console, "log").mockImplementation(() => undefined); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); + vi.spyOn(console, "warn").mockImplementation(() => undefined); + + const resolve = requireDist("../../adapters/openshell/resolve.js"); + const runtime = requireDist("../../adapters/openshell/runtime.js"); + const destroyGateway = requireDist("./destroy-gateway.js"); + const sandboxProviderCleanup = requireDist("../../onboard/sandbox-provider-cleanup.js"); + const nim = requireDist("../../inference/nim.js"); + const ollamaProxy = requireDist("../../inference/ollama/proxy.js"); + const tunnelServices = requireDist("../../tunnel/services.js"); + const onboardSession = requireDist("../../state/onboard-session.js"); + const registry = requireDist("../../state/registry.js"); + const sandboxSession = requireDist("../../state/sandbox-session.js"); + const shields = requireDist("../../shields/index.js"); + const timerControl = requireDist("../../shields/timer-control.js"); + const mcpBridge = requireDist("./mcp-bridge.js"); + + vi.spyOn(resolve, "resolveOpenshell").mockReturnValue("/usr/bin/openshell"); + vi.spyOn(sandboxSession, "getActiveSandboxSessions").mockReturnValue({ + detected: true, + sessions: [{ pid: 1 }], + }); + vi.spyOn(registry, "getSandbox").mockReturnValue({ + ...sandboxEntry, + agent: options.agent ?? sandboxEntry.agent, + ...(options.mcpServers?.length + ? { + mcp: { + bridges: Object.fromEntries( + options.mcpServers.map((server) => [ + server, + { + server, + ...(options.mcpAddState ? { addState: options.mcpAddState } : {}), + }, + ]), + ), + }, + } + : {}), + }); + let registeredSandboxCount = options.registeredSandboxCount ?? 0; + vi.spyOn(registry, "listSandboxes").mockImplementation(() => ({ + sandboxes: Array.from({ length: registeredSandboxCount }, (_, index) => ({ + name: `sb-${index}`, + })), + })); + const removeSandboxSpy = vi.spyOn(registry, "removeSandbox").mockImplementation(() => { + registeredSandboxCount = Math.max(0, registeredSandboxCount - 1); + return true; + }); + vi.spyOn(onboardSession, "loadSession").mockReturnValue({ + sandboxName: "alpha", + }); + vi.spyOn(onboardSession, "updateSession").mockImplementation((mutator: unknown) => { + const session = { sandboxName: "alpha" }; + expect(typeof mutator).toBe("function"); + (mutator as (value: typeof session) => void)(session); + return session; + }); + const gatewayPinsAtSandboxList: Array = []; + const runOpenshellSpy = vi.spyOn(runtime, "runOpenshell").mockImplementation((args: unknown) => { + const argv = Array.isArray(args) ? args : []; + switch (`${String(argv[0])}:${String(argv[1])}`) { + case "sandbox:exec": + events.push("wipe"); + return { status: 0, stdout: "", stderr: "" }; + case "sandbox:list": + gatewayPinsAtSandboxList.push(process.env.OPENSHELL_GATEWAY); + return { + status: 0, + stdout: sandboxListJson(options.sandboxPresent === false ? [] : ["alpha"]), + stderr: "", + }; + case "sandbox:delete": + events.push("delete"); + return { + status: options.deleteStatus ?? 0, + stdout: options.deleteOutput ?? "", + stderr: "", + }; + default: + return { status: 0, stdout: "", stderr: "" }; + } + }); + vi.spyOn(runtime, "captureOpenshell").mockReturnValue({ + status: 0, + output: "", + }); + const selectGatewaySpy = vi + .spyOn(destroyGateway, "selectGatewayForSandboxDestroy") + .mockImplementation(() => undefined); + const cleanupGatewaySpy = vi + .spyOn(destroyGateway, "cleanupGatewayAfterLastSandbox") + .mockImplementation(() => undefined); + vi.spyOn(sandboxProviderCleanup, "runSandboxProviderPreDeleteCleanup").mockImplementation(() => { + events.push("detach"); + return { failures: [] }; + }); + vi.spyOn(sandboxProviderCleanup, "emitProviderDetachResidualHint").mockImplementation( + () => undefined, + ); + const stopNimByNameSpy = vi + .spyOn(nim, "stopNimContainerByName") + .mockImplementation(() => undefined); + vi.spyOn(nim, "stopNimContainer").mockImplementation(() => undefined); + const killStaleProxySpy = vi + .spyOn(ollamaProxy, "killStaleProxy") + .mockImplementation(() => undefined); + const unloadOllamaModelsSpy = vi + .spyOn(ollamaProxy, "unloadOllamaModels") + .mockImplementation(() => undefined); + const stopAllSpy = vi.spyOn(tunnelServices, "stopAll").mockImplementation(() => undefined); + vi.spyOn(timerControl, "readTimerMarker").mockReturnValue( + options.activeTimer + ? { + pid: 4242, + sandboxName: "alpha", + snapshotPath: "/tmp/policy.yaml", + restoreAt: "2026-06-27T06:00:00.000Z", + processToken: "a".repeat(32), + } + : null, + ); + vi.spyOn(shields, "shieldsUp").mockImplementation(() => { + events.push("harden"); + options.shieldsUpError === undefined + ? undefined + : (() => { + throw options.shieldsUpError; + })(); + }); + vi.spyOn(shields, "isShieldsDown").mockReturnValue(options.shieldsDown ?? true); + const shieldsDownSpy = vi.spyOn(shields, "shieldsDown").mockImplementation(() => { + events.push("unlock"); + }); + const killTimerSpy = vi.spyOn(timerControl, "killTimer").mockImplementation(() => { + events.push("timer-cleanup"); + return { warnings: [] }; + }); + const preparedServers = options.mcpAddState === "prepared" ? [] : (options.mcpServers ?? []); + const mcpPreparation = { + entries: preparedServers.map((server) => ({ server })), + detachedProviderEntries: preparedServers.map((server) => ({ server })), + scrubbedAdapterEntries: preparedServers.map((server) => ({ server })), + destroyAlreadyPrepared: false, + destroyAlreadyPending: false, + }; + const gatewayPinsAtMcpPrepare: Array = []; + const prepareMcpBridgesForDestroySpy = vi + .spyOn(mcpBridge, "prepareMcpBridgesForDestroy") + .mockImplementation(async () => { + gatewayPinsAtMcpPrepare.push(process.env.OPENSHELL_GATEWAY); + return mcpPreparation; + }); + const prepareMcpBridgesForAbsentSandboxDestroySpy = vi + .spyOn(mcpBridge, "prepareMcpBridgesForAbsentSandboxDestroy") + .mockImplementation(async () => { + gatewayPinsAtMcpPrepare.push(process.env.OPENSHELL_GATEWAY); + return mcpPreparation; + }); + const restoreMcpBridgesAfterDestroyAbortSpy = vi + .spyOn(mcpBridge, "restoreMcpBridgesAfterDestroyAbort") + .mockImplementation(async () => { + events.push("mcp-restore"); + return options.restoreMcpError === undefined + ? undefined + : Promise.reject(new Error(options.restoreMcpError)); + }); + const finalizeMcpBridgesAfterSandboxDeleteSpy = vi + .spyOn(mcpBridge, "finalizeMcpBridgesAfterSandboxDelete") + .mockImplementation(() => + options.finalizeMcpError + ? Promise.reject(new Error(options.finalizeMcpError)) + : Promise.resolve(), + ); + + logSpy.mockClear(); + + return { + cleanupGatewaySpy, + destroySandbox: requireDist(destroyModulePath).destroySandbox, + errorSpy, + events, + finalizeMcpBridgesAfterSandboxDeleteSpy, + gatewayPinsAtMcpPrepare, + gatewayPinsAtSandboxList, + killTimerSpy, + killStaleProxySpy, + logSpy, + prepareMcpBridgesForAbsentSandboxDestroySpy, + prepareMcpBridgesForDestroySpy, + removeSandboxSpy, + restoreMcpBridgesAfterDestroyAbortSpy, + runOpenshellSpy, + selectGatewaySpy, + shieldsDownSpy, + stopAllSpy, + stopNimByNameSpy, + unloadOllamaModelsSpy, + }; +} diff --git a/test/helpers/e2e-retries.ts b/test/helpers/e2e-retries.ts deleted file mode 100644 index 12f3984c89d..00000000000 --- a/test/helpers/e2e-retries.ts +++ /dev/null @@ -1,18 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -type Environment = Record; - -const DEFAULT_CI_E2E_RETRIES = 2; -const DEFAULT_LOCAL_E2E_RETRIES = 0; -const MAX_E2E_RETRIES = 5; - -export function resolveE2ERetryCount(env: Environment = process.env): number { - const override = env.NEMOCLAW_E2E_RETRIES?.trim(); - if (override && /^[0-9]+$/.test(override)) { - return Math.min(Number.parseInt(override, 10), MAX_E2E_RETRIES); - } - - const envIsCi = env.GITHUB_ACTIONS === "true" || env.CI === "true" || env.CI === "1"; - return envIsCi ? DEFAULT_CI_E2E_RETRIES : DEFAULT_LOCAL_E2E_RETRIES; -} diff --git a/test/helpers/langchain-deepagents-code-headless.ts b/test/helpers/langchain-deepagents-code-headless.ts new file mode 100644 index 00000000000..b11a2994034 --- /dev/null +++ b/test/helpers/langchain-deepagents-code-headless.ts @@ -0,0 +1,198 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { execFileSync, spawnSync } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; +import { expect } from "vitest"; + +const repoRoot = path.resolve(import.meta.dirname, "..", ".."); + +export const headlessCheckPath = path.join( + repoRoot, + "test", + "e2e", + "e2e-cloud-experimental", + "checks", + "07-deepagents-code-headless-inference.sh", +); + +export const DCODE_CANONICAL_PATH = + "/usr/local/bin:/opt/venv/bin:/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/bin"; + +export const PROXY_URL_ENV_NAMES = [ + "HTTP_PROXY", + "HTTPS_PROXY", + "http_proxy", + "https_proxy", +] as const; +export const NO_PROXY_ENV_NAMES = ["NO_PROXY", "no_proxy"] as const; +const CLEARED_PROXY_ENV_NAMES = ["ALL_PROXY", "all_proxy", "OPENAI_PROXY"] as const; +export const TRACING_ENABLE_ENV_NAMES = [ + "DEEPAGENTS_CODE_LANGSMITH_TRACING", + "DEEPAGENTS_CODE_LANGSMITH_TRACING_V2", + "DEEPAGENTS_CODE_LANGCHAIN_TRACING", + "DEEPAGENTS_CODE_LANGCHAIN_TRACING_V2", + "LANGSMITH_TRACING", + "LANGSMITH_TRACING_V2", + "LANGCHAIN_TRACING", + "LANGCHAIN_TRACING_V2", +] as const; + +export function makeStartScriptFixture( + tempDir: string, + original: string, +): { + envFile: string; + scriptPath: string; +} { + const envFile = path.join(tempDir, "proxy-env.sh"); + const scriptPath = path.join(tempDir, "start.sh"); + const hostFile = path.join(tempDir, "trusted-proxy-host"); + const portFile = path.join(tempDir, "trusted-proxy-port"); + expect(original).toContain("local target=/tmp/nemoclaw-proxy-env.sh"); + expect(original).toContain('tmp="$(mktemp /tmp/nemoclaw-proxy-env.XXXXXX)"'); + const fixture = original + .replace( + 'readonly MANAGED_PROXY_HOST_FILE="/usr/local/share/nemoclaw/dcode-proxy-host"', + `readonly MANAGED_PROXY_HOST_FILE="${hostFile}"`, + ) + .replace( + 'readonly MANAGED_PROXY_PORT_FILE="/usr/local/share/nemoclaw/dcode-proxy-port"', + `readonly MANAGED_PROXY_PORT_FILE="${portFile}"`, + ) + .replace( + "readonly MANAGED_PROXY_OWNER_UID=0", + `readonly MANAGED_PROXY_OWNER_UID=${process.getuid?.() ?? 0}`, + ) + .replace("local target=/tmp/nemoclaw-proxy-env.sh", `local target="${envFile}"`) + .replace( + 'tmp="$(mktemp /tmp/nemoclaw-proxy-env.XXXXXX)"', + `tmp="$(mktemp "${tempDir}/nemoclaw-proxy-env.XXXXXX")"`, + ); + expect(fixture).toContain(`local target="${envFile}"`); + expect(fixture).toContain(`tmp="$(mktemp "${tempDir}/nemoclaw-proxy-env.XXXXXX")"`); + expect(fixture).not.toContain("local target=/tmp/nemoclaw-proxy-env.sh"); + expect(fixture).not.toContain('tmp="$(mktemp /tmp/nemoclaw-proxy-env.XXXXXX)"'); + fs.writeFileSync(hostFile, "10.200.0.1\n", "utf8"); + fs.writeFileSync(portFile, "3128\n", "utf8"); + fs.chmodSync(hostFile, 0o444); + fs.chmodSync(portFile, 0o444); + fs.writeFileSync(scriptPath, fixture, "utf8"); + fs.chmodSync(scriptPath, 0o755); + return { envFile, scriptPath }; +} + +type HeadlessCheckOperation = + | "classify-output" + | "contains-secret" + | "managed-placeholder" + | "managed-route" + | "positive-integer"; + +type HeadlessCheckEnvironment = Partial< + Record< + "CONFIG" | "DCODE_EXIT" | "DEEPAGENTS_HEADLESS_TIMEOUT" | "HEADLESS_OUTPUT" | "TOKEN", + string + > +>; + +const HEADLESS_CHECK_HELPER_SCRIPT = ` +source test/e2e/e2e-cloud-experimental/checks/07-deepagents-code-headless-inference.sh +case "$1" in + managed-route) + printf "%s" "$CONFIG" | references_managed_inference_route && printf route + ;; + managed-placeholder) + printf "%s" "$CONFIG" | references_managed_placeholder_key && printf key + ;; + classify-output) + if classification="$(classify_headless_output "$DCODE_EXIT" "$HEADLESS_OUTPUT")"; then + printf "pass:%s" "$classification" + else + printf "fail:%s" "$classification" + fi + ;; + positive-integer) + if is_positive_integer "$HEADLESS_TIMEOUT"; then printf valid; else printf invalid; fi + ;; + contains-secret) + if printf "%s" "$TOKEN" | contains_secret; then printf secret; else printf clean; fi + ;; + *) + printf "unsupported helper operation\\n" >&2 + exit 64 + ;; +esac +`; + +export function runStartScriptProxyProbe( + scriptPath: string, + envFile: string, + env: NodeJS.ProcessEnv, +): { envFileText: string; output: string } { + const probe = [ + ...[ + ...PROXY_URL_ENV_NAMES, + ...NO_PROXY_ENV_NAMES, + ...CLEARED_PROXY_ENV_NAMES, + ...TRACING_ENABLE_ENV_NAMES, + ].map((name) => `printf 'RUNTIME_${name}=%s\\n' "\${${name}-__unset__}"`), + "unset HTTP_PROXY HTTPS_PROXY NO_PROXY http_proxy https_proxy no_proxy ALL_PROXY all_proxy", + "export ALL_PROXY=socks5://persisted-user:persisted-password@persisted-all-proxy.example:1080", + "export all_proxy=socks5://lower-persisted-user:lower-persisted-password@lower-persisted-all-proxy.example:1080", + '. "$NEMOCLAW_TEST_PROXY_ENV"', + ...[ + ...PROXY_URL_ENV_NAMES, + ...NO_PROXY_ENV_NAMES, + ...CLEARED_PROXY_ENV_NAMES, + ...TRACING_ENABLE_ENV_NAMES, + ].map((name) => `printf 'SOURCED_${name}=%s\\n' "\${${name}-__unset__}"`), + ].join("\n"); + const result = spawnSync("bash", [scriptPath, "bash", "-c", probe], { + env: { + PATH: process.env.PATH ?? "/usr/bin:/bin", + ...env, + NEMOCLAW_TEST_PROXY_ENV: envFile, + }, + encoding: "utf8", + }); + expect(result.status, result.stderr).toBe(0); + return { + envFileText: fs.readFileSync(envFile, "utf8"), + output: `${result.stdout}\n${result.stderr}`, + }; +} + +export function runHeadlessCheckHelper( + operation: HeadlessCheckOperation, + env: HeadlessCheckEnvironment = {}, +): string { + return execFileSync("/bin/bash", ["-c", HEADLESS_CHECK_HELPER_SCRIPT, "bash", operation], { + cwd: repoRoot, + encoding: "utf8", + env: { + CONFIG: env.CONFIG ?? "", + DCODE_EXIT: env.DCODE_EXIT ?? "", + DEEPAGENTS_HEADLESS_TIMEOUT: env.DEEPAGENTS_HEADLESS_TIMEOUT ?? "", + HEADLESS_OUTPUT: env.HEADLESS_OUTPUT ?? "", + PATH: "/usr/bin:/bin", + TOKEN: env.TOKEN ?? "", + }, + }); +} + +export function runHeadlessCheckSnippet( + snippet: string, + env: NodeJS.ProcessEnv = {}, + sourcePath = headlessCheckPath, +): string { + const source = fs + .readFileSync(sourcePath, "utf8") + .replace("${BASH_SOURCE[0]}", "${BASH_SOURCE[0]-}"); + return execFileSync("/bin/bash", ["-s"], { + encoding: "utf8", + env: { ...process.env, ...env }, + input: `${source}\n${snippet}\n`, + }); +} diff --git a/test/helpers/mcp-lifecycle-lock-properties.ts b/test/helpers/mcp-lifecycle-lock-properties.ts new file mode 100644 index 00000000000..693932cc864 --- /dev/null +++ b/test/helpers/mcp-lifecycle-lock-properties.ts @@ -0,0 +1,233 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fc from "fast-check"; +import { describe, expect, it } from "vitest"; + +import { + classifyMcpLifecycleLock, + type LockObservation, + type McpLifecycleLockIdentityProbes, + type McpLifecycleLockOwner, +} from "../../src/lib/state/mcp-lifecycle-lock-identity"; + +const PROPERTY_TIMEOUT_MS = 15_000; +const PROPERTY_PARAMETERS = { numRuns: 250, seed: 0x5876c0de } as const; +const SANDBOX_NAME = "property-sandbox"; +const LOCAL_HOST = "host:local"; +const LOCAL_NAMESPACE = "pid:[4026531836]"; + +const pidArbitrary = fc.integer({ min: 2, max: Number.MAX_SAFE_INTEGER }); +const durationArbitrary = fc.integer({ min: 1, max: 1_000_000 }); +const identityArbitrary = fc + .tuple(fc.uuid(), fc.bigInt({ min: 0n, max: (1n << 64n) - 1n })) + .map(([bootId, startTicks]) => `linux:${bootId}:${startTicks}`); + +function owner( + pid: number, + processIdentity: string, + overrides: Partial = {}, +): McpLifecycleLockOwner { + return { + version: 1, + sandboxName: SANDBOX_NAME, + pid, + processIdentity, + hostIdentity: LOCAL_HOST, + pidNamespaceIdentity: LOCAL_NAMESPACE, + token: "property-owner", + acquiredAt: "2026-07-01T00:00:00.000Z", + ...overrides, + }; +} + +function observation(lockOwner: McpLifecycleLockOwner | null, mtimeMs = 0): LockObservation { + return { owner: lockOwner, mtimeMs, dev: 1, ino: 1 }; +} + +function probes( + overrides: Partial = {}, +): McpLifecycleLockIdentityProbes { + return { + localHostIdentity: LOCAL_HOST, + localPidNamespaceIdentity: LOCAL_NAMESPACE, + processIsAlive: () => true, + readProcessIdentity: () => null, + ...overrides, + }; +} + +describe("MCP lifecycle lock classifier properties", () => { + it("makes corrupt or wrong-sandbox generations stale exactly at the grace boundary", { + timeout: PROPERTY_TIMEOUT_MS, + }, () => { + fc.assert( + fc.property( + durationArbitrary, + durationArbitrary, + fc.boolean(), + pidArbitrary, + identityArbitrary, + (graceMs, ageMs, hasWrongSandboxOwner, pid, identity) => { + const lockOwner = hasWrongSandboxOwner + ? owner(pid, identity, { sandboxName: `${SANDBOX_NAME}-other` }) + : null; + const localProbes = probes({ + processIsAlive: () => { + throw new Error("corrupt ownership reached the local PID table"); + }, + readProcessIdentity: () => { + throw new Error("corrupt ownership reached process identity probing"); + }, + }); + + expect( + classifyMcpLifecycleLock( + observation(lockOwner, 0), + SANDBOX_NAME, + ageMs, + graceMs, + localProbes, + ), + ).toBe(ageMs >= graceMs ? "stale" : "wait"); + }, + ), + PROPERTY_PARAMETERS, + ); + }); + + it("keeps a valid matching live owner active across lock age and grace values", { + timeout: PROPERTY_TIMEOUT_MS, + }, () => { + fc.assert( + fc.property( + pidArbitrary, + identityArbitrary, + durationArbitrary, + durationArbitrary, + (pid, identity, ageMs, graceMs) => { + expect( + classifyMcpLifecycleLock( + observation(owner(pid, identity), 0), + SANDBOX_NAME, + ageMs, + graceMs, + probes({ readProcessIdentity: () => identity }), + ), + ).toBe("active"); + }, + ), + PROPERTY_PARAMETERS, + ); + }); + + it("keeps foreign-host and foreign-namespace contenders active without local probing", { + timeout: PROPERTY_TIMEOUT_MS, + }, () => { + fc.assert( + fc.property( + pidArbitrary, + identityArbitrary, + fc.constantFrom("host", "namespace"), + (pid, identity, foreignDimension) => { + const lockOwner = owner(pid, identity, { + ...(foreignDimension === "host" + ? { hostIdentity: `${LOCAL_HOST}:foreign` } + : { pidNamespaceIdentity: `${LOCAL_NAMESPACE}:foreign` }), + }); + const localProbes = probes({ + processIsAlive: () => { + throw new Error("foreign contender reached the local PID table"); + }, + readProcessIdentity: () => { + throw new Error("foreign contender reached process identity probing"); + }, + }); + + expect( + classifyMcpLifecycleLock( + observation(lockOwner), + SANDBOX_NAME, + Number.MAX_SAFE_INTEGER, + 1, + localProbes, + ), + ).toBe("active"); + }, + ), + PROPERTY_PARAMETERS, + ); + }); + + it("applies the same liveness contract to main and reaper owner records", { + timeout: PROPERTY_TIMEOUT_MS, + }, () => { + fc.assert( + fc.property( + pidArbitrary, + identityArbitrary, + fc.constantFrom("main", "reaper"), + fc.boolean(), + (pid, identity, lockRole, isAlive) => { + // Reaper locks intentionally use the same owner schema as the main + // lock. The token prefix only identifies the role in this property. + const lockOwner = owner(pid, identity, { token: `${lockRole}-owner` }); + + expect( + classifyMcpLifecycleLock( + observation(lockOwner), + SANDBOX_NAME, + 0, + 30_000, + probes({ + processIsAlive: () => isAlive, + readProcessIdentity: () => identity, + }), + ), + ).toBe(isAlive ? "active" : "stale"); + }, + ), + PROPERTY_PARAMETERS, + ); + }); + + it("reaps a live PID only when a fresh identity read confirms the mismatch", { + timeout: PROPERTY_TIMEOUT_MS, + }, () => { + fc.assert( + fc.property( + pidArbitrary, + identityArbitrary, + fc.constantFrom("match", "mismatch", "unavailable"), + (pid, identity, freshResult) => { + const reads: Array<{ pid: number; fresh: boolean }> = []; + const replacementIdentity = `${identity}:replacement`; + const freshIdentityByResult = { + match: identity, + mismatch: replacementIdentity, + unavailable: null, + } as const; + const readProcessIdentity = (readPid: number, fresh = false): string | null => { + reads.push({ pid: readPid, fresh }); + return fresh ? freshIdentityByResult[freshResult] : replacementIdentity; + }; + + expect( + classifyMcpLifecycleLock( + observation(owner(pid, identity)), + SANDBOX_NAME, + 0, + 30_000, + probes({ readProcessIdentity }), + ), + ).toBe(freshResult === "mismatch" ? "stale" : "active"); + expect(reads).toEqual([ + { pid, fresh: false }, + { pid, fresh: true }, + ]); + }, + ), + PROPERTY_PARAMETERS, + ); + }); +}); diff --git a/test/helpers/rebuild-flow-harness.ts b/test/helpers/rebuild-flow-harness.ts index bde6fe0403c..05c773d3fe6 100644 --- a/test/helpers/rebuild-flow-harness.ts +++ b/test/helpers/rebuild-flow-harness.ts @@ -103,8 +103,6 @@ export type RebuildFlowHarness = { session: RebuildFlowSession; }; -const originalSandboxName = process.env.NEMOCLAW_SANDBOX_NAME; - // Snapshot the given env vars and return a restore fn that reinstates their // prior values exactly — vars that were unset stay unset, set ones are put back. // Branchless on purpose (filter, not conditional restore) so it both restores @@ -124,18 +122,20 @@ export function snapshotEnv(names: readonly string[]): () => void { }; } +const restoreRebuildFlowEnv = snapshotEnv([ + "NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE", + "NEMOCLAW_SANDBOX_NAME", +]); + export function resetRebuildFlowTestEnvironment(): void { delete process.env.NEMOCLAW_SANDBOX_NAME; + process.env.NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE = "1"; } export function restoreRebuildFlowTestEnvironment(): void { vi.restoreAllMocks(); delete require.cache[requireDist.resolve(rebuildModulePath)]; - if (originalSandboxName === undefined) { - delete process.env.NEMOCLAW_SANDBOX_NAME; - } else { - process.env.NEMOCLAW_SANDBOX_NAME = originalSandboxName; - } + restoreRebuildFlowEnv(); } function createStep(status: string): RebuildFlowStep { @@ -240,6 +240,7 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): name: agentName, expectedVersion: "0.2.0", dockerfileBasePath: "/tmp/Dockerfile.base", + runtime: { kind: "terminal" }, }; vi.spyOn(gatewayDrift, "detectOpenShellStateRpcPreflightIssue").mockReturnValue(null); @@ -279,6 +280,7 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): }, ); vi.spyOn(onboardSession, "loadSession").mockReturnValue(session); + vi.spyOn(onboardSession, "acquireOnboardLock").mockReturnValue({ acquired: true }); vi.spyOn(onboardSession, "updateSession").mockImplementation((mutator: unknown) => { if (typeof mutator !== "function") { throw new TypeError("updateSession expected a mutator function"); @@ -325,7 +327,7 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): }; }); vi.spyOn(registry, "listSandboxes").mockReturnValue({ sandboxes: [] }); - const registryUpdateSpy = vi.spyOn(registry, "updateSandbox").mockImplementation(() => undefined); + const registryUpdateSpy = vi.spyOn(registry, "updateSandbox").mockReturnValue(true); const restoreSandboxEntrySpy = vi .spyOn(registry, "restoreSandboxEntry") .mockImplementation(() => undefined); @@ -424,6 +426,7 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): const onboardSpy = vi.spyOn(onboardMod, "onboard").mockImplementation(async () => { await overrides.onboard?.(session); }); + vi.spyOn(onboardMod, "preflightAuthoritativeRebuildTarget").mockResolvedValue(undefined); const applyPresetSpy = vi .spyOn(policies, "applyPreset") .mockImplementation((_sandboxName: unknown, presetName: unknown) => { diff --git a/test/helpers/rebuild-flow-lifecycle-cases.ts b/test/helpers/rebuild-flow-lifecycle-cases.ts new file mode 100644 index 00000000000..35143511be4 --- /dev/null +++ b/test/helpers/rebuild-flow-lifecycle-cases.ts @@ -0,0 +1,319 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; +import { + createRebuildFlowHarness, + installRebuildFlowTestHooks, + originalSandboxName, + snapshotEnv, +} from "./rebuild-flow-test-harness"; + +export function registerRebuildFlowLifecycleTests(): void { + describe("rebuildSandbox flow: lifecycle", () => { + installRebuildFlowTestHooks(); + it("backs up, recreates, restores, reapplies policy, and relocks on a successful OpenClaw rebuild", async () => { + const mcpEntry = { + server: "github", + url: "https://mcp.example.test/mcp", + env: ["GITHUB_TOKEN"], + providerName: "nemoclaw-mcp-alpha-github", + policyName: "mcp-bridge-github", + adapter: "mcporter", + createdAt: "2026-06-01T00:00:00.000Z", + updatedAt: "2026-06-01T00:00:00.000Z", + }; + const harness = createRebuildFlowHarness({ + applyPreset: () => true, + sandboxEntry: { policyPresetsFinalized: true, policyTier: "balanced" }, + mcpPreparation: { + entries: [mcpEntry], + detachedProviderEntries: [mcpEntry], + }, + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes", "--verbose"], { throwOnError: true }), + ).resolves.toBeUndefined(); + + expect(harness.backupSandboxStateSpy).toHaveBeenCalledWith("alpha"); + expect(harness.prepareMcpBridgesForRebuildSpy).toHaveBeenCalledWith("alpha"); + expect(harness.prepareMcpBridgesForRebuildSpy.mock.invocationCallOrder[0]).toBeLessThan( + harness.warnUnpreservedUserManagedFilesSpy.mock.invocationCallOrder[0], + ); + expect(harness.runOpenshellSpy).toHaveBeenCalledWith( + ["sandbox", "delete", "alpha"], + expect.objectContaining({ ignoreError: true }), + ); + expect(harness.onboardSpy).toHaveBeenCalledWith( + expect.objectContaining({ + resume: true, + nonInteractive: true, + recreateSandbox: true, + authoritativeResumeConfig: true, + autoYes: true, + }), + ); + expect(harness.registryUpdateSpy).toHaveBeenCalledWith( + "alpha", + expect.objectContaining({ + provider: "ollama-local", + model: "nvidia/nemotron", + webSearchEnabled: false, + fromDockerfile: null, + hermesAuthMethod: null, + }), + ); + const deleteCall = harness.runOpenshellSpy.mock.calls.findIndex( + (call) => Array.isArray(call[0]) && call[0].join(" ") === "sandbox delete alpha", + ); + expect(harness.registryUpdateSpy.mock.invocationCallOrder[0]).toBeLessThan( + harness.runOpenshellSpy.mock.invocationCallOrder[deleteCall], + ); + expect(harness.session.policyPresets).toEqual(["npm", "bad", "throw"]); + expect(harness.session.steps.gateway.status).toBe("complete"); + expect(harness.session.steps.preflight.status).toBe("complete"); + expect(harness.session.steps.sandbox.status).toBe("pending"); + expect(harness.restoreSandboxStateSpy).toHaveBeenCalledWith( + "alpha", + "/tmp/nemoclaw-rebuild-backup", + ); + expect(harness.restoreMcpBridgesAfterRebuildSpy).toHaveBeenCalledWith("alpha", [mcpEntry]); + expect(harness.removeSandboxRegistryEntrySpy).not.toHaveBeenCalled(); + expect(harness.errorSpy.mock.calls.map((call) => String(call[0])).join("\n")).toContain( + "Preserving MCP-bearing registry entry across sandbox recreation", + ); + expect(harness.applyPresetSpy).toHaveBeenCalledWith("alpha", "npm"); + expect(harness.applyPresetSpy).toHaveBeenCalledWith("alpha", "bad"); + expect(harness.applyPresetSpy).toHaveBeenCalledWith("alpha", "throw"); + expect(harness.registryUpdateSpy).toHaveBeenCalledWith("alpha", { + agentVersion: "0.2.0", + policies: ["npm", "bad", "throw"], + policyTier: "balanced", + policyPresetsFinalized: true, + }); + expect(harness.executeSandboxCommandSpy).toHaveBeenCalledWith( + "alpha", + "openclaw doctor --fix", + ); + expect(harness.relockSpy).toHaveBeenCalledWith("alpha", expect.any(Object), true, "nemoclaw"); + expect(process.env.NEMOCLAW_SANDBOX_NAME).toBe(originalSandboxName); + expect(harness.logSpy.mock.calls.map((call) => String(call[0])).join("\n")).toContain( + "rebuilt successfully", + ); + }); + + it("relocks as absent when registry cleanup throws after confirmed delete", async () => { + const harness = createRebuildFlowHarness({ + removeSandboxRegistryEntry: () => { + throw new Error("registry cleanup after delete failed"); + }, + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).rejects.toThrow("registry cleanup after delete failed"); + + expect(harness.onboardSpy).not.toHaveBeenCalled(); + expect(harness.relockSpy).toHaveBeenLastCalledWith( + "alpha", + expect.any(Object), + false, + "nemoclaw", + ); + }); + + it("relocks as present when shields postwork throws after successful onboard", async () => { + const harness = createRebuildFlowHarness({ + staleRecovery: true, + clearShieldsState: () => { + throw new Error("post-onboard shields cleanup failed"); + }, + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).rejects.toThrow("post-onboard shields cleanup failed"); + + expect(harness.onboardSpy).toHaveBeenCalledOnce(); + expect(harness.relockSpy).toHaveBeenLastCalledWith( + "alpha", + expect.any(Object), + true, + "nemoclaw", + ); + }); + + it("uses the no-exec MCP preparation path when recovering an absent sandbox", async () => { + const overrideEnvVar = "NEMOCLAW_HERMES_SANDBOX_BASE_IMAGE_REF"; + const restoreEnv = snapshotEnv([overrideEnvVar]); + process.env[overrideEnvVar] = "nemoclaw-hermes-sandbox-base-local:image-caller"; + const mcpEntry = { + server: "github", + agent: "openclaw", + adapter: "mcporter", + url: "https://mcp.example.test/mcp", + env: ["GITHUB_TOKEN"], + providerName: "alpha-mcp-github", + policyName: "mcp-bridge-github", + addedAt: "2026-06-01T00:00:00.000Z", + }; + try { + const harness = createRebuildFlowHarness({ + staleRecovery: true, + sandboxEntry: { mcp: { bridges: { github: mcpEntry } } }, + baseImagePreflight: { + ok: true, + imageRef: "nemoclaw-hermes-sandbox-base-local:image-preflighted", + overrideEnvVar, + }, + mcpPreparation: { + entries: [mcpEntry], + detachedProviderEntries: [], + scrubbedAdapterEntries: [], + }, + onboard: () => { + expect(process.env[overrideEnvVar]).toBe( + "nemoclaw-hermes-sandbox-base-local:image-preflighted", + ); + }, + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).resolves.toBeUndefined(); + + expect(process.env[overrideEnvVar]).toBe("nemoclaw-hermes-sandbox-base-local:image-caller"); + expect(harness.backupSandboxStateSpy).not.toHaveBeenCalled(); + expect(harness.prepareMcpBridgesForAbsentSandboxRebuildSpy).toHaveBeenCalledWith("alpha"); + expect(harness.prepareMcpBridgesForRebuildSpy).not.toHaveBeenCalled(); + expect(harness.warnUnpreservedUserManagedFilesSpy).not.toHaveBeenCalled(); + expect(harness.reattachMcpProvidersAfterRebuildAbortSpy).not.toHaveBeenCalled(); + expect(harness.restoreMcpBridgesAfterRebuildSpy).toHaveBeenCalledWith("alpha", [mcpEntry]); + } finally { + restoreEnv(); + } + }); + + it("pins compatible-endpoint reasoning for an MCP-bearing rebuild", async () => { + const restoreEnv = snapshotEnv(["COMPATIBLE_API_KEY", "NEMOCLAW_REASONING"]); + process.env.COMPATIBLE_API_KEY = "compat-key"; + process.env.NEMOCLAW_REASONING = "false"; + const mcpEntry = { + server: "github", + agent: "openclaw", + adapter: "mcporter", + url: "https://mcp.example.test/mcp", + env: ["GITHUB_TOKEN"], + providerName: "alpha-mcp-github", + policyName: "mcp-bridge-github", + addedAt: "2026-06-01T00:00:00.000Z", + }; + let reasoningSeenInsideOnboard: string | undefined; + try { + const harness = createRebuildFlowHarness({ + applyPreset: () => true, + sandboxEntry: { + provider: "compatible-endpoint", + model: "reasoning-model", + endpointUrl: "https://compatible.example.test/v1", + compatibleEndpointReasoning: "true", + mcp: { bridges: { github: mcpEntry } }, + }, + sessionSandboxName: "other", + mcpPreparation: { + entries: [mcpEntry], + detachedProviderEntries: [mcpEntry], + }, + onboard: (session) => { + reasoningSeenInsideOnboard = process.env.NEMOCLAW_REASONING; + expect(session.compatibleEndpointReasoning).toBe("true"); + }, + }); + harness.session.compatibleEndpointReasoning = "false"; + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).resolves.toBeUndefined(); + + expect(reasoningSeenInsideOnboard).toBeUndefined(); + expect(harness.session.compatibleEndpointReasoning).toBe("true"); + expect(process.env.NEMOCLAW_REASONING).toBe("false"); + expect(harness.restoreMcpBridgesAfterRebuildSpy).toHaveBeenCalledWith("alpha", [mcpEntry]); + } finally { + restoreEnv(); + } + }); + + it("restores enabled messaging presets while pruning disabled ones from final policies", async () => { + const disabledSlackPlan = { + schemaVersion: 1, + sandboxName: "alpha", + agent: "openclaw", + workflow: "rebuild", + channels: [ + { channelId: "telegram", disabled: false }, + { channelId: "discord", disabled: false }, + { channelId: "whatsapp", disabled: false }, + { channelId: "wechat", disabled: false }, + { channelId: "slack", disabled: true }, + ], + disabledChannels: ["slack"], + credentialBindings: [], + networkPolicy: { presets: [], entries: [] }, + agentRender: [], + buildSteps: [], + stateUpdates: [], + healthChecks: [], + }; + const harness = createRebuildFlowHarness({ + applyPreset: () => true, + backupPolicyPresets: ["slack", "npm", "pypi", "telegram"], + buildMessagingRebuildPlan: () => disabledSlackPlan, + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).resolves.toBeUndefined(); + + expect(harness.applyPresetSpy.mock.calls.map((call) => call[1])).toEqual([ + "npm", + "pypi", + "telegram", + "discord", + "whatsapp", + "wechat", + ]); + expect(harness.registryUpdateSpy).toHaveBeenCalledWith("alpha", { + agentVersion: "0.2.0", + policies: ["npm", "pypi", "telegram", "discord", "whatsapp", "wechat"], + policyTier: null, + policyPresetsFinalized: undefined, + }); + }); + + it("preserves a finalized empty policy selection and its tier", async () => { + const harness = createRebuildFlowHarness({ + applyPreset: () => true, + backupPolicyPresets: [], + sandboxEntry: { + policies: [], + policyPresetsFinalized: true, + policyTier: "restricted", + }, + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).resolves.toBeUndefined(); + + expect(harness.session.policyPresets).toEqual([]); + expect(harness.registryUpdateSpy).toHaveBeenCalledWith("alpha", { + agentVersion: "0.2.0", + policies: [], + policyTier: "restricted", + policyPresetsFinalized: true, + }); + }); + }); +} diff --git a/test/helpers/rebuild-flow-recovery-cases.ts b/test/helpers/rebuild-flow-recovery-cases.ts new file mode 100644 index 00000000000..070d7ddd999 --- /dev/null +++ b/test/helpers/rebuild-flow-recovery-cases.ts @@ -0,0 +1,432 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; +import { + makeActiveTeamsMessagingPlan, + makePreparedRecoveryManifest, +} from "../../src/lib/actions/sandbox/rebuild-flow-test-fixtures"; +import { createRebuildFlowHarness, installRebuildFlowTestHooks } from "./rebuild-flow-test-harness"; + +export function registerRebuildFlowRecoveryTests(): void { + describe("rebuildSandbox flow: recovery", () => { + installRebuildFlowTestHooks(); + + it("restores a validated prepared manifest without taking a second backup (#6114)", async () => { + const harness = createRebuildFlowHarness({ sandboxListOutput: "alpha Error" }); + const recoveryManifest = makePreparedRecoveryManifest(); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { + throwOnError: true, + recoveryManifest, + }), + ).resolves.toBeUndefined(); + + expect(harness.backupSandboxStateSpy).not.toHaveBeenCalled(); + expect(harness.runOpenshellSpy).toHaveBeenCalledWith( + ["sandbox", "delete", "alpha"], + expect.objectContaining({ ignoreError: true }), + ); + expect(harness.restoreSandboxStateSpy).toHaveBeenCalledWith( + "alpha", + recoveryManifest.backupPath, + ); + }); + + it("rejects a mismatched prepared manifest before deleting the sandbox (#6114)", async () => { + const harness = createRebuildFlowHarness({ + recoveryManifestValidation: () => ({ + ok: false, + reason: "manifest sandbox 'beta' does not match 'alpha'", + }), + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { + throwOnError: true, + recoveryManifest: makePreparedRecoveryManifest(), + }), + ).rejects.toThrow("Invalid recovery manifest"); + + expect(harness.backupSandboxStateSpy).not.toHaveBeenCalled(); + expect(harness.runOpenshellSpy).not.toHaveBeenCalledWith( + ["sandbox", "delete", "alpha"], + expect.anything(), + ); + expect(harness.onboardSpy).not.toHaveBeenCalled(); + }); + + it("revalidates a prepared manifest immediately before deletion (#6114)", async () => { + let validationCount = 0; + const harness = createRebuildFlowHarness({ + recoveryManifestValidation: (manifest) => { + validationCount++; + return validationCount === 1 + ? { ok: true, manifest } + : { ok: false, reason: "persisted backup identity changed during validation" }; + }, + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { + throwOnError: true, + recoveryManifest: makePreparedRecoveryManifest(), + }), + ).rejects.toThrow("Invalid recovery manifest"); + + expect(validationCount).toBe(2); + expect(harness.backupSandboxStateSpy).not.toHaveBeenCalled(); + expect(harness.runOpenshellSpy).not.toHaveBeenCalledWith( + ["sandbox", "delete", "alpha"], + expect.anything(), + ); + }); + + it("rejects registry configuration drift before prepared recovery deletion (#6114)", async () => { + const harness = createRebuildFlowHarness({ + preDeleteSandboxEntry: { + name: "alpha", + provider: "compatible-endpoint", + model: "new-model", + policies: ["npm", "github"], + agent: null, + agentVersion: "0.1.0", + nemoclawVersion: "0.0.71", + }, + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { + throwOnError: true, + recoveryManifest: makePreparedRecoveryManifest(), + }), + ).rejects.toThrow("Recovery registry configuration changed during preflight"); + + expect(harness.backupSandboxStateSpy).not.toHaveBeenCalled(); + expect(harness.runOpenshellSpy).not.toHaveBeenCalledWith( + ["sandbox", "delete", "alpha"], + expect.anything(), + ); + }); + + it("uses the refreshed registry snapshot for prepared-recovery rollback (#6114)", async () => { + const harness = createRebuildFlowHarness({ + defaultSandbox: "alpha", + preDeleteDefaultSandbox: "beta", + onboard: () => { + throw new Error("recreate failed"); + }, + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { + throwOnError: true, + recoveryManifest: makePreparedRecoveryManifest(), + }), + ).rejects.toThrow("Recreate failed"); + + expect(harness.restoreSandboxEntrySpy).toHaveBeenCalledWith( + expect.objectContaining({ name: "alpha", agentVersion: "0.1.0" }), + { reclaimDefault: null }, + ); + }); + + it("rejects a latest-backup change before prepared recovery deletion (#6114)", async () => { + const harness = createRebuildFlowHarness({ + preDeleteLatestManifest: { + ...makePreparedRecoveryManifest(), + timestamp: "2026-07-01T07-00-00-000Z", + backupPath: "/tmp/rebuild-backups/alpha/2026-07-01T07-00-00-000Z", + }, + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { + throwOnError: true, + recoveryManifest: makePreparedRecoveryManifest(), + }), + ).rejects.toThrow("Recovery backup identity changed during preflight"); + + expect(harness.backupSandboxStateSpy).not.toHaveBeenCalled(); + expect(harness.runOpenshellSpy).not.toHaveBeenCalledWith( + ["sandbox", "delete", "alpha"], + expect.anything(), + ); + }); + + it("restores the registry entry when prepared-backup recreation fails (#6114)", async () => { + const harness = createRebuildFlowHarness({ + defaultSandbox: "alpha", + onboard: () => { + throw new Error("recreate failed"); + }, + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { + throwOnError: true, + recoveryManifest: makePreparedRecoveryManifest(), + }), + ).rejects.toThrow("Recreate failed"); + + expect(harness.backupSandboxStateSpy).not.toHaveBeenCalled(); + expect(harness.restoreSandboxEntrySpy).toHaveBeenCalledWith( + expect.objectContaining({ name: "alpha", agentVersion: "0.1.0" }), + { reclaimDefault: "alpha" }, + ); + expect(harness.restoreSandboxStateSpy).not.toHaveBeenCalled(); + }); + + it("performs exactly one prepared-recovery rollback when MCP state is present", async () => { + const mcpEntry = { server: "github", providerName: "nemoclaw-mcp-alpha-github" }; + const harness = createRebuildFlowHarness({ + defaultSandbox: "alpha", + mcpPreparation: { + entries: [mcpEntry], + detachedProviderEntries: [mcpEntry], + scrubbedAdapterEntries: [mcpEntry], + }, + onboard: () => { + throw new Error("recreate failed"); + }, + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { + throwOnError: true, + recoveryManifest: makePreparedRecoveryManifest(), + }), + ).rejects.toThrow("Recreate failed"); + + expect(harness.restoreSandboxEntrySpy.mock.calls).toEqual([ + [expect.objectContaining({ name: "alpha" }), { reclaimDefault: "alpha" }], + ]); + }); + + it("blocks installer recovery when MCP post-restore verification is incomplete", async () => { + const mcpEntry = { server: "github", providerName: "nemoclaw-mcp-alpha-github" }; + const harness = createRebuildFlowHarness({ + mcpPreparation: { + entries: [mcpEntry], + detachedProviderEntries: [mcpEntry], + scrubbedAdapterEntries: [mcpEntry], + }, + restoreMcpBridgesAfterRebuild: () => Promise.reject(new Error("MCP restore boom")), + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { + throwOnError: true, + recoveryManifest: makePreparedRecoveryManifest(), + }), + ).rejects.toThrow("Prepared backup recovery"); + + expect(harness.errorSpy).toHaveBeenCalledWith( + expect.stringContaining("MCP bridge restore incomplete: MCP restore boom"), + ); + expect(harness.relockSpy).toHaveBeenCalled(); + }); + + it("prunes the disabled Teams preset from the final registry policies after rebuild", async () => { + const disabledTeamsPlan = { + schemaVersion: 1, + sandboxName: "alpha", + agent: "openclaw", + workflow: "rebuild", + channels: [], + disabledChannels: ["teams"], + credentialBindings: [], + networkPolicy: { presets: [], entries: [] }, + agentRender: [], + buildSteps: [], + stateUpdates: [], + healthChecks: [], + }; + const harness = createRebuildFlowHarness({ + applyPreset: () => true, + backupPolicyPresets: ["teams", "npm"], + buildMessagingRebuildPlan: () => disabledTeamsPlan, + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).resolves.toBeUndefined(); + + expect(harness.applyPresetSpy).toHaveBeenCalledWith("alpha", "npm"); + expect(harness.applyPresetSpy).not.toHaveBeenCalledWith("alpha", "teams"); + expect(harness.registryUpdateSpy).toHaveBeenCalledWith("alpha", { + agentVersion: "0.2.0", + policies: ["npm"], + policyTier: null, + policyPresetsFinalized: undefined, + }); + }); + + it("aborts before backup/delete when messaging manifest staging fails", async () => { + const harness = createRebuildFlowHarness({ + buildMessagingRebuildPlan: () => { + throw new Error("manifest boom"); + }, + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).rejects.toThrow("manifest boom"); + + const errors = harness.errorSpy.mock.calls.map((call) => String(call[0])).join("\n"); + expect(errors).toContain("messaging manifest plan could not be staged"); + expect(harness.releaseOnboardLockSpy).toHaveBeenCalledOnce(); + expect(harness.backupSandboxStateSpy).not.toHaveBeenCalled(); + expect(harness.runOpenshellSpy).not.toHaveBeenCalledWith( + ["sandbox", "delete", "alpha"], + expect.anything(), + ); + expect(harness.onboardSpy).not.toHaveBeenCalled(); + }); + + it("reattaches exactly the MCP providers detached when sandbox deletion fails", async () => { + const attached = { + server: "attached", + providerName: "nemoclaw-mcp-alpha-attached", + }; + const alreadyDetached = { + server: "already-detached", + providerName: "nemoclaw-mcp-alpha-already-detached", + }; + const harness = createRebuildFlowHarness({ + mcpPreparation: { + entries: [attached, alreadyDetached], + detachedProviderEntries: [attached], + }, + runOpenshell: (args) => + args.join(" ") === "sandbox delete alpha" + ? { status: 7, output: "delete failed", stderr: "delete failed" } + : { status: 0, output: "" }, + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).rejects.toThrow("Failed to delete sandbox"); + + expect(harness.reattachMcpProvidersAfterRebuildAbortSpy).toHaveBeenCalledWith( + "alpha", + [attached], + undefined, + ); + expect(harness.onboardSpy).not.toHaveBeenCalled(); + }); + + it("does not reclaim the default sandbox when an MCP rebuild recreate fails", async () => { + const mcpEntry = { + server: "github", + providerName: "nemoclaw-mcp-alpha-github", + }; + const harness = createRebuildFlowHarness({ + defaultSandbox: "alpha", + mcpPreparation: { + entries: [mcpEntry], + detachedProviderEntries: [mcpEntry], + }, + onboard: () => { + throw new Error("inner recreate boom"); + }, + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).rejects.toThrow("Recreate failed"); + + expect(harness.removeSandboxRegistryEntrySpy).not.toHaveBeenCalled(); + expect(harness.restoreSandboxEntrySpy.mock.calls).toEqual([ + [expect.objectContaining({ name: "alpha" })], + ]); + }); + + it("starts the active Teams host forward after a successful rebuild", async () => { + const plan = makeActiveTeamsMessagingPlan(); + const harness = createRebuildFlowHarness({ + applyPreset: () => true, + buildMessagingRebuildPlan: () => plan, + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).resolves.toBeUndefined(); + + expect(harness.ensureMessagingHostForwardAfterRebuildSpy).toHaveBeenCalledWith("alpha", plan); + expect( + harness.ensureMessagingHostForwardAfterRebuildSpy.mock.invocationCallOrder[0], + ).toBeGreaterThan(harness.onboardSpy.mock.invocationCallOrder[0]); + }); + + it("finishes the rebuild while surfacing incomplete post-restore work", async () => { + const harness = createRebuildFlowHarness({ + sandboxEntry: { policyPresetsFinalized: true, policyTier: "balanced" }, + executeSandboxCommand: () => ({ status: 1, stdout: "", stderr: "hash refresh failed" }), + repairMutableConfigPerms: () => ({ + applied: false, + skipReason: "unreadable", + reason: "cannot stat mutable config", + }), + restoreSandboxState: () => ({ + success: false, + restoredDirs: ["workspace"], + restoredFiles: [], + failedDirs: ["config"], + failedFiles: ["user.md"], + }), + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).resolves.toBeUndefined(); + + const output = harness.logSpy.mock.calls.map((call) => String(call[0])).join("\n"); + expect(output).toContain("rebuilt but some post-restore steps were incomplete"); + expect(output).toContain("State restore was incomplete"); + expect(output).toContain("Mutable config permissions were not verified"); + expect(output).toContain("Mutable OpenClaw config hash was not refreshed"); + expect(harness.applyPresetSpy).toHaveBeenCalledWith("alpha", "bad"); + expect(harness.applyPresetSpy).toHaveBeenCalledWith("alpha", "throw"); + expect(harness.errorSpy).toHaveBeenCalledWith(expect.stringContaining("bad, throw")); + expect(harness.relockSpy).toHaveBeenCalledWith("alpha", expect.any(Object), true, "nemoclaw"); + expect(harness.registryUpdateSpy).toHaveBeenCalledWith("alpha", { + agentVersion: "0.2.0", + policies: ["npm"], + policyTier: "balanced", + policyPresetsFinalized: undefined, + }); + expect(output).toContain("Policy presets failed to reapply: bad, throw"); + }); + + it("reports both MCP and policy recovery when both restores are incomplete", async () => { + const mcpEntry = { + server: "github", + providerName: "nemoclaw-mcp-alpha-github", + }; + const harness = createRebuildFlowHarness({ + applyPreset: () => false, + backupPolicyPresets: ["npm"], + mcpPreparation: { + entries: [mcpEntry], + detachedProviderEntries: [mcpEntry], + }, + restoreMcpBridgesAfterRebuild: () => Promise.reject(new Error("MCP restore boom")), + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).resolves.toBeUndefined(); + + const output = harness.logSpy.mock.calls.map((call) => String(call[0])).join("\n"); + expect(output).toContain("rebuilt but some post-restore steps were incomplete"); + expect(output).toContain("MCP bridge definitions were preserved but not fully refreshed"); + expect(output).toContain("Policy presets failed to reapply: npm"); + expect(output).not.toContain("rebuilt successfully"); + expect(harness.errorSpy).toHaveBeenCalledWith( + expect.stringContaining("MCP bridge restore incomplete: MCP restore boom"), + ); + }); + }); +} diff --git a/test/helpers/rebuild-flow-target-credentials-cases.ts b/test/helpers/rebuild-flow-target-credentials-cases.ts new file mode 100644 index 00000000000..ee53b749e0b --- /dev/null +++ b/test/helpers/rebuild-flow-target-credentials-cases.ts @@ -0,0 +1,310 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; +import { + createRebuildFlowHarness, + installRebuildFlowTestHooks, + snapshotEnv, +} from "./rebuild-flow-test-harness"; + +export function registerRebuildFlowTargetCredentialsTests(): void { + describe("rebuildSandbox flow: target credentials", () => { + installRebuildFlowTestHooks(); + it("aborts before backup/delete when durable Brave credential validation fails", async () => { + const harness = createRebuildFlowHarness({ + sandboxEntry: { webSearchEnabled: true }, + sessionSandboxName: "some-other-sandbox", + ensureValidatedBraveSearchCredential: async () => { + throw new Error("invalid Brave credential"); + }, + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).rejects.toThrow("Brave Search credential preflight failed"); + + expect(harness.backupSandboxStateSpy).not.toHaveBeenCalled(); + expect(harness.runOpenshellSpy).not.toHaveBeenCalledWith( + ["sandbox", "delete", "alpha"], + expect.anything(), + ); + }); + + it("rejects recorded web search when the target agent does not support it", async () => { + const harness = createRebuildFlowHarness({ + sandboxEntry: { agent: "hermes", webSearchEnabled: true }, + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).rejects.toThrow("Recorded Brave Search is unsupported"); + + expect(harness.backupSandboxStateSpy).not.toHaveBeenCalled(); + }); + + it("rejects a Tavily credential already owned by MCP before rebuild mutation", async () => { + const harness = createRebuildFlowHarness({ + sandboxEntry: { + webSearchEnabled: true, + webSearchProvider: "tavily", + mcp: { + bridges: { + search: { + server: "search", + agent: "openclaw", + url: "https://mcp.example.com/mcp", + env: ["TAVILY_API_KEY"], + policyName: "alpha-mcp-search", + addedAt: "2026-07-03T00:00:00.000Z", + }, + }, + }, + }, + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).rejects.toThrow("Web Search and MCP credential ownership conflict"); + + expect(harness.backupSandboxStateSpy).not.toHaveBeenCalled(); + expect(harness.prepareMcpBridgesForRebuildSpy).not.toHaveBeenCalled(); + expect(harness.runOpenshellSpy).not.toHaveBeenCalledWith( + ["sandbox", "delete", "alpha"], + expect.anything(), + ); + }); + + it("preserves legacy Brave web search during a nonmatching-session rebuild", async () => { + const harness = createRebuildFlowHarness({ + applyPreset: () => true, + sandboxEntry: { policies: ["brave"], webSearchEnabled: undefined }, + sessionSandboxName: "some-other-sandbox", + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).resolves.toBeUndefined(); + + expect(harness.ensureValidatedBraveSearchCredentialSpy).toHaveBeenCalledWith( + { fetchEnabled: true, provider: "brave" }, + true, + ); + expect(harness.session.webSearchConfig).toEqual({ fetchEnabled: true, provider: "brave" }); + }); + + it("reconciles stale Brave policy state to the durable Tavily provider", async () => { + const harness = createRebuildFlowHarness({ + applyPreset: (name) => name === "tavily", + backupPolicyPresets: ["brave"], + sandboxEntry: { + policies: ["brave"], + webSearchEnabled: true, + webSearchProvider: "tavily", + }, + sessionSandboxName: "some-other-sandbox", + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).resolves.toBeUndefined(); + + expect(harness.applyPresetSpy).toHaveBeenCalledWith("alpha", "tavily"); + expect(harness.applyPresetSpy).not.toHaveBeenCalledWith("alpha", "brave"); + expect(harness.session.webSearchConfig).toEqual({ + fetchEnabled: true, + provider: "tavily", + }); + }); + + it("restores the caller Tavily credential environment after rebuild", async () => { + const restoreEnv = snapshotEnv(["TAVILY_API_KEY"]); + process.env.TAVILY_API_KEY = "caller-tavily-key"; + try { + const harness = createRebuildFlowHarness({ + applyPreset: () => true, + sandboxEntry: { webSearchEnabled: true, webSearchProvider: "tavily" }, + ensureValidatedWebSearchCredential: async () => { + process.env.TAVILY_API_KEY = "validated-tavily-key"; + return "validated-tavily-key"; + }, + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).resolves.toBeUndefined(); + expect(process.env.TAVILY_API_KEY).toBe("caller-tavily-key"); + } finally { + restoreEnv(); + } + }); + + it("recreates unrelated-session targets from durable web, image, and Hermes auth state", async () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-rebuild-from-")); + const dockerfile = path.join(tempDir, "Dockerfile.custom"); + fs.writeFileSync(dockerfile, "FROM scratch\nARG NEMOCLAW_WEB_SEARCH_ENABLED=0\n"); + const harness = createRebuildFlowHarness({ + applyPreset: () => true, + sessionSandboxName: "some-other-sandbox", + sandboxEntry: { + provider: "hermes-provider", + model: "hermes-model", + webSearchEnabled: true, + fromDockerfile: dockerfile, + hermesAuthMethod: "api_key", + }, + hermesCredentialKeys: ["NOUS_API_KEY"], + }); + harness.session.webSearchConfig = null; + harness.session.hermesAuthMethod = "oauth"; + harness.session.metadata = { fromDockerfile: "/tmp/unrelated.Dockerfile" }; + + try { + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).resolves.toBeUndefined(); + + expect(harness.ensureValidatedBraveSearchCredentialSpy).toHaveBeenCalledWith( + { fetchEnabled: true, provider: "brave" }, + true, + ); + expect(harness.session.webSearchConfig).toEqual({ + fetchEnabled: true, + provider: "brave", + }); + expect(harness.session.hermesAuthMethod).toBe("api_key"); + expect(harness.session.credentialEnv).toBe("NOUS_API_KEY"); + expect(harness.session.metadata).toMatchObject({ fromDockerfile: dockerfile }); + expect(harness.onboardSpy).toHaveBeenCalledWith( + expect.objectContaining({ fromDockerfile: dockerfile }), + ); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } + }); + + it("keeps the Hermes OAuth credential binding with durable OAuth auth", async () => { + const harness = createRebuildFlowHarness({ + applyPreset: () => true, + sandboxEntry: { + agent: "hermes", + provider: "hermes-provider", + model: "hermes-model", + hermesAuthMethod: "oauth", + }, + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).resolves.toBeUndefined(); + + expect(harness.session.hermesAuthMethod).toBe("oauth"); + expect(harness.session.credentialEnv).toBe("OPENAI_API_KEY"); + }); + + it("rejects a shared Hermes Provider whose credential binding changed", async () => { + const harness = createRebuildFlowHarness({ + sandboxEntry: { + provider: "hermes-provider", + model: "hermes-model", + hermesAuthMethod: "api_key", + }, + hermesCredentialKeys: ["OPENAI_API_KEY"], + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).rejects.toThrow("Missing Hermes Provider credentials"); + expect(harness.backupSandboxStateSpy).not.toHaveBeenCalled(); + }); + + it("does not use a generic provider alias to recreate a missing Hermes API-key binding", async () => { + const restoreEnv = snapshotEnv(["NOUS_API_KEY", "NEMOCLAW_PROVIDER_KEY"]); + delete process.env.NOUS_API_KEY; + process.env.NEMOCLAW_PROVIDER_KEY = "unrelated-provider-key"; + try { + const harness = createRebuildFlowHarness({ + sandboxEntry: { + provider: "hermes-provider", + model: "hermes-model", + hermesAuthMethod: "api_key", + }, + hermesProviderExists: false, + }); + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).rejects.toThrow("Missing Hermes Provider credentials"); + expect(harness.backupSandboxStateSpy).not.toHaveBeenCalled(); + } finally { + restoreEnv(); + } + }); + + it("ignores a stale matching-session credential for a resolved local target", async () => { + const harness = createRebuildFlowHarness(); + harness.session.credentialEnv = "NVIDIA_INFERENCE_API_KEY"; + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).resolves.toBeUndefined(); + + expect(harness.session.credentialEnv).toBeNull(); + }); + + it("fails closed when a legacy matching session recovers Hermes without auth state", async () => { + const harness = createRebuildFlowHarness({ + sandboxEntry: { provider: null, model: null, hermesAuthMethod: undefined }, + }); + harness.session.provider = "hermes-provider"; + harness.session.model = "hermes-model"; + harness.session.hermesAuthMethod = null; + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).rejects.toThrow("Cannot determine recorded Hermes Provider authentication method"); + + expect(harness.backupSandboxStateSpy).not.toHaveBeenCalled(); + expect(harness.onboardSpy).not.toHaveBeenCalled(); + }); + + it("treats durable web-search false and Dockerfile null as authoritative", async () => { + const harness = createRebuildFlowHarness({ + applyPreset: () => true, + sandboxEntry: { + webSearchEnabled: false, + fromDockerfile: null, + hermesAuthMethod: null, + }, + }); + harness.session.webSearchConfig = { fetchEnabled: true }; + harness.session.hermesAuthMethod = "oauth"; + harness.session.metadata = { fromDockerfile: "/tmp/stale.Dockerfile" }; + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).resolves.toBeUndefined(); + + expect(harness.ensureValidatedBraveSearchCredentialSpy).not.toHaveBeenCalled(); + expect(harness.session.webSearchConfig).toBeNull(); + expect(harness.session.hermesAuthMethod).toBeNull(); + expect(harness.session.metadata).toMatchObject({ fromDockerfile: null }); + }); + + it("aborts before backup/delete when the durable custom Dockerfile is missing", async () => { + const harness = createRebuildFlowHarness({ + sandboxEntry: { fromDockerfile: "/definitely/missing/NemoClaw.Dockerfile" }, + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).rejects.toThrow("Recorded custom Dockerfile is unavailable"); + + expect(harness.backupSandboxStateSpy).not.toHaveBeenCalled(); + expect(harness.onboardSpy).not.toHaveBeenCalled(); + }); + }); +} diff --git a/test/helpers/rebuild-flow-target-image-cases.ts b/test/helpers/rebuild-flow-target-image-cases.ts new file mode 100644 index 00000000000..dd828484871 --- /dev/null +++ b/test/helpers/rebuild-flow-target-image-cases.ts @@ -0,0 +1,167 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; +import { + createRebuildFlowHarness, + installRebuildFlowTestHooks, + originalSandboxName, + snapshotEnv, +} from "./rebuild-flow-test-harness"; + +export function registerRebuildFlowTargetImageTests(): void { + describe("rebuildSandbox flow: target image", () => { + installRebuildFlowTestHooks(); + it("aborts before backup/delete when the durable custom Dockerfile is unreadable", async () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-rebuild-from-")); + const dockerfile = path.join(tempDir, "Dockerfile.unreadable"); + fs.writeFileSync(dockerfile, "FROM scratch\n", { mode: 0o000 }); + const harness = createRebuildFlowHarness({ sandboxEntry: { fromDockerfile: dockerfile } }); + try { + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).rejects.toThrow("Recorded custom Dockerfile is unavailable"); + expect(harness.backupSandboxStateSpy).not.toHaveBeenCalled(); + expect(harness.onboardSpy).not.toHaveBeenCalled(); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } + }); + + it("fails closed on a corrupt durable custom Dockerfile value", async () => { + const harness = createRebuildFlowHarness({ sandboxEntry: { fromDockerfile: 42 } }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).rejects.toThrow("Recorded custom Dockerfile is invalid"); + + expect(harness.backupSandboxStateSpy).not.toHaveBeenCalled(); + expect(harness.onboardSpy).not.toHaveBeenCalled(); + }); + + it("rebuilds a known-remote target even when the session belongs to another sandbox (#5735)", async () => { + const restoreEnv = snapshotEnv(["NVIDIA_INFERENCE_API_KEY"]); + process.env.NVIDIA_INFERENCE_API_KEY = "nvapi-key"; // pass credential preflight + try { + const harness = createRebuildFlowHarness({ + applyPreset: () => true, + sandboxEntry: { provider: "nvidia-prod", model: "nvidia/nemotron" }, + sessionSandboxName: "some-other-sandbox", + }); + const staleEndpoint = "https://stale.example.test/v1"; + harness.session.endpointUrl = staleEndpoint; + harness.session.metadata = { + gatewayName: "nemoclaw", + fromDockerfile: "/tmp/unrelated.Dockerfile", + }; + harness.session.webSearchConfig = { fetchEnabled: true }; + harness.session.policyPresets = ["foreign-preset"]; + harness.session.gpuPassthrough = true; + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).resolves.toBeUndefined(); + + expect(harness.onboardSpy).toHaveBeenCalled(); + const providerPreflightCall = harness.runOpenshellSpy.mock.calls.findIndex( + ([args]) => Array.isArray(args) && args[0] === "provider", + ); + expect(providerPreflightCall).toBeGreaterThanOrEqual(0); + expect(harness.ensureTargetGatewaySpy.mock.invocationCallOrder[0]).toBeLessThan( + harness.runOpenshellSpy.mock.invocationCallOrder[providerPreflightCall], + ); + expect(harness.session.endpointUrl).not.toBe(staleEndpoint); + expect(harness.session.metadata).toMatchObject({ fromDockerfile: null }); + expect(harness.session.webSearchConfig).toBeNull(); + expect(harness.session.policyPresets).toEqual(["npm", "bad", "throw"]); + expect(harness.session.gpuPassthrough).toBe(false); + expect(harness.runOpenshellSpy).toHaveBeenCalledWith( + ["sandbox", "delete", "alpha"], + expect.objectContaining({ ignoreError: true }), + ); + } finally { + restoreEnv(); + } + }); + + it("does not abort a routed (nvidia-router) target with a non-matching session (#5735)", async () => { + const harness = createRebuildFlowHarness({ + applyPreset: () => true, + sandboxEntry: { provider: "nvidia-router", model: "router-model" }, + sessionSandboxName: "some-other-sandbox", + }); + harness.session.routerPid = 4242; + harness.session.routerCredentialHash = "router-credential-hash"; + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).resolves.toBeUndefined(); + + expect(harness.runOpenshellSpy).toHaveBeenCalledWith( + ["sandbox", "delete", "alpha"], + expect.objectContaining({ ignoreError: true }), + ); + expect(harness.onboardSpy).toHaveBeenCalled(); + expect(harness.session.routerPid).toBe(4242); + expect(harness.session.routerCredentialHash).toBe("router-credential-hash"); + }); + + it("marks recreate onboarding failures as terminal and preserves retry cleanup", async () => { + const overrideEnvVar = "NEMOCLAW_HERMES_SANDBOX_BASE_IMAGE_REF"; + const restoreEnv = snapshotEnv([overrideEnvVar]); + process.env[overrideEnvVar] = "nemoclaw-hermes-sandbox-base-local:image-caller"; + try { + const harness = createRebuildFlowHarness({ + baseImagePreflight: { + ok: true, + imageRef: "nemoclaw-hermes-sandbox-base-local:image-preflighted", + overrideEnvVar, + }, + onboard: (session) => { + expect(process.env[overrideEnvVar]).toBe( + "nemoclaw-hermes-sandbox-base-local:image-preflighted", + ); + session.lastStepStarted = "sandbox"; + throw new Error("inner recreate boom"); + }, + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).rejects.toThrow("Recreate failed"); + + expect(process.env[overrideEnvVar]).toBe("nemoclaw-hermes-sandbox-base-local:image-caller"); + expect(harness.releaseOnboardLockSpy).toHaveBeenCalled(); + expect(harness.markStepFailedSpy).toHaveBeenCalledWith( + "sandbox", + "Rebuild recreate failed", + expect.objectContaining({ updateMachine: true }), + ); + expect(harness.session).toMatchObject({ + status: "failed", + failure: { step: "sandbox", message: "Rebuild recreate failed" }, + machine: { state: "failed" }, + steps: { sandbox: { status: "failed", error: "Rebuild recreate failed" } }, + }); + expect(harness.relockSpy).toHaveBeenCalledWith( + "alpha", + expect.any(Object), + false, + "nemoclaw", + ); + expect(process.env.NEMOCLAW_SANDBOX_NAME).toBe(originalSandboxName); + + const errors = harness.errorSpy.mock.calls.map((call) => String(call[0])).join("\n"); + expect(errors).toContain("Recreate failed after sandbox was destroyed"); + expect(errors).toContain("Backup is preserved at: /tmp/nemoclaw-rebuild-backup"); + expect(errors).toContain("onboard --resume"); + } finally { + restoreEnv(); + } + }); + }); +} diff --git a/test/helpers/rebuild-flow-target-session-cases.ts b/test/helpers/rebuild-flow-target-session-cases.ts new file mode 100644 index 00000000000..e873b9c2d0a --- /dev/null +++ b/test/helpers/rebuild-flow-target-session-cases.ts @@ -0,0 +1,192 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; +import { + createRebuildFlowHarness, + installRebuildFlowTestHooks, + snapshotEnv, +} from "./rebuild-flow-test-harness"; + +export function registerRebuildFlowTargetSessionTests(): void { + describe("rebuildSandbox flow: target session", () => { + installRebuildFlowTestHooks(); + it("isolates ambient onboard-selection env during recreate, then restores it (#5735)", async () => { + const restoreEnv = snapshotEnv([ + "NEMOCLAW_AGENT", + "NEMOCLAW_PROVIDER_KEY", + "NVIDIA_INFERENCE_API_KEY", + ]); + process.env.NEMOCLAW_AGENT = "langchain-deepagents-code"; + process.env.NEMOCLAW_PROVIDER_KEY = "sk-bogus-installer-key"; + process.env.NVIDIA_INFERENCE_API_KEY = "hosted-source-key"; + + let envSeenInsideOnboard: { + agent: string | undefined; + providerKey: string | undefined; + hostedSourceKey: string | undefined; + } | null = null; + + try { + const harness = createRebuildFlowHarness({ + applyPreset: () => true, + onboard: () => { + envSeenInsideOnboard = { + agent: process.env.NEMOCLAW_AGENT, + providerKey: process.env.NEMOCLAW_PROVIDER_KEY, + hostedSourceKey: process.env.NVIDIA_INFERENCE_API_KEY, + }; + }, + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).resolves.toBeUndefined(); + + expect(envSeenInsideOnboard).toEqual({ + agent: undefined, + providerKey: undefined, + hostedSourceKey: "hosted-source-key", + }); + const logged = harness.logSpy.mock.calls.map((call) => String(call[0])).join("\n"); + expect(logged).toContain("Ignoring ambient NEMOCLAW_AGENT='langchain-deepagents-code'"); + expect(process.env.NEMOCLAW_AGENT).toBe("langchain-deepagents-code"); + expect(process.env.NEMOCLAW_PROVIDER_KEY).toBe("sk-bogus-installer-key"); + expect(process.env.NVIDIA_INFERENCE_API_KEY).toBe("hosted-source-key"); + } finally { + restoreEnv(); + } + }); + + it("uses the exact preflighted agent base image only for the recreate", async () => { + const overrideEnvVar = "NEMOCLAW_HERMES_SANDBOX_BASE_IMAGE_REF"; + const restoreEnv = snapshotEnv([overrideEnvVar]); + delete process.env[overrideEnvVar]; + let refSeenInsideOnboard: string | undefined; + + try { + const harness = createRebuildFlowHarness({ + sandboxEntry: { agent: "hermes" }, + baseImagePreflight: { + ok: true, + imageRef: "nemoclaw-hermes-sandbox-base-local:12345678", + overrideEnvVar, + }, + onboard: () => { + refSeenInsideOnboard = process.env[overrideEnvVar]; + }, + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).resolves.toBeUndefined(); + + expect(refSeenInsideOnboard).toBe("nemoclaw-hermes-sandbox-base-local:12345678"); + expect(process.env[overrideEnvVar]).toBeUndefined(); + } finally { + restoreEnv(); + } + }); + + it("restores caller messaging config and plan env after rebuild", async () => { + const keys = ["NEMOCLAW_MESSAGING_PLAN_B64", "TELEGRAM_REQUIRE_MENTION"]; + const restoreEnv = snapshotEnv(keys); + process.env.NEMOCLAW_MESSAGING_PLAN_B64 = "caller-plan"; + delete process.env.TELEGRAM_REQUIRE_MENTION; + try { + const harness = createRebuildFlowHarness({ + applyPreset: () => true, + onboard: () => { + process.env.NEMOCLAW_MESSAGING_PLAN_B64 = "target-plan"; + process.env.TELEGRAM_REQUIRE_MENTION = "1"; + }, + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).resolves.toBeUndefined(); + + expect(process.env.NEMOCLAW_MESSAGING_PLAN_B64).toBe("caller-plan"); + expect(process.env.TELEGRAM_REQUIRE_MENTION).toBeUndefined(); + } finally { + restoreEnv(); + } + }); + + it("recreates a matching-session custom-endpoint sandbox from a validated session endpoint while ignoring hostile ambient values for PRA-4 (#5735)", async () => { + const restoreEnv = snapshotEnv([ + "NEMOCLAW_ENDPOINT_URL", + "NEMOCLAW_PROVIDER", + "NEMOCLAW_MODEL", + "COMPATIBLE_API_KEY", + ]); + process.env.NEMOCLAW_ENDPOINT_URL = "https://attacker.example.test/v1"; + process.env.NEMOCLAW_PROVIDER = "build"; + process.env.NEMOCLAW_MODEL = "attacker-model"; + process.env.COMPATIBLE_API_KEY = "compat-key"; // pass credential preflight + + let envSeenInsideOnboard: Record | null = null; + try { + const harness = createRebuildFlowHarness({ + applyPreset: () => true, + sandboxEntry: { provider: "compatible-endpoint", model: "session-model" }, + onboard: () => { + envSeenInsideOnboard = { + endpoint: process.env.NEMOCLAW_ENDPOINT_URL, + provider: process.env.NEMOCLAW_PROVIDER, + model: process.env.NEMOCLAW_MODEL, + }; + }, + }); + harness.session.provider = "compatible-endpoint"; + harness.session.model = "session-model"; + harness.session.endpointUrl = "https://my-custom-endpoint.example/v1?x=1#frag"; + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).resolves.toBeUndefined(); + + expect(envSeenInsideOnboard).toEqual({ + endpoint: undefined, + provider: undefined, + model: undefined, + }); + expect(harness.session.endpointUrl).toBe("https://my-custom-endpoint.example/v1"); + expect(harness.session.provider).toBe("compatible-endpoint"); + expect(harness.session.model).toBe("session-model"); + expect(process.env.NEMOCLAW_ENDPOINT_URL).toBe("https://attacker.example.test/v1"); + expect(process.env.NEMOCLAW_PROVIDER).toBe("build"); + expect(process.env.NEMOCLAW_MODEL).toBe("attacker-model"); + } finally { + restoreEnv(); + } + }); + + it("aborts before backup/delete when a custom-endpoint target has no matching session (#5735)", async () => { + const restoreEnv = snapshotEnv(["COMPATIBLE_API_KEY"]); + process.env.COMPATIBLE_API_KEY = "compat-key"; // pass credential preflight first + try { + const harness = createRebuildFlowHarness({ + sandboxEntry: { provider: "compatible-endpoint", model: "custom-model" }, + sessionSandboxName: "some-other-sandbox", + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).rejects.toThrow("Cannot determine recreate endpoint"); + + const errors = harness.errorSpy.mock.calls.map((call) => String(call[0])).join("\n"); + expect(errors).toContain("cannot determine the inference endpoint"); + expect(errors).toContain("Sandbox is untouched"); + expect(harness.backupSandboxStateSpy).not.toHaveBeenCalled(); + expect(harness.runOpenshellSpy).not.toHaveBeenCalledWith( + ["sandbox", "delete", "alpha"], + expect.anything(), + ); + expect(harness.onboardSpy).not.toHaveBeenCalled(); + } finally { + restoreEnv(); + } + }); + }); +} diff --git a/test/helpers/rebuild-flow-test-harness.ts b/test/helpers/rebuild-flow-test-harness.ts new file mode 100644 index 00000000000..002b09aac8f --- /dev/null +++ b/test/helpers/rebuild-flow-test-harness.ts @@ -0,0 +1,327 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createRequire } from "node:module"; +import { afterEach, beforeEach, vi } from "vitest"; +import { makePreparedRecoveryManifest } from "../../src/lib/actions/sandbox/rebuild-flow-test-fixtures"; +import { + createRebuildFlowSession, + installTerminalStepFailureMock, + originalSandboxName, + type RebuildFlowHarness, + type RebuildFlowOverrides, +} from "./rebuild-flow-test-support"; + +export { originalSandboxName, snapshotEnv } from "./rebuild-flow-test-support"; + +const requireDist = createRequire( + new URL("../../src/lib/actions/sandbox/rebuild-flow.test.ts", import.meta.url), +); +const rebuildModulePath = "./rebuild.js"; +requireDist(rebuildModulePath); +delete require.cache[requireDist.resolve(rebuildModulePath)]; + +export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): RebuildFlowHarness { + delete require.cache[requireDist.resolve(rebuildModulePath)]; + + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => undefined); + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => undefined); + + const gatewayDrift = requireDist("../../adapters/openshell/gateway-drift.js"); + const openshellRuntime = requireDist("../../adapters/openshell/runtime.js"); + const sandboxList = requireDist("../../openshell-sandbox-list.js"); + const resolve = requireDist("../../adapters/openshell/resolve.js"); + const agentDefs = requireDist("../../agent/defs.js"); + const agentRuntime = requireDist("../../agent/runtime.js"); + const onboardMod = requireDist("../../onboard.js"); + const hermesProviderAuth = requireDist("../../hermes-provider-auth.js"); + const onboardSession = requireDist("../../state/onboard-session.js"); + const registry = requireDist("../../state/registry.js"); + const sandboxState = requireDist("../../state/sandbox.js"); + const sandboxSession = requireDist("../../state/sandbox-session.js"); + const sandboxVersion = requireDist("../../sandbox/version.js"); + const destroy = requireDist("./destroy.js"); + const gatewayState = requireDist("./gateway-state.js"); + const rebuildFlowHelpers = requireDist("./rebuild-flow-helpers.js"); + const rebuildCustomImagePreflight = requireDist("./rebuild-custom-image-preflight.js"); + const rebuildUsageNotice = requireDist("./rebuild-usage-notice.js"); + const rebuildShields = requireDist("./rebuild-shields.js"); + const nim = requireDist("../../inference/nim.js"); + const policies = requireDist("../../policy/index.js"); + const processRecovery = requireDist("./process-recovery.js"); + const messagingHostForwardLifecycle = requireDist("./messaging-host-forward-lifecycle.js"); + const mcpBridge = requireDist("./mcp-bridge.js"); + const messaging = requireDist("../../messaging/index.js"); + const shields = requireDist("../../shields/index.js"); + + const session = createRebuildFlowSession(onboardSession.MACHINE_SNAPSHOT_VERSION); + const rebuildShieldsWindow = { relocked: false, wasLocked: false }; + const agentDef = { + name: + typeof overrides.sandboxEntry?.agent === "string" ? overrides.sandboxEntry.agent : "openclaw", + expectedVersion: "0.2.0", + }; + + vi.spyOn(gatewayDrift, "detectOpenShellStateRpcPreflightIssue").mockReturnValue(null); + vi.spyOn(gatewayDrift, "detectOpenShellStateRpcResultIssue").mockReturnValue(null); + vi.spyOn(sandboxList, "captureSandboxListWithGatewayRecovery").mockResolvedValue({ + result: { + status: 0, + output: overrides.sandboxListOutput ?? (overrides.staleRecovery ? "" : "alpha Ready"), + }, + }); + vi.spyOn(gatewayState, "getReconciledSandboxGatewayState").mockResolvedValue({ + state: overrides.staleRecovery ? "missing" : "present", + output: "", + }); + vi.spyOn(rebuildFlowHelpers, "ensureRebuildAgentBaseImage").mockReturnValue( + overrides.baseImagePreflight ?? { ok: true, imageRef: null, overrideEnvVar: null }, + ); + const ensureTargetGatewaySpy = vi + .spyOn(rebuildFlowHelpers, "ensureRebuildTargetGatewaySelected") + .mockResolvedValue(true); + vi.spyOn(rebuildCustomImagePreflight, "preflightRebuildImage").mockResolvedValue( + overrides.customImagePreflight ?? { ok: true, imageTag: null }, + ); + vi.spyOn(rebuildUsageNotice, "ensureRebuildUsageNoticeAccepted").mockResolvedValue(true); + const warnUnpreservedUserManagedFilesSpy = vi + .spyOn(rebuildFlowHelpers, "warnUnpreservedUserManagedFiles") + .mockImplementation(() => undefined); + vi.spyOn(resolve, "resolveOpenshell").mockReturnValue(null); + vi.spyOn(agentDefs, "loadAgent").mockReturnValue(agentDef); + vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue({ name: "openclaw" }); + vi.spyOn(agentRuntime, "getAgentDisplayName").mockReturnValue("OpenClaw"); + vi.spyOn(hermesProviderAuth, "inspectHermesProviderBinding").mockReturnValue({ + exists: overrides.hermesProviderExists ?? true, + credentialKeys: + (overrides.hermesProviderExists ?? true) + ? (overrides.hermesCredentialKeys ?? ["OPENAI_API_KEY"]) + : null, + }); + vi.spyOn(onboardSession, "loadSession").mockReturnValue(session); + vi.spyOn(onboardSession, "updateSession").mockImplementation((mutator: unknown) => { + if (typeof mutator !== "function") { + throw new TypeError("updateSession expected a mutator function"); + } + (mutator as (value: typeof session) => typeof session | void)(session); + return session; + }); + const releaseOnboardLockSpy = vi + .spyOn(onboardSession, "releaseOnboardLock") + .mockImplementation(() => undefined); + vi.spyOn(onboardSession, "acquireOnboardLock").mockReturnValue({ acquired: true }); + const markStepFailedSpy = installTerminalStepFailureMock(onboardSession, session); + session.sandboxName = overrides.sessionSandboxName ?? session.sandboxName; + const sandboxEntry = { + name: "alpha", + provider: "ollama-local", + model: "nvidia/nemotron", + policies: ["npm"], + agent: null, + agentVersion: "0.1.0", + nimContainer: null, + nemoclawVersion: "0.1.0", + dashboardPort: 18789, + gatewayName: "nemoclaw", + gatewayPort: 8080, + ...(overrides.sandboxEntry ?? {}), + }; + vi.spyOn(registry, "getSandbox").mockReturnValue(sandboxEntry); + vi.spyOn(registry, "getDefault").mockReturnValue(overrides.defaultSandbox ?? null); + let registryLoadCount = 0; + vi.spyOn(registry, "load").mockImplementation(() => { + const isPreDeleteRead = registryLoadCount > 0; + registryLoadCount++; + const defaultSandbox = isPreDeleteRead + ? overrides.preDeleteDefaultSandbox !== undefined + ? overrides.preDeleteDefaultSandbox + : (overrides.defaultSandbox ?? null) + : (overrides.defaultSandbox ?? null); + return { + sandboxes: { + alpha: + isPreDeleteRead && overrides.preDeleteSandboxEntry + ? overrides.preDeleteSandboxEntry + : sandboxEntry, + }, + defaultSandbox, + }; + }); + vi.spyOn(registry, "listSandboxes").mockReturnValue({ sandboxes: [] }); + const registryUpdateSpy = vi.spyOn(registry, "updateSandbox").mockReturnValue(true); + const restoreSandboxEntrySpy = vi + .spyOn(registry, "restoreSandboxEntry") + .mockImplementation(() => undefined); + vi.spyOn(sandboxSession, "getActiveSandboxSessions").mockReturnValue({ + detected: false, + sessions: [], + }); + vi.spyOn(sandboxVersion, "checkAgentVersion").mockReturnValue({ + expectedVersion: "0.2.0", + sandboxVersion: "0.1.0", + }); + vi.spyOn(rebuildShields, "openRebuildShieldsWindow").mockReturnValue(rebuildShieldsWindow); + const relockSpy = vi + .spyOn(rebuildShields, "relockRebuildShieldsWindow") + .mockImplementation((...args: unknown[]) => { + const window = args[1] as typeof rebuildShieldsWindow; + window.relocked = true; + return true; + }); + const backupSandboxStateSpy = vi.spyOn(sandboxState, "backupSandboxState").mockReturnValue({ + success: true, + backedUpDirs: ["workspace"], + backedUpFiles: ["user.md"], + failedDirs: [], + failedFiles: [], + manifest: { + backupPath: "/tmp/nemoclaw-rebuild-backup", + timestamp: "2026-06-01T00:00:00.000Z", + policyPresets: overrides.backupPolicyPresets ?? ["npm", "bad", "throw"], + }, + }); + vi.spyOn(sandboxState, "validateRebuildRecoveryManifest").mockImplementation( + (...args: unknown[]) => { + const manifest = args[2] as Record; + return overrides.recoveryManifestValidation?.(manifest) ?? { ok: true, manifest }; + }, + ); + vi.spyOn(sandboxState, "getLatestBackup").mockImplementation( + () => + (overrides.preDeleteLatestManifest === undefined + ? makePreparedRecoveryManifest() + : overrides.preDeleteLatestManifest) as ReturnType, + ); + vi.spyOn(sandboxState, "hasPositiveManagedImageEvidence").mockReturnValue( + overrides.managedImageEvidence ?? true, + ); + const restoreSandboxStateSpy = vi.spyOn(sandboxState, "restoreSandboxState").mockImplementation( + overrides.restoreSandboxState ?? + (() => ({ + success: true, + restoredDirs: ["workspace"], + restoredFiles: ["user.md"], + failedDirs: [], + failedFiles: [], + })), + ); + const runOpenshellSpy = vi + .spyOn(openshellRuntime, "runOpenshell") + .mockImplementation((args: unknown) => { + const argv = Array.isArray(args) ? args.map(String) : []; + return overrides.runOpenshell ? overrides.runOpenshell(argv) : { status: 0, output: "" }; + }); + const removeSandboxRegistryEntrySpy = vi + .spyOn(destroy, "removeSandboxRegistryEntry") + .mockImplementation(overrides.removeSandboxRegistryEntry ?? (() => undefined)); + vi.spyOn(nim, "stopNimContainer").mockImplementation(() => undefined); + vi.spyOn(nim, "stopNimContainerByName").mockImplementation(() => undefined); + const onboardSpy = vi.spyOn(onboardMod, "onboard").mockImplementation(async () => { + await overrides.onboard?.(session); + }); + vi.spyOn(onboardMod, "preflightAuthoritativeRebuildTarget").mockResolvedValue(undefined); + const ensureValidatedBraveSearchCredentialSpy = vi + .spyOn(onboardMod, "ensureValidatedWebSearchCredential") + .mockImplementation( + overrides.ensureValidatedWebSearchCredential ?? + overrides.ensureValidatedBraveSearchCredential ?? + (async () => "web-search-key"), + ); + const applyPresetSpy = vi + .spyOn(policies, "applyPreset") + .mockImplementation((_sandboxName: unknown, presetName: unknown) => { + const normalizedPresetName = String(presetName); + if (overrides.applyPreset) return overrides.applyPreset(normalizedPresetName); + if (normalizedPresetName === "throw") throw new Error("preset boom"); + return normalizedPresetName === "npm"; + }); + const executeSandboxCommandSpy = vi + .spyOn(processRecovery, "executeSandboxCommand") + .mockImplementation( + overrides.executeSandboxCommand ?? (() => ({ status: 0, stdout: "doctor ok", stderr: "" })), + ); + vi.spyOn(shields, "repairMutableConfigPerms").mockImplementation( + overrides.repairMutableConfigPerms ?? (() => ({ applied: true, verified: true, errors: [] })), + ); + vi.spyOn(shields, "isShieldsDown").mockReturnValue(true); + vi.spyOn(shields, "clearShieldsState").mockImplementation( + overrides.clearShieldsState ?? (() => undefined), + ); + const messagingRebuildPlanSpy = vi + .spyOn(messaging.MessagingWorkflowPlanner.prototype, "buildRebuildPlanFromSandboxEntry") + .mockImplementation(overrides.buildMessagingRebuildPlan ?? (() => null)); + const ensureMessagingHostForwardAfterRebuildSpy = vi + .spyOn(messagingHostForwardLifecycle, "ensureMessagingHostForwardAfterRebuild") + .mockReturnValue(true); + const prepareMcpBridgesForRebuildSpy = vi + .spyOn(mcpBridge, "prepareMcpBridgesForRebuild") + .mockResolvedValue( + overrides.mcpPreparation ?? { + entries: [], + detachedProviderEntries: [], + }, + ); + const prepareMcpBridgesForAbsentSandboxRebuildSpy = vi + .spyOn(mcpBridge, "prepareMcpBridgesForAbsentSandboxRebuild") + .mockResolvedValue( + overrides.mcpPreparation ?? { + entries: [], + detachedProviderEntries: [], + scrubbedAdapterEntries: [], + }, + ); + const reattachMcpProvidersAfterRebuildAbortSpy = vi + .spyOn(mcpBridge, "reattachMcpProvidersAfterRebuildAbort") + .mockResolvedValue(undefined); + const restoreMcpBridgesAfterRebuildSpy = vi + .spyOn(mcpBridge, "restoreMcpBridgesAfterRebuild") + .mockImplementation(overrides.restoreMcpBridgesAfterRebuild ?? (() => Promise.resolve())); + + errorSpy.mockClear(); + logSpy.mockClear(); + warnSpy.mockClear(); + + return { + rebuildSandbox: requireDist(rebuildModulePath).rebuildSandbox, + applyPresetSpy, + backupSandboxStateSpy, + errorSpy, + executeSandboxCommandSpy, + ensureMessagingHostForwardAfterRebuildSpy, + ensureTargetGatewaySpy, + ensureValidatedBraveSearchCredentialSpy, + logSpy, + markStepFailedSpy, + onboardSpy, + registryUpdateSpy, + releaseOnboardLockSpy, + relockSpy, + restoreSandboxStateSpy, + runOpenshellSpy, + messagingRebuildPlanSpy, + prepareMcpBridgesForAbsentSandboxRebuildSpy, + prepareMcpBridgesForRebuildSpy, + reattachMcpProvidersAfterRebuildAbortSpy, + removeSandboxRegistryEntrySpy, + restoreSandboxEntrySpy, + restoreMcpBridgesAfterRebuildSpy, + warnUnpreservedUserManagedFilesSpy, + session, + }; +} + +export function installRebuildFlowTestHooks(): void { + beforeEach(() => { + delete process.env.NEMOCLAW_SANDBOX_NAME; + }); + afterEach(() => { + vi.restoreAllMocks(); + delete require.cache[requireDist.resolve(rebuildModulePath)]; + if (originalSandboxName === undefined) { + delete process.env.NEMOCLAW_SANDBOX_NAME; + } else { + process.env.NEMOCLAW_SANDBOX_NAME = originalSandboxName; + } + }); +} diff --git a/test/helpers/rebuild-flow-test-support.ts b/test/helpers/rebuild-flow-test-support.ts new file mode 100644 index 00000000000..638820299a2 --- /dev/null +++ b/test/helpers/rebuild-flow-test-support.ts @@ -0,0 +1,177 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { type MockInstance, vi } from "vitest"; + +export type RebuildSandbox = + typeof import("../../src/lib/actions/sandbox/rebuild")["rebuildSandbox"]; +export type RebuildFlowStep = { + status: string; + startedAt: string | null; + completedAt: string | null; + error: string | null; +}; +export type RebuildFlowSession = Record & { + lastStepStarted: string | null; + status: string; + failure: { step: string; message: string | null; recordedAt: string } | null; + machine: { + version: number; + state: string; + stateEnteredAt: string; + revision: number; + }; + steps: Record; +}; +export type RebuildFlowOverrides = { + applyPreset?: (presetName: string) => boolean; + baseImagePreflight?: { + ok: boolean; + imageRef: string | null; + overrideEnvVar: string | null; + }; + executeSandboxCommand?: () => { status: number; stdout: string; stderr: string } | null; + onboard?: (session: RebuildFlowSession) => Promise | void; + repairMutableConfigPerms?: () => + | { applied: false; skipReason: "agent" | "locked" | "unreadable"; reason: string } + | { applied: true; verified: boolean; errors: string[] }; + restoreSandboxState?: () => { + success: boolean; + restoredDirs: string[]; + restoredFiles: string[]; + failedDirs: string[]; + failedFiles: string[]; + }; + restoreMcpBridgesAfterRebuild?: () => Promise; + buildMessagingRebuildPlan?: () => Promise | unknown; + sandboxEntry?: Record; + sessionSandboxName?: string; + sandboxListOutput?: string; + defaultSandbox?: string | null; + preDeleteSandboxEntry?: Record; + preDeleteDefaultSandbox?: string | null; + preDeleteLatestManifest?: Record | null; + recoveryManifestValidation?: ( + manifest: Record, + ) => { ok: true; manifest: Record } | { ok: false; reason: string }; + managedImageEvidence?: boolean; + staleRecovery?: boolean; + mcpPreparation?: { + entries: Array>; + detachedProviderEntries: Array>; + scrubbedAdapterEntries?: Array>; + }; + runOpenshell?: (args: string[]) => { + status: number; + output: string; + stdout?: string; + stderr?: string; + }; + backupPolicyPresets?: string[]; + ensureValidatedBraveSearchCredential?: () => Promise; + ensureValidatedWebSearchCredential?: () => Promise; + hermesCredentialKeys?: string[] | null; + hermesProviderExists?: boolean; + customImagePreflight?: { ok: true; imageTag: string | null } | { ok: false; detail: string }; + removeSandboxRegistryEntry?: () => void; + clearShieldsState?: () => void; +}; +export type RebuildFlowHarness = { + rebuildSandbox: RebuildSandbox; + applyPresetSpy: MockInstance; + backupSandboxStateSpy: MockInstance; + errorSpy: MockInstance; + executeSandboxCommandSpy: MockInstance; + ensureMessagingHostForwardAfterRebuildSpy: MockInstance; + ensureTargetGatewaySpy: MockInstance; + ensureValidatedBraveSearchCredentialSpy: MockInstance; + logSpy: MockInstance; + markStepFailedSpy: MockInstance; + onboardSpy: MockInstance; + registryUpdateSpy: MockInstance; + releaseOnboardLockSpy: MockInstance; + relockSpy: MockInstance; + restoreSandboxStateSpy: MockInstance; + runOpenshellSpy: MockInstance; + messagingRebuildPlanSpy: MockInstance; + prepareMcpBridgesForAbsentSandboxRebuildSpy: MockInstance; + prepareMcpBridgesForRebuildSpy: MockInstance; + reattachMcpProvidersAfterRebuildAbortSpy: MockInstance; + removeSandboxRegistryEntrySpy: MockInstance; + restoreSandboxEntrySpy: MockInstance; + restoreMcpBridgesAfterRebuildSpy: MockInstance; + warnUnpreservedUserManagedFilesSpy: MockInstance; + session: RebuildFlowSession; +}; +export const originalSandboxName = process.env.NEMOCLAW_SANDBOX_NAME; +export function snapshotEnv(names: readonly string[]): () => void { + const saved = names.map((name) => [name, process.env[name]] as const); + return () => { + for (const [name] of saved) { + delete process.env[name]; + } + Object.assign( + process.env, + Object.fromEntries( + saved.filter((entry): entry is [string, string] => entry[1] !== undefined), + ), + ); + }; +} +function createStep(status: string): RebuildFlowStep { + return { status, startedAt: null, completedAt: null, error: null }; +} +export function createRebuildFlowSession(machineSnapshotVersion: number): RebuildFlowSession { + return { + sandboxName: "alpha", + provider: "ollama-local", + model: "nvidia/nemotron", + credentialEnv: null, + metadata: {}, + hermesToolGateways: [], + lastStepStarted: null, + status: "in_progress", + failure: null, + machine: { + version: machineSnapshotVersion, + state: "gateway", + stateEnteredAt: "2026-06-01T00:00:00.000Z", + revision: 2, + }, + steps: { + preflight: createStep("complete"), + gateway: createStep("complete"), + provider_selection: createStep("pending"), + inference: createStep("pending"), + sandbox: createStep("pending"), + openclaw: createStep("pending"), + agent_setup: createStep("pending"), + policies: createStep("pending"), + }, + }; +} +export function installTerminalStepFailureMock( + onboardSession: { markStepFailed: (...args: unknown[]) => unknown }, + session: RebuildFlowSession, +): MockInstance { + return vi + .spyOn(onboardSession, "markStepFailed") + .mockImplementation((stepName: unknown, message: unknown, options: unknown) => { + const stepKey = String(stepName); + const step = session.steps[stepKey] ?? createStep("pending"); + session.steps[stepKey] = step; + step.status = "failed"; + step.error = typeof message === "string" ? message : null; + session.status = "failed"; + session.failure = { + step: stepKey, + message: typeof message === "string" ? message : null, + recordedAt: "2026-06-01T00:02:00.000Z", + }; + const updateMachine = + (options as { updateMachine?: boolean } | undefined)?.updateMachine === true; + session.machine.state = updateMachine ? "failed" : session.machine.state; + session.machine.revision += updateMachine ? 1 : 0; + return session; + }); +} diff --git a/test/hermes-doctor-config-hash.test.ts b/test/hermes-doctor-config-hash.test.ts index 75a66be7cd7..f766fb032b6 100644 --- a/test/hermes-doctor-config-hash.test.ts +++ b/test/hermes-doctor-config-hash.test.ts @@ -18,6 +18,11 @@ describe("Hermes doctor and config hash boundary", () => { const binDir = path.join(tmp, "usr-local-bin"); const libDir = path.join(tmp, "usr-local-lib-nemoclaw"); const preloadsDir = path.join(libDir, "preloads"); + const mcpConfigTransactionPath = path.join(libDir, "hermes-mcp-config-transaction.py"); + const mcpCredentialBoundaryPath = path.join( + libDir, + "openshell-child-visible-credentials.v0.0.72.json", + ); const nestedDir = path.join(preloadsDir, "nested"); const profileDir = path.join(tmp, "etc-profile.d"); const bashrcPath = path.join(tmp, "bash.bashrc"); @@ -36,6 +41,8 @@ describe("Hermes doctor and config hash boundary", () => { path.join(libDir, "validate-hermes-env-secret-boundary.py"), path.join(libDir, "seed-hermes-dashboard-config.py"), path.join(libDir, "hermes-runtime-config-guard.py"), + mcpConfigTransactionPath, + mcpCredentialBoundaryPath, path.join(libDir, "state-dir-guard.py"), path.join(libDir, "managed-gateway-control.py"), path.join(libDir, "sandbox-rlimits.sh"), @@ -69,12 +76,14 @@ describe("Hermes doctor and config hash boundary", () => { expect(result.stderr).toBe(""); expect(fs.readFileSync(chownLogPath, "utf-8")).toBe( [ - `root:root ${path.join(binDir, "nemoclaw-gateway-control")} ${path.join(libDir, "gateway-supervisor.sh")} ${path.join(libDir, "state-dir-guard.py")} ${path.join(libDir, "managed-gateway-control.py")}`, + `root:root ${path.join(binDir, "nemoclaw-gateway-control")} ${path.join(libDir, "gateway-supervisor.sh")} ${path.join(libDir, "state-dir-guard.py")} ${path.join(libDir, "managed-gateway-control.py")} ${mcpCredentialBoundaryPath}`, `-R 0:0 ${preloadsDir}`, "", ].join("\n"), ); expect(mode(path.join(binDir, "nemoclaw-gateway-control"))).toBe("700"); + expect(mode(mcpConfigTransactionPath)).toBe("755"); + expect(mode(mcpCredentialBoundaryPath)).toBe("444"); expect(mode(path.join(libDir, "gateway-supervisor.sh"))).toBe("444"); expect(mode(path.join(libDir, "state-dir-guard.py"))).toBe("500"); expect(mode(path.join(libDir, "managed-gateway-control.py"))).toBe("500"); diff --git a/test/hermes-gateway-supervisor-recovery.test.ts b/test/hermes-gateway-supervisor-recovery.test.ts index 28d7e9f55b7..7a6ef0f3d1e 100644 --- a/test/hermes-gateway-supervisor-recovery.test.ts +++ b/test/hermes-gateway-supervisor-recovery.test.ts @@ -527,6 +527,25 @@ echo "state=1 lock=1 owner_active=1 token_match=0 original_locked=0 recovery_saf }); describe("Hermes supervised auxiliary recovery", () => { + it("rejects public health from a relay that loses its tracked identity during the probe", () => { + const source = fs.readFileSync(START_SCRIPT, "utf-8"); + const result = runBashHarness([ + 'trace() { printf "%s\\n" "$*"; }', + "CHECKS=0", + 'hermes_socat_bridge_healthy() { CHECKS=$((CHECKS + 1)); trace "identity-check:$CHECKS"; [ "$CHECKS" -eq 1 ]; }', + 'curl() { printf "200"; }', + extractShellFunction(source, "hermes_api_socat_bridge_healthy"), + "if hermes_api_socat_bridge_healthy 101 8642; then trace unsafe-success; else trace refused; fi", + ]); + + expect(result.status, result.stderr).toBe(0); + expect(result.stdout.trim().split("\n")).toEqual([ + "identity-check:1", + "identity-check:2", + "refused", + ]); + }); + it("re-prepares runtime inputs and retries a refused non-root gateway respawn", () => { const source = fs.readFileSync(START_SCRIPT, "utf-8"); const result = runBashHarness([ @@ -886,6 +905,7 @@ describe("Hermes supervised auxiliary recovery", () => { 'hermes_role_identity_value() { printf "777"; }', "hermes_tracked_role_is_current() { return 0; }", 'gateway_control_stop_tracked_pid() { trace "stop:$1:$2"; }', + 'kill() { [ "$1" = "-0" ] && return 1; trace "unexpected-signal:$*"; }', 'hermes_set_role_identity() { trace "clear:$1:$2"; }', extractShellFunction(source, "hermes_stop_tracked_role"), "hermes_stop_tracked_role gateway 4242 gateway 18642", @@ -1082,11 +1102,13 @@ describe("Hermes supervised auxiliary recovery", () => { 'hermes_tracked_role_is_current() { gateway_control_pid_is_live "$2"; }', 'gateway_control_pid_owns_tcp_listener() { trace "listener:$1:$2"; [ "$1:$2" = "101:8642" ] || [ "$1:$2" = "303:18789" ]; }', 'hermes_tracked_service_owns_listener() { trace "service-listener:$1:$2:$3"; return 1; }', + 'curl() { printf "200"; }', 'hermes_stop_tracked_role() { trace "stop:$2"; return 0; }', "start_hermes_dashboard_sandbox_user() { trace start-dashboard; DASHBOARD_PID=404; DASHBOARD_SOCAT_PID=505; }", 'start_socat_forwarder() { trace "start-forward:$*"; return 0; }', "ensure_gateway_log_stream() { trace gateway-log; }", extractShellFunction(source, "hermes_socat_bridge_healthy"), + extractShellFunction(source, "hermes_api_socat_bridge_healthy"), extractShellFunction(source, "hermes_dashboard_healthy"), extractShellFunction(source, "ensure_hermes_supervised_auxiliaries"), "PUBLIC_PORT=8642", @@ -1104,6 +1126,8 @@ describe("Hermes supervised auxiliary recovery", () => { expect(result.status, result.stderr).toBe(0); expect(result.stdout.trim().split("\n")).toEqual([ + "live:101", + "listener:101:8642", "live:101", "listener:101:8642", "live:202", @@ -1133,6 +1157,7 @@ describe("Hermes supervised auxiliary recovery", () => { "start_hermes_dashboard_sandbox_user() { trace unexpected-dashboard-start; return 1; }", "ensure_gateway_log_stream() { trace gateway-log; }", extractShellFunction(source, "hermes_socat_bridge_healthy"), + extractShellFunction(source, "hermes_api_socat_bridge_healthy"), extractShellFunction(source, "hermes_dashboard_healthy"), extractShellFunction(source, "ensure_hermes_supervised_auxiliaries"), "PUBLIC_PORT=8642", @@ -1153,6 +1178,10 @@ describe("Hermes supervised auxiliary recovery", () => { "listener:101:8642", "stop:101", "start-forward:8642 18642 API SOCAT_PID 4242 gateway", + "live:111", + "listener:111:8642", + "live:111", + "listener:111:8642", "live:202", "live:303", "listener:303:18789", @@ -1162,6 +1191,95 @@ describe("Hermes supervised auxiliary recovery", () => { ]); }); + it("replaces a listener-owning API bridge that fails public HTTP health", () => { + const source = fs.readFileSync(START_SCRIPT, "utf-8"); + const result = runBashHarness([ + 'trace() { printf "%s\\n" "$*"; }', + 'id() { [ "${1:-}" = "-u" ] && printf "1000\\n"; }', + 'gateway_control_pid_is_live() { trace "live:$1"; return 0; }', + 'hermes_tracked_role_is_current() { gateway_control_pid_is_live "$2"; }', + 'gateway_control_pid_owns_tcp_listener() { trace "listener:$1:$2"; return 0; }', + 'curl() { if [ "$PUBLIC_HEALTH" = "stale" ]; then printf "503"; else printf "200"; fi; }', + 'hermes_stop_tracked_role() { trace "stop:$2"; return 0; }', + 'start_socat_forwarder() { trace "start-forward:$*"; printf -v "$4" 111; PUBLIC_HEALTH=ready; return 0; }', + "hermes_dashboard_healthy() { trace dashboard-healthy; return 0; }", + "ensure_gateway_log_stream() { trace gateway-log; }", + extractShellFunction(source, "hermes_socat_bridge_healthy"), + extractShellFunction(source, "hermes_api_socat_bridge_healthy"), + extractShellFunction(source, "ensure_hermes_supervised_auxiliaries"), + "PUBLIC_PORT=8642", + "INTERNAL_PORT=18642", + "DASHBOARD_PUBLIC_PORT=18789", + "DASHBOARD_INTERNAL_PORT=19119", + "PUBLIC_HEALTH=stale", + "SOCAT_PID=101", + "DASHBOARD_PID=202", + "DASHBOARD_SOCAT_PID=303", + "GATEWAY_PID=4242", + 'if ensure_hermes_supervised_auxiliaries; then trace success; else trace "failure:$?"; fi', + 'trace "final-api-bridge:$SOCAT_PID"', + ]); + + expect(result.status, result.stderr).toBe(0); + expect(result.stdout.trim().split("\n")).toEqual([ + "live:101", + "listener:101:8642", + "stop:101", + "start-forward:8642 18642 API SOCAT_PID 4242 current", + "live:111", + "listener:111:8642", + "live:111", + "listener:111:8642", + "dashboard-healthy", + "live:303", + "listener:303:18789", + "gateway-log", + "success", + "final-api-bridge:111", + ]); + }); + + it("fails closed when a replacement API bridge still cannot serve public health", () => { + const source = fs.readFileSync(START_SCRIPT, "utf-8"); + const result = runBashHarness([ + 'trace() { printf "%s\\n" "$*"; }', + 'id() { [ "${1:-}" = "-u" ] && printf "1000\\n"; }', + 'gateway_control_pid_is_live() { trace "live:$1"; return 0; }', + 'hermes_tracked_role_is_current() { gateway_control_pid_is_live "$2"; }', + 'gateway_control_pid_owns_tcp_listener() { trace "listener:$1:$2"; return 0; }', + 'curl() { printf "503"; }', + 'hermes_stop_tracked_role() { trace "stop:$2"; return 0; }', + 'start_socat_forwarder() { trace "start-forward:$*"; printf -v "$4" 111; return 0; }', + "hermes_dashboard_healthy() { trace unexpected-dashboard-health; return 0; }", + "ensure_gateway_log_stream() { trace unexpected-gateway-log; }", + extractShellFunction(source, "hermes_socat_bridge_healthy"), + extractShellFunction(source, "hermes_api_socat_bridge_healthy"), + extractShellFunction(source, "ensure_hermes_supervised_auxiliaries"), + "PUBLIC_PORT=8642", + "INTERNAL_PORT=18642", + "DASHBOARD_PUBLIC_PORT=18789", + "DASHBOARD_INTERNAL_PORT=19119", + "SOCAT_PID=101", + "DASHBOARD_PID=202", + "DASHBOARD_SOCAT_PID=303", + "GATEWAY_PID=4242", + 'if ensure_hermes_supervised_auxiliaries; then trace success; else trace "failure:$?"; fi', + 'trace "final-api-bridge:$SOCAT_PID"', + ]); + + expect(result.status, result.stderr).toBe(0); + expect(result.stdout.trim().split("\n")).toEqual([ + "live:101", + "listener:101:8642", + "stop:101", + "start-forward:8642 18642 API SOCAT_PID 4242 current", + "live:111", + "listener:111:8642", + "failure:1", + "final-api-bridge:111", + ]); + }); + it("restarts a dashboard that owns its listener but fails HTTP health", () => { const source = fs.readFileSync(START_SCRIPT, "utf-8"); const result = runBashHarness([ @@ -1171,12 +1289,13 @@ describe("Hermes supervised auxiliary recovery", () => { 'hermes_tracked_role_is_current() { gateway_control_pid_is_live "$2"; }', 'gateway_control_pid_owns_tcp_listener() { trace "listener:$1:$2"; return 0; }', 'hermes_tracked_service_owns_listener() { trace "service-listener:$1:$2:$3"; return 0; }', - 'curl() { trace dashboard-http; printf "500"; }', + 'curl() { case "$*" in *:8642/health*) printf "200" ;; *) trace dashboard-http; printf "500" ;; esac; }', 'hermes_stop_tracked_role() { trace "stop:$2"; return 0; }', "start_hermes_dashboard_sandbox_user() { trace start-dashboard; DASHBOARD_PID=404; DASHBOARD_SOCAT_PID=505; }", 'start_socat_forwarder() { trace "unexpected-forward:$*"; return 1; }', "ensure_gateway_log_stream() { trace gateway-log; }", extractShellFunction(source, "hermes_socat_bridge_healthy"), + extractShellFunction(source, "hermes_api_socat_bridge_healthy"), extractShellFunction(source, "hermes_dashboard_healthy"), extractShellFunction(source, "ensure_hermes_supervised_auxiliaries"), "PUBLIC_PORT=8642", @@ -1192,6 +1311,8 @@ describe("Hermes supervised auxiliary recovery", () => { expect(result.status, result.stderr).toBe(0); expect(result.stdout.trim().split("\n")).toEqual([ + "live:101", + "listener:101:8642", "live:101", "listener:101:8642", "live:202", @@ -1217,6 +1338,7 @@ describe("Hermes supervised auxiliary recovery", () => { 'hermes_stop_tracked_role() { trace "stop:$2"; return 0; }', "ensure_gateway_log_stream() { trace unexpected-gateway-log; }", extractShellFunction(source, "hermes_socat_bridge_healthy"), + extractShellFunction(source, "hermes_api_socat_bridge_healthy"), extractShellFunction(source, "ensure_hermes_supervised_auxiliaries"), "PUBLIC_PORT=8642", "INTERNAL_PORT=18642", diff --git a/test/hermes-mcp-config-transaction.test.ts b/test/hermes-mcp-config-transaction.test.ts new file mode 100644 index 00000000000..07ff0e08b01 --- /dev/null +++ b/test/hermes-mcp-config-transaction.test.ts @@ -0,0 +1,1481 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import crypto from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { + normalizeMcpServerUrl, + validateMcpCredentialEnvName, +} from "../src/lib/actions/sandbox/mcp-bridge-validation"; +import credentialBoundaryManifest from "../src/lib/actions/sandbox/openshell-child-visible-credentials.v0.0.72.json"; + +const TRANSACTION = path.resolve( + import.meta.dirname, + "..", + "agents/hermes/mcp-config-transaction.py", +); +const GUARD = path.resolve(import.meta.dirname, "..", "agents/hermes/runtime-config-guard.py"); + +function runPython(source: string, args: string[] = []) { + return spawnSync("python3", ["-c", source, TRANSACTION, GUARD, ...args], { + encoding: "utf8", + }); +} + +describe("Hermes managed MCP config transaction", () => { + it("rejects raw credentials, plaintext targets, and non-boolean control flags", () => { + const result = runPython(` +import importlib.util, json, sys +spec = importlib.util.spec_from_file_location("mcp_tx", sys.argv[1]) +module = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) +bad = [ + {"server": "fake", "url": "https://mcp.example.test/mcp", "headers": {"Authorization": "Bearer raw-secret"}}, + {"server": "fake", "url": "http://host.openshell.internal/mcp", "headers": {"Authorization": "Bearer openshell:resolve:env:FAKE_TOKEN"}}, + {"server": "fake", "url": "https://mcp.example.test/mcp", "headers": {"Authorization": "Bearer openshell:resolve:env:FAKE_TOKEN"}, "replace_existing": "yes"}, +] +errors = [] +for payload in bad: + try: + module._validate_payload("add", payload) + except ValueError as error: + errors.append(str(error)) +print(json.dumps(errors)) +if len(errors) != len(bad): + raise SystemExit(9) +`); + + expect(result.status, result.stderr).toBe(0); + expect(JSON.parse(result.stdout)).toHaveLength(3); + }); + + it("keeps Hermes and host MCP URL rejection boundaries in parity", () => { + const cases = [ + { url: "https://mcp.example.com/mcp", accepted: true }, + { url: "https://mcp.example.com./mcp", accepted: false }, + { url: "https://host.openshell.internal:31337/mcp", accepted: false }, + { url: "https://host.docker.internal:31337/mcp", accepted: false }, + { url: "https://host.containers.internal:31337/mcp", accepted: false }, + { url: "https://8.8.8.8/mcp", accepted: true }, + { url: "http://mcp.example.com/mcp", accepted: false }, + { url: "https://localhost/mcp", accepted: false }, + { url: "https://service.internal/mcp", accepted: false }, + { url: "https://127.0.0.1/mcp", accepted: false }, + { url: "https://10.0.0.1/mcp", accepted: false }, + { url: "https://100.64.0.1/mcp", accepted: false }, + { url: "https://169.254.169.254/mcp", accepted: false }, + { url: "https://192.0.2.1/mcp", accepted: false }, + { url: "https://198.18.0.1/mcp", accepted: false }, + { url: "https://224.0.0.1/mcp", accepted: false }, + { url: "https://[::1]/mcp", accepted: false }, + { url: "https://[fc00::1]/mcp", accepted: false }, + { url: "https://[fe80::1]/mcp", accepted: false }, + { url: "https://[2001:db8::1]/mcp", accepted: false }, + { url: "https://[ff02::1]/mcp", accepted: false }, + { url: "https://[::ffff:127.0.0.1]/mcp", accepted: false }, + { url: "https://[2606:4700:4700::1111]/mcp", accepted: false }, + { url: "https://2130706433/mcp", accepted: false }, + { url: "https://user:password@mcp.example.com/mcp", accepted: false }, + { url: "https://mcp.example.com//mcp", accepted: false }, + { url: "https://mcp.example.com/mcp\\child", accepted: false }, + { url: "https://mcp.example.com/%2f", accepted: false }, + { url: "https://mcp.example.com/%", accepted: false }, + { url: "https://mcp.example.com/%GG", accepted: false }, + { url: "https://mcp.example.com/%2", accepted: false }, + { url: "https://mcp.example.com/mcp?token=x", accepted: false }, + { url: "https://mcp.example.com/mcp#fragment", accepted: false }, + { url: "wss://mcp.example.com/mcp", accepted: false }, + ]; + const expected = cases.map(({ accepted }) => accepted); + const hostResults = cases.map(({ url }) => { + try { + normalizeMcpServerUrl(url); + return true; + } catch { + return false; + } + }); + const result = runPython( + ` +import importlib.util, json, sys +spec = importlib.util.spec_from_file_location("mcp_tx", sys.argv[1]) +module = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) +results = [] +for url in json.loads(sys.argv[3]): + payload = { + "server": "fake", + "url": url, + "headers": {"Authorization": "Bearer openshell:resolve:env:FAKE_TOKEN"}, + "replace_existing": False, + } + try: + module._validate_payload("add", payload) + except ValueError: + results.append(False) + else: + results.append(True) +print(json.dumps(results)) +`, + [JSON.stringify(cases.map(({ url }) => url))], + ); + + expect(hostResults).toEqual(expected); + expect(result.status, result.stderr).toBe(0); + expect(JSON.parse(result.stdout)).toEqual(expected); + }); + + it("rejects every OpenShell host alias when the Hermes validator is called directly", () => { + const result = runPython(` +import importlib.util, json, sys +spec = importlib.util.spec_from_file_location("mcp_tx", sys.argv[1]) +module = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) +errors = [] +for host in ("host.openshell.internal", "host.docker.internal", "host.containers.internal"): + payload = { + "server": "fake", + "url": f"https://{host}:31337/mcp", + "headers": {"Authorization": "Bearer openshell:resolve:env:FAKE_TOKEN"}, + "replace_existing": False, + } + try: + module._validate_payload("add", payload) + except ValueError as error: + errors.append(str(error)) +print(json.dumps(errors)) +if len(errors) != 3: + raise SystemExit(9) +`); + + expect(result.status, result.stderr).toBe(0); + expect(JSON.parse(result.stdout)).toEqual([ + "Authenticated MCP OpenShell host aliases are unavailable with OpenShell v0.0.72", + "Authenticated MCP OpenShell host aliases are unavailable with OpenShell v0.0.72", + "Authenticated MCP OpenShell host aliases are unavailable with OpenShell v0.0.72", + ]); + }); + + it("accepts a legacy host alias only for exact cleanup payloads", () => { + const result = runPython(` +import importlib.util, json, sys +spec = importlib.util.spec_from_file_location("mcp_tx", sys.argv[1]) +module = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) +payload = { + "server": "fake", + "url": "https://host.openshell.internal:31337/mcp", + "headers": {"Authorization": "Bearer openshell:resolve:env:GCP_PROJECT_ID"}, + "force": True, +} +module._validate_payload("remove", payload) +print(json.dumps({"ok": True})) +`); + + expect(result.status, result.stderr).toBe(0); + expect(JSON.parse(result.stdout)).toEqual({ ok: true }); + }); + + it("shares the host credential-name boundary while preserving exact cleanup", () => { + const blockedNames = [ + ...credentialBoundaryManifest.rawChildValueKeys, + ...credentialBoundaryManifest.rewrittenChildValueKeys, + ...credentialBoundaryManifest.runtimeControlKeys, + ...credentialBoundaryManifest.runtimeControlPrefixes.map((prefix) => `${prefix}MCP_TOKEN`), + ]; + const result = runPython( + ` +import importlib.util, json, sys +spec = importlib.util.spec_from_file_location("mcp_tx", sys.argv[1]) +module = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) + +def payload(name, action): + return { + "server": "fake", + "url": "https://mcp.example.test/mcp", + "headers": {"Authorization": f"Bearer openshell:resolve:env:{name}"}, + "replace_existing" if action == "add" else "force": False, + } + +blocked = json.loads(sys.argv[3]) +add_rejected = [] +cleanup_accepted = [] +for name in blocked: + try: + module._validate_payload("add", payload(name, "add")) + except ValueError: + add_rejected.append(name) + try: + module._validate_payload("remove", payload(name, "remove")) + except ValueError: + pass + else: + cleanup_accepted.append(name) +module._validate_payload("add", payload("MY_SERVICE_MCP_TOKEN", "add")) +print(json.dumps({ + "addRejected": add_rejected, + "cleanupAccepted": cleanup_accepted, + "safeAccepted": True, +})) +`, + [JSON.stringify(blockedNames)], + ); + + expect(credentialBoundaryManifest.openshellVersion).toBe("0.0.72"); + for (const name of blockedNames) { + expect(() => validateMcpCredentialEnvName(name)).toThrow(); + } + expect(() => validateMcpCredentialEnvName("MY_SERVICE_MCP_TOKEN")).not.toThrow(); + expect(result.status, result.stderr).toBe(0); + expect(JSON.parse(result.stdout)).toEqual({ + addRejected: blockedNames, + cleanupAccepted: blockedNames, + safeAccepted: true, + }); + }); + + it("accepts only HTTPS endpoint definitions with one OpenShell placeholder", () => { + const result = runPython(` +import importlib.util, json, sys +spec = importlib.util.spec_from_file_location("mcp_tx", sys.argv[1]) +module = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) + +base = { + "server": "safe_name-1", + "url": "https://mcp.example.test/mcp", + "headers": {"Authorization": "Bearer openshell:resolve:env:SAFE_MCP_TOKEN"}, +} +valid = [ + ("add", {**base, "replace_existing": False}), + ("remove", {**base, "force": False}), +] +invalid = [ + ("restart", {**base, "force": False}), + ("add", {**base, "replace_existing": False, "command": "touch /tmp/pwned"}), + ("add", {**base, "replace_existing": False, "args": ["--token", "raw"]}), + ("add", {**base, "replace_existing": False, "transport": "stdio"}), + ("add", {**base, "replace_existing": False, "env": {"SAFE_MCP_TOKEN": "raw"}}), + ("add", {**base, "replace_existing": False, "url": "http://mcp.example.test/mcp"}), + ("add", {**base, "replace_existing": False, "url": "https://mcp.example.test/../mcp"}), + ("add", {**base, "replace_existing": False, "url": "https://mcp.example.test/./mcp"}), + ("add", {**base, "replace_existing": False, "url": "https://mcp.example.test/mcp?transport=sse"}), + ("add", {**base, "replace_existing": False, "headers": {}}), + ("add", {**base, "replace_existing": False, "headers": {"authorization": base["headers"]["Authorization"]}}), + ("add", {**base, "replace_existing": False, "headers": {**base["headers"], "X-Api-Key": "raw"}}), + ("add", {**base, "replace_existing": False, "headers": {"Authorization": "Bearer raw-secret"}}), + ("add", {**base, "replace_existing": False, "headers": {"Authorization": "openshell:resolve:env:SAFE_MCP_TOKEN"}}), + ("add", {**base, "replace_existing": False, "headers": {"Authorization": "Bearer openshell:resolve:env:1INVALID"}}), + ("add", {**base, "replace_existing": False, "headers": {"Authorization": "Bearer openshell:resolve:env:SAFE_MCP_TOKEN extra"}}), +] + +accepted = [] +for action, payload in valid + invalid: + try: + module._validate_payload(action, payload) + except (TypeError, ValueError): + accepted.append(False) + else: + accepted.append(True) +print(json.dumps(accepted)) +`); + + expect(result.status, result.stderr).toBe(0); + expect(JSON.parse(result.stdout)).toEqual([true, true, ...Array(16).fill(false)]); + }); + + it("rejects command, YAML-tag, and terminal-control injection without executing it", () => { + const temp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-hermes-mcp-injection-")); + const sentinel = path.join(temp, "executed"); + const invalidPayload = { + server: `safe;touch ${sentinel}\n\u001b[31mFORGED`, + url: "https://mcp.example.test/mcp", + headers: { Authorization: "Bearer openshell:resolve:env:SAFE_MCP_TOKEN" }, + replace_existing: false, + }; + const commandResult = spawnSync( + "python3", + [TRANSACTION, "add", "--payload", JSON.stringify(invalidPayload)], + { encoding: "utf8" }, + ); + + try { + expect(commandResult.status).toBe(2); + expect(commandResult.stderr).not.toContain("\u001b"); + expect(commandResult.stderr.trim().split("\n")).toHaveLength(1); + expect(fs.existsSync(sentinel)).toBe(false); + + const hermesDir = path.join(temp, ".hermes"); + fs.mkdirSync(hermesDir); + fs.writeFileSync( + path.join(hermesDir, "config.yaml"), + `model: !!python/object/apply:os.system ["touch ${sentinel}"]\n`, + { mode: 0o600 }, + ); + fs.writeFileSync(path.join(hermesDir, ".env"), "HERMES_TEST=1\n", { + mode: 0o600, + }); + fs.writeFileSync(path.join(hermesDir, ".config-hash"), "untrusted\n", { + mode: 0o600, + }); + const yamlResult = runPython( + ` +import importlib.util, json, os, sys +spec = importlib.util.spec_from_file_location("mcp_tx", sys.argv[1]) +module = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) +module.GUARD_PATH = sys.argv[2] +module.HERMES_DIR = sys.argv[3] +module.CONFIG_PATH = os.path.join(module.HERMES_DIR, "config.yaml") +module.os.geteuid = lambda: 1000 +module._assert_non_root_lifecycle_identity = lambda: None +payload = { + "server": "safe", + "url": "https://mcp.example.test/mcp", + "headers": {"Authorization": "Bearer openshell:resolve:env:SAFE_MCP_TOKEN"}, + "replace_existing": False, +} +sys.argv = [sys.argv[1], "add", "--payload", json.dumps(payload)] +print(json.dumps({"exit_code": module.main()})) +`, + [hermesDir], + ); + + expect(yamlResult.status, `${yamlResult.stdout}\n${yamlResult.stderr}`).toBe(0); + expect(JSON.parse(yamlResult.stdout)).toEqual({ exit_code: 2 }); + expect(yamlResult.stderr.trim()).toBe("Invalid Hermes config: YAML parsing failed"); + expect(yamlResult.stderr).not.toContain("python/object"); + expect(fs.existsSync(sentinel)).toBe(false); + } finally { + fs.rmSync(temp, { recursive: true, force: true }); + } + }); + + it("preserves falsey non-map YAML roots across mutation and reload transactions", () => { + const result = runPython(` +import importlib.util, json, sys, types +spec = importlib.util.spec_from_file_location("mcp_tx", sys.argv[1]) +module = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) +payload = { + "server": "safe", + "url": "https://mcp.example.test/mcp", + "headers": {"Authorization": "Bearer openshell:resolve:env:SAFE_MCP_TOKEN"}, + "replace_existing": False, +} +snapshot = types.SimpleNamespace(mode=0o600) +module.os.geteuid = lambda: 1000 +module._assert_mutable_snapshot = lambda received: None +module._managed_hash_paths = lambda privileged: [] +module._refresh_and_verify_hashes = lambda guard, privileged: None +module.reload_gateway = lambda: True + +def run(method_name, original): + state = {"text": original, "writes": []} + def read_text(path): + return state["text"], snapshot + def write_existing(path, text, received_snapshot, mode): + state["writes"].append(text) + state["text"] = text + module._load_guard = lambda: types.SimpleNamespace( + _read_text=read_text, + _write_existing=write_existing, + ) + error = "" + try: + getattr(module, method_name)("add", payload) + except (TypeError, ValueError) as caught: + error = str(caught) + return { + "error": error, + "preserved": state["text"] == original, + "writes": len(state["writes"]), + } + +falsey_roots = ["[]\\n", "false\\n", "0\\n", '""\\n'] +results = { + method: [run(method, original) for original in falsey_roots] + for method in ("apply_transaction", "apply_transaction_and_reload") +} +null_result = run("apply_transaction", "null\\n") +print(json.dumps({"results": results, "null_result": null_result})) +`); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + const payload = JSON.parse(result.stdout) as { + results: Record>; + null_result: { error: string; preserved: boolean; writes: number }; + }; + for (const outcomes of Object.values(payload.results)) { + expect(outcomes).toHaveLength(4); + for (const outcome of outcomes) { + expect(outcome.error).toContain("expected a YAML object"); + expect(outcome.preserved).toBe(true); + expect(outcome.writes).toBe(0); + } + } + expect(payload.null_result.error).toBe(""); + expect(payload.null_result.preserved).toBe(false); + expect(payload.null_result.writes).toBe(1); + }); + + it("emits bounded one-line errors with payload and runtime secrets redacted", () => { + const result = runPython(` +import importlib.util, json, sys +spec = importlib.util.spec_from_file_location("mcp_tx", sys.argv[1]) +module = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) +payload = { + "server": "safe", + "url": "https://mcp.example.test/mcp", + "headers": {"Authorization": "Bearer openshell:resolve:env:SAFE_MCP_TOKEN"}, + "replace_existing": False, +} +def fail(action, received): + raise RuntimeError( + "reload failed Authorization: " + received["headers"]["Authorization"] + + " Bearer runtime-secret-123 token=second-secret-456 " + + "https://user:password@example.test/mcp?token=query-secret-789 " + + "\\x1b[31m\\nFORGED\\u202e" + ("A" * 1000) + ) +module.execute = fail +sys.argv = [sys.argv[1], "add", "--payload", json.dumps(payload)] +print(json.dumps({"exit_code": module.main()})) +`); + + expect(result.status, result.stdout).toBe(0); + expect(JSON.parse(result.stdout)).toEqual({ exit_code: 2 }); + expect(result.stderr).toContain(""); + for (const secret of [ + "SAFE_MCP_TOKEN", + "runtime-secret-123", + "second-secret-456", + "password", + "query-secret-789", + ]) { + expect(result.stderr).not.toContain(secret); + } + expect(result.stderr).not.toContain("\u001b"); + expect(result.stderr).not.toContain("\u202e"); + expect(result.stderr.trim().split("\n")).toHaveLength(1); + expect(result.stderr.trim().length).toBeLessThanOrEqual(512); + + const representations = runPython(` +import importlib.util, json, sys +spec = importlib.util.spec_from_file_location("mcp_tx", sys.argv[1]) +module = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) +messages = [ + "failed {'api_key': 'raw-secret-1'}", + "failed {'token': 'raw-secret-2'}", + 'Authorization: Bearer "runtime secret with spaces", comma-secret', + "Bearer 'quoted bearer secret', suffix-secret", +] +print(json.dumps([ + module._sanitize_error_message(RuntimeError(message)) for message in messages +])) +`); + expect(representations.status, representations.stderr).toBe(0); + const sanitized = JSON.parse(representations.stdout) as string[]; + expect(sanitized).toHaveLength(4); + for (const message of sanitized) expect(message).toContain(""); + for (const secret of [ + "raw-secret-1", + "raw-secret-2", + "runtime secret with spaces", + "comma-secret", + "quoted bearer secret", + "suffix-secret", + ]) { + expect(sanitized.join("\n")).not.toContain(secret); + } + }); + + it("refuses a locked config snapshot", () => { + const result = runPython(` +import importlib.util, sys, types +spec = importlib.util.spec_from_file_location("mcp_tx", sys.argv[1]) +module = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) +module.os.geteuid = lambda: 1000 +try: + module._assert_mutable_snapshot(types.SimpleNamespace(mode=0o440, uid=1000, gid=1000)) +except RuntimeError as error: + print(str(error)) +else: + raise SystemExit(9) +`); + + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toContain("locked"); + }); + + it("blocks symlink, hardlink, permission, inode-race, and atomic-write guard bypasses", () => { + const result = runPython(` +import hashlib, importlib.util, json, os, shutil, sys, tempfile + +TRANSACTION_PATH = sys.argv[1] +GUARD_PATH = sys.argv[2] +CONFIG_TEXT = "model: test\\n" +ENV_TEXT = "HERMES_TEST=1\\n" +PAYLOAD = { + "server": "safe", + "url": "https://mcp.example.test/mcp", + "headers": {"Authorization": "Bearer openshell:resolve:env:SAFE_MCP_TOKEN"}, + "replace_existing": False, +} + +def load_transaction(name): + spec = importlib.util.spec_from_file_location(name, TRANSACTION_PATH) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + +def fixture(name): + root = tempfile.mkdtemp(prefix="nemoclaw-hermes-mcp-" + name + "-") + hermes_dir = os.path.join(root, ".hermes") + os.mkdir(hermes_dir, 0o700) + config_path = os.path.join(hermes_dir, "config.yaml") + env_path = os.path.join(hermes_dir, ".env") + hash_path = os.path.join(hermes_dir, ".config-hash") + for path, text in ((config_path, CONFIG_TEXT), (env_path, ENV_TEXT)): + with open(path, "w", encoding="utf-8") as handle: + handle.write(text) + os.chmod(path, 0o600) + hash_text = ( + hashlib.sha256(CONFIG_TEXT.encode()).hexdigest() + " " + config_path + "\\n" + + hashlib.sha256(ENV_TEXT.encode()).hexdigest() + " " + env_path + "\\n" + ) + with open(hash_path, "w", encoding="utf-8") as handle: + handle.write(hash_text) + os.chmod(hash_path, 0o600) + return root, hermes_dir, config_path, hash_path, hash_text + +def configure(module, hermes_dir): + module.GUARD_PATH = GUARD_PATH + module.HERMES_DIR = hermes_dir + module.CONFIG_PATH = os.path.join(hermes_dir, "config.yaml") + module.os.geteuid = lambda: 1000 + module._assert_mutable_snapshot = lambda snapshot: None + +def blocked(operation): + try: + operation() + except Exception as error: + return True, type(error).__name__ + return False, "none" + +results = {} +roots = [] +try: + root, hermes_dir, config_path, _, _ = fixture("config-symlink") + roots.append(root) + target = os.path.join(root, "config-target") + os.replace(config_path, target) + os.symlink(target, config_path) + module = load_transaction("mcp_tx_config_symlink") + configure(module, hermes_dir) + was_blocked, error = blocked(lambda: module.apply_transaction("add", PAYLOAD)) + results["config_symlink"] = { + "blocked": was_blocked, + "error": error, + "preserved": open(target, encoding="utf-8").read() == CONFIG_TEXT, + } + + root, hermes_dir, config_path, _, _ = fixture("config-hardlink") + roots.append(root) + alias = os.path.join(root, "config-alias") + os.link(config_path, alias) + module = load_transaction("mcp_tx_config_hardlink") + configure(module, hermes_dir) + was_blocked, error = blocked(lambda: module.apply_transaction("add", PAYLOAD)) + results["config_hardlink"] = { + "blocked": was_blocked, + "error": error, + "preserved": open(alias, encoding="utf-8").read() == CONFIG_TEXT, + } + + root, hermes_dir, config_path, _, _ = fixture("config-mode") + roots.append(root) + os.chmod(config_path, 0o620) + module = load_transaction("mcp_tx_config_mode") + configure(module, hermes_dir) + was_blocked, error = blocked(lambda: module.apply_transaction("add", PAYLOAD)) + results["config_group_writable"] = { + "blocked": was_blocked, + "error": error, + "preserved": open(config_path, encoding="utf-8").read() == CONFIG_TEXT, + } + + for kind in ("symlink", "hardlink"): + root, hermes_dir, config_path, hash_path, hash_text = fixture("hash-" + kind) + roots.append(root) + alias = os.path.join(root, "hash-alias") + if kind == "symlink": + os.replace(hash_path, alias) + os.symlink(alias, hash_path) + else: + os.link(hash_path, alias) + module = load_transaction("mcp_tx_hash_" + kind) + configure(module, hermes_dir) + was_blocked, error = blocked(lambda: module.apply_transaction("add", PAYLOAD)) + results["hash_" + kind] = { + "blocked": was_blocked, + "error": error, + "config_preserved": open(config_path, encoding="utf-8").read() == CONFIG_TEXT, + "hash_preserved": open(alias, encoding="utf-8").read() == hash_text, + } + + root, hermes_dir, config_path, _, _ = fixture("config-race") + roots.append(root) + module = load_transaction("mcp_tx_config_race") + configure(module, hermes_dir) + guard = module._load_guard() + module._load_guard = lambda: guard + original_write = guard._write_existing + raced = {"done": False} + def race_before_write(path, text, snapshot, mode=None): + if path == config_path and not raced["done"]: + raced["done"] = True + replacement = os.path.join(hermes_dir, "attacker-config") + with open(replacement, "w", encoding="utf-8") as handle: + handle.write("attacker: preserved\\n") + os.chmod(replacement, 0o600) + os.replace(replacement, config_path) + return original_write(path, text, snapshot, mode=mode) + guard._write_existing = race_before_write + was_blocked, error = blocked(lambda: module.apply_transaction("add", PAYLOAD)) + results["config_inode_race"] = { + "blocked": was_blocked, + "error": error, + "attacker_preserved": open(config_path, encoding="utf-8").read() == "attacker: preserved\\n", + } + + root, hermes_dir, config_path, _, _ = fixture("atomic-failure") + roots.append(root) + module = load_transaction("mcp_tx_atomic_failure") + configure(module, hermes_dir) + guard = module._load_guard() + module._load_guard = lambda: guard + original_replace = guard.os.replace + def fail_config_replace(source, destination, *args, **kwargs): + if destination == "config.yaml": + raise OSError("simulated atomic replace failure") + return original_replace(source, destination, *args, **kwargs) + guard.os.replace = fail_config_replace + was_blocked, error = blocked(lambda: module.apply_transaction("add", PAYLOAD)) + guard.os.replace = original_replace + results["atomic_replace_failure"] = { + "blocked": was_blocked, + "error": error, + "config_preserved": open(config_path, encoding="utf-8").read() == CONFIG_TEXT, + "temp_cleaned": not any(".nemoclaw." in name for name in os.listdir(hermes_dir)), + } + + root, hermes_dir, config_path, hash_path, hash_text = fixture("hash-race") + roots.append(root) + module = load_transaction("mcp_tx_hash_race") + configure(module, hermes_dir) + guard = module._load_guard() + original_hash_text = guard._hash_text + raced = {"done": False} + def race_after_hash(*args): + value = original_hash_text(*args) + if not raced["done"]: + raced["done"] = True + replacement = os.path.join(hermes_dir, "raced-config") + with open(replacement, "w", encoding="utf-8") as handle: + handle.write("attacker: after-hash\\n") + os.chmod(replacement, 0o600) + os.replace(replacement, config_path) + return value + guard._hash_text = race_after_hash + was_blocked, error = blocked(lambda: module._refresh_and_verify_hashes(guard, False)) + results["hash_inode_race"] = { + "blocked": was_blocked, + "error": error, + "hash_preserved": open(hash_path, encoding="utf-8").read() == hash_text, + } +finally: + for root in roots: + shutil.rmtree(root, ignore_errors=True) + +print(json.dumps(results, sort_keys=True)) +`); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + const scenarios = JSON.parse(result.stdout) as Record>; + expect(Object.keys(scenarios).sort()).toEqual([ + "atomic_replace_failure", + "config_group_writable", + "config_hardlink", + "config_inode_race", + "config_symlink", + "hash_hardlink", + "hash_inode_race", + "hash_symlink", + ]); + const expectedErrors: Record = { + atomic_replace_failure: "OSError", + config_group_writable: "UnsafePathError", + config_hardlink: "UnsafePathError", + config_inode_race: "UnsafePathError", + config_symlink: "OSError", + hash_hardlink: "UnsafePathError", + hash_inode_race: "UnsafePathError", + hash_symlink: "OSError", + }; + for (const [name, scenario] of Object.entries(scenarios)) { + expect(scenario.blocked, name).toBe(true); + expect(scenario.error, `${name}.error`).toBe(expectedErrors[name]); + for (const [property, value] of Object.entries(scenario).filter( + ([property]) => property.endsWith("preserved") || property === "temp_cleaned", + )) { + expect(value, `${name}.${property}`).toBe(true); + } + } + }); + + it("keeps config ownership and gateway lifecycle identities separated", () => { + const result = runPython(` +import importlib.util, json, stat, sys, types +spec = importlib.util.spec_from_file_location("mcp_tx", sys.argv[1]) +module = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) + +errors = [] +module.os.geteuid = lambda: 1000 +for snapshot in ( + types.SimpleNamespace(mode=0o600, uid=2000, gid=1000), + types.SimpleNamespace(mode=0o400, uid=1000, gid=1000), +): + try: + module._assert_mutable_snapshot(snapshot) + except RuntimeError as error: + errors.append(str(error)) + +module.os.geteuid = lambda: 0 +module.pwd.getpwnam = lambda name: types.SimpleNamespace(pw_uid=1000) +module.grp.getgrnam = lambda name: types.SimpleNamespace(gr_gid=1000) +for snapshot in ( + types.SimpleNamespace(mode=0o600, uid=2000, gid=1000), + types.SimpleNamespace(mode=0o600, uid=1000, gid=2000), +): + try: + module._assert_mutable_snapshot(snapshot) + except RuntimeError as error: + errors.append(str(error)) + +module.os.geteuid = lambda: 1000 +unsafe_markers = ( + types.SimpleNamespace(st_mode=stat.S_IFLNK | 0o777, st_uid=0), + types.SimpleNamespace(st_mode=stat.S_IFREG | 0o444, st_uid=1000), +) +for marker in unsafe_markers: + module.os.lstat = lambda path, marker=marker: marker + try: + module._assert_non_root_lifecycle_identity() + except PermissionError as error: + errors.append(str(error)) + +print(json.dumps(errors)) +`); + + expect(result.status, result.stderr).toBe(0); + const errors = JSON.parse(result.stdout) as string[]; + expect(errors).toHaveLength(6); + expect(errors.slice(0, 4).every((error) => error.includes("not owned"))).toBe(true); + expect(errors.slice(4).every((error) => error.includes("marker is unsafe"))).toBe(true); + }); + + it("treats edits to any managed field as drift during removal", () => { + const result = runPython(` +import importlib.util, sys +spec = importlib.util.spec_from_file_location("mcp_tx", sys.argv[1]) +module = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) +payload = { + "server": "fake", + "url": "https://mcp.example.test/mcp", + "headers": {"Authorization": "Bearer openshell:resolve:env:FAKE_TOKEN"}, +} +candidate = module._managed_candidate(payload) +candidate["enabled"] = False +try: + module._mutate({"mcp_servers": {"fake": candidate}}, "remove", payload) +except ValueError as error: + print(str(error)) +else: + raise SystemExit(9) +`); + + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toContain("Refusing to remove modified Hermes MCP server"); + }); + + it("treats a null same-name Hermes server as drift rather than absence", () => { + const result = runPython(` +import importlib.util, sys +spec = importlib.util.spec_from_file_location("mcp_tx", sys.argv[1]) +module = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) +payload = { + "server": "fake", + "url": "https://mcp.example.test/mcp", + "headers": {"Authorization": "Bearer openshell:resolve:env:FAKE_TOKEN"}, +} +try: + module._mutate({"mcp_servers": {"fake": None}}, "remove", payload) +except ValueError as error: + print(str(error)) +else: + raise SystemExit(9) +`); + + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toContain("Refusing to remove modified Hermes MCP server"); + }); + + it("allows root reload control to signal only the gateway service identity", () => { + const result = runPython(` +import importlib.util, sys, types +spec = importlib.util.spec_from_file_location("mcp_tx", sys.argv[1]) +module = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) +gateway = types.ModuleType("gateway") +status = types.ModuleType("gateway.status") +status.get_running_pid = lambda cleanup_stale=False: 4242 +status.get_process_start_time = lambda pid: 99 +sys.modules["gateway"] = gateway +sys.modules["gateway.status"] = status +module.os.geteuid = lambda: 0 +module.pwd.getpwnam = lambda name: types.SimpleNamespace(pw_uid=2000) +module._is_trusted_gateway_process = lambda pid: True +module.os.stat = lambda path: types.SimpleNamespace(st_uid=1000) +try: + module._gateway_identity() +except PermissionError as error: + print(str(error)) +else: + raise SystemExit(9) +module.os.stat = lambda path: types.SimpleNamespace(st_uid=2000) +if module._gateway_identity() != (4242, 99): + raise SystemExit(10) +module._is_trusted_gateway_process = lambda pid: False +try: + module._gateway_identity() +except PermissionError as error: + print(str(error)) +else: + raise SystemExit(11) +`); + + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toContain("expected gateway identity"); + expect(result.stdout).toContain("does not identify the trusted launcher"); + }); + + it("recognizes the wrapped Hermes gateway from its bounded PID record", () => { + const temp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-hermes-gateway-pid-")); + const pidPath = path.join(temp, "gateway.pid"); + fs.writeFileSync(pidPath, JSON.stringify({ pid: 4242, start_time: 99 }), { mode: 0o600 }); + + try { + const result = runPython( + ` +import importlib.util, json, os, sys, types +spec = importlib.util.spec_from_file_location("mcp_tx", sys.argv[1]) +module = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) + +expected_uid = os.geteuid() +gateway = types.ModuleType("gateway") +status = types.ModuleType("gateway.status") +status.get_running_pid = lambda cleanup_stale=False: None +status.get_process_start_time = lambda pid: 99 +runtime = {"lock_active": True} +status.is_gateway_runtime_lock_active = lambda: runtime["lock_active"] +sys.modules["gateway"] = gateway +sys.modules["gateway.status"] = status + +module.GATEWAY_PID_PATH = sys.argv[3] +module.os.stat = lambda path: types.SimpleNamespace(st_uid=expected_uid) +module._is_trusted_gateway_process = lambda pid: pid == 4242 + +recognized = module._gateway_identity() +runtime["lock_active"] = False +unlocked = module._gateway_identity() +runtime["lock_active"] = True +status.get_process_start_time = lambda pid: 100 +reused = module._gateway_identity() +start_times = iter((99, 100)) +status.get_process_start_time = lambda pid: next(start_times) +unstable = module._gateway_identity() +status.get_process_start_time = lambda pid: 99 +module._is_trusted_gateway_process = lambda pid: False +try: + module._gateway_identity() +except PermissionError as error: + untrusted = str(error) +else: + raise SystemExit(9) +print(json.dumps({ + "recognized": recognized, + "reused": reused, + "unstable": unstable, + "unlocked": unlocked, + "untrusted": untrusted, +})) +`, + [pidPath], + ); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + expect(JSON.parse(result.stdout)).toEqual({ + recognized: [4242, 99], + reused: null, + unstable: null, + unlocked: null, + untrusted: "Hermes gateway PID does not identify the trusted launcher", + }); + } finally { + fs.rmSync(temp, { recursive: true, force: true }); + } + }); + + it("rejects a FIFO gateway PID record without blocking", () => { + const temp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-hermes-gateway-fifo-")); + const fifoPath = path.join(temp, "gateway.pid"); + + try { + const result = runPython( + ` +import importlib.util, os, signal, sys +spec = importlib.util.spec_from_file_location("mcp_tx", sys.argv[1]) +module = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) +module.GATEWAY_PID_PATH = sys.argv[3] +os.mkfifo(module.GATEWAY_PID_PATH, 0o600) +signal.alarm(2) +try: + module._gateway_pid_record_candidate(os.geteuid()) +except PermissionError as error: + print(str(error)) +else: + raise SystemExit(9) +finally: + signal.alarm(0) +`, + [fifoPath], + ); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + expect(result.stdout).toContain("Hermes gateway PID record is unsafe"); + } finally { + fs.rmSync(temp, { recursive: true, force: true }); + } + }); + + it("requires the public relay and stable identity before acknowledging reload health", () => { + const result = runPython(` +import importlib.util, json, signal, sys, types +spec = importlib.util.spec_from_file_location("mcp_tx", sys.argv[1]) +module = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) + +statuses = { + module.GATEWAY_INTERNAL_PORT: 200, + module.GATEWAY_PUBLIC_PORT: 401, +} +ports = [] +class Connection: + def __init__(self, host, port, timeout): + if host != "127.0.0.1" or timeout != 2: + raise AssertionError("unexpected Hermes health endpoint") + self.port = port + ports.append(port) + def request(self, method, path): + if method != "GET" or path != "/health": + raise AssertionError("unexpected Hermes health request") + def getresponse(self): + status = statuses[self.port] + if isinstance(status, list): + status = status.pop(0) + return types.SimpleNamespace(status=status, read=lambda: b"") + def close(self): + pass + +module.http.client.HTTPConnection = Connection +ready = module._gateway_healthy() +statuses[module.GATEWAY_PUBLIC_PORT] = 503 +public_down = module._gateway_healthy() +statuses[module.GATEWAY_INTERNAL_PORT] = 503 +statuses[module.GATEWAY_PUBLIC_PORT] = 401 +internal_down = module._gateway_healthy() +health_ports = list(ports) + +ports.clear() +statuses[module.GATEWAY_INTERNAL_PORT] = 200 +statuses[module.GATEWAY_PUBLIC_PORT] = [503, 401, 401] +identities = iter(((1, 10), (2, 20), (2, 20), (3, 30), (3, 30), (3, 30))) +module._gateway_identity = lambda: next(identities) +signals = [] +module.os.kill = lambda pid, sent_signal: signals.append((pid, signal.Signals(sent_signal).name)) +module.time.monotonic = lambda: 0 +sleeps = [] +module.time.sleep = sleeps.append +reloaded = module.reload_gateway() +print(json.dumps({ + "ready": ready, + "public_down": public_down, + "internal_down": internal_down, + "health_ports": health_ports, + "reloaded": reloaded, + "reload_ports": ports, + "signals": signals, + "sleeps": sleeps, +})) +`); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + expect(JSON.parse(result.stdout)).toEqual({ + ready: true, + public_down: false, + internal_down: false, + health_ports: [18642, 8642, 18642, 8642, 18642], + reloaded: true, + reload_ports: [18642, 8642, 18642, 8642, 18642, 8642], + signals: [[1, "SIGUSR1"]], + sleeps: [1, 1], + }); + }); + + it("trusts the current real Hermes launcher and retained compatibility paths", () => { + const result = runPython(` +import importlib.util, json, sys +spec = importlib.util.spec_from_file_location("mcp_tx", sys.argv[1]) +module = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) +arguments = { + 1: [b"/usr/local/bin/hermes.real", b"gateway", b"run"], + 2: [b"/opt/hermes/.venv/bin/python", b"/usr/local/bin/hermes.real", b"gateway", b"run"], + 3: [b"/usr/local/lib/nemoclaw/hermes", b"gateway", b"run"], + 4: [b"/opt/hermes/.venv/bin/hermes", b"gateway", b"run"], + 5: [b"/usr/local/bin/hermes", b"gateway", b"run"], +} +module._process_arguments = lambda pid: arguments[pid] +print(json.dumps({str(pid): module._is_trusted_gateway_process(pid) for pid in arguments})) +`); + + expect(result.status, result.stderr).toBe(0); + expect(JSON.parse(result.stdout)).toEqual({ + "1": true, + "2": true, + "3": true, + "4": true, + "5": false, + }); + }); + + it("allows an ordinary same-UID sandbox exec to reload the trusted gateway", () => { + const result = runPython(` +import importlib.util, json, signal, sys, types +spec = importlib.util.spec_from_file_location("mcp_tx", sys.argv[1]) +module = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) + +sandbox_uid = 1000 +gateway_pid = 4242 +gateway_state = {"start_time": 99} +observed = { + "trusted_pids": [], +} +module.os.geteuid = lambda: sandbox_uid +module.os.lstat = lambda path: (_ for _ in ()).throw(FileNotFoundError(path)) +observed["entrypoint_uid"] = module.os.geteuid() +module.pwd.getpwnam = lambda name: (_ for _ in ()).throw( + AssertionError("same-UID reload must not resolve a separate gateway identity") +) + +snapshot = types.SimpleNamespace(mode=0o600, uid=sandbox_uid, gid=sandbox_uid) +guard = types.SimpleNamespace( + _read_text=lambda path: ("model: test\\n", snapshot), +) +module._load_guard = lambda: guard +def apply_transaction(action, payload): + observed["helper_uid"] = module.os.geteuid() + observed["action"] = action + return True +module.apply_transaction = apply_transaction + +gateway = types.ModuleType("gateway") +status = types.ModuleType("gateway.status") +status.get_running_pid = lambda cleanup_stale=False: gateway_pid +status.get_process_start_time = lambda pid: gateway_state["start_time"] +sys.modules["gateway"] = gateway +sys.modules["gateway.status"] = status + +def stat_gateway(path): + observed["gateway_owner_uid"] = sandbox_uid + observed["gateway_check_uid"] = module.os.geteuid() + return types.SimpleNamespace(st_uid=sandbox_uid) +module.os.stat = stat_gateway +def trusted_gateway(pid): + observed["trusted_pids"].append(pid) + return True +module._is_trusted_gateway_process = trusted_gateway +module._gateway_has_managed_parent = lambda pid: True +def signal_gateway(pid, sent_signal): + observed["signal_uid"] = module.os.geteuid() + observed["signal_pid"] = pid + observed["signal_name"] = signal.Signals(sent_signal).name + gateway_state["start_time"] = 100 +module.os.kill = signal_gateway +def gateway_health_phase(deadline=None): + observed["health_uid"] = module.os.geteuid() + return True, "waiting-for-stable-replacement-identity" +module._gateway_health_phase = gateway_health_phase + +payload = { + "server": "fake", + "url": "https://mcp.example.test/mcp", + "headers": {"Authorization": "Bearer openshell:resolve:env:FAKE_TOKEN"}, + "replace_existing": False, +} +sys.argv = [sys.argv[1], "add", "--payload", json.dumps(payload)] +exit_code = module.main() +observed["exit_code"] = exit_code +print(json.dumps(observed, sort_keys=True)) +`); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + const lines = result.stdout.trim().split("\n"); + expect(JSON.parse(lines[0] ?? "{}")).toEqual({ + changed: true, + ok: true, + reloaded: true, + }); + expect(JSON.parse(lines[1] ?? "{}")).toEqual({ + action: "add", + entrypoint_uid: 1000, + exit_code: 0, + gateway_check_uid: 1000, + gateway_owner_uid: 1000, + health_uid: 1000, + helper_uid: 1000, + signal_name: "SIGUSR1", + signal_pid: 4242, + signal_uid: 1000, + trusted_pids: [4242, 4242, 4242, 4242, 4242], + }); + }); + + it("repairs and verifies strict and compatibility hashes on an unchanged retry", () => { + const temp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-hermes-mcp-tx-")); + const hermesDir = path.join(temp, ".hermes"); + const configPath = path.join(hermesDir, "config.yaml"); + const envPath = path.join(hermesDir, ".env"); + const strictHash = path.join(temp, "hermes.config-hash"); + const compatHash = path.join(hermesDir, ".config-hash"); + fs.mkdirSync(hermesDir); + const config = `model: test +mcp_servers: + fake: + url: https://mcp.example.test/mcp + enabled: true + timeout: 120 + connect_timeout: 60 + tools: + resources: true + prompts: true + headers: + Authorization: Bearer openshell:resolve:env:FAKE_TOKEN +`; + fs.writeFileSync(configPath, config, { mode: 0o600 }); + fs.writeFileSync(envPath, "HERMES_TEST=1\n", { mode: 0o600 }); + fs.writeFileSync(strictHash, "stale\n", { mode: 0o600 }); + fs.writeFileSync(compatHash, "different-stale\n", { mode: 0o600 }); + + try { + const result = runPython( + ` +import importlib.util, json, os, sys +spec = importlib.util.spec_from_file_location("mcp_tx", sys.argv[1]) +module = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) +module.GUARD_PATH = sys.argv[2] +module.HERMES_DIR = sys.argv[3] +module.CONFIG_PATH = os.path.join(module.HERMES_DIR, "config.yaml") +module.STRICT_HASH_PATH = sys.argv[4] +module.os.geteuid = lambda: 0 +module._require_lifecycle_identity = lambda: None +module._assert_mutable_snapshot = lambda snapshot: None +changed = module.apply_transaction("add", { + "server": "fake", + "url": "https://mcp.example.test/mcp", + "headers": {"Authorization": "Bearer openshell:resolve:env:FAKE_TOKEN"}, + "replace_existing": True, +}) +print(json.dumps({"changed": changed})) +`, + [hermesDir, strictHash], + ); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + expect(result.stdout).toContain('"changed": false'); + const strict = fs.readFileSync(strictHash, "utf8"); + const compat = fs.readFileSync(compatHash, "utf8"); + expect(strict).toBe(compat); + expect(strict).toContain(crypto.createHash("sha256").update(config).digest("hex")); + expect(strict).toContain( + crypto.createHash("sha256").update(fs.readFileSync(envPath)).digest("hex"), + ); + } finally { + fs.rmSync(temp, { recursive: true, force: true }); + } + }); + + it("rejects ordinary exec in a root-separated Hermes topology", () => { + const result = runPython(` +import importlib.util, json, stat, sys, types +spec = importlib.util.spec_from_file_location("mcp_tx", sys.argv[1]) +module = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) +module.os.geteuid = lambda: 1000 +module.os.lstat = lambda path: types.SimpleNamespace(st_mode=stat.S_IFREG | 0o444, st_uid=0) +payload = { + "server": "fake", + "url": "https://mcp.example.test/mcp", + "headers": {"Authorization": "Bearer openshell:resolve:env:FAKE_TOKEN"}, + "replace_existing": False, +} +errors = [] +for operation in (lambda: module.execute("add", payload), module.probe): + try: + operation() + except PermissionError as error: + errors.append(str(error)) +if len(errors) != 2: + raise SystemExit(9) +print(json.dumps(errors)) +`); + + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toContain("requires a same-uid OpenShell sandbox runtime"); + }); + + it("rejects a same-UID bare gateway before mutating managed MCP state", () => { + const result = runPython(` +import importlib.util, json, sys +spec = importlib.util.spec_from_file_location("mcp_tx", sys.argv[1]) +module = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) +module.os.geteuid = lambda: 1000 +module.os.lstat = lambda path: (_ for _ in ()).throw(FileNotFoundError(path)) +module._gateway_identity = lambda: (123, 456) +module._gateway_has_managed_parent = lambda pid: False +calls = [] +module.apply_transaction_and_reload = lambda action, payload: calls.append((action, payload)) +payload = { + "server": "fake", + "url": "https://mcp.example.test/mcp", + "headers": {"Authorization": "Bearer openshell:resolve:env:FAKE_TOKEN"}, + "replace_existing": False, +} +errors = [] +for operation in (lambda: module.execute("add", payload), module.probe): + try: + operation() + except RuntimeError as error: + errors.append(str(error)) +if calls or len(errors) != 2: + raise SystemExit(9) +print(json.dumps(errors)) +`); + + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toContain("not running under the managed service lifecycle"); + }); + + it("does not mistake a one-shot nemoclaw-start wrapper for the service manager", () => { + const result = runPython(` +import importlib.util, json, sys +spec = importlib.util.spec_from_file_location("mcp_tx", sys.argv[1]) +module = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) +arguments = { + 1: [b"bash", module.SERVICE_MANAGER_PATH], + 2: [b"bash", module.SERVICE_MANAGER_PATH, b"true"], + 3: [b"bash", b"-c", b"text mentioning /usr/local/bin/nemoclaw-start"], +} +module._process_arguments = lambda pid: arguments[pid] +print(json.dumps({str(pid): module._is_service_manager_process(pid) for pid in arguments})) +`); + + expect(result.status, result.stderr).toBe(0); + expect(JSON.parse(result.stdout)).toEqual({ + "1": true, + "2": false, + "3": false, + }); + }); + + it("runs a one-shot mutation through the stock OpenShell exec topology", () => { + const result = runPython(` +import importlib.util, json, sys +spec = importlib.util.spec_from_file_location("mcp_tx", sys.argv[1]) +module = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) +module.os.geteuid = lambda: 1000 +module.os.lstat = lambda path: (_ for _ in ()).throw(FileNotFoundError(path)) +module._gateway_identity = lambda: (123, 456) +module._gateway_has_managed_parent = lambda pid: True +module.apply_transaction_and_reload = lambda action, payload: { + "ok": True, "changed": True, "reloaded": True +} +result = module.execute("add", { + "server": "fake", + "url": "https://mcp.example.test/mcp", + "headers": {"Authorization": "Bearer openshell:resolve:env:FAKE_TOKEN"}, + "replace_existing": False, +}) +print(json.dumps(result, sort_keys=True)) +`); + + expect(result.status, result.stderr).toBe(0); + expect(JSON.parse(result.stdout)).toEqual({ + changed: true, + ok: true, + reloaded: true, + }); + }); + + it("probes the same-UID helper without mutating config", () => { + const result = runPython(` +import importlib.util, json, sys +spec = importlib.util.spec_from_file_location("mcp_tx", sys.argv[1]) +module = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) +module.os.geteuid = lambda: 1000 +module.os.lstat = lambda path: (_ for _ in ()).throw(FileNotFoundError(path)) +module._gateway_identity = lambda: (123, 456) +module._gateway_has_managed_parent = lambda pid: True +print(json.dumps(module.probe(), sort_keys=True)) +`); + + expect(result.status, result.stderr).toBe(0); + expect(JSON.parse(result.stdout)).toEqual({ ok: true }); + }); + + it("restores config and hashes after both desired-config reload signals fail", () => { + const temp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-hermes-mcp-rollback-")); + const hermesDir = path.join(temp, ".hermes"); + const configPath = path.join(hermesDir, "config.yaml"); + const envPath = path.join(hermesDir, ".env"); + const compatHash = path.join(hermesDir, ".config-hash"); + const strictHash = path.join(temp, "strict-hash"); + const config = "model: test\n"; + const env = "HERMES_TEST=1\n"; + const originalHash = `${crypto.createHash("sha256").update(config).digest("hex")} ${configPath}\n${crypto.createHash("sha256").update(env).digest("hex")} ${envPath}\n`; + fs.mkdirSync(hermesDir); + fs.writeFileSync(configPath, config, { mode: 0o600 }); + fs.writeFileSync(envPath, env, { mode: 0o600 }); + fs.writeFileSync(compatHash, originalHash, { mode: 0o600 }); + fs.writeFileSync(strictHash, originalHash, { mode: 0o600 }); + + try { + const result = runPython( + ` +import importlib.util, json, os, signal, sys +spec = importlib.util.spec_from_file_location("mcp_tx", sys.argv[1]) +module = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) +module.GUARD_PATH = sys.argv[2] +module.HERMES_DIR = sys.argv[3] +module.CONFIG_PATH = os.path.join(module.HERMES_DIR, "config.yaml") +module.STRICT_HASH_PATH = sys.argv[4] +module.os.geteuid = lambda: 0 +module._assert_mutable_snapshot = lambda snapshot: None +module.RELOAD_TIMEOUT_SECONDS = 4 +clock = {"now": 0} +gateway = {"identity": (4242, 99)} +signals = [] +module._gateway_identity = lambda: gateway["identity"] +module._gateway_has_managed_parent = lambda pid: True +module._gateway_health_phase = lambda deadline=None: ( + True, "waiting-for-stable-replacement-identity" +) +module.time.monotonic = lambda: clock["now"] +module.time.sleep = lambda seconds: clock.__setitem__("now", clock["now"] + seconds) +def signal_gateway(pid, sent_signal): + signals.append((pid, signal.Signals(sent_signal).name)) + if len(signals) == 3: + gateway["identity"] = (4243, 100) +module.os.kill = signal_gateway +try: + module.apply_transaction_and_reload("add", { + "server": "fake", + "url": "https://mcp.example.test/mcp", + "headers": {"Authorization": "Bearer openshell:resolve:env:FAKE_TOKEN"}, + "replace_existing": False, + }) +except RuntimeError as error: + print(json.dumps({"error": str(error), "signals": signals})) +else: + raise SystemExit(9) +`, + [hermesDir, strictHash], + ); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + expect(JSON.parse(result.stdout)).toMatchObject({ + signals: [ + [4242, "SIGUSR1"], + [4242, "SIGUSR1"], + [4242, "SIGUSR1"], + ], + }); + expect(result.stdout).toContain("re-kick sent: yes"); + expect(fs.readFileSync(configPath, "utf8")).toBe(config); + expect(fs.readFileSync(compatHash, "utf8")).toBe(originalHash); + expect(fs.readFileSync(strictHash, "utf8")).toBe(originalHash); + } finally { + fs.rmSync(temp, { recursive: true, force: true }); + } + }); +}); diff --git a/test/hermes-mcp-force-cleanup.test.ts b/test/hermes-mcp-force-cleanup.test.ts new file mode 100644 index 00000000000..7144658dbea --- /dev/null +++ b/test/hermes-mcp-force-cleanup.test.ts @@ -0,0 +1,71 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +const TRANSACTION = path.resolve( + import.meta.dirname, + "..", + "agents/hermes/mcp-config-transaction.py", +); + +describe("Hermes MCP forced cleanup", () => { + it("removes legacy percent-path entries by validated server name", () => { + const result = spawnSync( + "python3", + [ + "-c", + ` +import importlib.util, json, sys +spec = importlib.util.spec_from_file_location("mcp_tx", sys.argv[1]) +module = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) +results = [] +for suffix in ("%", "%GG", "%2"): + url = f"https://mcp.example.test/{suffix}" + payload = { + "server": "legacy", + "url": url, + "headers": {"Authorization": "Bearer openshell:resolve:env:LEGACY_TOKEN"}, + "force": True, + } + module._validate_payload("remove", payload) + updated, changed = module._mutate( + {"mcp_servers": {"legacy": {"url": url}, "other": {"url": "https://other.test/mcp"}}}, + "remove", + payload, + ) + try: + module._validate_payload("remove", {**payload, "force": False}) + except ValueError: + non_force_rejected = True + else: + non_force_rejected = False + results.append({ + "changed": changed, + "legacy_removed": "legacy" not in updated["mcp_servers"], + "other_preserved": "other" in updated["mcp_servers"], + "non_force_rejected": non_force_rejected, + }) +print(json.dumps(results)) +`, + TRANSACTION, + ], + { encoding: "utf8" }, + ); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + expect(JSON.parse(result.stdout)).toEqual( + Array.from({ length: 3 }, () => ({ + changed: true, + legacy_removed: true, + other_preserved: true, + non_force_rejected: true, + })), + ); + }); +}); diff --git a/test/hermes-mcp-reload-convergence.test.ts b/test/hermes-mcp-reload-convergence.test.ts new file mode 100644 index 00000000000..c1c3c6a7191 --- /dev/null +++ b/test/hermes-mcp-reload-convergence.test.ts @@ -0,0 +1,323 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +const TRANSACTION = path.resolve( + import.meta.dirname, + "..", + "agents/hermes/mcp-config-transaction.py", +); +const GUARD = path.resolve(import.meta.dirname, "..", "agents/hermes/runtime-config-guard.py"); + +function runPython(source: string, args: string[] = []) { + return spawnSync("python3", ["-c", source, TRANSACTION, GUARD, ...args], { + encoding: "utf8", + }); +} + +describe("Hermes managed MCP reload convergence", () => { + it("re-kicks one revalidated gateway identity within the original reload deadline", () => { + const result = runPython(` +import importlib.util, json, signal, sys +spec = importlib.util.spec_from_file_location("mcp_tx", sys.argv[1]) +module = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) + +module.RELOAD_TIMEOUT_SECONDS = 4 +clock = {"now": 0} +gateway = {"identity": (4242, 99)} +signals = [] +sleeps = [] +module._gateway_identity = lambda: gateway["identity"] +module._gateway_has_managed_parent = lambda pid: True +module._gateway_health_phase = lambda deadline=None: ( + (True, "waiting-for-stable-replacement-identity") + if len(signals) >= 2 + else (False, "waiting-for-internal-health-on-18642") +) +module.time.monotonic = lambda: clock["now"] +def sleep(seconds): + sleeps.append(seconds) + clock["now"] += seconds +module.time.sleep = sleep +def signal_gateway(pid, sent_signal): + signals.append((pid, signal.Signals(sent_signal).name)) + if len(signals) == 1: + gateway["identity"] = (4243, 100) + elif len(signals) == 2: + gateway["identity"] = (4244, 101) +module.os.kill = signal_gateway + +print(json.dumps({ + "reloaded": module.reload_gateway(), + "signals": signals, + "sleeps": sleeps, + "elapsed": clock["now"], +})) +`); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + expect(JSON.parse(result.stdout)).toEqual({ + elapsed: 3, + reloaded: true, + signals: [ + [4242, "SIGUSR1"], + [4243, "SIGUSR1"], + ], + sleeps: [1, 1, 1], + }); + }); + + it("does not re-kick without a currently trusted gateway identity", () => { + const result = runPython(` +import importlib.util, json, signal, sys +spec = importlib.util.spec_from_file_location("mcp_tx", sys.argv[1]) +module = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) + +module.RELOAD_TIMEOUT_SECONDS = 4 +clock = {"now": 0} +identity_calls = {"count": 0} +signals = [] +def identity(): + identity_calls["count"] += 1 + return (4242, 99) if identity_calls["count"] == 1 else None +module._gateway_identity = identity +module.time.monotonic = lambda: clock["now"] +module.time.sleep = lambda seconds: clock.__setitem__("now", clock["now"] + seconds) +module.os.kill = lambda pid, sent_signal: signals.append( + (pid, signal.Signals(sent_signal).name) +) +try: + module.reload_gateway() +except TimeoutError as error: + print(json.dumps({"error": str(error), "signals": signals})) +else: + raise SystemExit(9) +`); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + expect(JSON.parse(result.stdout)).toEqual({ + error: + "Hermes gateway did not complete its managed MCP reload (last safe phase: waiting-for-replacement-identity; re-kick attempted: no; re-kick sent: no)", + signals: [[4242, "SIGUSR1"]], + }); + }); + + it("attempts a vanished re-kick target only once", () => { + const result = runPython(` +import importlib.util, json, signal, sys +spec = importlib.util.spec_from_file_location("mcp_tx", sys.argv[1]) +module = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) + +module.RELOAD_TIMEOUT_SECONDS = 5 +clock = {"now": 0} +attempts = [] +module._gateway_identity = lambda: (4242, 99) +module._gateway_has_managed_parent = lambda pid: True +module.time.monotonic = lambda: clock["now"] +module.time.sleep = lambda seconds: clock.__setitem__("now", clock["now"] + seconds) +def signal_gateway(pid, sent_signal): + attempts.append((pid, signal.Signals(sent_signal).name)) + if len(attempts) == 2: + raise ProcessLookupError(pid) +module.os.kill = signal_gateway +try: + module.reload_gateway() +except TimeoutError as error: + print(json.dumps({"attempts": attempts, "error": str(error)})) +else: + raise SystemExit(9) +`); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + expect(JSON.parse(result.stdout)).toEqual({ + attempts: [ + [4242, "SIGUSR1"], + [4242, "SIGUSR1"], + ], + error: + "Hermes gateway did not complete its managed MCP reload (last safe phase: waiting-for-replacement-identity; re-kick attempted: yes; re-kick sent: no)", + }); + }); + + it("reports whether reload stopped at internal health, public relay, or stable identity", () => { + const result = runPython(` +import importlib.util, json, sys, types +spec = importlib.util.spec_from_file_location("mcp_tx", sys.argv[1]) +module = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) + +statuses = { + module.GATEWAY_INTERNAL_PORT: 503, + module.GATEWAY_PUBLIC_PORT: 401, +} +class Connection: + def __init__(self, host, port, timeout): + self.port = port + def request(self, method, path): + pass + def getresponse(self): + return types.SimpleNamespace(status=statuses[self.port], read=lambda: b"") + def close(self): + pass +module.http.client.HTTPConnection = Connection + +internal = module._gateway_health_phase() +statuses[module.GATEWAY_INTERNAL_PORT] = 200 +statuses[module.GATEWAY_PUBLIC_PORT] = 503 +public = module._gateway_health_phase() +statuses[module.GATEWAY_PUBLIC_PORT] = 401 +stable = module._gateway_health_phase() +print(json.dumps({"internal": internal, "public": public, "stable": stable})) +`); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + expect(JSON.parse(result.stdout)).toEqual({ + internal: [false, "waiting-for-internal-health-on-18642"], + public: [false, "waiting-for-public-relay-health-on-8642"], + stable: [true, "waiting-for-stable-replacement-identity"], + }); + }); + + it("does not re-kick after a health probe exhausts the shared deadline", () => { + const result = runPython(` +import importlib.util, json, signal, sys +spec = importlib.util.spec_from_file_location("mcp_tx", sys.argv[1]) +module = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) + +module.RELOAD_TIMEOUT_SECONDS = 4 +clock = {"now": 0} +identity_calls = {"count": 0} +signals = [] +def identity(): + identity_calls["count"] += 1 + return (4242, 99) if identity_calls["count"] == 1 else (4243, 100) +def health_phase(deadline=None): + clock["now"] = deadline + return False, "waiting-for-internal-health-on-18642" +module._gateway_identity = identity +module._gateway_health_phase = health_phase +module._gateway_has_managed_parent = lambda pid: (_ for _ in ()).throw( + AssertionError("deadline exhaustion must precede re-kick authority checks") +) +module.time.monotonic = lambda: clock["now"] +module.time.sleep = lambda seconds: (_ for _ in ()).throw( + AssertionError("deadline exhaustion must not sleep") +) +module.os.kill = lambda pid, sent_signal: signals.append( + (pid, signal.Signals(sent_signal).name) +) +try: + module.reload_gateway() +except TimeoutError as error: + print(json.dumps({"error": str(error), "signals": signals})) +else: + raise SystemExit(9) +`); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + expect(JSON.parse(result.stdout)).toEqual({ + error: + "Hermes gateway did not complete its managed MCP reload (last safe phase: waiting-for-internal-health-on-18642; re-kick attempted: no; re-kick sent: no)", + signals: [[4242, "SIGUSR1"]], + }); + }); + + it("reports the furthest safe phase reached when reload exhausts its deadline", () => { + const result = runPython(` +import importlib.util, json, signal, sys +spec = importlib.util.spec_from_file_location("mcp_tx", sys.argv[1]) +module = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) + +def run_case(name): + module.RELOAD_TIMEOUT_SECONDS = 4 + clock = {"now": 0} + first_identity = {"pending": True} + churn = {"count": 0} + signals = [] + + def identity(): + if first_identity["pending"]: + first_identity["pending"] = False + return (4242, 99) + if name == "replacement": + return None + if name == "internal" and clock["now"] >= 3: + return None + if name == "stable": + churn["count"] += 1 + return (4243, 100) if churn["count"] % 2 else (4244, 101) + return (4243, 100) + + phases = { + "internal": (False, "waiting-for-internal-health-on-18642"), + "public": (False, "waiting-for-public-relay-health-on-8642"), + "stable": (True, "waiting-for-stable-replacement-identity"), + } + module._gateway_identity = identity + module._gateway_has_managed_parent = lambda pid: True + module._gateway_health_phase = lambda deadline=None: phases[name] + module.time.monotonic = lambda: clock["now"] + module.time.sleep = lambda seconds: clock.__setitem__("now", clock["now"] + seconds) + module.os.kill = lambda pid, sent_signal: signals.append( + (pid, signal.Signals(sent_signal).name) + ) + try: + module.reload_gateway() + except TimeoutError as error: + return {"error": str(error), "signals": signals} + raise AssertionError("reload unexpectedly succeeded") + +print(json.dumps({name: run_case(name) for name in ( + "replacement", "internal", "public", "stable" +)})) +`); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + expect(JSON.parse(result.stdout)).toEqual({ + replacement: { + error: + "Hermes gateway did not complete its managed MCP reload (last safe phase: waiting-for-replacement-identity; re-kick attempted: no; re-kick sent: no)", + signals: [[4242, "SIGUSR1"]], + }, + internal: { + error: + "Hermes gateway did not complete its managed MCP reload (last safe phase: waiting-for-internal-health-on-18642; re-kick attempted: yes; re-kick sent: yes)", + signals: [ + [4242, "SIGUSR1"], + [4243, "SIGUSR1"], + ], + }, + public: { + error: + "Hermes gateway did not complete its managed MCP reload (last safe phase: waiting-for-public-relay-health-on-8642; re-kick attempted: yes; re-kick sent: yes)", + signals: [ + [4242, "SIGUSR1"], + [4243, "SIGUSR1"], + ], + }, + stable: { + error: + "Hermes gateway did not complete its managed MCP reload (last safe phase: waiting-for-stable-replacement-identity; re-kick attempted: yes; re-kick sent: yes)", + signals: [ + [4242, "SIGUSR1"], + [4243, "SIGUSR1"], + ], + }, + }); + }); +}); diff --git a/test/hermes-mcp-runtime-capability.test.ts b/test/hermes-mcp-runtime-capability.test.ts new file mode 100644 index 00000000000..8562207f118 --- /dev/null +++ b/test/hermes-mcp-runtime-capability.test.ts @@ -0,0 +1,85 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; + +const ROOT = path.resolve(import.meta.dirname, ".."); +const HERMES_DOCKERFILE = path.join(ROOT, "agents", "hermes", "Dockerfile"); + +function dockerRunCommandBetween( + dockerfile: string, + startMarker: string, + endMarker: string, +): string { + const start = dockerfile.indexOf(startMarker); + const end = dockerfile.indexOf(endMarker, start); + expect(start, `Expected Dockerfile start marker ${startMarker}`).toBeGreaterThanOrEqual(0); + expect(end, `Expected Dockerfile end marker ${endMarker}`).toBeGreaterThan(start); + const runIndex = dockerfile.indexOf("RUN ", start); + expect(runIndex, `Expected RUN instruction after ${startMarker}`).toBeGreaterThanOrEqual(start); + expect(runIndex, `Expected RUN instruction before ${endMarker}`).toBeLessThan(end); + const blockLines = dockerfile.slice(runIndex, end).split("\n"); + const runEnd = blockLines.findIndex((line) => !line.trimEnd().endsWith("\\")); + expect(runEnd, `Expected complete RUN instruction before ${endMarker}`).toBeGreaterThanOrEqual(0); + const runLines = blockLines.slice(0, runEnd + 1); + return runLines + .join("\n") + .trim() + .replace(/^RUN\s+/, "") + .replace(/\\\n/g, " "); +} + +function runHermesMcpClientImportValidation({ + mcpAvailable, + httpAvailable, +}: { + mcpAvailable: boolean; + httpAvailable: boolean; +}) { + const dockerfile = fs.readFileSync(HERMES_DOCKERFILE, "utf-8"); + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-hermes-mcp-runtime-")); + const toolsDir = path.join(tmp, "tools"); + const command = dockerRunCommandBetween( + dockerfile, + "# Managed MCP requires the packaged Hermes client surface", + "# Published base images can lag Dockerfile.base", + ).replaceAll("/opt/hermes/.venv/bin/python", "python3"); + try { + fs.mkdirSync(toolsDir, { recursive: true }); + fs.writeFileSync(path.join(tmp, "mcp.py"), "# MCP SDK fixture\n"); + fs.writeFileSync(path.join(toolsDir, "__init__.py"), ""); + fs.writeFileSync( + path.join(toolsDir, "mcp_tool.py"), + `_MCP_AVAILABLE = ${mcpAvailable ? "True" : "False"}\n` + + `_MCP_HTTP_AVAILABLE = ${httpAvailable ? "True" : "False"}\n`, + ); + return spawnSync("bash", ["-c", command], { + encoding: "utf-8", + env: { ...process.env, PYTHONPATH: tmp }, + timeout: 5000, + }); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } +} + +describe("Hermes managed MCP client import capability", () => { + it("fails the final image build without packaged Streamable HTTP client support", () => { + const complete = runHermesMcpClientImportValidation({ + mcpAvailable: true, + httpAvailable: true, + }); + expect(complete.status, complete.stderr).toBe(0); + + const missingHttp = runHermesMcpClientImportValidation({ + mcpAvailable: true, + httpAvailable: false, + }); + expect(missingHttp.status).toBe(1); + expect(missingHttp.stderr).toContain("Hermes MCP Streamable HTTP runtime is unavailable"); + }); +}); diff --git a/test/hermes-mcp-shields-order.test.ts b/test/hermes-mcp-shields-order.test.ts new file mode 100644 index 00000000000..5d0e15ba679 --- /dev/null +++ b/test/hermes-mcp-shields-order.test.ts @@ -0,0 +1,146 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +describe("Hermes MCP shields ordering", () => { + it("refuses add, resumed add, restart, and remove before external mutation", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-hermes-mcp-shields-")); + const script = String.raw` +process.env.HOME = ${JSON.stringify(home)}; +process.env.GITHUB_TOKEN = "host-only-secret"; +const registry = require("./src/lib/state/registry.js"); +const globalActions = require("./src/lib/actions/global.js"); +const gatewayRuntime = require("./src/lib/gateway-runtime-action.js"); +const policies = require("./src/lib/policy/index.js"); +const processRecovery = require("./src/lib/actions/sandbox/process-recovery.js"); +const shields = require("./src/lib/shields/index.js"); + +const mutations = []; +const providerId = "11111111-2222-4333-8444-555555555555"; +shields.isShieldsDown = () => false; +gatewayRuntime.recoverNamedGatewayRuntime = async () => { + mutations.push("gateway:recover"); + return { + recovered: true, + attempted: false, + before: { state: "healthy_named" }, + after: { state: "healthy_named" }, + }; +}; +globalActions.runOpenshellProviderCommand = (args) => { + const command = args.join(" "); + if (command === "status --output json") { + return { status: 0, stdout: JSON.stringify({ gateway: "nemoclaw" }), stderr: "" }; + } + if (args[0] === "provider" && args[1] === "get") { + return { status: 1, stdout: "", stderr: "Provider not found" }; + } + if (args[0] === "sandbox" && args[1] === "provider" && args[2] === "list") { + return { status: 0, stdout: "No providers attached.\n", stderr: "" }; + } + mutations.push("openshell:" + command); + return { status: 0, stdout: "", stderr: "" }; +}; +policies.getPresetContentGatewayState = () => "match"; +policies.applyPresetContent = () => { mutations.push("policy:apply"); return true; }; +policies.removePreset = () => { mutations.push("policy:remove"); return true; }; +processRecovery.executeSandboxCommand = (_sandboxName, command) => { + mutations.push("adapter:" + command); + return { status: 0, stdout: '{"ok":true}\n', stderr: "" }; +}; + +const bridge = require("./src/lib/actions/sandbox/mcp-bridge.js"); +const makeEntry = (server, addState) => ({ + server, + agent: "hermes", + adapter: "hermes-config", + url: "https://8.8.8.8/mcp", + env: ["GITHUB_TOKEN"], + providerName: "provider-" + server, + providerId, + policyName: "mcp-bridge-" + server, + addedAt: "2026-06-30T00:00:00.000Z", + ...(addState ? { addState } : {}), +}); +const register = (name, entry) => { + registry.registerSandbox({ + name, + agent: "hermes", + gatewayName: "nemoclaw", + ...(entry ? { mcp: { bridges: { [entry.server]: entry } } } : {}), + }); + if (entry) { + registry.addCustomPolicy(name, { + name: entry.policyName, + content: bridge.buildMcpBridgePolicyYaml( + entry.server, + entry.url, + "hermes-config", + ["8.8.8.8"], + ), + sourcePath: "generated:nemoclaw-mcp-bridge", + }); + } +}; +const messages = []; +const capture = async (operation) => { + try { await operation(); } + catch (error) { messages.push(error instanceof Error ? error.message : String(error)); } +}; + +(async () => { + register("fresh", null); + await capture(() => bridge.addMcpBridge("fresh", { + server: "github", + url: "https://8.8.8.8/mcp", + env: [{ name: "GITHUB_TOKEN" }], + })); + const freshManifest = registry.getSandbox("fresh")?.mcp; + + const resumed = makeEntry("resumed", "preflighted"); + register("resume", resumed); + await capture(() => bridge.addMcpBridge("resume", { + server: resumed.server, + url: resumed.url, + env: [{ name: "GITHUB_TOKEN" }], + })); + + const restarted = makeEntry("restarted"); + register("restart", restarted); + await capture(() => bridge.restartMcpBridge("restart", restarted.server)); + + const removed = makeEntry("removed"); + register("remove", removed); + await capture(() => bridge.removeMcpBridge("remove", removed.server)); + + process.stdout.write(JSON.stringify({ messages, mutations, freshManifest })); +})().catch((error) => { console.error(error); process.exit(1); }); +`; + const result = spawnSync(process.execPath, ["-e", script], { + cwd: process.cwd(), + encoding: "utf8", + env: { ...process.env, HOME: home }, + timeout: 30_000, + }); + fs.rmSync(home, { recursive: true, force: true }); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + const payload = JSON.parse(result.stdout) as { + messages: string[]; + mutations: string[]; + freshManifest?: unknown; + }; + expect(payload.messages).toHaveLength(4); + for (const message of payload.messages) { + expect(message).toContain("has shields up or an unreadable shields posture"); + } + expect(payload.mutations).toEqual([]); + expect(payload.freshManifest).toBeUndefined(); + }); +}); diff --git a/test/hermes-mcp-startup-probe.test.ts b/test/hermes-mcp-startup-probe.test.ts new file mode 100644 index 00000000000..b7a1b0ff187 --- /dev/null +++ b/test/hermes-mcp-startup-probe.test.ts @@ -0,0 +1,222 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; + +import { describe, expect, it } from "vitest"; + +type ProbeResult = { status: number; stdout: string; stderr: string }; +type SupervisorResult = ProbeResult | null; + +function runHermesProbe( + results: ProbeResult[], + shieldsDown = true, + supervisorResults: SupervisorResult[] = [], +) { + const script = String.raw` +const globalActions = require("./src/lib/actions/global.js"); +const processRecovery = require("./src/lib/actions/sandbox/process-recovery.js"); +const wait = require("./src/lib/core/wait.js"); +const shields = require("./src/lib/shields/index.js"); +const results = ${JSON.stringify(results)}; +const supervisorResults = ${JSON.stringify(supervisorResults)}; +let calls = 0; +let recoveryCalls = 0; +const recoveryActions = []; +globalActions.runOpenshellProviderCommand = () => results[calls++]; +processRecovery.executeGatewaySupervisorAction = (_sandbox, action, timeout) => { + recoveryActions.push({ action, timeout }); + return supervisorResults[recoveryCalls++] ?? null; +}; +wait.waitUntil = (condition, optionsOrTimeout) => { + const maxAttempts = typeof optionsOrTimeout === "object" + ? (optionsOrTimeout.maxAttempts ?? Number.POSITIVE_INFINITY) + : Number.POSITIVE_INFINITY; + let attempts = 0; + while (calls < results.length && attempts < maxAttempts) { + attempts += 1; + if (condition()) return true; + } + return false; +}; +shields.isShieldsDown = () => ${JSON.stringify(shieldsDown)}; +const adapters = require("./src/lib/actions/sandbox/mcp-bridge-adapters.js"); +let message = ""; +try { + adapters.assertAgentMcpMutationRuntimeCapability("hermes-box", "hermes-config"); +} catch (error) { + message = error instanceof Error ? error.message : String(error); +} +process.stdout.write(JSON.stringify({ calls, recoveryActions, message })); +`; + const result = spawnSync(process.execPath, ["-e", script], { + cwd: process.cwd(), + encoding: "utf8", + env: process.env, + timeout: 30_000, + }); + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + return JSON.parse(result.stdout) as { + calls: number; + recoveryActions: Array<{ action: string; timeout: number }>; + message: string; + }; +} + +const starting: ProbeResult = { + status: 1, + stdout: "", + stderr: "Hermes gateway is not running for managed MCP reload", +}; +const ready: ProbeResult = { + status: 0, + stdout: '{"ok":true}\n', + stderr: "", +}; +const recovered: SupervisorResult = { + status: 0, + stdout: `v1 ${"a".repeat(64)} complete ok 0 4242\nGATEWAY_PID=4242`, + stderr: "", +}; + +describe("Hermes managed MCP startup probe", () => { + it("refuses shields-up config before invoking the sandbox helper", () => { + const result = runHermesProbe([ready], false); + + expect(result.calls).toBe(0); + expect(result.recoveryActions).toEqual([]); + expect(result.message).toContain("has shields up or an unreadable shields posture"); + expect(result.message).toContain("nemohermes hermes-box shields down"); + }); + + it("retries only the exact transient gateway-starting result", () => { + expect(runHermesProbe([starting, ready])).toEqual({ + calls: 2, + recoveryActions: [], + message: "", + }); + }); + + it("does not recover when the third exact startup probe is ready", () => { + expect(runHermesProbe([starting, starting, ready])).toEqual({ + calls: 3, + recoveryActions: [], + message: "", + }); + }); + + it("uses one host-authenticated recovery after repeated exact not-ready probes", () => { + expect(runHermesProbe([starting, starting, starting, ready], true, [recovered])).toEqual({ + calls: 4, + recoveryActions: [{ action: "recover", timeout: 210_000 }], + message: "", + }); + }); + + it("keeps the fresh helper wait when privileged recovery is unavailable", () => { + expect(runHermesProbe([starting, starting, starting, ready])).toEqual({ + calls: 4, + recoveryActions: [{ action: "recover", timeout: 210_000 }], + message: "", + }); + }); + + it("does not treat controller success as transaction-helper readiness", () => { + const result = runHermesProbe([starting, starting, starting, starting, starting], true, [ + recovered, + ]); + + expect(result.calls).toBe(5); + expect(result.recoveryActions).toEqual([{ action: "recover", timeout: 210_000 }]); + expect(result.message).toContain("after managed gateway recovery"); + }); + + it.each([ + "GATEWAY_CONFIG_HASH_MISMATCH", + "SUPERVISOR_REBUILD_REQUIRED", + "SUPERVISOR_UNSAFE_CONTROL_DIR", + "SUPERVISOR_INVALID_STATUS", + "GATEWAY_HEALTH_TIMEOUT", + "SUPERVISOR_TIMEOUT", + "SUPERVISOR_BUSY", + ])("fails typed managed-recovery integrity refusal %s without another sandbox probe", (marker) => { + const result = runHermesProbe([starting, starting, starting, ready], true, [ + { status: 1, stdout: "", stderr: marker }, + ]); + + expect(result.calls).toBe(3); + expect(result.recoveryActions).toEqual([{ action: "recover", timeout: 210_000 }]); + expect(result.message).toContain("managed gateway recovery failed before MCP mutation"); + expect(result.message).toContain(marker); + }); + + it.each([ + { + label: "non-numeric PID", + result: { status: 0, stdout: "GATEWAY_PID=garbage", stderr: "" }, + }, + { + label: "failure output beside a completion", + result: { ...recovered!, stderr: "SUPERVISOR_UNSAFE_CONTROL_DIR" }, + }, + { + label: "failure status beside a completion", + result: { ...recovered!, status: 1 }, + }, + { + label: "partial completion protocol", + result: { + status: 1, + stdout: `v1 ${"a".repeat(64)} complete ok 0 4242`, + stderr: "", + }, + }, + ])("rejects invalid controller response: $label", ({ result: invalidResult }) => { + const result = runHermesProbe([starting, starting, starting, ready], true, [invalidResult]); + + expect(result.calls).toBe(3); + expect(result.recoveryActions).toEqual([{ action: "recover", timeout: 210_000 }]); + expect(result.message).toContain("managed gateway recovery failed before MCP mutation"); + }); + + it("fails immediately on trust and topology errors", () => { + const result = runHermesProbe([ + { + status: 1, + stdout: "", + stderr: "Hermes gateway PID does not identify the trusted launcher", + }, + ready, + ]); + + expect(result.calls).toBe(1); + expect(result.recoveryActions).toEqual([]); + expect(result.message).toContain("does not identify the trusted launcher"); + expect(result.message).not.toContain("nemoclaw hermes-box recover"); + }); + + it("directs an unmanaged but trusted gateway to recovery before mutation", () => { + const result = runHermesProbe([ + { + status: 1, + stdout: "", + stderr: "Hermes gateway is not running under the managed service lifecycle", + }, + ready, + ]); + + expect(result.calls).toBe(1); + expect(result.recoveryActions).toEqual([]); + expect(result.message).toContain("nemoclaw hermes-box recover"); + expect(result.message).toContain("managed service lifecycle"); + }); + + it("fails clearly when the gateway never becomes ready", () => { + const result = runHermesProbe([starting, starting, starting]); + + expect(result.calls).toBe(3); + expect(result.recoveryActions).toEqual([{ action: "recover", timeout: 210_000 }]); + expect(result.message).toContain("after managed gateway recovery"); + expect(result.message).toContain("no controller result"); + }); +}); diff --git a/test/hermes-share-mount-deps.test.ts b/test/hermes-share-mount-deps.test.ts index 0609a2ec712..cba9d509027 100644 --- a/test/hermes-share-mount-deps.test.ts +++ b/test/hermes-share-mount-deps.test.ts @@ -113,7 +113,7 @@ function runHermesInstallLayer( 'ln() { printf "ln %s\\n" "$*" >> "$call_log"; }', 'export HERMES_SEMVER="0.16.0"', 'export HERMES_NPM_INTEGRITY="sha512-test"', - 'export HERMES_UV_EXTRAS="messaging"', + 'export HERMES_UV_EXTRAS="messaging mcp"', command.replaceAll("/opt/hermes", fixture), ].join("\n"); fs.writeFileSync(scriptPath, script, { mode: 0o700 }); diff --git a/test/hermes-tool-gateway-broker.test.ts b/test/hermes-tool-gateway-broker.test.ts index 0448d7bb773..0bd374341a8 100644 --- a/test/hermes-tool-gateway-broker.test.ts +++ b/test/hermes-tool-gateway-broker.test.ts @@ -13,6 +13,7 @@ import path from "node:path"; import zlib from "node:zlib"; import { afterEach, describe, expect, it } from "vitest"; +import { testTimeout } from "./helpers/timeouts"; const SCRIPT = path.join( import.meta.dirname, @@ -32,6 +33,8 @@ const BROKER_WRAPPER = path.join( ); let children: ChildProcess[] = []; +const BROKER_READINESS_TIMEOUT_MS = 15_000; +const BROKER_TEST_TIMEOUT_MS = testTimeout(45_000); function sha256(value: string): string { return crypto.createHash("sha256").update(value).digest("hex"); @@ -66,25 +69,38 @@ function close(server: http.Server): Promise { return new Promise((resolve) => server.close(() => resolve())); } -async function waitForHealth(port: number): Promise { - for (let i = 0; i < 50; i++) { - try { - const resp = await fetch(`http://127.0.0.1:${port}/health`); - if (resp.status === 200) return; - } catch { - // keep polling - } - await new Promise((resolve) => setTimeout(resolve, 100)); - } - throw new Error("broker did not become healthy"); +function brokerDiagnostics(child: ChildProcess, output: () => string): string { + const captured = output().trim() || ""; + return [ + `exit=${child.exitCode ?? "pending"}, signal=${child.signalCode ?? "none"}`, + `captured output:\n${captured}`, + ].join("; "); } -async function waitUntil(predicate: () => boolean): Promise { - for (let i = 0; i < 50; i++) { - if (predicate()) return; +async function waitForBrokerCondition( + description: string, + child: ChildProcess, + output: () => string, + predicate: () => boolean | Promise, +): Promise { + const deadline = Date.now() + BROKER_READINESS_TIMEOUT_MS; + let lastError: unknown; + while (Date.now() < deadline) { + if (child.exitCode !== null || child.signalCode !== null) { + throw new Error(`${description}: broker exited early; ${brokerDiagnostics(child, output)}`); + } + try { + if (await predicate()) return; + } catch (error) { + lastError = error; + } await new Promise((resolve) => setTimeout(resolve, 100)); } - throw new Error("condition was not met"); + const lastErrorDetail = lastError instanceof Error ? `; last error: ${lastError.message}` : ""; + throw new Error( + `${description}: condition was not met within ${BROKER_READINESS_TIMEOUT_MS}ms; ` + + `${brokerDiagnostics(child, output)}${lastErrorDetail}`, + ); } afterEach(() => { @@ -123,7 +139,9 @@ describe("Hermes managed-tool gateway broker", () => { ).toBe(true); }); - it("refreshes via header, replaces upstream auth, normalizes responses, and rotates OpenShell storage", async () => { + it("refreshes via header, replaces upstream auth, normalizes responses, and rotates OpenShell storage", { + timeout: BROKER_TEST_TIMEOUT_MS, + }, async () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-hermes-tool-broker-")); const stateDir = path.join(tmp, "state"); const binDir = path.join(tmp, "bin"); @@ -264,14 +282,31 @@ describe("Hermes managed-tool gateway broker", () => { }); try { - await waitForHealth(brokerPort); - await waitUntil(() => { - try { - return fs.readFileSync(openshellLog, "utf8").includes("provider update hermes-provider"); - } catch { - return false; - } - }); + await waitForBrokerCondition( + "broker health", + child, + () => output, + async () => { + const response = await fetch(`http://127.0.0.1:${brokerPort}/health`, { + signal: AbortSignal.timeout(1_000), + }); + return response.status === 200; + }, + ); + await waitForBrokerCondition( + "inference provider refresh", + child, + () => output, + () => { + try { + return fs + .readFileSync(openshellLog, "utf8") + .includes("provider update hermes-provider"); + } catch { + return false; + } + }, + ); const unknown = await fetch(`http://127.0.0.1:${brokerPort}/unknown`); expect(unknown.status).toBe(404); diff --git a/test/install-build-dependency-preflight.test.ts b/test/install-build-dependency-preflight.test.ts new file mode 100644 index 00000000000..a1491ee81b5 --- /dev/null +++ b/test/install-build-dependency-preflight.test.ts @@ -0,0 +1,118 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { writeExecutable } from "./helpers/installer-sourced-env"; + +const INSTALLER = path.join(import.meta.dirname, "..", "install.sh"); + +function writeNodeStub(fakeBin: string) { + writeExecutable( + path.join(fakeBin, "node"), + `#!/usr/bin/env bash +if [ "$1" = "--version" ] || [ "$1" = "-v" ]; then echo "v22.16.0"; exit 0; fi +if [ -n "\${1:-}" ] && [ -f "$1" ]; then exec ${JSON.stringify(process.execPath)} "$@"; fi +if [ "$1" = "-e" ]; then exec ${JSON.stringify(process.execPath)} "$@"; fi +exit 99`, + ); +} + +function writeNpmStub(fakeBin: string, installSnippet = "exit 0") { + writeExecutable( + path.join(fakeBin, "npm"), + `#!/usr/bin/env bash +set -euo pipefail +if [ "$1" = "--version" ]; then echo "10.9.2"; exit 0; fi +if [ "$1" = "config" ] && [ "$2" = "get" ] && [ "$3" = "prefix" ]; then echo "$NPM_PREFIX"; exit 0; fi +if [ "$1" = "install" ] || [ "$1" = "link" ] || [ "$1" = "uninstall" ] || [ "$1" = "pack" ] || [ "$1" = "run" ]; then + ${installSnippet} +fi +echo "unexpected npm invocation: $*" >&2; exit 98`, + ); +} + +function writeDockerOkStub(fakeBin: string) { + writeExecutable( + path.join(fakeBin, "docker"), + `#!/usr/bin/env bash +if [ "$1" = "info" ]; then + echo '{"ServerVersion":"29.3.1","OperatingSystem":"Ubuntu 24.04","CgroupVersion":"2"}' +fi +exit 0`, + ); + writeExecutable( + path.join(fakeBin, "systemctl"), + `#!/usr/bin/env bash +if [ "$1" = "is-active" ] && [ "$2" = "docker" ]; then echo "active"; fi +exit 0`, + ); +} + +function buildSystemPathWithout(nameToExclude: string): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-preflight-nodep-")); + const exclude = new Set(["node", "npm", "npx", nameToExclude]); + for (const sysDir of ["/usr/bin", "/bin"]) { + for (const name of (fs.existsSync(sysDir) ? fs.readdirSync(sysDir) : []).filter( + (entry) => !exclude.has(entry), + )) { + try { + fs.symlinkSync(path.join(sysDir, name), path.join(dir, name)); + } catch (err) { + (err as NodeJS.ErrnoException).code === "EEXIST" || throwError(err); + } + } + } + return dir; +} + +function throwError(error: unknown): never { + throw error; +} + +function runWithoutStrings(env: Record = {}) { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-install-no-strings-")); + const fakeBin = path.join(tmp, "bin"); + fs.mkdirSync(fakeBin); + writeNodeStub(fakeBin); + writeDockerOkStub(fakeBin); + env.NEMOCLAW_DEFER_OPENSHELL_INSTALL === "1" && + (() => { + writeNpmStub(fakeBin, 'echo "npm stub stop" >&2; exit 91'); + env.NPM_PREFIX = path.join(tmp, "prefix"); + })(); + return spawnSync("bash", [INSTALLER], { + cwd: path.join(import.meta.dirname, ".."), + encoding: "utf-8", + env: { + ...process.env, + HOME: tmp, + PATH: `${fakeBin}:${buildSystemPathWithout("strings")}`, + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", + ...env, + }, + }); +} + +describe("installer build-dependency preflight (#4415)", { timeout: 30_000 }, () => { + it("fails fast when binutils strings is missing, before clone/build work", () => { + const result = runWithoutStrings(); + const output = `${result.stdout}${result.stderr}`; + expect(result.status).not.toBe(0); + expect(output).toMatch(/'strings' \(from binutils\) is required/); + expect(output).toMatch(/sudo apt-get install -y binutils/); + expect(output).not.toMatch(/Installing OpenShell/); + expect(output).not.toMatch(/Cloning into/); + }); + + it("does not fire the binutils preflight when OpenShell install is deferred", () => { + const result = runWithoutStrings({ NEMOCLAW_DEFER_OPENSHELL_INSTALL: "1" }); + expect(`${result.stdout}${result.stderr}`).not.toMatch( + /'strings' \(from binutils\) is required/, + ); + }); +}); diff --git a/test/install-openshell-version-check.test.ts b/test/install-openshell-version-check.test.ts index 323dc923f54..161e140643b 100644 --- a/test/install-openshell-version-check.test.ts +++ b/test/install-openshell-version-check.test.ts @@ -7,6 +7,9 @@ import os from "node:os"; import path from "node:path"; import { describe, expect, it } from "vitest"; +import credentialBoundaryManifest from "../src/lib/actions/sandbox/openshell-child-visible-credentials.v0.0.72.json"; +import { buildRebuildHermesChildEnv } from "./e2e/live/rebuild-hermes-env.ts"; + const SCRIPT = path.join(import.meta.dirname, "..", "scripts", "install-openshell.sh"); const PINNED_OPEN_SHELL_SHA256 = { cliDarwinArm64: "117b5354cc42d80bc4d5e070ea5ac4e341208ff6d3c29b516d8a9c80e2310f8d", @@ -17,13 +20,20 @@ const PINNED_OPEN_SHELL_SHA256 = { gatewayLinuxX64: "03225fb9388b682af1a5f1614b26b75f828da6031e3ffc1fd920b6fbe5f70877", sandboxLinuxArm64: "2cf62cbd651e55d0f8750804e2b4025e0d6c8eea4564c87cda47a2c922941db0", sandboxLinuxX64: "811f914b6a6a3a3f4533449ddebebb6422333861a27a5fa848db6cbfdffdd230", + sandboxBinaryLinuxX64: "f9f991a24d10772ad5d24ae27a8ea6baad8cac671695bd90fcd0355e0e0ad198", }; const ZERO_SHA256 = "0000000000000000000000000000000000000000000000000000000000000000"; +const REQUIRED_OPENSHELL_VERSION = credentialBoundaryManifest.openshellVersion; +const LEGACY_OPENSHELL_VERSION = "0.0.44"; +const OPENSHELL_REWRITE_FEATURE_MARKERS = + "request-body-credential-rewrite websocket-credential-rewrite"; +const OPENSHELL_MCP_FEATURE_MARKER = "allow_all_known_mcp_methods"; +const OPENSHELL_FEATURE_MARKERS = `${OPENSHELL_REWRITE_FEATURE_MARKERS} ${OPENSHELL_MCP_FEATURE_MARKER}`; +type OpenShellFeaturePlacement = "openshell" | "gateway" | "split-mcp-gateway" | "none"; function writeExecutable(target: string, contents: string) { fs.writeFileSync(target, contents, { mode: 0o755 }); } - /** * Run install-openshell.sh with a fake `openshell` binary that reports the * given version. The download/install code path is never reached because we @@ -36,16 +46,41 @@ function runWithInstalledVersion( extraEnv: NodeJS.ProcessEnv = {}, options: { capability?: boolean; + featurePlacement?: OpenShellFeaturePlacement; driverBins?: boolean | "gateway" | "gateway-vm"; + driverLocation?: "path" | "explicit" | "symlink"; + driverVersion?: string; + sandboxVersion?: string; + sandboxVersionExit?: number; + sandboxBinaryDigest?: string; + driverVersionExit?: number; + driverReadable?: boolean; os?: string; arch?: string; } = {}, ) { const capability = options.capability ?? true; + const featurePlacement: OpenShellFeaturePlacement = capability + ? (options.featurePlacement ?? "openshell") + : "none"; + const openshellMarkers = + featurePlacement === "openshell" + ? OPENSHELL_FEATURE_MARKERS + : featurePlacement === "split-mcp-gateway" + ? OPENSHELL_REWRITE_FEATURE_MARKERS + : ""; + const gatewayMarkers = + featurePlacement === "gateway" + ? OPENSHELL_FEATURE_MARKERS + : featurePlacement === "split-mcp-gateway" + ? OPENSHELL_MCP_FEATURE_MARKER + : ""; const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-openshell-ver-")); try { const fakeBin = path.join(tmp, "bin"); + const driverBin = options.driverLocation ? path.join(tmp, "driver-bin") : fakeBin; fs.mkdirSync(fakeBin); + fs.mkdirSync(driverBin, { recursive: true }); writeExecutable( path.join(fakeBin, "uname"), @@ -58,30 +93,61 @@ if [ "\${1:-}" = "-m" ]; then echo "${options.arch ?? "x86_64"}"; else echo "${o path.join(fakeBin, "openshell"), `#!/usr/bin/env bash if [ "\${1:-}" = "--version" ]; then echo "openshell ${version}"; exit 0; fi -${capability ? "# request-body-credential-rewrite websocket-credential-rewrite" : ""} +${openshellMarkers ? `# ${openshellMarkers}` : ""} exit 99`, ); - if (options.driverBins !== false) { + const driverFixtures: Array<{ name: string; markers: string }> = + options.driverBins === false + ? [] + : [ + { name: "openshell-gateway", markers: gatewayMarkers }, + ...(options.driverBins === "gateway" + ? [] + : [ + { + name: "openshell-sandbox", + markers: OPENSHELL_MCP_FEATURE_MARKER, + }, + ]), + ...(options.driverBins === "gateway-vm" + ? [ + { + name: "openshell-driver-vm", + markers: OPENSHELL_MCP_FEATURE_MARKER, + }, + ] + : []), + ]; + for (const fixture of driverFixtures) { writeExecutable( - path.join(fakeBin, "openshell-gateway"), + path.join(driverBin, fixture.name), `#!/usr/bin/env bash +if [ "\${1:-}" = "--version" ]; then echo "${fixture.name} ${fixture.name === "openshell-sandbox" ? (options.sandboxVersion ?? options.driverVersion ?? version) : (options.driverVersion ?? version)}"; exit ${fixture.name === "openshell-sandbox" ? (options.sandboxVersionExit ?? options.driverVersionExit ?? 0) : (options.driverVersionExit ?? 0)}; fi +# ${fixture.markers} exit 0`, ); + if (options.driverReadable === false) fs.chmodSync(path.join(driverBin, fixture.name), 0o111); + if (options.driverLocation === "symlink") { + fs.symlinkSync(path.join(driverBin, fixture.name), path.join(fakeBin, fixture.name)); + } } - if (options.driverBins !== false && options.driverBins !== "gateway") { - writeExecutable( - path.join(fakeBin, "openshell-sandbox"), - `#!/usr/bin/env bash -exit 0`, - ); - } - if (options.driverBins === "gateway-vm") { - writeExecutable( - path.join(fakeBin, "openshell-driver-vm"), - `#!/usr/bin/env bash -exit 0`, - ); + + switch (options.sandboxBinaryDigest) { + case undefined: + break; + default: + writeExecutable( + path.join(fakeBin, "sha256sum"), + `#!/usr/bin/env bash +case "\${1:-}" in + */openshell-sandbox) + printf '%s %s\\n' '${options.sandboxBinaryDigest}' "$1" + exit 0 + ;; +esac +exit 1`, + ); } // Stub curl to fail so the install path exits without doing real network I/O @@ -120,12 +186,20 @@ exit 0`, ); } + const explicitDriverEnv = + options.driverLocation === "explicit" + ? { + NEMOCLAW_OPENSHELL_GATEWAY_BIN: path.join(driverBin, "openshell-gateway"), + NEMOCLAW_OPENSHELL_SANDBOX_BIN: path.join(driverBin, "openshell-sandbox"), + } + : {}; return spawnSync("bash", [SCRIPT], { env: { ...process.env, NEMOCLAW_OPENSHELL_CHANNEL: "stable", + ...explicitDriverEnv, ...extraEnv, - PATH: `${fakeBin}:/usr/bin:/bin`, + PATH: `${fakeBin}:${driverBin}:/usr/bin:/bin`, }, encoding: "utf8", }); @@ -135,29 +209,151 @@ exit 0`, } describe("install-openshell.sh version check", { timeout: 15_000 }, () => { - it("exits cleanly when openshell 0.0.72 and driver binaries are already installed", () => { - const result = runWithInstalledVersion("0.0.72"); + it("exits cleanly when the required OpenShell and driver binaries are already installed", () => { + const result = runWithInstalledVersion(REQUIRED_OPENSHELL_VERSION); expect(result.status).toBe(0); - expect(result.stdout).toMatch(/already installed.*0\.0\.72/); + expect(result.stdout).toContain(`already installed: ${REQUIRED_OPENSHELL_VERSION}`); }); - it("triggers reinstall when openshell 0.0.72 is missing Docker-driver binaries", () => { - const result = runWithInstalledVersion("0.0.72", {}, { driverBins: false, os: "Linux" }); + it("accepts MCP L7 support from the installed gateway sidecar", () => { + const result = runWithInstalledVersion( + REQUIRED_OPENSHELL_VERSION, + {}, + { featurePlacement: "split-mcp-gateway" }, + ); + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + expect(result.stdout).toContain(`already installed: ${REQUIRED_OPENSHELL_VERSION}`); + }); + + it("does not combine the OpenShell CLI with driver binaries from another PATH root", () => { + const result = runWithInstalledVersion( + REQUIRED_OPENSHELL_VERSION, + {}, + { driverLocation: "path" }, + ); expect(result.status).not.toBe(0); expect(result.stdout).toMatch(/missing Docker-driver binaries/); - expect(result.stdout).toMatch(/Installing OpenShell from release 'v0\.0\.72'/); + expect(result.stdout).toContain( + `Installing OpenShell from release 'v${REQUIRED_OPENSHELL_VERSION}'`, + ); + }); + + it("accepts cross-prefix driver binaries only through explicit overrides", () => { + const result = runWithInstalledVersion( + REQUIRED_OPENSHELL_VERSION, + {}, + { driverLocation: "explicit" }, + ); + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + expect(result.stdout).toContain(`already installed: ${REQUIRED_OPENSHELL_VERSION}`); + }); + + it("rejects mixed release components hidden behind one symlink directory", () => { + const result = runWithInstalledVersion( + REQUIRED_OPENSHELL_VERSION, + {}, + { driverLocation: "symlink" }, + ); + expect(result.status).toBe(1); + expect(result.stderr).toMatch(/gateway resolves outside the active CLI install root/); }); - it("fails closed when openshell 0.0.72 lacks required messaging rewrite support", () => { - const result = runWithInstalledVersion("0.0.72", {}, { capability: false }); + it("rejects stale components copied into the active install root", () => { + const result = runWithInstalledVersion( + REQUIRED_OPENSHELL_VERSION, + {}, + { driverVersion: "0.0.71" }, + ); + expect(result.status).toBe(1); + expect(result.stderr).toMatch(/gateway does not match the active CLI build/); + }); + + it("rejects a component whose version probe fails after printing a version", () => { + const result = runWithInstalledVersion( + REQUIRED_OPENSHELL_VERSION, + {}, + { driverVersionExit: 42 }, + ); + expect(result.status).toBe(1); + expect(result.stderr).toMatch(/gateway does not match the active CLI build/); + }); + + it("accepts the exact pinned sandbox when its host-side version probe cannot load", () => { + const result = runWithInstalledVersion( + REQUIRED_OPENSHELL_VERSION, + {}, + { + sandboxVersionExit: 127, + sandboxBinaryDigest: PINNED_OPEN_SHELL_SHA256.sandboxBinaryLinuxX64, + }, + ); + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + expect(result.stdout).toContain(`already installed: ${REQUIRED_OPENSHELL_VERSION}`); + }); + + it("rejects a non-runnable sandbox whose digest is not a pinned release artifact", () => { + const result = runWithInstalledVersion( + REQUIRED_OPENSHELL_VERSION, + {}, + { sandboxVersionExit: 127, sandboxBinaryDigest: ZERO_SHA256 }, + ); + expect(result.status).toBe(1); + expect(result.stderr).toMatch(/sandbox does not match the active CLI build/); + }); + + it("rejects a selected component that cannot be scanned", () => { + const result = runWithInstalledVersion( + REQUIRED_OPENSHELL_VERSION, + {}, + { driverReadable: false }, + ); + expect(result.status).toBe(1); + expect(result.stderr).toMatch(/gateway is not readable and executable/); + }); + + it("rejects an executable directory supplied as an explicit component", () => { + const explicitDirectory = fs.mkdtempSync( + path.join(os.tmpdir(), "nemoclaw-openshell-component-dir-"), + ); + try { + const result = runWithInstalledVersion( + REQUIRED_OPENSHELL_VERSION, + { + NEMOCLAW_OPENSHELL_GATEWAY_BIN: explicitDirectory, + NEMOCLAW_OPENSHELL_SANDBOX_BIN: explicitDirectory, + }, + { os: "Darwin", arch: "arm64" }, + ); + expect(result.status).toBe(1); + expect(result.stderr).toMatch(/explicit OpenShell gateway binary.*missing.*not executable/); + } finally { + fs.rmSync(explicitDirectory, { recursive: true, force: true }); + } + }); + + it("triggers reinstall when the required OpenShell is missing Docker-driver binaries", () => { + const result = runWithInstalledVersion( + REQUIRED_OPENSHELL_VERSION, + {}, + { driverBins: false, os: "Linux" }, + ); + expect(result.status).not.toBe(0); + expect(result.stdout).toMatch(/missing Docker-driver binaries/); + expect(result.stdout).toContain( + `Installing OpenShell from release 'v${REQUIRED_OPENSHELL_VERSION}'`, + ); + }); + + it("fails closed when the required OpenShell lacks required messaging rewrite support", () => { + const result = runWithInstalledVersion(REQUIRED_OPENSHELL_VERSION, {}, { capability: false }); expect(result.status).toBe(1); // `fail()` writes to stderr as of #3446; previously stdout. expect(result.stderr).toMatch(/missing request-body-credential-rewrite support/); }); - it("accepts macOS openshell 0.0.72 when the gateway binary is installed", () => { + it("accepts macOS OpenShell when the gateway binary is installed", () => { const result = runWithInstalledVersion( - "0.0.72", + REQUIRED_OPENSHELL_VERSION, {}, { driverBins: "gateway", @@ -166,7 +362,17 @@ describe("install-openshell.sh version check", { timeout: 15_000 }, () => { }, ); expect(result.status).toBe(0); - expect(result.stdout).toMatch(/already installed.*0\.0\.72/); + expect(result.stdout).toContain(`already installed: ${REQUIRED_OPENSHELL_VERSION}`); + }); + + it("ignores a stale sibling sandbox binary for a macOS VM-driver install", () => { + const result = runWithInstalledVersion( + REQUIRED_OPENSHELL_VERSION, + {}, + { os: "Darwin", arch: "arm64", sandboxVersion: LEGACY_OPENSHELL_VERSION }, + ); + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + expect(result.stdout).toContain(`already installed: ${REQUIRED_OPENSHELL_VERSION}`); }); it("does not require the macOS VM driver entitlement for Docker-driver onboarding", () => { @@ -175,7 +381,7 @@ describe("install-openshell.sh version check", { timeout: 15_000 }, () => { const state = path.join(tmp, "codesign-state"); const log = path.join(tmp, "codesign.log"); const result = runWithInstalledVersion( - "0.0.72", + REQUIRED_OPENSHELL_VERSION, { NEMOCLAW_FAKE_CODESIGN_HAS_ENTITLEMENT: "0", NEMOCLAW_FAKE_CODESIGN_STATE: state, @@ -189,7 +395,7 @@ describe("install-openshell.sh version check", { timeout: 15_000 }, () => { ); expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); - expect(result.stdout).toMatch(/already installed.*0\.0\.72/); + expect(result.stdout).toContain(`already installed: ${REQUIRED_OPENSHELL_VERSION}`); expect(result.stdout).not.toMatch(/missing the macOS Hypervisor entitlement/); expect(result.stdout).not.toMatch(/Signing openshell-driver-vm/); expect(result.stdout).not.toMatch(/Installing OpenShell from release/); @@ -199,9 +405,9 @@ describe("install-openshell.sh version check", { timeout: 15_000 }, () => { } }); - it("triggers reinstall on macOS when openshell 0.0.72 is missing required gateway binaries", () => { + it("triggers reinstall on macOS when OpenShell is missing required gateway binaries", () => { const result = runWithInstalledVersion( - "0.0.72", + REQUIRED_OPENSHELL_VERSION, {}, { driverBins: false, @@ -211,7 +417,9 @@ describe("install-openshell.sh version check", { timeout: 15_000 }, () => { ); expect(result.status).not.toBe(0); expect(result.stdout).toMatch(/missing Docker-driver binaries/); - expect(result.stdout).toMatch(/Installing OpenShell from release 'v0\.0\.72'/); + expect(result.stdout).toContain( + `Installing OpenShell from release 'v${REQUIRED_OPENSHELL_VERSION}'`, + ); }); it("downloads the macOS arm64 gateway asset during reinstall", () => { @@ -276,6 +484,22 @@ exit 0`, writeExecutable( path.join(fakeBin, "tar"), `#!/usr/bin/env bash +outdir="" +prev="" +for arg in "$@"; do + if [ "$prev" = "-C" ]; then + outdir="$arg" + break + fi + prev="$arg" +done +[ -n "$outdir" ] || exit 1 +case "$*" in +*openshell-gateway*) name="openshell-gateway" ;; +*) name="openshell" ;; +esac +printf '#!/usr/bin/env bash\nexit 0\n' > "$outdir/$name" +chmod 755 "$outdir/$name" exit 0`, ); writeExecutable( @@ -283,10 +507,10 @@ exit 0`, `#!/usr/bin/env bash dest="\${@: -1}" mkdir -p "$(dirname "$dest")" -cat > "$dest" <<'EOF' -#!/usr/bin/env bash -if [ "\${1:-}" = "--version" ]; then echo "openshell 0.0.72"; exit 0; fi -# request-body-credential-rewrite websocket-credential-rewrite + cat > "$dest" <<'EOF' + #!/usr/bin/env bash +if [ "\${1:-}" = "--version" ]; then echo "openshell ${REQUIRED_OPENSHELL_VERSION}"; exit 0; fi +# ${OPENSHELL_FEATURE_MARKERS} exit 0 EOF chmod +x "$dest" @@ -393,8 +617,17 @@ dest="\${@: -1}" mkdir -p "$(dirname "$dest")" case "$(basename "$dest")" in openshell) - printf '#!/usr/bin/env bash\nif [ "$1" = "--version" ]; then echo "openshell 0.0.72"; else exit 0; fi\n# request-body-credential-rewrite websocket-credential-rewrite\n' > "$dest" ;; -*) printf '#!/usr/bin/env bash\nexit 0\n' > "$dest" ;; + printf '#!/usr/bin/env bash\nif [ "$1" = "--version" ]; then echo "openshell ${REQUIRED_OPENSHELL_VERSION}"; else exit 0; fi\n# ${OPENSHELL_FEATURE_MARKERS}\n' > "$dest" + ;; +openshell-sandbox) + printf '#!/usr/bin/env bash\nif [ "$1" = "--version" ]; then echo "openshell-sandbox ${REQUIRED_OPENSHELL_VERSION}"; exit 0; fi\n# ${OPENSHELL_MCP_FEATURE_MARKER}\nexit 0\n' > "$dest" + ;; +openshell-gateway) + printf '#!/usr/bin/env bash\nif [ "$1" = "--version" ]; then echo "openshell-gateway ${REQUIRED_OPENSHELL_VERSION}"; exit 0; fi\nexit 0\n' > "$dest" + ;; +*) + printf '#!/usr/bin/env bash\nexit 0\n' > "$dest" + ;; esac chmod 755 "$dest"`, ); @@ -517,7 +750,16 @@ printf '%s\\n' "$dest" >> ${JSON.stringify(installLog)} mkdir -p "$(dirname "$dest")" case "$(basename "$dest")" in openshell) - printf '#!/usr/bin/env bash\\nif [ "$1" = "--version" ]; then echo "openshell 0.0.72"; else exit 0; fi\\n# request-body-credential-rewrite websocket-credential-rewrite\\n' > "$dest" + printf '#!/usr/bin/env bash\\nif [ "$1" = "--version" ]; then echo "openshell ${REQUIRED_OPENSHELL_VERSION}"; else exit 0; fi\\n# ${OPENSHELL_FEATURE_MARKERS}\\n' > "$dest" + ;; +openshell-sandbox) + printf '#!/usr/bin/env bash\\nif [ "$1" = "--version" ]; then echo "openshell-sandbox ${REQUIRED_OPENSHELL_VERSION}"; exit 0; fi\\n# ${OPENSHELL_MCP_FEATURE_MARKER}\\nexit 0\\n' > "$dest" + ;; +openshell-gateway) + printf '#!/usr/bin/env bash\\nif [ "$1" = "--version" ]; then echo "openshell-gateway ${REQUIRED_OPENSHELL_VERSION}"; exit 0; fi\\nexit 0\\n' > "$dest" + ;; +openshell-driver-vm) + printf '#!/usr/bin/env bash\\n# ${OPENSHELL_MCP_FEATURE_MARKER}\\nexit 0\\n' > "$dest" ;; *) printf '#!/usr/bin/env bash\\nexit 0\\n' > "$dest" @@ -671,16 +913,26 @@ exit 0`, it("reinstalls the pinned release when openshell is above MAX_VERSION", () => { const result = runWithInstalledVersion("0.0.73"); expect(result.status).not.toBe(0); - expect(result.stdout).toMatch(/above the maximum.*reinstalling pinned OpenShell 0\.0\.72/); - expect(result.stdout).toMatch(/Installing OpenShell from release 'v0\.0\.72'/); + expect(result.stdout).toContain( + `above the maximum (${REQUIRED_OPENSHELL_VERSION}) supported by this NemoClaw release`, + ); + expect(result.stdout).toContain(`reinstalling pinned OpenShell ${REQUIRED_OPENSHELL_VERSION}`); + expect(result.stdout).toContain( + `Installing OpenShell from release 'v${REQUIRED_OPENSHELL_VERSION}'`, + ); expect(result.stderr).not.toMatch(/Upgrade NemoClaw first/); }); it("reinstalls the pinned release when openshell is at a much newer version", () => { const result = runWithInstalledVersion("0.1.0"); expect(result.status).not.toBe(0); - expect(result.stdout).toMatch(/above the maximum.*reinstalling pinned OpenShell 0\.0\.72/); - expect(result.stdout).toMatch(/Installing OpenShell from release 'v0\.0\.72'/); + expect(result.stdout).toContain( + `above the maximum (${REQUIRED_OPENSHELL_VERSION}) supported by this NemoClaw release`, + ); + expect(result.stdout).toContain(`reinstalling pinned OpenShell ${REQUIRED_OPENSHELL_VERSION}`); + expect(result.stdout).toContain( + `Installing OpenShell from release 'v${REQUIRED_OPENSHELL_VERSION}'`, + ); expect(result.stderr).not.toMatch(/Upgrade NemoClaw first/); }); @@ -704,13 +956,121 @@ exit 0`, ); }); + it("accepts coherent dev components with different git-prefix lengths", () => { + const result = runWithInstalledVersion( + "0.0.72-dev.8+g7bce1223d", + { + NEMOCLAW_ACCEPT_DEV_UNVERIFIED_INSTALL: "1", + NEMOCLAW_OPENSHELL_CHANNEL: "dev", + }, + { driverVersion: "0.0.72-dev.8+g7bce1223" }, + ); + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + expect(result.stdout).toMatch(/dev channel/); + }); + + it("refreshes a dev build when Docker-driver binaries are missing", () => { + const result = runWithInstalledVersion( + `${LEGACY_OPENSHELL_VERSION}.dev84+g6b2180425`, + { + NEMOCLAW_ACCEPT_DEV_UNVERIFIED_INSTALL: "1", + NEMOCLAW_OPENSHELL_CHANNEL: "dev", + }, + { driverBins: false, os: "Linux" }, + ); + expect(result.status).not.toBe(0); + expect(result.stdout).toMatch(/required dev-channel messaging-rewrite\/MCP-L7 build/); + expect(result.stdout).toContain("Installing OpenShell from release 'dev'"); + }); + + it("refreshes a Linux dev build when the sandbox binary alone is missing", () => { + const result = runWithInstalledVersion( + `${LEGACY_OPENSHELL_VERSION}.dev84+g6b2180425`, + { + NEMOCLAW_ACCEPT_DEV_UNVERIFIED_INSTALL: "1", + NEMOCLAW_OPENSHELL_CHANNEL: "dev", + }, + { driverBins: "gateway", os: "Linux" }, + ); + expect(result.status).not.toBe(0); + expect(result.stdout).toContain("Installing OpenShell from release 'dev'"); + }); + + it("reuses a macOS dev build with its required standalone gateway", () => { + const result = runWithInstalledVersion( + "0.0.72-dev.8+g7bce1223d", + { + NEMOCLAW_ACCEPT_DEV_UNVERIFIED_INSTALL: "1", + NEMOCLAW_OPENSHELL_CHANNEL: "dev", + }, + { driverBins: "gateway", os: "Darwin", arch: "arm64" }, + ); + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + expect(result.stdout).toMatch(/dev channel/); + }); + + it("refreshes an installed dev build when current main is required", () => { + const result = runWithInstalledVersion("0.0.72-dev.8+g7bce1223d", { + NEMOCLAW_ACCEPT_DEV_UNVERIFIED_INSTALL: "1", + NEMOCLAW_OPENSHELL_CHANNEL: "dev", + NEMOCLAW_OPENSHELL_FORCE_INSTALL: "1", + }); + expect(result.status).not.toBe(0); + expect(result.stdout).toContain("refreshing the moving dev release"); + expect(result.stdout).toContain("Installing OpenShell from release 'dev'"); + }); + + it("keeps auto on the stable release-selection contract", () => { + const result = runWithInstalledVersion("0.0.36", { + NEMOCLAW_OPENSHELL_CHANNEL: "auto", + }); + + expect(result.status).not.toBe(0); + expect(result.stdout).toContain( + `Installing OpenShell from release 'v${REQUIRED_OPENSHELL_VERSION}'`, + ); + expect(result.stdout).not.toContain("Installing OpenShell from release 'dev'"); + }); + + it("preserves the rebuild Hermes requested channel through the real installer boundary", () => { + const childEnv = buildRebuildHermesChildEnv( + { + HOME: process.env.HOME, + PATH: process.env.PATH, + NEMOCLAW_ACCEPT_DEV_UNVERIFIED_INSTALL: "1", + NEMOCLAW_OPENSHELL_CHANNEL: "dev", + NVIDIA_API_KEY: "must-not-reach-child", + }, + {}, + ); + const result = runWithInstalledVersion("0.0.36", childEnv); + + expect(childEnv.NEMOCLAW_ACCEPT_DEV_UNVERIFIED_INSTALL).toBe("1"); + expect(childEnv.NEMOCLAW_OPENSHELL_CHANNEL).toBe("dev"); + expect(childEnv.NVIDIA_API_KEY).toBeUndefined(); + expect(result.status).not.toBe(0); + expect(result.stdout).toContain("Installing OpenShell from release 'dev'"); + expect(result.stdout).not.toContain( + `Installing OpenShell from release 'v${REQUIRED_OPENSHELL_VERSION}'`, + ); + }); + it("upgrades stable OpenShell when the dev channel is requested", () => { const result = runWithInstalledVersion("0.0.36", { NEMOCLAW_OPENSHELL_CHANNEL: "dev", NEMOCLAW_ACCEPT_DEV_UNVERIFIED_INSTALL: "1", }); expect(result.status).not.toBe(0); - expect(result.stdout).toMatch(/required dev-channel messaging-rewrite build/); + expect(result.stdout).toMatch(/required dev-channel messaging-rewrite\/MCP-L7 build/); + }); + + it("rejects the removed artifact channel", () => { + const result = runWithInstalledVersion("0.0.72", { + NEMOCLAW_OPENSHELL_CHANNEL: "artifact", + }); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("NEMOCLAW_OPENSHELL_CHANNEL must be one of: stable, dev, auto"); }); it("proceeds to install when openshell is not present", () => { diff --git a/test/install-preflight.test.ts b/test/install-preflight.test.ts index 3167c947d51..94e93b754d8 100644 --- a/test/install-preflight.test.ts +++ b/test/install-preflight.test.ts @@ -286,7 +286,7 @@ exit 98 }, }); - expect(result.status).toBe(0); + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); const gitCalls = fs.readFileSync(gitLog, "utf-8"); expect(gitCalls).not.toMatch(/clone/); expect(gitCalls).not.toMatch(/fetch/); @@ -390,7 +390,7 @@ exit 98 }); const output = `${result.stdout}${result.stderr}`; - expect(result.status).toBe(0); + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); expect(output).toMatch(/NemoClaw Installer/); expect(output).not.toMatch(/deprecated compatibility wrapper/); }); @@ -410,7 +410,7 @@ exit 98 }); const output = `${result.stdout}${result.stderr}`; - expect(result.status).toBe(0); + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); expect(output).toMatch(/NemoClaw Installer/); expect(output).not.toMatch(/deprecated compatibility wrapper/); }); @@ -421,7 +421,7 @@ exit 98 encoding: "utf-8", }); - expect(result.status).toBe(0); + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); const output = `${result.stdout}${result.stderr}`; expect(output).toMatch(/NemoClaw Installer/); expect(output).toMatch(/--non-interactive/); @@ -444,7 +444,7 @@ exit 98 }); const output = `${result.stdout}${result.stderr}`; - expect(result.status).toBe(0); + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); expect(output).toMatch(/build \| openai \| anthropic \| anthropicCompatible/); expect(output).toMatch(/gemini \| ollama \| custom \| nim-local \| vllm \| routed/); expect(output).toMatch(/aliases: cloud -> build, nim -> nim-local/); @@ -456,7 +456,7 @@ exit 98 encoding: "utf-8", }); - expect(result.status).toBe(0); + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); const output = `${result.stdout}${result.stderr}`; expect(output.trim()).toMatch(/^nemoclaw-installer(?: v\d+\.\d+\.\d+(?:-.+)?)?$/); expect(output).not.toMatch(/0\.1\.0/); @@ -518,6 +518,8 @@ exit 98 fs.mkdirSync(path.join(prefix, "bin"), { recursive: true }); writeNodeStub(fakeBin); + writeDockerOkStub(fakeBin); + writeOpenShellOkStub(fakeBin); writeExecutable( path.join(fakeBin, "git"), `#!/usr/bin/env bash @@ -598,7 +600,7 @@ fi`, }, }); - expect(result.status).toBe(0); + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); const log = fs.readFileSync(npmLog, "utf-8"); // install (no -g) and link must both have been called expect(log).toMatch(/^install(?!\s+-g)/m); @@ -624,6 +626,7 @@ fi`, fs.mkdirSync(path.join(prefix, "bin"), { recursive: true }); writeNodeStub(fakeBin); + writeDockerOkStub(fakeBin); writeNpmStub( fakeBin, `printf '%s\\n' "$*" >> "$NPM_LOG_PATH" @@ -2170,6 +2173,8 @@ exit 99`, fs.mkdirSync(path.join(prefix, "bin"), { recursive: true }); writeNodeStub(fakeBin); + writeDockerOkStub(fakeBin); + writeOpenShellOkStub(fakeBin); writeNpmStub( fakeBin, `if [ "$1" = "pack" ]; then exit 1; fi @@ -2231,7 +2236,7 @@ exit 0`, }, }); - expect(result.status).toBe(0); + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); // git clone / git fetch should NOT have been called in the source-checkout path. // git may be called for version resolution (git describe), so we check // that no clone or fetch was attempted rather than no git calls at all. @@ -2253,6 +2258,8 @@ exit 0`, fs.mkdirSync(path.join(prefix, "bin"), { recursive: true }); writeNodeStub(fakeBin); + writeDockerOkStub(fakeBin); + writeOpenShellOkStub(fakeBin); writeExecutable( path.join(fakeBin, "curl"), @@ -2308,7 +2315,7 @@ fi`, }, }); - expect(result.status).toBe(0); + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); const gitCalls = fs.readFileSync(gitLog, "utf-8"); expect(gitCalls).not.toMatch(/clone/); expect(gitCalls).not.toMatch(/fetch/); @@ -3894,40 +3901,15 @@ sys.exit(exit_code) }); }); -// --------------------------------------------------------------------------- -// Build-dependency preflight (#4415): missing binutils/`strings` should fail -// fast at preflight, before any clone/build/download work, instead of ~5 -// minutes in at OpenShell verification. -// --------------------------------------------------------------------------- - -/** - * Like buildIsolatedSystemPath but lets the caller exclude additional binary - * names (in addition to node/npm/npx). Used to simulate a host that is missing - * `strings` (binutils) while keeping the rest of coreutils available. - */ -function buildSystemPathExcluding(extra: readonly string[]): string { - const dir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-preflight-nodep-")); - const EXCLUDE = new Set(["node", "npm", "npx", ...extra]); - for (const sysDir of ["/usr/bin", "/bin"]) { - if (!fs.existsSync(sysDir)) continue; - for (const name of fs.readdirSync(sysDir)) { - if (EXCLUDE.has(name)) continue; - try { - fs.symlinkSync(path.join(sysDir, name), path.join(dir, name)); - } catch (err) { - if ((err as NodeJS.ErrnoException).code !== "EEXIST") throw err; - } - } - } - return dir; -} - /** docker stub whose `info` always succeeds, so ensure_docker passes. */ function writeDockerOkStub(fakeBin: string) { writeExecutable( path.join(fakeBin, "docker"), `#!/usr/bin/env bash -if [ "$1" = "info" ]; then exit 0; fi +if [ "$1" = "info" ]; then + echo '{"ServerVersion":"29.3.1","OperatingSystem":"Ubuntu 24.04","CgroupVersion":"2"}' + exit 0 +fi exit 0 `, ); @@ -3940,66 +3922,13 @@ exit 0 ); } -describe("installer build-dependency preflight (#4415)", { timeout: 30_000 }, () => { - it("fails fast at preflight when binutils (strings) is missing, before any clone/build", () => { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-install-no-strings-")); - const fakeBin = path.join(tmp, "bin"); - fs.mkdirSync(fakeBin); - writeNodeStub(fakeBin); - writeDockerOkStub(fakeBin); - const noStringsPath = buildSystemPathExcluding(["strings"]); - - const result = spawnSync("bash", [INSTALLER], { - cwd: path.join(import.meta.dirname, ".."), - encoding: "utf-8", - env: { - ...process.env, - HOME: tmp, - PATH: `${fakeBin}:${noStringsPath}`, - NEMOCLAW_NON_INTERACTIVE: "1", - NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", - }, - }); - - const output = `${result.stdout}${result.stderr}`; - expect(result.status).not.toBe(0); - expect(output).toMatch(/'strings' \(from binutils\) is required/); - expect(output).toMatch(/sudo apt-get install -y binutils/); - // Fail-fast guarantee: never reached the OpenShell install/verify or the - // CLI build, which is the ~5-minutes-in failure point the issue reports. - expect(output).not.toMatch(/Installing OpenShell/); - expect(output).not.toMatch(/Cloning into/); - }); - - it("does not fire the binutils preflight when OpenShell install is deferred", () => { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-install-no-strings-deferred-")); - const fakeBin = path.join(tmp, "bin"); - fs.mkdirSync(fakeBin); - writeNodeStub(fakeBin); - // npm stub that fails fast on install, so the run stops shortly AFTER the - // (skipped) binutils preflight rather than doing real work. The assertion - // only cares that our binutils error never fires under DEFER. - writeNpmStub(fakeBin, 'echo "npm stub stop" >&2; exit 91'); - writeDockerOkStub(fakeBin); - const noStringsPath = buildSystemPathExcluding(["strings"]); - - const result = spawnSync("bash", [INSTALLER], { - cwd: path.join(import.meta.dirname, ".."), - encoding: "utf-8", - env: { - ...process.env, - HOME: tmp, - PATH: `${fakeBin}:${noStringsPath}`, - NEMOCLAW_NON_INTERACTIVE: "1", - NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", - NEMOCLAW_DEFER_OPENSHELL_INSTALL: "1", - NPM_PREFIX: path.join(tmp, "prefix"), - }, - }); - - const output = `${result.stdout}${result.stderr}`; - // The deferred path postpones all OpenShell work (and its own strings - // check) to a later phase, so the early preflight must stay silent. - expect(output).not.toMatch(/'strings' \(from binutils\) is required/); - }); -}); +function writeOpenShellOkStub(fakeBin: string, version = "0.0.72") { + writeExecutable( + path.join(fakeBin, "openshell"), + `#!/usr/bin/env bash +if [ "$1" = "--version" ] || [ "$1" = "version" ]; then echo "openshell ${version}"; exit 0; fi +# request-body-credential-rewrite websocket-credential-rewrite allow_all_known_mcp_methods +exit 0 +`, + ); +} diff --git a/test/issue-5667-hosted-inference-model-namespace.test.ts b/test/issue-5667-hosted-inference-model-namespace.test.ts index f8be6441c0b..792a72efc10 100644 --- a/test/issue-5667-hosted-inference-model-namespace.test.ts +++ b/test/issue-5667-hosted-inference-model-namespace.test.ts @@ -86,12 +86,19 @@ printf '200' function writeDcodeWrapperFixture(tmpDir: string, home: string): string { const wrapperPath = path.join(tmpDir, "dcode-wrapper.sh"); + const managedMcpValidator = [ + 'managed_mcp_config="$(', + " /opt/venv/bin/python3 -I -c \\", + " 'from deepagents_code._nemoclaw_managed import managed_mcp_config_path; print(managed_mcp_config_path() or \"\")'", + ')"', + ].join("\n"); const wrapper = fs .readFileSync( path.join(REPO_ROOT, "agents", "langchain-deepagents-code", "dcode-wrapper.sh"), "utf8", ) .replace("export HOME=/sandbox", `export HOME=${JSON.stringify(home)}`) + .replace(managedMcpValidator, 'managed_mcp_config=""') .replace( "exec /opt/venv/bin/python3 -I -m deepagents_code", `exec env PYTHONPATH=${JSON.stringify(path.join(tmpDir, "python"))} python3 -m deepagents_code`, diff --git a/test/langchain-deepagents-code-direct-module-patch.test.ts b/test/langchain-deepagents-code-direct-module-patch.test.ts index 07737bc97d6..8665698a4f9 100644 --- a/test/langchain-deepagents-code-direct-module-patch.test.ts +++ b/test/langchain-deepagents-code-direct-module-patch.test.ts @@ -679,8 +679,8 @@ describe("LangChain Deep Agents Code managed package patch", () => { const main = fs.readFileSync(path.join(packageDir, "main.py"), "utf8"); for (const expected of [ 'args.sandbox = "none"', - "args.no_mcp = True", - "args.mcp_config = None", + "args.no_mcp = not has_managed_mcp", + "args.mcp_config = managed_mcp_config if has_managed_mcp else None", "args.shell_allow_list = None", 'getattr(args, "update", False)', 'getattr(args, "auto_update", False)', @@ -792,15 +792,127 @@ describe("LangChain Deep Agents Code managed package patch", () => { expect(result.stdout).toContain("managed-posture-ok"); }); + it("accepts only exact same-name OpenShell credential placeholders", () => { + const tempDir = createPackageFixture(); + patchFixture(tempDir); + const run = (name: string, value: string) => + spawnSync("python3", ["-m", "deepagents_code"], { + env: { PATH: process.env.PATH, PYTHONPATH: tempDir, [name]: value }, + encoding: "utf8", + }); + + for (const value of [ + "openshell:resolve:env:GITHUB_MCP_TOKEN", + "openshell:resolve:env:v0_GITHUB_MCP_TOKEN", + `openshell:resolve:env:v${"1".repeat(20)}_GITHUB_MCP_TOKEN`, + ]) { + const result = run("GITHUB_MCP_TOKEN", value); + expect(result.status, result.stderr).toBe(0); + } + + for (const [name, value] of [ + ["GITHUB_MCP_TOKEN", "prefix-openshell:resolve:env:GITHUB_MCP_TOKEN"], + ["GITHUB_MCP_TOKEN", "openshell:resolve:env:OTHER_TOKEN"], + ["GITHUB_MCP_TOKEN", `openshell:resolve:env:v${"1".repeat(21)}_GITHUB_MCP_TOKEN`], + ["OPENSHELL_TLS_KEY", "openshell:resolve:env:OPENSHELL_TLS_KEY"], + ]) { + const result = run(name, value); + expect(result.status, `${name}=${value} was allowed`).not.toBe(0); + expect(result.stderr).toContain("invalid OpenShell credential placeholder"); + } + }); + + it("loads only strict HTTPS-only managed MCP configuration", () => { + const tempDir = createPackageFixture(); + patchFixture(tempDir); + const configPath = path.join(tempDir, ".mcp.json"); + const validate = (config: unknown, mode = 0o600) => { + fs.writeFileSync(configPath, `${JSON.stringify(config)}\n`, { mode }); + fs.chmodSync(configPath, mode); + return spawnSync( + "python3", + [ + "-c", + [ + "import sys", + "from pathlib import Path", + "from deepagents_code import _nemoclaw_managed as managed", + "managed._MCP_CONFIG_FILE = Path(sys.argv[1])", + "print(managed.managed_mcp_config_path() or 'absent')", + ].join("; "), + configPath, + ], + { + env: { PATH: process.env.PATH, PYTHONPATH: tempDir }, + encoding: "utf8", + }, + ); + }; + const validServer = { + type: "http", + url: "https://api.githubcopilot.com/mcp/", + headers: { + Authorization: "Bearer openshell:resolve:env:v20_GITHUB_MCP_TOKEN", + }, + }; + + const valid = validate({ mcpServers: { github: validServer } }); + expect(valid.status, valid.stderr).toBe(0); + expect(valid.stdout.trim()).toBe(configPath); + + for (const config of [ + { mcpServers: { github: { command: "bash", args: ["-c", "id"] } } }, + { mcpServers: { github: validServer }, ui: { theme: "dark" } }, + { + mcpServers: { + github: { ...validServer, headers: { "X-Test": "value" } }, + }, + }, + { + mcpServers: { + github: { ...validServer, headers: { Authorization: "Bearer raw-secret-value" } }, + }, + }, + { + mcpServers: { + github: { ...validServer, url: "https://127.0.0.1/mcp/" }, + }, + }, + ]) { + const result = validate(config); + expect(result.status, JSON.stringify(config)).not.toBe(0); + } + + const badMode = validate({ mcpServers: { github: validServer } }, 0o644); + expect(badMode.status).not.toBe(0); + expect(badMode.stderr).toContain("unsafe ownership or mode"); + }); + it("blocks TUI commands, credential screens, dotenv, OAuth, and install backends", () => { const tempDir = createPackageFixture(); patchFixture(tempDir); + const managedMcpPath = path.join(tempDir, "managed-mcp.json"); + fs.writeFileSync( + managedMcpPath, + `${JSON.stringify({ + mcpServers: { + github: { + type: "http", + url: "https://api.githubcopilot.com/mcp/", + headers: { + Authorization: "Bearer openshell:resolve:env:GITHUB_MCP_TOKEN", + }, + }, + }, + })}\n`, + { mode: 0o600 }, + ); const validation = ` import asyncio import os from pathlib import Path -from deepagents_code import agent, app, auth_store, config, hooks, model_config, non_interactive, server, subagents, update_check +from deepagents_code import agent, app, auth_store, config, hooks, main as dcode_main, model_config, non_interactive, server, subagents, update_check from deepagents_code import _nemoclaw_managed from deepagents_code import config_manifest from deepagents_code.integrations import openai_codex @@ -980,6 +1092,21 @@ async def validate(): assert headless_kwargs["interpreter_ptc"] is None assert headless_kwargs["rubric_model"] is None assert non_interactive.settings.shell_allow_list is None + _nemoclaw_managed._MCP_CONFIG_FILE = Path(${JSON.stringify(managedMcpPath)}) + managed_args = dcode_main.parse_args() + assert managed_args.mcp_config == ${JSON.stringify(managedMcpPath)} + assert managed_args.no_mcp is False + assert managed_args.trust_project_mcp is False + managed_headless_kwargs = await non_interactive.run_non_interactive( + "message", + "assistant", + mcp_config_path="attacker.json", + no_mcp=True, + trust_project_mcp=True, + ) + assert managed_headless_kwargs["mcp_config_path"] == ${JSON.stringify(managedMcpPath)} + assert managed_headless_kwargs["no_mcp"] is False + assert managed_headless_kwargs["trust_project_mcp"] is False assert model_config.ModelConfig().get_class_path("openai") is None managed_kwargs = config._get_provider_kwargs("openai") assert managed_kwargs == { diff --git a/test/langchain-deepagents-code-headless-runtime.test.ts b/test/langchain-deepagents-code-headless-runtime.test.ts new file mode 100644 index 00000000000..1e00ec4123a --- /dev/null +++ b/test/langchain-deepagents-code-headless-runtime.test.ts @@ -0,0 +1,137 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; + +import { + headlessCheckPath, + runHeadlessCheckHelper, + runHeadlessCheckSnippet, +} from "./helpers/langchain-deepagents-code-headless.ts"; + +describe("LangChain Deep Agents Code headless runtime contracts", () => { + it("requires exit zero and PONG from Deep Agents Code headless inference (#6191)", () => { + const classify = (exitCode: string, output: string) => + runHeadlessCheckHelper("classify-output", { + DCODE_EXIT: exitCode, + HEADLESS_OUTPUT: output, + }); + + expect(classify("0", "startup log\n PONG \nDCODE_EXIT:0")).toBe("pass:pong"); + expect( + classify("1", "OpenAI provider returned HTTP 401 for inference.local\nDCODE_EXIT:1"), + ).toBe("fail:actionable-inference-error"); + expect(classify("1", "PONG\nDCODE_EXIT:1")).toBe("fail:nonzero-exit"); + expect(classify("1", "openai.APIConnectionError\nDCODE_EXIT:1")).toBe( + "fail:inference-connection-failure", + ); + expect(classify("1", "Could not resolve host inference.local\nDCODE_EXIT:1")).toBe( + "fail:inference-connection-failure", + ); + expect(classify("0", "OpenAI provider unavailable\nDCODE_EXIT:0")).toBe( + "fail:actionable-inference-error", + ); + expect(classify("0", "dcode version 0.1.12\nOpenAI provider unavailable\nDCODE_EXIT:0")).toBe( + "fail:actionable-inference-error", + ); + expect(classify("124", "still waiting\nDCODE_EXIT:124")).toBe("fail:timeout"); + expect(classify("1", "usage: dcode [-h]\nDCODE_EXIT:1")).toBe("fail:local-execution-failure"); + expect(classify("1", "Traceback (most recent call last):\nDCODE_EXIT:1")).toBe( + "fail:local-execution-failure", + ); + expect(classify("127", "bash: dcode: command not found\nDCODE_EXIT:127")).toBe( + "fail:wrapper-missing", + ); + expect(classify("1", "No module named deepagents_code\nDCODE_EXIT:1")).toBe( + "fail:wrapper-missing", + ); + // The word 'dcode' appearing in a non-error context (e.g. a version + // banner) must not be misclassified as a wrapper-missing failure. The + // is_dcode_wrapper_failure regex requires a specific error indicator + // ("command not found", "No such file or directory", "Permission denied", + // or "No module named deepagents_code") after the dcode path segment. + // See PR #6206 / advisor PRA-2. + expect(classify("0", " PONG \nDCODE_EXIT:0")).toBe("pass:pong"); + expect(classify("0", "dcode version 0.1.12\nPONG\nDCODE_EXIT:0")).toBe("pass:pong"); + expect(classify("0", "something happened\nDCODE_EXIT:0")).toBe("fail:ambiguous-output"); + expect(classify("0", "Reply with exactly one word: PONG\nDCODE_EXIT:0")).toBe( + "fail:ambiguous-output", + ); + expect(classify("0", "PONG because the route works\nDCODE_EXIT:0")).toBe( + "fail:ambiguous-output", + ); + expect(classify("1", "something happened\nDCODE_EXIT:1")).toBe("fail:nonzero-exit"); + }); + + it("accepts only the normalized login-shell proxy contract (#6191)", () => { + const validate = (proxyUrl: string, noProxy: string, lowerProxy = proxyUrl) => { + const loginHome = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-dcode-login-")); + const hostFile = path.join(loginHome, "trusted-proxy-host"); + const portFile = path.join(loginHome, "trusted-proxy-port"); + const proxyEnvFile = path.join(loginHome, "proxy-env.sh"); + const checkFixture = path.join(loginHome, "headless-check.sh"); + const runtimeUid = process.getuid?.() ?? 0; + fs.writeFileSync(hostFile, "10.200.0.1\n", "utf8"); + fs.writeFileSync(portFile, "3128\n", "utf8"); + fs.chmodSync(hostFile, 0o444); + fs.chmodSync(portFile, 0o444); + fs.writeFileSync( + checkFixture, + fs + .readFileSync(headlessCheckPath, "utf8") + .replaceAll("/usr/local/share/nemoclaw/dcode-proxy-host", hostFile) + .replaceAll("/usr/local/share/nemoclaw/dcode-proxy-port", portFile) + .replaceAll("/tmp/nemoclaw-proxy-env.sh", proxyEnvFile) + .replace('= "0:444"', `= "${runtimeUid}:444"`) + .replace( + 'runtime_uid="$(id -u)" || contract_fail runtime-user; sandbox_uid="$(id -u sandbox)" || contract_fail runtime-user;', + `runtime_uid=${runtimeUid}; sandbox_uid=${runtimeUid};`, + ), + "utf8", + ); + fs.writeFileSync( + proxyEnvFile, + [ + `export HTTP_PROXY=${JSON.stringify(proxyUrl)}`, + `export HTTPS_PROXY=${JSON.stringify(proxyUrl)}`, + `export http_proxy=${JSON.stringify(lowerProxy)}`, + `export https_proxy=${JSON.stringify(lowerProxy)}`, + `export NO_PROXY=${JSON.stringify(noProxy)}`, + `export no_proxy=${JSON.stringify(noProxy)}`, + "unset ALL_PROXY all_proxy", + "", + ].join("\n"), + "utf8", + ); + fs.chmodSync(proxyEnvFile, 0o444); + fs.writeFileSync( + path.join(loginHome, ".profile"), + `export HOME=/sandbox\n. ${JSON.stringify(proxyEnvFile)}\n`, + "utf8", + ); + return runHeadlessCheckSnippet( + [ + "sandbox_login_exec() {", + " case \"$1\" in *$'\\n'*|*$'\\r'*) return 97 ;; esac", + ' env -u HTTP_PROXY -u HTTPS_PROXY -u NO_PROXY -u http_proxy -u https_proxy -u no_proxy -u ALL_PROXY -u all_proxy HOME="$TEST_LOGIN_HOME" bash -lc "$1"', + "}", + "if sandbox_login_proxy_contract >/dev/null 2>&1; then printf pass; else printf fail; fi", + ].join("\n"), + { TEST_LOGIN_HOME: loginHome }, + checkFixture, + ); + }; + + const managedProxy = "http://10.200.0.1:3128"; + const managedNoProxy = "localhost,127.0.0.1,::1,10.200.0.1"; + expect(validate(managedProxy, managedNoProxy)).toBe("pass"); + expect(validate(managedProxy, `${managedNoProxy},inference.local`)).toBe("fail"); + expect(validate("http://corp-user:corp-password@proxy.example:8080", managedNoProxy)).toBe( + "fail", + ); + expect(validate(managedProxy, managedNoProxy, "http://other-proxy.example:3128")).toBe("fail"); + }); +}); diff --git a/test/langchain-deepagents-code-image.test.ts b/test/langchain-deepagents-code-image.test.ts index b9af5d3e360..6417b1ddf29 100644 --- a/test/langchain-deepagents-code-image.test.ts +++ b/test/langchain-deepagents-code-image.test.ts @@ -10,7 +10,17 @@ import YAML from "yaml"; import { CONTEXT_PATTERNS, TOKEN_PREFIX_PATTERNS } from "../src/lib/security/secret-patterns.ts"; import { cloudExperimentalChecksForOnboarding } from "./e2e/live/cloud-experimental-check-list.ts"; -import { makeStartScriptFixture } from "./support/dcode-start-script-fixture.ts"; +import { + DCODE_CANONICAL_PATH, + headlessCheckPath, + makeStartScriptFixture as makeHeadlessStartScriptFixture, + NO_PROXY_ENV_NAMES, + PROXY_URL_ENV_NAMES, + runHeadlessCheckHelper, + runStartScriptProxyProbe, + TRACING_ENABLE_ENV_NAMES, +} from "./helpers/langchain-deepagents-code-headless.ts"; +import { makeStartScriptFixture as makeIdentityStartScriptFixture } from "./support/dcode-start-script-fixture.ts"; function fingerprint(patterns: readonly RegExp[]): string[] { return patterns.map((re) => `${re.source}::${re.flags}`); @@ -30,30 +40,33 @@ function fakePrivateKeyBlock(type = "", newline = "\\n"): string { return `-----BEGIN ${label} ${newline}opaque-test-body${newline}-----END ${label}`; } -const agentDir = path.join(process.cwd(), "agents", "langchain-deepagents-code"); -const headlessCheckPath = path.join( - process.cwd(), - "test", - "e2e", - "e2e-cloud-experimental", - "checks", - "07-deepagents-code-headless-inference.sh", -); +const repoRoot = path.resolve(import.meta.dirname, ".."); +const agentDir = path.join(repoRoot, "agents", "langchain-deepagents-code"); const tuiStartupCheckPath = path.join( - process.cwd(), + repoRoot, "test", "e2e", "e2e-cloud-experimental", "checks", "10-deepagents-code-tui-startup.sh", ); -const DCODE_CANONICAL_PATH = - "/usr/local/bin:/opt/venv/bin:/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/bin"; function readAgentFile(name: string): string { return fs.readFileSync(path.join(agentDir, name), "utf8"); } +const MANAGED_MCP_VALIDATOR_INVOCATION = [ + 'managed_mcp_config="$(', + " /opt/venv/bin/python3 -I -c \\", + " 'from deepagents_code._nemoclaw_managed import managed_mcp_config_path; print(managed_mcp_config_path() or \"\")'", + ')"', +].join("\n"); + +function stubManagedMcpValidator(source: string): string { + expect(source).toContain(MANAGED_MCP_VALIDATOR_INVOCATION); + return source.replace(MANAGED_MCP_VALIDATOR_INVOCATION, 'managed_mcp_config=""'); +} + function makeWrapperFixture( tempDir: string, envFileOverride?: string, @@ -69,7 +82,7 @@ function makeWrapperFixture( const envFile = envFileOverride ?? path.join(tempDir, ".env"); const authFile = path.join(tempDir, "auth.json"); const codexAuthFile = path.join(tempDir, "chatgpt-auth.json"); - const fixture = readAgentFile("dcode-wrapper.sh") + const fixture = stubManagedMcpValidator(readAgentFile("dcode-wrapper.sh")) .replace( 'readonly DEEPAGENTS_ENV_FILE="/sandbox/.deepagents/.env"', `readonly DEEPAGENTS_ENV_FILE="${envFile}"`, @@ -101,7 +114,7 @@ function makeNetworkSimulatingFixture(tempDir: string): { const wrapperPath = path.join(tempDir, "dcode-wrapper.sh"); const networkLog = path.join(tempDir, "network.log"); const envFile = path.join(tempDir, ".env"); - const fixture = readAgentFile("dcode-wrapper.sh") + const fixture = stubManagedMcpValidator(readAgentFile("dcode-wrapper.sh")) .replace( 'readonly DEEPAGENTS_ENV_FILE="/sandbox/.deepagents/.env"', `readonly DEEPAGENTS_ENV_FILE="${envFile}"`, @@ -143,69 +156,6 @@ function policyBinaryPaths(policyText: string, policyName: string): string[] { }); } -const PROXY_URL_ENV_NAMES = ["HTTP_PROXY", "HTTPS_PROXY", "http_proxy", "https_proxy"] as const; -const NO_PROXY_ENV_NAMES = ["NO_PROXY", "no_proxy"] as const; -const CLEARED_PROXY_ENV_NAMES = ["ALL_PROXY", "all_proxy", "OPENAI_PROXY"] as const; -const TRACING_ENABLE_ENV_NAMES = [ - "DEEPAGENTS_CODE_LANGSMITH_TRACING", - "DEEPAGENTS_CODE_LANGSMITH_TRACING_V2", - "DEEPAGENTS_CODE_LANGCHAIN_TRACING", - "DEEPAGENTS_CODE_LANGCHAIN_TRACING_V2", - "LANGSMITH_TRACING", - "LANGSMITH_TRACING_V2", - "LANGCHAIN_TRACING", - "LANGCHAIN_TRACING_V2", -] as const; - -function runStartScriptProxyProbe( - scriptPath: string, - envFile: string, - env: NodeJS.ProcessEnv, -): { envFileText: string; output: string } { - const probe = [ - ...[ - ...PROXY_URL_ENV_NAMES, - ...NO_PROXY_ENV_NAMES, - ...CLEARED_PROXY_ENV_NAMES, - ...TRACING_ENABLE_ENV_NAMES, - ].map((name) => `printf 'RUNTIME_${name}=%s\\n' "\${${name}-__unset__}"`), - "unset HTTP_PROXY HTTPS_PROXY NO_PROXY http_proxy https_proxy no_proxy ALL_PROXY all_proxy", - "export ALL_PROXY=socks5://persisted-user:persisted-password@persisted-all-proxy.example:1080", - "export all_proxy=socks5://lower-persisted-user:lower-persisted-password@lower-persisted-all-proxy.example:1080", - '. "$NEMOCLAW_TEST_PROXY_ENV"', - ...[ - ...PROXY_URL_ENV_NAMES, - ...NO_PROXY_ENV_NAMES, - ...CLEARED_PROXY_ENV_NAMES, - ...TRACING_ENABLE_ENV_NAMES, - ].map((name) => `printf 'SOURCED_${name}=%s\\n' "\${${name}-__unset__}"`), - ].join("\n"); - const result = spawnSync("bash", [scriptPath, "bash", "-c", probe], { - env: { - PATH: process.env.PATH ?? "/usr/bin:/bin", - ...env, - NEMOCLAW_TEST_PROXY_ENV: envFile, - }, - encoding: "utf8", - }); - expect(result.status, result.stderr).toBe(0); - return { - envFileText: fs.readFileSync(envFile, "utf8"), - output: `${result.stdout}\n${result.stderr}`, - }; -} - -function runHeadlessCheckHelper( - snippet: string, - env: NodeJS.ProcessEnv = {}, - sourcePath = headlessCheckPath, -): string { - return execFileSync("bash", ["-c", `source "$1"; ${snippet}`, "bash", sourcePath], { - encoding: "utf8", - env: { ...process.env, ...env }, - }); -} - describe("LangChain Deep Agents Code image contracts", () => { it("hardens copied NemoClaw blueprints against sandbox-user mutation", () => { const dockerfile = readAgentFile("Dockerfile"); @@ -257,7 +207,7 @@ describe("LangChain Deep Agents Code image contracts", () => { it("serializes the sandbox name into the shell env file for in-sandbox identity", () => { const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-dcode-start-")); try { - const { envFile, scriptPath } = makeStartScriptFixture(tempDir); + const { envFile, scriptPath } = makeIdentityStartScriptFixture(tempDir); execFileSync("bash", [scriptPath, "sh", "-c", ":"], { env: { @@ -275,7 +225,10 @@ describe("LangChain Deep Agents Code image contracts", () => { it("replaces inherited host proxy values with the managed runtime proxy (#6191)", () => { const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-dcode-start-")); - const { envFile, scriptPath } = makeStartScriptFixture(tempDir); + const { envFile, scriptPath } = makeHeadlessStartScriptFixture( + tempDir, + readAgentFile("start.sh"), + ); const inheritedSecrets = { NVIDIA_API_KEY: `nvapi-${"A".repeat(10)}`, OPENAI_API_KEY: `sk-${"B".repeat(20)}`, @@ -357,7 +310,10 @@ describe("LangChain Deep Agents Code image contracts", () => { expect(wrapper).toContain("unset PYTHONHOME PYTHONPATH"); expect(wrapper).toContain('/opt/venv/bin/python3 -I - "$auth_file"'); expect(wrapper).toContain("exec /opt/venv/bin/python3 -I -m deepagents_code"); - expect(wrapper).toContain("extra_args=(--sandbox none --no-mcp)"); + expect(wrapper).toContain("extra_args=(--sandbox none)"); + expect(wrapper).toContain('extra_args+=(--mcp-config "$managed_mcp_config")'); + expect(wrapper).not.toContain("--mcp-config /sandbox/.mcp.json"); + expect(wrapper).toContain("extra_args+=(--no-mcp)"); expect(wrapper).toContain("assert_no_auth_store_credentials"); expect(wrapper).toContain("assert_no_codex_auth_credentials"); for (const s of [ @@ -393,6 +349,39 @@ describe("LangChain Deep Agents Code image contracts", () => { expect(policy).not.toContain("dcode.upstream"); }); + it("exposes an exact managed MCP capability marker without starting dcode", () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-dcode-mcp-capability-")); + try { + const { wrapperPath, ranMarker } = makeWrapperFixture(tempDir); + const result = runWrapper(wrapperPath, ["--nemoclaw-mcp-capability"], {}); + + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toBe("NEMOCLAW_DEEPAGENTS_MCP_CAPABILITY=1\n"); + expect(fs.existsSync(ranMarker)).toBe(false); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } + }); + + it("uses the pinned Deep Agents Code user-level MCP discovery path", () => { + const requirements = readAgentFile("requirements.lock"); + const wrapper = readAgentFile("dcode-wrapper.sh"); + const patcher = readAgentFile("patch-managed-deepagents-code.py"); + const manifest = readAgentFile("manifest.yaml"); + const userLevelPath = "/sandbox/.deepagents/.mcp.json"; + + // The pinned Deep Agents Code release discovers ~/.deepagents/.mcp.json as user-level + // config. /sandbox/.mcp.json is project-level and headless `dcode -n` + // rejects it unless the project trust gate has been satisfied. + expect(requirements).toContain("deepagents-code==0.1.30"); + expect(wrapper).toContain("managed_mcp_config_path"); + expect(patcher).toContain(`_MCP_CONFIG_FILE = Path("${userLevelPath}")`); + expect(patcher).toContain("managed_mcp_config = _nemoclaw_managed_mcp_config_path()"); + expect(manifest).toContain("- .deepagents/.mcp.json"); + expect(wrapper).not.toContain("--mcp-config /sandbox/.mcp.json"); + expect(patcher).not.toContain('managed_mcp_config = "/sandbox/.mcp.json"'); + }); + it("puts the managed Python venv before system Python in every dcode entry path", () => { const baseDockerfile = readAgentFile("Dockerfile.base"); const dockerfile = readAgentFile("Dockerfile"); @@ -709,78 +698,20 @@ describe("LangChain Deep Agents Code image contracts", () => { it("requires the managed inference route and placeholder key in Deep Agents Code config", () => { expect( - runHeadlessCheckHelper( - 'printf "%s" "$CONFIG" | references_managed_inference_route && printf route', - { CONFIG: 'base_url = "https://inference.local/v1"' }, - ), + runHeadlessCheckHelper("managed-route", { + CONFIG: 'base_url = "https://inference.local/v1"', + }), ).toBe("route"); expect( - runHeadlessCheckHelper( - 'printf "%s" "$CONFIG" | references_managed_placeholder_key && printf key', - { CONFIG: 'api_key_env = "DEEPAGENTS_CODE_OPENAI_API_KEY"' }, - ), + runHeadlessCheckHelper("managed-placeholder", { + CONFIG: 'api_key_env = "DEEPAGENTS_CODE_OPENAI_API_KEY"', + }), ).toBe("key"); }); - it("requires exit zero and PONG from Deep Agents Code headless inference (#6191)", () => { - const classify = (exitCode: string, output: string) => - runHeadlessCheckHelper( - [ - 'if classification="$(classify_headless_output "$DCODE_EXIT" "$HEADLESS_OUTPUT")"; then', - ' printf "pass:%s" "$classification";', - "else", - ' printf "fail:%s" "$classification";', - "fi", - ].join(" "), - { DCODE_EXIT: exitCode, HEADLESS_OUTPUT: output }, - ); - - const cases: Array<[string, string, string]> = [ - ["0", "startup log\n PONG \nDCODE_EXIT:0", "pass:pong"], - [ - "1", - "OpenAI provider returned HTTP 401 for inference.local\nDCODE_EXIT:1", - "fail:actionable-inference-error", - ], - ["1", "PONG\nDCODE_EXIT:1", "fail:nonzero-exit"], - ["1", "openai.APIConnectionError\nDCODE_EXIT:1", "fail:inference-connection-failure"], - [ - "1", - "Could not resolve host inference.local\nDCODE_EXIT:1", - "fail:inference-connection-failure", - ], - ["0", "OpenAI provider unavailable\nDCODE_EXIT:0", "fail:actionable-inference-error"], - [ - "0", - "dcode version 0.1.12\nOpenAI provider unavailable\nDCODE_EXIT:0", - "fail:actionable-inference-error", - ], - ["124", "still waiting\nDCODE_EXIT:124", "fail:timeout"], - ["1", "usage: dcode [-h]\nDCODE_EXIT:1", "fail:local-execution-failure"], - ["1", "Traceback (most recent call last):\nDCODE_EXIT:1", "fail:local-execution-failure"], - ["127", "bash: dcode: command not found\nDCODE_EXIT:127", "fail:wrapper-missing"], - ["1", "No module named deepagents_code\nDCODE_EXIT:1", "fail:wrapper-missing"], - // The word 'dcode' in a non-error context (e.g. version banner) must not - // be misclassified as wrapper-missing; the regex requires a specific error - // indicator after the dcode path segment. See PR #6206 / advisor PRA-2. - ["0", " PONG \nDCODE_EXIT:0", "pass:pong"], - ["0", "dcode version 0.1.12\nPONG\nDCODE_EXIT:0", "pass:pong"], - ["0", "something happened\nDCODE_EXIT:0", "fail:ambiguous-output"], - ["0", "Reply with exactly one word: PONG\nDCODE_EXIT:0", "fail:ambiguous-output"], - ["0", "PONG because the route works\nDCODE_EXIT:0", "fail:ambiguous-output"], - ["1", "something happened\nDCODE_EXIT:1", "fail:nonzero-exit"], - ]; - for (const [exitCode, output, expected] of cases) { - expect(classify(exitCode, output)).toBe(expected); - } - }); - it("rejects unsafe headless timeout values before sandbox execution", () => { const validate = (timeout: string) => - runHeadlessCheckHelper( - 'if is_positive_integer "$HEADLESS_TIMEOUT"; then printf valid; else printf invalid; fi', - { DEEPAGENTS_HEADLESS_TIMEOUT: timeout }, - ); + runHeadlessCheckHelper("positive-integer", { DEEPAGENTS_HEADLESS_TIMEOUT: timeout }); expect(validate("120")).toBe("valid"); expect(validate("0")).toBe("invalid"); @@ -789,10 +720,7 @@ describe("LangChain Deep Agents Code image contracts", () => { it("detects representative secret families in headless inference artifacts", () => { const detectsSecret = (token: string) => - runHeadlessCheckHelper( - 'if printf "%s" "$TOKEN" | contains_secret; then printf secret; else printf clean; fi', - { TOKEN: token }, - ); + runHeadlessCheckHelper("contains-secret", { TOKEN: token }); const secretSamples = [ "nvapi-" + "A".repeat(10), "nvcf-" + "A".repeat(10), @@ -885,13 +813,86 @@ describe("LangChain Deep Agents Code image contracts", () => { expect(fs.existsSync(ranMarker)).toBe(false); }); + it("allows only exact same-name OpenShell env placeholders in runtime and dotenv inputs", () => { + const name = "GITHUB_MCP_TOKEN"; + const validPlaceholders = [ + `openshell:resolve:env:${name}`, + `openshell:resolve:env:v0_${name}`, + `openshell:resolve:env:v1442987827285932589_${name}`, + ]; + + for (const [index, placeholder] of validPlaceholders.entries()) { + const runtimeDir = fs.mkdtempSync( + path.join(os.tmpdir(), `nemoclaw-dcode-placeholder-runtime-${index}-`), + ); + const runtimeFixture = makeWrapperFixture(runtimeDir); + const runtimeResult = runWrapper(runtimeFixture.wrapperPath, ["-n", "hi"], { + [name]: placeholder, + }); + expect(runtimeResult.status, placeholder).toBe(0); + expect(runtimeResult.stdout).toContain("dcode-stub-ran"); + expect(fs.existsSync(runtimeFixture.ranMarker)).toBe(true); + + const dotenvDir = fs.mkdtempSync( + path.join(os.tmpdir(), `nemoclaw-dcode-placeholder-dotenv-${index}-`), + ); + const dotenvFixture = makeWrapperFixture(dotenvDir); + fs.writeFileSync(dotenvFixture.envFile, `${name}="${placeholder}"\n`, "utf8"); + const dotenvResult = runWrapper(dotenvFixture.wrapperPath, ["-n", "hi"], {}); + expect(dotenvResult.status, placeholder).toBe(0); + expect(dotenvResult.stdout).toContain("dcode-stub-ran"); + expect(fs.existsSync(dotenvFixture.ranMarker)).toBe(true); + } + }); + + it("rejects mismatched, malformed, wrapped, and raw credential placeholders", () => { + const invalidCases = [ + { name: "MODEL_NAME", value: "openshell:resolve:env:OTHER_NAME" }, + { name: "MODEL_NAME", value: "openshell:resolve:env:v12_OTHER_NAME" }, + { name: "MODEL_NAME", value: "openshell:resolve:env:v_MODEL_NAME" }, + { name: "MODEL_NAME", value: "openshell:resolve:env:v12x_MODEL_NAME" }, + { name: "MODEL_NAME", value: "openshell:resolve:env:v12__MODEL_NAME" }, + { name: "MODEL_NAME", value: "Bearer openshell:resolve:env:MODEL_NAME" }, + { name: "MODEL_NAME", value: "openshell:resolve:env:MODEL_NAME:suffix" }, + { name: "MODEL-NAME", value: "openshell:resolve:env:MODEL-NAME" }, + { name: "OPENSHELL_TLS_KEY", value: "openshell:resolve:env:OPENSHELL_TLS_KEY" }, + { name: "OPENSHELL_TLS_KEY", value: "openshell:resolve:env:v12_OPENSHELL_TLS_KEY" }, + { name: "GITHUB_MCP_TOKEN", value: "opaqueRawCredentialValue12345" }, + ]; + + for (const [index, { name, value }] of invalidCases.entries()) { + const runtimeDir = fs.mkdtempSync( + path.join(os.tmpdir(), `nemoclaw-dcode-placeholder-invalid-runtime-${index}-`), + ); + const runtimeFixture = makeWrapperFixture(runtimeDir); + const runtimeResult = runWrapper(runtimeFixture.wrapperPath, ["-n", "hi"], { + [name]: value, + }); + expect(runtimeResult.status, `runtime accepted ${value}`).not.toBe(0); + expect(runtimeResult.stderr).toContain(name); + expect(runtimeResult.stderr).not.toContain(value); + expect(fs.existsSync(runtimeFixture.ranMarker)).toBe(false); + + const dotenvDir = fs.mkdtempSync( + path.join(os.tmpdir(), `nemoclaw-dcode-placeholder-invalid-dotenv-${index}-`), + ); + const dotenvFixture = makeWrapperFixture(dotenvDir); + fs.writeFileSync(dotenvFixture.envFile, `${name}=${value}\n`, "utf8"); + const dotenvResult = runWrapper(dotenvFixture.wrapperPath, ["-n", "hi"], {}); + expect(dotenvResult.status, `dotenv accepted ${value}`).not.toBe(0); + expect(dotenvResult.stderr).toContain(name); + expect(dotenvResult.stderr).not.toContain(value); + expect(fs.existsSync(dotenvFixture.ranMarker)).toBe(false); + } + }); + it("allows nemoclaw-managed messaging tokens whose values are intentionally credential-shaped", () => { const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-dcode-wrapper-")); const { wrapperPath, ranMarker } = makeWrapperFixture(tempDir); const result = runWrapper(wrapperPath, ["-n", "hi"], { - SLACK_BOT_TOKEN: "xoxb-1234567890-abcdefghij", - SLACK_APP_TOKEN: "xapp-1-A1B2C3-1234567890-abcdefghij", + SLACK_BOT_TOKEN: ["xoxb", "1234567890", "abcdefghij"].join("-"), + SLACK_APP_TOKEN: ["xapp", "1", "A1B2C3", "1234567890", "abcdefghij"].join("-"), TELEGRAM_BOT_TOKEN: "123456789:AbcDefGhiJklMnoPqrStuVwxYz012345678", DISCORD_BOT_TOKEN: "ABCDEFGHIJKLMNOPQRSTUVWX.Abcdef.ZZZZZZZZZZZZZZZZZZZZZZZZZZZ", }); @@ -1457,11 +1458,11 @@ describe("LangChain Deep Agents Code image contracts", () => { { name: "sk", sample: "sk-abcdefghijklmnopqrstuvwx" }, { name: "xoxb", sample: "xoxb-1234567890" }, { name: "xoxp", sample: "xoxp-1234567890" }, - { name: "xoxa", sample: "xoxa-1234567890" }, + { name: "xoxa", sample: ["xoxa", "1234567890"].join("-") }, { name: "xoxs", sample: "xoxs-1234567890" }, - { name: "xapp", sample: "xapp-1-A1B2C3-12345-abcde" }, - { name: "akia", sample: "AKIAABCDEFGHIJKLMNOP" }, - { name: "asia", sample: "ASIAABCDEFGHIJKLMNOP" }, + { name: "xapp", sample: ["xapp", "1", "A1B2C3", "12345", "abcde"].join("-") }, + { name: "akia", sample: ["AKIA", "ABCDEFGHIJKLMNOP"].join("") }, + { name: "asia", sample: ["ASIA", "ABCDEFGHIJKLMNOP"].join("") }, { name: "hf", sample: "hf_abcdefghijklmnopq" }, { name: "glpat", sample: "glpat-abcdefghijklmn" }, { name: "gsk", sample: "gsk_abcdefghijklmnop" }, diff --git a/test/langchain-deepagents-code-managed-entrypoints.test.ts b/test/langchain-deepagents-code-managed-entrypoints.test.ts index e0c559811ce..e465f9fab0d 100644 --- a/test/langchain-deepagents-code-managed-entrypoints.test.ts +++ b/test/langchain-deepagents-code-managed-entrypoints.test.ts @@ -24,6 +24,13 @@ function readAgentFile(name: string): string { return fs.readFileSync(path.join(agentDir, name), "utf8"); } +const MANAGED_MCP_VALIDATOR_INVOCATION = [ + 'managed_mcp_config="$(', + " /opt/venv/bin/python3 -I -c \\", + " 'from deepagents_code._nemoclaw_managed import managed_mcp_config_path; print(managed_mcp_config_path() or \"\")'", + ')"', +].join("\n"); + function makeWrapperFixture(tempDir: string): { wrapperPath: string; ranMarker: string } { const wrapperPath = path.join(tempDir, "dcode-wrapper.sh"); const ranMarker = path.join(tempDir, "dcode-ran"); @@ -31,6 +38,7 @@ function makeWrapperFixture(tempDir: string): { wrapperPath: string; ranMarker: const authFile = path.join(tempDir, "auth.json"); const codexAuthFile = path.join(tempDir, "chatgpt-auth.json"); const fixture = readAgentFile("dcode-wrapper.sh") + .replace(MANAGED_MCP_VALIDATOR_INVOCATION, 'managed_mcp_config=""') .replace( 'readonly DEEPAGENTS_ENV_FILE="/sandbox/.deepagents/.env"', `readonly DEEPAGENTS_ENV_FILE="${envFile}"`, diff --git a/test/langchain-deepagents-code-proxy-launcher.test.ts b/test/langchain-deepagents-code-proxy-launcher.test.ts index d2fb46af985..ad205598340 100644 --- a/test/langchain-deepagents-code-proxy-launcher.test.ts +++ b/test/langchain-deepagents-code-proxy-launcher.test.ts @@ -147,7 +147,7 @@ describe("Deep Agents Code direct-exec proxy launcher", () => { env: hostileEnv, encoding: "utf8", }); - const startResult = spawnSync(scriptPath, ["/bin/true"], { + const startResult = spawnSync(scriptPath, ["/usr/bin/true"], { env: hostileEnv, encoding: "utf8", }); diff --git a/test/mcp-add-crash-consistency.test.ts b/test/mcp-add-crash-consistency.test.ts new file mode 100644 index 00000000000..a3b09baa5ca --- /dev/null +++ b/test/mcp-add-crash-consistency.test.ts @@ -0,0 +1,767 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +type CrashBoundary = + | "provider" + | "policy" + | "policy-failure" + | "policy-drift" + | "credential-collision" + | "adapter" + | "adapter-mismatch" + | "attach-race" + | "race" + | "late-race" + | "preupdate-observation-forbidden" + | ""; + +function runAddProcess(home: string, crashAfter: CrashBoundary, includeSecret = true) { + const script = String.raw` +process.env.HOME = ${JSON.stringify(home)}; +const includeSecret = ${JSON.stringify(includeSecret)}; +includeSecret ? (process.env.FAKE_MCP_SECRET = "host-only-secret") : delete process.env.FAKE_MCP_SECRET; +const fs = require("node:fs"); +const path = require("node:path"); +const crashAfter = ${JSON.stringify(crashAfter)}; +const marker = (name) => path.join(process.env.HOME, name + ".marker"); +const mark = (name) => fs.writeFileSync(marker(name), "yes\n", { mode: 0o600 }); +const marked = (name) => fs.existsSync(marker(name)); +const providerPresentAtStart = marked("provider"); +const providerId = "11111111-2222-4333-8444-555555555555"; +const foreignProviderId = "99999999-8888-4777-8666-555555555555"; +let providerGetCount = 0; +let observedProviderName = null; +let attachmentAttemptedThisProcess = false; + +const registry = require("./src/lib/state/registry.js"); +const globalActions = require("./src/lib/actions/global.js"); +const gatewayRuntime = require("./src/lib/gateway-runtime-action.js"); +const policies = require("./src/lib/policy/index.js"); +const processRecovery = require("./src/lib/actions/sandbox/process-recovery.js"); + +gatewayRuntime.recoverNamedGatewayRuntime = async () => ({ + recovered: true, + attempted: false, + before: { state: "healthy_named" }, + after: { state: "healthy_named" }, +}); + +globalActions.runOpenshellProviderCommand = (args) => { + if (args[0] === "status" && args[1] === "--output" && args[2] === "json") { + return { status: 0, stdout: JSON.stringify({ gateway: "nemoclaw" }), stderr: "" }; + } + if (args[0] === "provider" && args[1] === "get") { + if (args[2] === "foreign-attached") { + return { status: 0, stdout: "Id: " + foreignProviderId + "\nType: generic\nResource version: 1\nCredential keys: FAKE_MCP_SECRET\n", stderr: "" }; + } + observedProviderName = args[2]; + providerGetCount += 1; + if (crashAfter === "race" && providerGetCount === 2) mark("provider"); + if (crashAfter === "late-race" && providerGetCount === 3) mark("provider"); + return marked("provider") + ? { status: 0, stdout: "Id: " + (marked("foreign-provider") ? foreignProviderId : providerId) + "\nType: generic\nResource version: " + (marked("updated") ? "2" : "1") + "\nCredential keys: FAKE_MCP_SECRET\n", stderr: "" } + : { status: 1, stdout: "", stderr: "NotFound: provider" }; + } + if (args[0] === "provider" && (args[1] === "create" || args[1] === "update")) { + if (!marked("policy")) { + return { status: 1, stdout: "", stderr: "provider mutation preceded policy attestation" }; + } + if (args[1] === "create") observedProviderName = args[args.indexOf("--name") + 1]; + if (args[1] === "update") observedProviderName = args[2]; + mark("provider"); + if (args[1] === "update") mark("updated"); + if (crashAfter === "provider") process.exit(86); + return { status: 0, stdout: args[1] === "create" ? "Created provider" : "Updated provider", stderr: "" }; + } + if (args[0] === "sandbox" && args[1] === "provider" && args[2] === "list") { + if (crashAfter === "credential-collision") { + return { + status: 0, + stdout: "NAME TYPE CREDENTIAL_KEYS CONFIG_KEYS\nforeign-attached generic 1 0\n", + stderr: "", + }; + } + if (crashAfter === "attach-race" && marked("provider") && !marked("attached")) { + mark("foreign-provider"); + } + const attached = marked("attached"); + const providerName = observedProviderName ?? registry.getSandbox("crash-test")?.mcp?.bridges?.fake?.providerName; + if (attached && !marked("provider")) { + return { status: 1, stdout: "", stderr: "FailedPrecondition: provider '" + providerName + "' not found" }; + } + return { + status: 0, + stdout: attached + ? "NAME TYPE CREDENTIAL_KEYS CONFIG_KEYS\n" + providerName + " generic 1 0\n" + : "No providers attached to sandbox crash-test.\n", + stderr: "", + }; + } + if (args[0] === "sandbox" && args[1] === "provider" && args[2] === "attach") { + observedProviderName = args[4]; + attachmentAttemptedThisProcess = true; + mark("attached"); + return { status: 0, stdout: "attached", stderr: "" }; + } + if (args[0] === "sandbox" && args[1] === "provider" && args[2] === "detach") { + fs.rmSync(marker("attached"), { force: true }); + return { status: 0, stdout: "Detached provider", stderr: "" }; + } + if (args[0] === "provider" && args[1] === "delete") { + fs.rmSync(marker("provider"), { force: true }); + return { status: 0, stdout: "deleted", stderr: "" }; + } + return { status: 0, stdout: "", stderr: "" }; +}; + +policies.getPresetContentGatewayState = () => { + if (!marked("policy")) return "absent"; + return crashAfter === "policy-drift" ? "drift" : "match"; +}; +policies.applyPresetContent = () => { + if (crashAfter === "policy-failure") return false; + fs.appendFileSync(marker("policy-apply-log"), "apply\n", { mode: 0o600 }); + mark("policy"); + if (crashAfter === "policy") process.exit(86); + return true; +}; +policies.removePreset = () => { + fs.rmSync(marker("policy"), { force: true }); + return true; +}; + +processRecovery.executeSandboxExecCommand = (_sandbox, command) => { + const encoded = command.match(/printf '%s' '([A-Za-z0-9+/=]+)' \| base64 -d/)?.[1] || ""; + const proof = encoded ? Buffer.from(encoded, "base64").toString("utf8") : command; + const isObservation = proof.includes("printf '%s\\n' absent"); + const isPreupdateObservation = + isObservation && + providerPresentAtStart && + !marked("updated") && + !attachmentAttemptedThisProcess; + isPreupdateObservation && mark("observation"); + return { + status: crashAfter === "preupdate-observation-forbidden" && isPreupdateObservation ? 1 : 0, + stdout: isObservation ? (marked("updated") ? "v2" : marked("provider") ? "v1" : "absent") : "", + stderr: "", + }; +}; +processRecovery.executeSandboxCommand = (_sandbox, command) => { + if (command === "command -v mcporter") { + return { status: 0, stdout: "/usr/local/bin/mcporter\n", stderr: "" }; + } + if (command.includes("config' 'add")) { + mark("adapter"); + if (crashAfter === "adapter") process.exit(86); + return { status: 0, stdout: "", stderr: "" }; + } + if (command.includes("config' 'remove") || command.includes('["config", "remove"')) { + fs.rmSync(marker("adapter"), { force: true }); + return { status: 0, stdout: "", stderr: "" }; + } + if ( + crashAfter === "adapter-mismatch" && + marked("adapter") && + command.includes('["config", "get"') + ) { + return { status: 0, stdout: "mismatch\n", stderr: "" }; + } + return { + status: 0, + stdout: marked("adapter") ? "registered\n" : "absent\n", + stderr: "", + }; +}; + +if (!registry.getSandbox("crash-test")) { + registry.registerSandbox({ + name: "crash-test", + agent: "openclaw", + gatewayName: "nemoclaw", + }); +} +const bridge = require("./src/lib/actions/sandbox/mcp-bridge.js"); +bridge.addMcpBridge("crash-test", { + server: "fake", + url: "https://8.8.8.8/mcp", + env: [{ name: "FAKE_MCP_SECRET" }], +}).then( + () => process.exit(0), + (error) => { + console.error(error && error.stack || error); + process.exit(2); + }, +); +`; + return spawnSync(process.execPath, ["-e", script], { + cwd: process.cwd(), + encoding: "utf8", + env: { ...process.env, HOME: home }, + timeout: 30_000, + }); +} + +function runRemoveProcess(home: string, crashAfterProviderDelete: boolean) { + const script = String.raw` +process.env.HOME = ${JSON.stringify(home)}; +process.env.FAKE_MCP_SECRET = "host-only-secret"; +const fs = require("node:fs"); +const path = require("node:path"); +const crashAfterProviderDelete = ${JSON.stringify(crashAfterProviderDelete)}; +const marker = (name) => path.join(process.env.HOME, name + ".marker"); +const marked = (name) => fs.existsSync(marker(name)); +const providerId = "11111111-2222-4333-8444-555555555555"; +let observedProviderName = null; + +const globalActions = require("./src/lib/actions/global.js"); +const gatewayRuntime = require("./src/lib/gateway-runtime-action.js"); +const policies = require("./src/lib/policy/index.js"); +const processRecovery = require("./src/lib/actions/sandbox/process-recovery.js"); + +gatewayRuntime.recoverNamedGatewayRuntime = async () => ({ + recovered: true, + attempted: false, + before: { state: "healthy_named" }, + after: { state: "healthy_named" }, +}); + +globalActions.runOpenshellProviderCommand = (args) => { + if (args[0] === "status" && args[1] === "--output" && args[2] === "json") { + return { status: 0, stdout: JSON.stringify({ gateway: "nemoclaw" }), stderr: "" }; + } + if (args[0] === "provider" && args[1] === "get") { + observedProviderName = args[2]; + return marked("provider") + ? { status: 0, stdout: "Id: " + providerId + "\nType: generic\nResource version: 1\nCredential keys: FAKE_MCP_SECRET\n", stderr: "" } + : { status: 1, stdout: "", stderr: "NotFound: provider" }; + } + if (args[0] === "sandbox" && args[1] === "provider" && args[2] === "detach") { + observedProviderName = args[4]; + const wasAttached = marked("attached"); + fs.rmSync(marker("attached"), { force: true }); + return { + status: 0, + stdout: wasAttached + ? "Detached provider " + observedProviderName + " from sandbox crash-test.\n" + : "Provider " + observedProviderName + " was not attached to sandbox crash-test.\n", + stderr: "", + }; + } + if (args[0] === "sandbox" && args[1] === "provider" && args[2] === "list") { + const attached = marked("attached"); + const providerName = observedProviderName ?? require("./src/lib/state/registry.js").getSandbox("crash-test")?.mcp?.bridges?.fake?.providerName; + if (attached && !marked("provider")) { + return { status: 1, stdout: "", stderr: "FailedPrecondition: provider '" + providerName + "' not found" }; + } + return { + status: 0, + stdout: attached + ? "NAME TYPE CREDENTIAL_KEYS CONFIG_KEYS\n" + providerName + " generic 1 0\n" + : "No providers attached to sandbox crash-test.\n", + stderr: "", + }; + } + if (args[0] === "provider" && args[1] === "delete") { + if (!marked("provider")) { + return { status: 1, stdout: "", stderr: "NotFound: provider" }; + } + fs.rmSync(marker("provider"), { force: true }); + if (crashAfterProviderDelete) process.exit(87); + return { status: 0, stdout: "deleted", stderr: "" }; + } + return { status: 0, stdout: "", stderr: "" }; +}; + +policies.getPresetContentGatewayState = () => marked("policy") ? "match" : "absent"; +policies.removePreset = () => { + fs.rmSync(marker("policy"), { force: true }); + return true; +}; + +processRecovery.executeSandboxCommand = (_sandbox, command) => { + if (command.includes('["config", "remove"')) { + fs.rmSync(marker("adapter"), { force: true }); + } + return { status: 0, stdout: "", stderr: "" }; +}; +processRecovery.executeSandboxExecCommand = () => ({ status: 0, stdout: "", stderr: "" }); + +const bridge = require("./src/lib/actions/sandbox/mcp-bridge.js"); +bridge.removeMcpBridge("crash-test", "fake").then( + () => process.exit(0), + (error) => { + console.error(error && error.stack || error); + process.exit(2); + }, +); +`; + return spawnSync(process.execPath, ["-e", script], { + cwd: process.cwd(), + encoding: "utf8", + env: { ...process.env, HOME: home }, + timeout: 30_000, + }); +} + +function runStatusProcess(home: string) { + const script = String.raw` +process.env.HOME = ${JSON.stringify(home)}; +const globalActions = require("./src/lib/actions/global.js"); +const gatewayRuntime = require("./src/lib/gateway-runtime-action.js"); +const policies = require("./src/lib/policy/index.js"); +const processRecovery = require("./src/lib/actions/sandbox/process-recovery.js"); + +gatewayRuntime.recoverNamedGatewayRuntime = async () => ({ + recovered: true, + attempted: false, + before: { state: "healthy_named" }, + after: { state: "healthy_named" }, +}); +globalActions.runOpenshellProviderCommand = (args) => { + if (args[0] === "provider" && args[1] === "get") { + return { + status: 0, + stdout: "Type: generic\nCredential keys: FAKE_MCP_SECRET\n", + stderr: "", + }; + } + if (args[0] === "sandbox" && args[1] === "provider" && args[2] === "list") { + return { status: 0, stdout: "No providers attached to sandbox crash-test.\n", stderr: "" }; + } + return { status: 0, stdout: "", stderr: "" }; +}; +policies.presetContentMatchesGateway = () => { + throw new Error("unowned prepared policy must not be inspected as registered"); +}; +processRecovery.executeSandboxCommand = () => ({ + status: 0, + stdout: "absent\n", + stderr: "", +}); + +const bridge = require("./src/lib/actions/sandbox/mcp-bridge.js"); +bridge.statusMcpBridge("crash-test", "fake").then( + (status) => { + process.stdout.write(JSON.stringify(status[0])); + process.exit(0); + }, + (error) => { + console.error(error && error.stack || error); + process.exit(2); + }, +); +`; + return spawnSync(process.execPath, ["-e", script], { + cwd: process.cwd(), + encoding: "utf8", + env: { ...process.env, HOME: home }, + timeout: 30_000, + }); +} + +function readBridge(home: string): Record { + const parsed = JSON.parse( + fs.readFileSync(path.join(home, ".nemoclaw", "sandboxes.json"), "utf8"), + ) as { + sandboxes: { + "crash-test": { mcp: { bridges: { fake: Record } } }; + }; + }; + return parsed.sandboxes["crash-test"].mcp.bridges.fake; +} + +describe("MCP add crash consistency", () => { + it("rejects a missing host credential before creating durable MCP state", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mcp-add-missing-secret-")); + try { + const result = runAddProcess(home, "", false); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(2); + expect(result.stderr).toContain("Host environment variable 'FAKE_MCP_SECRET' is required"); + expect(fs.existsSync(path.join(home, "policy.marker"))).toBe(false); + expect(fs.existsSync(path.join(home, "provider.marker"))).toBe(false); + expect(fs.existsSync(path.join(home, "observation.marker"))).toBe(false); + const registry = JSON.parse( + fs.readFileSync(path.join(home, ".nemoclaw", "sandboxes.json"), "utf8"), + ) as { sandboxes: { "crash-test": { mcp?: unknown } } }; + expect(registry.sandboxes["crash-test"].mcp).toBeUndefined(); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + + it("creates a fresh provider without an update-only prior revision observation", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mcp-add-no-prior-observation-")); + try { + const result = runAddProcess(home, "preupdate-observation-forbidden"); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + expect(fs.existsSync(path.join(home, "observation.marker"))).toBe(false); + expect(fs.existsSync(path.join(home, "provider.marker"))).toBe(true); + expect(fs.existsSync(path.join(home, "attached.marker"))).toBe(true); + expect(fs.existsSync(path.join(home, "adapter.marker"))).toBe(true); + expect(readBridge(home).addState).toBeUndefined(); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + + it("resumes an exact provider without a host credential or prior revision observation", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mcp-add-reuse-no-observation-")); + try { + const interrupted = runAddProcess(home, "adapter"); + expect(interrupted.status, `${interrupted.stdout}\n${interrupted.stderr}`).toBe(86); + expect(fs.existsSync(path.join(home, "observation.marker"))).toBe(false); + + const resumed = runAddProcess(home, "", false); + expect(resumed.status, `${resumed.stdout}\n${resumed.stderr}`).toBe(0); + expect(fs.existsSync(path.join(home, "observation.marker"))).toBe(false); + expect(readBridge(home).addState).toBeUndefined(); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + + it("does not reapply policy when a resumed provider is missing its host credential", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mcp-add-resume-no-secret-")); + try { + const interrupted = runAddProcess(home, "adapter"); + expect(interrupted.status, `${interrupted.stdout}\n${interrupted.stderr}`).toBe(86); + const policyApplyLog = path.join(home, "policy-apply-log.marker"); + expect(fs.readFileSync(policyApplyLog, "utf8").trim().split("\n")).toHaveLength(1); + fs.rmSync(path.join(home, "provider.marker")); + + const resumed = runAddProcess(home, "", false); + expect(resumed.status, `${resumed.stdout}\n${resumed.stderr}`).toBe(2); + expect(resumed.stderr).toContain("is missing. Export host environment variable"); + expect(fs.readFileSync(policyApplyLog, "utf8").trim().split("\n")).toHaveLength(1); + expect(fs.existsSync(path.join(home, "policy.marker"))).toBe(false); + expect(fs.existsSync(path.join(home, "observation.marker"))).toBe(false); + expect(fs.existsSync(path.join(home, "attached.marker"))).toBe(false); + expect(readBridge(home)).toMatchObject({ addState: "preflighted" }); + + const recovered = runAddProcess(home, ""); + expect(recovered.status, `${recovered.stdout}\n${recovered.stderr}`).toBe(0); + expect(fs.readFileSync(policyApplyLog, "utf8").trim().split("\n")).toHaveLength(2); + expect(fs.existsSync(path.join(home, "provider.marker"))).toBe(true); + expect(fs.existsSync(path.join(home, "attached.marker"))).toBe(true); + expect(readBridge(home).addState).toBeUndefined(); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + + it("requires a host credential before retrying a prepared provider create", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mcp-add-prepared-no-secret-")); + try { + const providerMarker = path.join(home, "provider.marker"); + fs.writeFileSync(providerMarker, "foreign\n", { mode: 0o600 }); + const staged = runAddProcess(home, ""); + expect(staged.status, `${staged.stdout}\n${staged.stderr}`).toBe(2); + expect(readBridge(home).addState).toBe("prepared"); + fs.rmSync(providerMarker); + + const resumed = runAddProcess(home, "", false); + expect(resumed.status, `${resumed.stdout}\n${resumed.stderr}`).toBe(2); + expect(resumed.stderr).toContain("Host environment variable 'FAKE_MCP_SECRET' is required"); + expect(readBridge(home).addState).toBe("prepared"); + expect(fs.existsSync(path.join(home, "policy.marker"))).toBe(false); + expect(fs.existsSync(path.join(home, "policy-apply-log.marker"))).toBe(false); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + + it("rejects and rolls back an adapter definition that differs after a successful add", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mcp-add-mismatch-")); + try { + const result = runAddProcess(home, "adapter-mismatch"); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(2); + expect(result.stderr).toContain("mcporter config verification failed"); + expect(fs.existsSync(path.join(home, "provider.marker"))).toBe(false); + expect(fs.existsSync(path.join(home, "policy.marker"))).toBe(false); + expect(fs.existsSync(path.join(home, "adapter.marker"))).toBe(false); + expect(readBridge(home)).toMatchObject({ + server: "fake", + addState: "preflighted", + }); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + + it("fails closed after process death between provider create and provider-ID persistence", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mcp-add-provider-")); + try { + const crashed = runAddProcess(home, "provider"); + expect(crashed.status, `${crashed.stdout}\n${crashed.stderr}`).toBe(86); + expect(readBridge(home)).toMatchObject({ addState: "preflighted" }); + expect(readBridge(home)).not.toHaveProperty("providerId"); + expect(fs.existsSync(path.join(home, "policy.marker"))).toBe(true); + + const resumed = runAddProcess(home, ""); + expect(resumed.status, `${resumed.stdout}\n${resumed.stderr}`).toBe(2); + expect(resumed.stderr).toContain("has no stable provider ID and cannot safely adopt it"); + expect(fs.existsSync(path.join(home, "provider.marker"))).toBe(true); + expect(fs.existsSync(path.join(home, "policy.marker"))).toBe(false); + expect(readBridge(home)).not.toHaveProperty("providerId"); + + // After the operator independently removes the unowned provider, the + // local preflight manifest can be cleaned without adopting/deleting it. + fs.rmSync(path.join(home, "provider.marker")); + const cleaned = runRemoveProcess(home, false); + expect(cleaned.status, `${cleaned.stdout}\n${cleaned.stderr}`).toBe(0); + const registry = JSON.parse( + fs.readFileSync(path.join(home, ".nemoclaw", "sandboxes.json"), "utf8"), + ) as { sandboxes: { "crash-test": { mcp?: unknown } } }; + expect(registry.sandboxes["crash-test"].mcp).toBeUndefined(); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + + it("does not create a credential provider unless the generated policy is effective", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mcp-add-policy-drift-")); + try { + const rejected = runAddProcess(home, "policy-drift"); + + expect(rejected.status, `${rejected.stdout}\n${rejected.stderr}`).toBe(2); + expect(rejected.stderr).toContain("Failed to activate generated MCP policy"); + expect(rejected.stderr).toContain("effective state: drift"); + expect(`${rejected.stdout}\n${rejected.stderr}`).not.toContain("host-only-secret"); + expect(fs.existsSync(path.join(home, "provider.marker"))).toBe(false); + expect(fs.existsSync(path.join(home, "attached.marker"))).toBe(false); + expect(fs.existsSync(path.join(home, "adapter.marker"))).toBe(false); + expect(readBridge(home)).toMatchObject({ addState: "preflighted" }); + expect(readBridge(home)).not.toHaveProperty("providerId"); + const registry = JSON.parse( + fs.readFileSync(path.join(home, ".nemoclaw", "sandboxes.json"), "utf8"), + ) as { + sandboxes: { "crash-test": { customPolicies?: Array<{ name: string }> } }; + }; + expect(registry.sandboxes["crash-test"].customPolicies).toEqual([ + expect.objectContaining({ + name: "mcp-bridge-fake", + content: expect.any(String), + sourcePath: "generated:nemoclaw-mcp-bridge", + }), + ]); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + + it("releases a generated-policy reservation when policy activation definitely fails", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mcp-add-policy-failure-")); + try { + const rejected = runAddProcess(home, "policy-failure"); + + expect(rejected.status, `${rejected.stdout}\n${rejected.stderr}`).toBe(2); + expect(rejected.stderr).toContain("Failed to activate generated MCP policy"); + expect(rejected.stderr).toContain("effective state: absent"); + expect(fs.existsSync(path.join(home, "policy.marker"))).toBe(false); + expect(fs.existsSync(path.join(home, "provider.marker"))).toBe(false); + expect(fs.existsSync(path.join(home, "attached.marker"))).toBe(false); + expect(fs.existsSync(path.join(home, "adapter.marker"))).toBe(false); + expect(readBridge(home)).toMatchObject({ addState: "preflighted" }); + const registry = JSON.parse( + fs.readFileSync(path.join(home, ".nemoclaw", "sandboxes.json"), "utf8"), + ) as { + sandboxes: { "crash-test": { customPolicies?: Array<{ name: string }> } }; + }; + expect(registry.sandboxes["crash-test"].customPolicies).toBeUndefined(); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + + it("rejects an attached credential-key collision before activating the MCP policy", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mcp-add-key-collision-")); + try { + const rejected = runAddProcess(home, "credential-collision"); + + expect(rejected.status, `${rejected.stdout}\n${rejected.stderr}`).toBe(2); + expect(rejected.stderr).toContain( + "Credential key 'FAKE_MCP_SECRET' is already supplied by attached provider 'foreign-attached'", + ); + expect(fs.existsSync(path.join(home, "policy.marker"))).toBe(false); + expect(fs.existsSync(path.join(home, "provider.marker"))).toBe(false); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + + for (const [boundary, expectedProviderId, expectedProviderMarker, expectedObservationMarker] of [ + ["policy", undefined, false, false], + ["adapter", "11111111-2222-4333-8444-555555555555", true, true], + ] as const) { + it(`resumes exact resources after process death at the ${boundary} boundary`, () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), `nemoclaw-mcp-add-${boundary}-`)); + try { + const crashed = runAddProcess(home, boundary); + expect(crashed.status, `${crashed.stdout}\n${crashed.stderr}`).toBe(86); + const pending = readBridge(home); + expect(pending.addState).toBe("preflighted"); + expect(pending.providerId).toBe(expectedProviderId); + expect(fs.existsSync(path.join(home, "provider.marker"))).toBe(expectedProviderMarker); + expect(fs.existsSync(path.join(home, "policy.marker"))).toBe(true); + expect(JSON.stringify(pending)).not.toContain("host-only-secret"); + + const resumed = runAddProcess(home, ""); + expect(resumed.status, `${resumed.stdout}\n${resumed.stderr}`).toBe(0); + const committed = readBridge(home); + expect(committed.addState).toBeUndefined(); + expect(committed).toMatchObject({ + server: "fake", + env: ["FAKE_MCP_SECRET"], + policyName: "mcp-bridge-fake", + }); + expect(committed.providerName).toBe(pending.providerName); + expect(fs.existsSync(path.join(home, "provider.marker"))).toBe(true); + expect(fs.existsSync(path.join(home, "policy.marker"))).toBe(true); + expect(fs.existsSync(path.join(home, "adapter.marker"))).toBe(true); + expect(fs.existsSync(path.join(home, "observation.marker"))).toBe( + expectedObservationMarker, + ); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + } + + it("rejects a same-name provider created after preflight and before the first mutation", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mcp-add-race-")); + try { + const raced = runAddProcess(home, "race"); + expect(raced.status, `${raced.stdout}\n${raced.stderr}`).toBe(2); + expect(raced.stderr).toContain("already exists but is not owned"); + expect(readBridge(home)).toMatchObject({ addState: "preflighted" }); + expect(readBridge(home)).not.toHaveProperty("providerId"); + expect(fs.existsSync(path.join(home, "provider.marker"))).toBe(true); + expect(fs.existsSync(path.join(home, "policy.marker"))).toBe(false); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + + it("rechecks absence immediately before provider create", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mcp-add-late-race-")); + try { + const raced = runAddProcess(home, "late-race"); + expect(raced.status, `${raced.stdout}\n${raced.stderr}`).toBe(2); + expect(raced.stderr).toContain("changed before create"); + expect(readBridge(home)).toMatchObject({ addState: "preflighted" }); + expect(readBridge(home)).not.toHaveProperty("providerId"); + expect(fs.existsSync(path.join(home, "provider.marker"))).toBe(true); + expect(fs.existsSync(path.join(home, "policy.marker"))).toBe(false); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + + it("rechecks stable identity immediately before provider attach", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mcp-add-attach-race-")); + try { + const raced = runAddProcess(home, "attach-race"); + expect(raced.status, `${raced.stdout}\n${raced.stderr}`).toBe(2); + expect(raced.stderr).toContain("changed before attach"); + expect(readBridge(home)).toMatchObject({ + addState: "preflighted", + providerId: "11111111-2222-4333-8444-555555555555", + }); + expect(fs.existsSync(path.join(home, "provider.marker"))).toBe(true); + expect(fs.existsSync(path.join(home, "policy.marker"))).toBe(false); + expect(fs.existsSync(path.join(home, "adapter.marker"))).toBe(false); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + + it("does not claim or delete a same-name resource found before preflight", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mcp-add-foreign-provider-")); + try { + const providerMarker = path.join(home, "provider.marker"); + fs.writeFileSync(providerMarker, "foreign\n", { mode: 0o600 }); + + const rejected = runAddProcess(home, ""); + expect(rejected.status, `${rejected.stdout}\n${rejected.stderr}`).toBe(2); + expect(rejected.stderr).toContain("could not prove provider"); + expect(readBridge(home).addState).toBe("prepared"); + + const statusResult = runStatusProcess(home); + expect(statusResult.status, `${statusResult.stdout}\n${statusResult.stderr}`).toBe(0); + const status = JSON.parse(statusResult.stdout) as { + addState?: string; + policy: { registryPresent: boolean; gatewayPresent: boolean | null }; + }; + expect(status.addState).toBe("prepared"); + expect(status.policy).toEqual({ + name: "mcp-bridge-fake", + registryPresent: false, + gatewayPresent: null, + }); + + const cancelScript = ` +process.env.HOME = ${JSON.stringify(home)}; +const registry = require("./src/lib/state/registry.js"); +const bridge = require("./src/lib/actions/sandbox/mcp-bridge.js"); +bridge.removeMcpBridge("crash-test", "fake", { force: true }).then( + () => process.exit(0), + (error) => { console.error(error); process.exit(2); }, +); +`; + const cancelled = spawnSync(process.execPath, ["-e", cancelScript], { + cwd: process.cwd(), + encoding: "utf8", + env: { ...process.env, HOME: home }, + timeout: 30_000, + }); + expect(cancelled.status, `${cancelled.stdout}\n${cancelled.stderr}`).toBe(0); + expect(fs.existsSync(providerMarker)).toBe(true); + const registry = JSON.parse( + fs.readFileSync(path.join(home, ".nemoclaw", "sandboxes.json"), "utf8"), + ) as { sandboxes: { "crash-test": { mcp?: unknown } } }; + expect(registry.sandboxes["crash-test"].mcp).toBeUndefined(); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); +}); + +describe("MCP remove crash consistency", () => { + it("converges when the process dies after provider deletion", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mcp-remove-provider-")); + try { + const added = runAddProcess(home, ""); + expect(added.status, `${added.stdout}\n${added.stderr}`).toBe(0); + const providerName = readBridge(home).providerName; + + const crashed = runRemoveProcess(home, true); + expect(crashed.status, `${crashed.stdout}\n${crashed.stderr}`).toBe(87); + expect(readBridge(home)).toMatchObject({ + server: "fake", + providerName, + }); + expect(fs.existsSync(path.join(home, "provider.marker"))).toBe(false); + expect(fs.existsSync(path.join(home, "policy.marker"))).toBe(false); + expect(fs.existsSync(path.join(home, "adapter.marker"))).toBe(false); + + const resumed = runRemoveProcess(home, false); + expect(resumed.status, `${resumed.stdout}\n${resumed.stderr}`).toBe(0); + const registry = JSON.parse( + fs.readFileSync(path.join(home, ".nemoclaw", "sandboxes.json"), "utf8"), + ) as { sandboxes: { "crash-test": { mcp?: unknown } } }; + expect(registry.sandboxes["crash-test"].mcp).toBeUndefined(); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); +}); diff --git a/test/mcp-artifact-secret-scan.test.ts b/test/mcp-artifact-secret-scan.test.ts new file mode 100644 index 00000000000..0825e607cdd --- /dev/null +++ b/test/mcp-artifact-secret-scan.test.ts @@ -0,0 +1,81 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { scanMcpArtifactSecrets } from "../tools/e2e/assert-mcp-artifact-secrets-absent.mts"; +import { MCP_BRIDGE_TEST_CREDENTIALS } from "./e2e/fixtures/mcp-bridge-credentials.ts"; + +const roots: string[] = []; + +function artifactRoot(): string { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mcp-artifact-scan-")); + roots.push(root); + return root; +} + +afterEach(() => { + for (const root of roots.splice(0)) fs.rmSync(root, { force: true, recursive: true }); +}); + +describe("MCP artifact credential scan", () => { + it("accepts clean trees and missing artifact directories", () => { + const root = artifactRoot(); + fs.mkdirSync(path.join(root, "nested")); + fs.writeFileSync(path.join(root, "nested", "result.json"), '{"status":"clean"}\n'); + + expect(scanMcpArtifactSecrets(root)).toEqual({ filesScanned: 1, leaks: [] }); + expect(scanMcpArtifactSecrets(path.join(root, "missing"))).toEqual({ + filesScanned: 0, + leaks: [], + }); + }); + + it("finds raw and directly encoded fixture credentials without reporting their values", () => { + const root = artifactRoot(); + fs.writeFileSync(path.join(root, "raw.txt"), MCP_BRIDGE_TEST_CREDENTIALS.host); + fs.writeFileSync( + path.join(root, "encoded.txt"), + Buffer.from(MCP_BRIDGE_TEST_CREDENTIALS.rotatedHost).toString("base64url"), + ); + + const result = scanMcpArtifactSecrets(root); + expect(result.leaks).toEqual( + expect.arrayContaining([ + { credential: "host", encoding: "raw", file: "raw.txt" }, + { credential: "rotatedHost", encoding: "base64", file: "encoded.txt" }, + ]), + ); + expect(JSON.stringify(result)).not.toContain(MCP_BRIDGE_TEST_CREDENTIALS.host); + expect(JSON.stringify(result)).not.toContain(MCP_BRIDGE_TEST_CREDENTIALS.rotatedHost); + }); + + it("decodes larger base64 payloads before checking for embedded credentials", () => { + const root = artifactRoot(); + const encoded = Buffer.from( + `prefix:${MCP_BRIDGE_TEST_CREDENTIALS.rebindHost}:suffix`, + "utf8", + ).toString("base64"); + const wrapped = encoded.match(/.{1,7}/gu)?.join("\n") ?? encoded; + fs.writeFileSync(path.join(root, "wrapped.json"), JSON.stringify({ payload: wrapped })); + + expect(scanMcpArtifactSecrets(root).leaks).toContainEqual({ + credential: "rebindHost", + encoding: "base64", + file: "wrapped.json", + }); + }); + + it("fails closed on symbolic links inside the upload tree", () => { + const root = artifactRoot(); + const outside = path.join(artifactRoot(), "outside"); + fs.writeFileSync(outside, "outside"); + fs.symlinkSync(outside, path.join(root, "linked")); + + expect(() => scanMcpArtifactSecrets(root)).toThrow(/refuses symbolic link/); + }); +}); diff --git a/test/mcp-bridge-servers.test.ts b/test/mcp-bridge-servers.test.ts new file mode 100644 index 00000000000..0e2f1fff8b6 --- /dev/null +++ b/test/mcp-bridge-servers.test.ts @@ -0,0 +1,485 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { execFileSync } from "node:child_process"; +import fs from "node:fs"; +import https from "node:https"; +import os from "node:os"; +import path from "node:path"; + +import { afterAll, afterEach, describe, expect, it, vi } from "vitest"; + +import { MCP_BRIDGE_ALLOWED_METHODS } from "../src/lib/actions/sandbox/mcp-bridge-policy"; +import { + buildCloudflaredQuickTunnelArgs, + parseTryCloudflareOrigin, + type StartedHttpServer, + startCompatibleMock, + startFakeMcpHttpsServer, + startPublicMcpHttpsTunnel, +} from "./e2e/live/mcp-bridge-servers"; + +const servers: StartedHttpServer[] = []; +const tlsDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mcp-fixture-tls-")); +execFileSync( + "openssl", + [ + "req", + "-x509", + "-newkey", + "rsa:2048", + "-sha256", + "-nodes", + "-days", + "1", + "-subj", + "/CN=127.0.0.1", + "-addext", + "subjectAltName=IP:127.0.0.1", + "-keyout", + path.join(tlsDir, "server.key"), + "-out", + path.join(tlsDir, "server.crt"), + ], + { stdio: "ignore" }, +); +const fixtureTls = { + cert: fs.readFileSync(path.join(tlsDir, "server.crt")), + key: fs.readFileSync(path.join(tlsDir, "server.key")), +}; + +afterAll(() => { + fs.rmSync(tlsDir, { recursive: true, force: true }); +}); + +afterEach(async () => { + await Promise.all(servers.splice(0).map((server) => server.close())); +}); + +describe("authenticated MCP live fixtures", () => { + it("builds a bounded public HTTPS quick-tunnel origin without embedding credentials", () => { + expect(buildCloudflaredQuickTunnelArgs(43123)).toEqual([ + "tunnel", + "--no-autoupdate", + "--protocol", + "http2", + "--url", + "https://127.0.0.1:43123", + "--no-tls-verify", + "--loglevel", + "info", + ]); + expect(() => buildCloudflaredQuickTunnelArgs(0)).toThrow(/invalid local MCP HTTPS port/); + expect(() => buildCloudflaredQuickTunnelArgs(65_536)).toThrow(/invalid local MCP HTTPS port/); + }); + + it("accepts only an exact public trycloudflare origin from tunnel output", () => { + expect( + parseTryCloudflareOrigin( + '{"message":"https://mcp-fixture-123.trycloudflare.com registered"}', + ), + ).toBe("https://mcp-fixture-123.trycloudflare.com"); + expect(parseTryCloudflareOrigin("http://mcp-fixture.trycloudflare.com")).toBeNull(); + expect( + parseTryCloudflareOrigin("https://mcp-fixture.trycloudflare.com.attacker.invalid"), + ).toBeNull(); + }); + + it("waits for public readiness and registers unconditional process cleanup", async () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cloudflared-fixture-")); + const cloudflared = path.join(directory, "cloudflared"); + const priorAmbientSecret = process.env.MCP_TUNNEL_MUST_NOT_LEAK; + const priorOpenShellSecret = process.env.OPENSHELL_OIDC_CLIENT_SECRET; + process.env.MCP_TUNNEL_MUST_NOT_LEAK = "ambient-ci-secret"; + process.env.OPENSHELL_OIDC_CLIENT_SECRET = "ambient-openshell-secret"; + fs.writeFileSync( + cloudflared, + [ + "#!/bin/sh", + '[ -z "${MCP_TUNNEL_MUST_NOT_LEAK:-}" ] || exit 9', + '[ -z "${OPENSHELL_OIDC_CLIENT_SECRET:-}" ] || exit 10', + "printf '%s\\n' 'https://fixture-cleanup-123.trycloudflare.com' >&2", + "trap 'exit 0' TERM INT", + "while :; do sleep 1; done", + "", + ].join("\n"), + { mode: 0o755 }, + ); + const fetchMock = vi + .spyOn(globalThis, "fetch") + .mockResolvedValueOnce({ body: null, status: 502 } as Response) + .mockResolvedValue({ body: null, status: 405 } as Response); + let cleanupName = ""; + let cleanupProcess: (() => Promise) | undefined; + + try { + const tunnel = await startPublicMcpHttpsTunnel({ + cloudflaredBin: cloudflared, + cleanup: { + add: (name, run) => { + cleanupName = name; + cleanupProcess = async () => { + await run(); + }; + }, + }, + label: "unit MCP fixture", + server: { port: 43123, close: async () => {} }, + }); + + expect(tunnel).toMatchObject({ + origin: "https://fixture-cleanup-123.trycloudflare.com", + url: "https://fixture-cleanup-123.trycloudflare.com/mcp", + }); + expect(fetchMock).toHaveBeenCalledTimes(2); + expect(cleanupName).toBe("stop unit MCP fixture cloudflared quick tunnel"); + expect(cleanupProcess).toBeTypeOf("function"); + } finally { + await cleanupProcess?.(); + fetchMock.mockRestore(); + priorAmbientSecret === undefined + ? delete process.env.MCP_TUNNEL_MUST_NOT_LEAK + : (process.env.MCP_TUNNEL_MUST_NOT_LEAK = priorAmbientSecret); + priorOpenShellSecret === undefined + ? delete process.env.OPENSHELL_OIDC_CLIENT_SECRET + : (process.env.OPENSHELL_OIDC_CLIENT_SECRET = priorOpenShellSecret); + fs.rmSync(directory, { force: true, recursive: true }); + } + }); + + it("implements stateless Streamable HTTP and validates the tool challenge", async () => { + const secret = "fixture-secret"; + const challenge = "fixture-challenge"; + const resultToken = `MCP_AUTH_REWRITE_OK::${challenge}`; + const server = await startFakeMcpHttpsServer({ + secret, + challenge, + resultToken, + tls: fixtureTls, + }); + servers.push(server); + const url = `https://127.0.0.1:${server.port}/mcp`; + const headers = { + authorization: `Bearer ${secret}`, + "content-type": "application/json", + }; + + const request = async ( + method: string, + body?: Record, + ): Promise<{ status: number; body: string; json(): unknown }> => + await new Promise((resolve, reject) => { + const encoded = body ? JSON.stringify(body) : ""; + const req = https.request( + url, + { + method, + ca: fixtureTls.cert, + headers: encoded + ? { ...headers, "content-length": Buffer.byteLength(encoded) } + : headers, + }, + (response) => { + let responseBody = ""; + response.setEncoding("utf8"); + response.on("data", (chunk: string) => { + responseBody += chunk; + }); + response.on("end", () => + resolve({ + status: response.statusCode ?? 0, + body: responseBody, + json: () => JSON.parse(responseBody), + }), + ); + }, + ); + req.on("error", reject); + req.end(encoded); + }); + + expect((await request("HEAD")).status).toBe(405); + expect(server.requests, "public tunnel readiness must not pollute security assertions").toEqual( + [], + ); + const initialize = await request("POST", { + jsonrpc: "2.0", + id: 1, + method: "initialize", + params: { protocolVersion: "2025-06-18" }, + }); + expect(initialize.json()).toMatchObject({ + result: { protocolVersion: "2025-06-18" }, + }); + const initialized = await request("POST", { + jsonrpc: "2.0", + method: "notifications/initialized", + }); + expect(initialized.status).toBe(202); + + const list = await request("POST", { + jsonrpc: "2.0", + id: 2, + method: "tools/list", + }); + expect(list.json()).toMatchObject({ + result: { + tools: [ + { + name: "fake_echo", + inputSchema: { required: ["challenge"] }, + }, + ], + }, + }); + + const call = await request("POST", { + jsonrpc: "2.0", + id: 3, + method: "tools/call", + params: { name: "fake_echo", arguments: { challenge } }, + }); + expect(call.json()).toMatchObject({ + result: { + content: [{ type: "text", text: resultToken }], + isError: false, + }, + }); + const paramsByMethod: Partial> = { + initialize: { + protocolVersion: "2025-11-25", + capabilities: {}, + clientInfo: { name: "fixture", version: "1.0.0" }, + }, + "tools/call": { name: "fake_echo", arguments: { challenge } }, + "resources/read": { uri: "file:///empty" }, + "resources/subscribe": { uri: "file:///empty" }, + "resources/unsubscribe": { uri: "file:///empty" }, + "prompts/get": { name: "empty", arguments: {} }, + "tasks/get": { taskId: "fake-task" }, + "tasks/update": { taskId: "fake-task", inputResponses: {} }, + "tasks/result": { taskId: "fake-task" }, + "tasks/cancel": { taskId: "fake-task" }, + "completion/complete": { + ref: { type: "ref/prompt", name: "empty" }, + argument: { name: "value", value: "" }, + }, + "logging/setLevel": { level: "info" }, + "notifications/cancelled": { requestId: 1 }, + "notifications/progress": { progressToken: 1, progress: 1 }, + "notifications/elicitation/complete": { + elicitationId: "fake-elicitation", + }, + }; + + for (const rpcMethod of MCP_BRIDGE_ALLOWED_METHODS.filter((method) => + method.startsWith("notifications/"), + )) { + const params = paramsByMethod[rpcMethod]; + const response = await request("POST", { + jsonrpc: "2.0", + method: rpcMethod, + ...(params !== undefined ? { params } : {}), + }); + + expect({ status: response.status, body: response.body }, rpcMethod).toEqual({ + status: 202, + body: "", + }); + } + + for (const [index, rpcMethod] of MCP_BRIDGE_ALLOWED_METHODS.filter( + (method) => !method.startsWith("notifications/"), + ).entries()) { + const id = index + 1; + const params = paramsByMethod[rpcMethod]; + const response = await request("POST", { + jsonrpc: "2.0", + id, + method: rpcMethod, + ...(params !== undefined ? { params } : {}), + }); + + expect(response.status, rpcMethod).toBe(200); + expect(JSON.parse(response.body), rpcMethod).toMatchObject({ + jsonrpc: "2.0", + id, + }); + expect(JSON.parse(response.body), rpcMethod).not.toHaveProperty("error"); + expect(JSON.parse(response.body), rpcMethod).toHaveProperty("result"); + } + + expect( + server.requests.every( + (request) => request.auth !== "Bearer openshell:resolve:env:FAKE_TOKEN", + ), + ).toBe(true); + }); + + it("emits an MCP tool call and withholds success until the tool result returns", async () => { + const resultToken = "MCP_AUTH_REWRITE_OK::fixture"; + const server = await startCompatibleMock({ + apiKey: "compatible-key", + model: "mock/model", + toolChallenge: "fixture", + toolResultToken: resultToken, + toolNames: ["mcp_fake_fake_echo"], + }); + servers.push(server); + const url = `http://127.0.0.1:${server.port}/v1/chat/completions`; + const headers = { + authorization: "Bearer compatible-key", + "content-type": "application/json", + }; + const first = await fetch(url, { + method: "POST", + headers, + body: JSON.stringify({ + model: "mock/model", + messages: [{ role: "user", content: "use the tool" }], + tools: [ + { + type: "function", + function: { name: "mcp_fake_fake_echo", parameters: {} }, + }, + ], + }), + }); + const firstBody = (await first.json()) as { + choices: Array<{ + message: { + tool_calls: Array<{ function: { name: string; arguments: string } }>; + }; + }>; + }; + expect(firstBody.choices[0].message.tool_calls[0]).toMatchObject({ + function: { + name: "mcp_fake_fake_echo", + arguments: JSON.stringify({ challenge: "fixture" }), + }, + }); + expect(JSON.stringify(firstBody)).not.toContain(resultToken); + + const final = await fetch(url, { + method: "POST", + headers, + body: JSON.stringify({ + model: "mock/model", + messages: [{ role: "tool", content: resultToken }], + tools: [ + { + type: "function", + function: { name: "mcp_fake_fake_echo", parameters: {} }, + }, + ], + }), + }); + expect(await final.json()).toMatchObject({ + choices: [{ message: { content: resultToken } }], + }); + + const streamed = await fetch(url, { + method: "POST", + headers, + body: JSON.stringify({ + model: "mock/model", + stream: true, + messages: [{ role: "user", content: "use the tool" }], + tools: [ + { + type: "function", + function: { name: "mcp_fake_fake_echo", parameters: {} }, + }, + ], + }), + }); + const firstDataLine = (await streamed.text()) + .split("\n") + .find((line) => line.startsWith("data: {") && line.includes("tool_calls")); + expect(firstDataLine).toBeDefined(); + const firstChunk = JSON.parse(firstDataLine!.slice("data: ".length)); + expect(firstChunk).toMatchObject({ + model: "mock/model", + choices: [ + { + delta: { + tool_calls: [ + { + index: 0, + function: { name: "mcp_fake_fake_echo" }, + }, + ], + }, + }, + ], + }); + }); + + it("uses Hermes progressive disclosure when the MCP tool is deferred", async () => { + const resultToken = "MCP_AUTH_REWRITE_OK::deferred-fixture"; + const server = await startCompatibleMock({ + apiKey: "compatible-key", + model: "mock/model", + toolChallenge: "deferred-fixture", + toolResultToken: resultToken, + toolNames: ["mcp_fake_fake_echo"], + deferredToolName: "mcp_fake_fake_echo", + }); + servers.push(server); + const url = `http://127.0.0.1:${server.port}/v1/chat/completions`; + const headers = { + authorization: "Bearer compatible-key", + "content-type": "application/json", + }; + + const first = await fetch(url, { + method: "POST", + headers, + body: JSON.stringify({ + model: "mock/model", + messages: [{ role: "user", content: "use the deferred tool" }], + tools: [ + { + type: "function", + function: { name: "tool_call", parameters: {} }, + }, + ], + }), + }); + const firstBody = (await first.json()) as { + choices: Array<{ + message: { + tool_calls: Array<{ function: { name: string; arguments: string } }>; + }; + }>; + }; + expect(firstBody.choices[0].message.tool_calls[0]).toMatchObject({ + function: { + name: "tool_call", + arguments: JSON.stringify({ + name: "mcp_fake_fake_echo", + arguments: { challenge: "deferred-fixture" }, + }), + }, + }); + expect(JSON.stringify(firstBody)).not.toContain(resultToken); + + const final = await fetch(url, { + method: "POST", + headers, + body: JSON.stringify({ + model: "mock/model", + messages: [{ role: "tool", content: JSON.stringify({ result: resultToken }) }], + tools: [ + { + type: "function", + function: { name: "tool_call", parameters: {} }, + }, + ], + }), + }); + expect(await final.json()).toMatchObject({ + choices: [{ message: { content: resultToken } }], + }); + }); +}); diff --git a/test/mcp-destroy-lifecycle.test.ts b/test/mcp-destroy-lifecycle.test.ts new file mode 100644 index 00000000000..9ac200ec20e --- /dev/null +++ b/test/mcp-destroy-lifecycle.test.ts @@ -0,0 +1,896 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +function runDestroyLifecycleScenario(body: string) { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mcp-destroy-")); + const script = ` +process.env.HOME = ${JSON.stringify(home)}; +const registry = require("./src/lib/state/registry.js"); +const globalActions = require("./src/lib/actions/global.js"); +const gatewayRuntime = require("./src/lib/gateway-runtime-action.js"); +const policies = require("./src/lib/policy/index.js"); +const processRecovery = require("./src/lib/actions/sandbox/process-recovery.js"); + +const providers = new Map([ + [ + "alpha-mcp-github", + { credential: "GITHUB_TOKEN", id: "11111111-2222-4333-8444-555555555555" }, + ], + [ + "alpha-mcp-slack", + { credential: "SLACK_TOKEN", id: "66666666-7777-4888-8999-000000000000" }, + ], +]); +const attachedProviders = new Set(providers.keys()); +const calls = []; +const adapterCalls = []; +let adapterRegistered = true; +let policyApplyCalls = 0; +let failProviderDelete = null; +let failProviderDetach = null; +globalActions.runOpenshellProviderCommand = (args) => { + calls.push(args.join(" ")); + if (args.join(" ") === "status --output json") { + return { + status: 0, + stdout: "ready", + stderr: "", + }; + } + if (args[0] === "provider" && args[1] === "get") { + const provider = providers.get(args[2]); + return provider + ? { status: 0, stdout: "Id: " + provider.id + "\\nType: generic\\nResource version: 1\\nCredential keys: " + provider.credential + "\\n", stderr: "" } + : { status: 1, stdout: "", stderr: "Provider not found" }; + } + if (args[0] === "sandbox" && args[1] === "provider" && args[2] === "list") { + const names = [...attachedProviders]; + const danglingName = names.find((name) => !providers.has(name)); + if (danglingName) { + return { + status: 9, + stdout: "", + stderr: "FailedPrecondition: provider '" + danglingName + "' not found", + }; + } + return { + status: 0, + stdout: + names.length > 0 + ? "NAME TYPE CREDENTIAL_KEYS CONFIG_KEYS\\n" + + names + .map((name) => name + " generic 1 0") + .join("\\n") + + "\\n" + : "No providers attached to sandbox " + args[3] + ".\\n", + stderr: "", + }; + } + if (args[0] === "sandbox" && args[1] === "provider" && args[2] === "detach") { + if (failProviderDetach === args[4]) { + return { status: 9, stdout: "", stderr: "provider detach failed" }; + } + attachedProviders.delete(args[4]); + return { status: 0, stdout: "Detached provider", stderr: "" }; + } + if (args[0] === "sandbox" && args[1] === "provider" && args[2] === "attach") { + attachedProviders.add(args[4]); + return { status: 0, stdout: "Attached provider", stderr: "" }; + } + if (args[0] === "provider" && args[1] === "delete") { + if (failProviderDelete === args[2]) { + return { status: 9, stdout: "", stderr: "provider delete failed" }; + } + attachedProviders.delete(args[2]); + providers.delete(args[2]); + return { status: 0, stdout: "Deleted provider", stderr: "" }; + } + throw new Error("Unexpected OpenShell call: " + args.join(" ")); +}; +gatewayRuntime.recoverNamedGatewayRuntime = async () => ({ + recovered: true, + attempted: false, + before: { state: "healthy_named" }, + after: { state: "healthy_named" }, +}); +policies.applyPresetContent = () => { + policyApplyCalls += 1; + return true; +}; +policies.getPresetContentGatewayState = () => "match"; +policies.removePreset = () => true; +processRecovery.executeSandboxCommand = (_sandbox, command) => { + adapterCalls.push(command); + if (command.includes("'config' 'add'")) { + adapterRegistered = true; + return { status: 0, stdout: "", stderr: "" }; + } + if (command.includes('["config", "remove"')) { + adapterRegistered = false; + return { status: 0, stdout: "", stderr: "" }; + } + if (command.includes('["config", "get"')) { + return { + status: 0, + stdout: adapterRegistered ? "registered\\n" : "absent\\n", + stderr: "", + }; + } + return { + status: 0, + stdout: command === "command -v mcporter" ? "/usr/local/bin/mcporter\\n" : "", + stderr: "", + }; +}; +processRecovery.executeSandboxExecCommand = (_sandbox, command) => { + const encoded = command.match(/printf '%s' '([A-Za-z0-9+/=]+)' \| base64 -d/)?.[1] || ""; + const proof = encoded ? Buffer.from(encoded, "base64").toString("utf8") : command; + const isRevisionObservation = proof.includes("printf '%s\\\\n' absent"); + const observedCredential = proof.includes("openshell:resolve:env:GITHUB_TOKEN") + ? "GITHUB_TOKEN" + : proof.includes("openshell:resolve:env:SLACK_TOKEN") + ? "SLACK_TOKEN" + : null; + const credentialAttached = + observedCredential !== null && + [...attachedProviders].some( + (providerName) => providers.get(providerName)?.credential === observedCredential, + ); + return { + status: + proof.includes("allow_all_known_mcp_methods") || + proof.includes('[ -z "\${') || + proof.includes("openshell:resolve:env:GITHUB_TOKEN") || + proof.includes("openshell:resolve:env:SLACK_TOKEN") + ? 0 + : 1, + stdout: isRevisionObservation ? (credentialAttached ? "canonical" : "absent") : "", + stderr: "", + }; +}; + +const bridgeEntry = (server, credential) => ({ + server, + agent: "openclaw", + adapter: "mcporter", + url: "https://8.8.8.8/" + server, + env: [credential], + providerName: "alpha-mcp-" + server, + providerId: providers.get("alpha-mcp-" + server).id, + policyName: "mcp-bridge-" + server, + addedAt: "2026-06-27T00:00:00.000Z", +}); +const bridgeEntries = { + github: bridgeEntry("github", "GITHUB_TOKEN"), + slack: bridgeEntry("slack", "SLACK_TOKEN"), +}; +const ownedPolicy = (server) => ({ + name: "mcp-bridge-" + server, + content: "network_policies: {}\\n", + sourcePath: "generated:nemoclaw-mcp-bridge", +}); +${body} +`; + const result = spawnSync(process.execPath, ["-e", script], { + cwd: process.cwd(), + encoding: "utf8", + env: { ...process.env, HOME: home }, + }); + fs.rmSync(home, { recursive: true, force: true }); + return result; +} + +describe("authenticated MCP sandbox destroy lifecycle", () => { + for (const method of [ + "prepareMcpBridgesForAbsentSandboxDestroy", + "prepareMcpBridgesForAbsentSandboxRebuild", + ] as const) { + it(`clears a providerless preflighted add during ${method}`, () => { + const result = runDestroyLifecycleScenario(` +providers.delete("alpha-mcp-github"); +attachedProviders.delete("alpha-mcp-github"); +const pending = { ...bridgeEntries.github, addState: "preflighted" }; +delete pending.providerId; +registry.registerSandbox({ + name: "alpha", + agent: "openclaw", + mcp: { bridges: { github: pending } }, +}); +registry.addCustomPolicy("alpha", ownedPolicy("github")); +policies.getPresetContentGatewayState = () => { throw new Error("absent rebuild queried live policy"); }; +const bridge = require("./src/lib/actions/sandbox/mcp-bridge.js"); +(async () => { + const preparation = await bridge.${method}("alpha"); + process.stdout.write(JSON.stringify({ preparation, sandbox: registry.getSandbox("alpha") })); +})().catch((error) => { console.error(error); process.exit(1); }); +`); + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + const payload = JSON.parse(result.stdout) as { + preparation: { entries: unknown[] }; + sandbox: { mcp?: unknown; customPolicies?: unknown }; + }; + expect(payload.preparation.entries).toEqual([]); + expect(payload.sandbox.mcp).toBeUndefined(); + expect(payload.sandbox.customPolicies).toBeUndefined(); + }); + } + + for (const method of [ + "prepareMcpBridgesForRebuild", + "prepareMcpBridgesForAbsentSandboxRebuild", + ] as const) { + for (const marker of ["destroyPreparedAt", "destroyPendingAt"] as const) { + it(`rejects ${method} while ${marker} is durable`, () => { + const result = runDestroyLifecycleScenario(` +registry.registerSandbox({ + name: "alpha", + agent: "openclaw", + gatewayName: "nemoclaw", + mcp: { + bridges: { github: bridgeEntries.github }, + ${marker}: "2026-07-02T22:49:42.000Z", + }, +}); +registry.addCustomPolicy("alpha", ownedPolicy("github")); +const bridge = require("./src/lib/actions/sandbox/mcp-bridge.js"); +(async () => { + let message = ""; + try { + await bridge.${method}("alpha"); + } catch (error) { + message = error.message; + } + process.stdout.write(JSON.stringify({ + message, + sandbox: registry.getSandbox("alpha"), + calls, + adapterCalls, + })); +})().catch((error) => { console.error(error); process.exit(1); }); +`); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + const payload = JSON.parse(result.stdout) as { + message: string; + sandbox: { mcp: Record }; + calls: string[]; + adapterCalls: string[]; + }; + expect(payload.message).toContain("incomplete MCP destroy transaction"); + expect(payload.sandbox.mcp).toHaveProperty(marker); + expect(payload.calls).toEqual([]); + expect(payload.adapterCalls).toEqual([]); + }); + } + } + + it("prepares an absent-sandbox rebuild without adapter exec or provider detach", () => { + const result = runDestroyLifecycleScenario(` +delete process.env.GITHUB_TOKEN; +registry.registerSandbox({ + name: "alpha", + agent: "openclaw", + gatewayName: "nemoclaw", + mcp: { bridges: { github: bridgeEntries.github } }, +}); +registry.addCustomPolicy("alpha", ownedPolicy("github")); +policies.getPresetContentGatewayState = () => { throw new Error("absent rebuild queried live policy"); }; +const bridge = require("./src/lib/actions/sandbox/mcp-bridge.js"); +(async () => { + const preparation = await bridge.prepareMcpBridgesForAbsentSandboxRebuild("alpha"); + process.stdout.write(JSON.stringify({ + preparation, + providers: [...providers.keys()], + calls, + adapterCalls, + })); +})().catch((error) => { console.error(error); process.exit(1); }); +`); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + const payload = JSON.parse(result.stdout) as { + preparation: { + entries: unknown[]; + detachedProviderEntries: unknown[]; + scrubbedAdapterEntries: unknown[]; + }; + providers: string[]; + calls: string[]; + adapterCalls: string[]; + }; + expect(payload.preparation.entries).toHaveLength(1); + expect(payload.preparation.detachedProviderEntries).toEqual([]); + expect(payload.preparation.scrubbedAdapterEntries).toEqual([]); + expect(payload.calls).toEqual(["provider get alpha-mcp-github"]); + expect(payload.adapterCalls).toEqual([]); + expect(payload.providers).toContain("alpha-mcp-github"); + }); + + for (const method of ["prepareMcpBridgesForRebuild"] as const) { + it(`rejects policy drift before ${method} mutates adapter or provider state`, () => { + const result = runDestroyLifecycleScenario(` +registry.registerSandbox({ + name: "alpha", + agent: "openclaw", + gatewayName: "nemoclaw", + mcp: { bridges: { github: bridgeEntries.github } }, +}); +registry.addCustomPolicy("alpha", ownedPolicy("github")); +policies.getPresetContentGatewayState = () => "drift"; +const bridge = require("./src/lib/actions/sandbox/mcp-bridge.js"); +(async () => { + let message = ""; + try { + await bridge.${method}("alpha"); + } catch (error) { + message = error.message; + } + process.stdout.write(JSON.stringify({ message, calls, adapterCalls })); +})().catch((error) => { console.error(error); process.exit(1); }); +`); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + const payload = JSON.parse(result.stdout) as { + message: string; + calls: string[]; + adapterCalls: string[]; + }; + expect(payload.message).toMatch(/policy.*drift/i); + expect(payload.calls).toEqual([]); + expect(payload.adapterCalls).toEqual([]); + }); + } + + it("rejects an unowned same-name policy record during absent-sandbox rebuild", () => { + const result = runDestroyLifecycleScenario(` +registry.registerSandbox({ + name: "alpha", + agent: "openclaw", + gatewayName: "nemoclaw", + mcp: { bridges: { github: bridgeEntries.github } }, +}); +registry.addCustomPolicy("alpha", { + ...ownedPolicy("github"), + content: "operator-owned-content", + sourcePath: "/operator/policy.yaml", +}); +policies.getPresetContentGatewayState = () => { throw new Error("absent rebuild queried live policy"); }; +const bridge = require("./src/lib/actions/sandbox/mcp-bridge.js"); +(async () => { + let message = ""; + try { + await bridge.prepareMcpBridgesForAbsentSandboxRebuild("alpha"); + } catch (error) { + message = error.message; + } + process.stdout.write(JSON.stringify({ message, calls, adapterCalls })); +})().catch((error) => { console.error(error); process.exit(1); }); +`); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + const payload = JSON.parse(result.stdout) as { + message: string; + calls: string[]; + adapterCalls: string[]; + }; + expect(payload.message).toMatch(/unowned same-name registry record/); + expect(payload.calls).toEqual([]); + expect(payload.adapterCalls).toEqual([]); + }); + + it("finalizes an externally absent sandbox without attempting sandbox adapter exec", () => { + const result = runDestroyLifecycleScenario(` +registry.registerSandbox({ + name: "alpha", + agent: "openclaw", + mcp: { bridges: { github: bridgeEntries.github } }, +}); +registry.addCustomPolicy("alpha", ownedPolicy("github")); +const bridge = require("./src/lib/actions/sandbox/mcp-bridge.js"); +(async () => { + const preparation = await bridge.prepareMcpBridgesForAbsentSandboxDestroy("alpha"); + await bridge.finalizeMcpBridgesAfterSandboxDelete("alpha", preparation); + process.stdout.write(JSON.stringify({ + preparation, + sandbox: registry.getSandbox("alpha"), + providers: [...providers.keys()], + calls, + adapterCalls, + })); +})().catch((error) => { console.error(error); process.exit(1); }); +`); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + const payload = JSON.parse(result.stdout) as { + preparation: { entries: unknown[] }; + sandbox: { mcp?: unknown; customPolicies?: unknown }; + providers: string[]; + calls: string[]; + adapterCalls: string[]; + }; + expect(payload.preparation.entries).toHaveLength(1); + expect(payload.adapterCalls).toEqual([]); + expect(payload.calls.some((call) => call.includes("sandbox provider"))).toBe(false); + expect(payload.providers).not.toContain("alpha-mcp-github"); + expect(payload.sandbox.mcp).toBeUndefined(); + expect(payload.sandbox.customPolicies).toBeUndefined(); + }); + + it("restores policy, attachment, and adapter without rotating an exported host secret", () => { + const result = runDestroyLifecycleScenario(` +process.env.GITHUB_TOKEN = "ambient-value-that-must-not-rotate"; +registry.registerSandbox({ + name: "alpha", + agent: "openclaw", + mcp: { bridges: { github: bridgeEntries.github } }, +}); +registry.addCustomPolicy("alpha", ownedPolicy("github")); +const bridge = require("./src/lib/actions/sandbox/mcp-bridge.js"); +(async () => { + const preparation = await bridge.prepareMcpBridgesForDestroy("alpha"); + await bridge.restoreMcpBridgesAfterDestroyAbort("alpha", preparation); + process.stdout.write(JSON.stringify({ + sandbox: registry.getSandbox("alpha"), + providers: [...providers.keys()], + calls, + adapterCalls, + policyApplyCalls, + secretPresent: Object.prototype.hasOwnProperty.call(process.env, "GITHUB_TOKEN"), + })); +})().catch((error) => { console.error(error); process.exit(1); }); +`); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + const payload = JSON.parse(result.stdout.slice(result.stdout.indexOf("{"))) as { + sandbox: { + mcp: { + bridges: Record; + destroyPreparedAt?: string; + destroyPendingAt?: string; + }; + }; + providers: string[]; + calls: string[]; + adapterCalls: string[]; + policyApplyCalls: number; + secretPresent: boolean; + }; + expect(payload.secretPresent).toBe(true); + expect(payload.providers).toContain("alpha-mcp-github"); + expect( + payload.calls.some((call) => call === "sandbox provider attach alpha alpha-mcp-github"), + ).toBe(true); + expect(payload.calls.some((call) => /^provider (create|update) /.test(call))).toBe(false); + expect(payload.policyApplyCalls).toBe(1); + expect(payload.adapterCalls).toContain("command -v mcporter"); + expect( + payload.adapterCalls.some((call) => call.includes("openshell:resolve:env:GITHUB_TOKEN")), + ).toBe(true); + expect(payload.sandbox.mcp.bridges).toHaveProperty("github"); + expect(payload.sandbox.mcp.destroyPreparedAt).toBeUndefined(); + expect(payload.sandbox.mcp.destroyPendingAt).toBeUndefined(); + }); + + it("restores the durable destroy marker when abort rollback fails", () => { + const result = runDestroyLifecycleScenario(` +registry.registerSandbox({ + name: "alpha", + agent: "openclaw", + mcp: { bridges: { github: bridgeEntries.github } }, +}); +registry.addCustomPolicy("alpha", ownedPolicy("github")); +const bridge = require("./src/lib/actions/sandbox/mcp-bridge.js"); +(async () => { + const preparation = await bridge.prepareMcpBridgesForDestroy("alpha"); + policies.applyPresetContent = () => false; + let error = ""; + try { + await bridge.restoreMcpBridgesAfterDestroyAbort("alpha", preparation); + } catch (caught) { + error = caught instanceof Error ? caught.message : String(caught); + } + process.stdout.write(JSON.stringify({ + error, + sandbox: registry.getSandbox("alpha"), + attached: [...attachedProviders], + adapterRegistered, + })); +})().catch((error) => { console.error(error); process.exit(1); }); +`); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + const payload = JSON.parse(result.stdout) as { + error: string; + sandbox: { + mcp: { bridges: Record; destroyPreparedAt?: string }; + }; + attached: string[]; + adapterRegistered: boolean; + }; + expect(payload.error).toMatch(/failed to activate generated MCP policy/i); + expect(payload.sandbox.mcp.bridges).toHaveProperty("github"); + expect(payload.sandbox.mcp.destroyPreparedAt).toBeTruthy(); + expect(payload.attached).not.toContain("alpha-mcp-github"); + expect(payload.adapterRegistered).toBe(false); + }); + + it("preserves credentials and bridge state until sandbox deletion is confirmed", () => { + const result = runDestroyLifecycleScenario(` +registry.registerSandbox({ + name: "alpha", + agent: "openclaw", + mcp: { bridges: { github: bridgeEntries.github } }, +}); +registry.addCustomPolicy("alpha", ownedPolicy("github")); +registry.addCustomPolicy("alpha", { name: "operator", content: "version: 1\\n" }); +const bridge = require("./src/lib/actions/sandbox/mcp-bridge.js"); +(async () => { + const preparation = await bridge.prepareMcpBridgesForDestroy("alpha"); + const afterPrepare = registry.getSandbox("alpha"); + await bridge.finalizeMcpBridgesAfterSandboxDelete("alpha", preparation); + const afterFinalize = registry.getSandbox("alpha"); + process.stdout.write(JSON.stringify({ + afterPrepare, + afterFinalize, + providers: [...providers.keys()], + calls, + adapterCalls, + })); +})().catch((error) => { console.error(error); process.exit(1); }); +`); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + const payload = JSON.parse(result.stdout) as { + afterPrepare: { + mcp: { + bridges: Record; + destroyPreparedAt?: string; + destroyPendingAt?: string; + }; + customPolicies: Array<{ name: string }>; + }; + afterFinalize: { + mcp?: unknown; + customPolicies: Array<{ name: string }>; + }; + providers: string[]; + calls: string[]; + adapterCalls: string[]; + }; + expect(payload.afterPrepare.mcp.bridges).toHaveProperty("github"); + expect(payload.afterPrepare.mcp.destroyPreparedAt).toBeTruthy(); + expect(payload.afterPrepare.mcp.destroyPendingAt).toBeUndefined(); + expect(payload.afterPrepare.customPolicies.map((policy) => policy.name)).toContain( + "mcp-bridge-github", + ); + expect(payload.afterFinalize.mcp).toBeUndefined(); + expect(payload.afterFinalize.customPolicies.map((policy) => policy.name)).toEqual(["operator"]); + expect(payload.providers).not.toContain("alpha-mcp-github"); + expect( + payload.calls.some((call) => call === "sandbox provider detach alpha alpha-mcp-github"), + ).toBe(true); + expect( + payload.adapterCalls.some((call) => call.includes("config") && call.includes("remove")), + ).toBe(true); + }); + + it("restores a rebuilt sandbox without rotating an exported MCP credential", () => { + const result = runDestroyLifecycleScenario(` +process.env.GITHUB_TOKEN = "ambient-value-that-must-not-rotate"; +attachedProviders.delete("alpha-mcp-github"); +adapterRegistered = false; +registry.registerSandbox({ + name: "alpha", + agent: "openclaw", + mcp: { bridges: { github: bridgeEntries.github } }, +}); +registry.addCustomPolicy("alpha", ownedPolicy("github")); +const bridge = require("./src/lib/actions/sandbox/mcp-bridge.js"); +(async () => { + await bridge.restoreMcpBridgesAfterRebuild("alpha", [bridgeEntries.github]); + process.stdout.write(JSON.stringify({ + calls, + attached: [...attachedProviders], + adapterRegistered, + policyApplyCalls, + })); +})().catch((error) => { console.error(error); process.exit(1); }); +`); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + const payload = JSON.parse(result.stdout.slice(result.stdout.indexOf("{"))) as { + calls: string[]; + attached: string[]; + adapterRegistered: boolean; + policyApplyCalls: number; + }; + expect(payload.calls.some((call) => /^provider (create|update) /.test(call))).toBe(false); + expect(payload.attached).toContain("alpha-mcp-github"); + expect(payload.adapterRegistered).toBe(true); + expect(payload.policyApplyCalls).toBe(1); + }); + + for (const [label, prepareFunction] of [ + ["destroy", "prepareMcpBridgesForDestroy"], + ["rebuild", "prepareMcpBridgesForRebuild"], + ] as const) { + it(`reattaches an already-absent first provider when a later ${label} detach fails`, () => { + const result = runDestroyLifecycleScenario(` +registry.registerSandbox({ + name: "alpha", + agent: "openclaw", + gatewayName: "nemoclaw", + mcp: { bridges: bridgeEntries }, +}); +registry.addCustomPolicy("alpha", ownedPolicy("github")); +registry.addCustomPolicy("alpha", ownedPolicy("slack")); +// Simulate a prior process dying after the first detach but before a durable +// prepared marker. The retry must own rollback of this already-absent binding. +attachedProviders.delete("alpha-mcp-github"); +failProviderDetach = "alpha-mcp-slack"; +const bridge = require("./src/lib/actions/sandbox/mcp-bridge.js"); +(async () => { + let message = ""; + try { + await bridge.${prepareFunction}("alpha"); + } catch (error) { + message = error.message; + } + process.stdout.write(JSON.stringify({ + message, + attached: [...attachedProviders].sort(), + calls, + adapterRegistered, + })); +})().catch((error) => { console.error(error); process.exit(1); }); +`); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + const payload = JSON.parse(result.stdout) as { + message: string; + attached: string[]; + calls: string[]; + adapterRegistered: boolean; + }; + expect(payload.message).toContain("provider detach failed"); + expect(payload.attached).toEqual(["alpha-mcp-github", "alpha-mcp-slack"]); + expect( + payload.calls.some((call) => call === "sandbox provider attach alpha alpha-mcp-github"), + ).toBe(true); + expect(payload.adapterRegistered).toBe(true); + }); + } + + it("reattaches every desired provider when rebuild deletion aborts after a retry", () => { + const result = runDestroyLifecycleScenario(` +registry.registerSandbox({ + name: "alpha", + agent: "openclaw", + gatewayName: "nemoclaw", + mcp: { bridges: bridgeEntries }, +}); +registry.addCustomPolicy("alpha", ownedPolicy("github")); +registry.addCustomPolicy("alpha", ownedPolicy("slack")); +// The first rebuild process died after detaching github. A retry completes +// preparation, then sandbox deletion is modeled as failed by invoking abort. +attachedProviders.delete("alpha-mcp-github"); +const bridge = require("./src/lib/actions/sandbox/mcp-bridge.js"); +(async () => { + const preparation = await bridge.prepareMcpBridgesForRebuild("alpha"); + const detachedBeforeAbort = [...attachedProviders].sort(); + await bridge.reattachMcpProvidersAfterRebuildAbort( + "alpha", + preparation.detachedProviderEntries, + preparation.scrubbedAdapterEntries, + ); + process.stdout.write(JSON.stringify({ + preparation, + detachedBeforeAbort, + attachedAfterAbort: [...attachedProviders].sort(), + calls, + adapterRegistered, + })); +})().catch((error) => { console.error(error); process.exit(1); }); +`); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + const payload = JSON.parse(result.stdout) as { + preparation: { detachedProviderEntries: unknown[] }; + detachedBeforeAbort: string[]; + attachedAfterAbort: string[]; + calls: string[]; + adapterRegistered: boolean; + }; + expect(payload.preparation.detachedProviderEntries).toHaveLength(2); + expect(payload.detachedBeforeAbort).toEqual([]); + expect(payload.attachedAfterAbort).toEqual(["alpha-mcp-github", "alpha-mcp-slack"]); + expect( + payload.calls.some((call) => call === "sandbox provider attach alpha alpha-mcp-github"), + ).toBe(true); + expect(payload.adapterRegistered).toBe(true); + }); + + it("keeps a pending manifest after partial provider deletion and completes on retry", () => { + const result = runDestroyLifecycleScenario(` +registry.registerSandbox({ + name: "alpha", + agent: "openclaw", + mcp: { bridges: bridgeEntries }, +}); +registry.addCustomPolicy("alpha", ownedPolicy("github")); +registry.addCustomPolicy("alpha", ownedPolicy("slack")); +const bridge = require("./src/lib/actions/sandbox/mcp-bridge.js"); +(async () => { + const preparation = await bridge.prepareMcpBridgesForDestroy("alpha"); + failProviderDelete = "alpha-mcp-slack"; + let firstError = ""; + try { + await bridge.finalizeMcpBridgesAfterSandboxDelete("alpha", preparation, { force: true }); + } catch (error) { + firstError = error.message; + } + const afterFailure = registry.getSandbox("alpha"); + failProviderDelete = null; + const retry = await bridge.prepareMcpBridgesForDestroy("alpha"); + await bridge.finalizeMcpBridgesAfterSandboxDelete("alpha", retry, { force: true }); + process.stdout.write(JSON.stringify({ + firstError, + afterFailure, + retry, + afterRetry: registry.getSandbox("alpha"), + providers: [...providers.keys()], + calls, + })); +})().catch((error) => { console.error(error); process.exit(1); }); +`); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + const payload = JSON.parse(result.stdout) as { + firstError: string; + afterFailure: { + mcp: { + bridges: Record; + destroyPreparedAt?: string; + destroyPendingAt?: string; + }; + customPolicies: Array<{ name: string }>; + }; + retry: { destroyAlreadyPending: boolean }; + afterRetry: { mcp?: unknown; customPolicies?: unknown }; + providers: string[]; + calls: string[]; + }; + expect(payload.firstError).toContain("provider delete failed"); + expect(payload.afterFailure.mcp.destroyPendingAt).toBeTruthy(); + expect(payload.afterFailure.mcp.destroyPreparedAt).toBeUndefined(); + expect(Object.keys(payload.afterFailure.mcp.bridges)).toEqual(["github", "slack"]); + expect(payload.afterFailure.customPolicies).toHaveLength(2); + expect(payload.retry.destroyAlreadyPending).toBe(true); + expect(payload.afterRetry.mcp).toBeUndefined(); + expect(payload.afterRetry.customPolicies).toBeUndefined(); + expect(payload.providers).toEqual([]); + expect( + payload.calls.filter((call) => call === "sandbox provider detach alpha alpha-mcp-github"), + ).toHaveLength(1); + }); + + it("resumes from the durable prepared phase after delete-before-finalize interruption", () => { + const result = runDestroyLifecycleScenario(` +registry.registerSandbox({ + name: "alpha", + agent: "openclaw", + mcp: { bridges: { github: bridgeEntries.github } }, +}); +registry.addCustomPolicy("alpha", ownedPolicy("github")); +const bridge = require("./src/lib/actions/sandbox/mcp-bridge.js"); +(async () => { + await bridge.prepareMcpBridgesForDestroy("alpha"); + const callsAfterFirstPrepare = calls.length; + const adapterCallsAfterFirstPrepare = adapterCalls.length; + const retry = await bridge.prepareMcpBridgesForDestroy("alpha"); + await bridge.finalizeMcpBridgesAfterSandboxDelete("alpha", retry); + process.stdout.write(JSON.stringify({ + callsAfterFirstPrepare, + adapterCallsAfterFirstPrepare, + calls, + adapterCalls, + retry, + sandbox: registry.getSandbox("alpha"), + providers: [...providers.keys()], + })); +})().catch((error) => { console.error(error); process.exit(1); }); +`); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + const payload = JSON.parse(result.stdout) as { + callsAfterFirstPrepare: number; + adapterCallsAfterFirstPrepare: number; + calls: string[]; + adapterCalls: string[]; + retry: { + destroyAlreadyPrepared: boolean; + destroyAlreadyPending: boolean; + }; + sandbox: { mcp?: unknown }; + providers: string[]; + }; + expect(payload.retry.destroyAlreadyPrepared).toBe(true); + expect(payload.retry.destroyAlreadyPending).toBe(false); + expect( + payload.calls + .slice(0, payload.callsAfterFirstPrepare) + .some((call) => call === "sandbox provider detach alpha alpha-mcp-github"), + ).toBe(true); + expect( + payload.calls + .slice(payload.callsAfterFirstPrepare) + .filter((call) => call.includes("sandbox provider detach")), + ).toEqual([]); + expect(payload.adapterCalls).toHaveLength(payload.adapterCallsAfterFirstPrepare); + expect(payload.sandbox.mcp).toBeUndefined(); + expect(payload.providers).not.toContain("alpha-mcp-github"); + }); + + it("does not let force delete a drifted global provider", () => { + const result = runDestroyLifecycleScenario(` +providers.set("alpha-mcp-github", { + credential: "OTHER_TOKEN", + id: "11111111-2222-4333-8444-555555555555", +}); +registry.registerSandbox({ + name: "alpha", + agent: "openclaw", + mcp: { + bridges: { github: bridgeEntries.github }, + destroyPendingAt: "2026-06-27T01:00:00.000Z", + }, +}); +registry.addCustomPolicy("alpha", ownedPolicy("github")); +const bridge = require("./src/lib/actions/sandbox/mcp-bridge.js"); +(async () => { + const sandbox = registry.getSandbox("alpha"); + const preparation = { + entries: Object.values(sandbox.mcp.bridges), + detachedProviderEntries: [], + scrubbedAdapterEntries: [], + destroyAlreadyPrepared: false, + destroyAlreadyPending: true, + }; + let message = ""; + try { + await bridge.finalizeMcpBridgesAfterSandboxDelete("alpha", preparation, { force: true }); + } catch (error) { + message = error.message; + } + process.stdout.write(JSON.stringify({ + message, + sandbox: registry.getSandbox("alpha"), + providers: [...providers.keys()], + calls, + })); +})().catch((error) => { console.error(error); process.exit(1); }); +`); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + const payload = JSON.parse(result.stdout) as { + message: string; + sandbox: { mcp: { bridges: Record } }; + providers: string[]; + calls: string[]; + }; + expect(payload.message).toContain("no longer exactly matches"); + expect(payload.message).toContain("--force does not delete"); + expect(payload.sandbox.mcp.bridges).toHaveProperty("github"); + expect(payload.providers).toContain("alpha-mcp-github"); + expect(payload.calls.some((call) => call.startsWith("provider delete alpha-mcp-github "))).toBe( + false, + ); + }); +}); diff --git a/test/mcp-lifecycle-lock.test.ts b/test/mcp-lifecycle-lock.test.ts new file mode 100644 index 00000000000..1a2639b5472 --- /dev/null +++ b/test/mcp-lifecycle-lock.test.ts @@ -0,0 +1,645 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { type ChildProcess, spawn } from "node:child_process"; +import fs from "node:fs"; +import { createRequire } from "node:module"; +import { createServer } from "node:net"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import "./helpers/mcp-lifecycle-lock-properties"; + +type LifecycleLockModule = typeof import("../src/lib/state/mcp-lifecycle-lock"); + +const requireDist = createRequire(import.meta.url); +const lockModulePath = requireDist.resolve("../src/lib/state/mcp-lifecycle-lock.js"); +const lifecycleLock = requireDist(lockModulePath) as LifecycleLockModule; +const currentProcessIdentity = lifecycleLock.readMcpLockProcessIdentity(process.pid); +const currentHostIdentity = lifecycleLock.readMcpLockHostIdentity(); +const currentPidNamespaceIdentity = lifecycleLock.readMcpLockPidNamespaceIdentity(); + +let stateDir: string; +const children = new Set(); + +function options(overrides: Record = {}) { + return { + stateDir, + pollIntervalMs: 5, + timeoutMs: 1_000, + corruptLockGraceMs: 10, + ...overrides, + }; +} + +function deferred(): { promise: Promise; resolve: () => void } { + let resolve!: () => void; + const promise = new Promise((done) => { + resolve = done; + }); + return { promise, resolve }; +} + +function waitForLine(child: ChildProcess, expected: string): Promise { + return new Promise((resolve, reject) => { + let output = ""; + const timeout = setTimeout(() => reject(new Error(`Timed out waiting for ${expected}`)), 2_000); + child.once("error", reject); + child.stdout?.on("data", (chunk: Buffer) => { + output += chunk.toString("utf8"); + const matched = output.split(/\r?\n/).includes(expected); + switch (matched) { + case true: + clearTimeout(timeout); + resolve(); + } + }); + }); +} + +beforeEach(() => { + stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mcp-lock-")); +}); + +afterEach(() => { + vi.restoreAllMocks(); + for (const child of children) child.kill("SIGKILL"); + children.clear(); + fs.rmSync(stateDir, { recursive: true, force: true }); +}); + +describe("MCP lifecycle lock", () => { + it("does not forward an MCP credential to the macOS process-identity probe", () => { + const childProcess = requireDist("node:child_process"); + const platform = vi.spyOn(process, "platform", "get").mockReturnValue("darwin"); + const spawnSync = vi.spyOn(childProcess, "spawnSync").mockReturnValue({ + status: 0, + stdout: "Mon Jun 30 12:00:00 2026\n", + stderr: "", + } as never); + const priorSecret = process.env.TEST_MCP_RAW_TOKEN; + const priorGateway = process.env.OPENSHELL_GATEWAY; + process.env.TEST_MCP_RAW_TOKEN = "must-reach-only-provider-mutation"; + process.env.OPENSHELL_GATEWAY = "nemoclaw-19080"; + + try { + expect(lifecycleLock.readMcpLockProcessIdentity(4242, true)).toBe( + "darwin:Mon Jun 30 12:00:00 2026", + ); + const options = spawnSync.mock.calls[0]?.[2] as { env?: NodeJS.ProcessEnv }; + expect(options.env?.TEST_MCP_RAW_TOKEN).toBeUndefined(); + expect(options.env?.OPENSHELL_GATEWAY).toBe("nemoclaw-19080"); + expect(options.env?.PATH).toBe(process.env.PATH); + } finally { + priorSecret === undefined + ? delete process.env.TEST_MCP_RAW_TOKEN + : (process.env.TEST_MCP_RAW_TOKEN = priorSecret); + priorGateway === undefined + ? delete process.env.OPENSHELL_GATEWAY + : (process.env.OPENSHELL_GATEWAY = priorGateway); + platform.mockRestore(); + } + }); + + it.skipIf(process.platform === "win32")( + "does not follow a symlink when observing lock ownership", + async () => { + const lockPath = lifecycleLock.getMcpLifecycleLockPath("alpha", stateDir); + const targetPath = path.join(stateDir, "operator-owned-target"); + const target = `${JSON.stringify({ + version: 1, + sandboxName: "alpha", + pid: process.pid, + processIdentity: currentProcessIdentity, + token: "operator-owned-token", + acquiredAt: new Date().toISOString(), + })}\n`; + fs.mkdirSync(path.dirname(lockPath), { recursive: true }); + fs.writeFileSync(targetPath, target); + fs.symlinkSync(targetPath, lockPath); + + await expect( + lifecycleLock.withMcpLifecycleLock("alpha", () => "acquired", options()), + ).resolves.toBe("acquired"); + expect(fs.readFileSync(targetPath, "utf8")).toBe(target); + }, + ); + + it.skipIf(process.platform === "win32")( + "reaps a non-regular Unix socket found at the lock path", + async () => { + const shortStateDir = path.join("/tmp", `m${process.pid}`); + fs.rmSync(shortStateDir, { recursive: true, force: true }); + const lockPath = lifecycleLock.getMcpLifecycleLockPath("alpha", shortStateDir); + expect(Buffer.byteLength(lockPath)).toBeLessThan(104); + fs.mkdirSync(path.dirname(lockPath), { recursive: true }); + const server = createServer(); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(lockPath, resolve); + }); + expect(fs.lstatSync(lockPath).isSocket()).toBe(true); + + try { + await expect( + lifecycleLock.withMcpLifecycleLock("alpha", () => "acquired", { + ...options(), + stateDir: shortStateDir, + }), + ).resolves.toBe("acquired"); + } finally { + await new Promise((resolve) => server.close(() => resolve())); + fs.rmSync(shortStateDir, { recursive: true, force: true }); + } + }, + ); + + it("serializes separate top-level promises in one process", async () => { + const firstEntered = deferred(); + const releaseFirst = deferred(); + const order: string[] = []; + + const first = lifecycleLock.withMcpLifecycleLock( + "alpha", + async () => { + order.push("first-enter"); + firstEntered.resolve(); + await releaseFirst.promise; + order.push("first-exit"); + }, + options(), + ); + await firstEntered.promise; + + const second = lifecycleLock.withMcpLifecycleLock( + "alpha", + () => { + order.push("second-enter"); + }, + options(), + ); + await new Promise((resolve) => setTimeout(resolve, 30)); + expect(order).toEqual(["first-enter"]); + + releaseFirst.resolve(); + await Promise.all([first, second]); + expect(order).toEqual(["first-enter", "first-exit", "second-enter"]); + }); + + it("is reentrant only inside the same async lifecycle context", async () => { + const events: string[] = []; + await lifecycleLock.withMcpLifecycleLock( + "alpha", + async () => { + events.push("outer"); + await lifecycleLock.withMcpLifecycleLock( + "alpha", + () => events.push("nested"), + options({ timeoutMs: 50 }), + ); + }, + options(), + ); + expect(events).toEqual(["outer", "nested"]); + expect(fs.existsSync(lifecycleLock.getMcpLifecycleLockPath("alpha", stateDir))).toBe(false); + }); + + it("does not let a detached promise reuse an ended operation's lease", async () => { + const startDetached = deferred(); + const lockPath = lifecycleLock.getMcpLifecycleLockPath("alpha", stateDir); + let detached: Promise | undefined; + + await lifecycleLock.withMcpLifecycleLock( + "alpha", + () => { + detached = (async () => { + await startDetached.promise; + await lifecycleLock.withMcpLifecycleLock( + "alpha", + () => expect(fs.existsSync(lockPath)).toBe(true), + options(), + ); + })(); + }, + options(), + ); + + startDetached.resolve(); + await detached; + expect(fs.existsSync(lockPath)).toBe(false); + }); + + it("serializes a second Node process on the same sandbox", async () => { + const releasePath = path.join(stateDir, "release-child"); + const script = String.raw` +const fs = require("node:fs"); +const lock = require(process.argv[1]); +const stateDir = process.argv[2]; +const releasePath = process.argv[3]; +(async () => { + await lock.withMcpLifecycleLock("alpha", async () => { + process.stdout.write("READY\n"); + while (!fs.existsSync(releasePath)) { + await new Promise((resolve) => setTimeout(resolve, 5)); + } + }, { stateDir, pollIntervalMs: 5, timeoutMs: 2000 }); +})().then(() => process.exit(0), (error) => { + console.error(error); + process.exit(1); +}); +`; + const child = spawn(process.execPath, ["-e", script, lockModulePath, stateDir, releasePath], { + stdio: ["ignore", "pipe", "pipe"], + }); + children.add(child); + const childExit = new Promise((resolve, reject) => { + child.once("exit", (code) => (code === 0 ? resolve() : reject(new Error(`child ${code}`)))); + }); + await waitForLine(child, "READY"); + + let parentEntered = false; + const parent = lifecycleLock.withMcpLifecycleLock( + "alpha", + () => { + parentEntered = true; + }, + options({ timeoutMs: 2_000 }), + ); + await new Promise((resolve) => setTimeout(resolve, 40)); + expect(parentEntered).toBe(false); + + fs.writeFileSync(releasePath, "release\n"); + await parent; + expect(parentEntered).toBe(true); + await childExit; + children.delete(child); + }); + + it("recovers an atomic lock left by a dead owner", async () => { + const lockPath = lifecycleLock.getMcpLifecycleLockPath("alpha", stateDir); + fs.mkdirSync(path.dirname(lockPath), { recursive: true }); + fs.writeFileSync( + lockPath, + `${JSON.stringify({ + version: 1, + sandboxName: "alpha", + pid: 2_147_483_647, + processIdentity: "dead-process", + hostIdentity: currentHostIdentity, + pidNamespaceIdentity: currentPidNamespaceIdentity, + token: "stale-token", + acquiredAt: "2026-01-01T00:00:00.000Z", + })}\n`, + ); + + let entered = false; + await lifecycleLock.withMcpLifecycleLock( + "alpha", + () => { + entered = true; + }, + options(), + ); + expect(entered).toBe(true); + expect(fs.existsSync(lockPath)).toBe(false); + }); + + it("waits for a foreign-host owner instead of reaping it with local PID checks", async () => { + const lockPath = lifecycleLock.getMcpLifecycleLockPath("alpha", stateDir); + fs.mkdirSync(path.dirname(lockPath), { recursive: true }); + fs.writeFileSync( + lockPath, + `${JSON.stringify({ + version: 1, + sandboxName: "alpha", + pid: 2_147_483_647, + processIdentity: "foreign-process", + hostIdentity: `${currentHostIdentity}-foreign`, + pidNamespaceIdentity: currentPidNamespaceIdentity, + token: "foreign-host-token", + acquiredAt: "2026-01-01T00:00:00.000Z", + })}\n`, + ); + const old = new Date("2020-01-01T00:00:00.000Z"); + fs.utimesSync(lockPath, old, old); + + await expect( + lifecycleLock.withMcpLifecycleLock("alpha", () => undefined, options({ timeoutMs: 40 })), + ).rejects.toThrow("Timed out waiting for the sandbox mutation lock"); + expect(JSON.parse(fs.readFileSync(lockPath, "utf8")).token).toBe("foreign-host-token"); + }); + + it.each([ + ["unknown legacy host", {}], + [ + "foreign PID namespace", + { + hostIdentity: currentHostIdentity, + pidNamespaceIdentity: `${currentPidNamespaceIdentity ?? "unknown"}-foreign`, + }, + ], + ])("fails closed for an owner from an %s", async (_label, ownerLocation) => { + const lockPath = lifecycleLock.getMcpLifecycleLockPath("alpha", stateDir); + fs.mkdirSync(path.dirname(lockPath), { recursive: true }); + fs.writeFileSync( + lockPath, + `${JSON.stringify({ + version: 1, + sandboxName: "alpha", + pid: 2_147_483_647, + processIdentity: "unknown-process", + ...ownerLocation, + token: "untrusted-owner-token", + acquiredAt: "2026-01-01T00:00:00.000Z", + })}\n`, + ); + + await expect( + lifecycleLock.withMcpLifecycleLock("alpha", () => undefined, options({ timeoutMs: 40 })), + ).rejects.toThrow("Timed out waiting for the sandbox mutation lock"); + expect(JSON.parse(fs.readFileSync(lockPath, "utf8")).token).toBe("untrusted-owner-token"); + }); + + it("accepts ownership when LINK succeeded but its NFS reply reports EEXIST", async () => { + const lockPath = lifecycleLock.getMcpLifecycleLockPath("alpha", stateDir); + const link = fs.promises.link.bind(fs.promises); + let injectedAmbiguousReply = false; + const linkSpy = vi.spyOn(fs.promises, "link").mockImplementation(async (from, to) => { + await link(from, to); + const shouldInject = + !injectedAmbiguousReply && String(to) === lockPath && String(from).includes(".candidate-"); + switch (shouldInject) { + case true: + injectedAmbiguousReply = true; + throw Object.assign(new Error("simulated replayed LINK response"), { code: "EEXIST" }); + } + }); + + try { + await expect( + lifecycleLock.withMcpLifecycleLock("alpha", () => "acquired", options()), + ).resolves.toBe("acquired"); + } finally { + linkSpy.mockRestore(); + } + expect(injectedAmbiguousReply).toBe(true); + expect(fs.existsSync(lockPath)).toBe(false); + }); + + it("does not strand a canonical self-lock when candidate cleanup fails", async () => { + const lockPath = lifecycleLock.getMcpLifecycleLockPath("alpha", stateDir); + const rm = fs.promises.rm.bind(fs.promises); + let injectedCleanupFailure = false; + const rmSpy = vi.spyOn(fs.promises, "rm").mockImplementation(async (target, options) => { + const shouldInject = !injectedCleanupFailure && String(target).includes(".candidate-"); + switch (shouldInject) { + case true: + injectedCleanupFailure = true; + throw Object.assign(new Error("simulated candidate cleanup failure"), { code: "EIO" }); + } + return rm(target, options); + }); + + let entered = false; + try { + await expect( + lifecycleLock.withMcpLifecycleLock( + "alpha", + () => { + entered = true; + }, + options(), + ), + ).resolves.toBeUndefined(); + } finally { + rmSpy.mockRestore(); + } + expect(injectedCleanupFailure).toBe(true); + expect(entered).toBe(true); + expect(fs.existsSync(lockPath)).toBe(false); + }); + + it("waits for grace then recovers a stable truncated owner record", async () => { + const lockPath = lifecycleLock.getMcpLifecycleLockPath("alpha", stateDir); + fs.mkdirSync(path.dirname(lockPath), { recursive: true }); + fs.writeFileSync(lockPath, '{"version":1,"sandboxName":"alpha"'); + + await expect( + lifecycleLock.withMcpLifecycleLock( + "alpha", + () => undefined, + options({ timeoutMs: 30, corruptLockGraceMs: 100 }), + ), + ).rejects.toThrow("Timed out waiting for the sandbox mutation lock"); + expect(fs.readFileSync(lockPath, "utf8")).toContain('"sandboxName":"alpha"'); + + const future = new Date(Date.now() + 24 * 60 * 60_000); + fs.utimesSync(lockPath, future, future); + + await expect( + lifecycleLock.withMcpLifecycleLock( + "alpha", + () => "acquired", + options({ timeoutMs: 200, corruptLockGraceMs: 20 }), + ), + ).resolves.toBe("acquired"); + expect(fs.existsSync(lockPath)).toBe(false); + }); + + it("recovers a reaper whose owner was killed during stale-lock cleanup", async () => { + const lockPath = lifecycleLock.getMcpLifecycleLockPath("alpha", stateDir); + const reaperPath = `${lockPath}.reaper`; + fs.mkdirSync(path.dirname(lockPath), { recursive: true }); + fs.writeFileSync( + reaperPath, + `${JSON.stringify({ + version: 1, + sandboxName: "alpha", + pid: 2_147_483_647, + processIdentity: "killed-reaper", + hostIdentity: currentHostIdentity, + pidNamespaceIdentity: currentPidNamespaceIdentity, + token: "stale-reaper-token", + acquiredAt: "2026-01-01T00:00:00.000Z", + })}\n`, + ); + + let entered = false; + await lifecycleLock.withMcpLifecycleLock( + "alpha", + () => { + entered = true; + }, + options(), + ); + expect(entered).toBe(true); + expect(fs.existsSync(lockPath)).toBe(false); + expect(fs.existsSync(reaperPath)).toBe(false); + }); + + it("does not unlink a replacement reaper published during stale recovery", async () => { + const lockPath = lifecycleLock.getMcpLifecycleLockPath("alpha", stateDir); + const reaperPath = `${lockPath}.reaper`; + fs.mkdirSync(path.dirname(lockPath), { recursive: true }); + fs.writeFileSync( + reaperPath, + `${JSON.stringify({ + version: 1, + sandboxName: "alpha", + pid: 2_147_483_647, + processIdentity: "dead-reaper", + hostIdentity: currentHostIdentity, + pidNamespaceIdentity: currentPidNamespaceIdentity, + token: "observed-stale-token", + acquiredAt: "2026-01-01T00:00:00.000Z", + })}\n`, + ); + const replacement = { + version: 1, + sandboxName: "alpha", + pid: process.pid, + processIdentity: lifecycleLock.readMcpLockProcessIdentity(process.pid), + hostIdentity: currentHostIdentity, + pidNamespaceIdentity: currentPidNamespaceIdentity, + token: "replacement-reaper-token", + acquiredAt: new Date().toISOString(), + }; + const rename = fs.promises.rename.bind(fs.promises); + let injectedReplacement = false; + const renameSpy = vi.spyOn(fs.promises, "rename").mockImplementation(async (from, to) => { + const shouldInject = !injectedReplacement && String(from) === reaperPath; + switch (shouldInject) { + case true: + injectedReplacement = true; + fs.unlinkSync(reaperPath); + fs.writeFileSync(reaperPath, `${JSON.stringify(replacement)}\n`); + } + return rename(from, to); + }); + + try { + await expect( + lifecycleLock.withMcpLifecycleLock("alpha", () => undefined, options({ timeoutMs: 50 })), + ).rejects.toThrow("Timed out waiting for the sandbox mutation lock"); + } finally { + renameSpy.mockRestore(); + } + expect(JSON.parse(fs.readFileSync(reaperPath, "utf8")).token).toBe("replacement-reaper-token"); + }); + + it("does not delete a replacement main lock during stale recovery", async () => { + const lockPath = lifecycleLock.getMcpLifecycleLockPath("alpha", stateDir); + fs.mkdirSync(path.dirname(lockPath), { recursive: true }); + fs.writeFileSync( + lockPath, + `${JSON.stringify({ + version: 1, + sandboxName: "alpha", + pid: 2_147_483_647, + processIdentity: "dead-process", + hostIdentity: currentHostIdentity, + pidNamespaceIdentity: currentPidNamespaceIdentity, + token: "observed-stale-token", + acquiredAt: "2026-01-01T00:00:00.000Z", + })}\n`, + ); + const replacement = { + version: 1, + sandboxName: "alpha", + pid: process.pid, + processIdentity: lifecycleLock.readMcpLockProcessIdentity(process.pid), + hostIdentity: currentHostIdentity, + pidNamespaceIdentity: currentPidNamespaceIdentity, + token: "replacement-main-token", + acquiredAt: new Date().toISOString(), + }; + const rename = fs.promises.rename.bind(fs.promises); + let injectedReplacement = false; + const renameSpy = vi.spyOn(fs.promises, "rename").mockImplementation(async (from, to) => { + const shouldInject = !injectedReplacement && String(from) === lockPath; + switch (shouldInject) { + case true: + injectedReplacement = true; + fs.unlinkSync(lockPath); + fs.writeFileSync(lockPath, `${JSON.stringify(replacement)}\n`); + } + return rename(from, to); + }); + + try { + await expect( + lifecycleLock.withMcpLifecycleLock("alpha", () => undefined, options({ timeoutMs: 50 })), + ).rejects.toThrow("Timed out waiting for the sandbox mutation lock"); + } finally { + renameSpy.mockRestore(); + } + expect(JSON.parse(fs.readFileSync(lockPath, "utf8")).token).toBe("replacement-main-token"); + }); + + it.skipIf(currentProcessIdentity === null)( + "recovers a recycled PID by comparing process-start identity", + async () => { + const lockPath = lifecycleLock.getMcpLifecycleLockPath("alpha", stateDir); + fs.mkdirSync(path.dirname(lockPath), { recursive: true }); + fs.writeFileSync( + lockPath, + `${JSON.stringify({ + version: 1, + sandboxName: "alpha", + pid: process.pid, + processIdentity: `${String(currentProcessIdentity)}-different-start`, + hostIdentity: currentHostIdentity, + pidNamespaceIdentity: currentPidNamespaceIdentity, + token: "recycled-token", + acquiredAt: "2026-01-01T00:00:00.000Z", + })}\n`, + ); + + await expect( + lifecycleLock.withMcpLifecycleLock("alpha", () => undefined, options()), + ).resolves.toBeUndefined(); + expect(fs.existsSync(lockPath)).toBe(false); + }, + ); + + it("does not break a long-lived lock owned by the same process identity", async () => { + const lockPath = lifecycleLock.getMcpLifecycleLockPath("alpha", stateDir); + fs.mkdirSync(path.dirname(lockPath), { recursive: true }); + fs.writeFileSync( + lockPath, + `${JSON.stringify({ + version: 1, + sandboxName: "alpha", + pid: process.pid, + processIdentity: lifecycleLock.readMcpLockProcessIdentity(process.pid), + hostIdentity: currentHostIdentity, + pidNamespaceIdentity: currentPidNamespaceIdentity, + token: "active-token", + acquiredAt: "2020-01-01T00:00:00.000Z", + })}\n`, + ); + const old = new Date("2020-01-01T00:00:00.000Z"); + fs.utimesSync(lockPath, old, old); + + await expect( + lifecycleLock.withMcpLifecycleLock("alpha", () => undefined, options({ timeoutMs: 40 })), + ).rejects.toThrow("Timed out waiting for the sandbox mutation lock"); + expect(JSON.parse(fs.readFileSync(lockPath, "utf8")).token).toBe("active-token"); + }); + + it("never releases a lock whose owner token changed", async () => { + const lockPath = lifecycleLock.getMcpLifecycleLockPath("alpha", stateDir); + await lifecycleLock.withMcpLifecycleLock( + "alpha", + () => { + const owner = JSON.parse(fs.readFileSync(lockPath, "utf8")); + fs.writeFileSync(lockPath, `${JSON.stringify({ ...owner, token: "replacement-token" })}\n`); + }, + options(), + ); + + expect(JSON.parse(fs.readFileSync(lockPath, "utf8")).token).toBe("replacement-token"); + }); +}); diff --git a/test/mcp-openshell-workflow.test.ts b/test/mcp-openshell-workflow.test.ts new file mode 100644 index 00000000000..6be4b01d62f --- /dev/null +++ b/test/mcp-openshell-workflow.test.ts @@ -0,0 +1,49 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; + +import { describe, expect, it } from "vitest"; + +import credentialBoundaryManifest from "../src/lib/actions/sandbox/openshell-child-visible-credentials.v0.0.72.json"; +import { validateMcpOpenShellWorkflowBoundary } from "../tools/e2e/mcp-workflow-boundary.mts"; +import { readYaml } from "./helpers/e2e-workflow-contract"; + +type Blueprint = { + min_openshell_version?: string; + max_openshell_version?: string; +}; + +type E2eWorkflow = { + jobs?: Record }>; +}; + +describe("MCP OpenShell workflow boundary", () => { + it("keeps the setup docs aligned with the stable default", () => { + const setupDocs = fs.readFileSync("docs/deployment/set-up-mcp-bridge.mdx", "utf8"); + + expect(setupDocs).toContain( + `NemoClaw v0.0.74 defaults to the pinned stable OpenShell \`${credentialBoundaryManifest.openshellVersion}\` release`, + ); + expect(setupDocs).toContain( + "The optional OpenShell development channel is compatibility evidence only and is not a shipping target.", + ); + expect(setupDocs).not.toContain("requires an OpenShell build from current main"); + }); + + it("validates the unified stable and explicit-dev MCP workflow contract", () => { + expect(validateMcpOpenShellWorkflowBoundary()).toEqual([]); + }); + + it("keeps the credential manifest aligned with every shipping OpenShell version pin", () => { + const expected = credentialBoundaryManifest.openshellVersion; + const blueprint = readYaml("nemoclaw-blueprint/blueprint.yaml"); + const workflow = readYaml(".github/workflows/e2e.yaml"); + + expect(blueprint.min_openshell_version).toBe(expected); + expect(blueprint.max_openshell_version).toBe(expected); + expect( + workflow.jobs?.["openshell-gateway-auth-contract"]?.env?.NEMOCLAW_OPENSHELL_PIN_VERSION, + ).toBe(expected); + }); +}); diff --git a/test/mcp-policy-key-ownership.test.ts b/test/mcp-policy-key-ownership.test.ts new file mode 100644 index 00000000000..f82cfa95c20 --- /dev/null +++ b/test/mcp-policy-key-ownership.test.ts @@ -0,0 +1,561 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +const PRESET = `network_policies: + example: + name: generated-policy + endpoints: [] +`; + +function runApply( + expectedExistingNetworkPolicyContent: string | null, + liveName: string | null = "operator-owned", +) { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mcp-policy-owner-")); + const binDir = path.join(home, ".local", "bin"); + const callsPath = path.join(home, "calls.log"); + fs.mkdirSync(binDir, { recursive: true }); + fs.writeFileSync( + path.join(binDir, "openshell"), + `#!/bin/sh +printf '%s\n' "$*" >> ${JSON.stringify(callsPath)} +if [ "$1 $2" = "policy get" ]; then + printf 'Version: 1\nHash: test\n---\nversion: 1\n${ + liveName === null + ? "network_policies: {}" + : `network_policies:\n example:\n name: ${liveName}\n endpoints: []` + }\n' +fi +exit 0 +`, + { mode: 0o755 }, + ); + const script = ` +const registry = require("./src/lib/state/registry.js"); +const policies = require("./src/lib/policy/index.js"); +registry.registerSandbox({ name: "alpha" }); +const result = policies.applyPresetContent( + "alpha", + "mcp-bridge-example", + ${JSON.stringify(PRESET)}, + { + custom: { sourcePath: "generated:nemoclaw-mcp-bridge" }, + expectedExistingNetworkPolicyContent: ${JSON.stringify(expectedExistingNetworkPolicyContent)}, + }, +); +process.stdout.write("\\n__RESULT__" + JSON.stringify(result)); +`; + const result = spawnSync(process.execPath, ["-e", script], { + cwd: process.cwd(), + encoding: "utf8", + env: { + ...process.env, + HOME: home, + NEMOCLAW_OPENSHELL_BIN: path.join(binDir, "openshell"), + PATH: `${binDir}:/usr/bin:/bin`, + }, + }); + const calls = fs.existsSync(callsPath) ? fs.readFileSync(callsPath, "utf8") : ""; + fs.rmSync(home, { recursive: true, force: true }); + return { calls, result }; +} + +function runContentMatch(liveName: string) { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mcp-policy-match-")); + const binDir = path.join(home, ".local", "bin"); + fs.mkdirSync(binDir, { recursive: true }); + fs.writeFileSync( + path.join(binDir, "openshell"), + `#!/bin/sh +printf 'Version: 1\nHash: test\n---\nversion: 1\nnetwork_policies:\n example:\n name: ${liveName}\n endpoints: []\n' +`, + { mode: 0o755 }, + ); + const script = ` +const policies = require("./src/lib/policy/index.js"); +process.stdout.write(String(policies.presetContentMatchesGateway("alpha", ${JSON.stringify(PRESET)}))); +`; + const result = spawnSync(process.execPath, ["-e", script], { + cwd: process.cwd(), + encoding: "utf8", + env: { + ...process.env, + HOME: home, + NEMOCLAW_OPENSHELL_BIN: path.join(binDir, "openshell"), + PATH: `${binDir}:/usr/bin:/bin`, + }, + }); + fs.rmSync(home, { recursive: true, force: true }); + return result; +} + +function runFailedPolicyMutation(operation: "apply" | "remove") { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mcp-policy-failure-")); + const binDir = path.join(home, ".local", "bin"); + fs.mkdirSync(binDir, { recursive: true }); + fs.writeFileSync( + path.join(binDir, "openshell"), + `#!/bin/sh +if [ "$1 $2" = "policy get" ]; then + printf 'Version: 1\nHash: test\n---\nversion: 1\nnetwork_policies:\n example:\n name: generated-policy\n endpoints: []\n' + exit 0 +fi +if [ "$1 $2" = "policy set" ]; then + exit 19 +fi +exit 0 +`, + { mode: 0o755 }, + ); + const script = ` +const registry = require("./src/lib/state/registry.js"); +const policies = require("./src/lib/policy/index.js"); +registry.registerSandbox({ name: "alpha" }); +${ + operation === "remove" + ? `registry.addCustomPolicy("alpha", { + name: "mcp-bridge-example", + content: ${JSON.stringify(PRESET)}, + sourcePath: "generated:nemoclaw-mcp-bridge", +});` + : "" +} +const result = ${ + operation === "apply" + ? `policies.applyPresetContent( + "alpha", + "mcp-bridge-example", + ${JSON.stringify(PRESET)}, + { + custom: { sourcePath: "generated:nemoclaw-mcp-bridge" }, + expectedExistingNetworkPolicyContent: ${JSON.stringify(PRESET)}, + nonFatal: true, + }, +)` + : `policies.removePreset("alpha", "mcp-bridge-example", { nonFatal: true })` + }; +process.stdout.write("\\n__RESULT__" + JSON.stringify({ + result, + policies: registry.getCustomPolicies("alpha").map((policy) => policy.name), +})); +`; + const result = spawnSync(process.execPath, ["-e", script], { + cwd: process.cwd(), + encoding: "utf8", + env: { + ...process.env, + HOME: home, + NEMOCLAW_OPENSHELL_BIN: path.join(binDir, "openshell"), + PATH: `${binDir}:/usr/bin:/bin`, + }, + }); + fs.rmSync(home, { recursive: true, force: true }); + return result; +} + +function runSuccessfulPolicyRemoval(skipRegistryUpdate: boolean) { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mcp-policy-remove-success-")); + const binDir = path.join(home, ".local", "bin"); + fs.mkdirSync(binDir, { recursive: true }); + fs.writeFileSync( + path.join(binDir, "openshell"), + `#!/bin/sh +if [ "$1 $2" = "policy get" ]; then + printf 'Version: 1\nHash: test\n---\nversion: 1\nnetwork_policies:\n example:\n name: generated-policy\n endpoints: []\n' +fi +exit 0 +`, + { mode: 0o755 }, + ); + const script = ` +const registry = require("./src/lib/state/registry.js"); +const policies = require("./src/lib/policy/index.js"); +registry.registerSandbox({ name: "alpha" }); +registry.addCustomPolicy("alpha", { + name: "mcp-bridge-example", + content: ${JSON.stringify(PRESET)}, + sourcePath: "generated:nemoclaw-mcp-bridge", +}); +const result = policies.removePreset("alpha", "mcp-bridge-example", { + nonFatal: true, + skipRegistryUpdate: ${JSON.stringify(skipRegistryUpdate)}, +}); +process.stdout.write("\\n__RESULT__" + JSON.stringify({ + result, + policies: registry.getCustomPolicies("alpha").map((policy) => policy.name), +})); +`; + const result = spawnSync(process.execPath, ["-e", script], { + cwd: process.cwd(), + encoding: "utf8", + env: { + ...process.env, + HOME: home, + NEMOCLAW_OPENSHELL_BIN: path.join(binDir, "openshell"), + PATH: `${binDir}:/usr/bin:/bin`, + }, + }); + fs.rmSync(home, { recursive: true, force: true }); + return result; +} + +describe("MCP-generated network policy ownership", () => { + it("refuses to replace a same-key policy the bridge does not own", () => { + const { calls, result } = runApply(null); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + expect(result.stdout).toContain("__RESULT__false"); + expect(result.stderr).toContain("does not match the exact state owned"); + expect(calls).not.toContain("policy set"); + }); + + it("allows a registered bridge to refresh its owned key", () => { + const { calls, result } = runApply(PRESET, "generated-policy"); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + expect(result.stdout).toContain("__RESULT__true"); + expect(calls).toContain("policy set"); + }); + + it("refuses a same-key value changed after the caller's ownership proof", () => { + const { calls, result } = runApply(PRESET, "concurrent-writer"); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + expect(result.stdout).toContain("__RESULT__false"); + expect(result.stderr).toContain("does not match the exact state owned"); + expect(calls).not.toContain("policy set"); + }); + + it("refuses an owned key removed after the caller's ownership proof", () => { + const { calls, result } = runApply(PRESET, null); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + expect(result.stdout).toContain("__RESULT__false"); + expect(result.stderr).toContain("does not match the exact state owned"); + expect(calls).not.toContain("policy set"); + }); + + it("detects same-key live policy drift instead of reporting presence", () => { + expect(runContentMatch("operator-widened").stdout).toBe("false"); + expect(runContentMatch("generated-policy").stdout).toBe("true"); + }); + + it("returns control to MCP rollback when policy apply fails", () => { + const result = runFailedPolicyMutation("apply"); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + expect(result.stdout).toContain('__RESULT__{"result":false,"policies":[]}'); + expect(result.stderr).toContain("Failed to update policy"); + }); + + it("preserves MCP policy ownership state when policy removal fails", () => { + const result = runFailedPolicyMutation("remove"); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + expect(result.stdout).toContain('__RESULT__{"result":false,"policies":["mcp-bridge-example"]}'); + expect(result.stderr).toContain("Failed to update policy"); + }); + + it.each([ + [false, []], + [true, ["mcp-bridge-example"]], + ] as const)("supports ownership-preserving policy removal (skipRegistryUpdate=%s)", (skipRegistryUpdate, expectedPolicies) => { + const result = runSuccessfulPolicyRemoval(skipRegistryUpdate); + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + expect(result.stdout).toContain( + `__RESULT__${JSON.stringify({ result: true, policies: expectedPolicies })}`, + ); + }); + + it("does not delete an operator-owned same-key policy when add rolls back", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mcp-policy-lifecycle-")); + const binDir = path.join(home, ".local", "bin"); + const callsPath = path.join(home, "calls.log"); + fs.mkdirSync(binDir, { recursive: true }); + fs.writeFileSync( + path.join(binDir, "openshell"), + `#!/bin/sh +printf '%s\n' "$*" >> ${JSON.stringify(callsPath)} +if [ "$1 $2 $3" = "status --output json" ]; then + printf '%s\n' 'ready' + exit 0 +fi +if [ "$1 $2" = "provider get" ]; then + printf 'Provider not found\n' >&2 + exit 1 +fi +if [ "$1 $2" = "policy get" ]; then + printf 'Version: 1\nHash: test\n---\nversion: 1\nnetwork_policies:\n mcp_bridge_example:\n name: operator-owned\n endpoints: []\n' +fi +exit 0 +`, + { mode: 0o755 }, + ); + const script = ` +process.env.HOME = ${JSON.stringify(home)}; +process.env.COLLISION_TOKEN = "host-only-secret"; +const registry = require("./src/lib/state/registry.js"); +const gatewayRuntime = require("./src/lib/gateway-runtime-action.js"); +const processRecovery = require("./src/lib/actions/sandbox/process-recovery.js"); +gatewayRuntime.recoverNamedGatewayRuntime = async () => ({ + recovered: true, + attempted: false, + before: { state: "healthy_named" }, + after: { state: "healthy_named" }, +}); +processRecovery.executeSandboxCommand = () => ({ + status: 0, + stdout: "absent\\n", + stderr: "", +}); +processRecovery.executeSandboxExecCommand = () => ({ + status: 0, + stdout: "", + stderr: "", +}); +registry.registerSandbox({ name: "alpha", agent: "openclaw" }); +const bridge = require("./src/lib/actions/sandbox/mcp-bridge.js"); +bridge.addMcpBridge("alpha", { + server: "example", + url: "https://8.8.8.8/mcp", + env: [{ name: "COLLISION_TOKEN" }], +}).then( + () => process.exit(2), + (error) => { + process.stdout.write("\\n__RESULT__" + JSON.stringify({ + message: error.message, + customPolicies: registry.getCustomPolicies("alpha"), + })); + }, +); +`; + const result = spawnSync(process.execPath, ["-e", script], { + cwd: process.cwd(), + encoding: "utf8", + env: { + ...process.env, + HOME: home, + NEMOCLAW_OPENSHELL_BIN: path.join(binDir, "openshell"), + PATH: `${binDir}:/usr/bin:/bin`, + }, + }); + const calls = fs.existsSync(callsPath) ? fs.readFileSync(callsPath, "utf8") : ""; + fs.rmSync(home, { recursive: true, force: true }); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + expect(result.stdout).toContain( + "could not prove generated policy key 'mcp_bridge_example' absent", + ); + expect(result.stdout).toContain('"customPolicies":[]'); + expect(calls).not.toContain("provider create"); + expect(calls).not.toContain("provider delete"); + expect(calls).not.toContain("policy set"); + }); + + it("reserves policy ownership before the live gateway mutation", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mcp-policy-registry-failure-")); + const binDir = path.join(home, ".local", "bin"); + const callsPath = path.join(home, "calls.log"); + const providerStatePath = path.join(home, "provider.state"); + fs.mkdirSync(binDir, { recursive: true }); + fs.writeFileSync( + path.join(binDir, "openshell"), + `#!/bin/sh +printf '%s\n' "$*" >> ${JSON.stringify(callsPath)} +if [ "$1 $2 $3" = "status --output json" ]; then + printf '%s\n' 'ready' + exit 0 +fi +if [ "$1 $2 $3" = "sandbox provider list" ]; then + printf '%s\n' 'No providers attached to sandbox alpha.' + exit 0 +fi +if [ "$1 $2" = "provider get" ]; then + if [ -f ${JSON.stringify(providerStatePath)} ]; then + printf 'Id: 11111111-2222-4333-8444-555555555555\nType: generic\nResource version: 1\nCredential keys: RESERVATION_TOKEN\n' + exit 0 + fi + printf 'Provider not found\n' >&2 + exit 1 +fi +if [ "$1 $2" = "provider create" ]; then + : > ${JSON.stringify(providerStatePath)} + printf '%s\n' 'Created provider.' +fi +if [ "$1 $2" = "provider delete" ]; then + rm -f -- ${JSON.stringify(providerStatePath)} +fi +if [ "$1 $2" = "policy get" ]; then + printf 'Version: 1\nHash: test\n---\nversion: 1\nnetwork_policies: {}\n' +fi +exit 0 +`, + { mode: 0o755 }, + ); + const script = ` +process.env.HOME = ${JSON.stringify(home)}; +process.env.RESERVATION_TOKEN = "host-only-secret"; +const registry = require("./src/lib/state/registry.js"); +const gatewayRuntime = require("./src/lib/gateway-runtime-action.js"); +const processRecovery = require("./src/lib/actions/sandbox/process-recovery.js"); +gatewayRuntime.recoverNamedGatewayRuntime = async () => ({ + recovered: true, + attempted: false, + before: { state: "healthy_named" }, + after: { state: "healthy_named" }, +}); +processRecovery.executeSandboxCommand = () => ({ + status: 0, + stdout: "absent\\n", + stderr: "", +}); +processRecovery.executeSandboxExecCommand = () => ({ + status: 0, + stdout: "", + stderr: "", +}); +registry.registerSandbox({ name: "alpha", agent: "openclaw" }); +registry.addCustomPolicy = () => { throw new Error("injected registry write failure"); }; +const bridge = require("./src/lib/actions/sandbox/mcp-bridge.js"); +bridge.addMcpBridge("alpha", { + server: "reservation", + url: "https://8.8.8.8/mcp", + env: [{ name: "RESERVATION_TOKEN" }], +}).then( + () => process.exit(2), + (error) => process.stdout.write("\\n__RESULT__" + error.message), +); +`; + const result = spawnSync(process.execPath, ["-e", script], { + cwd: process.cwd(), + encoding: "utf8", + env: { + ...process.env, + HOME: home, + NEMOCLAW_OPENSHELL_BIN: path.join(binDir, "openshell"), + PATH: `${binDir}:/usr/bin:/bin`, + }, + }); + const calls = fs.existsSync(callsPath) ? fs.readFileSync(callsPath, "utf8") : ""; + fs.rmSync(home, { recursive: true, force: true }); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + expect(result.stdout).toContain("injected registry write failure"); + expect(calls).not.toContain("provider create"); + expect(calls).not.toContain("provider delete"); + expect(calls).not.toContain("policy set"); + }); + + it("refuses to overwrite a drifted owned policy during restart", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mcp-policy-drift-")); + const script = ` +process.env.HOME = ${JSON.stringify(home)}; +const registry = require("./src/lib/state/registry.js"); +const globalActions = require("./src/lib/actions/global.js"); +const gatewayRuntime = require("./src/lib/gateway-runtime-action.js"); +const policies = require("./src/lib/policy/index.js"); +const processRecovery = require("./src/lib/actions/sandbox/process-recovery.js"); +let applyCalled = false; +const providerCalls = []; + +gatewayRuntime.recoverNamedGatewayRuntime = async () => ({ + recovered: true, + attempted: false, + before: { state: "healthy_named" }, + after: { state: "healthy_named" }, +}); +globalActions.runOpenshellProviderCommand = (args) => { + if (args.join(" ") === "status --output json") { + return { + status: 0, + stdout: "ready", + stderr: "", + }; + } + providerCalls.push(args.join(" ")); + if (args[0] === "provider" && args[1] === "get") { + return { + status: 0, + stdout: "Type: generic\\nCredential keys: DRIFT_TOKEN\\n", + stderr: "", + }; + } + return { status: 0, stdout: "", stderr: "" }; +}; +policies.getPresetContentGatewayState = () => "drift"; +policies.applyPresetContent = () => { + applyCalled = true; + return true; +}; +processRecovery.executeSandboxExecCommand = () => ({ + status: 0, + stdout: "", + stderr: "", +}); +processRecovery.executeSandboxCommand = () => ({ + status: 0, + stdout: "registered\\n", + stderr: "", +}); + +const bridge = require("./src/lib/actions/sandbox/mcp-bridge.js"); +const entry = { + server: "example", + agent: "openclaw", + adapter: "mcporter", + url: "https://8.8.8.8/mcp", + env: ["DRIFT_TOKEN"], + providerName: "alpha-mcp-example", + policyName: "mcp-bridge-example", + addedAt: "2026-06-01T00:00:00.000Z", +}; +registry.registerSandbox({ + name: "alpha", + agent: "openclaw", + gatewayName: "nemoclaw", + mcp: { bridges: { example: entry } }, +}); +registry.addCustomPolicy("alpha", { + name: entry.policyName, + content: bridge.buildMcpBridgePolicyYaml(entry.server, entry.url, entry.adapter), + sourcePath: "generated:nemoclaw-mcp-bridge", +}); + +bridge.restartMcpBridge("alpha", "example").then( + () => process.exit(9), + (error) => { + process.stdout.write(JSON.stringify({ + message: error.message, + applyCalled, + providerCalls, + })); + process.exit(0); + }, +); +`; + const result = spawnSync(process.execPath, ["-e", script], { + cwd: process.cwd(), + encoding: "utf8", + env: { ...process.env, HOME: home }, + timeout: 30_000, + }); + fs.rmSync(home, { recursive: true, force: true }); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + const payload = JSON.parse(result.stdout) as { + message: string; + applyCalled: boolean; + providerCalls: string[]; + }; + expect(payload.message).toMatch(/policy.*drift/i); + expect(payload.applyCalled).toBe(false); + expect(payload.providerCalls).toEqual([]); + }); +}); diff --git a/test/mcp-policy-transition.test.ts b/test/mcp-policy-transition.test.ts new file mode 100644 index 00000000000..f5abe092761 --- /dev/null +++ b/test/mcp-policy-transition.test.ts @@ -0,0 +1,349 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +function runPolicyTransition( + mode: "crash-retry" | "post-set-crash" | "foreign-after-crash" | "rejected", +) { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mcp-policy-transition-")); + const script = String.raw` +process.env.HOME = ${JSON.stringify(home)}; +const mode = ${JSON.stringify(mode)}; +const registry = require("./src/lib/state/registry.js"); +const policies = require("./src/lib/policy/index.js"); +const generated = require("./src/lib/actions/sandbox/mcp-bridge-policy.js"); + +const entry = { + server: "example", + agent: "openclaw", + adapter: "mcporter", + url: "https://mcp.example.test/mcp", + env: ["MCP_TOKEN"], + providerName: "alpha-mcp-example", + providerId: "11111111-2222-4333-8444-555555555555", + policyName: "mcp-bridge-example", + addedAt: "2026-06-01T00:00:00.000Z", +}; +const oldContent = generated.buildMcpBridgePolicyYaml( + entry.server, + entry.url, + entry.adapter, + ["1.1.1.1"], +); +const desiredContent = generated.buildMcpBridgePolicyYaml( + entry.server, + entry.url, + entry.adapter, + ["8.8.8.8"], +); +let liveContent = oldContent; +let applyCalls = 0; + +registry.registerSandbox({ + name: "alpha", + agent: "openclaw", + mcp: { bridges: { example: entry } }, +}); +registry.addCustomPolicy("alpha", { + name: entry.policyName, + content: oldContent, + sourcePath: "generated:nemoclaw-mcp-bridge", +}); +policies.getPresetContentGatewayState = (_sandbox, candidate) => + candidate === liveContent ? "match" : "drift"; +policies.applyPresetContent = () => { + applyCalls += 1; + if (mode === "rejected") return false; + if (applyCalls === 1) { + if (mode === "post-set-crash") liveContent = desiredContent; + if (mode === "foreign-after-crash") liveContent = "foreign-policy-content"; + throw new Error("simulated process death after reservation"); + } + liveContent = desiredContent; + return true; +}; + +let firstError = ""; +try { + generated.applyGeneratedPolicy("alpha", entry, ["8.8.8.8"]); +} catch (error) { + firstError = error instanceof Error ? error.message : String(error); +} + +const afterFirst = registry.getCustomPolicies("alpha")[0]; +const presenceAfterFirst = generated.getPolicyPresence("alpha", entry); +const afterPresence = registry.getCustomPolicies("alpha")[0]; + +let retryError = ""; +if (mode !== "rejected") { + try { + generated.applyGeneratedPolicy("alpha", entry, ["8.8.8.8"]); + } catch (error) { + retryError = error instanceof Error ? error.message : String(error); + } +} +const afterRetry = registry.getCustomPolicies("alpha")[0]; + +process.stdout.write(JSON.stringify({ + firstError, + retryError, + applyCalls, + presenceAfterFirst, + pendingPreservedByStatus: afterPresence?.pendingContent === desiredContent, + afterFirst: { + contentIsOld: afterFirst?.content === oldContent, + pendingIsDesired: afterFirst?.pendingContent === desiredContent, + }, + afterRetry: { + contentIsOld: afterRetry?.content === oldContent, + contentIsDesired: afterRetry?.content === desiredContent, + hasPending: Object.hasOwn(afterRetry ?? {}, "pendingContent"), + }, +})); +`; + const result = spawnSync(process.execPath, ["-e", script], { + cwd: process.cwd(), + encoding: "utf8", + env: { ...process.env, HOME: home }, + timeout: 30_000, + }); + fs.rmSync(home, { recursive: true, force: true }); + return result; +} + +function runUnownedRegistryCollision(operation: "assert" | "apply") { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mcp-policy-unowned-")); + const script = ` +process.env.HOME = ${JSON.stringify(home)}; +const registry = require("./src/lib/state/registry.js"); +const policies = require("./src/lib/policy/index.js"); +const generated = require("./src/lib/actions/sandbox/mcp-bridge-policy.js"); +const entry = { + server: "example", + agent: "openclaw", + adapter: "mcporter", + url: "https://mcp.example.test/mcp", + env: ["MCP_TOKEN"], + providerName: "alpha-mcp-example", + policyName: "mcp-bridge-example", + addedAt: "2026-06-01T00:00:00.000Z", +}; +registry.registerSandbox({ name: "alpha", agent: "openclaw" }); +registry.addCustomPolicy("alpha", { + name: entry.policyName, + content: "operator-owned-content", + sourcePath: "/operator/policy.yaml", +}); +let applyCalled = false; +policies.getPresetContentGatewayState = () => "absent"; +policies.applyPresetContent = () => { applyCalled = true; return true; }; +let message = ""; +try { + if (${JSON.stringify(operation)} === "assert") { + generated.assertGeneratedPolicyMutationSafe("alpha", entry); + } else { + generated.applyGeneratedPolicy("alpha", entry, ["8.8.8.8"]); + } +} catch (error) { + message = error instanceof Error ? error.message : String(error); +} +process.stdout.write(JSON.stringify({ + message, + applyCalled, + policies: registry.getCustomPolicies("alpha"), +})); +`; + const result = spawnSync(process.execPath, ["-e", script], { + cwd: process.cwd(), + encoding: "utf8", + env: { ...process.env, HOME: home }, + }); + fs.rmSync(home, { recursive: true, force: true }); + return result; +} + +function runGeneratedPolicyRemoval(postRemovalState: "absent" | "match") { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mcp-policy-remove-")); + const script = ` +process.env.HOME = ${JSON.stringify(home)}; +const registry = require("./src/lib/state/registry.js"); +const policies = require("./src/lib/policy/index.js"); +const generated = require("./src/lib/actions/sandbox/mcp-bridge-policy.js"); +const entry = { + server: "example", + agent: "openclaw", + adapter: "mcporter", + url: "https://mcp.example.test/mcp", + env: ["MCP_TOKEN"], + providerName: "alpha-mcp-example", + policyName: "mcp-bridge-example", + addedAt: "2026-06-01T00:00:00.000Z", +}; +const content = generated.buildMcpBridgePolicyYaml( + entry.server, + entry.url, + entry.adapter, + ["8.8.8.8"], +); +registry.registerSandbox({ name: "alpha", agent: "openclaw" }); +registry.addCustomPolicy("alpha", { + name: entry.policyName, + content, + sourcePath: "generated:nemoclaw-mcp-bridge", +}); +let state = "match"; +let skipRegistryUpdate = false; +policies.getPresetContentGatewayState = () => state; +policies.removePreset = (_sandbox, _policyName, options) => { + skipRegistryUpdate = options?.skipRegistryUpdate === true; + if (!skipRegistryUpdate) registry.removeCustomPolicyByName("alpha", entry.policyName); + state = ${JSON.stringify(postRemovalState)}; + return true; +}; +let message = ""; +try { + generated.removeGeneratedPolicy("alpha", entry); +} catch (error) { + message = error instanceof Error ? error.message : String(error); +} +process.stdout.write(JSON.stringify({ + message, + skipRegistryUpdate, + policies: registry.getCustomPolicies("alpha"), +})); +`; + const result = spawnSync(process.execPath, ["-e", script], { + cwd: process.cwd(), + encoding: "utf8", + env: { ...process.env, HOME: home }, + }); + fs.rmSync(home, { recursive: true, force: true }); + return result; +} + +describe("generated MCP policy transitions", () => { + it.each([ + "assert", + "apply", + ] as const)("preserves an unowned same-name registry record during %s", (operation) => { + const result = runUnownedRegistryCollision(operation); + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + const payload = JSON.parse(result.stdout) as { + message: string; + applyCalled: boolean; + policies: Array<{ content: string; sourcePath: string }>; + }; + expect(payload.message).toMatch(/unowned same-name registry record/); + expect(payload.applyCalled).toBe(false); + expect(payload.policies).toEqual([ + expect.objectContaining({ + content: "operator-owned-content", + sourcePath: "/operator/policy.yaml", + }), + ]); + }); + + it("preserves the confirmed and desired policy across an interrupted refresh", () => { + const result = runPolicyTransition("crash-retry"); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + const payload = JSON.parse(result.stdout) as { + firstError: string; + retryError: string; + applyCalls: number; + presenceAfterFirst: boolean; + pendingPreservedByStatus: boolean; + afterFirst: { contentIsOld: boolean; pendingIsDesired: boolean }; + afterRetry: { contentIsDesired: boolean; hasPending: boolean }; + }; + expect(payload).toMatchObject({ + firstError: "simulated process death after reservation", + retryError: "", + applyCalls: 2, + presenceAfterFirst: true, + pendingPreservedByStatus: true, + afterFirst: { contentIsOld: true, pendingIsDesired: true }, + afterRetry: { contentIsDesired: true, hasPending: false }, + }); + }); + + it("restores confirmed ownership when a changed policy is rejected", () => { + const result = runPolicyTransition("rejected"); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + const payload = JSON.parse(result.stdout) as { + firstError: string; + applyCalls: number; + afterFirst: { contentIsOld: boolean; pendingIsDesired: boolean }; + afterRetry: { contentIsOld: boolean; hasPending: boolean }; + }; + expect(payload.firstError).toContain("Failed to activate generated MCP policy"); + expect(payload).toMatchObject({ + applyCalls: 1, + afterFirst: { contentIsOld: true, pendingIsDesired: false }, + afterRetry: { contentIsOld: true, hasPending: false }, + }); + }); + + it("finalizes desired ownership after policy load wins the crash boundary", () => { + const result = runPolicyTransition("post-set-crash"); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + const payload = JSON.parse(result.stdout) as { + retryError: string; + applyCalls: number; + presenceAfterFirst: boolean; + pendingPreservedByStatus: boolean; + afterFirst: { contentIsOld: boolean; pendingIsDesired: boolean }; + afterRetry: { contentIsDesired: boolean; hasPending: boolean }; + }; + expect(payload).toMatchObject({ + retryError: "", + applyCalls: 2, + presenceAfterFirst: true, + pendingPreservedByStatus: true, + afterFirst: { contentIsOld: true, pendingIsDesired: true }, + afterRetry: { contentIsDesired: true, hasPending: false }, + }); + }); + + it("keeps both versions and fails closed when live policy matches neither", () => { + const result = runPolicyTransition("foreign-after-crash"); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + const payload = JSON.parse(result.stdout) as { + retryError: string; + applyCalls: number; + afterRetry: { contentIsOld: boolean; hasPending: boolean }; + }; + expect(payload.retryError).toMatch(/drifted|could not be inspected/); + expect(payload).toMatchObject({ + applyCalls: 1, + afterRetry: { contentIsOld: true, hasPending: true }, + }); + }); + + it.each([ + ["absent", false], + ["match", true], + ] as const)("requires exact post-removal state %s before dropping ownership", (postRemovalState, preservesOwnership) => { + const result = runGeneratedPolicyRemoval(postRemovalState); + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + const payload = JSON.parse(result.stdout) as { + message: string; + skipRegistryUpdate: boolean; + policies: Array<{ content: string; sourcePath: string }>; + }; + expect(payload.skipRegistryUpdate).toBe(true); + expect(payload.message).toMatch(preservesOwnership ? /effective state: match/ : /^$/); + expect(payload.policies.map((policy) => policy.sourcePath)).toEqual( + preservesOwnership ? ["generated:nemoclaw-mcp-bridge"] : [], + ); + }); +}); diff --git a/test/mcp-provider-ownership.test.ts b/test/mcp-provider-ownership.test.ts new file mode 100644 index 00000000000..f1b9033ef9e --- /dev/null +++ b/test/mcp-provider-ownership.test.ts @@ -0,0 +1,575 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +function runRemoveIdentityRace(swapAt: "detach" | "delete") { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mcp-provider-race-")); + const script = ` +process.env.HOME = ${JSON.stringify(home)}; +const swapAt = ${JSON.stringify(swapAt)}; +const registry = require("./src/lib/state/registry.js"); +const agentDefs = require("./src/lib/agent/defs.js"); +const gatewayRuntime = require("./src/lib/gateway-runtime-action.js"); +const policies = require("./src/lib/policy/index.js"); +const processRecovery = require("./src/lib/actions/sandbox/process-recovery.js"); +const globalActions = require("./src/lib/actions/global.js"); +const expectedId = "11111111-2222-4333-8444-555555555555"; +const foreignId = "99999999-8888-4777-8666-555555555555"; +let liveId = expectedId; +let attached = true; +let policyState = "match"; +const calls = []; +agentDefs.loadAgent = () => { throw new Error("persisted adapter must be used"); }; +gatewayRuntime.recoverNamedGatewayRuntime = async () => ({ + recovered: true, + attempted: false, + before: { state: "healthy_named" }, + after: { state: "healthy_named" }, +}); +globalActions.runOpenshellProviderCommand = (args) => { + calls.push(args.join(" ")); + if (args[0] === "status") { + return { status: 0, stdout: "ready", stderr: "" }; + } + if (args[0] === "provider" && args[1] === "get") { + return { + status: 0, + stdout: "Id: " + liveId + "\\nType: generic\\nResource version: 4\\nCredential keys: EXPECTED_TOKEN\\n", + stderr: "", + }; + } + if (args[0] === "sandbox" && args[1] === "provider" && args[2] === "list") { + return { + status: 0, + stdout: attached + ? "NAME TYPE CREDENTIAL_KEYS CONFIG_KEYS\\nalpha-mcp-fake generic 1 0\\n" + : "No providers attached to sandbox alpha.\\n", + stderr: "", + }; + } + if (args[0] === "sandbox" && args[1] === "provider" && args[2] === "detach") { + attached = false; + return { status: 0, stdout: "detached", stderr: "" }; + } + return { status: 0, stdout: "", stderr: "" }; +}; +policies.getPresetContentGatewayState = () => policyState; +policies.removePreset = () => { + if (swapAt === "delete") liveId = foreignId; + policyState = "absent"; + return true; +}; +processRecovery.executeSandboxCommand = () => { + if (swapAt === "detach") liveId = foreignId; + return { status: 0, stdout: "", stderr: "" }; +}; +processRecovery.executeSandboxExecCommand = () => ({ status: 0, stdout: "", stderr: "" }); +const entry = { + server: "fake", + agent: "openclaw", + url: "https://mcp.example.test/mcp", + env: ["EXPECTED_TOKEN"], + providerName: "alpha-mcp-fake", + providerId: expectedId, + policyName: "mcp-bridge-fake", + adapter: "mcporter", + addedAt: "2026-06-01T00:00:00.000Z", +}; +registry.registerSandbox({ + name: "alpha", + agent: "legacy-disabled", + mcp: { bridges: { fake: entry } }, +}); +registry.addCustomPolicy("alpha", { + name: entry.policyName, + content: "network_policies: {}\\n", + sourcePath: "generated:nemoclaw-mcp-bridge", +}); +const bridge = require("./src/lib/actions/sandbox/mcp-bridge.js"); +bridge.removeMcpBridge("alpha", "fake").then( + () => process.exit(9), + (error) => process.stdout.write(JSON.stringify({ + message: error.message, + calls, + bridgePresent: !!registry.getSandbox("alpha")?.mcp?.bridges?.fake, + })), +); +`; + const result = spawnSync(process.execPath, ["-e", script], { + cwd: process.cwd(), + encoding: "utf8", + env: { ...process.env, HOME: home }, + }); + fs.rmSync(home, { recursive: true, force: true }); + return result; +} + +function runLegacyReservedCredentialCleanup() { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mcp-provider-legacy-cleanup-")); + const script = String.raw` +process.env.HOME = ${JSON.stringify(home)}; +const registry = require("./src/lib/state/registry.js"); +const agentDefs = require("./src/lib/agent/defs.js"); +const gatewayRuntime = require("./src/lib/gateway-runtime-action.js"); +const policies = require("./src/lib/policy/index.js"); +const processRecovery = require("./src/lib/actions/sandbox/process-recovery.js"); +const globalActions = require("./src/lib/actions/global.js"); +const expectedId = "11111111-2222-4333-8444-555555555555"; +let providerExists = true; +let attached = true; +let policyState = "match"; +const calls = []; +agentDefs.loadAgent = () => { throw new Error("persisted adapter must be used"); }; +gatewayRuntime.recoverNamedGatewayRuntime = async () => ({ + recovered: true, + attempted: false, + before: { state: "healthy_named" }, + after: { state: "healthy_named" }, +}); +globalActions.runOpenshellProviderCommand = (args) => { + calls.push(args.join(" ")); + if (args[0] === "status") return { status: 0, stdout: "ready", stderr: "" }; + if (args[0] === "provider" && args[1] === "get") { + return providerExists + ? { + status: 0, + stdout: "Id: " + expectedId + "\nType: generic\nResource version: 4\nCredential keys: LD_PRELOAD\n", + stderr: "", + } + : { status: 1, stdout: "", stderr: "NotFound: provider" }; + } + if (args[0] === "sandbox" && args[1] === "provider" && args[2] === "list") { + return { + status: 0, + stdout: attached + ? "NAME TYPE CREDENTIAL_KEYS CONFIG_KEYS\nalpha-mcp-fake generic 1 0\n" + : "No providers attached to sandbox alpha.\n", + stderr: "", + }; + } + if (args[0] === "sandbox" && args[1] === "provider" && args[2] === "detach") { + attached = false; + return { status: 0, stdout: "Detached provider alpha-mcp-fake from sandbox alpha.", stderr: "" }; + } + if (args[0] === "provider" && args[1] === "delete") { + providerExists = false; + return { status: 0, stdout: "deleted", stderr: "" }; + } + throw new Error("unexpected call: " + args.join(" ")); +}; +policies.getPresetContentGatewayState = () => policyState; +policies.removePreset = () => { policyState = "absent"; return true; }; +const runSandboxChild = () => { + calls.push("sandbox-child attached=" + attached); + if (attached) throw new Error("sandbox child started while LD_PRELOAD remained attached"); + return { status: 0, stdout: "", stderr: "" }; +}; +processRecovery.executeSandboxCommand = runSandboxChild; +processRecovery.executeSandboxExecCommand = runSandboxChild; +const entry = { + server: "fake", + agent: "openclaw", + adapter: "mcporter", + url: "https://mcp.example.test/mcp", + env: ["LD_PRELOAD"], + providerName: "alpha-mcp-fake", + providerId: expectedId, + policyName: "mcp-bridge-fake", + addedAt: "2026-06-01T00:00:00.000Z", +}; +registry.registerSandbox({ + name: "alpha", + agent: "legacy-disabled", + mcp: { bridges: { fake: entry } }, +}); +registry.addCustomPolicy("alpha", { + name: entry.policyName, + content: "network_policies: {}\n", + sourcePath: "generated:nemoclaw-mcp-bridge", +}); +const bridge = require("./src/lib/actions/sandbox/mcp-bridge.js"); +bridge.removeMcpBridge("alpha", "fake").then( + () => process.stdout.write(JSON.stringify({ + calls, + bridgePresent: !!registry.getSandbox("alpha")?.mcp?.bridges?.fake, + providerExists, + attached, + })), + (error) => { console.error(error); process.exit(1); }, +); +`; + const result = spawnSync(process.execPath, ["-e", script], { + cwd: process.cwd(), + encoding: "utf8", + env: { ...process.env, HOME: home }, + }); + fs.rmSync(home, { recursive: true, force: true }); + return result; +} + +describe("MCP provider ownership", () => { + for (const boundary of ["detach", "delete"] as const) { + it(`rechecks stable identity immediately before provider ${boundary}`, () => { + const result = runRemoveIdentityRace(boundary); + + expect(result.status, `${result.stdout}\\n${result.stderr}`).toBe(0); + const payload = JSON.parse(result.stdout) as { + message: string; + calls: string[]; + bridgePresent: boolean; + }; + expect(payload.message).toContain("Expected stable provider ID"); + expect(payload.calls.some((call) => call.startsWith("provider delete alpha-mcp-fake"))).toBe( + false, + ); + expect( + payload.calls.some((call) => + call.startsWith("sandbox provider detach alpha alpha-mcp-fake"), + ), + ).toBe(boundary === "delete"); + expect(payload.bridgePresent).toBe(true); + }); + } + + it("removes an exact legacy provider whose credential name is now reserved", () => { + const result = runLegacyReservedCredentialCleanup(); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + const payload = JSON.parse(result.stdout.slice(result.stdout.indexOf("{"))) as { + calls: string[]; + bridgePresent: boolean; + providerExists: boolean; + attached: boolean; + }; + expect(payload).toMatchObject({ + bridgePresent: false, + providerExists: false, + attached: false, + }); + expect(payload.calls).toContain("sandbox provider detach alpha alpha-mcp-fake"); + expect(payload.calls).toContain("provider delete alpha-mcp-fake"); + expect(payload.calls.indexOf("sandbox provider detach alpha alpha-mcp-fake")).toBeLessThan( + payload.calls.indexOf("sandbox-child attached=false"), + ); + }); + + it("reports a same-shape provider with a different stable ID as drift", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mcp-provider-status-owner-")); + const script = String.raw` +process.env.HOME = ${JSON.stringify(home)}; +process.env.EXPECTED_TOKEN = "host-only-secret"; +const registry = require("./src/lib/state/registry.js"); +const agentDefs = require("./src/lib/agent/defs.js"); +const gatewayRuntime = require("./src/lib/gateway-runtime-action.js"); +const globalActions = require("./src/lib/actions/global.js"); +const processRecovery = require("./src/lib/actions/sandbox/process-recovery.js"); +agentDefs.loadAgent = () => ({ + name: "openclaw", + displayName: "OpenClaw", + mcpCapability: { support: "bridge", adapter: "mcporter" }, +}); +gatewayRuntime.recoverNamedGatewayRuntime = async () => ({ + recovered: true, + attempted: false, + before: { state: "healthy_named" }, + after: { state: "healthy_named" }, +}); +globalActions.runOpenshellProviderCommand = (args) => { + if (args[0] === "provider" && args[1] === "get") { + return { + status: 0, + stdout: "Id: 99999999-8888-4777-8666-555555555555\nType: generic\nResource version: 4\nCredential keys: EXPECTED_TOKEN\n", + stderr: "", + }; + } + if (args[0] === "sandbox" && args[1] === "provider" && args[2] === "list") { + return { + status: 0, + stdout: "NAME TYPE CREDENTIAL_KEYS CONFIG_KEYS\nalpha-mcp-fake generic 1 0\n", + stderr: "", + }; + } + throw new Error("unexpected call: " + args.join(" ")); +}; +processRecovery.executeSandboxCommand = () => ({ + status: 0, + stdout: "registered\\n", + stderr: "", +}); +registry.registerSandbox({ + name: "alpha", + agent: "openclaw", + mcp: { bridges: { fake: { + server: "fake", + agent: "openclaw", + url: "https://mcp.example.test/mcp", + env: ["EXPECTED_TOKEN"], + providerName: "alpha-mcp-fake", + providerId: "11111111-2222-4333-8444-555555555555", + policyName: "mcp-bridge-fake", + adapter: "mcporter", + addedAt: "2026-06-01T00:00:00.000Z", + } } }, +}); +const bridge = require("./src/lib/actions/sandbox/mcp-bridge.js"); +bridge.statusMcpBridge("alpha", "fake").then( + (statuses) => process.stdout.write(JSON.stringify(statuses[0])), + (error) => { console.error(error); process.exit(1); }, +); +`; + const result = spawnSync(process.execPath, ["-e", script], { + cwd: process.cwd(), + encoding: "utf8", + env: { ...process.env, HOME: home }, + }); + fs.rmSync(home, { recursive: true, force: true }); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + const status = JSON.parse(result.stdout) as { + env: { ready: boolean }; + provider: { credentialReady: boolean; detail?: string }; + }; + expect(status.env.ready).toBe(false); + expect(status.provider.credentialReady).toBe(false); + expect(status.provider.detail).toContain("Expected stable provider ID"); + }); + + it("clears multiple dangling stock OpenShell provider references without listing between them", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mcp-provider-dangling-")); + const script = String.raw` +process.env.HOME = ${JSON.stringify(home)}; +const globalActions = require("./src/lib/actions/global.js"); +const calls = []; +const attached = new Set(["alpha-mcp-fake", "alpha-mcp-second"]); +globalActions.runOpenshellProviderCommand = (args) => { + calls.push(args.join(" ")); + if (args[0] === "provider" && args[1] === "get") { + return { status: 1, stdout: "", stderr: "NotFound: provider" }; + } + if (args[0] === "sandbox" && args[1] === "provider" && args[2] === "list") { + return attached.size > 0 + ? { status: 9, stdout: "", stderr: "FailedPrecondition: provider '" + [...attached][0] + "' not found" } + : { status: 0, stdout: "No providers attached to sandbox alpha.\n", stderr: "" }; + } + if (args[0] === "sandbox" && args[1] === "provider" && args[2] === "detach") { + attached.delete(args[4]); + return { + status: 0, + stdout: "Detached provider " + args[4] + " from sandbox alpha.\n", + stderr: "", + }; + } + throw new Error("unexpected call: " + args.join(" ")); +}; +const providerActions = require("./src/lib/actions/sandbox/mcp-bridge-provider.js"); +const entry = { + server: "fake", + agent: "openclaw", + url: "https://mcp.example.test/mcp", + env: ["EXPECTED_TOKEN"], + providerName: "alpha-mcp-fake", + providerId: "11111111-2222-4333-8444-555555555555", + policyName: "mcp-bridge-fake", + adapter: "mcporter", + addedAt: "2026-06-01T00:00:00.000Z", +}; +const before = providerActions.inspectMcpProviderAttachments("alpha"); +const firstOutcome = providerActions.detachMissingProviderReference("alpha", entry); +const afterFirst = providerActions.inspectMcpProviderAttachments("alpha"); +const secondOutcome = providerActions.detachMissingProviderReference("alpha", { + ...entry, + server: "second", + providerName: "alpha-mcp-second", + providerId: "22222222-3333-4444-8555-666666666666", + policyName: "mcp-bridge-second", +}); +const after = providerActions.inspectMcpProviderAttachments("alpha"); +process.stdout.write(JSON.stringify({ before, firstOutcome, afterFirst, secondOutcome, after, calls })); +`; + const result = spawnSync(process.execPath, ["-e", script], { + cwd: process.cwd(), + encoding: "utf8", + env: { ...process.env, HOME: home }, + }); + fs.rmSync(home, { recursive: true, force: true }); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + const payload = JSON.parse(result.stdout) as { + before: { attachments: null; error: string }; + firstOutcome: string; + afterFirst: { attachments: null; error: string }; + secondOutcome: string; + after: { attachments: unknown[] }; + calls: string[]; + }; + expect(payload.before.attachments).toBeNull(); + expect(payload.before.error).toContain("provider 'alpha-mcp-fake' not found"); + expect(payload.firstOutcome).toBe("detached"); + expect(payload.afterFirst.attachments).toBeNull(); + expect(payload.afterFirst.error).toContain("provider 'alpha-mcp-second' not found"); + expect(payload.secondOutcome).toBe("detached"); + expect(payload.after.attachments).toEqual([]); + expect(payload.calls).toEqual([ + "sandbox provider list alpha", + "provider get alpha-mcp-fake", + "sandbox provider detach alpha alpha-mcp-fake", + "provider get alpha-mcp-fake", + "sandbox provider list alpha", + "provider get alpha-mcp-second", + "sandbox provider detach alpha alpha-mcp-second", + "provider get alpha-mcp-second", + "sandbox provider list alpha", + ]); + }); + + it("does not treat a concurrent writer's resource-version advance as our update", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mcp-provider-update-race-")); + const script = String.raw` +process.env.HOME = ${JSON.stringify(home)}; +process.env.EXPECTED_TOKEN = "host-only-secret"; +const globalActions = require("./src/lib/actions/global.js"); +const calls = []; +let resourceVersion = 4; +globalActions.runOpenshellProviderCommand = (args) => { + calls.push(args.join(" ")); + if (args[0] === "provider" && args[1] === "get") { + return { + status: 0, + stdout: "Id: 11111111-2222-4333-8444-555555555555\nType: generic\nResource version: " + resourceVersion + "\nCredential keys: EXPECTED_TOKEN\n", + stderr: "", + }; + } + if (args[0] === "provider" && args[1] === "update") { + resourceVersion = 5; + return { + status: 9, + stdout: "", + stderr: "Aborted: provider was modified concurrently (current resource_version: 5)", + }; + } + throw new Error("unexpected call: " + args.join(" ")); +}; +const providerActions = require("./src/lib/actions/sandbox/mcp-bridge-provider.js"); +let message = ""; +try { + providerActions.upsertMcpProvider( + "alpha-mcp-fake", + [{ name: "EXPECTED_TOKEN" }], + { + allowExisting: true, + expectedProviderId: "11111111-2222-4333-8444-555555555555", + }, + ); +} catch (error) { + message = error.message; +} +process.stdout.write(JSON.stringify({ message, resourceVersion, calls })); +`; + const result = spawnSync(process.execPath, ["-e", script], { + cwd: process.cwd(), + encoding: "utf8", + env: { ...process.env, HOME: home }, + }); + fs.rmSync(home, { recursive: true, force: true }); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + const payload = JSON.parse(result.stdout) as { + message: string; + resourceVersion: number; + calls: string[]; + }; + expect(payload.resourceVersion).toBe(5); + expect(payload.message).toContain("modified concurrently"); + expect(payload.calls).toEqual([ + "provider get alpha-mcp-fake", + "provider get alpha-mcp-fake", + "provider update alpha-mcp-fake --credential EXPECTED_TOKEN", + ]); + expect(JSON.stringify(payload.calls)).not.toContain("host-only-secret"); + }); + + it("never detaches or deletes a non-matching provider in force mode", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mcp-provider-owner-")); + const script = ` +process.env.HOME = ${JSON.stringify(home)}; +const registry = require("./src/lib/state/registry.js"); +const agentDefs = require("./src/lib/agent/defs.js"); +const gatewayRuntime = require("./src/lib/gateway-runtime-action.js"); +const policies = require("./src/lib/policy/index.js"); +const processRecovery = require("./src/lib/actions/sandbox/process-recovery.js"); +const globalActions = require("./src/lib/actions/global.js"); +const calls = []; +agentDefs.loadAgent = () => { throw new Error("persisted adapter must be used"); }; +gatewayRuntime.recoverNamedGatewayRuntime = async () => ({ + recovered: true, + attempted: false, + before: { state: "healthy_named" }, + after: { state: "healthy_named" }, +}); +policies.getPresetContentGatewayState = () => "absent"; +policies.removePreset = () => true; +processRecovery.executeSandboxCommand = () => ({ status: 0, stdout: "", stderr: "" }); +processRecovery.executeSandboxExecCommand = () => ({ status: 0, stdout: "", stderr: "" }); +globalActions.runOpenshellProviderCommand = (args) => { + calls.push(args.join(" ")); + if (args[0] === "status") { + return { status: 0, stdout: "ready", stderr: "" }; + } + if (args[0] === "provider" && args[1] === "get") { + return { + status: 0, + stdout: "Id: 99999999-8888-4777-8666-555555555555\\nType: generic\\nResource version: 4\\nCredential keys: EXPECTED_TOKEN\\n", + stderr: "", + }; + } + return { status: 0, stdout: "", stderr: "" }; +}; +registry.registerSandbox({ + name: "alpha", + agent: "legacy-disabled", + mcp: { bridges: { fake: { + server: "fake", + url: "https://mcp.example.test/mcp", + env: ["EXPECTED_TOKEN"], + providerName: "alpha-mcp-fake", + providerId: "11111111-2222-4333-8444-555555555555", + policyName: "mcp-bridge-fake", + adapter: "mcporter", + addedAt: "2026-06-01T00:00:00.000Z", + } } }, +}); +const bridge = require("./src/lib/actions/sandbox/mcp-bridge.js"); +bridge.removeMcpBridge("alpha", "fake", { force: true }).then( + () => process.exit(9), + (error) => process.stdout.write(JSON.stringify({ + message: error.message, + calls, + bridgePresent: !!registry.getSandbox("alpha")?.mcp?.bridges?.fake, + })), +); +`; + const result = spawnSync(process.execPath, ["-e", script], { + cwd: process.cwd(), + encoding: "utf8", + env: { ...process.env, HOME: home }, + }); + fs.rmSync(home, { recursive: true, force: true }); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + const payload = JSON.parse(result.stdout.slice(result.stdout.indexOf("{"))) as { + message: string; + calls: string[]; + bridgePresent: boolean; + }; + expect(payload.message).toContain("registry entry was preserved"); + expect(result.stderr).toContain("Expected stable provider ID"); + expect(payload.calls.some((call) => call === "provider get alpha-mcp-fake")).toBe(true); + expect(payload.bridgePresent).toBe(true); + }); +}); diff --git a/test/mcp-restart-policy-order.test.ts b/test/mcp-restart-policy-order.test.ts new file mode 100644 index 00000000000..bc602143099 --- /dev/null +++ b/test/mcp-restart-policy-order.test.ts @@ -0,0 +1,252 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +describe("MCP restart policy ordering", () => { + it("rejects a foreign attached credential key before policy or provider mutation", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mcp-restart-order-")); + const script = String.raw` +process.env.HOME = ${JSON.stringify(home)}; +process.env.MCP_TOKEN = "host-only-secret"; +const registry = require("./src/lib/state/registry.js"); +const globalActions = require("./src/lib/actions/global.js"); +const gatewayRuntime = require("./src/lib/gateway-runtime-action.js"); +const policies = require("./src/lib/policy/index.js"); +const processRecovery = require("./src/lib/actions/sandbox/process-recovery.js"); +const generated = require("./src/lib/actions/sandbox/mcp-bridge-policy.js"); + +const providerCalls = []; +let policyApplyCalls = 0; +const entry = { + server: "example", + agent: "openclaw", + adapter: "mcporter", + url: "https://8.8.8.8/mcp", + env: ["MCP_TOKEN"], + providerName: "alpha-mcp-example", + providerId: "11111111-2222-4333-8444-555555555555", + policyName: "mcp-bridge-example", + addedAt: "2026-06-01T00:00:00.000Z", +}; + +gatewayRuntime.recoverNamedGatewayRuntime = async () => ({ + recovered: true, + attempted: false, + before: { state: "healthy_named" }, + after: { state: "healthy_named" }, +}); +globalActions.runOpenshellProviderCommand = (args) => { + if (args.join(" ") === "status --output json") { + return { status: 0, stdout: JSON.stringify({ gateway: "nemoclaw" }), stderr: "" }; + } + if (args[0] === "provider" && args[1] === "get") { + if (args[2] === "foreign-attached") { + return { + status: 0, + stdout: "Id: 99999999-8888-4777-8666-555555555555\nType: generic\nResource version: 1\nCredential keys: MCP_TOKEN\n", + stderr: "", + }; + } + return { + status: 0, + stdout: "Id: " + entry.providerId + "\nType: generic\nResource version: 1\nCredential keys: MCP_TOKEN\n", + stderr: "", + }; + } + if (args.join(" ") === "sandbox provider list alpha") { + return { + status: 0, + stdout: "NAME TYPE CREDENTIAL_KEYS CONFIG_KEYS\nforeign-attached generic 1 0\n", + stderr: "", + }; + } + if (args[0] === "provider" && (args[1] === "create" || args[1] === "update")) { + providerCalls.push(args.join(" ")); + } + return { status: 0, stdout: "", stderr: "" }; +}; +policies.getPresetContentGatewayState = () => "match"; +policies.applyPresetContent = () => { + policyApplyCalls += 1; + return true; +}; +processRecovery.executeSandboxExecCommand = () => ({ status: 0, stdout: "", stderr: "" }); +processRecovery.executeSandboxCommand = (_sandbox, command) => ({ + status: 0, + stdout: command === "command -v mcporter" ? "/usr/local/bin/mcporter\n" : "registered\n", + stderr: "", +}); + +registry.registerSandbox({ + name: "alpha", + agent: "openclaw", + gatewayName: "nemoclaw", + mcp: { bridges: { example: entry } }, +}); +registry.addCustomPolicy("alpha", { + name: entry.policyName, + content: generated.buildMcpBridgePolicyYaml(entry.server, entry.url, entry.adapter, ["8.8.8.8"]), + sourcePath: "generated:nemoclaw-mcp-bridge", +}); + +const bridge = require("./src/lib/actions/sandbox/mcp-bridge.js"); +bridge.restartMcpBridge("alpha", "example").then( + () => process.exit(9), + (error) => { + process.stdout.write(JSON.stringify({ + message: error instanceof Error ? error.message : String(error), + policyApplyCalls, + providerCalls, + })); + }, +); +`; + const result = spawnSync(process.execPath, ["-e", script], { + cwd: process.cwd(), + encoding: "utf8", + env: { ...process.env, HOME: home }, + timeout: 30_000, + }); + fs.rmSync(home, { recursive: true, force: true }); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + const payload = JSON.parse(result.stdout) as { + message: string; + policyApplyCalls: number; + providerCalls: string[]; + }; + expect(payload.message).toContain( + "Credential key 'MCP_TOKEN' is already supplied by attached provider 'foreign-attached'", + ); + expect(payload.policyApplyCalls).toBe(0); + expect(payload.providerCalls).toEqual([]); + }); + + it("compares bounded provider revision observations on the host during restart", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mcp-restart-revision-")); + const script = String.raw` +process.env.HOME = ${JSON.stringify(home)}; +process.env.MCP_TOKEN = "host-only-secret"; +const registry = require("./src/lib/state/registry.js"); +const globalActions = require("./src/lib/actions/global.js"); +const gatewayRuntime = require("./src/lib/gateway-runtime-action.js"); +const policies = require("./src/lib/policy/index.js"); +const processRecovery = require("./src/lib/actions/sandbox/process-recovery.js"); +const generated = require("./src/lib/actions/sandbox/mcp-bridge-policy.js"); + +let resourceVersion = 1; +const observations = []; +const proofScripts = []; +const providerCalls = []; +const entry = { + server: "example", + agent: "openclaw", + adapter: "mcporter", + url: "https://8.8.8.8/mcp", + env: ["MCP_TOKEN"], + providerName: "alpha-mcp-example", + providerId: "11111111-2222-4333-8444-555555555555", + policyName: "mcp-bridge-example", + addedAt: "2026-06-01T00:00:00.000Z", +}; + +gatewayRuntime.recoverNamedGatewayRuntime = async () => ({ + recovered: true, + attempted: false, + before: { state: "healthy_named" }, + after: { state: "healthy_named" }, +}); +globalActions.runOpenshellProviderCommand = (args) => { + const command = args.join(" "); + if (command === "status --output json") { + return { status: 0, stdout: JSON.stringify({ gateway: "nemoclaw" }), stderr: "" }; + } + if (args[0] === "provider" && args[1] === "get") { + return { + status: 0, + stdout: "Id: " + entry.providerId + "\nType: generic\nResource version: " + resourceVersion + "\nCredential keys: MCP_TOKEN\n", + stderr: "", + }; + } + if (args[0] === "provider" && args[1] === "update") { + providerCalls.push(command); + resourceVersion = 2; + return { status: 0, stdout: "Updated provider", stderr: "" }; + } + if (args[0] === "sandbox" && args[1] === "provider" && args[2] === "list") { + return { + status: 0, + stdout: "NAME TYPE CREDENTIAL_KEYS CONFIG_KEYS\n" + entry.providerName + " generic 1 0\n", + stderr: "", + }; + } + if (args[0] === "sandbox" && args[1] === "provider" && args[2] === "attach") { + return { status: 0, stdout: "attached", stderr: "" }; + } + return { status: 0, stdout: "", stderr: "" }; +}; +policies.getPresetContentGatewayState = () => "match"; +policies.applyPresetContent = () => true; +processRecovery.executeSandboxExecCommand = (_sandbox, command) => { + const encoded = command.match(/printf '%s' '([A-Za-z0-9+/=]+)' \| base64 -d/)?.[1] || ""; + const proof = encoded ? Buffer.from(encoded, "base64").toString("utf8") : command; + proofScripts.push(proof); + if (proof.includes("printf '%s\\n' absent")) { + const observation = "v" + resourceVersion; + observations.push(observation); + return { status: 0, stdout: observation, stderr: "" }; + } + return { status: 0, stdout: "", stderr: "" }; +}; +processRecovery.executeSandboxCommand = (_sandbox, command) => ({ + status: 0, + stdout: command === "command -v mcporter" ? "/usr/local/bin/mcporter\n" : "registered\n", + stderr: "", +}); + +registry.registerSandbox({ + name: "alpha", + agent: "openclaw", + gatewayName: "nemoclaw", + mcp: { bridges: { example: entry } }, +}); +registry.addCustomPolicy("alpha", { + name: entry.policyName, + content: generated.buildMcpBridgePolicyYaml(entry.server, entry.url, entry.adapter, ["8.8.8.8"]), + sourcePath: "generated:nemoclaw-mcp-bridge", +}); + +const bridge = require("./src/lib/actions/sandbox/mcp-bridge.js"); +bridge.restartMcpBridge("alpha", "example").then( + () => process.stdout.write(JSON.stringify({ observations, proofScripts, providerCalls })), + (error) => { console.error(error); process.exit(1); }, +); +`; + const result = spawnSync(process.execPath, ["-e", script], { + cwd: process.cwd(), + encoding: "utf8", + env: { ...process.env, HOME: home }, + timeout: 30_000, + }); + fs.rmSync(home, { recursive: true, force: true }); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + const payload = JSON.parse(result.stdout.slice(result.stdout.indexOf("{"))) as { + observations: string[]; + proofScripts: string[]; + providerCalls: string[]; + }; + expect(payload.observations).toEqual(["v1", "v2"]); + expect(payload.providerCalls).toEqual([ + "provider update alpha-mcp-example --credential MCP_TOKEN", + ]); + expect(payload.proofScripts).toHaveLength(2); + expect(payload.proofScripts.join("\n")).not.toMatch(/\/tmp|snapshot/); + }); +}); diff --git a/test/mcp-url-target.test.ts b/test/mcp-url-target.test.ts new file mode 100644 index 00000000000..3d578de6eff --- /dev/null +++ b/test/mcp-url-target.test.ts @@ -0,0 +1,37 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { isBlockedMcpUrlTargetHost } from "../src/lib/security/mcp-url-target"; + +describe("MCP URL target special-use filtering", () => { + it.each([ + "192.31.196.1", + "192.52.193.1", + "192.88.99.1", + "192.175.48.1", + "::7f00:1", + "::a00:1", + "::ffff:127.0.0.1", + "::ffff:7f00:1", + "::ffff:a00:1", + "::ffff:c0a8:101", + "2001:2::1", + "2001:20::1", + "2620:4f:8000::1", + "3fff::1", + "5f00::1", + "fec0::1", + ])("blocks non-global special-purpose address %s", (address) => { + expect(isBlockedMcpUrlTargetHost(address)).toBe(true); + }); + + it.each([ + "8.8.8.8", + "1.1.1.1", + "2606:4700:4700::1111", + ])("keeps globally routable address %s eligible", (address) => { + expect(isBlockedMcpUrlTargetHost(address)).toBe(false); + }); +}); diff --git a/test/mcporter-supply-chain.test.ts b/test/mcporter-supply-chain.test.ts new file mode 100644 index 00000000000..da7eb226673 --- /dev/null +++ b/test/mcporter-supply-chain.test.ts @@ -0,0 +1,64 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +const repoRoot = path.join(import.meta.dirname, ".."); +const runtimeDirectory = path.join(repoRoot, "agents", "openclaw", "mcporter-runtime"); +const dockerfiles = ["Dockerfile.base", "Dockerfile"].map((name) => ({ + name, + contents: fs.readFileSync(path.join(repoRoot, name), "utf8"), +})); +const expectedVersion = "0.7.3"; +const expectedIntegrity = + "sha512-egoPVYqTnWb3NjRIxo+xc8OrAI0dlPrJm9pAiZx0pImuNIV5rKhGtTnIfH/Y1ldGPVu74ibj3KR5c9U/QSdQFA=="; +const runtimePrefix = "npm --prefix /usr/local/lib/nemoclaw/mcporter-runtime"; + +describe("mcporter image supply-chain controls", () => { + it("resolves the committed production graph through npm's lockfile boundary", () => { + const result = spawnSync( + "npm", + ["ls", "--package-lock-only", "--omit=dev", "--all", "--json"], + { cwd: runtimeDirectory, encoding: "utf8" }, + ); + expect(result.status, result.stderr).toBe(0); + const graph = JSON.parse(result.stdout) as { + dependencies?: Record; + problems?: string[]; + }; + expect(graph.problems).toBeUndefined(); + expect(graph.dependencies?.mcporter?.version).toBe(expectedVersion); + }); + + it.each(dockerfiles)("pins and verifies the package in $name", ({ contents }) => { + const flattenedContents = contents.replace(/\\\s*\n/g, " ").replace(/\s+/g, " "); + + expect(contents).toContain(`ARG MCPORTER_VERSION=${expectedVersion}`); + expect(contents).toContain(`ARG MCPORTER_0_7_3_INTEGRITY=${expectedIntegrity}`); + expect(contents).toContain('npm view "mcporter@${MCPORTER_VERSION}" dist.integrity'); + expect(contents).toContain( + "COPY agents/openclaw/mcporter-runtime/package.json /usr/local/lib/nemoclaw/mcporter-runtime/package.json", + ); + expect(contents).toContain( + "COPY agents/openclaw/mcporter-runtime/package-lock.json /usr/local/lib/nemoclaw/mcporter-runtime/package-lock.json", + ); + expect(flattenedContents).toContain( + `${runtimePrefix} ci --ignore-scripts --omit=dev --no-audit --no-fund --no-progress`, + ); + expect(contents).toContain( + "ln -s /usr/local/lib/nemoclaw/mcporter-runtime/node_modules/.bin/mcporter /usr/local/bin/mcporter", + ); + expect(contents).toContain('test "$(mcporter --version)" = "$MCPORTER_VERSION"'); + expect(contents).not.toMatch(/npm install -g[^\n]*mcporter/); + expect(contents).not.toContain("mcporter shrinkwrap"); + }); + + it.each(dockerfiles)("audits the committed dependency graph in $name", ({ contents }) => { + expect(contents).toContain(`${runtimePrefix} audit --omit=dev --audit-level=low`); + expect(contents).toContain(`${runtimePrefix} audit signatures`); + }); +}); diff --git a/test/onboard-openshell-install-stream.test.ts b/test/onboard-openshell-install-stream.test.ts index ee43e5c8291..b74e9c2380e 100644 --- a/test/onboard-openshell-install-stream.test.ts +++ b/test/onboard-openshell-install-stream.test.ts @@ -1,15 +1,16 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { beforeEach, describe, expect, it, vi } from "vitest"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; const { spawnSyncMock } = vi.hoisted(() => ({ spawnSyncMock: vi.fn() })); vi.mock("node:child_process", () => ({ spawnSync: spawnSyncMock })); import { - runOpenshellInstall, type RunOpenshellInstallDeps, + runOpenshellInstall, } from "../src/lib/onboard/openshell-pin"; function makeDeps(overrides: Partial = {}): RunOpenshellInstallDeps { @@ -30,6 +31,10 @@ describe("runOpenshellInstall progress streaming (#4431)", () => { spawnSyncMock.mockReset(); }); + afterEach(() => { + vi.unstubAllEnvs(); + }); + it("inherits stdio so install-openshell.sh output streams live", () => { spawnSyncMock.mockReturnValue({ status: 0 }); runOpenshellInstall(makeDeps()); @@ -45,6 +50,34 @@ describe("runOpenshellInstall progress streaming (#4431)", () => { expect(options.stdio).not.toContain("pipe"); }); + it("normalizes relative component overrides before changing the installer cwd", () => { + vi.stubEnv("NEMOCLAW_OPENSHELL_GATEWAY_BIN", "components/openshell-gateway"); + vi.stubEnv("NEMOCLAW_OPENSHELL_SANDBOX_BIN", "components/openshell-sandbox"); + spawnSyncMock.mockReturnValue({ status: 0 }); + + runOpenshellInstall(makeDeps()); + + const options = spawnSyncMock.mock.calls[0][2]; + expect(options.env.NEMOCLAW_OPENSHELL_GATEWAY_BIN).toBe( + path.resolve("components/openshell-gateway"), + ); + expect(options.env.NEMOCLAW_OPENSHELL_SANDBOX_BIN).toBe( + path.resolve("components/openshell-sandbox"), + ); + }); + + it("removes whitespace-only component overrides before invoking the installer", () => { + vi.stubEnv("NEMOCLAW_OPENSHELL_GATEWAY_BIN", " "); + vi.stubEnv("NEMOCLAW_OPENSHELL_SANDBOX_BIN", "\t"); + spawnSyncMock.mockReturnValue({ status: 0 }); + + runOpenshellInstall(makeDeps()); + + const options = spawnSyncMock.mock.calls[0][2]; + expect(options.env.NEMOCLAW_OPENSHELL_GATEWAY_BIN).toBeUndefined(); + expect(options.env.NEMOCLAW_OPENSHELL_SANDBOX_BIN).toBeUndefined(); + }); + it("returns a not-installed result without throwing on non-zero exit", () => { spawnSyncMock.mockReturnValue({ status: 1 }); const result = runOpenshellInstall(makeDeps()); diff --git a/test/onboard-openshell-version.test.ts b/test/onboard-openshell-version.test.ts index 8d14d8bb060..1d047e0b028 100644 --- a/test/onboard-openshell-version.test.ts +++ b/test/onboard-openshell-version.test.ts @@ -26,12 +26,13 @@ const installModule = require("../src/lib/onboard/openshell-install") as { parseOpenshellReleaseTag: (tag: unknown) => string | null; resolveOpenshellInstallVersion: ( available: readonly string[], - options: { max: string | null }, + options: { min?: string | null; max: string | null }, helpers: { versionGte: (a: string, b: string) => boolean }, ) => { kind: "pin" | "no-max" | "incompatible"; version?: string; latest?: string | null; + min?: string | null; max?: string; message?: string; reason?: "latest" | "max-cap"; @@ -40,6 +41,7 @@ const installModule = require("../src/lib/onboard/openshell-install") as { const pinModule = require("../src/lib/onboard/openshell-pin") as { resolveOpenshellInstallPin: (deps: { + getBlueprintMinOpenshellVersion?: () => string | null; getBlueprintMaxOpenshellVersion: () => string | null; versionGte: (a: string, b: string) => boolean; listReleases?: () => string[] | null; @@ -253,6 +255,29 @@ describe("resolveOpenshellInstallVersion", () => { expect(result.latest).toBe("0.0.36"); }); + it("rejects published releases below the supported minimum", () => { + const result = installModule.resolveOpenshellInstallVersion( + ["v0.0.71"], + { min: "0.0.72", max: "0.0.72" }, + helpers, + ); + expect(result.kind).toBe("incompatible"); + expect(result.min).toBe("0.0.72"); + expect(result.max).toBe("0.0.72"); + expect(result.message).toContain("0.0.72 through 0.0.72"); + }); + + it("selects the release that satisfies both minimum and maximum", () => { + const result = installModule.resolveOpenshellInstallVersion( + ["v0.0.71", "v0.0.72"], + { min: "0.0.72", max: "0.0.72" }, + helpers, + ); + expect(result.kind).toBe("pin"); + expect(result.version).toBe("0.0.72"); + expect(result.reason).toBe("latest"); + }); + it("returns incompatible when no release ≤ max exists", () => { const result = installModule.resolveOpenshellInstallVersion( ["v0.0.38", "0.0.39"], @@ -350,6 +375,17 @@ describe("resolveOpenshellInstallPin", () => { expect(logged.join("\n")).toContain("0.0.38"); }); + it("surfaces incompatible before download when all releases are below min", () => { + const result = pinModule.resolveOpenshellInstallPin({ + getBlueprintMinOpenshellVersion: () => "0.0.72", + getBlueprintMaxOpenshellVersion: () => "0.0.72", + versionGte, + listReleases: () => ["v0.0.71"], + }); + expect(result.kind).toBe("incompatible"); + expect(result.message ?? "").toContain("0.0.72 through 0.0.72"); + }); + it("surfaces incompatible when no published release ≤ max exists", () => { const result = pinModule.resolveOpenshellInstallPin({ getBlueprintMaxOpenshellVersion: () => "0.0.36", @@ -363,6 +399,27 @@ describe("resolveOpenshellInstallPin", () => { }); describe("computeOpenshellInstallEnv", () => { + it("does not apply stable release discovery to the dev channel", () => { + const channel = "dev"; + const result = pinModule.computeOpenshellInstallEnv( + { + NEMOCLAW_OPENSHELL_CHANNEL: channel, + NEMOCLAW_OPENSHELL_PIN_VERSION: "0.0.71", + }, + { + getBlueprintMinOpenshellVersion: () => "0.0.72", + getBlueprintMaxOpenshellVersion: () => "0.0.72", + versionGte, + listReleases: () => ["v0.0.71"], + }, + ); + expect(result.env).not.toBe(null); + expect(result.env?.NEMOCLAW_OPENSHELL_CHANNEL).toBe(channel); + expect(result.env?.NEMOCLAW_OPENSHELL_PIN_VERSION).toBeUndefined(); + expect(result.env?.NEMOCLAW_OPENSHELL_MIN_VERSION).toBe("0.0.72"); + expect(result.env?.NEMOCLAW_OPENSHELL_MAX_VERSION).toBe("0.0.72"); + }); + it("overlays MIN/MAX/PIN env vars from blueprint when latest exceeds max", () => { const result = pinModule.computeOpenshellInstallEnv( { EXISTING: "preserved" }, diff --git a/test/onboard-prepared-build-context.test.ts b/test/onboard-prepared-build-context.test.ts index fbe69a84c4f..a810272d954 100644 --- a/test/onboard-prepared-build-context.test.ts +++ b/test/onboard-prepared-build-context.test.ts @@ -186,6 +186,7 @@ const { createSandbox } = require(${onboardPath}); null, null, [], + null, preparedBuildContext, ); } catch (error) { diff --git a/test/onboard-prompt-default-case.test.ts b/test/onboard-prompt-default-case.test.ts index 83ec399c7f2..1287b2394b0 100644 --- a/test/onboard-prompt-default-case.test.ts +++ b/test/onboard-prompt-default-case.test.ts @@ -12,7 +12,6 @@ const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts const credentialsPath = JSON.stringify( path.join(repoRoot, "src", "lib", "credentials", "store.ts"), ); -const onboardSourcePath = path.join(repoRoot, "src", "lib", "onboard.ts"); type RunResult = { result: boolean; @@ -241,22 +240,3 @@ describe("promptYesNoOrDefault (interactive)", () => { expect(out.promptCalls).toEqual([" Apply this configuration? [Y/n]: "]); }); }); - -describe("under-provisioned runtime prompt defaults (#4236)", () => { - it("defaults the preflight warning prompt to abort for interactive runs", () => { - const source = fs.readFileSync(onboardSourcePath, "utf-8"); - expect(source).toMatch( - /promptYesNoOrDefault\(\s*" Continue with onboarding\?",\s*null,\s*false\s*\)/, - ); - expect(source).not.toMatch( - /promptYesNoOrDefault\(\s*" Continue with onboarding\?",\s*null,\s*true\s*\)/, - ); - }); - - it("keeps non-interactive runs warning-only so automation can continue", () => { - const source = fs.readFileSync(onboardSourcePath, "utf-8"); - expect(source).toContain( - "WARNING: Non-interactive mode is continuing despite under-provisioned runtime.", - ); - }); -}); diff --git a/test/onboard-sandbox-create-failure.test.ts b/test/onboard-sandbox-create-failure.test.ts index 75bbefb8bf5..364d978d606 100644 --- a/test/onboard-sandbox-create-failure.test.ts +++ b/test/onboard-sandbox-create-failure.test.ts @@ -7,7 +7,10 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; -import { collectSandboxCreateFailureDiagnostics } from "../src/lib/onboard/sandbox-create-failure.js"; +import { + collectSandboxCreateFailureDiagnostics, + printSandboxCreateFailureDiagnostics, +} from "../src/lib/onboard/sandbox-create-failure.js"; describe("sandbox create failure diagnostics", () => { it("preserves gateway failure lines and VM console output before cleanup", () => { @@ -56,4 +59,61 @@ describe("sandbox create failure diagnostics", () => { "backup_path=/tmp/pre-upgrade-backup", ); }); + + it("prints saved diagnostics and retained backup details", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-create-failure-print-")); + const homeDir = path.join(tmp, "home"); + const messages: string[] = []; + const originalError = console.error; + console.error = (message?: unknown) => { + messages.push(String(message ?? "")); + }; + + try { + const diagnostics = printSandboxCreateFailureDiagnostics("my-assistant", { + homeDir, + backupPath: "/tmp/pre-upgrade-backup", + now: new Date("2026-05-12T20:35:00.000Z"), + }); + + expect(diagnostics?.dir).toContain(path.join(homeDir, ".nemoclaw", "onboard-failures")); + expect(messages).toContain(` Diagnostics saved: ${diagnostics!.dir}`); + expect(messages).toContain(" State backup retained: /tmp/pre-upgrade-backup"); + } finally { + console.error = originalError; + } + }); + + it("preserves a bounded gateway tail when sandbox-specific lines are absent", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-create-failure-tail-")); + const homeDir = path.join(tmp, "home"); + const logDir = path.join(homeDir, ".local", "state", "nemoclaw", "openshell-docker-gateway"); + const gatewayLogPath = path.join(logDir, "openshell-gateway.log"); + fs.mkdirSync(logDir, { recursive: true }); + fs.writeFileSync( + gatewayLogPath, + [ + "2026-05-12T20:30:00Z INFO gateway starting", + "2026-05-12T20:30:01Z WARN gateway exited before request dispatch", + ].join("\n"), + ); + + const diagnostics = collectSandboxCreateFailureDiagnostics("my-assistant", { + homeDir, + now: new Date("2026-05-12T20:35:00.000Z"), + }); + + expect(diagnostics?.gatewayTailPath).toBe( + path.join(diagnostics!.dir, "openshell-gateway-tail.log"), + ); + expect(fs.readFileSync(diagnostics!.gatewayTailPath!, "utf-8")).toContain( + "gateway exited before request dispatch", + ); + expect(diagnostics?.summaryLines).toContain( + "2026-05-12T20:30:01Z WARN gateway exited before request dispatch", + ); + expect(fs.readFileSync(path.join(diagnostics!.dir, "summary.txt"), "utf-8")).toContain( + "gateway_tail=", + ); + }); }); diff --git a/test/openshell-channel-workflow.test.ts b/test/openshell-channel-workflow.test.ts new file mode 100644 index 00000000000..8b4b6a7e0f4 --- /dev/null +++ b/test/openshell-channel-workflow.test.ts @@ -0,0 +1,95 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +const REPO_ROOT = path.resolve(import.meta.dirname, ".."); +const LAUNCHABLE = path.join(REPO_ROOT, "scripts", "brev-launchable-ci-cpu.sh"); + +function resolveLaunchableVersion(options: { channel: string; explicit?: string }) { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-launchable-channel-")); + const fakeBin = path.join(tempDir, "bin"); + fs.mkdirSync(fakeBin); + const getent = path.join(fakeBin, "getent"); + fs.writeFileSync( + getent, + "#!/usr/bin/env bash\nprintf 'tester:x:501:20:tester:%s:/bin/bash\\n' \"$HOME\"\n", + { encoding: "utf8", mode: 0o755 }, + ); + try { + return spawnSync("bash", [LAUNCHABLE, "--print-openshell-version"], { + encoding: "utf8", + env: { + HOME: tempDir, + LAUNCH_LOG: path.join(tempDir, "launch.log"), + LOGNAME: "tester", + NEMOCLAW_OPENSHELL_CHANNEL: options.channel, + PATH: `${fakeBin}:/usr/bin:/bin`, + SUDO_USER: "tester", + USER: "tester", + ...(options.explicit === undefined ? {} : { OPENSHELL_VERSION: options.explicit }), + }, + }); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } +} + +function runLaunchableDevGate() { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-launchable-dev-gate-")); + try { + return spawnSync("bash", [LAUNCHABLE], { + encoding: "utf8", + env: { + HOME: tempDir, + LAUNCH_LOG: path.join(tempDir, "launch.log"), + LOGNAME: "tester", + NEMOCLAW_OPENSHELL_CHANNEL: "dev", + PATH: "/usr/bin:/bin", + SUDO_USER: "tester", + USER: "tester", + }, + }); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } +} + +describe("OpenShell channel workflow boundary", () => { + it.each([ + { channel: "dev", expected: "dev" }, + { channel: "stable", expected: "v0.0.72" }, + { channel: "auto", expected: "v0.0.72" }, + { channel: "dev", explicit: "v9.9.9", expected: "v9.9.9" }, + ])("resolves launchable channel $channel to $expected", ({ channel, explicit, expected }) => { + const result = resolveLaunchableVersion({ channel, explicit }); + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + expect(result.stdout.trim()).toBe(expected); + }); + + it("rejects an invalid launchable channel", () => { + const result = resolveLaunchableVersion({ channel: "artifact" }); + expect(result.status).not.toBe(0); + expect(`${result.stdout}${result.stderr}`).toContain( + "NEMOCLAW_OPENSHELL_CHANNEL must be one of: stable, dev, auto", + ); + }); + + it("requires explicit opt-in before a launchable consumes unverified dev artifacts", () => { + const result = runLaunchableDevGate(); + expect(result.status).not.toBe(0); + expect(`${result.stdout}${result.stderr}`).toContain( + "NEMOCLAW_ACCEPT_DEV_UNVERIFIED_INSTALL=1", + ); + + const source = fs.readFileSync(LAUNCHABLE, "utf8"); + expect(source).toContain( + 'if [[ "$OPENSHELL_VERSION" != "dev" ]]; then\n verify_openshell_cli_asset', + ); + }); +}); diff --git a/test/package-contract/cli/command-registry.test.ts b/test/package-contract/cli/command-registry.test.ts index 4a41297396a..e1f52031328 100644 --- a/test/package-contract/cli/command-registry.test.ts +++ b/test/package-contract/cli/command-registry.test.ts @@ -56,14 +56,16 @@ describe("command-registry", () => { }); describe("sandboxCommands()", () => { - it("should return exactly 52 entries", () => { - // 44 visible + 8 hidden (shields×3 + config get/set/rotate-token + - // inference get/set). 44 visible includes the sessions group (root + - // list + reset + delete + export), the agents quartet (add + apply + - // delete + list), the singular `agent` passthrough that forwards to - // `openclaw agent`, and the download + upload host-side openshell - // wrappers. - expect(sandboxCommands()).toHaveLength(52); + it("should return exactly 57 entries", () => { + // 49 visible + 8 hidden (shields×3 + config get/set/rotate-token + + // inference get/set). + // 49 visible includes the sessions group (root + list + reset + delete + + // export), the agents quartet (add + apply + delete + list), the + // singular `agent` passthrough that forwards to `openclaw agent`, and + // the download + upload host-side openshell wrappers, plus five MCP + // bridge display entries under the `mcp` parent and the gateway restart + // command under the `gateway` parent. + expect(sandboxCommands()).toHaveLength(57); }); it("every entry has scope sandbox", () => { @@ -221,9 +223,9 @@ describe("command-registry", () => { }); describe("sandboxActionTokens()", () => { - it("returns exactly 31 unique action tokens including empty string", () => { + it("returns exactly 32 unique action tokens including empty string", () => { const tokens = sandboxActionTokens(); - expect(tokens).toHaveLength(31); + expect(tokens).toHaveLength(32); // Must contain every first-level sandbox action plus the empty default action. const expected = new Set([ "agent", @@ -253,6 +255,7 @@ describe("command-registry", () => { "shields", "config", "channels", + "mcp", "gateway", "gateway-token", "upload", @@ -311,6 +314,7 @@ describe("command-registry", () => { "Skills", "Policy Presets", "Messaging Channels", + "MCP Servers", "Compatibility Commands", "Services", "Troubleshooting", diff --git a/test/package-contract/cli/credentials-cli-command.test.ts b/test/package-contract/cli/credentials-cli-command.test.ts index 48bc2a43530..69bc0d109d5 100644 --- a/test/package-contract/cli/credentials-cli-command.test.ts +++ b/test/package-contract/cli/credentials-cli-command.test.ts @@ -38,6 +38,7 @@ type RuntimeRecovery = { }; type RuntimeBridgeRunOptions = { env?: Record; + replaceEnv?: boolean; stdio?: unknown; ignoreError?: boolean; timeout?: number; @@ -177,7 +178,13 @@ describe("credentials oclif commands", () => { expect(calls).toEqual([ { args: ["provider", "list", "--names"], - opts: { ignoreError: true, stdio: ["ignore", "pipe", "pipe"], timeout: 30_000 }, + opts: { + env: expect.any(Object), + ignoreError: true, + replaceEnv: true, + stdio: ["ignore", "pipe", "pipe"], + timeout: 30_000, + }, }, ]); expect(output.stdout).toContain("openai-prod"); @@ -243,7 +250,13 @@ describe("credentials oclif commands", () => { expect(calls).toEqual([ { args: ["provider", "delete", "nvidia-prod"], - opts: { ignoreError: true, stdio: ["ignore", "pipe", "pipe"], timeout: 30_000 }, + opts: { + env: expect.any(Object), + ignoreError: true, + replaceEnv: true, + stdio: ["ignore", "pipe", "pipe"], + timeout: 30_000, + }, }, ]); expect(output.stdout).toContain("Removed provider 'nvidia-prod'"); @@ -304,6 +317,7 @@ describe("credentials oclif commands", () => { it("credentials add forwards env-key-only --credential to OpenShell provider create", async () => { process.env.TAVILY_API_KEY = "tvly-test-12345"; + process.env.UNRELATED_API_KEY = "unrelated-secret-67890"; const extraProviderCalls: string[] = []; const calls = installRuntimeBridge({ runOpenshell: (args, opts) => { @@ -331,7 +345,13 @@ describe("credentials oclif commands", () => { expect(calls).toEqual([ { args: ["provider", "profile", "import", "--file", TAVILY_PROFILE_PATH], - opts: { ignoreError: true, stdio: ["ignore", "pipe", "pipe"], timeout: 30_000 }, + opts: { + env: expect.any(Object), + ignoreError: true, + replaceEnv: true, + stdio: ["ignore", "pipe", "pipe"], + timeout: 30_000, + }, }, { args: [ @@ -344,14 +364,25 @@ describe("credentials oclif commands", () => { "--credential", "TAVILY_API_KEY", ], - opts: { ignoreError: true, stdio: ["ignore", "pipe", "pipe"], timeout: 30_000 }, + opts: { + env: expect.any(Object), + ignoreError: true, + replaceEnv: true, + stdio: ["ignore", "pipe", "pipe"], + timeout: 30_000, + }, }, ]); + expect(calls[0]?.opts?.env?.TAVILY_API_KEY).toBeUndefined(); + expect(calls[1]?.opts?.env?.UNRELATED_API_KEY).toBeUndefined(); + expect(calls[1]?.opts?.env?.TAVILY_API_KEY).toBe("tvly-test-12345"); + expect(calls[1]?.args).not.toContain("tvly-test-12345"); expect(extraProviderCalls).toEqual(["tavily-search"]); expect(output.stdout).toContain("Registered provider 'tavily-search'"); expect(output.stdout).toContain("rebuild"); } finally { delete process.env.TAVILY_API_KEY; + delete process.env.UNRELATED_API_KEY; } }); diff --git a/test/package-contract/cli/public-argv-translation.test.ts b/test/package-contract/cli/public-argv-translation.test.ts index f7bbd4336a6..6e40055abfe 100644 --- a/test/package-contract/cli/public-argv-translation.test.ts +++ b/test/package-contract/cli/public-argv-translation.test.ts @@ -268,6 +268,26 @@ describe("translatePublicSandboxArgv", () => { "sandbox:channels:add", ["alpha", "slack"], ); + expectNative( + translatePublicSandboxArgv("alpha", "mcp", [ + "add", + "github", + "--url", + "https://api.githubcopilot.com/mcp/", + "--env", + "GITHUB_TOKEN", + ]), + "sandbox:mcp", + [ + "alpha", + "add", + "github", + "--url", + "https://api.githubcopilot.com/mcp/", + "--env", + "GITHUB_TOKEN", + ], + ); expectNative( translatePublicSandboxArgv("alpha", "snapshot", ["restore", "latest"]), "sandbox:snapshot:restore", diff --git a/test/pr-workflow-contract.test.ts b/test/pr-workflow-contract.test.ts index d6d5f3e50e6..260d41c1acf 100644 --- a/test/pr-workflow-contract.test.ts +++ b/test/pr-workflow-contract.test.ts @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { spawnSync } from "node:child_process"; -import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; @@ -1057,11 +1057,100 @@ describe("pull request and main workflow contracts", () => { expect(runs).toContain("docker image inspect"); expect(runs).toContain("${image}@sha256:"); + expect(runs).toContain("mcp_client_imports_ok"); + expect(runs).toContain("Build-time package/import guard only"); + expect(runs).toContain("_MCP_HTTP_AVAILABLE"); expect(runs).toContain("layout_ok"); expect(runs).toContain("HERMES_BASE_IMAGE=${digest_ref}"); expect(runs).toContain("HERMES_BASE_IMAGE=nemoclaw-hermes-base-local"); }); + it("rejects a pulled Hermes base without MCP HTTP imports and falls back locally", () => { + const temp = mkdtempSync(join(tmpdir(), "nemoclaw-hermes-base-resolver-")); + const fakeBin = join(temp, "bin"); + const dockerLog = join(temp, "docker.log"); + const githubEnv = join(temp, "github.env"); + const remoteDigest = `ghcr.io/nvidia/nemoclaw/hermes-sandbox-base@sha256:${"a".repeat(64)}`; + const resolver = requiredStep(resolveHermesBaseAction, "Resolve Hermes sandbox base image").run; + + try { + mkdirSync(fakeBin); + writeFileSync(githubEnv, ""); + writeFileSync( + join(fakeBin, "docker"), + [ + "#!/usr/bin/env node", + 'const fs = require("node:fs");', + "const args = process.argv.slice(2);", + 'fs.appendFileSync(process.env.DOCKER_LOG, JSON.stringify(args) + "\\n");', + 'if (args[0] === "pull" || args[0] === "build") process.exit(0);', + 'if (args[0] === "image" && args[1] === "inspect") {', + ' process.stdout.write(process.env.REMOTE_DIGEST + "\\n");', + " process.exit(0);", + "}", + 'if (args[0] === "run") {', + ' const entrypointIndex = args.indexOf("--entrypoint");', + " const entrypoint = args[entrypointIndex + 1];", + " const image = args[entrypointIndex + 2];", + ' if (entrypoint === "/usr/bin/ldd") {', + ' process.stdout.write("ldd (Ubuntu GLIBC 2.39) 2.39\\n");', + " process.exit(0);", + " }", + ' if (entrypoint === "sh") process.exit(0);', + ' if (entrypoint === "/opt/hermes/.venv/bin/python") {', + " process.exit(image === process.env.REMOTE_DIGEST ? 42 : 0);", + " }", + "}", + "console.error(`unexpected docker invocation: ${JSON.stringify(args)}`);", + "process.exit(2);", + "", + ].join("\n"), + { mode: 0o755 }, + ); + // Keep the fake executable in a dedicated PATH directory so every other + // command in the composite action remains the real host utility. + const result = spawnSync("bash", ["-c", resolver ?? ""], { + cwd: process.cwd(), + encoding: "utf8", + timeout: 10_000, + env: { + ...process.env, + DOCKER_LOG: dockerLog, + GITHUB_ENV: githubEnv, + GITHUB_SHA: "", + PATH: `${fakeBin}:${process.env.PATH ?? ""}`, + REMOTE_DIGEST: remoteDigest, + }, + }); + + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toContain("lacks the packaged MCP Streamable HTTP client imports"); + expect(result.stdout).toContain("building locally"); + expect(readFileSync(githubEnv, "utf8").trim()).toBe( + "HERMES_BASE_IMAGE=nemoclaw-hermes-base-local", + ); + + const calls = readFileSync(dockerLog, "utf8") + .trim() + .split("\n") + .map((line) => JSON.parse(line) as string[]); + const remoteProbe = calls.findIndex( + (args) => args.includes("/opt/hermes/.venv/bin/python") && args.includes(remoteDigest), + ); + const localBuild = calls.findIndex((args) => args[0] === "build"); + const localProbe = calls.findIndex( + (args) => + args.includes("/opt/hermes/.venv/bin/python") && + args.includes("nemoclaw-hermes-base-local"), + ); + expect(remoteProbe).toBeGreaterThanOrEqual(0); + expect(localBuild).toBeGreaterThan(remoteProbe); + expect(localProbe).toBeGreaterThan(localBuild); + } finally { + rmSync(temp, { force: true, recursive: true }); + } + }); + it("does not run npm lifecycle scripts during CI dependency installs", () => { for (const [actionName, action] of Object.entries(sharedActions)) { const installRuns = stepRuns(action).filter((run) => run.includes("npm install")); diff --git a/test/process-recovery-primitives.test.ts b/test/process-recovery-primitives.test.ts index f0fbad62100..d79547ded7f 100644 --- a/test/process-recovery-primitives.test.ts +++ b/test/process-recovery-primitives.test.ts @@ -13,6 +13,7 @@ const requireSource = createRequire(import.meta.url); const { classifyForwardHealthWithReachability, classifySandboxForwardHealth, + executeSandboxCommand, executeSandboxExecCommand, resolveSandboxDashboardPort, } = requireSource( @@ -256,6 +257,38 @@ describe("classifyForwardHealthWithReachability", () => { }); describe("executeSandboxExecCommand", () => { + it("does not forward an MCP credential to the OpenShell child process", () => { + const childProcess = requireSource("node:child_process"); + const spawn = vi.spyOn(childProcess, "spawnSync").mockReturnValue({ + status: 0, + stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\nREADY\n", + stderr: "", + } as never); + const priorSecret = process.env.TEST_MCP_RAW_TOKEN; + const priorGateway = process.env.OPENSHELL_GATEWAY; + process.env.TEST_MCP_RAW_TOKEN = "must-reach-only-provider-mutation"; + process.env.OPENSHELL_GATEWAY = "nemoclaw-19080"; + + try { + const result = withFakeOpenshellBinary(() => + executeSandboxExecCommand("hermes-box", "printf READY"), + ); + const options = spawn.mock.calls[0]?.[2] as { env?: NodeJS.ProcessEnv }; + + expect(result).toEqual({ status: 0, stdout: "READY", stderr: "" }); + expect(options.env?.TEST_MCP_RAW_TOKEN).toBeUndefined(); + expect(options.env?.OPENSHELL_GATEWAY).toBe("nemoclaw-19080"); + expect(options.env?.PATH).toBe(process.env.PATH); + } finally { + priorSecret === undefined + ? delete process.env.TEST_MCP_RAW_TOKEN + : (process.env.TEST_MCP_RAW_TOKEN = priorSecret); + priorGateway === undefined + ? delete process.env.OPENSHELL_GATEWAY + : (process.env.OPENSHELL_GATEWAY = priorGateway); + } + }); + it("parses stdout-framed root exec output after the startup marker", () => { const childProcess = requireSource("node:child_process"); vi.spyOn(childProcess, "spawnSync").mockReturnValue({ @@ -347,9 +380,19 @@ describe("executeSandboxExecCommand", () => { stderr: "", } as never); + const priorSecret = process.env.TEST_MCP_RAW_TOKEN; + const priorGateway = process.env.OPENSHELL_GATEWAY; + process.env.TEST_MCP_RAW_TOKEN = "must-reach-only-provider-mutation"; + process.env.OPENSHELL_GATEWAY = "nemoclaw-19080"; const result = withFakeOpenshellBinary(() => executeSandboxExecCommand("hermes-box", "echo SECRET_BOUNDARY_OK"), ); + priorSecret === undefined + ? delete process.env.TEST_MCP_RAW_TOKEN + : (process.env.TEST_MCP_RAW_TOKEN = priorSecret); + priorGateway === undefined + ? delete process.env.OPENSHELL_GATEWAY + : (process.env.OPENSHELL_GATEWAY = priorGateway); expect(result).toEqual({ status: 0, stdout: "SECRET_BOUNDARY_OK", stderr: "" }); expect(privilegedArgv).toHaveBeenCalledWith("hermes-box", [ @@ -366,5 +409,75 @@ describe("executeSandboxExecCommand", () => { "-c", "marked-command", ]); + const dockerOptions = dockerSpawnSync.mock.calls[0]?.[1] as { env?: NodeJS.ProcessEnv }; + expect(dockerOptions.env?.TEST_MCP_RAW_TOKEN).toBeUndefined(); + expect(dockerOptions.env?.OPENSHELL_GATEWAY).toBe("nemoclaw-19080"); + expect(dockerOptions.env?.PATH).toBe(process.env.PATH); + }); + + it("does not let Docker fallback satisfy a strict provider credential proof", () => { + const childProcess = requireSource("node:child_process"); + const dockerExec = requireSource("../src/lib/adapters/docker/exec.ts"); + const privilegedExec = requireSource("../src/lib/sandbox/privileged-exec.ts"); + const spawn = vi.spyOn(childProcess, "spawnSync").mockReturnValue({ + status: 1, + stdout: "OpenShell transport failed before the child marker\n", + stderr: "gateway unavailable\n", + } as never); + const privilegedArgv = vi.spyOn(privilegedExec, "privilegedSandboxExecArgv"); + const dockerSpawnSync = vi.spyOn(dockerExec, "dockerSpawnSync"); + + const result = withFakeOpenshellBinary(() => + executeSandboxExecCommand("hermes-box", '[ -z "${FAKE_MCP_SECRET+x}" ]', undefined, { + allowLocalDockerFallback: false, + }), + ); + + expect(result).toBeNull(); + expect(privilegedArgv).not.toHaveBeenCalled(); + expect(dockerSpawnSync).not.toHaveBeenCalled(); + const args = spawn.mock.calls[0]?.[1] as string[]; + const shellPayload = args.at(-1) ?? ""; + expect(shellPayload).not.toMatch(/[\r\n]/); + expect(shellPayload).toContain("printf '%s\\n' '__NEMOCLAW_SANDBOX_EXEC_STARTED__'"); + }); +}); + +describe("executeSandboxCommand", () => { + it("does not forward an MCP credential to the SSH child process", () => { + const openshellRuntime = requireSource("../src/lib/adapters/openshell/runtime.ts"); + const childProcess = requireSource("node:child_process"); + vi.spyOn(openshellRuntime, "captureSandboxSshConfig").mockReturnValue({ + status: 0, + output: "Host openshell-alpha\n HostName 127.0.0.1\n", + } as never); + const spawn = vi.spyOn(childProcess, "spawnSync").mockReturnValue({ + status: 0, + stdout: "registered\n", + stderr: "", + } as never); + const priorSecret = process.env.TEST_MCP_RAW_TOKEN; + const priorGateway = process.env.OPENSHELL_GATEWAY; + process.env.TEST_MCP_RAW_TOKEN = "must-reach-only-provider-mutation"; + process.env.OPENSHELL_GATEWAY = "nemoclaw-19080"; + + try { + expect(executeSandboxCommand("alpha", "mcporter config get fake --json")).toEqual({ + status: 0, + stdout: "registered", + stderr: "", + }); + const options = spawn.mock.calls[0]?.[2] as { env?: NodeJS.ProcessEnv }; + expect(options.env?.TEST_MCP_RAW_TOKEN).toBeUndefined(); + expect(options.env?.OPENSHELL_GATEWAY).toBe("nemoclaw-19080"); + expect(options.env?.PATH).toBe(process.env.PATH); + } finally { + priorSecret === undefined + ? delete process.env.TEST_MCP_RAW_TOKEN + : (process.env.TEST_MCP_RAW_TOKEN = priorSecret); + priorGateway === undefined + ? delete process.env.OPENSHELL_GATEWAY + : (process.env.OPENSHELL_GATEWAY = priorGateway); + } }); }); diff --git a/test/rebuild-credential-preflight.test.ts b/test/rebuild-credential-preflight.test.ts index 0bd115efdab..a331a781a4c 100644 --- a/test/rebuild-credential-preflight.test.ts +++ b/test/rebuild-credential-preflight.test.ts @@ -18,6 +18,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; +import { execTimeout, testTimeout } from "./helpers/timeouts"; const REPO_ROOT = path.join(import.meta.dirname, ".."); const NODE_BIN = path.dirname(process.execPath); @@ -121,6 +122,11 @@ function createFixture(opts: { model: "meta/llama-3.3-70b-instruct", provider, gpuEnabled: false, + sandboxGpuMode: "0", + gatewayName: "nemoclaw", + gatewayPort: 8080, + dashboardPort: 18789, + fromDockerfile: null, policies: [], agent, ...(agent === "langchain-deepagents-code" @@ -253,12 +259,18 @@ function createFixture(opts: { ].join("\\n"); const registeredProvidersLiteral = JSON.stringify(registeredProviders ?? null); + const hermesProviderStatePath = path.join(tmpDir, "hermes-provider-credential-key"); + const initialHermesCredentialKey = + hermesAuthMethod === "api_key" ? "NOUS_API_KEY" : "OPENAI_API_KEY"; fs.writeFileSync( path.join(tmpDir, "openshell"), `#!/usr/bin/env node -const fs = require("node:fs"); +const fs = require("fs"); const a = process.argv.slice(2); const registeredProviders = ${registeredProvidersLiteral}; +const hermesProviderStatePath = ${JSON.stringify(hermesProviderStatePath)}; +const requiredFeatures = "request-body-credential-rewrite websocket-credential-rewrite allow_all_known_mcp_methods"; +if (a[0]==="-V" || a[0]==="--version") { process.stdout.write("openshell 0.0.72\\n"); process.exit(0); } if (a[0]==="sandbox" && a[1]==="list") { process.stdout.write("${sandboxName} Ready\\n"); process.exit(0); } if (a[0]==="sandbox" && a[1]==="ssh-config") { process.stdout.write("${sshConfig}\\n"); process.exit(0); } if (a[0]==="sandbox" && a[1]==="delete") { fs.writeFileSync(${JSON.stringify(deleteMarker)}, "deleted\\n"); process.exit(0); } @@ -277,21 +289,53 @@ if (a[0]==="sandbox" && a[1]==="exec") { } process.exit(0); } -if (a[0]==="status") { process.stdout.write("Status: Connected\\nGateway: nemoclaw\\n"); process.exit(0); } -if (a[0]==="gateway" && a[1]==="info") { process.stdout.write("Gateway: nemoclaw\\n"); process.exit(0); } +if (a[0]==="status") { process.stdout.write("Server Status\\n Gateway: nemoclaw\\n Status: Connected\\n"); process.exit(0); } +if (a[0]==="gateway" && a[1]==="info") { const i=a.indexOf("-g"); const name=i>=0?a[i+1]:"nemoclaw"; process.stdout.write("Gateway Info\\n\\nGateway: " + name + "\\n"); process.exit(0); } if (a[0]==="gateway" && a[1]==="select") { process.exit(0); } -if (a[0]==="inference" && a[1]==="get") { process.stdout.write('{"provider":"${provider}","model":"meta/llama-3.3-70b-instruct"}\\n'); process.exit(0); } +if (a[0]==="inference" && a[1]==="get") { process.stdout.write("Gateway inference:\\n Provider: ${provider}\\n Model: meta/llama-3.3-70b-instruct\\n"); process.exit(0); } if (a[0]==="inference" && a[1]==="set") { process.exit(0); } if (a[0]==="provider" && a[1]==="get") { - if (Array.isArray(registeredProviders)) process.exit(registeredProviders.includes(a[2]) ? 0 : 1); - process.exit(${providerRegistered ? 0 : 1}); + const providerName = a[2]; + const persistedHermes = providerName === "hermes-provider" && fs.existsSync(hermesProviderStatePath); + const exists = persistedHermes || (Array.isArray(registeredProviders) + ? registeredProviders.includes(providerName) + : ${providerRegistered ? "true" : "false"}); + if (!exists) process.exit(1); + if (providerName === "hermes-provider") { + const credentialKey = persistedHermes + ? fs.readFileSync(hermesProviderStatePath, "utf8").trim() + : ${JSON.stringify(initialHermesCredentialKey)}; + process.stdout.write("Provider:\\n Name: hermes-provider\\n Credential keys: " + credentialKey + "\\n"); + } + process.exit(0); +} +if (a[0]==="provider" && (a[1]==="create" || a[1]==="update")) { + const nameIndex = a.indexOf("--name"); + const providerName = a[1] === "create" ? a[nameIndex + 1] : a[2]; + const credentialIndex = a.indexOf("--credential"); + if (providerName === "hermes-provider" && credentialIndex >= 0) { + fs.writeFileSync(hermesProviderStatePath, a[credentialIndex + 1]); + } + process.exit(0); } if (a[0]==="provider") { process.exit(0); } +if (a[0]==="forward" && a[1]==="list") { process.stdout.write("SANDBOX BIND PORT PID STATUS\\n${sandboxName} 127.0.0.1 18789 4242 running\\n"); process.exit(0); } if (a[0]==="forward") { process.exit(0); } process.exit(0); `, { mode: 0o755 }, ); + for (const component of ["openshell-gateway", "openshell-sandbox"]) { + fs.writeFileSync( + path.join(tmpDir, component), + `#!/usr/bin/env node +const requiredFeatures = "request-body-credential-rewrite websocket-credential-rewrite allow_all_known_mcp_methods"; +if (process.argv[2] === "-V" || process.argv[2] === "--version") process.stdout.write("${component} 0.0.72\\n"); +process.exit(0); +`, + { mode: 0o755 }, + ); + } // ── Fake ps for active SSH session detection ────────────────── const activeSessionLines = Array.from( @@ -315,8 +359,25 @@ process.exit(0); path.join(tmpDir, "docker"), `#!/usr/bin/env node const a = process.argv.slice(2); +if (a[0]==="info") { + process.stdout.write(JSON.stringify({ServerVersion:"27.0.0", OperatingSystem:"Docker Engine", NCPU:8, MemTotal:17179869184}) + "\\n"); + process.exit(0); +} if (a[0]==="build") { process.exit(${dockerBuildExitCode}); } -if (a[0]==="image" && a[1]==="inspect") { process.exit(0); } +if (a[0]==="image" && a[1]==="inspect") { + const formatIndex = a.indexOf("--format"); + const format = formatIndex >= 0 ? a[formatIndex + 1] : ""; + if (format === "{{.Id}}") process.stdout.write("sha256:${"a".repeat(64)}\\n"); + if (format === "{{json .RepoDigests}}") process.stdout.write("[]\\n"); + process.exit(0); +} +if (a[0]==="tag" || a[0]==="rmi") { process.exit(0); } +if (a[0]==="run") { + if (a.includes("nslookup")) process.stdout.write("Server: 127.0.0.11\\n** server can't find nemoclaw.invalid: NXDOMAIN\\n"); + else if (a.includes("/usr/bin/ldd")) process.stdout.write("ldd (GNU libc) 2.41\\n"); + else process.stdout.write("nemoclaw-hermes-mcp-runtime-ok\\n"); + process.exit(0); +} if (a[0]==="inspect") { process.stdout.write("true\\n"); process.exit(0); } if (a[0]==="ps") { process.exit(0); } process.stderr.write("unexpected docker call: " + a.join(" ") + "\\n"); @@ -361,11 +422,11 @@ process.exit(0); function runRebuild( fixture: ReturnType, extraEnv: Record = {}, - options: { yes?: boolean; input?: string } = {}, + options: { yes?: boolean; input?: string; timeoutMs?: number } = {}, ) { const args = [fixture.sandboxName, "rebuild"]; if (options.yes !== false) args.push("--yes"); - return runCli(fixture, args, extraEnv, options.input); + return runCli(fixture, args, extraEnv, options.input, options.timeoutMs); } function runCli( @@ -373,6 +434,7 @@ function runCli( args: string[], extraEnv: Record = {}, input?: string, + timeoutMs = 60_000, ) { const argv = [path.join(REPO_ROOT, "bin", "nemoclaw.js"), ...args]; return spawnSync(process.execPath, argv, { @@ -382,12 +444,14 @@ function runCli( env: { HOME: fixture.tmpDir, PATH: fixture.tmpDir + ":" + NODE_BIN + ":/usr/bin:/bin", + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", + NEMOCLAW_SKIP_HOST_DNS_PREFLIGHT: "1", NEMOCLAW_NON_INTERACTIVE: "1", NEMOCLAW_NO_CONNECT_HINT: "1", NO_COLOR: "1", ...extraEnv, }, - timeout: 30_000, + timeout: execTimeout(timeoutMs), }); } @@ -624,7 +688,7 @@ describe("atomic rebuild (#2273)", () => { }); it("copies Hermes messaging channels from the registry into the rebuild resume session", { - timeout: 60_000, + timeout: testTimeout(120_000), }, () => { const f = createFixture({ agent: "hermes", @@ -636,7 +700,7 @@ describe("atomic rebuild (#2273)", () => { }, }); - const result = runRebuild(f); + const result = runRebuild(f, {}, { timeoutMs: 120_000 }); const output = (result.stderr || "") + (result.stdout || ""); expect(output).toContain("Creating new sandbox with current image"); @@ -862,7 +926,13 @@ describe("atomic rebuild (#2273)", () => { expect(output).not.toContain("Missing credential: NOUS_API_KEY"); expect(output).not.toContain("provider credential not found"); + expect(output).toContain( + "Hermes Provider is not registered in OpenShell; registering it from the configured exported API-key environment variable before rebuild.", + ); + expect(output).not.toContain("NOUS_API_KEY"); + expect(output).not.toContain("nous-key-from-env"); expect(output).toContain("Backing up sandbox state"); + expect(output).toContain("State backed up"); }); it("uses the registered nvidia-prod provider in OpenShell instead of requiring NVIDIA_INFERENCE_API_KEY", { diff --git a/test/rebuild-messaging-conflict-preflight.test.ts b/test/rebuild-messaging-conflict-preflight.test.ts index bde4ad00c6f..b74e124612f 100644 --- a/test/rebuild-messaging-conflict-preflight.test.ts +++ b/test/rebuild-messaging-conflict-preflight.test.ts @@ -121,6 +121,11 @@ function createConflictFixture() { model: "meta/llama-3.3-70b-instruct", provider: "nvidia-prod", gpuEnabled: false, + sandboxGpuMode: "0", + gatewayName: "nemoclaw", + gatewayPort: 8080, + dashboardPort: 18789, + fromDockerfile: null, policies: [], agent: null, messaging: { schemaVersion: 1, plan: teamsPlan(name, "shared-teams-hash") }, @@ -160,8 +165,8 @@ const a = process.argv.slice(2); if (a[0]==="sandbox" && a[1]==="list") { process.stdout.write("my-assistant\\n"); process.exit(0); } if (a[0]==="sandbox" && a[1]==="ssh-config") { process.stdout.write("${sshConfig}\\n"); process.exit(0); } if (a[0]==="sandbox" && a[1]==="delete") { process.exit(0); } -if (a[0]==="status") { process.stdout.write("running\\n"); process.exit(0); } -if (a[0]==="gateway" && a[1]==="info") { process.stdout.write("nemoclaw\\n"); process.exit(0); } +if (a[0]==="status") { process.stdout.write("Status: Connected\\nGateway: nemoclaw\\n"); process.exit(0); } +if (a[0]==="gateway" && a[1]==="info") { process.stdout.write("Gateway: nemoclaw\\n"); process.exit(0); } if (a[0]==="gateway" && a[1]==="select") { process.exit(0); } if (a[0]==="inference" && a[1]==="get") { process.stdout.write('{"provider":"nvidia-prod","model":"meta/llama-3.3-70b-instruct"}\\n'); process.exit(0); } if (a[0]==="inference") { process.exit(0); } @@ -209,6 +214,7 @@ function runRebuild(tmpDir: string) { env: { HOME: tmpDir, PATH: `${tmpDir}:${NODE_BIN}:/usr/bin:/bin`, + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", NEMOCLAW_NON_INTERACTIVE: "1", NEMOCLAW_NO_CONNECT_HINT: "1", NO_COLOR: "1", diff --git a/test/rebuild-shields-auto-unlock.test.ts b/test/rebuild-shields-auto-unlock.test.ts index 515d075ca49..0dd8e854287 100644 --- a/test/rebuild-shields-auto-unlock.test.ts +++ b/test/rebuild-shields-auto-unlock.test.ts @@ -97,6 +97,11 @@ function createFixture(opts: { shieldsLocked: boolean }) { model: "meta/llama-3.3-70b-instruct", provider: "nvidia-prod", gpuEnabled: false, + sandboxGpuMode: "0", + gatewayName: "nemoclaw", + gatewayPort: 8080, + dashboardPort: 18789, + fromDockerfile: null, policies: [], agent: null, openshellDriver: "vm", @@ -172,22 +177,36 @@ function createFixture(opts: { shieldsLocked: boolean }) { path.join(tmpDir, "openshell"), `#!/usr/bin/env node const a = process.argv.slice(2); +const requiredFeatures = "request-body-credential-rewrite websocket-credential-rewrite allow_all_known_mcp_methods"; +if (a[0]==="-V" || a[0]==="--version") { process.stdout.write("openshell 0.0.72\\n"); process.exit(0); } if (a[0]==="sandbox" && a[1]==="list") { process.stdout.write("${sandboxName}\\n"); process.exit(0); } if (a[0]==="sandbox" && a[1]==="ssh-config") { process.stdout.write("${sshConfig}\\n"); process.exit(0); } if (a[0]==="sandbox" && a[1]==="delete") { process.exit(0); } if (a[0]==="policy" && a[1]==="get") { process.stdout.write("version: 1\\nnetwork_policies:\\n test: {}\\n"); process.exit(0); } if (a[0]==="policy" && a[1]==="set") { process.exit(0); } -if (a[0]==="status") { process.stdout.write("running\\n"); process.exit(0); } -if (a[0]==="gateway" && a[1]==="info") { process.stdout.write("nemoclaw\\n"); process.exit(0); } +if (a[0]==="status") { process.stdout.write("Server Status\\n Gateway: nemoclaw\\n Status: Connected\\n"); process.exit(0); } +if (a[0]==="gateway" && a[1]==="info") { const i=a.indexOf("-g"); const name=i>=0?a[i+1]:"nemoclaw"; process.stdout.write("Gateway Info\\n\\nGateway: " + name + "\\n"); process.exit(0); } if (a[0]==="gateway" && a[1]==="select") { process.exit(0); } -if (a[0]==="inference" && a[1]==="get") { process.stdout.write('{"provider":"nvidia-prod","model":"meta/llama-3.3-70b-instruct"}\\n'); process.exit(0); } +if (a[0]==="inference" && a[1]==="get") { process.stdout.write("Gateway inference:\\n Provider: nvidia-prod\\n Model: meta/llama-3.3-70b-instruct\\n"); process.exit(0); } if (a[0]==="inference" && a[1]==="set") { process.exit(0); } if (a[0]==="provider") { process.exit(0); } +if (a[0]==="forward" && a[1]==="list") { process.stdout.write("SANDBOX BIND PORT PID STATUS\\n${sandboxName} 127.0.0.1 18789 4242 running\\n"); process.exit(0); } if (a[0]==="forward") { process.exit(0); } process.exit(0); `, { mode: 0o755 }, ); + for (const component of ["openshell-gateway", "openshell-sandbox"]) { + fs.writeFileSync( + path.join(tmpDir, component), + `#!/usr/bin/env node +const requiredFeatures = "request-body-credential-rewrite websocket-credential-rewrite allow_all_known_mcp_methods"; +if (process.argv[2] === "-V" || process.argv[2] === "--version") process.stdout.write("${component} 0.0.72\\n"); +process.exit(0); +`, + { mode: 0o755 }, + ); + } // Fake docker — covers both the basic cases and kubectl exec proxying. // For shields lock/unlock, we return zero exit with the data shields.ts @@ -204,10 +223,26 @@ function readLockState() { function writeLockState(state) { fs.writeFileSync(lockStatePath, state); } +if (a[0]==="info") { + process.stdout.write(JSON.stringify({ServerVersion:"27.0.0", OperatingSystem:"Docker Engine", NCPU:8, MemTotal:17179869184}) + "\\n"); + process.exit(0); +} if (a[0]==="build") { process.exit(0); } -if (a[0]==="image" && a[1]==="inspect") { process.exit(0); } +if (a[0]==="image" && a[1]==="inspect") { + const formatIndex = a.indexOf("--format"); + const format = formatIndex >= 0 ? a[formatIndex + 1] : ""; + if (format === "{{.Id}}") process.stdout.write("sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\\n"); + if (format === "{{json .RepoDigests}}") process.stdout.write("[]\\n"); + process.exit(0); +} +if (a[0]==="tag" || a[0]==="rmi") { process.exit(0); } +if (a[0]==="run") { + if (a.includes("nslookup")) process.stdout.write("Server: 127.0.0.11\\n** server can't find nemoclaw.invalid: NXDOMAIN\\n"); + else if (a.includes("/usr/bin/ldd")) process.stdout.write("ldd (GNU libc) 2.41\\n"); + process.exit(0); +} if (a[0]==="inspect") { process.stdout.write("true\\n"); process.exit(0); } -if (a[0]==="ps") { process.stdout.write("openshell-${sandboxName}-abc123\\n"); process.exit(0); } +if (a[0]==="ps") { process.stdout.write("abc123\\topenshell-${sandboxName}-abc123\\n"); process.exit(0); } // Supports both direct exec ("docker exec --user root ") // and legacy kubectl proxying ("docker exec kubectl exec ... -- "). if (a[0]==="exec") { @@ -346,6 +381,7 @@ function runRebuild(fixture: ReturnType) { HOME: fixture.tmpDir, PATH: fixture.tmpDir + ":" + NODE_BIN + ":/usr/bin:/bin", NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", + NEMOCLAW_SKIP_HOST_DNS_PREFLIGHT: "1", NEMOCLAW_NON_INTERACTIVE: "1", NEMOCLAW_NO_CONNECT_HINT: "1", NO_COLOR: "1", diff --git a/test/rebuild-stale-recovery.test.ts b/test/rebuild-stale-recovery.test.ts index 23f85e44e05..a91c09a4f30 100644 --- a/test/rebuild-stale-recovery.test.ts +++ b/test/rebuild-stale-recovery.test.ts @@ -63,6 +63,8 @@ function createStaleFixture( const sandboxName = "my-assistant"; const provider = "nvidia-prod"; const credentialEnv = "NVIDIA_INFERENCE_API_KEY"; + const targetGatewayName = gatewayName ?? "nemoclaw"; + const targetGatewayPort = targetGatewayName === "nemoclaw-9000" ? 9000 : 8080; const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-4497-")); tmpFixtures.push(tmpDir); @@ -79,9 +81,13 @@ function createStaleFixture( model: "meta/llama-3.3-70b-instruct", provider, gpuEnabled: false, + sandboxGpuMode: "0", + gatewayName: targetGatewayName, + gatewayPort: targetGatewayPort, + dashboardPort: 28789, + fromDockerfile: null, policies: [], agent: null, - ...(gatewayName ? { gatewayName } : {}), }, }, }), @@ -113,7 +119,7 @@ function createStaleFixture( webSearchConfig: null, policyPresets: [], messagingPlan: null, - metadata: { gatewayName: "nemoclaw", fromDockerfile: null }, + metadata: { gatewayName: targetGatewayName, fromDockerfile: null }, steps: {}, }), { mode: 0o600 }, @@ -133,38 +139,72 @@ function createStaleFixture( const listBody = liveListIncludesSandbox ? `process.stdout.write("${sandboxName}\\n"); process.exit(0);` : `process.stdout.write("\\n"); process.exit(0);`; - // When a foreign gateway is active, `status` reports a different active - // gateway even though the named nemoclaw gateway still exists. This models - // the multi-gateway data-loss risk: the sandbox is hidden from the active - // gateway's list but rebuild must NOT destroy it. - const statusBody = foreignGatewayActive + // The authoritative target preflights run before liveness reconciliation. + // Report the recorded target as healthy until `sandbox list` is queried, + // then expose the drift that these guard tests are specifically exercising. + const healthyTargetStatus = `process.stdout.write("Server Status\\n\\n Gateway: ${targetGatewayName}\\n Server: http://127.0.0.1:${targetGatewayPort}\\n Status: Connected\\n"); process.exit(0);`; + const lateDriftStatus = foreignGatewayActive ? `process.stdout.write("Server Status\\n\\n Gateway: other-gw\\n Server: http://127.0.0.1:9090\\n Status: Connected\\n"); process.exit(0);` - : `process.stdout.write("Server Status\\n\\n Gateway: nemoclaw\\n Server: http://127.0.0.1:8080\\n Status: Connected\\n"); process.exit(0);`; + : gatewayName + ? `process.stdout.write("Server Status\\n\\n Gateway: nemoclaw\\n Server: http://127.0.0.1:8080\\n Status: Connected\\n"); process.exit(0);` + : healthyTargetStatus; + const livenessProbeMarker = path.join(tmpDir, "sandbox-list-probed"); fs.writeFileSync( path.join(tmpDir, "openshell"), `#!/usr/bin/env node +const fs = require("fs"); const a = process.argv.slice(2); -if (a[0]==="sandbox" && a[1]==="list") { ${listBody} } +const requiredFeatures = "request-body-credential-rewrite websocket-credential-rewrite allow_all_known_mcp_methods"; +const livenessProbeMarker = ${JSON.stringify(livenessProbeMarker)}; +if (a[0]==="-V" || a[0]==="--version") { process.stdout.write("openshell 0.0.72\\n"); process.exit(0); } +if (a[0]==="sandbox" && a[1]==="list") { fs.writeFileSync(livenessProbeMarker, "1"); ${listBody} } if (a[0]==="sandbox" && a[1]==="delete") { process.exit(0); } if (a[0]==="sandbox" && a[1]==="get") { process.stderr.write("Error: × Not Found: sandbox not found\\n"); process.exit(1); } -if (a[0]==="status") { ${statusBody} } -if (a[0]==="gateway" && a[1]==="info") { process.stdout.write("Gateway Info\\n\\nGateway: nemoclaw\\nGateway endpoint: https://127.0.0.1:8080/\\n"); process.exit(0); } +if (a[0]==="status") { if (fs.existsSync(livenessProbeMarker)) { ${lateDriftStatus} } ${healthyTargetStatus} } +if (a[0]==="gateway" && a[1]==="info") { process.stdout.write("Gateway Info\\n\\nGateway: ${targetGatewayName}\\nGateway endpoint: https://127.0.0.1:${targetGatewayPort}/\\n"); process.exit(0); } if (a[0]==="gateway" && a[1]==="select") { process.exit(0); } -if (a[0]==="gateway") { process.stdout.write("nemoclaw\\n"); process.exit(0); } -if (a[0]==="inference" && a[1]==="get") { process.stdout.write('{"provider":"${provider}","model":"meta/llama-3.3-70b-instruct"}\\n'); process.exit(0); } +if (a[0]==="gateway") { process.stdout.write("${targetGatewayName}\\n"); process.exit(0); } +if (a[0]==="inference" && a[1]==="get") { process.stdout.write("Gateway inference:\\n Provider: ${provider}\\n Model: meta/llama-3.3-70b-instruct\\n"); process.exit(0); } if (a[0]==="provider" && a[1]==="get") { process.exit(0); } process.exit(0); `, { mode: 0o755 }, ); + for (const component of ["openshell-gateway", "openshell-sandbox"]) { + fs.writeFileSync( + path.join(tmpDir, component), + `#!/usr/bin/env node +const requiredFeatures = "request-body-credential-rewrite websocket-credential-rewrite allow_all_known_mcp_methods"; +if (process.argv[2] === "-V" || process.argv[2] === "--version") process.stdout.write("${component} 0.0.72\\n"); +process.exit(0); +`, + { mode: 0o755 }, + ); + } // Fake docker — recreate path may shell out; succeed on common probes. fs.writeFileSync( path.join(tmpDir, "docker"), `#!/usr/bin/env node const a = process.argv.slice(2); +if (a[0]==="info") { + process.stdout.write(JSON.stringify({ServerVersion:"27.0.0", OperatingSystem:"Docker Engine", NCPU:8, MemTotal:17179869184}) + "\\n"); + process.exit(0); +} if (a[0]==="build") { process.exit(0); } -if (a[0]==="image" && a[1]==="inspect") { process.exit(0); } +if (a[0]==="image" && a[1]==="inspect") { + const formatIndex = a.indexOf("--format"); + const format = formatIndex >= 0 ? a[formatIndex + 1] : ""; + if (format === "{{.Id}}") process.stdout.write("sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\\n"); + if (format === "{{json .RepoDigests}}") process.stdout.write("[]\\n"); + process.exit(0); +} +if (a[0]==="tag" || a[0]==="rmi") { process.exit(0); } +if (a[0]==="run") { + if (a.includes("nslookup")) process.stdout.write("Server: 127.0.0.11\\n** server can't find nemoclaw.invalid: NXDOMAIN\\n"); + else if (a.includes("/usr/bin/ldd")) process.stdout.write("ldd (GNU libc) 2.41\\n"); + process.exit(0); +} if (a[0]==="inspect") { process.stdout.write("true\\n"); process.exit(0); } if (a[0]==="ps") { process.exit(0); } process.exit(0); @@ -185,6 +225,8 @@ function runRebuild(fixture: { tmpDir: string; sandboxName: string }) { env: { HOME: fixture.tmpDir, PATH: fixture.tmpDir + ":" + NODE_BIN + ":/usr/bin:/bin", + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", + NEMOCLAW_SKIP_HOST_DNS_PREFLIGHT: "1", NEMOCLAW_NON_INTERACTIVE: "1", NEMOCLAW_NO_CONNECT_HINT: "1", NO_COLOR: "1", diff --git a/test/registry.test.ts b/test/registry.test.ts index c6a500d36d9..0ab05f4e2b1 100644 --- a/test/registry.test.ts +++ b/test/registry.test.ts @@ -89,6 +89,33 @@ describe("registry", () => { expect(data.sandboxes.alpha.nimContainer).toBeNull(); }); + it("stores rebuild fidelity metadata at registration time", () => { + registry.registerSandbox({ + name: "alpha", + webSearchEnabled: true, + fromDockerfile: "/tmp/Dockerfile.custom", + hermesAuthMethod: "oauth", + }); + expect(registry.getSandbox("alpha")).toMatchObject({ + webSearchEnabled: true, + fromDockerfile: "/tmp/Dockerfile.custom", + hermesAuthMethod: "oauth", + }); + }); + + it("stores normalized compatible-endpoint reasoning state", () => { + registry.registerSandbox({ + name: "alpha", + provider: "compatible-endpoint", + model: "reasoning-model", + endpointUrl: "https://example.test/v1", + compatibleEndpointReasoning: "true", + }); + const data = JSON.parse(fs.readFileSync(regFile, "utf-8")); + expect(data.sandboxes.alpha.compatibleEndpointReasoning).toBe("true"); + expect(registry.getSandbox("alpha").compatibleEndpointReasoning).toBe("true"); + }); + it("persists distinct gateway bindings for two sandboxes on different ports (#4422)", () => { registry.registerSandbox({ name: "first", @@ -129,6 +156,67 @@ describe("registry", () => { expect(data.sandboxes.alpha.livePhase).toBeUndefined(); }); + it("persists MCP server state without local proxy secrets", () => { + registry.registerSandbox({ + name: "alpha", + agent: "openclaw", + mcp: { + bridges: { + github: { + server: "github", + agent: "openclaw", + adapter: "mcporter", + url: "https://api.githubcopilot.com/mcp/", + env: ["GITHUB_TOKEN"], + providerName: "alpha-mcp-github", + providerId: "11111111-2222-4333-8444-555555555555", + policyName: "mcp-bridge-github", + addedAt: new Date(0).toISOString(), + }, + }, + }, + }); + + const raw = JSON.parse(fs.readFileSync(regFile, "utf-8")); + const entry = raw.sandboxes.alpha.mcp.bridges.github; + + expect(entry).toMatchObject({ + url: "https://api.githubcopilot.com/mcp/", + env: ["GITHUB_TOKEN"], + providerName: "alpha-mcp-github", + providerId: "11111111-2222-4333-8444-555555555555", + policyName: "mcp-bridge-github", + }); + expect(entry.token).toBeUndefined(); + expect(entry.command).toBeUndefined(); + expect(entry.port).toBeUndefined(); + }); + + it("normalizes MCP bridge maps by the recovered server name", () => { + registry.registerSandbox({ + name: "alpha", + agent: "openclaw", + mcp: { + bridges: { + stale_key: { + server: "github", + agent: "openclaw", + adapter: "mcporter", + url: "https://api.githubcopilot.com/mcp/", + env: ["GITHUB_TOKEN"], + providerName: "alpha-mcp-github", + policyName: "mcp-bridge-github", + addedAt: new Date(0).toISOString(), + }, + }, + }, + }); + + const raw = JSON.parse(fs.readFileSync(regFile, "utf-8")); + expect(raw.sandboxes.alpha.mcp.bridges.github.server).toBe("github"); + expect(raw.sandboxes.alpha.mcp.bridges.stale_key).toBeUndefined(); + }); + it("normalizes configured inference fields into a discriminated view", () => { const configured = { name: "alpha", provider: "nvidia-prod", model: "nvidia/test" }; const missingProvider = { name: "beta", provider: null, model: "nvidia/test" }; @@ -172,6 +260,124 @@ describe("registry", () => { expect(sb.model).toBe("new-model"); }); + it("persists MCP env names without raw host env values", () => { + registry.registerSandbox({ name: "mcp-sb", agent: "openclaw" }); + registry.updateSandbox("mcp-sb", { + mcp: { + bridges: { + github: { + server: "github", + agent: "openclaw", + adapter: "mcporter", + url: "https://api.githubcopilot.com/mcp/", + env: ["GITHUB_TOKEN"], + providerName: "mcp-sb-mcp-github", + providerId: "11111111-2222-4333-8444-555555555555", + policyName: "mcp-bridge-github", + addedAt: new Date(0).toISOString(), + }, + }, + }, + }); + + const raw = fs.readFileSync(regFile, "utf-8"); + const data = JSON.parse(raw); + expect(data.sandboxes["mcp-sb"].mcp.bridges.github.env).toEqual(["GITHUB_TOKEN"]); + expect(data.sandboxes["mcp-sb"].mcp.bridges.github.providerName).toBe("mcp-sb-mcp-github"); + expect(data.sandboxes["mcp-sb"].mcp.bridges.github.providerId).toBe( + "11111111-2222-4333-8444-555555555555", + ); + expect(data.sandboxes["mcp-sb"].mcp.bridges.github.token).toBeUndefined(); + expect(raw).not.toContain("ghp_"); + expect(raw).not.toContain("secret-value"); + }); + + it("drops invalid persisted MCP bridge entries during registry serialization", () => { + registry.registerSandbox({ name: "mcp-safe", agent: "openclaw" }); + registry.updateSandbox("mcp-safe", { + mcp: { + bridges: { + ok: { + server: "ok", + agent: "openclaw", + adapter: "mcporter", + url: "https://api.githubcopilot.com/mcp/#ignored", + env: ["GITHUB_TOKEN", "GITHUB_TOKEN"], + providerName: "mcp-safe-mcp-ok", + policyName: "mcp-bridge-ok", + addedAt: new Date(0).toISOString(), + }, + credentialUrl: { + server: "credentialUrl", + agent: "openclaw", + adapter: "mcporter", + url: "https://user:secret@example.test/mcp", + env: ["TOKEN"], + providerName: "mcp-safe-mcp-credential", + policyName: "mcp-bridge-credential", + addedAt: new Date(0).toISOString(), + }, + privateIp: { + server: "privateIp", + agent: "openclaw", + adapter: "mcporter", + url: "http://127.0.0.1:31337/mcp", + env: ["TOKEN"], + providerName: "mcp-safe-mcp-private", + policyName: "mcp-bridge-private", + addedAt: new Date(0).toISOString(), + }, + invalidEnv: { + server: "invalidEnv", + agent: "openclaw", + adapter: "mcporter", + url: "https://api.githubcopilot.com/mcp/", + env: ["TOKEN=secret"], + providerName: "mcp-safe-mcp-invalid-env", + policyName: "mcp-bridge-invalid-env", + addedAt: new Date(0).toISOString(), + }, + unknownAdapter: { + server: "unknownAdapter", + agent: "openclaw", + adapter: "unknown", + url: "https://api.githubcopilot.com/mcp/", + env: ["TOKEN"], + providerName: "mcp-safe-mcp-unknown", + policyName: "mcp-bridge-unknown", + addedAt: new Date(0).toISOString(), + }, + invalidProviderId: { + server: "invalidProviderId", + agent: "openclaw", + adapter: "mcporter", + url: "https://api.githubcopilot.com/mcp/", + env: ["TOKEN"], + providerName: "mcp-safe-mcp-invalid-provider-id", + providerId: "invalid provider id", + policyName: "mcp-bridge-invalid-provider-id", + addedAt: new Date(0).toISOString(), + }, + oversizedUrl: { + server: "oversizedUrl", + agent: "openclaw", + adapter: "mcporter", + url: `https://api.githubcopilot.com/${"a".repeat(2_048)}`, + env: ["TOKEN"], + providerName: "mcp-safe-mcp-oversized", + policyName: "mcp-bridge-oversized", + addedAt: new Date(0).toISOString(), + }, + }, + }, + }); + + const bridges = registry.getSandbox("mcp-safe").mcp.bridges; + expect(Object.keys(bridges)).toEqual(["ok"]); + expect(bridges.ok.url).toBe("https://api.githubcopilot.com/mcp/"); + expect(bridges.ok.env).toEqual(["GITHUB_TOKEN"]); + }); + it("updateSandbox returns false for nonexistent sandbox", () => { expect(registry.updateSandbox("nope", {})).toBe(false); }); diff --git a/test/repro-2201.test.ts b/test/repro-2201.test.ts index ccf100709a0..7fe9c2fa490 100644 --- a/test/repro-2201.test.ts +++ b/test/repro-2201.test.ts @@ -104,6 +104,13 @@ function createFixture({ tmpFixtures.push(tmpDir); const nemoclawDir = path.join(tmpDir, ".nemoclaw"); fs.mkdirSync(nemoclawDir, { recursive: true, mode: 0o700 }); + const durableFromDockerfile = fromDockerfile + ? path.join(tmpDir, "custom-image", "Dockerfile") + : null; + for (const dockerfilePath of durableFromDockerfile ? [durableFromDockerfile] : []) { + fs.mkdirSync(path.dirname(dockerfilePath), { recursive: true }); + fs.writeFileSync(dockerfilePath, "FROM scratch\n"); + } const rebuildTargetMessagingPlan = rebuildTarget.messagingPlanChannels ? makeMessagingPlan( rebuildTarget.name, @@ -130,6 +137,11 @@ function createFixture({ model: "m", provider: "p", gpuEnabled: false, + sandboxGpuMode: "0", + gatewayName: "nemoclaw", + gatewayPort: 8080, + dashboardPort: 18789, + fromDockerfile: durableFromDockerfile, policies: [], agent: rebuildTarget.agent, ...(rebuildTargetMessagingPlan @@ -141,6 +153,11 @@ function createFixture({ model: "m", provider: "p", gpuEnabled: false, + sandboxGpuMode: "0", + gatewayName: "nemoclaw", + gatewayPort: 8080, + dashboardPort: 18790, + fromDockerfile: lastOnboarded.name === rebuildTarget.name ? durableFromDockerfile : null, policies: [], agent: lastOnboarded.agent, ...(lastOnboardedMessagingPlan @@ -177,7 +194,7 @@ function createFixture({ webSearchConfig: null, policyPresets: [], messagingPlan: lastOnboardedMessagingPlan, - metadata: { gatewayName: "nemoclaw", fromDockerfile: fromDockerfile }, + metadata: { gatewayName: "nemoclaw", fromDockerfile: durableFromDockerfile }, steps: { preflight: { status: "complete", startedAt: null, completedAt: null, error: null }, gateway: { status: "complete", startedAt: null, completedAt: null, error: null }, @@ -213,6 +230,12 @@ function createFixture({ path.join(tmpDir, "openshell"), `#!/usr/bin/env node const a = process.argv.slice(2); +const requiredFeatures = "request-body-credential-rewrite websocket-credential-rewrite allow_all_known_mcp_methods"; +if (a[0]==="-V" || a[0]==="--version") { process.stdout.write("openshell 0.0.72\\n"); process.exit(0); } +if (a[0]==="status") { process.stdout.write("Server Status\\n Gateway: nemoclaw\\n Status: Connected\\n"); process.exit(0); } +if (a[0]==="gateway" && a[1]==="info") { const i=a.indexOf("-g"); const name=i>=0?a[i+1]:"nemoclaw"; process.stdout.write("Gateway Info\\n\\nGateway: " + name + "\\n"); process.exit(0); } +if (a[0]==="gateway" && a[1]==="select") { process.exit(0); } +if (a[0]==="inference" && a[1]==="get") { process.stdout.write("Gateway inference:\\n Provider: p\\n Model: m\\n"); process.exit(0); } if (a[0]==="sandbox" && a[1]==="list") { process.stdout.write("${sandboxName}\\n"); process.exit(0); } if (a[0]==="sandbox" && a[1]==="ssh-config") { process.stdout.write("${sshConfig}\\n"); process.exit(0); } if (a[0]==="sandbox" && a[1]==="delete") { process.exit(0); } @@ -220,6 +243,17 @@ process.exit(0); `, { mode: 0o755 }, ); + for (const component of ["openshell-gateway", "openshell-sandbox"]) { + fs.writeFileSync( + path.join(tmpDir, component), + `#!/usr/bin/env node +const requiredFeatures = "request-body-credential-rewrite websocket-credential-rewrite allow_all_known_mcp_methods"; +if (process.argv[2] === "-V" || process.argv[2] === "--version") process.stdout.write("${component} 0.0.72\\n"); +process.exit(0); +`, + { mode: 0o755 }, + ); + } // ── Fake docker ───────────────────────────────────────────────── // Hermes rebuilds refresh the local agent base image before deleting the @@ -228,8 +262,29 @@ process.exit(0); path.join(tmpDir, "docker"), `#!/usr/bin/env node const a = process.argv.slice(2); +if (a[0]==="info") { + process.stdout.write(JSON.stringify({ServerVersion:"27.0.0", OperatingSystem:"Docker Engine", NCPU:8, MemTotal:17179869184}) + "\\n"); + process.exit(0); +} if (a[0]==="build") { process.exit(0); } +if (a[0]==="image" && a[1]==="inspect" && a[2]==="--format") { + if (a[3]==="{{.Id}}") process.stdout.write("sha256:${"a".repeat(64)}\\n"); + if (a[3]==="{{json .RepoDigests}}") process.stdout.write("[]\\n"); + process.exit(0); +} if (a[0]==="image" && a[1]==="inspect") { process.exit(0); } +if (a[0]==="run" && a.includes("nslookup")) { + process.stdout.write("Server: 127.0.0.11\\n** server can't find nemoclaw.invalid: NXDOMAIN\\n"); + process.exit(0); +} +if (a[0]==="run" && a.includes("/usr/bin/ldd")) { + process.stdout.write("ldd (GNU libc) 2.41\\n"); + process.exit(0); +} +if (a[0]==="run" && a.includes("/opt/hermes/.venv/bin/python")) { + process.stdout.write("nemoclaw-hermes-mcp-runtime-ok\\n"); + process.exit(0); +} if (a[0]==="inspect") { process.stdout.write("true\\n"); process.exit(0); } process.exit(0); `, @@ -278,6 +333,8 @@ function runRebuild(fixture: ReturnType) { env: { HOME: fixture.tmpDir, PATH: fixture.tmpDir + ":" + NODE_BIN + ":/usr/bin:/bin", + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", + NEMOCLAW_SKIP_HOST_DNS_PREFLIGHT: "1", NEMOCLAW_NON_INTERACTIVE: "1", NEMOCLAW_NO_CONNECT_HINT: "1", NO_COLOR: "1", diff --git a/test/runner.test.ts b/test/runner.test.ts index 31720f74e8d..1946e47f2f1 100644 --- a/test/runner.test.ts +++ b/test/runner.test.ts @@ -79,7 +79,11 @@ describe("runner helpers", () => { const calls: SpawnCall[] = []; const originalSpawnSync = childProcess.spawnSync; // @ts-expect-error — intentional partial mock for testing - childProcess.spawnSync = captureSpawnCall(calls, { status: 0, stdout: "", stderr: "" }); + childProcess.spawnSync = captureSpawnCall(calls, { + status: 0, + stdout: "", + stderr: "", + }); try { delete require.cache[require.resolve(runnerPath)]; @@ -101,7 +105,11 @@ describe("runner helpers", () => { const calls: SpawnCall[] = []; const originalSpawnSync = childProcess.spawnSync; // @ts-expect-error — intentional partial mock for testing - childProcess.spawnSync = captureSpawnCall(calls, { status: 0, stdout: "", stderr: "" }); + childProcess.spawnSync = captureSpawnCall(calls, { + status: 0, + stdout: "", + stderr: "", + }); try { delete require.cache[require.resolve(runnerPath)]; @@ -180,14 +188,20 @@ describe("runner env merging", () => { const originalSpawnSync = childProcess.spawnSync; const originalPath = process.env.PATH; // @ts-expect-error — intentional partial mock for testing - childProcess.spawnSync = captureSpawnCall(calls, { status: 0, stdout: "", stderr: "" }); + childProcess.spawnSync = captureSpawnCall(calls, { + status: 0, + stdout: "", + stderr: "", + }); try { delete require.cache[require.resolve(runnerPath)]; const { run } = require(runnerPath); process.env.PATH = "/usr/local/bin:/usr/bin"; run(["echo", "test"], { - env: { OPENSHELL_CLUSTER_IMAGE: "ghcr.io/nvidia/openshell/cluster:0.0.12" }, + env: { + OPENSHELL_CLUSTER_IMAGE: "ghcr.io/nvidia/openshell/cluster:0.0.12", + }, }); } finally { if (originalPath === undefined) { @@ -212,14 +226,20 @@ describe("runner env merging", () => { const originalSpawnSync = childProcess.spawnSync; const originalPath = process.env.PATH; // @ts-expect-error — intentional partial mock for testing - childProcess.spawnSync = captureSpawnCall(calls, { status: 0, stdout: "", stderr: "" }); + childProcess.spawnSync = captureSpawnCall(calls, { + status: 0, + stdout: "", + stderr: "", + }); try { delete require.cache[require.resolve(runnerPath)]; const { runFile } = require(runnerPath); process.env.PATH = "/usr/local/bin:/usr/bin"; runFile("bash", ["/tmp/setup.sh"], { - env: { OPENSHELL_CLUSTER_IMAGE: "ghcr.io/nvidia/openshell/cluster:0.0.12" }, + env: { + OPENSHELL_CLUSTER_IMAGE: "ghcr.io/nvidia/openshell/cluster:0.0.12", + }, }); } finally { if (originalPath === undefined) { @@ -251,7 +271,11 @@ describe("runner env merging", () => { const originalNoProxy = process.env.NO_PROXY; const originalNoProxyLower = process.env.no_proxy; // @ts-expect-error — intentional partial mock for testing - childProcess.spawnSync = captureSpawnCall(calls, { status: 0, stdout: "", stderr: "" }); + childProcess.spawnSync = captureSpawnCall(calls, { + status: 0, + stdout: "", + stderr: "", + }); try { delete require.cache[require.resolve(runnerPath)]; @@ -300,7 +324,9 @@ describe("shellQuote", () => { const dangerous = "test; rm -rf /"; const quoted = shellQuote(dangerous); expect(quoted).toBe("'test; rm -rf /'"); - const result = spawnSync("bash", ["-c", `echo ${quoted}`], { encoding: "utf-8" }); + const result = spawnSync("bash", ["-c", `echo ${quoted}`], { + encoding: "utf-8", + }); expect(result.stdout.trim()).toBe(dangerous); }); @@ -308,7 +334,9 @@ describe("shellQuote", () => { const { shellQuote } = require(runnerPath); const payload = "test`whoami`$HOME"; const quoted = shellQuote(payload); - const result = spawnSync("bash", ["-c", `echo ${quoted}`], { encoding: "utf-8" }); + const result = spawnSync("bash", ["-c", `echo ${quoted}`], { + encoding: "utf-8", + }); expect(result.stdout.trim()).toBe(payload); }); }); @@ -667,8 +695,8 @@ describe("regression guards", () => { const tmpBin = fs.mkdtempSync(path.join(os.tmpdir(), "gh-absent-")); const stub = ` #!/usr/bin/env bash - openshell() { echo "openshell 0.0.1"; } - export -f openshell + printf '%s\n' '#!/bin/sh' 'echo "openshell 0.0.1"' > "${tmpBin}/openshell" + chmod +x "${tmpBin}/openshell" export PATH="${tmpBin}:/usr/bin:/bin" command() { if [ "\${1:-}" = "-v" ] && [ "\${2:-}" = "gh" ]; then return 1; fi; builtin command "$@"; } curl() { @@ -710,10 +738,16 @@ describe("regression guards", () => { export -f curl sha256sum() { cat >/dev/null; echo "checksum OK"; return 0; } export -f sha256sum - strings() { echo "request-body-credential-rewrite websocket-credential-rewrite"; } + strings() { echo "request-body-credential-rewrite websocket-credential-rewrite allow_all_known_mcp_methods"; } export -f strings - tar() { return 0; }; export -f tar - install() { return 0; }; export -f install + tar() { + local destination="\${@: -1}" + printf '%s\n' '#!/bin/sh' 'echo "openshell 0.0.72"' > "$destination/openshell" + printf '%s\n' '#!/bin/sh' 'echo "openshell-gateway 0.0.72"' > "$destination/openshell-gateway" + printf '%s\n' '#!/bin/sh' 'echo "openshell-sandbox 0.0.72"' > "$destination/openshell-sandbox" + chmod +x "$destination/openshell" "$destination/openshell-gateway" "$destination/openshell-sandbox" + }; export -f tar + install() { /usr/bin/install "$@"; }; export -f install source "${scriptPath}" `; try { @@ -740,8 +774,8 @@ describe("regression guards", () => { const stub = ` #!/usr/bin/env bash - openshell() { echo "openshell 0.0.1"; } - export -f openshell + printf '%s\n' '#!/bin/sh' 'echo "openshell 0.0.1"' > "${tmpBin}/openshell" + chmod +x "${tmpBin}/openshell" export PATH="${tmpBin}:/usr/bin:/bin" curl() { echo "CURL_FALLBACK $*" @@ -782,10 +816,16 @@ describe("regression guards", () => { export -f curl sha256sum() { echo "SHA256SUM $*" >> ${JSON.stringify(checksumLog)}; echo "checksum OK"; return 0; } export -f sha256sum - strings() { echo "request-body-credential-rewrite websocket-credential-rewrite"; } + strings() { echo "request-body-credential-rewrite websocket-credential-rewrite allow_all_known_mcp_methods"; } export -f strings - tar() { return 0; }; export -f tar - install() { return 0; }; export -f install + tar() { + local destination="\${@: -1}" + printf '%s\n' '#!/bin/sh' 'echo "openshell 0.0.72"' > "$destination/openshell" + printf '%s\n' '#!/bin/sh' 'echo "openshell-gateway 0.0.72"' > "$destination/openshell-gateway" + printf '%s\n' '#!/bin/sh' 'echo "openshell-sandbox 0.0.72"' > "$destination/openshell-sandbox" + chmod +x "$destination/openshell" "$destination/openshell-gateway" "$destination/openshell-sandbox" + }; export -f tar + install() { /usr/bin/install "$@"; }; export -f install source "${scriptPath}" `; try { @@ -827,7 +867,11 @@ describe("regression guards", () => { [path.join(import.meta.dirname, "..", script), "--version"], { encoding: "utf-8", - env: { ...process.env, HOME: tmp, PATH: `${fakeBin}:/usr/bin:/bin` }, + env: { + ...process.env, + HOME: tmp, + PATH: `${fakeBin}:/usr/bin:/bin`, + }, timeout: 15000, }, ); diff --git a/test/sandbox-build-context.test.ts b/test/sandbox-build-context.test.ts index b09b16bdec0..dbe8eb33b98 100644 --- a/test/sandbox-build-context.test.ts +++ b/test/sandbox-build-context.test.ts @@ -31,6 +31,8 @@ describe("sandbox build context staging", () => { writeFixture("Dockerfile"); writeFixture("tsconfig.runtime-preloads.json", "{}\n"); + writeFixture(path.join("agents", "openclaw", "mcporter-runtime", "package.json"), "{}\n"); + writeFixture(path.join("agents", "openclaw", "mcporter-runtime", "package-lock.json"), "{}\n"); for (const fileName of [ "package.json", "package-lock.json", @@ -143,6 +145,17 @@ describe("sandbox build context staging", () => { expect((fs.statSync(stagedPlugin).mode & 0o777).toString(8)).toBe("644"); } + function expectStagedMcporterRuntime(buildCtx: string) { + const runtimeDir = path.join(buildCtx, "agents", "openclaw", "mcporter-runtime"); + expect(fs.readdirSync(runtimeDir).sort()).toEqual(["package-lock.json", "package.json"]); + expect((fs.statSync(path.join(runtimeDir, "package.json")).mode & 0o777).toString(8)).toBe( + "644", + ); + expect((fs.statSync(path.join(runtimeDir, "package-lock.json")).mode & 0o777).toString(8)).toBe( + "644", + ); + } + it("normalizes copied blueprint modes with chmod a+rX semantics", () => { const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-build-context-unit-")); const blueprintDir = path.join(tmpDir, "nemoclaw-blueprint"); @@ -183,6 +196,7 @@ describe("sandbox build context staging", () => { writeBuildContextFixture(sourceRoot); const { buildCtx } = stageOptimizedSandboxBuildContext(sourceRoot, tmpDir); expectStagedBlueprintModes(buildCtx); + expectStagedMcporterRuntime(buildCtx); } finally { fs.rmSync(sourceRoot, { recursive: true, force: true }); fs.rmSync(tmpDir, { recursive: true, force: true }); @@ -211,6 +225,7 @@ describe("sandbox build context staging", () => { writeBuildContextFixture(sourceRoot); const { buildCtx } = stageLegacySandboxBuildContext(sourceRoot, tmpDir); expectStagedBlueprintModes(buildCtx); + expectStagedMcporterRuntime(buildCtx); } finally { fs.rmSync(sourceRoot, { recursive: true, force: true }); fs.rmSync(tmpDir, { recursive: true, force: true }); @@ -241,6 +256,7 @@ describe("sandbox build context staging", () => { const { buildCtx, stagedDockerfile } = stageOptimizedSandboxBuildContext(repoRoot, tmpDir); expectDockerfileScriptCopiesExist(buildCtx, stagedDockerfile); expect(fs.existsSync(path.join(buildCtx, "tsconfig.runtime-preloads.json"))).toBe(true); + expectStagedMcporterRuntime(buildCtx); expect(fs.existsSync(path.join(buildCtx, "nemoclaw-blueprint", ".venv"))).toBe(false); expect(fs.existsSync(path.join(buildCtx, "nemoclaw-blueprint", "blueprint.yaml"))).toBe(true); expect( diff --git a/test/sandbox-connect-inference/auto-pair-approval.test.ts b/test/sandbox-connect-inference/auto-pair-approval.test.ts index 07e19f7d5ec..f43d255a66f 100644 --- a/test/sandbox-connect-inference/auto-pair-approval.test.ts +++ b/test/sandbox-connect-inference/auto-pair-approval.test.ts @@ -225,11 +225,11 @@ describe("sandbox connect auto-pair approval pass (#4263)", () => { ); // Force the approval-pass sandbox-exec to fail with exit status 7 - // (simulated via the NEMOCLAW_TEST_FAIL_APPROVAL_PASS hook in the + // (simulated via the OPENSHELL_TEST_FAIL_APPROVAL_PASS hook in the // fake openshell). The connect flow must still reach SSH handoff — // the approval pass is best-effort and must not surface failures. const result = runConnect(tmpDir, sandboxName, { - NEMOCLAW_TEST_FAIL_APPROVAL_PASS: "1", + OPENSHELL_TEST_FAIL_APPROVAL_PASS: "1", }); expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); const state = JSON.parse(fs.readFileSync(stateFile, "utf-8")); @@ -286,7 +286,7 @@ describe("sandbox connect scope-upgrade approval on recover/probe (#4504)", () = expect(controlExec?.slice(userIndex, userIndex + 5)).toEqual([ "--user", "root", - `openshell-${sandboxName}-fixture`, + "sandbox-container-id", "/usr/local/bin/nemoclaw-gateway-control", "recover", ]); @@ -325,7 +325,7 @@ describe("sandbox connect scope-upgrade approval on recover/probe (#4504)", () = { gatewaySupervisorRecovery: true }, ); - const result = runConnect(tmpDir, sandboxName, { NEMOCLAW_TEST_FAIL_APPROVAL_PASS: "1" }, [ + const result = runConnect(tmpDir, sandboxName, { OPENSHELL_TEST_FAIL_APPROVAL_PASS: "1" }, [ "--probe-only", ]); expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); @@ -356,7 +356,7 @@ describe("sandbox connect scope-upgrade approval on recover/probe (#4504)", () = "claude-sonnet-4-20250514", ); - const result = runConnect(tmpDir, sandboxName, { NEMOCLAW_TEST_GATEWAY_DOWN: "1" }, [ + const result = runConnect(tmpDir, sandboxName, { OPENSHELL_TEST_GATEWAY_DOWN: "1" }, [ "--probe-only", ]); expect(result.status).toBe(1); diff --git a/test/sandbox-connect-inference/helpers.ts b/test/sandbox-connect-inference/helpers.ts index fdee51ac555..c9533feaa86 100644 --- a/test/sandbox-connect-inference/helpers.ts +++ b/test/sandbox-connect-inference/helpers.ts @@ -242,7 +242,7 @@ if (args[0] === "sandbox" && args[1] === "exec") { } } if ( - process.env.NEMOCLAW_TEST_FAIL_APPROVAL_PASS === "1" && + process.env.OPENSHELL_TEST_FAIL_APPROVAL_PASS === "1" && approvalCmd.includes("openclaw") && approvalCmd.includes("devices") && approvalCmd.includes("approve") @@ -254,7 +254,7 @@ if (args[0] === "sandbox" && args[1] === "exec") { // STOPPED so the probe path takes the not-running branch and (when recovery // also fails) the probe-failure exit — where the approval sweep must NOT run. if ( - process.env.NEMOCLAW_TEST_GATEWAY_DOWN === "1" && + process.env.OPENSHELL_TEST_GATEWAY_DOWN === "1" && command.includes("/health") && command.includes("HTTP_CODE") ) { @@ -332,11 +332,23 @@ const sanitizedPrefix = index % 2 === 0 ? value === "--env" : /^[A-Z0-9_]+=.*$/.test(value) ); -if (args[0] === "ps") { +const isDirectSandboxDiscovery = + args[0] === "ps" && + args.includes("--no-trunc") && + args.includes("label=openshell.ai/managed-by=openshell") && + args.includes("label=openshell.ai/sandbox-name=${sandboxName}") && + args.includes("{{.ID}}\\t{{.Names}}"); + +if (isDirectSandboxDiscovery) { const directContainer = state.gatewaySupervisorRecovery - ? "openshell-${sandboxName}-fixture\\n" + ? "sandbox-container-id\\topenshell-${sandboxName}-fixture\\n" : ""; - process.stdout.write("openshell-cluster-nemoclaw\\n" + directContainer); + process.stdout.write(directContainer); + process.exit(0); +} + +if (args[0] === "ps") { + process.stdout.write("openshell-cluster-nemoclaw\\n"); process.exit(0); } @@ -348,7 +360,7 @@ if ( args.includes("PYTHONNOUSERSITE=1") && args.length === userIndex + 6 && args[userIndex + 1] === "root" && - args[userIndex + 2] === "openshell-${sandboxName}-fixture" && + args[userIndex + 2] === "sandbox-container-id" && args[userIndex + 3] === "/usr/local/bin/nemoclaw-gateway-control" && args[userIndex + 4] === "recover" ) { diff --git a/test/sandbox-provisioning.test.ts b/test/sandbox-provisioning.test.ts index 72ffb84c7c0..274eb639739 100644 --- a/test/sandbox-provisioning.test.ts +++ b/test/sandbox-provisioning.test.ts @@ -1188,6 +1188,11 @@ describe("Hermes sandbox provisioning", () => { const bashrcPath = path.join(etcDir, "bash.bashrc"); const gatewayControlPath = path.join(localBin, "nemoclaw-gateway-control"); const gatewaySupervisorPath = path.join(localLib, "gateway-supervisor.sh"); + const mcpConfigTransactionPath = path.join(localLib, "hermes-mcp-config-transaction.py"); + const mcpCredentialBoundaryPath = path.join( + localLib, + "openshell-child-visible-credentials.v0.0.72.json", + ); const stateDirGuardPath = path.join(localLib, "state-dir-guard.py"); const managedGatewayControlPath = path.join(localLib, "managed-gateway-control.py"); const files = [ @@ -1197,6 +1202,8 @@ describe("Hermes sandbox provisioning", () => { path.join(localLib, "validate-hermes-env-secret-boundary.py"), path.join(localLib, "seed-hermes-dashboard-config.py"), path.join(localLib, "hermes-runtime-config-guard.py"), + mcpConfigTransactionPath, + mcpCredentialBoundaryPath, gatewaySupervisorPath, stateDirGuardPath, managedGatewayControlPath, @@ -1225,9 +1232,11 @@ describe("Hermes sandbox provisioning", () => { expect(result.status, result.stderr).toBe(0); expect(calls).toContain( - `chown root:root ${gatewayControlPath} ${gatewaySupervisorPath} ${stateDirGuardPath} ${managedGatewayControlPath}`, + `chown root:root ${gatewayControlPath} ${gatewaySupervisorPath} ${stateDirGuardPath} ${managedGatewayControlPath} ${mcpCredentialBoundaryPath}`, ); expect((fs.statSync(gatewayControlPath).mode & 0o777).toString(8)).toBe("700"); + expect((fs.statSync(mcpConfigTransactionPath).mode & 0o777).toString(8)).toBe("755"); + expect((fs.statSync(mcpCredentialBoundaryPath).mode & 0o777).toString(8)).toBe("444"); expect((fs.statSync(gatewaySupervisorPath).mode & 0o777).toString(8)).toBe("444"); expect((fs.statSync(stateDirGuardPath).mode & 0o777).toString(8)).toBe("500"); expect((fs.statSync(managedGatewayControlPath).mode & 0o777).toString(8)).toBe("500"); @@ -1357,6 +1366,8 @@ describe("Hermes sandbox provisioning", () => { "web", "--extra", "pty", + "--extra", + "mcp", ]); } finally { fs.rmSync(tmp, { recursive: true, force: true }); diff --git a/test/sandbox-rlimit-hooks.test.ts b/test/sandbox-rlimit-hooks.test.ts index c61b65615b7..83f2073ca15 100644 --- a/test/sandbox-rlimit-hooks.test.ts +++ b/test/sandbox-rlimit-hooks.test.ts @@ -408,6 +408,14 @@ describe("sandbox rlimit system hooks (#2173)", () => { const validator = path.join(localLib, "validate-hermes-env-secret-boundary.py"); const dashboardSeeder = path.join(localLib, "seed-hermes-dashboard-config.py"); const runtimeGuard = path.join(localLib, "hermes-runtime-config-guard.py"); + const mcpTransaction = path.join(localLib, "hermes-mcp-config-transaction.py"); + const mcpCredentialBoundary = path.join( + localLib, + "openshell-child-visible-credentials.v0.0.72.json", + ); + const preloadDir = path.join(localLib, "preloads"); + const safetyNet = path.join(preloadDir, "sandbox-safety-net.js"); + const ciaoGuard = path.join(preloadDir, "ciao-network-guard.js"); const gatewaySupervisor = path.join(localLib, "gateway-supervisor.sh"); const stateDirGuard = path.join(localLib, "state-dir-guard.py"); const managedGatewayControl = path.join(localLib, "managed-gateway-control.py"); @@ -424,12 +432,21 @@ describe("sandbox rlimit system hooks (#2173)", () => { fs.writeFileSync(validator, "# validator fixture\n"); fs.writeFileSync(dashboardSeeder, "# dashboard seeder fixture\n"); fs.writeFileSync(runtimeGuard, "# runtime guard fixture\n"); + fs.writeFileSync(mcpTransaction, "# MCP transaction fixture\n"); + fs.writeFileSync(mcpCredentialBoundary, "{}\n"); + fs.mkdirSync(preloadDir, { mode: 0o777 }); + fs.writeFileSync(safetyNet, "module.exports = 'safety net fixture';\n", { mode: 0o666 }); + fs.writeFileSync(ciaoGuard, "module.exports = 'ciao guard fixture';\n", { mode: 0o666 }); + fs.chmodSync(preloadDir, 0o777); + fs.chmodSync(safetyNet, 0o666); + fs.chmodSync(ciaoGuard, 0o666); fs.writeFileSync(gatewaySupervisor, "# gateway supervisor fixture\n"); fs.writeFileSync(stateDirGuard, "# state-dir guard fixture\n"); fs.writeFileSync(managedGatewayControl, "# managed gateway control fixture\n"); fs.writeFileSync(startBin, "#!/usr/bin/env bash\n"); fs.writeFileSync(gatewayControl, "#!/usr/bin/env sh\n"); fs.writeFileSync(bashrc, "# stale hermes bashrc\n"); + const fixtureOwner = fs.statSync(startBin); const replay = dockerRunCommandBetween( dockerfile, "# Copy startup script and the secret-boundary validator.", @@ -442,6 +459,14 @@ describe("sandbox rlimit system hooks (#2173)", () => { .replaceAll("/usr/local/lib/nemoclaw/validate-hermes-env-secret-boundary.py", validator) .replaceAll("/usr/local/lib/nemoclaw/seed-hermes-dashboard-config.py", dashboardSeeder) .replaceAll("/usr/local/lib/nemoclaw/hermes-runtime-config-guard.py", runtimeGuard) + .replaceAll("/usr/local/lib/nemoclaw/hermes-mcp-config-transaction.py", mcpTransaction) + .replaceAll( + "/usr/local/lib/nemoclaw/openshell-child-visible-credentials.v0.0.72.json", + mcpCredentialBoundary, + ) + .replaceAll("/usr/local/lib/nemoclaw/preloads/sandbox-safety-net.js", safetyNet) + .replaceAll("/usr/local/lib/nemoclaw/preloads/ciao-network-guard.js", ciaoGuard) + .replaceAll("/usr/local/lib/nemoclaw/preloads", preloadDir) .replaceAll("/usr/local/lib/nemoclaw/state-dir-guard.py", stateDirGuard) .replaceAll("/usr/local/lib/nemoclaw/managed-gateway-control.py", managedGatewayControl) .replaceAll("/usr/local/lib/nemoclaw/sandbox-rlimits.sh", rlimitLib) @@ -459,6 +484,19 @@ describe("sandbox rlimit system hooks (#2173)", () => { expectSystemRlimitHookEnforcesLimits(profileHook); expectSystemRlimitHookEnforcesLimits(bashrc); expectSystemRlimitHookIsSilentWhenVerificationFails(bashrc, rlimitLib); + const hardenedDir = fs.statSync(preloadDir); + const hardenedSafetyNet = fs.statSync(safetyNet); + const hardenedCiaoGuard = fs.statSync(ciaoGuard); + expect(hardenedDir.mode & 0o777).toBe(0o755); + expect(hardenedSafetyNet.mode & 0o777).toBe(0o444); + expect(hardenedCiaoGuard.mode & 0o777).toBe(0o444); + expect(fs.statSync(mcpCredentialBoundary).mode & 0o777).toBe(0o444); + expect(hardenedDir.uid).toBe(fixtureOwner.uid); + expect(hardenedDir.gid).toBe(fixtureOwner.gid); + expect(hardenedSafetyNet.uid).toBe(fixtureOwner.uid); + expect(hardenedSafetyNet.gid).toBe(fixtureOwner.gid); + expect(hardenedCiaoGuard.uid).toBe(fixtureOwner.uid); + expect(hardenedCiaoGuard.gid).toBe(fixtureOwner.gid); } finally { fs.rmSync(tmp, { recursive: true, force: true }); } diff --git a/test/tavily-preset.test.ts b/test/tavily-preset.test.ts index 4e6fa01dc71..53686f9b4a6 100644 --- a/test/tavily-preset.test.ts +++ b/test/tavily-preset.test.ts @@ -54,6 +54,14 @@ describe("tavily opt-in preset", () => { { path: "/usr/local/bin/curl" }, { path: "/usr/bin/curl" }, ]); + expect(policy?.binaries).not.toEqual( + expect.arrayContaining([ + { path: "/usr/bin/python3*" }, + { path: "/usr/local/bin/python3*" }, + { path: "/sandbox/**/bin/python3*" }, + ]), + ); + expect(policy).not.toHaveProperty("access", "full"); expect(policy?.endpoints?.[0]).not.toHaveProperty("access"); expect(policy?.endpoints?.[0]).not.toHaveProperty("tls", "skip"); }); diff --git a/test/update-hermes-agent-script.test.ts b/test/update-hermes-agent-script.test.ts index c98bb1d27e3..8adc842dd76 100644 --- a/test/update-hermes-agent-script.test.ts +++ b/test/update-hermes-agent-script.test.ts @@ -8,6 +8,14 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; const SCRIPT = path.join(import.meta.dirname, "..", "scripts", "update-hermes-agent.sh"); +const HERMES_BASE_DOCKERFILE = path.join( + import.meta.dirname, + "..", + "agents", + "hermes", + "Dockerfile.base", +); +const HERMES_MANIFEST = path.join(import.meta.dirname, "..", "agents", "hermes", "manifest.yaml"); const TARGET_TAG = "v2026.6.19"; const CURRENT_INSTALLED_BASE = [ @@ -22,6 +30,8 @@ const CURRENT_INSTALLED_BASE = [ const CURRENT_INSTALLED_DOCKERFILE = [ "COPY agents/hermes/validate-hermes-env-secret-boundary.py /usr/local/lib/nemoclaw/validate-hermes-env-secret-boundary.py", "COPY agents/hermes/seed-dashboard-config.py /usr/local/lib/nemoclaw/seed-hermes-dashboard-config.py", + "COPY agents/hermes/mcp-config-transaction.py /usr/local/lib/nemoclaw/hermes-mcp-config-transaction.py", + "COPY src/lib/actions/sandbox/openshell-child-visible-credentials.v0.0.72.json /usr/local/lib/nemoclaw/openshell-child-visible-credentials.v0.0.72.json", "RUN HERMES_HOME=/sandbox/.hermes /usr/local/bin/hermes doctor --fix \\", " && node --experimental-strip-types /opt/nemoclaw-hermes-config/generate-config.ts", "RUN mkdir -p /sandbox/.hermes/dashboard-home", @@ -37,7 +47,93 @@ function writeInstalledHermesCopy(baseDockerfile: string, baseText = CURRENT_INS ); } +function writeExecutable(file: string, body: string) { + fs.writeFileSync(file, body, { mode: 0o755 }); +} + describe("scripts/update-hermes-agent.sh", () => { + it("pins rebuild overrides to the accepted full image-ID local tag family", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-hermes-update-rebuild-")); + const repo = path.join(tmp, "repo"); + const script = path.join(repo, "scripts", "update-hermes-agent.sh"); + const fakeBin = path.join(tmp, "bin"); + const dockerLog = path.join(tmp, "docker.log"); + const nemohermesLog = path.join(tmp, "nemohermes.log"); + const imageId = `sha256:${"a".repeat(64)}`; + const pinnedRef = `nemoclaw-hermes-sandbox-base-local:image-${"a".repeat(64)}`; + const baseRef = "nemoclaw-hermes-base-local:test"; + fs.mkdirSync(path.dirname(script), { recursive: true }); + fs.mkdirSync(path.join(repo, "agents", "hermes"), { recursive: true }); + fs.mkdirSync(fakeBin, { recursive: true }); + fs.copyFileSync(SCRIPT, script); + fs.chmodSync(script, 0o755); + fs.copyFileSync(HERMES_BASE_DOCKERFILE, path.join(repo, "agents", "hermes", "Dockerfile.base")); + fs.copyFileSync(HERMES_MANIFEST, path.join(repo, "agents", "hermes", "manifest.yaml")); + writeExecutable( + path.join(fakeBin, "curl"), + `#!/usr/bin/env bash +set -euo pipefail +output="" +previous="" +for arg in "$@"; do + case "$previous" in + -o) output="$arg" ;; + esac + previous="$arg" +done +printf 'fake archive' > "$output" +`, + ); + writeExecutable( + path.join(fakeBin, "tar"), + "#!/usr/bin/env bash\nprintf 'version = \"0.17.0\"\\n'\n", + ); + writeExecutable(path.join(fakeBin, "npm"), "#!/usr/bin/env bash\nprintf 'sha512-test\\n'\n"); + writeExecutable( + path.join(fakeBin, "docker"), + `#!/usr/bin/env bash +set -euo pipefail +printf '%s\\n' "$*" >> "$FAKE_DOCKER_LOG" +case "\${1:-}" in + image) printf '%s\\n' ${JSON.stringify(imageId)} ;; +esac +`, + ); + writeExecutable( + path.join(fakeBin, "nemohermes"), + `#!/usr/bin/env bash +set -euo pipefail +printf '%s|%s\\n' "\${NEMOCLAW_HERMES_SANDBOX_BASE_IMAGE_REF:-}" "$*" >> "$FAKE_NEMOHERMES_LOG" +if [[ "$*" == "hermes exec -- hermes --version" ]]; then + printf '0.17.0\\n' +fi +`, + ); + + try { + const run = spawnSync("bash", [script, "--tag", TARGET_TAG, "--rebuild"], { + encoding: "utf8", + env: { + ...process.env, + PATH: `${fakeBin}:${process.env.PATH}`, + HOME: path.join(tmp, "home"), + HERMES_BASE_REF: baseRef, + FAKE_DOCKER_LOG: dockerLog, + FAKE_NEMOHERMES_LOG: nemohermesLog, + NEMOCLAW_SOURCE_ROOT: undefined, + }, + timeout: 10_000, + }); + + expect(run.status, `${run.stdout}\n${run.stderr}`).toBe(0); + expect(fs.readFileSync(dockerLog, "utf8")).toContain(`tag ${baseRef} ${pinnedRef}`); + expect(fs.readFileSync(nemohermesLog, "utf8")).toContain(`${pinnedRef}|hermes rebuild`); + expect(run.stdout).toContain("OK: sandbox reports Hermes Agent v0.17.0"); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + it("keeps installed-copy scanning opt-in unless rebuild needs it", () => { const tmpHome = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-hermes-update-home-")); const installedDockerfile = path.join( @@ -164,4 +260,49 @@ describe("scripts/update-hermes-agent.sh", () => { fs.rmSync(tmpHome, { recursive: true, force: true }); } }); + + it("refuses installed copies that predate the transactional MCP boundary", () => { + const tmpHome = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-hermes-update-pre-mcp-")); + const installedDockerfile = path.join( + tmpHome, + ".nemoclaw", + "source", + "agents", + "hermes", + "Dockerfile.base", + ); + const installedAgentDockerfile = path.join(path.dirname(installedDockerfile), "Dockerfile"); + const preMcpDockerfile = CURRENT_INSTALLED_DOCKERFILE.replace( + /^COPY (?:agents\/hermes\/mcp-config-transaction\.py|src\/lib\/actions\/sandbox\/openshell-child-visible-credentials\.v0\.0\.72\.json) .*\n/gm, + "", + ); + fs.mkdirSync(path.dirname(installedDockerfile), { recursive: true }); + fs.writeFileSync(installedDockerfile, CURRENT_INSTALLED_BASE); + fs.writeFileSync(installedAgentDockerfile, preMcpDockerfile); + + const run = spawnSync( + "bash", + [SCRIPT, "--tag", TARGET_TAG, "--check", "--update-installed-copies"], + { + encoding: "utf-8", + env: { + ...process.env, + HOME: tmpHome, + NEMOCLAW_SOURCE_ROOT: undefined, + }, + timeout: 5000, + }, + ); + + try { + expect(run.status).toBe(1); + expect(run.stdout).toContain("INVALID: installed copy"); + expect(run.stdout).toContain("marker hermes-mcp-config-transaction.py"); + expect(run.stdout).toContain("marker openshell-child-visible-credentials.v0.0.72.json"); + expect(fs.readFileSync(installedDockerfile, "utf-8")).toBe(CURRENT_INSTALLED_BASE); + expect(fs.readFileSync(installedAgentDockerfile, "utf-8")).toBe(preMcpDockerfile); + } finally { + fs.rmSync(tmpHome, { recursive: true, force: true }); + } + }); }); diff --git a/test/validate-config-schemas.test.ts b/test/validate-config-schemas.test.ts index 372d130a612..24dd6203951 100644 --- a/test/validate-config-schemas.test.ts +++ b/test/validate-config-schemas.test.ts @@ -10,10 +10,10 @@ */ import { existsSync, readFileSync } from "node:fs"; -import { join, dirname } from "node:path"; +import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; -import { describe, it, expect } from "vitest"; import Ajv, { type ValidateFunction } from "ajv/dist/2020.js"; +import { describe, expect, it } from "vitest"; import YAML from "yaml"; import { discoverTargets } from "../scripts/validate-configs"; @@ -377,6 +377,172 @@ describe("sandbox-policy.schema.json", () => { expectValid(validate, valid, "rest body rewrite policy"); }); + it("accepts sandbox-policy JSON-RPC and MCP endpoints with explicit L7 matchers", () => { + const valid = { + version: 1, + network_policies: { + mcp_bridge: { + name: "MCP Bridge", + binaries: [{ path: "/usr/local/bin/mcporter" }], + endpoints: [ + { + host: "host.openshell.internal", + port: 31337, + protocol: "json-rpc", + enforcement: "enforce", + json_rpc: { max_body_bytes: 131072 }, + rules: [{ allow: { method: "tools/list", path: "/mcp" } }], + }, + { + host: "host.openshell.internal", + port: 31337, + protocol: "mcp", + enforcement: "enforce", + mcp: { max_body_bytes: 131072, strict_tool_names: true }, + rules: [ + { + allow: { + method: "tools/call", + path: "/mcp", + tool: { any: ["search", "read"] }, + params: { query: { any: ["safe", "readonly"] } }, + }, + }, + ], + deny_rules: [{ tool: "admin" }], + }, + ], + }, + }, + }; + expectValid(validate, valid, "json-rpc and mcp policy"); + }); + + it("rejects sandbox-policy MCP endpoints without rules or explicit MCP allow-all", () => { + const bad = { + version: 1, + network_policies: { + mcp_bridge: { + name: "MCP Bridge", + binaries: [{ path: "/usr/local/bin/mcporter" }], + endpoints: [ + { + host: "host.openshell.internal", + port: 31337, + protocol: "mcp", + mcp: { max_body_bytes: 131072 }, + }, + ], + }, + }, + }; + expect(validate(bad)).toBe(false); + }); + + it("accepts sandbox-policy MCP endpoint allow-all without REST access presets", () => { + const valid = { + version: 1, + network_policies: { + mcp_bridge: { + name: "MCP Bridge", + binaries: [{ path: "/usr/local/bin/mcporter" }], + endpoints: [ + { + host: "host.openshell.internal", + port: 31337, + protocol: "mcp", + mcp: { max_body_bytes: 131072, allow_all_known_mcp_methods: true }, + }, + ], + }, + }, + }; + expectValid(validate, valid, "mcp policy allow-all"); + }); + + it("rejects sandbox-policy JSON-RPC and MCP endpoints above the body-size cap", () => { + const oversizedJsonRpc = { + version: 1, + network_policies: { + rpc: { + name: "RPC", + binaries: [{ path: "/usr/local/bin/tool" }], + endpoints: [ + { + host: "mcp.example.com", + port: 443, + protocol: "json-rpc", + json_rpc: { max_body_bytes: 1048577 }, + rules: [{ allow: { method: "initialize" } }], + }, + ], + }, + }, + }; + expect(validate(oversizedJsonRpc)).toBe(false); + + const oversizedMcp = { + version: 1, + network_policies: { + mcp_bridge: { + name: "MCP Bridge", + binaries: [{ path: "/usr/local/bin/mcporter" }], + endpoints: [ + { + host: "mcp.example.com", + port: 443, + protocol: "mcp", + mcp: { max_body_bytes: 1048577, allow_all_known_mcp_methods: true }, + }, + ], + }, + }, + }; + expect(validate(oversizedMcp)).toBe(false); + }); + + it("rejects sandbox-policy JSON-RPC and MCP endpoints with REST access presets", () => { + const base = { + version: 1, + network_policies: { + rpc: { + name: "RPC", + binaries: [{ path: "/usr/local/bin/mcporter" }], + endpoints: [ + { + host: "rpc.example.com", + port: 443, + protocol: "json-rpc", + access: "full", + rules: [{ allow: { method: "initialize" } }], + }, + ], + }, + }, + }; + expect(validate(base)).toBe(false); + + const mcp = { + version: 1, + network_policies: { + rpc: { + name: "RPC", + binaries: [{ path: "/usr/local/bin/mcporter" }], + endpoints: [ + { + host: "mcp.example.com", + port: 443, + protocol: "mcp", + access: "full", + mcp: { allow_all_known_mcp_methods: true }, + }, + ], + }, + }, + }; + expect(validate(mcp)).toBe(false); + }); + it("rejects sandbox-policy endpoint with protocol websocket but no rules or access", () => { const bad = { version: 1, @@ -498,6 +664,182 @@ describe("policy-preset.schema.json", () => { expectValid(validate, valid, "rest body rewrite preset"); }); + it("accepts preset JSON-RPC and MCP endpoints with focused option objects", () => { + const valid = { + preset: { name: "mcp", description: "MCP" }, + network_policies: { + mcp_bridge: { + name: "MCP Bridge", + binaries: [{ path: "/usr/local/bin/mcporter" }], + endpoints: [ + { + host: "mcp.example.com", + port: 443, + protocol: "json-rpc", + json_rpc: { max_body_bytes: 131072 }, + rules: [{ allow: { method: "initialize", path: "/mcp" } }], + }, + { + host: "mcp.example.com", + port: 443, + protocol: "mcp", + mcp: { max_body_bytes: 131072, allow_all_known_mcp_methods: false }, + rules: [{ allow: { method: "tools/call", path: "/mcp", tool: "search" } }], + deny_rules: [{ params: { mode: "admin" } }], + }, + ], + }, + }, + }; + expectValid(validate, valid, "json-rpc and mcp preset"); + }); + + it("rejects preset MCP endpoints with missing rules, invalid options, or invalid matchers", () => { + const base = { + preset: { name: "mcp", description: "MCP" }, + network_policies: { + mcp_bridge: { + name: "MCP Bridge", + binaries: [{ path: "/usr/local/bin/mcporter" }], + endpoints: [ + { + host: "mcp.example.com", + port: 443, + protocol: "mcp", + mcp: { max_body_bytes: 131072 }, + rules: [{ allow: { method: "tools/list", path: "/mcp" } }], + }, + ], + }, + }, + }; + type McpPresetFixture = { + network_policies: { + mcp_bridge: { + endpoints: Array<{ + rules?: unknown[]; + deny_rules?: unknown[]; + mcp: { allow_all_known_mcp_methods?: unknown }; + }>; + }; + }; + }; + const missingRules = cloneObject(base) as McpPresetFixture; + delete missingRules.network_policies.mcp_bridge.endpoints[0]!.rules; + expect(validate(missingRules)).toBe(false); + + const invalidOptions = cloneObject(base) as McpPresetFixture; + invalidOptions.network_policies.mcp_bridge.endpoints[0]!.mcp.allow_all_known_mcp_methods = + "yes"; + expect(validate(invalidOptions)).toBe(false); + + const invalidMatcher = cloneObject(base) as McpPresetFixture; + invalidMatcher.network_policies.mcp_bridge.endpoints[0]!.deny_rules = [{ tool: { any: [] } }]; + expect(validate(invalidMatcher)).toBe(false); + }); + + it("accepts preset MCP allow-all and rejects JSON-RPC or MCP access presets", () => { + const allowAll = { + preset: { name: "mcp", description: "MCP" }, + network_policies: { + mcp_bridge: { + name: "MCP Bridge", + binaries: [{ path: "/usr/local/bin/mcporter" }], + endpoints: [ + { + host: "mcp.example.com", + port: 443, + protocol: "mcp", + mcp: { allow_all_known_mcp_methods: true }, + }, + ], + }, + }, + }; + expectValid(validate, allowAll, "mcp preset allow-all"); + + const jsonRpcAccess = { + preset: { name: "mcp", description: "MCP" }, + network_policies: { + mcp_bridge: { + name: "MCP Bridge", + binaries: [{ path: "/usr/local/bin/mcporter" }], + endpoints: [ + { + host: "rpc.example.com", + port: 443, + protocol: "json-rpc", + access: "full", + rules: [{ allow: { method: "initialize" } }], + }, + ], + }, + }, + }; + expect(validate(jsonRpcAccess)).toBe(false); + + const mcpAccess = { + preset: { name: "mcp", description: "MCP" }, + network_policies: { + mcp_bridge: { + name: "MCP Bridge", + binaries: [{ path: "/usr/local/bin/mcporter" }], + endpoints: [ + { + host: "mcp.example.com", + port: 443, + protocol: "mcp", + access: "full", + mcp: { allow_all_known_mcp_methods: true }, + }, + ], + }, + }, + }; + expect(validate(mcpAccess)).toBe(false); + }); + + it("rejects preset JSON-RPC and MCP endpoints above the body-size cap", () => { + const oversizedJsonRpc = { + preset: { name: "rpc", description: "RPC" }, + network_policies: { + rpc: { + name: "RPC", + binaries: [{ path: "/usr/local/bin/tool" }], + endpoints: [ + { + host: "mcp.example.com", + port: 443, + protocol: "json-rpc", + json_rpc: { max_body_bytes: 1048577 }, + rules: [{ allow: { method: "initialize" } }], + }, + ], + }, + }, + }; + expect(validate(oversizedJsonRpc)).toBe(false); + + const oversizedMcp = { + preset: { name: "mcp", description: "MCP" }, + network_policies: { + mcp_bridge: { + name: "MCP Bridge", + binaries: [{ path: "/usr/local/bin/mcporter" }], + endpoints: [ + { + host: "mcp.example.com", + port: 443, + protocol: "mcp", + mcp: { max_body_bytes: 1048577, allow_all_known_mcp_methods: true }, + }, + ], + }, + }, + }; + expect(validate(oversizedMcp)).toBe(false); + }); + it("rejects preset endpoint with protocol websocket but no rules", () => { const bad = { preset: { name: "test", description: "test" }, diff --git a/test/vm-driver-privileged-exec-routing.test.ts b/test/vm-driver-privileged-exec-routing.test.ts index 6a4b4aaffdf..d721e80326f 100644 --- a/test/vm-driver-privileged-exec-routing.test.ts +++ b/test/vm-driver-privileged-exec-routing.test.ts @@ -65,8 +65,8 @@ function writeRegistry( ); } -function writeDockerPs(psFile: string, names: string[]): void { - fs.writeFileSync(psFile, `${names.join("\n")}\n`); +function writeDockerPs(psFile: string, rows: Array<[string, string]>): void { + fs.writeFileSync(psFile, `${rows.map((row) => row.join("\t")).join("\n")}\n`); } function assertDirect(args: string[], expectedContainer: string, label: string): void { @@ -128,37 +128,28 @@ describe("VM/Docker privileged-exec routing regression (#4245)", () => { const helper = loadHelperWithFakeHome(fakeHome, fakeBin, dockerPsFile, dockerLog); const cmd = ["stat", "-c", "%a", "/sandbox/.openclaw/openclaw.json"]; - writeDockerPs(dockerPsFile, [ - "openshell-gateway-nemoclaw", - "openshell-alpha-child", - "openshell-alpha-child-2026", - "openshell-alpha-abc123", - "openshell-dockerbox-987", - "openshell-unknown-driver", - ]); - - assertDirect( - helper.privilegedSandboxExecArgv("alpha", cmd), - "openshell-alpha-abc123", - "VM driver with prefix collision", - ); + writeDockerPs(dockerPsFile, [["alpha-id", "openshell-alpha-abc123"]]); + assertDirect(helper.privilegedSandboxExecArgv("alpha", cmd), "alpha-id", "VM driver"); + writeDockerPs(dockerPsFile, [["alpha-child-id", "openshell-alpha-child-2026"]]); assertDirect( helper.privilegedSandboxExecArgv("alpha-child", cmd), - "openshell-alpha-child", - "VM driver with exact container", + "alpha-child-id", + "VM driver child", ); + writeDockerPs(dockerPsFile, [["dockerbox-id", "openshell-dockerbox-987"]]); assertDirect( helper.privilegedSandboxExecArgv("dockerbox", cmd), - "openshell-dockerbox-987", + "dockerbox-id", "Docker driver", ); + writeDockerPs(dockerPsFile, [["unknown-id", "openshell-unknown-driver"]]); assertDirect( helper.privilegedSandboxExecArgv("unknown-driver", cmd), - "openshell-unknown-driver", + "unknown-id", "registry entry without a recorded driver", ); - writeDockerPs(dockerPsFile, ["openshell-gateway-nemoclaw", "openshell-other"]); + writeDockerPs(dockerPsFile, []); expect(() => helper.privilegedSandboxExecArgv("alpha", ["id"])).toThrow( /No running direct OpenShell sandbox container found for 'alpha'.*driver: vm/, ); diff --git a/tools/e2e-advisor/targets.mts b/tools/e2e-advisor/targets.mts index 69d2e56c9f7..08fbeb3e0e7 100755 --- a/tools/e2e-advisor/targets.mts +++ b/tools/e2e-advisor/targets.mts @@ -148,9 +148,7 @@ async function main(): Promise { fs.mkdirSync(outDir, { recursive: true }); - logProgress( - `Starting target advisor analysis: base=${baseRef} head=${headRef} outDir=${outDir}`, - ); + logProgress(`Starting target advisor analysis: base=${baseRef} head=${headRef} outDir=${outDir}`); const schema = readJson(schemaPath); const changedFiles = getChangedFiles(baseRef, headRef); logProgress(`Detected ${changedFiles.length} changed file(s)`); @@ -503,9 +501,7 @@ export function extractFreeStandingE2eJobs(workflowText: string): E2eWorkflowJob const body = bodyLines.join("\n"); if (!body.includes("inputs.jobs") || !body.includes(`,${id},`)) continue; const liveTestFiles = uniqueStrings( - [...body.matchAll(/test\/e2e\/live\/[A-Za-z0-9._-]+\.test\.ts/g)].map( - (item) => item[0], - ), + [...body.matchAll(/test\/e2e\/live\/[A-Za-z0-9._-]+\.test\.ts/g)].map((item) => item[0]), ).filter((file) => file !== REGISTRY_LIVE_ENTRYPOINT); if (liveTestFiles.length === 0) continue; jobs.push({ id, liveTestFiles }); @@ -537,11 +533,7 @@ function shouldSuppressFanoutForUnwiredLiveTests( } function isE2eTargetRelevantFile(file: string): boolean { - return ( - file === E2E_WORKFLOW_PATH || - file.startsWith("test/e2e/") || - file.startsWith("tools/e2e") - ); + return file === E2E_WORKFLOW_PATH || file.startsWith("test/e2e/") || file.startsWith("tools/e2e"); } function missingFreeStandingLiveWiringReason(files: string[]): string { diff --git a/tools/e2e/assert-mcp-artifact-secrets-absent.mts b/tools/e2e/assert-mcp-artifact-secrets-absent.mts new file mode 100644 index 00000000000..4c10593076d --- /dev/null +++ b/tools/e2e/assert-mcp-artifact-secrets-absent.mts @@ -0,0 +1,113 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; + +import { MCP_BRIDGE_TEST_CREDENTIALS } from "../../test/e2e/fixtures/mcp-bridge-credentials.ts"; + +export interface ArtifactSecretLeak { + credential: keyof typeof MCP_BRIDGE_TEST_CREDENTIALS; + encoding: "base64" | "raw"; + file: string; +} + +export interface ArtifactSecretScanResult { + filesScanned: number; + leaks: ArtifactSecretLeak[]; +} + +const BASE64_CANDIDATE = /[A-Za-z0-9+/_-]{16,}={0,2}/g; + +function listArtifactFiles(root: string): string[] { + if (!fs.existsSync(root)) return []; + const files: string[] = []; + const visit = (target: string): void => { + const stat = fs.lstatSync(target); + if (stat.isSymbolicLink()) { + throw new Error(`MCP artifact scan refuses symbolic link: ${path.relative(root, target)}`); + } + if (stat.isDirectory()) { + for (const entry of fs.readdirSync(target).sort()) visit(path.join(target, entry)); + return; + } + if (!stat.isFile()) { + throw new Error(`MCP artifact scan refuses non-regular file: ${path.relative(root, target)}`); + } + files.push(target); + }; + visit(root); + return files; +} + +function decodedBase64Candidates(text: string): Buffer[] { + return [text, text.replace(/(?:\s+|\\[rnt])+/gu, "")].flatMap((candidateText) => + [...candidateText.matchAll(BASE64_CANDIDATE)].map((match) => { + const normalized = match[0].replace(/-/g, "+").replace(/_/g, "/"); + const padded = normalized.padEnd( + normalized.length + ((4 - (normalized.length % 4)) % 4), + "=", + ); + return Buffer.from(padded, "base64"); + }), + ); +} + +export function scanMcpArtifactSecrets(rootDirectory: string): ArtifactSecretScanResult { + const root = path.resolve(rootDirectory); + const files = listArtifactFiles(root); + const leaks: ArtifactSecretLeak[] = []; + + for (const file of files) { + const data = fs.readFileSync(file); + const text = data.toString("utf8"); + const decodedCandidates = decodedBase64Candidates(text); + for (const [credential, secret] of Object.entries(MCP_BRIDGE_TEST_CREDENTIALS) as Array< + [keyof typeof MCP_BRIDGE_TEST_CREDENTIALS, string] + >) { + const secretBytes = Buffer.from(secret, "utf8"); + if (data.includes(secretBytes)) { + leaks.push({ credential, encoding: "raw", file: path.relative(root, file) }); + } + const encodedForms = [ + secretBytes.toString("base64"), + secretBytes.toString("base64").replace(/=+$/u, ""), + secretBytes.toString("base64url"), + ]; + if ( + encodedForms.some((encoded) => text.includes(encoded)) || + decodedCandidates.some((decoded) => decoded.includes(secretBytes)) + ) { + leaks.push({ credential, encoding: "base64", file: path.relative(root, file) }); + } + } + } + + return { filesScanned: files.length, leaks }; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + try { + const root = process.argv[2]; + if (!root || process.argv.length !== 3) { + throw new Error( + "Usage: npx tsx tools/e2e/assert-mcp-artifact-secrets-absent.mts ARTIFACT_DIR", + ); + } + const result = scanMcpArtifactSecrets(root); + if (result.leaks.length > 0) { + for (const leak of result.leaks) { + console.error( + `::error file=${leak.file}::MCP artifact contains ${leak.encoding}-encoded ${leak.credential} fixture credential`, + ); + } + process.exitCode = 1; + } else { + console.log(`MCP artifact credential scan passed (${result.filesScanned} files)`); + } + } catch (error) { + console.error(`::error::${error instanceof Error ? error.message : String(error)}`); + process.exitCode = 1; + } +} diff --git a/tools/e2e/brev-remote-vitest.mts b/tools/e2e/brev-remote-vitest.mts new file mode 100644 index 00000000000..19b8c308517 --- /dev/null +++ b/tools/e2e/brev-remote-vitest.mts @@ -0,0 +1,56 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { shellQuote } from "../../src/lib/core/shell-quote"; + +export type BrevVitestProject = "cli" | "e2e-live"; + +export const BREV_SECURITY_SUITE_TIMEOUT_MS = 20 * 60_000; +export const BREV_MESSAGING_PROVIDER_TIMEOUT_MS = 70 * 60_000; +export const BREV_MESSAGING_COMPAT_TIMEOUT_MS = 40 * 60_000; +export const BREV_REMOTE_WRAPPER_GRACE_MS = 120_000; +export const BREV_WORKFLOW_OWNERSHIP_ENV = "NEMOCLAW_BREV_WORKFLOW_OWNS_INSTANCE"; + +const BREV_SUITES_WITHOUT_HARNESS_SANDBOX = new Set([ + "all", + "full", + "gpu", + "messaging-compatible-endpoint", + "messaging-providers", +]); + +export function brevSuiteNeedsHarnessSandbox(testSuite: string): boolean { + return !BREV_SUITES_WITHOUT_HARNESS_SANDBOX.has(testSuite); +} + +export function brevSuiteHarnessSandboxName(testSuite: string): string | undefined { + return brevSuiteNeedsHarnessSandbox(testSuite) ? "e2e-test" : undefined; +} + +export function brevWorkflowOwnsInstance(env: NodeJS.ProcessEnv = process.env): boolean { + return env[BREV_WORKFLOW_OWNERSHIP_ENV] === "1"; +} + +export function buildBrevRemoteVitestCommand(project: BrevVitestProject, target: string): string { + const vitestCommand = [ + "./node_modules/.bin/vitest", + "run", + "--project", + project, + target, + "--silent=false", + "--reporter=default", + ] + .map(shellQuote) + .join(" "); + + return [ + // A nested live installer test may run npm link and prune the repository's + // dev dependencies. Restore the reviewed lockfile graph before the next + // remote suite, with lifecycle scripts disabled, instead of letting npx + // download an unpinned replacement. + "if [ ! -x ./node_modules/.bin/vitest ]; then npm ci --ignore-scripts --no-audit --no-fund; fi", + "test -x ./node_modules/.bin/vitest", + `NEMOCLAW_RUN_LIVE_E2E=1 ${vitestCommand}`, + ].join(" && "); +} diff --git a/tools/e2e/mcp-workflow-boundary.mts b/tools/e2e/mcp-workflow-boundary.mts new file mode 100644 index 00000000000..535e3c39496 --- /dev/null +++ b/tools/e2e/mcp-workflow-boundary.mts @@ -0,0 +1,408 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; + +import YAML from "yaml"; +import { UPLOAD_E2E_ARTIFACTS_ACTION } from "./upload-e2e-artifacts-workflow-boundary.mts"; + +const DEFAULT_WORKFLOW_PATH = ".github/workflows/e2e.yaml"; +const MCP_JOBS = ["mcp-bridge", "mcp-bridge-dev"] as const; +const TERMINAL_JOBS = ["report-to-pr", "scorecard"] as const; +const DOCKER_CLEANUP_RUN = "bash .github/scripts/docker-auth-cleanup.sh"; +const DEV_DOCKER_CLEANUP_NAME = "Revoke Docker auth before unverified dev tooling"; +const MCP_CLOUDFLARED_VERSION = "2026.6.1"; +const MCP_CLOUDFLARED_DEB_SHA256 = + "ccd02ec216c62bfa573395d8f72cb2e91e95cbdf8726a8acc06b3e2d9aa31526"; +const LEGACY_WORKFLOWS = [ + ".github/workflows/e2e-script.yaml", + ".github/workflows/e2e-vitest-scenarios.yaml", + ".github/workflows/nightly-e2e.yaml", +] as const; +const FORBIDDEN_INFERENCE_SECRETS = + /ANTHROPIC_API_KEY|AWS_(?:ACCESS_KEY_ID|SECRET_ACCESS_KEY)|COMPATIBLE_(?:ANTHROPIC_)?API_KEY|GITHUB_TOKEN|GH_TOKEN|NVIDIA_(?:INFERENCE_)?API_KEY|OPENAI_API_KEY/; + +type UnknownRecord = Record; + +function asRecord(value: unknown): UnknownRecord { + return value !== null && typeof value === "object" && !Array.isArray(value) + ? (value as UnknownRecord) + : {}; +} + +function asString(value: unknown): string { + return typeof value === "string" ? value : ""; +} + +function asSteps(job: UnknownRecord): UnknownRecord[] { + const steps = job.steps; + return Array.isArray(steps) ? steps.map(asRecord) : []; +} + +function namedStep(job: UnknownRecord, name: string): UnknownRecord { + return asSteps(job).find((step) => step.name === name) ?? {}; +} + +function isArtifactUploadStep(step: UnknownRecord): boolean { + const uses = asString(step.uses); + return uses === UPLOAD_E2E_ARTIFACTS_ACTION || uses.startsWith("actions/upload-artifact@"); +} + +function jobNeeds(job: UnknownRecord): string[] { + if (typeof job.needs === "string") return [job.needs]; + return Array.isArray(job.needs) + ? job.needs.filter((item): item is string => typeof item === "string") + : []; +} + +function requireEqual(errors: string[], actual: unknown, expected: unknown, message: string): void { + if (actual !== expected) errors.push(message); +} + +function requireContains( + errors: string[], + actual: unknown, + expected: string, + message: string, +): void { + if (!asString(actual).includes(expected)) errors.push(message); +} + +function validateJobIdentity( + errors: string[], + jobName: (typeof MCP_JOBS)[number], + job: UnknownRecord, +): void { + const env = asRecord(job.env); + requireEqual(errors, env.E2E_JOB, "1", `${jobName} must declare E2E_JOB=1`); + requireEqual( + errors, + env.E2E_TARGET_ID, + jobName, + `${jobName} must use its job id as E2E_TARGET_ID`, + ); + requireEqual( + errors, + env.NEMOCLAW_MCP_BRIDGE_AGENT_MATRIX, + "1", + `${jobName} must exercise all three MCP adapters`, + ); + requireEqual( + errors, + env.NEMOCLAW_RUN_LIVE_E2E, + "1", + `${jobName} must enable the unified live E2E project`, + ); + requireContains( + errors, + env.E2E_ARTIFACT_DIR, + `e2e-artifacts/live/${jobName}`, + `${jobName} must isolate its artifact directory`, + ); + if (jobName === "mcp-bridge") { + requireEqual( + errors, + env.NEMOCLAW_OPENSHELL_CHANNEL, + "stable", + "mcp-bridge must pin the stable OpenShell channel", + ); + if (Object.hasOwn(env, "E2E_DEFAULT_ENABLED")) { + errors.push("mcp-bridge must remain default-enabled"); + } + requireContains( + errors, + job.if, + "inputs.jobs == ''", + "mcp-bridge must run in default full-suite dispatches", + ); + } else { + requireEqual(errors, env.E2E_DEFAULT_ENABLED, "0", "mcp-bridge-dev must remain explicit-only"); + requireEqual( + errors, + env.NEMOCLAW_OPENSHELL_CHANNEL, + "dev", + "mcp-bridge-dev must select the OpenShell dev channel", + ); + if (Object.hasOwn(env, "NEMOCLAW_ACCEPT_DEV_UNVERIFIED_INSTALL")) { + errors.push("mcp-bridge-dev must scope unverified artifact opt-in to its installer step"); + } + if (asString(job.if).includes("inputs.jobs == ''")) { + errors.push("mcp-bridge-dev must not run in default full-suite dispatches"); + } + } +} + +function validateJobSecurity( + errors: string[], + jobName: (typeof MCP_JOBS)[number], + job: UnknownRecord, + canonicalDockerAuth: UnknownRecord, +): void { + const permissions = asRecord(job.permissions); + if (Object.keys(permissions).sort().join(",") !== "contents" || permissions.contents !== "read") { + errors.push(`${jobName} must use only contents:read permissions`); + } + + const checkouts = asSteps(job).filter((step) => + asString(step.uses).startsWith("actions/checkout@"), + ); + if (checkouts.length !== 1) errors.push(`${jobName} must use exactly one checkout step`); + for (const checkout of checkouts) { + if (!/^actions\/checkout@[0-9a-f]{40}$/.test(asString(checkout.uses))) { + errors.push(`${jobName} must use a SHA-pinned checkout`); + } + if (asRecord(checkout.with)["persist-credentials"] !== false) { + errors.push(`${jobName} checkout must set persist-credentials:false`); + } + } + if (FORBIDDEN_INFERENCE_SECRETS.test(JSON.stringify(job))) { + errors.push(`${jobName} must not receive inference or GitHub credentials`); + } + + const login = namedStep(job, "Authenticate to Docker Hub"); + const cleanup = namedStep(job, "Clean up Docker auth"); + if (JSON.stringify(login) !== JSON.stringify(canonicalDockerAuth)) { + errors.push(`${jobName} must reuse the canonical isolated Docker Hub auth step`); + } + const expectedCleanup = { + name: "Clean up Docker auth", + if: "always()", + shell: "bash", + run: DOCKER_CLEANUP_RUN, + }; + if (JSON.stringify(cleanup) !== JSON.stringify(expectedCleanup)) { + errors.push(`${jobName} must use the canonical unconditional Docker auth cleanup`); + } + const steps = asSteps(job); + const checkoutIndex = steps.findIndex((step) => + asString(step.uses).startsWith("actions/checkout@"), + ); + if (steps.indexOf(login) !== checkoutIndex + 1) { + errors.push(`${jobName} must authenticate immediately after credential-free checkout`); + } + if (steps.indexOf(cleanup) !== steps.length - 1) { + errors.push(`${jobName} Docker auth cleanup must remain the final step`); + } + if (jobName === "mcp-bridge-dev") { + const devCleanup = namedStep(job, DEV_DOCKER_CLEANUP_NAME); + const install = namedStep(job, "Install OpenShell CLI"); + const expectedDevCleanup = { + name: DEV_DOCKER_CLEANUP_NAME, + shell: "bash", + run: DOCKER_CLEANUP_RUN, + }; + if (JSON.stringify(devCleanup) !== JSON.stringify(expectedDevCleanup)) { + errors.push("mcp-bridge-dev must revoke Docker auth before unverified dev tooling"); + } + const devCleanupIndex = steps.indexOf(devCleanup); + const installIndex = steps.indexOf(install); + if (devCleanupIndex <= steps.indexOf(login) || installIndex <= devCleanupIndex) { + errors.push( + "mcp-bridge-dev Docker auth revocation must follow setup and precede the dev installer", + ); + } + if ( + devCleanupIndex >= 0 && + steps.slice(devCleanupIndex + 1).some((step) => step.name === "Authenticate to Docker Hub") + ) { + errors.push("mcp-bridge-dev must not restore Docker auth after dev-tooling revocation"); + } + } +} + +function validateJobExecution( + errors: string[], + jobName: (typeof MCP_JOBS)[number], + job: UnknownRecord, +): void { + const steps = asSteps(job); + const cloudflared = namedStep(job, "Install and verify cloudflared prerequisite"); + const tls = namedStep(job, "Generate MCP test TLS"); + const install = namedStep(job, "Install OpenShell CLI"); + const run = namedStep(job, "Run MCP OpenShell provider live test"); + const scan = namedStep(job, "Scan MCP artifacts for fixture credentials"); + const uploads = steps.filter(isArtifactUploadStep); + const upload = namedStep(job, "Upload MCP server artifacts"); + if (uploads.length !== 1 || uploads[0] !== upload) { + errors.push(`${jobName} must use exactly one reviewed MCP artifact upload step`); + } + + const cloudflaredEnv = asRecord(cloudflared.env); + requireEqual( + errors, + cloudflaredEnv.CLOUDFLARED_VERSION, + MCP_CLOUDFLARED_VERSION, + `${jobName} must pin cloudflared ${MCP_CLOUDFLARED_VERSION}`, + ); + requireEqual( + errors, + cloudflaredEnv.CLOUDFLARED_DEB_SHA256, + MCP_CLOUDFLARED_DEB_SHA256, + `${jobName} must pin the reviewed cloudflared package checksum`, + ); + for (const required of [ + "https://github.com/cloudflare/cloudflared/releases/download/${CLOUDFLARED_VERSION}/cloudflared-linux-amd64.deb", + "sha256sum -c -", + "dpkg-deb -f", + "sudo dpkg -i", + "cloudflared version ${CLOUDFLARED_VERSION}", + ]) { + requireContains( + errors, + cloudflared.run, + required, + `${jobName} cloudflared installation is not immutable and verified`, + ); + } + for (const forbidden of ["pkg.cloudflare.com", "apt-get install", "apt install"]) { + if (asString(cloudflared.run).includes(forbidden)) { + errors.push(`${jobName} cloudflared installation must not use mutable package repositories`); + } + } + if (steps.indexOf(cloudflared) < 0 || steps.indexOf(tls) <= steps.indexOf(cloudflared)) { + errors.push(`${jobName} must install verified cloudflared before creating MCP fixtures`); + } + + requireEqual( + errors, + tls.run, + "bash test/e2e/setup-mcp-test-tls.sh", + `${jobName} must generate its HTTPS fixture before installation`, + ); + if (steps.indexOf(tls) < 0 || steps.indexOf(install) <= steps.indexOf(tls)) { + errors.push(`${jobName} must generate HTTPS fixtures before installing OpenShell`); + } + requireEqual( + errors, + asRecord(install.env).NEMOCLAW_OPENSHELL_FORCE_INSTALL, + "1", + `${jobName} must force the selected OpenShell install`, + ); + const installEnv = asRecord(install.env); + if (jobName === "mcp-bridge-dev") { + requireEqual( + errors, + installEnv.NEMOCLAW_ACCEPT_DEV_UNVERIFIED_INSTALL, + "1", + "mcp-bridge-dev installer must explicitly authorize unverified dev artifacts", + ); + } else if (Object.hasOwn(installEnv, "NEMOCLAW_ACCEPT_DEV_UNVERIFIED_INSTALL")) { + errors.push("mcp-bridge stable installer must not authorize unverified dev artifacts"); + } + requireContains( + errors, + install.run, + "bash scripts/install-openshell.sh", + `${jobName} must use the repository OpenShell installer`, + ); + for (const required of ["--project e2e-live", "test/e2e/live/mcp-bridge.test.ts"]) { + requireContains(errors, run.run, required, `${jobName} must run the unified MCP live test`); + } + requireEqual( + errors, + scan.id, + "mcp_artifact_secret_scan", + `${jobName} secret scanner must expose its gated step id`, + ); + requireEqual( + errors, + scan.if, + "always()", + `${jobName} artifact secret scan must run unconditionally`, + ); + for (const required of [ + "tools/e2e/assert-mcp-artifact-secrets-absent.mts", + `e2e-artifacts/live/${jobName}`, + ]) { + requireContains(errors, scan.run, required, `${jobName} artifact secret scan is incomplete`); + } + requireEqual( + errors, + upload.uses, + UPLOAD_E2E_ARTIFACTS_ACTION, + `${jobName} artifact upload must use the reviewed shared uploader`, + ); + requireEqual( + errors, + upload.if, + "${{ always() && steps.mcp_artifact_secret_scan.outcome == 'success' }}", + `${jobName} artifact upload must be gated by the secret scanner`, + ); + const uploadOptions = asRecord(upload.with); + requireEqual( + errors, + uploadOptions.path, + `e2e-artifacts/live/${jobName}/`, + `${jobName} artifact upload must use exactly the scanned directory`, + ); + requireEqual( + errors, + uploadOptions.name, + `e2e-${jobName}`, + `${jobName} artifact upload must use its isolated artifact name`, + ); + if (Object.keys(uploadOptions).sort().join(",") !== "name,path") { + errors.push(`${jobName} artifact upload must delegate policy to the reviewed shared uploader`); + } + if (steps.indexOf(scan) < 0 || steps.indexOf(upload) <= steps.indexOf(scan)) { + errors.push(`${jobName} must scan artifacts before upload`); + } +} + +export function validateMcpOpenShellWorkflowBoundary( + workflowPath = DEFAULT_WORKFLOW_PATH, +): string[] { + const errors: string[] = []; + const workflowText = fs.readFileSync(workflowPath, "utf8"); + const workflow = asRecord(YAML.parse(workflowText)); + const jobs = asRecord(workflow.jobs); + const canonicalDockerAuth = namedStep(asRecord(jobs.live), "Authenticate to Docker Hub"); + const inputs = asRecord(asRecord(asRecord(workflow.on).workflow_dispatch).inputs); + const globalEnv = asRecord(workflow.env); + + if (Object.hasOwn(inputs, "openshell_channel")) { + errors.push("the unified workflow must not expose a fan-out-wide OpenShell channel input"); + } + if (Object.hasOwn(globalEnv, "NEMOCLAW_OPENSHELL_CHANNEL")) { + errors.push("the unified workflow must select OpenShell channels only inside MCP jobs"); + } + for (const legacy of LEGACY_WORKFLOWS) { + if (workflowPath === DEFAULT_WORKFLOW_PATH && fs.existsSync(legacy)) { + errors.push(`retired workflow must remain deleted: ${legacy}`); + } + } + for (const retiredToken of [ + "test/e2e-scenario/", + "tools/e2e-scenarios/", + "e2e-scenarios-live", + "NEMOCLAW_RUN_E2E_SCENARIOS", + "e2e-artifacts/vitest/", + ]) { + if (workflowText.includes(retiredToken)) { + errors.push(`unified MCP workflow must not reference retired token: ${retiredToken}`); + } + } + + for (const jobName of MCP_JOBS) { + const job = asRecord(jobs[jobName]); + if (Object.keys(job).length === 0) { + errors.push(`missing unified MCP job: ${jobName}`); + continue; + } + validateJobIdentity(errors, jobName, job); + validateJobSecurity(errors, jobName, job, canonicalDockerAuth); + validateJobExecution(errors, jobName, job); + } + + for (const terminalJobName of TERMINAL_JOBS) { + const terminal = asRecord(jobs[terminalJobName]); + const terminalNeeds = new Set(jobNeeds(terminal)); + for (const mcpJob of MCP_JOBS) { + if (!terminalNeeds.has(mcpJob)) { + errors.push(`${terminalJobName} must wait for ${mcpJob}`); + } + } + } + + return errors; +} diff --git a/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts b/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts index 476313f7c78..ef42515dda9 100644 --- a/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts +++ b/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts @@ -31,8 +31,10 @@ const UPLOAD_ARTIFACT_ACTION = "actions/upload-artifact@043fb46d1a93c77aae656e7c const UPLOAD_ARTIFACT_ACTION_PREFIX = "actions/upload-artifact@"; const INNER_ALWAYS = "${{ always() }}"; const CALLER_ALWAYS = "always()"; +const MCP_SCANNED_UPLOAD_CONDITION = + "${{ always() && steps.mcp_artifact_secret_scan.outcome == 'success' }}"; const TARGET_ID_PATTERN = /^[A-Za-z0-9_-]+$/; -const EXPECTED_UPLOAD_JOB_COUNT = 71; +const EXPECTED_UPLOAD_JOB_COUNT = 73; const EXPECTED_DEFAULT_CALLER_COUNT = 62; type WorkflowRecord = Record; @@ -134,6 +136,25 @@ const EXPLICIT_UPLOAD_CONTRACTS = new Map([ path: "e2e-artifacts/live/channels-stop-start/${{ matrix.agent }}/", }, ], + [ + "mcp-bridge", + { + name: "e2e-mcp-bridge", + path: "e2e-artifacts/live/mcp-bridge/", + }, + ], + [ + "mcp-bridge-dev", + { + name: "e2e-mcp-bridge-dev", + path: "e2e-artifacts/live/mcp-bridge-dev/", + }, + ], +]); + +const EXPLICIT_CALLER_CONDITIONS = new Map([ + ["mcp-bridge", MCP_SCANNED_UPLOAD_CONDITION], + ["mcp-bridge-dev", MCP_SCANNED_UPLOAD_CONDITION], ]); const EXPECTED_ACTION_INPUTS = { @@ -301,8 +322,13 @@ export function validateUploadE2eArtifactsInvocations(workflow: WorkflowRecord): if (typeof upload.name !== "string" || upload.name.length === 0) { errors.push(`${jobName} upload-e2e-artifacts invocation must retain a step name`); } - if (upload.if !== CALLER_ALWAYS) { - errors.push(`${jobName} upload-e2e-artifacts invocation must run with always()`); + const expectedCallerCondition = EXPLICIT_CALLER_CONDITIONS.get(jobName) ?? CALLER_ALWAYS; + if (upload.if !== expectedCallerCondition) { + errors.push( + expectedCallerCondition === CALLER_ALWAYS + ? `${jobName} upload-e2e-artifacts invocation must run with always()` + : `${jobName} upload-e2e-artifacts invocation must remain gated by its reviewed pre-upload checks`, + ); } const stepsAfterUpload = jobSteps.slice(jobSteps.indexOf(upload) + 1); if ( diff --git a/vitest.config.ts b/vitest.config.ts index 15d8683b28b..118175aac01 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -9,7 +9,6 @@ import { shouldRunBranchValidationE2E, shouldRunLiveE2E, } from "./test/e2e/fixtures/live-project-gate.ts"; -import { resolveE2ERetryCount } from "./test/helpers/e2e-retries"; import { testTimeout } from "./test/helpers/timeouts"; const isGithubActions = process.env.GITHUB_ACTIONS === "true"; @@ -17,7 +16,6 @@ const isCi = isGithubActions || process.env.CI === "true" || process.env.CI === const LIVE_E2E_PROJECT_TIMEOUT_MS = 30 * 60 * 1000; const runLiveE2E = shouldRunLiveE2E(); const runBranchValidationE2E = shouldRunBranchValidationE2E(); -const e2eRetryCount = resolveE2ERetryCount(); const sourceRequireHook = path.resolve("test/helpers/onboard-script-mocks.cjs"); const canonicalOpenShellPolicyBoundary = path.resolve( "nemoclaw/src/shared/openshell-policy-boundary.cts", @@ -88,6 +86,7 @@ export default defineConfig({ "test/e2e/support/**", "test/package-contract/**", "test/install-express-prompt.test.ts", + "test/install-build-dependency-preflight.test.ts", "test/install-preflight.test.ts", "test/install-preflight-docker-bootstrap.test.ts", "test/install-openshell-version-check.test.ts", @@ -101,6 +100,7 @@ export default defineConfig({ alias: canonicalOpenShellPolicyAlias, include: [ "test/install-express-prompt.test.ts", + "test/install-build-dependency-preflight.test.ts", "test/install-preflight.test.ts", "test/install-preflight-docker-bootstrap.test.ts", "test/install-openshell-version-check.test.ts", @@ -143,10 +143,11 @@ export default defineConfig({ name: "e2e-live", alias: canonicalOpenShellPolicyAlias, testTimeout: testTimeout(LIVE_E2E_PROJECT_TIMEOUT_MS), - // Vitest counts retries after the initial failure. In CI the default - // value of 2 gives live E2Es up to three total attempts while keeping - // local opt-in runs single-shot unless NEMOCLAW_E2E_RETRIES is set. - retry: e2eRetryCount, + // Live targets mutate host, Docker, gateway, and sandbox state. A + // whole-test retry reuses that state and can hide the first failure + // behind stale locks or exhausted storage. Transient operations must + // retry inside the target after proving their cleanup boundary. + retry: 0, include: runLiveE2E ? ["test/e2e/live/**/*.test.ts"] : [], // Live E2E tests are opt-in because they install, onboard, and // mutate real NemoClaw/OpenShell state. Run explicitly with: @@ -158,7 +159,10 @@ export default defineConfig({ test: { name: "e2e-branch-validation", alias: canonicalOpenShellPolicyAlias, - retry: e2eRetryCount, + // A branch-validation retry must provision a fresh remote instance. + // Retrying a stateful target inside one VM can overlap a timed-out + // installer that still legitimately owns the onboarding lock. + retry: 0, include: runBranchValidationE2E ? ["test/e2e/brev-e2e.test.ts"] : [], // Branch validation E2E: rsyncs the branch over a Brev instance // provisioned from the published NemoClaw launchable image and