From 98f5addc09fbaef07c18bdfd2eabe01d010fa2ba Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Sat, 18 Jul 2026 11:41:11 -0700 Subject: [PATCH 1/4] fix(installer): refresh packaged Station CDI --- ci/platform-matrix.json | 2 +- docs/get-started/dgx-station-preparation.mdx | 9 +- docs/reference/platform-support.mdx | 2 +- docs/reference/troubleshooting.mdx | 4 +- scripts/prepare-dgx-station-host.sh | 9 +- test/install-station-dgx-os.test.ts | 164 +++++++++++++++++++ 6 files changed, 184 insertions(+), 6 deletions(-) diff --git a/ci/platform-matrix.json b/ci/platform-matrix.json index 9eebe0c60db..6a602deeacf 100644 --- a/ci/platform-matrix.json +++ b/ci/platform-matrix.json @@ -66,7 +66,7 @@ "prd_priority": "P1", "ci_tested": false, "prerequisites_notes": "Deferred. For evaluation on a qualifying DGX Station GB300, see [Additional Setup for DGX Station](prerequisites/dgx-station-preparation) before the Quickstart.", - "notes": "The PRD marks this platform as P1. Workstation form-factor with NVIDIA GPUs and the same Docker + NVIDIA Container Toolkit + CDI requirements as DGX Spark. Station remains Deferred. For canonical hardware qualification, image requirements, preparation, repair limits, reboot handoff, and the explicit temporary metadata override, see [Prepare DGX Station to Install NemoClaw](../get-started/prerequisites/dgx-station-preparation). On a Station GB300, the installer accepts either a clean generic Ubuntu 24.04 ARM64 image or stock DGX OS `7.2.0`, `7.4.0`, or `7.5.0` when a safe, root-owned `/etc/dgx-release` marker identifies `DGX Server for GALAXY-GB300`. It also recognizes two exact no-OTA factory profiles for bounded qualification. Both require `DGX_PLATFORM=DGX Server for GALAXY-GB300` and require `DGX_OTA_VERSION` and `DGX_OTA_DATE` to be absent. The April 2026 NVIDIA Colossus BaseOS profile requires `DGX_PRETTY_NAME=NVIDIA DGX Server`, `DGX_SWBUILD_VERSION=7.5.0-GB300ws-GB200ws`, and `DGX_SWBUILD_DATE=2026-04-02-08-20-16`; the June 2026 NVIDIA AI Developer Tools profile requires `DGX_PRETTY_NAME=NVIDIA DGX GB300WS`, `DGX_SWBUILD_VERSION=7.5.0`, and `DGX_SWBUILD_DATE=2026-06-16-11-48-10`. The stock DGX OS and AI Developer Tools paths validate the factory stack in place without replacing host packages, enabling or restarting services, generating CDI configuration, or rewriting the Docker runtime. The Colossus BaseOS path preserves the factory kernel, driver, DKMS, Docker, and NVIDIA Container Toolkit packages and prepares only Docker access and packaged CDI. Only the Colossus BaseOS path registers the NVIDIA Docker runtime when the launch probe proves it is missing, with restoration of the prior daemon configuration if registration fails. Each path validates the factory GB300 driver and volatile ECC state, Docker and containerd, Buildx, `nvidia-ctk`, CDI, and GPU visibility through both CDI and `--gpus all` before it proceeds. By default, unknown DGX OS versions, unmatched no-OTA factory images, unsafe release markers, and other Station generations stop before host preparation. The installer offers express install with the pinned `nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4` recipe, including an approximately 352 GB model download, without follow-up provider, model, policy, or sandbox-name choices. On Station GB300 only, direct-GPU sandbox policies add the exact read-only BDF directory for each host-discovered NVIDIA display-class PCI device, plus only existing CPU, memory, NUMA topology, and NVIDIA module initialization paths; they do not add `/sys`, the PCI parent subtree, or sysfs write access. One physical DGX OS `7.5.0` GB300 validation completed with released OpenShell `0.0.85`, local Nemotron Ultra serving, sandbox `cuInit(0)`, and a Hermes write/read file-tool task. In that validation, reads of `/sys/fs/cgroup/cgroup.controllers` and `/sys/class/net/lo/address` remained denied. DGX Station remains Deferred pending repeat qualification from a clean host and CI coverage. Pass `--station-deepseek` to use `deepseek-ai/DeepSeek-V4-Flash` while retaining the one-confirmation express flow. The flag requires an interactive terminal, and `/dev/tty` must be available when the installer runs through `curl | bash`. For headless setup on a qualifying Station, select `NEMOCLAW_PROVIDER=install-vllm` and `NEMOCLAW_VLLM_MODEL=deepseek-v4-flash` instead. Direct managed-vLLM onboarding still defaults to `deepseek-ai/DeepSeek-V4-Flash` when no model override is set." + "notes": "The PRD marks this platform as P1. Workstation form-factor with NVIDIA GPUs and the same Docker + NVIDIA Container Toolkit + CDI requirements as DGX Spark. Station remains Deferred. For canonical hardware qualification, image requirements, preparation, repair limits, reboot handoff, and the explicit temporary metadata override, see [Prepare DGX Station to Install NemoClaw](../get-started/prerequisites/dgx-station-preparation). On a Station GB300, the installer accepts either a clean generic Ubuntu 24.04 ARM64 image or stock DGX OS `7.2.0`, `7.4.0`, or `7.5.0` when a safe, root-owned `/etc/dgx-release` marker identifies `DGX Server for GALAXY-GB300`. It also recognizes two exact no-OTA factory profiles for bounded qualification. Both require `DGX_PLATFORM=DGX Server for GALAXY-GB300` and require `DGX_OTA_VERSION` and `DGX_OTA_DATE` to be absent. The April 2026 NVIDIA Colossus BaseOS profile requires `DGX_PRETTY_NAME=NVIDIA DGX Server`, `DGX_SWBUILD_VERSION=7.5.0-GB300ws-GB200ws`, and `DGX_SWBUILD_DATE=2026-04-02-08-20-16`; the June 2026 NVIDIA AI Developer Tools profile requires `DGX_PRETTY_NAME=NVIDIA DGX GB300WS`, `DGX_SWBUILD_VERSION=7.5.0`, and `DGX_SWBUILD_DATE=2026-06-16-11-48-10`. The stock DGX OS path validates the factory stack in place without replacing host packages, enabling or restarting services, generating CDI configuration, or rewriting the Docker runtime. The AI Developer Tools path preserves those boundaries when `nvidia.com/gpu=all` is already present. When that CDI device is missing, it requires no active workloads, enables the packaged `nvidia-cdi-refresh.path` and `nvidia-cdi-refresh.service` units, starts the path unit, restarts the refresh service, and checks the device again. It stops with service diagnostics if packaged refresh fails or omits the device. It never generates CDI directly, rewrites the Docker runtime, or restarts Docker or containerd. The Colossus BaseOS path preserves the factory kernel, driver, DKMS, Docker, and NVIDIA Container Toolkit packages and prepares only Docker access and packaged CDI. Only the Colossus BaseOS path registers the NVIDIA Docker runtime when the launch probe proves it is missing, with restoration of the prior daemon configuration if registration fails. Each path validates the factory GB300 driver and volatile ECC state, Docker and containerd, Buildx, `nvidia-ctk`, CDI, and GPU visibility through both CDI and `--gpus all` before it proceeds. By default, unknown DGX OS versions, unmatched no-OTA factory images, unsafe release markers, and other Station generations stop before host preparation. The installer offers express install with the pinned `nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4` recipe, including an approximately 352 GB model download, without follow-up provider, model, policy, or sandbox-name choices. On Station GB300 only, direct-GPU sandbox policies add the exact read-only BDF directory for each host-discovered NVIDIA display-class PCI device, plus only existing CPU, memory, NUMA topology, and NVIDIA module initialization paths; they do not add `/sys`, the PCI parent subtree, or sysfs write access. One physical DGX OS `7.5.0` GB300 validation completed with released OpenShell `0.0.85`, local Nemotron Ultra serving, sandbox `cuInit(0)`, and a Hermes write/read file-tool task. In that validation, reads of `/sys/fs/cgroup/cgroup.controllers` and `/sys/class/net/lo/address` remained denied. DGX Station remains Deferred pending repeat qualification from a clean host and CI coverage. Pass `--station-deepseek` to use `deepseek-ai/DeepSeek-V4-Flash` while retaining the one-confirmation express flow. The flag requires an interactive terminal, and `/dev/tty` must be available when the installer runs through `curl | bash`. For headless setup on a qualifying Station, select `NEMOCLAW_PROVIDER=install-vllm` and `NEMOCLAW_VLLM_MODEL=deepseek-v4-flash` instead. Direct managed-vLLM onboarding still defaults to `deepseek-ai/DeepSeek-V4-Flash` when no model override is set." }, { "name": "NVIDIA RTX (consumer and Pro workstation GPUs)", diff --git a/docs/get-started/dgx-station-preparation.mdx b/docs/get-started/dgx-station-preparation.mdx index 2d6b7c4cb56..dcf8aefbfcd 100644 --- a/docs/get-started/dgx-station-preparation.mdx +++ b/docs/get-started/dgx-station-preparation.mdx @@ -69,11 +69,16 @@ After changing pinned packages, the installer exits with status `10`; reboot, si On a qualifying stock DGX OS or exact no-OTA factory image, express install validates the factory stack in place against the local default Docker daemon. It requires a loaded GB300 driver with zero volatile corrected and uncorrected ECC errors, active Docker and containerd services, working Docker Buildx, `nvidia-ctk`, the `nvidia.com/gpu=all` CDI device, and successful `nvidia-smi` device-visibility probes through both CDI and `--gpus all`. -The stock DGX OS and AI Developer Tools paths do not install or replace host packages, enable or restart services, generate CDI configuration, or rewrite the Docker runtime. +The stock DGX OS path does not install or replace host packages, enable or restart services, generate CDI configuration, or rewrite the Docker runtime. +The AI Developer Tools path preserves those boundaries when `nvidia.com/gpu=all` is already present. +When that CDI device is missing, the AI Developer Tools path first confirms that no workloads are active, then enables the packaged `nvidia-cdi-refresh.path` and `nvidia-cdi-refresh.service` units, starts the path unit, restarts the refresh service, and checks the device again. +It stops with service diagnostics if packaged refresh fails or omits the device. +It never generates CDI directly, rewrites the Docker runtime, or restarts Docker or containerd. The Colossus BaseOS path requires the exact reviewed package and failed-service fingerprints, preserves the factory kernel, driver, DKMS, Docker, and NVIDIA Container Toolkit packages, and prepares only Docker access and packaged CDI. Only the Colossus BaseOS path registers the NVIDIA Docker runtime when the launch probe proves that the runtime is missing, and it restores the prior daemon configuration if registration fails. The preparation helper writes a private audit log under `~/station-bootstrap-logs`. -Aside from the installer's separately disclosed `docker`-group membership grant when the current trusted account needs Docker access, the stock DGX OS and AI Developer Tools paths leave only the digest-pinned visibility-test image in the Docker cache. +Aside from the installer's separately disclosed `docker`-group membership grant when the current trusted account needs Docker access, the stock DGX OS path leaves only the digest-pinned visibility-test image in the Docker cache. +The AI Developer Tools path can also leave the packaged CDI refresh units enabled when it repairs a missing device. One physical DGX OS `7.5.0` GB300 validation completed with released OpenShell `0.0.85`, local Nemotron Ultra serving, sandbox `cuInit(0)`, and a Hermes write/read file-tool task. Repeat qualification from a clean host and CI coverage remain pending. diff --git a/docs/reference/platform-support.mdx b/docs/reference/platform-support.mdx index 7238a30c291..02cae02e2e5 100644 --- a/docs/reference/platform-support.mdx +++ b/docs/reference/platform-support.mdx @@ -79,7 +79,7 @@ For install requirements and the shorter setup-oriented platform view, refer to | OS | Container runtime | Status | PRD priority | CI | Notes | |----|-------------------|--------|--------------|----|-------| | DGX OS (Spark) | Docker | Tested | P1 | Yes | Use the standard installer and `$$nemoclaw onboard`. For an end-to-end walkthrough with local inference, see the [NVIDIA Spark playbook](https://build.nvidia.com/spark/nemoclaw). | -| DGX OS (Station) | Docker | Deferred | P1 | No | The PRD marks this platform as P1. Workstation form-factor with NVIDIA GPUs and the same Docker + NVIDIA Container Toolkit + CDI requirements as DGX Spark. Station remains Deferred. For canonical hardware qualification, image requirements, preparation, repair limits, reboot handoff, and the explicit temporary metadata override, see [Prepare DGX Station to Install NemoClaw](../get-started/prerequisites/dgx-station-preparation). On a Station GB300, the installer accepts either a clean generic Ubuntu 24.04 ARM64 image or stock DGX OS `7.2.0`, `7.4.0`, or `7.5.0` when a safe, root-owned `/etc/dgx-release` marker identifies `DGX Server for GALAXY-GB300`. It also recognizes two exact no-OTA factory profiles for bounded qualification. Both require `DGX_PLATFORM=DGX Server for GALAXY-GB300` and require `DGX_OTA_VERSION` and `DGX_OTA_DATE` to be absent. The April 2026 NVIDIA Colossus BaseOS profile requires `DGX_PRETTY_NAME=NVIDIA DGX Server`, `DGX_SWBUILD_VERSION=7.5.0-GB300ws-GB200ws`, and `DGX_SWBUILD_DATE=2026-04-02-08-20-16`; the June 2026 NVIDIA AI Developer Tools profile requires `DGX_PRETTY_NAME=NVIDIA DGX GB300WS`, `DGX_SWBUILD_VERSION=7.5.0`, and `DGX_SWBUILD_DATE=2026-06-16-11-48-10`. The stock DGX OS and AI Developer Tools paths validate the factory stack in place without replacing host packages, enabling or restarting services, generating CDI configuration, or rewriting the Docker runtime. The Colossus BaseOS path preserves the factory kernel, driver, DKMS, Docker, and NVIDIA Container Toolkit packages and prepares only Docker access and packaged CDI. Only the Colossus BaseOS path registers the NVIDIA Docker runtime when the launch probe proves it is missing, with restoration of the prior daemon configuration if registration fails. Each path validates the factory GB300 driver and volatile ECC state, Docker and containerd, Buildx, `nvidia-ctk`, CDI, and GPU visibility through both CDI and `--gpus all` before it proceeds. By default, unknown DGX OS versions, unmatched no-OTA factory images, unsafe release markers, and other Station generations stop before host preparation. The installer offers express install with the pinned `nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4` recipe, including an approximately 352 GB model download, without follow-up provider, model, policy, or sandbox-name choices. On Station GB300 only, direct-GPU sandbox policies add the exact read-only BDF directory for each host-discovered NVIDIA display-class PCI device, plus only existing CPU, memory, NUMA topology, and NVIDIA module initialization paths; they do not add `/sys`, the PCI parent subtree, or sysfs write access. One physical DGX OS `7.5.0` GB300 validation completed with released OpenShell `0.0.85`, local Nemotron Ultra serving, sandbox `cuInit(0)`, and a Hermes write/read file-tool task. In that validation, reads of `/sys/fs/cgroup/cgroup.controllers` and `/sys/class/net/lo/address` remained denied. DGX Station remains Deferred pending repeat qualification from a clean host and CI coverage. Pass `--station-deepseek` to use `deepseek-ai/DeepSeek-V4-Flash` while retaining the one-confirmation express flow. The flag requires an interactive terminal, and `/dev/tty` must be available when the installer runs through `curl \| bash`. For headless setup on a qualifying Station, select `NEMOCLAW_PROVIDER=install-vllm` and `NEMOCLAW_VLLM_MODEL=deepseek-v4-flash` instead. Direct managed-vLLM onboarding still defaults to `deepseek-ai/DeepSeek-V4-Flash` when no model override is set. | +| DGX OS (Station) | Docker | Deferred | P1 | No | The PRD marks this platform as P1. Workstation form-factor with NVIDIA GPUs and the same Docker + NVIDIA Container Toolkit + CDI requirements as DGX Spark. Station remains Deferred. For canonical hardware qualification, image requirements, preparation, repair limits, reboot handoff, and the explicit temporary metadata override, see [Prepare DGX Station to Install NemoClaw](../get-started/prerequisites/dgx-station-preparation). On a Station GB300, the installer accepts either a clean generic Ubuntu 24.04 ARM64 image or stock DGX OS `7.2.0`, `7.4.0`, or `7.5.0` when a safe, root-owned `/etc/dgx-release` marker identifies `DGX Server for GALAXY-GB300`. It also recognizes two exact no-OTA factory profiles for bounded qualification. Both require `DGX_PLATFORM=DGX Server for GALAXY-GB300` and require `DGX_OTA_VERSION` and `DGX_OTA_DATE` to be absent. The April 2026 NVIDIA Colossus BaseOS profile requires `DGX_PRETTY_NAME=NVIDIA DGX Server`, `DGX_SWBUILD_VERSION=7.5.0-GB300ws-GB200ws`, and `DGX_SWBUILD_DATE=2026-04-02-08-20-16`; the June 2026 NVIDIA AI Developer Tools profile requires `DGX_PRETTY_NAME=NVIDIA DGX GB300WS`, `DGX_SWBUILD_VERSION=7.5.0`, and `DGX_SWBUILD_DATE=2026-06-16-11-48-10`. The stock DGX OS path validates the factory stack in place without replacing host packages, enabling or restarting services, generating CDI configuration, or rewriting the Docker runtime. The AI Developer Tools path preserves those boundaries when `nvidia.com/gpu=all` is already present. When that CDI device is missing, it requires no active workloads, enables the packaged `nvidia-cdi-refresh.path` and `nvidia-cdi-refresh.service` units, starts the path unit, restarts the refresh service, and checks the device again. It stops with service diagnostics if packaged refresh fails or omits the device. It never generates CDI directly, rewrites the Docker runtime, or restarts Docker or containerd. The Colossus BaseOS path preserves the factory kernel, driver, DKMS, Docker, and NVIDIA Container Toolkit packages and prepares only Docker access and packaged CDI. Only the Colossus BaseOS path registers the NVIDIA Docker runtime when the launch probe proves it is missing, with restoration of the prior daemon configuration if registration fails. Each path validates the factory GB300 driver and volatile ECC state, Docker and containerd, Buildx, `nvidia-ctk`, CDI, and GPU visibility through both CDI and `--gpus all` before it proceeds. By default, unknown DGX OS versions, unmatched no-OTA factory images, unsafe release markers, and other Station generations stop before host preparation. The installer offers express install with the pinned `nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4` recipe, including an approximately 352 GB model download, without follow-up provider, model, policy, or sandbox-name choices. On Station GB300 only, direct-GPU sandbox policies add the exact read-only BDF directory for each host-discovered NVIDIA display-class PCI device, plus only existing CPU, memory, NUMA topology, and NVIDIA module initialization paths; they do not add `/sys`, the PCI parent subtree, or sysfs write access. One physical DGX OS `7.5.0` GB300 validation completed with released OpenShell `0.0.85`, local Nemotron Ultra serving, sandbox `cuInit(0)`, and a Hermes write/read file-tool task. In that validation, reads of `/sys/fs/cgroup/cgroup.controllers` and `/sys/class/net/lo/address` remained denied. DGX Station remains Deferred pending repeat qualification from a clean host and CI coverage. Pass `--station-deepseek` to use `deepseek-ai/DeepSeek-V4-Flash` while retaining the one-confirmation express flow. The flag requires an interactive terminal, and `/dev/tty` must be available when the installer runs through `curl \| bash`. For headless setup on a qualifying Station, select `NEMOCLAW_PROVIDER=install-vllm` and `NEMOCLAW_VLLM_MODEL=deepseek-v4-flash` instead. Direct managed-vLLM onboarding still defaults to `deepseek-ai/DeepSeek-V4-Flash` when no model override is set. | | Linux | Docker | Tested | P0 | Yes | Primary tested path. Ubuntu 24.04 has host-level onboarding validation. A digest-pinned Ubuntu 26.04 userspace lane builds the CLI and runs preflight, installer, and platform contracts on eligible main pushes; Docker-host, AppArmor, Landlock, and live onboarding validation on 26.04 remain pending. Other distros (Ubuntu 22.04, Fedora, Rocky, Alma, NixOS, Arch) may work but are not validated. | | macOS (Apple Silicon) | Colima, Docker Desktop | Tested with limitations | P0 | Yes | Start the container runtime (Colima or Docker Desktop) before running the installer. Homebrew Colima users must install both Colima and the Docker CLI (`brew install colima docker`) before `docker info` can work. Xcode Command Line Tools (`xcode-select --install`) are typically required for Node native modules during install. NemoClaw recommends them but does not enforce them during preflight. | | NVIDIA RTX (consumer and Pro workstation GPUs) | Docker | Deferred | P1 | No | The PRD marks this platform as P1. Covers RTX consumer cards and RTX Pro workstation cards on Linux hosts that meet the generic-Linux-GPU requirements (NVIDIA Container Toolkit + CDI present). The provider menu emits managed vLLM behind `NEMOCLAW_EXPERIMENTAL=1` or `NEMOCLAW_PROVIDER=install-vllm` for this host class today; the end-to-end onboard path on this hardware is not yet validated in CI. | diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index 36cbe512e42..d7f42e3be55 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -2270,7 +2270,9 @@ When shared gateway cleanup would be unsafe, follow the targeted destroy or gate Recent NVIDIA Container Toolkit installs configure the Docker daemon for Container Device Interface (CDI) device injection, which OpenShell's `gateway start --gpu` then auto-selects. If no `nvidia.com/gpu` CDI spec has been generated on the host yet, gateway start fails with `Docker responded with status code 500: CDI device injection failed: unresolvable CDI devices nvidia.com/gpu=all`. Outside Station Express, the standard NemoClaw installer detects this gap before onboarding, first tries to enable the NVIDIA CDI refresh systemd units, and can fall back to generating the spec directly with `nvidia-ctk`. -Station Express requires the packaged refresh lifecycle to work; if it fails or omits `nvidia.com/gpu=all`, inspect `nvidia-cdi-refresh.service`, repair it, and rerun the printed exact-commit install command. +Station Express never falls back to direct CDI generation. +The generic Ubuntu, Colossus BaseOS, and exact AI Developer Tools paths require the packaged refresh lifecycle to work; if it fails or omits `nvidia.com/gpu=all`, inspect `nvidia-cdi-refresh.service`, repair it, and rerun the printed exact-commit install command. +Other factory-runtime profiles stop when the CDI device is missing without enabling or restarting the refresh units. If you run `$$nemoclaw onboard` directly, preflight prints the manual remediation instead. The native Linux fix is the same on Docker hosts whose `docker info` advertises a non-empty `CDISpecDirs`. On WSL with Docker Desktop, Docker may advertise CDI directories even though `--device nvidia.com/gpu=all` is not usable from the WSL distro. diff --git a/scripts/prepare-dgx-station-host.sh b/scripts/prepare-dgx-station-host.sh index 53d4fc6eb77..4a2b13f27d4 100755 --- a/scripts/prepare-dgx-station-host.sh +++ b/scripts/prepare-dgx-station-host.sh @@ -5,7 +5,7 @@ set -Eeuo pipefail umask 077 -readonly SCRIPT_VERSION="2026-07-17.4" +readonly SCRIPT_VERSION="2026-07-18.1" readonly REBOOT_REQUIRED_EXIT=10 readonly LOGIN_REQUIRED_EXIT=11 readonly MIN_FREE_KIB=$((20 * 1024 * 1024)) @@ -1358,6 +1358,13 @@ verify_dgx_os_runtime_sudo() { || fatal "The local Docker daemon is not reachable with sudo on the Station factory image" station_sudo_local_default_docker buildx version >/dev/null 2>&1 \ || fatal "Docker Buildx is unavailable on the Station factory image" + if [[ "$STATION_HOST_PROFILE" == "ai-developer-tools" ]]; then + if sudo nvidia-ctk cdi list | grep -Fxq 'nvidia.com/gpu=all'; then + info "cdi=nvidia.com/gpu=all source=factory_runtime" + else + refresh_cdi + fi + fi sudo nvidia-ctk cdi list | grep -Fxq 'nvidia.com/gpu=all' \ || fatal "The Station factory image does not advertise the nvidia.com/gpu=all CDI device" ensure_dgx_os_acceptance_image diff --git a/test/install-station-dgx-os.test.ts b/test/install-station-dgx-os.test.ts index bffe5d85572..82449a0af24 100644 --- a/test/install-station-dgx-os.test.ts +++ b/test/install-station-dgx-os.test.ts @@ -789,6 +789,167 @@ run_apply expect(output).not.toContain("UNEXPECTED_REQUIREMENT"); }); + it("keeps healthy AI Developer Tools CDI validation-only", () => { + const { result, output } = runSourced( + STATION_PREPARE, + ` +require_command() { + [[ "$1" == "sudo" ]] || { printf 'UNEXPECTED_REQUIREMENT %s\n' "$1"; return 1; } +} +acquire_sudo() { :; } +common_preflight() { STATION_HOST_PROFILE=ai-developer-tools; } +check_dgx_os_runtime_commands() { printf 'FACTORY_GATES_OK\n'; } +systemctl() { + case "$*" in + 'is-active --quiet containerd.service'|'is-active --quiet docker.service') return 0 ;; + *) printf 'UNEXPECTED_SYSTEMCTL %s\n' "$*"; return 1 ;; + esac +} +station_sudo_local_default_docker() { + case "$*" in + 'info'|'buildx version'|'ps -aq') return 0 ;; + *) printf 'UNEXPECTED_DOCKER %s\n' "$*"; return 1 ;; + esac +} +sudo() { + case "$*" in + 'nvidia-ctk cdi list') printf 'nvidia.com/gpu=all\n' ;; + *) printf 'UNEXPECTED_SUDO %s\n' "$*"; return 1 ;; + esac +} +refresh_cdi() { printf 'UNEXPECTED_CDI_REFRESH\n'; return 1; } +ensure_dgx_os_acceptance_image() { printf 'IMAGE_CACHE_READY\n'; } +run_dgx_os_cdi_test_sudo() { printf 'DIGEST_PINNED_CDI_PROBE_OK\n'; } +run_dgx_os_gpus_test_sudo() { printf 'DIGEST_PINNED_GPUS_PROBE_OK\n'; } +ensure_docker_group() { printf 'DOCKER_GROUP_PRESENT\n'; } +install_packages() { printf 'UNEXPECTED_PACKAGE_MUTATION\n'; return 1; } +finish_runtime() { printf 'UNEXPECTED_RUNTIME_MUTATION\n'; return 1; } +run_apply +`, + ); + + expect(result.status, output).toBe(0); + expect(output).toContain("FACTORY_GATES_OK"); + expect(output).toContain("cdi=nvidia.com/gpu=all source=factory_runtime"); + expect(output).toContain("DIGEST_PINNED_CDI_PROBE_OK"); + expect(output).toContain("DIGEST_PINNED_GPUS_PROBE_OK"); + expect(output.indexOf("FACTORY_GATES_OK")).toBeLessThan( + output.indexOf("cdi=nvidia.com/gpu=all source=factory_runtime"), + ); + expect(output).toContain("APPLY_RESULT=COMPLETE"); + expect(output).not.toContain("UNEXPECTED_"); + }); + + it("repairs missing AI Developer Tools CDI through the packaged lifecycle before validation", () => { + const { result, output } = runSourced( + STATION_PREPARE, + ` +cdi_ready=0 +require_command() { + [[ "$1" == "sudo" ]] || { printf 'UNEXPECTED_REQUIREMENT %s\n' "$1"; return 1; } +} +acquire_sudo() { :; } +common_preflight() { STATION_HOST_PROFILE=ai-developer-tools; } +check_dgx_os_runtime_commands() { printf 'FACTORY_GATES_OK\n'; } +systemctl() { + case "$*" in + 'is-active --quiet containerd.service'|'is-active --quiet docker.service') return 0 ;; + *) printf 'UNEXPECTED_SYSTEMCTL %s\n' "$*"; return 1 ;; + esac +} +station_sudo_local_default_docker() { + case "$*" in + 'info'|'buildx version'|'ps -aq') return 0 ;; + *) printf 'UNEXPECTED_DOCKER %s\n' "$*"; return 1 ;; + esac +} +check_no_workloads() { printf 'WORKLOAD_GATE_OK\n'; } +sudo() { + case "$*" in + 'nvidia-ctk cdi list') + ((cdi_ready == 1)) && printf 'nvidia.com/gpu=all\n' + ;; + 'systemctl enable nvidia-cdi-refresh.path nvidia-cdi-refresh.service'|'systemctl start nvidia-cdi-refresh.path') + printf 'PACKAGED_LIFECYCLE %s\n' "$*" + ;; + 'systemctl restart nvidia-cdi-refresh.service') + printf 'PACKAGED_LIFECYCLE %s\n' "$*" + cdi_ready=1 + ;; + *) printf 'UNEXPECTED_SUDO %s\n' "$*"; return 1 ;; + esac +} +nvidia-ctk() { + [[ "$*" == "cdi list" && "$cdi_ready" == "1" ]] && printf 'nvidia.com/gpu=all\n' +} +ensure_dgx_os_acceptance_image() { printf 'IMAGE_CACHE_READY\n'; } +run_dgx_os_cdi_test_sudo() { printf 'DIGEST_PINNED_CDI_PROBE_OK\n'; } +run_dgx_os_gpus_test_sudo() { printf 'DIGEST_PINNED_GPUS_PROBE_OK\n'; } +ensure_docker_group() { printf 'DOCKER_GROUP_PRESENT\n'; } +install_packages() { printf 'UNEXPECTED_PACKAGE_MUTATION\n'; return 1; } +finish_runtime() { printf 'UNEXPECTED_RUNTIME_MUTATION\n'; return 1; } +configure_docker_runtime_if_needed() { printf 'UNEXPECTED_DOCKER_RUNTIME_MUTATION\n'; return 1; } +run_apply +`, + ); + + expect(result.status, output).toBe(0); + expect(output).toContain("FACTORY_GATES_OK"); + expect(output).toContain("WORKLOAD_GATE_OK"); + expect(output).toContain( + "PACKAGED_LIFECYCLE systemctl enable nvidia-cdi-refresh.path nvidia-cdi-refresh.service", + ); + expect(output).toContain("PACKAGED_LIFECYCLE systemctl start nvidia-cdi-refresh.path"); + expect(output).toContain("PACKAGED_LIFECYCLE systemctl restart nvidia-cdi-refresh.service"); + expect(output).toContain("cdi=nvidia.com/gpu=all source=packaged_refresh_service"); + expect(output).toContain("DIGEST_PINNED_CDI_PROBE_OK"); + expect(output).toContain("DIGEST_PINNED_GPUS_PROBE_OK"); + expect(output.indexOf("FACTORY_GATES_OK")).toBeLessThan( + output.indexOf("PACKAGED_LIFECYCLE systemctl enable"), + ); + expect(output.indexOf("cdi=nvidia.com/gpu=all source=packaged_refresh_service")).toBeLessThan( + output.indexOf("DIGEST_PINNED_CDI_PROBE_OK"), + ); + expect(output).toContain("APPLY_RESULT=COMPLETE"); + expect(output).not.toContain("UNEXPECTED_"); + expect(output).not.toContain("nvidia-ctk cdi generate"); + expect(output).not.toContain("systemctl restart docker.service"); + expect(output).not.toContain("systemctl restart containerd.service"); + }); + + it("keeps forced factory-runtime profiles out of packaged CDI repair", () => { + const { result, output } = runSourced( + STATION_PREPARE, + ` +STATION_HOST_PROFILE=forced-factory-runtime +check_dgx_os_runtime_commands() { printf 'FACTORY_GATES_OK\n'; } +systemctl() { + case "$*" in + 'is-active --quiet containerd.service'|'is-active --quiet docker.service') return 0 ;; + *) printf 'UNEXPECTED_SYSTEMCTL %s\n' "$*"; return 1 ;; + esac +} +station_sudo_local_default_docker() { + case "$*" in + 'info'|'buildx version') return 0 ;; + *) printf 'UNEXPECTED_DOCKER %s\n' "$*"; return 1 ;; + esac +} +sudo() { + [[ "$*" == "nvidia-ctk cdi list" ]] || { printf 'UNEXPECTED_SUDO %s\n' "$*"; return 1; } +} +refresh_cdi() { printf 'UNEXPECTED_CDI_REFRESH\n'; return 1; } +verify_dgx_os_runtime_sudo +`, + ); + + expect(result.status, output).not.toBe(0); + expect(output).toContain("FACTORY_GATES_OK"); + expect(output).toMatch(/does not advertise the nvidia\.com\/gpu=all CDI device/); + expect(output).not.toContain("UNEXPECTED_"); + expect(output).not.toContain("nvidia-cdi-refresh"); + }); + it("returns a relogin result instead of requesting a reboot for factory Docker access", () => { const { result, output } = runSourced( STATION_PREPARE, @@ -796,6 +957,9 @@ run_apply require_command() { :; } acquire_sudo() { :; } common_preflight() { STATION_HOST_PROFILE=ai-developer-tools; } +sudo() { + [[ "$*" == "nvidia-ctk cdi list" ]] && printf 'nvidia.com/gpu=all\n' +} verify_dgx_os_runtime_sudo() { printf 'FACTORY_RUNTIME_VALIDATED\n'; } ensure_docker_group() { DOCKER_GROUP_ADDED=1; } run_apply From 0fd63e903d90a998a52465a67b5ee14555a826a3 Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Sat, 18 Jul 2026 11:52:53 -0700 Subject: [PATCH 2/4] chore(ci): add platform matrix SPDX metadata Signed-off-by: Senthil Ravichandran --- ci/platform-matrix.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/platform-matrix.json b/ci/platform-matrix.json index 6a602deeacf..9c21175070f 100644 --- a/ci/platform-matrix.json +++ b/ci/platform-matrix.json @@ -1,5 +1,5 @@ { - "$comment": "Single source of truth for NemoClaw launch claims and platform support. Covers platforms, inference providers, supported agents, messaging integrations, and deployment paths. Scripts read this to generate README and docs tables. QA/CI update platform/provider rows; the engineering owner reviews other rows. Docs are derived.", + "$comment": "SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\nSPDX-License-Identifier: Apache-2.0\n\nSingle source of truth for NemoClaw launch claims and platform support. Covers platforms, inference providers, supported agents, messaging integrations, and deployment paths. Scripts read this to generate README and docs tables. QA/CI update platform/provider rows; the engineering owner reviews other rows. Docs are derived.", "version": "1.1", "updated": "2026-07-17", From 7dc32f0b7d2e8c398436e426ac5d3bcd75a9316f Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sat, 18 Jul 2026 12:46:09 -0700 Subject: [PATCH 3/4] test(installer): stub CDI mutation safety gate Signed-off-by: Carlos Villela --- test/install-station-dgx-os.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/install-station-dgx-os.test.ts b/test/install-station-dgx-os.test.ts index 43ad3d329e0..ec3fa2e3e7d 100644 --- a/test/install-station-dgx-os.test.ts +++ b/test/install-station-dgx-os.test.ts @@ -863,7 +863,7 @@ station_sudo_local_default_docker() { *) printf 'UNEXPECTED_DOCKER %s\n' "$*"; return 1 ;; esac } -check_no_workloads() { printf 'WORKLOAD_GATE_OK\n'; } +require_docker_mutation_quiescence() { printf 'WORKLOAD_GATE_OK %s\n' "$1"; } sudo() { case "$*" in 'nvidia-ctk cdi list') From 8503b58d86c5b260761c1b218befc6c5ec6dcbbe Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sat, 18 Jul 2026 12:59:04 -0700 Subject: [PATCH 4/4] test(installer): cover packaged CDI failures Signed-off-by: Carlos Villela --- docs/get-started/dgx-station-preparation.mdx | 1 + scripts/prepare-dgx-station-host.sh | 4 ++ test/install-station-dgx-os.test.ts | 52 ++++++++++++++++++++ 3 files changed, 57 insertions(+) diff --git a/docs/get-started/dgx-station-preparation.mdx b/docs/get-started/dgx-station-preparation.mdx index 9870fd56182..4bac497c0c7 100644 --- a/docs/get-started/dgx-station-preparation.mdx +++ b/docs/get-started/dgx-station-preparation.mdx @@ -78,6 +78,7 @@ The AI Developer Tools path preserves those boundaries when `nvidia.com/gpu=all` When that CDI device is missing, the AI Developer Tools path first confirms that no workloads are active, then enables the packaged `nvidia-cdi-refresh.path` and `nvidia-cdi-refresh.service` units, starts the path unit, restarts the refresh service, and checks the device again. It stops with service diagnostics if packaged refresh fails or omits the device. It never generates CDI directly, rewrites the Docker runtime, or restarts Docker or containerd. +This compatibility repair is owned by the June 2026 AI Developer Tools factory-image boundary, which can leave its packaged CDI units disabled; it can be removed after repeat clean-host qualification shows that the qualified image consistently advertises `nvidia.com/gpu=all` at boot. The Colossus BaseOS path requires the exact reviewed package and failed-service fingerprints, preserves the factory kernel, driver, DKMS, Docker, and NVIDIA Container Toolkit packages, and prepares only Docker access and packaged CDI. Only the Colossus BaseOS path registers the NVIDIA Docker runtime when the launch probe proves that the runtime is missing, and it restores the prior daemon configuration if registration fails. The preparation helper writes a private audit log under `~/station-bootstrap-logs`. diff --git a/scripts/prepare-dgx-station-host.sh b/scripts/prepare-dgx-station-host.sh index cdde17b4cfc..e37820de92a 100755 --- a/scripts/prepare-dgx-station-host.sh +++ b/scripts/prepare-dgx-station-host.sh @@ -1487,6 +1487,10 @@ verify_dgx_os_runtime_sudo() { || fatal "The local Docker daemon is not reachable with sudo on the Station factory image" station_sudo_local_default_docker buildx version >/dev/null 2>&1 \ || fatal "Docker Buildx is unavailable on the Station factory image" + # The June 2026 AI Developer Tools factory image can leave its packaged CDI + # refresh units disabled. Image production owns that source state; remove + # this repair after clean-host qualification consistently supplies the CDI + # device at boot. if [[ "$STATION_HOST_PROFILE" == "ai-developer-tools" ]]; then if sudo nvidia-ctk cdi list | grep -Fxq 'nvidia.com/gpu=all'; then info "cdi=nvidia.com/gpu=all source=factory_runtime" diff --git a/test/install-station-dgx-os.test.ts b/test/install-station-dgx-os.test.ts index ec3fa2e3e7d..3174cb33a2e 100644 --- a/test/install-station-dgx-os.test.ts +++ b/test/install-station-dgx-os.test.ts @@ -917,6 +917,58 @@ run_apply expect(output).not.toContain("systemctl restart containerd.service"); }); + it.each([ + ["restart failure", "return 1", "Packaged CDI refresh failed"], + ["missing device after restart", "return 0", "did not advertise nvidia.com/gpu=all"], + ] as const)("fails closed on AI Developer Tools CDI %s", (_scenario, restartResult, error) => { + const { result, output } = runSourced( + STATION_PREPARE, + ` +require_command() { :; } +check_dgx_os_runtime_commands() { printf 'FACTORY_GATES_OK\n'; } +systemctl() { + case "$*" in + 'is-active --quiet containerd.service'|'is-active --quiet docker.service') return 0 ;; + *) printf 'UNEXPECTED_SYSTEMCTL %s\n' "$*"; return 1 ;; + esac +} +station_sudo_local_default_docker() { + case "$*" in + 'info'|'buildx version') return 0 ;; + *) printf 'UNEXPECTED_DOCKER %s\n' "$*"; return 1 ;; + esac +} +require_docker_mutation_quiescence() { printf 'WORKLOAD_GATE_OK %s\n' "$1"; } +sudo() { + case "$*" in + 'nvidia-ctk cdi list') return 0 ;; + 'systemctl enable nvidia-cdi-refresh.path nvidia-cdi-refresh.service'|'systemctl start nvidia-cdi-refresh.path') return 0 ;; + 'systemctl restart nvidia-cdi-refresh.service') ${restartResult} ;; + 'systemctl status nvidia-cdi-refresh.service --no-pager'|'journalctl -u nvidia-cdi-refresh.service --no-pager -n 50') + printf 'SERVICE_DIAGNOSTICS %s\n' "$*" + ;; + *) printf 'UNEXPECTED_SUDO %s\n' "$*"; return 1 ;; + esac +} +nvidia-ctk() { return 0; } +ensure_dgx_os_acceptance_image() { printf 'UNEXPECTED_ACCEPTANCE_IMAGE\n'; return 1; } +run_dgx_os_cdi_test_sudo() { printf 'UNEXPECTED_CDI_PROBE\n'; return 1; } +run_dgx_os_gpus_test_sudo() { printf 'UNEXPECTED_GPUS_PROBE\n'; return 1; } +STATION_HOST_PROFILE=ai-developer-tools +verify_dgx_os_runtime_sudo +`, + ); + + expect(result.status, output).not.toBe(0); + expect(output).toContain("FACTORY_GATES_OK"); + expect(output).toContain("WORKLOAD_GATE_OK"); + expect(output).toContain("SERVICE_DIAGNOSTICS"); + expect(output).toContain(error); + expect(output).not.toContain("UNEXPECTED_"); + expect(output).not.toContain("systemctl restart docker.service"); + expect(output).not.toContain("systemctl restart containerd.service"); + }); + it("keeps forced factory-runtime profiles out of packaged CDI repair", () => { const { result, output } = runSourced( STATION_PREPARE,