From ef33b2c19a81799096c937546a1d092b98fcc923 Mon Sep 17 00:00:00 2001 From: Tinson Lai Date: Mon, 20 Jul 2026 12:30:40 +0000 Subject: [PATCH] fix(install): unblock forced DGX Station preparation on real GB300 hosts Accept GB300 PCI device id 0x31c3 alongside 0x31c2 so a genuine GB300 passes the forced factory-runtime identity check. Tolerate the InfiniBand communication manager and realtime audio scheduler as failed units on the --force-station-install path, since neither affects GPU or container capability; required GPU, container, and preparation-critical service checks still block. Signed-off-by: Tinson Lai --- docs/get-started/dgx-station-preparation.mdx | 1 + scripts/prepare-dgx-station-host.sh | 34 +++++++-- ...install-station-forced-preparation.test.ts | 75 +++++++++++++++++++ .../install-station-platform-identity.test.ts | 16 +++- 4 files changed, 118 insertions(+), 8 deletions(-) create mode 100644 test/install-station-forced-preparation.test.ts diff --git a/docs/get-started/dgx-station-preparation.mdx b/docs/get-started/dgx-station-preparation.mdx index 4bac497c0c7..c3b6d95d011 100644 --- a/docs/get-started/dgx-station-preparation.mdx +++ b/docs/get-started/dgx-station-preparation.mdx @@ -65,6 +65,7 @@ After successful registration, the runtime remains configured until the same acc Preparation requires Secure Boot to be disabled, matching headers for the running kernel, at least 20 GiB free on the root filesystem, and no active agent, inference, or Docker workloads. It also stops when systemd reports a failed unit unless the unit matches an exact, condition-qualified state from the generic Station image: the pinned OEM `cloud-init` telemetry failure, a network-wait failure while current network health is established, masked `fwupd`, or an SSSD socket on a host without SSSD configuration. Any other failed unit blocks preparation for administrator review. +On the `--force-station-install` path, preparation also tolerates the InfiniBand communication manager (`ibacm`) and the realtime audio scheduler (`rtkit-daemon`) as failed units, because neither affects GPU or container capability; the required GPU, container, and preparation-critical service checks still apply. It does not install a host CUDA toolkit or Docker Compose. If any other existing prerequisite version differs, preparation stops instead of changing it automatically. After changing pinned packages, the installer exits with status `10`; reboot, sign in, and run the printed command, which pins the exact accepted NemoClaw commit before resuming express setup. diff --git a/scripts/prepare-dgx-station-host.sh b/scripts/prepare-dgx-station-host.sh index 57f808c1765..d303fc9a83b 100755 --- a/scripts/prepare-dgx-station-host.sh +++ b/scripts/prepare-dgx-station-host.sh @@ -10,7 +10,7 @@ readonly REBOOT_REQUIRED_EXIT=10 readonly LOGIN_REQUIRED_EXIT=11 readonly MIN_FREE_KIB=$((20 * 1024 * 1024)) readonly GB300_PCI_VENDOR="0x10de" -readonly GB300_PCI_DEVICE="0x31c2" +readonly -a GB300_PCI_DEVICES=("0x31c2" "0x31c3") readonly GB300_PCI_CLASS_PREFIX="0x03" STATION_HOST_PROFILE="generic-ubuntu" FORCE_STATION_INSTALL=0 @@ -336,6 +336,22 @@ normalize_nvidia_pci_bus_id() { printf '%s:%s' "$domain" "$rest" } +gb300_pci_device_is_known() { + local candidate=$1 known + for known in "${GB300_PCI_DEVICES[@]}"; do + [[ "$candidate" == "$known" ]] && return 0 + done + return 1 +} + +gb300_pci_device_display() { + local rendered="" device + for device in "${GB300_PCI_DEVICES[@]}"; do + rendered+="${rendered:+/}${device#0x}" + done + printf '%s' "$rendered" +} + station_pci_device_is_gb300() { local bus_id=$1 pci_root=${2:-/sys/bus/pci/devices} pci_path vendor device class bus_id="$(normalize_nvidia_pci_bus_id "$bus_id")" || return 1 @@ -347,9 +363,9 @@ station_pci_device_is_gb300() { IFS= read -r vendor <"$pci_path/vendor" || return 1 IFS= read -r device <"$pci_path/device" || return 1 IFS= read -r class <"$pci_path/class" || return 1 - [[ "$vendor" == "$GB300_PCI_VENDOR" && - "$device" == "$GB300_PCI_DEVICE" && - "$class" == "${GB300_PCI_CLASS_PREFIX}"* ]] + [[ "$vendor" == "$GB300_PCI_VENDOR" ]] || return 1 + gb300_pci_device_is_known "$device" || return 1 + [[ "$class" == "${GB300_PCI_CLASS_PREFIX}"* ]] } station_has_exact_gb300_pci_gpu() { @@ -521,6 +537,12 @@ is_qualified_factory_failed_unit() { *) return 1 ;; esac ;; + forced-factory-runtime) + case "${1:-}" in + ibacm.service | rtkit-daemon.service) return 0 ;; + *) return 1 ;; + esac + ;; *) return 1 ;; esac } @@ -669,7 +691,7 @@ check_platform() { case "$release_state" in generic-ubuntu) station_has_exact_gb300_pci_gpu "$(station_pci_devices_path)" \ - || fatal "Expected an NVIDIA GB300 PCI GPU (${GB300_PCI_VENDOR#0x}:${GB300_PCI_DEVICE#0x}) before generic Ubuntu preparation" + || fatal "Expected an NVIDIA GB300 PCI GPU (${GB300_PCI_VENDOR#0x}:$(gb300_pci_device_display)) before generic Ubuntu preparation" STATION_HOST_PROFILE="generic-ubuntu" ;; supported-dgx-os) STATION_HOST_PROFILE="stock-dgx-os" ;; @@ -678,7 +700,7 @@ check_platform() { *) if ((FORCE_STATION_INSTALL == 1)); then station_has_exact_gb300_pci_gpu "$(station_pci_devices_path)" \ - || fatal "Expected an NVIDIA GB300 PCI GPU (${GB300_PCI_VENDOR#0x}:${GB300_PCI_DEVICE#0x}) before forced factory-runtime validation" + || fatal "Expected an NVIDIA GB300 PCI GPU (${GB300_PCI_VENDOR#0x}:$(gb300_pci_device_display)) before forced factory-runtime validation" STATION_HOST_PROFILE="forced-factory-runtime" warn "DGX release metadata allowlist bypassed by explicit --force-station-install intent; all hardware and factory-runtime health checks remain required" else diff --git a/test/install-station-forced-preparation.test.ts b/test/install-station-forced-preparation.test.ts new file mode 100644 index 00000000000..a9fcb06fff9 --- /dev/null +++ b/test/install-station-forced-preparation.test.ts @@ -0,0 +1,75 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { TEST_SYSTEM_PATH } from "./helpers/installer-sourced-env"; + +const REPO_ROOT = path.resolve(import.meta.dirname, ".."); +const STATION_PREPARE = path.join(REPO_ROOT, "scripts", "prepare-dgx-station-host.sh"); + +function runSourced(body: string, extraEnv: Record = {}) { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-station-forced-")); + const result = spawnSync( + "bash", + ["--noprofile", "--norc", "-c", `source "$STATION_PREPARE" >/dev/null\n${body}`], + { + cwd: REPO_ROOT, + encoding: "utf-8", + env: { + HOME: home, + PATH: TEST_SYSTEM_PATH, + STATION_PREPARE, + ...extraEnv, + }, + timeout: 15_000, + killSignal: "SIGKILL", + }, + ); + return { result, output: `${result.stdout}${result.stderr}` }; +} + +describe("DGX Station forced-factory-runtime preparation", () => { + it.each([ + ["ibacm.service"], + ["rtkit-daemon.service"], + ])("tolerates the unrelated failed unit %s (#7236)", (unit) => { + const tolerated = runSourced( + ` +STATION_HOST_PROFILE=forced-factory-runtime +systemctl() { printf '${unit} loaded failed failed Unrelated\n'; } +check_failed_units +`, + ); + expect(tolerated.result.status, tolerated.output).toBe(0); + expect(tolerated.output).toMatch( + new RegExp(`condition-qualified forced-factory-runtime failed unit: ${unit}`), + ); + }); + + it("still blocks unrelated and preparation-critical failed units (#7236)", () => { + const unrelated = runSourced( + ` +STATION_HOST_PROFILE=forced-factory-runtime +systemctl() { printf 'ssh.service loaded failed failed SSH\n'; } +check_failed_units +`, + ); + expect(unrelated.result.status, unrelated.output).not.toBe(0); + expect(unrelated.output).toMatch(/unqualified failed unit: ssh.service/); + expect(unrelated.output).toMatch(/Unqualified failed system units block Station preparation/); + + const critical = runSourced( + ` +STATION_HOST_PROFILE=forced-factory-runtime +systemctl() { printf 'containerd.service loaded failed failed containerd\n'; } +check_failed_units +`, + ); + expect(critical.result.status, critical.output).not.toBe(0); + expect(critical.output).toMatch(/failed preparation-critical unit: containerd.service/); + }); +}); diff --git a/test/install-station-platform-identity.test.ts b/test/install-station-platform-identity.test.ts index 0f62f7e7ed2..abba788c2f5 100644 --- a/test/install-station-platform-identity.test.ts +++ b/test/install-station-platform-identity.test.ts @@ -82,6 +82,18 @@ describe("DGX Station platform identity", () => { expect(result.status, output).toBe(0); }); + it.each([ + ["0x31c2", "0x31c2"], + ["0x31c3", "0x31c3"], + ])("accepts GB300 PCI device id %s (#7235)", (_scenario, device) => { + const pciRoot = writePciIdentityFixture("0x10de", device); + const { result, output } = runStationPrepare(`station_has_exact_gb300_pci_gpu "$PCI_ROOT"`, { + PCI_ROOT: pciRoot, + }); + + expect(result.status, output).toBe(0); + }); + it("selects the GB300 by PCI identity when an auxiliary GPU has the same name", () => { const pciRoot = writePciIdentityFixture(); const { result, output } = runStationPrepare( @@ -180,7 +192,7 @@ run_apply ); expect(result.status, output).not.toBe(0); - expect(output).toContain("Expected an NVIDIA GB300 PCI GPU (10de:31c2)"); + expect(output).toContain("Expected an NVIDIA GB300 PCI GPU (10de:31c2/31c3)"); expect(output).not.toContain("UNEXPECTED_MUTATION"); }); @@ -226,7 +238,7 @@ run_apply ); expect(result.status, output).not.toBe(0); - expect(output).toContain("Expected an NVIDIA GB300 PCI GPU (10de:31c2)"); + expect(output).toContain("Expected an NVIDIA GB300 PCI GPU (10de:31c2/31c3)"); expect(output).not.toContain("UNEXPECTED_MUTATION"); }); });