Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions docs/get-started/dgx-station-preparation.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ After successful registration, the runtime remains configured until the same acc
Preparation requires Secure Boot to be disabled, matching headers for the running kernel, at least 20 GiB free on the root filesystem, and no active agent, inference, or Docker workloads.
It also stops when systemd reports a failed unit unless the unit matches an exact, condition-qualified state from the generic Station image: the pinned OEM `cloud-init` telemetry failure, a network-wait failure while current network health is established, masked `fwupd`, or an SSSD socket on a host without SSSD configuration.
Any other failed unit blocks preparation for administrator review.
On the `--force-station-install` path, preparation also tolerates the InfiniBand communication manager (`ibacm`) and the realtime audio scheduler (`rtkit-daemon`) as failed units, because neither affects GPU or container capability; the required GPU, container, and preparation-critical service checks still apply.
It does not install a host CUDA toolkit or Docker Compose.
If any other existing prerequisite version differs, preparation stops instead of changing it automatically.
After changing pinned packages, the installer exits with status `10`; reboot, sign in, and run the printed command, which pins the exact accepted NemoClaw commit before resuming express setup.
Expand Down
34 changes: 28 additions & 6 deletions scripts/prepare-dgx-station-host.sh
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ readonly REBOOT_REQUIRED_EXIT=10
readonly LOGIN_REQUIRED_EXIT=11
readonly MIN_FREE_KIB=$((20 * 1024 * 1024))
readonly GB300_PCI_VENDOR="0x10de"
readonly GB300_PCI_DEVICE="0x31c2"
readonly -a GB300_PCI_DEVICES=("0x31c2" "0x31c3")
readonly GB300_PCI_CLASS_PREFIX="0x03"
STATION_HOST_PROFILE="generic-ubuntu"
FORCE_STATION_INSTALL=0
Expand Down Expand Up @@ -336,6 +336,22 @@ normalize_nvidia_pci_bus_id() {
printf '%s:%s' "$domain" "$rest"
}

gb300_pci_device_is_known() {
local candidate=$1 known
for known in "${GB300_PCI_DEVICES[@]}"; do
[[ "$candidate" == "$known" ]] && return 0
done
return 1
}

gb300_pci_device_display() {
local rendered="" device
for device in "${GB300_PCI_DEVICES[@]}"; do
rendered+="${rendered:+/}${device#0x}"
done
printf '%s' "$rendered"
}

station_pci_device_is_gb300() {
local bus_id=$1 pci_root=${2:-/sys/bus/pci/devices} pci_path vendor device class
bus_id="$(normalize_nvidia_pci_bus_id "$bus_id")" || return 1
Expand All @@ -347,9 +363,9 @@ station_pci_device_is_gb300() {
IFS= read -r vendor <"$pci_path/vendor" || return 1
IFS= read -r device <"$pci_path/device" || return 1
IFS= read -r class <"$pci_path/class" || return 1
[[ "$vendor" == "$GB300_PCI_VENDOR" &&
"$device" == "$GB300_PCI_DEVICE" &&
"$class" == "${GB300_PCI_CLASS_PREFIX}"* ]]
[[ "$vendor" == "$GB300_PCI_VENDOR" ]] || return 1
gb300_pci_device_is_known "$device" || return 1
[[ "$class" == "${GB300_PCI_CLASS_PREFIX}"* ]]
}

station_has_exact_gb300_pci_gpu() {
Expand Down Expand Up @@ -521,6 +537,12 @@ is_qualified_factory_failed_unit() {
*) return 1 ;;
esac
;;
forced-factory-runtime)
case "${1:-}" in
ibacm.service | rtkit-daemon.service) return 0 ;;
*) return 1 ;;
esac
;;
*) return 1 ;;
esac
}
Expand Down Expand Up @@ -669,7 +691,7 @@ check_platform() {
case "$release_state" in
generic-ubuntu)
station_has_exact_gb300_pci_gpu "$(station_pci_devices_path)" \
|| fatal "Expected an NVIDIA GB300 PCI GPU (${GB300_PCI_VENDOR#0x}:${GB300_PCI_DEVICE#0x}) before generic Ubuntu preparation"
|| fatal "Expected an NVIDIA GB300 PCI GPU (${GB300_PCI_VENDOR#0x}:$(gb300_pci_device_display)) before generic Ubuntu preparation"
STATION_HOST_PROFILE="generic-ubuntu"
;;
supported-dgx-os) STATION_HOST_PROFILE="stock-dgx-os" ;;
Expand All @@ -678,7 +700,7 @@ check_platform() {
*)
if ((FORCE_STATION_INSTALL == 1)); then
station_has_exact_gb300_pci_gpu "$(station_pci_devices_path)" \
|| fatal "Expected an NVIDIA GB300 PCI GPU (${GB300_PCI_VENDOR#0x}:${GB300_PCI_DEVICE#0x}) before forced factory-runtime validation"
|| fatal "Expected an NVIDIA GB300 PCI GPU (${GB300_PCI_VENDOR#0x}:$(gb300_pci_device_display)) before forced factory-runtime validation"
STATION_HOST_PROFILE="forced-factory-runtime"
warn "DGX release metadata allowlist bypassed by explicit --force-station-install intent; all hardware and factory-runtime health checks remain required"
else
Expand Down
75 changes: 75 additions & 0 deletions test/install-station-forced-preparation.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { spawnSync } from "node:child_process";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { describe, expect, it } from "vitest";
import { TEST_SYSTEM_PATH } from "./helpers/installer-sourced-env";

const REPO_ROOT = path.resolve(import.meta.dirname, "..");
const STATION_PREPARE = path.join(REPO_ROOT, "scripts", "prepare-dgx-station-host.sh");

function runSourced(body: string, extraEnv: Record<string, string> = {}) {
const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-station-forced-"));
const result = spawnSync(
"bash",
["--noprofile", "--norc", "-c", `source "$STATION_PREPARE" >/dev/null\n${body}`],
{
cwd: REPO_ROOT,
encoding: "utf-8",
env: {
HOME: home,
PATH: TEST_SYSTEM_PATH,
STATION_PREPARE,
...extraEnv,
},
timeout: 15_000,
killSignal: "SIGKILL",
},
);
return { result, output: `${result.stdout}${result.stderr}` };
}

describe("DGX Station forced-factory-runtime preparation", () => {
it.each([
["ibacm.service"],
["rtkit-daemon.service"],
])("tolerates the unrelated failed unit %s (#7236)", (unit) => {
const tolerated = runSourced(
`
STATION_HOST_PROFILE=forced-factory-runtime
systemctl() { printf '${unit} loaded failed failed Unrelated\n'; }
check_failed_units
`,
);
expect(tolerated.result.status, tolerated.output).toBe(0);
expect(tolerated.output).toMatch(
new RegExp(`condition-qualified forced-factory-runtime failed unit: ${unit}`),
);
});

it("still blocks unrelated and preparation-critical failed units (#7236)", () => {
const unrelated = runSourced(
`
STATION_HOST_PROFILE=forced-factory-runtime
systemctl() { printf 'ssh.service loaded failed failed SSH\n'; }
check_failed_units
`,
);
expect(unrelated.result.status, unrelated.output).not.toBe(0);
expect(unrelated.output).toMatch(/unqualified failed unit: ssh.service/);
expect(unrelated.output).toMatch(/Unqualified failed system units block Station preparation/);

const critical = runSourced(
`
STATION_HOST_PROFILE=forced-factory-runtime
systemctl() { printf 'containerd.service loaded failed failed containerd\n'; }
check_failed_units
`,
);
expect(critical.result.status, critical.output).not.toBe(0);
expect(critical.output).toMatch(/failed preparation-critical unit: containerd.service/);
});
});
16 changes: 14 additions & 2 deletions test/install-station-platform-identity.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,18 @@ describe("DGX Station platform identity", () => {
expect(result.status, output).toBe(0);
});

it.each([
["0x31c2", "0x31c2"],
["0x31c3", "0x31c3"],
])("accepts GB300 PCI device id %s (#7235)", (_scenario, device) => {
const pciRoot = writePciIdentityFixture("0x10de", device);
const { result, output } = runStationPrepare(`station_has_exact_gb300_pci_gpu "$PCI_ROOT"`, {
PCI_ROOT: pciRoot,
});

expect(result.status, output).toBe(0);
});

it("selects the GB300 by PCI identity when an auxiliary GPU has the same name", () => {
const pciRoot = writePciIdentityFixture();
const { result, output } = runStationPrepare(
Expand Down Expand Up @@ -180,7 +192,7 @@ run_apply
);

expect(result.status, output).not.toBe(0);
expect(output).toContain("Expected an NVIDIA GB300 PCI GPU (10de:31c2)");
expect(output).toContain("Expected an NVIDIA GB300 PCI GPU (10de:31c2/31c3)");
expect(output).not.toContain("UNEXPECTED_MUTATION");
});

Expand Down Expand Up @@ -226,7 +238,7 @@ run_apply
);

expect(result.status, output).not.toBe(0);
expect(output).toContain("Expected an NVIDIA GB300 PCI GPU (10de:31c2)");
expect(output).toContain("Expected an NVIDIA GB300 PCI GPU (10de:31c2/31c3)");
expect(output).not.toContain("UNEXPECTED_MUTATION");
});
});
Loading