diff --git a/scripts/checks/vitest-project-overlap.mts b/scripts/checks/vitest-project-overlap.mts index ddb30a5cf66..ecc6abd57d1 100644 --- a/scripts/checks/vitest-project-overlap.mts +++ b/scripts/checks/vitest-project-overlap.mts @@ -47,7 +47,9 @@ const INSTALLER_INTEGRATION_TESTS = new Set([ "test/install-preflight-docker-bootstrap.test.ts", "test/install-preflight.test.ts", "test/install-station-dgx-os.test.ts", + "test/install-station-docker-repository.test.ts", "test/install-station-host-preparation.test.ts", + "test/install-station-package-transaction.test.ts", ]); function normalizeRepoPath(file: string): string { diff --git a/scripts/install.sh b/scripts/install.sh index fc1194739f9..4d2876550bd 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -3499,7 +3499,25 @@ run_station_host_preparation() { if [ "${FORCE_STATION_INSTALL:-}" = "1" ]; then helper_args+=(--force-station-install) fi - bash "$helper" "${helper_args[@]}" + bash "$helper" "${helper_args[@]}" 2>&1 | filter_station_host_preparation_output +} + +filter_station_host_preparation_output() { + local line detail + while IFS= read -r line; do + case "$line" in + *" version="*" log="*) + info "DGX Station host preparation log: ${line##* log=}" + ;; + *" WARNING: "*) + detail="${line#* WARNING: }" + warn "$detail" + ;; + *" ERROR: "*) + printf '%s\n' "$line" >&2 + ;; + esac + done } ensure_station_express_host() { diff --git a/scripts/prepare-dgx-station-host.sh b/scripts/prepare-dgx-station-host.sh index e37820de92a..60e7e11b3ab 100755 --- a/scripts/prepare-dgx-station-host.sh +++ b/scripts/prepare-dgx-station-host.sh @@ -254,6 +254,9 @@ LOG_FILE="" DOCKER_GROUP_ADDED=0 CDI_LIFECYCLE_READY=0 NETWORK_VALIDATED=0 +PACKAGE_TRANSACTION_SPECS=() +APT_TRANSACTION_GUARD_DIR="" +APT_TRANSACTION_HOOK="" GPU_ROWS_ERROR="" info() { @@ -320,19 +323,39 @@ is_station_gb300_product() { "$product" =~ (^|[^[:alnum:]])[Gg][Bb]300([^[:alnum:]]|$) ]] } +normalize_nvidia_pci_bus_id() { + local bus_id domain rest + bus_id="$(printf '%s' "$1" | tr '[:upper:]' '[:lower:]')" + bus_id="${bus_id//[[:space:]]/}" + [[ "$bus_id" =~ ^([0-9a-f]{4}|[0-9a-f]{8}):[0-9a-f]{2}:[0-9a-f]{2}\.[0-7]$ ]] || return 1 + domain="${bus_id%%:*}" + rest="${bus_id#*:}" + if ((${#domain} == 8)); then + domain="${domain:4}" + fi + printf '%s:%s' "$domain" "$rest" +} + +station_pci_device_is_gb300() { + local bus_id=$1 pci_root=${2:-/sys/bus/pci/devices} pci_path vendor device class + bus_id="$(normalize_nvidia_pci_bus_id "$bus_id")" || return 1 + pci_path="${pci_root}/${bus_id}" + [[ -d "$pci_path" && + -r "$pci_path/vendor" && + -r "$pci_path/device" && + -r "$pci_path/class" ]] || return 1 + IFS= read -r vendor <"$pci_path/vendor" || return 1 + IFS= read -r device <"$pci_path/device" || return 1 + IFS= read -r class <"$pci_path/class" || return 1 + [[ "$vendor" == "$GB300_PCI_VENDOR" && + "$device" == "$GB300_PCI_DEVICE" && + "$class" == "${GB300_PCI_CLASS_PREFIX}"* ]] +} + station_has_exact_gb300_pci_gpu() { - local pci_root=${1:-/sys/bus/pci/devices} pci_path vendor device class + local pci_root=${1:-/sys/bus/pci/devices} pci_path for pci_path in "$pci_root"/*; do - [[ -d "$pci_path" && - -r "$pci_path/vendor" && - -r "$pci_path/device" && - -r "$pci_path/class" ]] || continue - IFS= read -r vendor <"$pci_path/vendor" || continue - IFS= read -r device <"$pci_path/device" || continue - IFS= read -r class <"$pci_path/class" || continue - [[ "$vendor" == "$GB300_PCI_VENDOR" && - "$device" == "$GB300_PCI_DEVICE" && - "$class" == "${GB300_PCI_CLASS_PREFIX}"* ]] && return 0 + station_pci_device_is_gb300 "${pci_path##*/}" "$pci_root" && return 0 done return 1 } @@ -930,11 +953,19 @@ exit_reboot_required() { } loaded_driver_version() { - local loaded + local rows row bus_id driver pci_root command -v nvidia-smi >/dev/null 2>&1 || return 0 - loaded="$(nvidia-smi --query-gpu=driver_version --format=csv,noheader 2>/dev/null | head -n1 | tr -d '[:space:]')" \ + rows="$(nvidia-smi --query-gpu=pci.bus_id,driver_version --format=csv,noheader 2>/dev/null)" \ || return 0 - printf '%s' "$loaded" + pci_root="$(station_pci_devices_path)" + while IFS= read -r row; do + [[ -n "${row//[[:space:]]/}" ]] || continue + IFS=',' read -r bus_id driver <<<"$row" + bus_id="$(normalize_nvidia_pci_bus_id "$bus_id")" || continue + station_pci_device_is_gb300 "$bus_id" "$pci_root" || continue + printf '%s' "${driver//[[:space:]]/}" + return 0 + done <<<"$rows" } driver_is_loaded() { @@ -1149,13 +1180,48 @@ install_exact_file_or_reuse() { info "${label}=installed path=${target}" } +ensure_docker_repository_source() { + local docker_asc=$1 docker_gpg=$2 docker_gpg_list=$3 docker_asc_list=$4 + local source_target=/etc/apt/sources.list.d/docker.list + local gpg_key_target=/etc/apt/keyrings/docker.gpg + local asc_key_target=/etc/apt/keyrings/docker.asc + + sudo test ! -L "$source_target" \ + || fatal "Docker repository source must not be a symbolic link: ${source_target}" + if ! sudo test -e "$source_target"; then + install_exact_file_or_reuse "$docker_gpg" "$gpg_key_target" 0644 docker_repository_key + install_exact_file_or_reuse "$docker_gpg_list" "$source_target" 0644 docker_repository_source + return 0 + fi + + assert_root_regular_file_safe "$source_target" 0644 "Docker repository source" + if sudo cmp -s "$docker_gpg_list" "$source_target"; then + assert_root_regular_file_safe "$gpg_key_target" 0644 "Docker repository key" + sudo cmp -s "$docker_gpg" "$gpg_key_target" \ + || fatal "Existing Docker repository key differs from the verified dearmored key: ${gpg_key_target}" + info "docker_repository_source=exact path=${source_target}" + return 0 + fi + + if sudo cmp -s "$docker_asc_list" "$source_target"; then + assert_root_regular_file_safe "$asc_key_target" 0644 "Docker repository ASCII key" + sudo cmp -s "$docker_asc" "$asc_key_target" \ + || fatal "Existing Docker repository ASCII key differs from the verified key: ${asc_key_target}" + info "docker_repository_source=verified_compatible path=${source_target}" + return 0 + fi + + fatal "Existing Docker repository source differs from the validated .gpg and .asc forms; refusing to overwrite ${source_target}" +} + configure_repositories() { - local tmp cuda_deb docker_asc docker_gpg docker_list + local tmp cuda_deb docker_asc docker_gpg docker_gpg_list docker_asc_list tmp="$(mktemp -d)" cuda_deb="${tmp}/cuda-keyring.deb" docker_asc="${tmp}/docker.asc" docker_gpg="${tmp}/docker.gpg" - docker_list="${tmp}/docker.list" + docker_gpg_list="${tmp}/docker-gpg.list" + docker_asc_list="${tmp}/docker-asc.list" info "Downloading and verifying official repository keys" ensure_cuda_keyring "$cuda_deb" @@ -1166,45 +1232,299 @@ configure_repositories() { gpg --batch --yes --dearmor --output "$docker_gpg" "$docker_asc" ensure_root_directory_safe /etc/apt/keyrings /etc/apt 0755 "Docker repository key directory" assert_root_directory_safe /etc/apt/sources.list.d "Docker repository source directory" - install_exact_file_or_reuse "$docker_gpg" /etc/apt/keyrings/docker.gpg 0644 docker_repository_key printf '%s\n' \ 'deb [arch=arm64 signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu noble stable' \ - >"$docker_list" - install_exact_file_or_reuse "$docker_list" /etc/apt/sources.list.d/docker.list 0644 docker_repository_source + >"$docker_gpg_list" + printf '%s\n' \ + 'deb [arch=arm64 signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu noble stable' \ + >"$docker_asc_list" + ensure_docker_repository_source "$docker_asc" "$docker_gpg" "$docker_gpg_list" "$docker_asc_list" rm -rf "$tmp" info "repository_keys=verified" } +collect_package_transaction_specs() { + local spec state + PACKAGE_TRANSACTION_SPECS=() + for spec in "${PACKAGE_SPECS[@]}"; do + state="$(package_state "$spec")" + case "$state" in + missing | approved-transition) PACKAGE_TRANSACTION_SPECS+=("$spec") ;; + exact) ;; + *) fatal "Package transaction contains an unapproved prerequisite state: ${spec} (${state})" ;; + esac + done + ((${#PACKAGE_TRANSACTION_SPECS[@]} > 0)) \ + || fatal "Package transaction has no missing or approved-transition prerequisites" +} + validate_package_availability() { local spec - for spec in "${PACKAGE_SPECS[@]}"; do + for spec in "$@"; do apt-cache show "$spec" >/dev/null 2>&1 || fatal "Exact package version is unavailable: ${spec}" done info "exact_package_versions=available" } +apt_guard_fatal() { + printf 'APT transaction guard: %s\n' "$*" >&2 + return 1 +} + +validate_apt_preinstall_plan() { + local targets_file=$1 line package old_version old_arch old_multiarch direction + local new_version new_arch new_multiarch action extra record_key + local target expected allowed_old native_arch target_line target_expected target_allowed_old target_native_arch + local manifest_native_arch="" + local configured='|' changed='|' seen_targets='|' target_names='|' + [[ -r "$targets_file" && -f "$targets_file" && ! -L "$targets_file" ]] \ + || apt_guard_fatal "target manifest is unavailable or unsafe: ${targets_file}" || return + + while IFS='|' read -r target expected allowed_old native_arch extra; do + [[ -n "$target" && -n "$expected" && -n "$native_arch" && -z "$extra" ]] \ + || apt_guard_fatal "target manifest contains an invalid record" || return + [[ "$target" =~ ^[a-z0-9][a-z0-9+.-]+$ ]] \ + || apt_guard_fatal "target manifest contains an invalid package name: ${target}" || return + [[ "$expected" != *[[:space:]]* && "$expected" != *"|"* && + "$allowed_old" != *[[:space:]]* && "$allowed_old" != *"|"* && + "$native_arch" =~ ^[a-z0-9][a-z0-9-]*$ ]] \ + || apt_guard_fatal "target manifest contains an invalid version for ${target}" || return + [[ "$target_names" != *"|${target}|"* ]] \ + || apt_guard_fatal "target manifest repeats package ${target}" || return + if [[ -z "$manifest_native_arch" ]]; then + manifest_native_arch=$native_arch + else + [[ "$native_arch" == "$manifest_native_arch" ]] \ + || apt_guard_fatal "target manifest mixes native architectures" || return + fi + target_names="${target_names}${target}|" + done <"$targets_file" + [[ "$target_names" != "|" ]] || apt_guard_fatal "target manifest is empty" || return + + IFS= read -r line || apt_guard_fatal "APT omitted the pre-install protocol header" || return + [[ "$line" == "VERSION 3" ]] \ + || apt_guard_fatal "APT pre-install protocol must be VERSION 3, found: ${line}" || return + while IFS= read -r line; do + [[ -n "$line" ]] || break + done + [[ -z "$line" ]] || apt_guard_fatal "APT pre-install protocol omitted its record separator" || return + + while read -r package old_version old_arch old_multiarch direction new_version new_arch new_multiarch action extra; do + [[ -n "$package" && -n "$old_version" && -n "$old_arch" && -n "$old_multiarch" && + -n "$direction" && -n "$new_version" && -n "$new_arch" && -n "$new_multiarch" && + -n "$action" && -z "$extra" ]] \ + || apt_guard_fatal "APT emitted a malformed package action" || return + [[ "$package" =~ ^[a-z0-9][a-z0-9+.-]+$ ]] \ + || apt_guard_fatal "APT emitted an invalid package name: ${package}" || return + [[ "$old_arch" == "-" || "$old_arch" =~ ^[a-z0-9][a-z0-9-]*$ ]] \ + || apt_guard_fatal "APT emitted an invalid old architecture for ${package}: ${old_arch}" || return + [[ "$new_arch" == "-" || "$new_arch" =~ ^[a-z0-9][a-z0-9-]*$ ]] \ + || apt_guard_fatal "APT emitted an invalid new architecture for ${package}: ${new_arch}" || return + [[ "$old_multiarch" =~ ^(same|foreign|allowed|none|no|-)$ && + "$new_multiarch" =~ ^(same|foreign|allowed|none|no|-)$ ]] \ + || apt_guard_fatal "APT emitted an invalid Multi-Arch field for ${package}" || return + record_key="${package}@${new_arch}" + case "$action" in + "**REMOVE**") apt_guard_fatal "APT proposed removing ${package}" || return ;; + "**CONFIGURE**") + configured="${configured}${record_key}|" + ;; + /*) + [[ "$changed" != *"|${record_key}|"* ]] \ + || apt_guard_fatal "APT proposed duplicate archive actions for ${record_key}" || return + changed="${changed}${record_key}|" + target="" + expected="" + allowed_old="" + native_arch="" + while IFS='|' read -r target_line target_expected target_allowed_old target_native_arch; do + if [[ "$target_line" == "$package" ]]; then + target=$target_line + expected=$target_expected + allowed_old=$target_allowed_old + native_arch=$target_native_arch + break + fi + done <"$targets_file" + if [[ -z "$native_arch" ]]; then + native_arch=$manifest_native_arch + fi + [[ "$new_arch" == "$native_arch" || "$new_arch" == "all" ]] \ + || apt_guard_fatal "APT selected foreign architecture ${new_arch} for ${package}; expected ${native_arch} or all" || return + if [[ -n "$target" ]]; then + [[ "$new_version" == "$expected" ]] \ + || apt_guard_fatal "APT selected ${package}=${new_version}; expected ${expected}" || return + if [[ -n "$allowed_old" ]]; then + [[ "$old_version" == "$allowed_old" && "$direction" == "<" && + ("$old_arch" == "$native_arch" || "$old_arch" == "all") ]] \ + || apt_guard_fatal "APT changed approved transition ${package} from ${old_version}; expected ${allowed_old}" || return + else + [[ "$old_version" == "-" && "$old_arch" == "-" && "$direction" == "<" ]] \ + || apt_guard_fatal "APT proposed changing retained target ${package}=${old_version}" || return + fi + seen_targets="${seen_targets}${package}|" + else + [[ "$old_version" == "-" && "$old_arch" == "-" && "$direction" == "<" ]] \ + || apt_guard_fatal "APT proposed changing retained package ${package}=${old_version}" || return + fi + ;; + *) apt_guard_fatal "APT emitted an unsupported action for ${package}: ${action}" || return ;; + esac + done + + while IFS='|' read -r target expected allowed_old native_arch; do + [[ "$seen_targets" == *"|${target}|"* ]] \ + || apt_guard_fatal "APT omitted required target ${target}=${expected}" || return + done <"$targets_file" + while [[ "$configured" != "|" ]]; do + configured="${configured#|}" + package="${configured%%|*}" + configured="|${configured#*|}" + [[ "$changed" == *"|${package}|"* ]] \ + || apt_guard_fatal "APT proposed configuring retained package ${package} without an archive action" || return + done +} + +cleanup_apt_transaction_guard() { + local guard_dir=${APT_TRANSACTION_GUARD_DIR:-} + APT_TRANSACTION_GUARD_DIR="" + APT_TRANSACTION_HOOK="" + [[ -n "$guard_dir" ]] || return 0 + if [[ ! "$guard_dir" =~ ^/run/nemoclaw-apt-transaction\.[A-Za-z0-9]+$ ]]; then + warn "refusing to clean unexpected APT transaction guard path: ${guard_dir}" + return 0 + fi + sudo rm -rf -- "$guard_dir" \ + || warn "could not remove APT transaction guard directory: ${guard_dir}" +} + +create_apt_transaction_guard() { + local spec state name expected allowed_old native_arch targets="" hook_path targets_path + native_arch="$(sudo dpkg --print-architecture)" + [[ "$native_arch" =~ ^[a-z0-9][a-z0-9-]*$ ]] \ + || fatal "Could not determine the native package architecture" + for spec in "${PACKAGE_TRANSACTION_SPECS[@]}"; do + state="$(package_state "$spec")" + name="$(package_name "$spec")" + expected="$(package_expected_version "$spec")" + case "$state" in + missing) allowed_old="" ;; + approved-transition) allowed_old="$(installed_version "$name")" ;; + *) fatal "Cannot authorize APT target ${spec} from state ${state}" ;; + esac + targets="${targets}${name}|${expected}|${allowed_old}|${native_arch}"$'\n' + done + + APT_TRANSACTION_GUARD_DIR="$(sudo mktemp -d /run/nemoclaw-apt-transaction.XXXXXXXXXX)" \ + || fatal "Could not create the root-owned APT transaction guard directory" + [[ "$APT_TRANSACTION_GUARD_DIR" =~ ^/run/nemoclaw-apt-transaction\.[A-Za-z0-9]+$ ]] \ + || fatal "APT transaction guard returned an unexpected path: ${APT_TRANSACTION_GUARD_DIR}" + hook_path="${APT_TRANSACTION_GUARD_DIR}/verify-plan" + targets_path="${APT_TRANSACTION_GUARD_DIR}/targets" + { + printf '%s\n' '#!/usr/bin/env bash' 'set -euo pipefail' + declare -f apt_guard_fatal + declare -f validate_apt_preinstall_plan + # The generated hook expands its own path at execution time. + # shellcheck disable=SC2016 + printf '%s\n' 'validate_apt_preinstall_plan "${0%/*}/targets"' + } | sudo tee "$hook_path" >/dev/null + printf '%s' "$targets" | sudo tee "$targets_path" >/dev/null + sudo chmod 0700 "$hook_path" + sudo chmod 0600 "$targets_path" + assert_root_directory_safe "$APT_TRANSACTION_GUARD_DIR" "APT transaction guard directory" + assert_root_regular_file_safe "$hook_path" 0700 "APT transaction guard" + assert_root_regular_file_safe "$targets_path" 0600 "APT transaction target manifest" + APT_TRANSACTION_HOOK=$hook_path +} + +validate_apt_simulation() { + local simulation=$1 + shift + local spec target_spec name expected actual line action version before_version + local simulated_targets='|' simulated_changes='|' + + while IFS= read -r line; do + [[ "$line" =~ ^(Inst|Conf|Remv|Purg)[[:space:]] ]] || continue + read -r action name _ <<<"$line" + case "$action" in + Remv | Purg) fatal "APT simulation proposed a package removal: ${line}" ;; + Conf) + [[ "$simulated_changes" == *"|${name}|"* ]] \ + || fatal "APT simulation proposed configuration without an approved install: ${line}" + ;; + Inst) + [[ "$simulated_changes" != *"|${name}|"* ]] \ + || fatal "APT simulation proposed a duplicate package change: ${line}" + simulated_changes="${simulated_changes}${name}|" + target_spec="" + for spec in "$@"; do + if [[ "$(package_name "$spec")" == "$name" ]]; then + target_spec=$spec + break + fi + done + if [[ -n "$target_spec" ]]; then + [[ "$line" == *"("* ]] || fatal "APT simulation omitted the target version: ${line}" + version="${line#*(}" + version="${version%%[[:space:]]*}" + version="${version%)}" + expected="$(package_expected_version "$target_spec")" + [[ "$version" == "$expected" ]] \ + || fatal "APT simulation selected ${name}=${version}; expected ${expected}" + simulated_targets="${simulated_targets}${name}|" + continue + fi + + # A target package may require a new dependency, but APT must not + # upgrade, downgrade, or reinstall any package retained from the host. + actual="$(installed_version "$name")" + before_version="${line%%(*}" + [[ -z "$actual" && "$before_version" != *"["* ]] \ + || fatal "APT simulation proposed changing retained package ${name}=${actual}: ${line}" + ;; + esac + done <<<"$simulation" + + for spec in "$@"; do + name="$(package_name "$spec")" + [[ "$simulated_targets" == *"|${name}|"* ]] \ + || fatal "APT simulation did not include required package ${spec}" + done +} + simulate_install() { local simulation - simulation="$(apt-get -s install --no-install-recommends "${PACKAGE_SPECS[@]}")" \ + [[ "$APT_TRANSACTION_HOOK" =~ ^/run/nemoclaw-apt-transaction\.[A-Za-z0-9]+/verify-plan$ ]] \ + || fatal "APT transaction guard is not ready" + simulation="$(sudo env DEBIAN_FRONTEND=noninteractive LC_ALL=C \ + apt-get -s install --no-install-recommends --no-remove \ + -o "DPkg::Pre-Install-Pkgs::=${APT_TRANSACTION_HOOK}" \ + -o "DPkg::Tools::options::${APT_TRANSACTION_HOOK}::Version=3" "$@")" \ || fatal "APT simulation failed" printf '%s\n' "$simulation" - if grep -Eq '^(Remv |Purg )' <<<"$simulation"; then - fatal "APT simulation proposed a package removal" - fi - info "apt_simulation=no_removals" + validate_apt_simulation "$simulation" "$@" + info "apt_simulation=missing_only retained_packages=unchanged" } install_packages() { + collect_package_transaction_specs configure_repositories info "Refreshing package metadata" sudo apt-get update - validate_package_availability - simulate_install + validate_package_availability "${PACKAGE_TRANSACTION_SPECS[@]}" + create_apt_transaction_guard + simulate_install "${PACKAGE_TRANSACTION_SPECS[@]}" + check_no_workloads require_docker_restart_quiescence "Station prerequisite package installation" - info "Installing pinned Station prerequisites" - sudo env DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - "${PACKAGE_SPECS[@]}" + info "Installing missing pinned Station prerequisites" + sudo env DEBIAN_FRONTEND=noninteractive LC_ALL=C \ + apt-get install -y --no-install-recommends --no-remove \ + -o "DPkg::Pre-Install-Pkgs::=${APT_TRANSACTION_HOOK}" \ + -o "DPkg::Tools::options::${APT_TRANSACTION_HOOK}::Version=3" \ + "${PACKAGE_TRANSACTION_SPECS[@]}" + cleanup_apt_transaction_guard local spec for spec in "${PACKAGE_SPECS[@]}"; do @@ -1274,7 +1594,7 @@ ensure_acceptance_image() { run_cdi_test_sudo() { local rows rows="$(sudo docker run --rm --device nvidia.com/gpu=all "$ACCEPTANCE_IMAGE" nvidia-smi \ - --query-gpu=name,driver_version,ecc.errors.corrected.volatile.total,ecc.errors.uncorrected.volatile.total \ + --query-gpu=pci.bus_id,name,driver_version,ecc.errors.corrected.volatile.total,ecc.errors.uncorrected.volatile.total \ --format=csv,noheader,nounits)" || return 1 gpu_rows_are_valid "$rows" || { warn "CDI container probe did not expose the qualified GB300: ${GPU_ROWS_ERROR}" @@ -1285,7 +1605,7 @@ run_cdi_test_sudo() { run_gpus_test_sudo() { local rows rows="$(sudo docker run --rm --gpus all "$ACCEPTANCE_IMAGE" nvidia-smi \ - --query-gpu=name,driver_version,ecc.errors.corrected.volatile.total,ecc.errors.uncorrected.volatile.total \ + --query-gpu=pci.bus_id,name,driver_version,ecc.errors.corrected.volatile.total,ecc.errors.uncorrected.volatile.total \ --format=csv,noheader,nounits)" || return 1 gpu_rows_are_valid "$rows" || { warn "Docker --gpus container probe did not expose the qualified GB300: ${GPU_ROWS_ERROR}" @@ -1296,7 +1616,7 @@ run_gpus_test_sudo() { run_cdi_test_user() { local rows rows="$(docker run --rm --device nvidia.com/gpu=all "$ACCEPTANCE_IMAGE" nvidia-smi \ - --query-gpu=name,driver_version,ecc.errors.corrected.volatile.total,ecc.errors.uncorrected.volatile.total \ + --query-gpu=pci.bus_id,name,driver_version,ecc.errors.corrected.volatile.total,ecc.errors.uncorrected.volatile.total \ --format=csv,noheader,nounits)" || return 1 gpu_rows_are_valid "$rows" } @@ -1304,7 +1624,7 @@ run_cdi_test_user() { run_gpus_test_user() { local rows rows="$(docker run --rm --gpus all "$ACCEPTANCE_IMAGE" nvidia-smi \ - --query-gpu=name,driver_version,ecc.errors.corrected.volatile.total,ecc.errors.uncorrected.volatile.total \ + --query-gpu=pci.bus_id,name,driver_version,ecc.errors.corrected.volatile.total,ecc.errors.uncorrected.volatile.total \ --format=csv,noheader,nounits)" || return 1 gpu_rows_are_valid "$rows" } @@ -1319,7 +1639,7 @@ ensure_dgx_os_acceptance_image() { run_dgx_os_cdi_test_sudo() { local rows rows="$(station_sudo_local_default_docker run --rm --device nvidia.com/gpu=all "$ACCEPTANCE_IMAGE" nvidia-smi \ - --query-gpu=name,driver_version,ecc.errors.corrected.volatile.total,ecc.errors.uncorrected.volatile.total \ + --query-gpu=pci.bus_id,name,driver_version,ecc.errors.corrected.volatile.total,ecc.errors.uncorrected.volatile.total \ --format=csv,noheader,nounits)" || return 1 gpu_rows_are_valid "$rows" || { warn "Factory-runtime CDI probe did not expose the qualified GB300: ${GPU_ROWS_ERROR}" @@ -1330,7 +1650,7 @@ run_dgx_os_cdi_test_sudo() { run_dgx_os_gpus_test_sudo() { local rows rows="$(station_sudo_local_default_docker run --rm --gpus all "$ACCEPTANCE_IMAGE" nvidia-smi \ - --query-gpu=name,driver_version,ecc.errors.corrected.volatile.total,ecc.errors.uncorrected.volatile.total \ + --query-gpu=pci.bus_id,name,driver_version,ecc.errors.corrected.volatile.total,ecc.errors.uncorrected.volatile.total \ --format=csv,noheader,nounits)" || return 1 gpu_rows_are_valid "$rows" || { warn "Factory-runtime Docker --gpus probe did not expose the qualified GB300: ${GPU_ROWS_ERROR}" @@ -1341,7 +1661,7 @@ run_dgx_os_gpus_test_sudo() { run_dgx_os_cdi_test_user() { local rows rows="$(station_local_default_docker run --rm --device nvidia.com/gpu=all "$ACCEPTANCE_IMAGE" nvidia-smi \ - --query-gpu=name,driver_version,ecc.errors.corrected.volatile.total,ecc.errors.uncorrected.volatile.total \ + --query-gpu=pci.bus_id,name,driver_version,ecc.errors.corrected.volatile.total,ecc.errors.uncorrected.volatile.total \ --format=csv,noheader,nounits)" || return 1 gpu_rows_are_valid "$rows" } @@ -1349,7 +1669,7 @@ run_dgx_os_cdi_test_user() { run_dgx_os_gpus_test_user() { local rows rows="$(station_local_default_docker run --rm --gpus all "$ACCEPTANCE_IMAGE" nvidia-smi \ - --query-gpu=name,driver_version,ecc.errors.corrected.volatile.total,ecc.errors.uncorrected.volatile.total \ + --query-gpu=pci.bus_id,name,driver_version,ecc.errors.corrected.volatile.total,ecc.errors.uncorrected.volatile.total \ --format=csv,noheader,nounits)" || return 1 gpu_rows_are_valid "$rows" } @@ -1554,22 +1874,26 @@ verify_apply_state() { } gpu_rows_are_valid() { - local rows=$1 row name driver corrected uncorrected row_index=0 gb300_count=0 expected_driver="" + local rows=$1 row bus_id name driver corrected uncorrected gb300_count=0 expected_driver="" pci_root GPU_ROWS_ERROR="" case "$STATION_HOST_PROFILE" in generic-ubuntu) expected_driver="$DRIVER_VERSION" ;; colossus-baseos) expected_driver="$BASEOS_DRIVER_VERSION" ;; esac + pci_root="$(station_pci_devices_path)" while IFS= read -r row; do [[ -n "${row//[[:space:]]/}" ]] || continue - IFS=',' read -r name driver corrected uncorrected <<<"$row" + IFS=',' read -r bus_id name driver corrected uncorrected <<<"$row" + if ! bus_id="$(normalize_nvidia_pci_bus_id "$bus_id")"; then + GPU_ROWS_ERROR="nvidia-smi returned an invalid PCI bus ID: ${bus_id}" + return 1 + fi name="${name#"${name%%[![:space:]]*}"}" driver="${driver//[[:space:]]/}" corrected="${corrected//[[:space:]]/}" uncorrected="${uncorrected//[[:space:]]/}" - if [[ "$name" != *"GB300"* ]]; then - info "gpu_index=${row_index} gpu=${name} role=auxiliary validation=skipped" - ((row_index += 1)) + if ! station_pci_device_is_gb300 "$bus_id" "$pci_root"; then + info "gpu_bdf=${bus_id} gpu=${name} role=auxiliary validation=skipped" continue fi if [[ -z "$driver" ]]; then @@ -1585,11 +1909,10 @@ gpu_rows_are_valid() { return 1 fi ((gb300_count += 1)) - info "gpu_index=${row_index} gpu=${name} role=inference driver=${driver} ecc_corrected=${corrected} ecc_uncorrected=${uncorrected}" - ((row_index += 1)) + info "gpu_bdf=${bus_id} gpu=${name} role=inference driver=${driver} ecc_corrected=${corrected} ecc_uncorrected=${uncorrected}" done <<<"$rows" if ((gb300_count != 1)); then - GPU_ROWS_ERROR="Expected exactly one NVIDIA GB300, found ${gb300_count}" + GPU_ROWS_ERROR="Expected exactly one NVIDIA GB300 PCI device, found ${gb300_count}" return 1 fi } @@ -1597,7 +1920,7 @@ gpu_rows_are_valid() { verify_gpu() { local rows rows="$(nvidia-smi \ - --query-gpu=name,driver_version,ecc.errors.corrected.volatile.total,ecc.errors.uncorrected.volatile.total \ + --query-gpu=pci.bus_id,name,driver_version,ecc.errors.corrected.volatile.total,ecc.errors.uncorrected.volatile.total \ --format=csv,noheader,nounits)" || fatal "nvidia-smi failed" gpu_rows_are_valid "$rows" || fatal "$GPU_ROWS_ERROR" } @@ -1752,6 +2075,7 @@ main() { info "version=${SCRIPT_VERSION} mode=${MODE} log=disabled_read_only" fi trap 'on_error "$LINENO"' ERR + trap 'cleanup_apt_transaction_guard' EXIT case "$MODE" in --check) run_check ;; --apply) run_apply ;; diff --git a/test/install-express-prompt.test.ts b/test/install-express-prompt.test.ts index b41e78e5fa3..41bd9aa9020 100644 --- a/test/install-express-prompt.test.ts +++ b/test/install-express-prompt.test.ts @@ -9,6 +9,24 @@ import { describe, expect, it } from "vitest"; import { INSTALLER_PAYLOAD, TEST_SYSTEM_PATH } from "./helpers/installer-sourced-env"; describe("installer express install prompt (sourced)", () => { + function runInstallerSourced(body: string) { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-express-sourced-")); + const result = spawnSync( + "bash", + ["--noprofile", "--norc", "-c", `source "$INSTALLER_UNDER_TEST" >/dev/null\n${body}`], + { + cwd: path.resolve(import.meta.dirname, ".."), + encoding: "utf-8", + env: { + HOME: home, + PATH: TEST_SYSTEM_PATH, + INSTALLER_UNDER_TEST: INSTALLER_PAYLOAD, + }, + }, + ); + return { result, output: `${result.stdout}${result.stderr}` }; + } + function runExpressPromptWithTty( answer: string, stdinMode: "pipe" | "tty", @@ -361,6 +379,45 @@ detect_express_platform expect(output).toMatch(/STATION_EXPRESS=1/); }); + it("keeps Station preparation details in the log while showing warnings and errors", () => { + const { result, output } = runInstallerSourced(` +printf '%s\n' \ + '[station-prepare] 2026-07-17T07:59:07Z version=2026-07-17.4 mode=--apply log=/tmp/station-prepare.log' \ + '[station-prepare] 2026-07-17T07:59:07Z platform=Dell Pro Max with Station GB300 profile=generic-ubuntu' \ + '[station-prepare] 2026-07-17T07:59:08Z WARNING: condition-qualified generic-image failed unit: cloud-init.service' \ + 'NVIDIA-SMI 610.43.02' \ + '[station-prepare] 2026-07-17T07:59:20Z ERROR: example failure' \ + | filter_station_host_preparation_output +`); + + expect(result.status, output).toBe(0); + expect(output).toContain("DGX Station host preparation log: /tmp/station-prepare.log"); + expect(output).toContain("condition-qualified generic-image failed unit: cloud-init.service"); + expect(output).toContain("ERROR: example failure"); + expect(output).not.toMatch(/platform=Dell Pro Max|NVIDIA-SMI/); + }); + + it("preserves the Station helper exit status while filtering installer output", () => { + const { result, output } = runInstallerSourced(` +bash() { + printf '%s\n' \ + '[station-prepare] 2026-07-17T07:59:07Z version=2026-07-17.4 mode=--apply log=/tmp/station-prepare.log' \ + '[station-prepare] 2026-07-17T07:59:08Z runtime_setup=complete' + return 10 +} +if run_station_host_preparation; then + printf 'STATUS=0\n' +else + printf 'STATUS=%s\n' "$?" +fi +`); + + expect(result.status, output).toBe(0); + expect(output).toContain("STATUS=10"); + expect(output).toContain("DGX Station host preparation log: /tmp/station-prepare.log"); + expect(output).not.toContain("runtime_setup=complete"); + }); + it.each([ [ "supported-colossus-baseos", diff --git a/test/install-station-dgx-os.test.ts b/test/install-station-dgx-os.test.ts index 3174cb33a2e..dd3e5fcd458 100644 --- a/test/install-station-dgx-os.test.ts +++ b/test/install-station-dgx-os.test.ts @@ -522,16 +522,17 @@ describe("DGX Station forced metadata installer handoff", () => { INSTALLER_PAYLOAD, ` SCRIPT_DIR="$HOME" -touch "$SCRIPT_DIR/prepare-dgx-station-host.sh" -bash() { printf 'HELPER_ARGS=%s\n' "$*"; } +cat >"$SCRIPT_DIR/prepare-dgx-station-host.sh" <<'HELPER' +printf '%s\n' "$*" >"$HOME/helper-args" +HELPER FORCE_STATION_INSTALL=1 run_station_host_preparation `, ); expect(result.status, output).toBe(0); - expect(output).toContain( - `HELPER_ARGS=${path.join(home, "prepare-dgx-station-host.sh")} --apply --force-station-install`, + expect(fs.readFileSync(path.join(home, "helper-args"), "utf8")).toBe( + "--apply --force-station-install\n", ); }); @@ -1029,7 +1030,8 @@ run_apply STATION_PREPARE, ` STATION_HOST_PROFILE=stock-dgx-os -nvidia-smi() { printf 'NVIDIA GB300, 595.71.05, 0, 0\n'; } +station_pci_device_is_gb300() { return 0; } +nvidia-smi() { printf '00000000:01:00.0, NVIDIA GB300, 595.71.05, 0, 0\n'; } verify_gpu `, ); @@ -1040,7 +1042,8 @@ verify_gpu STATION_PREPARE, ` STATION_HOST_PROFILE=generic-ubuntu -nvidia-smi() { printf 'NVIDIA GB300, 595.71.05, 0, 0\n'; } +station_pci_device_is_gb300() { return 0; } +nvidia-smi() { printf '00000000:01:00.0, NVIDIA GB300, 595.71.05, 0, 0\n'; } verify_gpu `, ); @@ -1053,18 +1056,19 @@ verify_gpu STATION_PREPARE, ` STATION_HOST_PROFILE=ai-developer-tools +station_pci_device_is_gb300() { [[ "$1" == "0000:01:00.0" ]]; } nvidia-smi() { - printf 'NVIDIA RTX PRO 6000 Blackwell Max-Q Workstation Edition, 610.43.03, [N/A], [N/A]\n' - printf 'NVIDIA GB300, 610.43.03, 0, 0\n' + printf '00000000:02:00.0, NVIDIA RTX PRO 6000 Blackwell Max-Q Workstation Edition, 610.43.03, [N/A], [N/A]\n' + printf '00000000:01:00.0, NVIDIA GB300, 610.43.03, 0, 0\n' } verify_gpu `, ); expect(result.status, output).toBe(0); - expect(output).toContain("gpu_index=0 gpu=NVIDIA RTX PRO 6000"); + expect(output).toContain("gpu_bdf=0000:02:00.0 gpu=NVIDIA RTX PRO 6000"); expect(output).toContain("role=auxiliary validation=skipped"); - expect(output).toContain("gpu_index=1 gpu=NVIDIA GB300 role=inference"); + expect(output).toContain("gpu_bdf=0000:01:00.0 gpu=NVIDIA GB300 role=inference"); }); it("requires both factory container probes to expose the GB300 on a mixed-GPU host", () => { @@ -1072,29 +1076,31 @@ verify_gpu STATION_PREPARE, ` STATION_HOST_PROFILE=ai-developer-tools +station_pci_device_is_gb300() { [[ "$1" == "0000:01:00.0" ]]; } station_sudo_local_default_docker() { - printf 'NVIDIA RTX PRO 6000 Blackwell Max-Q Workstation Edition, 610.43.03, [N/A], [N/A]\n' - printf 'NVIDIA GB300, 610.43.03, 0, 0\n' + printf '00000000:02:00.0, NVIDIA RTX PRO 6000 Blackwell Max-Q Workstation Edition, 610.43.03, [N/A], [N/A]\n' + printf '00000000:01:00.0, NVIDIA GB300, 610.43.03, 0, 0\n' } run_dgx_os_cdi_test_sudo run_dgx_os_gpus_test_sudo `, ); expect(mixed.result.status, mixed.output).toBe(0); - expect(mixed.output).toContain("gpu_index=1 gpu=NVIDIA GB300 role=inference"); + expect(mixed.output).toContain("gpu_bdf=0000:01:00.0 gpu=NVIDIA GB300 role=inference"); const rtxOnly = runSourced( STATION_PREPARE, ` STATION_HOST_PROFILE=ai-developer-tools +station_pci_device_is_gb300() { [[ "$1" == "0000:01:00.0" ]]; } station_sudo_local_default_docker() { - printf 'NVIDIA RTX PRO 6000 Blackwell Max-Q Workstation Edition, 610.43.03, [N/A], [N/A]\n' + printf '00000000:02:00.0, NVIDIA RTX PRO 6000 Blackwell Max-Q Workstation Edition, 610.43.03, [N/A], [N/A]\n' } run_dgx_os_cdi_test_sudo `, ); expect(rtxOnly.result.status, rtxOnly.output).not.toBe(0); - expect(rtxOnly.output).toContain("Expected exactly one NVIDIA GB300, found 0"); + expect(rtxOnly.output).toContain("Expected exactly one NVIDIA GB300 PCI device, found 0"); }); it("requires the qualified BaseOS driver to be loaded", () => { @@ -1102,7 +1108,8 @@ run_dgx_os_cdi_test_sudo STATION_PREPARE, ` STATION_HOST_PROFILE=colossus-baseos -nvidia-smi() { printf 'NVIDIA GB300, 595.71.05, 0, 0\n'; } +station_pci_device_is_gb300() { return 0; } +nvidia-smi() { printf '00000000:01:00.0, NVIDIA GB300, 595.71.05, 0, 0\n'; } verify_gpu `, ); @@ -1202,18 +1209,30 @@ verify_dgx_os_runtime_sudo }); it.each([ - ["wrong GPU", "NVIDIA GB200, 595.71.05, 0, 0", /Expected exactly one NVIDIA GB300, found 0/], - ["non-zero volatile ECC", "NVIDIA GB300, 595.71.05, 1, 0", /ECC must be 0\/0/], + [ + "a missing GB300 PCI identity", + "00000000:01:00.0, NVIDIA GB300, 595.71.05, 0, 0", + "return 1", + /Expected exactly one NVIDIA GB300 PCI device, found 0/, + ], + [ + "non-zero volatile ECC", + "00000000:01:00.0, NVIDIA GB300, 595.71.05, 1, 0", + "return 0", + /ECC must be 0\/0/, + ], [ "a failing second GPU row", - "NVIDIA GB300, 595.71.05, 0, 0\nNVIDIA GB300, 595.71.05, 0, 1", + "00000000:01:00.0, NVIDIA GB300, 595.71.05, 0, 0\n00000000:02:00.0, NVIDIA GB300, 595.71.05, 0, 1", + "return 0", /ECC must be 0\/0/, ], - ])("fails stock validation for %s", (_scenario, row, message) => { + ])("fails stock validation for %s", (_scenario, row, pciResult, message) => { const { result, output } = runSourced( STATION_PREPARE, ` STATION_HOST_PROFILE=stock-dgx-os +station_pci_device_is_gb300() { ${pciResult}; } nvidia-smi() { printf '%s\n' "$GPU_ROW"; } verify_gpu `, diff --git a/test/install-station-docker-repository.test.ts b/test/install-station-docker-repository.test.ts new file mode 100644 index 00000000000..31a98abce14 --- /dev/null +++ b/test/install-station-docker-repository.test.ts @@ -0,0 +1,182 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { TEST_SYSTEM_PATH } from "./helpers/installer-sourced-env"; + +const REPO_ROOT = path.resolve(import.meta.dirname, ".."); +const STATION_PREPARE = path.join(REPO_ROOT, "scripts", "prepare-dgx-station-host.sh"); + +function runSourced(body: string, extraEnv: Record = {}) { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-station-docker-repository-")); + const result = spawnSync( + "bash", + ["--noprofile", "--norc", "-c", `source "$SCRIPT_UNDER_TEST" >/dev/null\n${body}`], + { + cwd: REPO_ROOT, + encoding: "utf-8", + env: { + HOME: home, + PATH: TEST_SYSTEM_PATH, + SCRIPT_UNDER_TEST: STATION_PREPARE, + ...extraEnv, + }, + timeout: 15_000, + killSignal: "SIGKILL", + }, + ); + return { result, output: `${result.stdout}${result.stderr}` }; +} + +const DOCKER_REPOSITORY_FIXTURE = ` +prepare_docker_repository_fixture() { + mkdir -p "$HOME/root/etc/apt/keyrings" "$HOME/root/etc/apt/sources.list.d" + printf 'verified ascii key\n' >"$HOME/docker.asc" + printf 'verified dearmored key\n' >"$HOME/docker.gpg" + printf '%s\n' \\ + 'deb [arch=arm64 signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu noble stable' \\ + >"$HOME/docker-gpg.list" + printf '%s\n' \\ + 'deb [arch=arm64 signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu noble stable' \\ + >"$HOME/docker-asc.list" +} +assert_root_regular_file_safe() { printf 'ASSERT_SAFE %s\n' "$1"; } +sudo() { + if [[ "$*" == 'test ! -L /etc/apt/sources.list.d/docker.list' ]]; then + test ! -L "$HOME/root/etc/apt/sources.list.d/docker.list" + return + fi + if [[ "$*" == 'test -e /etc/apt/sources.list.d/docker.list' ]]; then + test -e "$HOME/root/etc/apt/sources.list.d/docker.list" + return + fi + if [[ "$1" == 'cmp' && "$2" == '-s' ]]; then + case "$4" in + /etc/apt/sources.list.d/docker.list) + cmp -s "$3" "$HOME/root/etc/apt/sources.list.d/docker.list" + ;; + /etc/apt/keyrings/docker.gpg) + cmp -s "$3" "$HOME/root/etc/apt/keyrings/docker.gpg" + ;; + /etc/apt/keyrings/docker.asc) + cmp -s "$3" "$HOME/root/etc/apt/keyrings/docker.asc" + ;; + *) return 1 ;; + esac + return + fi + return 1 +} +`; + +const VERIFY_REPOSITORY = ` +ensure_docker_repository_source \ + "$HOME/docker.asc" \ + "$HOME/docker.gpg" \ + "$HOME/docker-gpg.list" \ + "$HOME/docker-asc.list" +`; + +describe("DGX Station Docker repository compatibility", () => { + it("reuses the exact .gpg source with its verified key", () => { + const { result, output } = runSourced(` +${DOCKER_REPOSITORY_FIXTURE} +prepare_docker_repository_fixture +cp "$HOME/docker.gpg" "$HOME/root/etc/apt/keyrings/docker.gpg" +cp "$HOME/docker-gpg.list" "$HOME/root/etc/apt/sources.list.d/docker.list" +${VERIFY_REPOSITORY} +`); + + expect(result.status, output).toBe(0); + expect(output).toContain("ASSERT_SAFE /etc/apt/sources.list.d/docker.list"); + expect(output).toContain("ASSERT_SAFE /etc/apt/keyrings/docker.gpg"); + expect(output).toContain("docker_repository_source=exact"); + }); + + it("reuses the equivalent .asc source with its verified key", () => { + const { result, output } = runSourced(` +${DOCKER_REPOSITORY_FIXTURE} +prepare_docker_repository_fixture +cp "$HOME/docker.asc" "$HOME/root/etc/apt/keyrings/docker.asc" +cp "$HOME/docker-asc.list" "$HOME/root/etc/apt/sources.list.d/docker.list" +${VERIFY_REPOSITORY} +`); + + expect(result.status, output).toBe(0); + expect(output).toContain("ASSERT_SAFE /etc/apt/sources.list.d/docker.list"); + expect(output).toContain("ASSERT_SAFE /etc/apt/keyrings/docker.asc"); + expect(output).toContain("docker_repository_source=verified_compatible"); + }); + + it("rejects an .asc source when its installed key differs", () => { + const { result, output } = runSourced(` +${DOCKER_REPOSITORY_FIXTURE} +prepare_docker_repository_fixture +printf 'different ascii key\n' >"$HOME/root/etc/apt/keyrings/docker.asc" +cp "$HOME/docker-asc.list" "$HOME/root/etc/apt/sources.list.d/docker.list" +${VERIFY_REPOSITORY} +`); + + expect(result.status, output).not.toBe(0); + expect(output).toMatch(/ASCII key differs from the verified key/); + expect(output).not.toContain("docker_repository_source=verified_compatible"); + }); + + it.each([ + [ + "a changed URL", + "deb [arch=arm64 signed-by=/etc/apt/keyrings/docker.asc] https://mirror.invalid/linux/ubuntu noble stable\n", + ], + [ + "an extra source line", + "deb [arch=arm64 signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu noble stable\ndeb https://mirror.invalid/linux/ubuntu noble stable\n", + ], + ])("rejects a source with %s", (_case, sourceContent) => { + const { result, output } = runSourced( + ` +${DOCKER_REPOSITORY_FIXTURE} +prepare_docker_repository_fixture +cp "$HOME/docker.asc" "$HOME/root/etc/apt/keyrings/docker.asc" +printf '%s' "$SOURCE_CONTENT" >"$HOME/root/etc/apt/sources.list.d/docker.list" +${VERIFY_REPOSITORY} +`, + { SOURCE_CONTENT: sourceContent }, + ); + + expect(result.status, output).not.toBe(0); + expect(output).toMatch(/differs from the validated \.gpg and \.asc forms/); + }); + + it("rejects a symlinked source", () => { + const { result, output } = runSourced(` +${DOCKER_REPOSITORY_FIXTURE} +prepare_docker_repository_fixture +cp "$HOME/docker.asc" "$HOME/root/etc/apt/keyrings/docker.asc" +ln -s "$HOME/docker-asc.list" "$HOME/root/etc/apt/sources.list.d/docker.list" +${VERIFY_REPOSITORY} +`); + + expect(result.status, output).not.toBe(0); + expect(output).toMatch(/Docker repository source must not be a symbolic link/); + }); + + it("uses the dearmored key for a new source", () => { + const { result, output } = runSourced(` +${DOCKER_REPOSITORY_FIXTURE} +prepare_docker_repository_fixture +install_exact_file_or_reuse() { printf 'INSTALL %s -> %s\n' "$1" "$2"; } +${VERIFY_REPOSITORY} +`); + + expect(result.status, output).toBe(0); + expect(output).toMatch(/INSTALL .+\/docker\.gpg -> \/etc\/apt\/keyrings\/docker\.gpg/); + expect(output).toMatch( + /INSTALL .+\/docker-gpg\.list -> \/etc\/apt\/sources\.list\.d\/docker\.list/, + ); + expect(output).not.toContain("docker.asc -> /etc/apt/keyrings"); + }); +}); diff --git a/test/install-station-host-preparation.test.ts b/test/install-station-host-preparation.test.ts index 4e62cee188b..525650b2f33 100644 --- a/test/install-station-host-preparation.test.ts +++ b/test/install-station-host-preparation.test.ts @@ -79,9 +79,10 @@ describe("DGX Station host preparation", () => { STATION_PREPARE, ` STATION_HOST_PROFILE=generic-ubuntu +station_pci_device_is_gb300() { return 0; } sudo() { printf 'SUDO %s\\n' "$*" >&2 - printf 'NVIDIA GB300, 610.43.02, 0, 0\\n' + printf '00000000:01:00.0, NVIDIA GB300, 610.43.02, 0, 0\\n' } run_cdi_test_sudo run_gpus_test_sudo @@ -95,7 +96,7 @@ run_gpus_test_sudo `SUDO docker run --rm --device nvidia.com/gpu=all ${image} nvidia-smi --query-gpu=`, ); expect(output).toContain(`SUDO docker run --rm --gpus all ${image} nvidia-smi --query-gpu=`); - expect(output).toContain("gpu=NVIDIA GB300 role=inference"); + expect(output).toContain("gpu_bdf=0000:01:00.0 gpu=NVIDIA GB300 role=inference"); }); it.each([ @@ -415,8 +416,15 @@ run_apply configure_repositories() { printf 'CONFIGURE_REPOSITORIES\n'; } validate_package_availability() { printf 'VALIDATE_PACKAGES\n'; } simulate_install() { printf 'SIMULATE_INSTALL\n'; } +check_no_workloads() { printf 'RECHECK_ALL_WORKLOADS\n'; } require_docker_restart_quiescence() { printf 'RECHECK_RESTART_QUIESCENCE\n'; } +package_state() { printf 'missing\n'; } package_is_exact() { return 0; } +create_apt_transaction_guard() { + APT_TRANSACTION_GUARD_DIR=/run/nemoclaw-apt-transaction.TEST + APT_TRANSACTION_HOOK="$APT_TRANSACTION_GUARD_DIR/verify-plan" +} +cleanup_apt_transaction_guard() { :; } sudo() { printf 'SUDO %s\n' "$*"; } install_packages `, @@ -436,7 +444,6 @@ install_packages } expect(output).toContain("pinned_packages=installed"); }); - it("does not refresh CDI when the GPU launch probe already passes", () => { const { result, output } = runSourced( STATION_PREPARE, @@ -961,7 +968,10 @@ PAYLOAD set -euo pipefail case "\${1:-}" in --classify-dgx-release) printf 'CLASSIFY_STATION\\n' >&2; printf 'generic-ubuntu' ;; - --apply) printf 'PREPARE_STATION\\n' ;; + --apply) + printf '[station-prepare] 2026-07-17T07:59:07Z version=2026-07-17.4 mode=--apply log=/tmp/station-prepare.log\\n' + printf 'PREPARE_STATION\\n' + ;; *) exit 2 ;; esac HELPER @@ -992,12 +1002,14 @@ exit 0 killSignal: "SIGKILL", }); const output = `${result.stdout}${result.stderr}`; + const preparationLogIndex = output.indexOf("DGX Station host preparation log"); expect(result.status, output).toBe(0); expect(output).toContain("CLASSIFY_STATION"); expect(output).toContain("DGX Station host prerequisites are ready"); - expect(output.indexOf("PREPARE_STATION")).toBeGreaterThanOrEqual(0); - expect(output.indexOf("PREPARE_STATION")).toBeLessThan(output.indexOf("ENSURE_DOCKER")); + expect(preparationLogIndex).toBeGreaterThanOrEqual(0); + expect(preparationLogIndex).toBeLessThan(output.indexOf("ENSURE_DOCKER")); + expect(output).not.toContain("PREPARE_STATION"); expect(output.indexOf("ENSURE_DOCKER")).toBeLessThan(output.indexOf("ENSURE_BUILD_DEPS")); }); diff --git a/test/install-station-package-transaction.test.ts b/test/install-station-package-transaction.test.ts new file mode 100644 index 00000000000..1cfc13ae06e --- /dev/null +++ b/test/install-station-package-transaction.test.ts @@ -0,0 +1,415 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { TEST_SYSTEM_PATH } from "./helpers/installer-sourced-env"; + +const REPO_ROOT = path.resolve(import.meta.dirname, ".."); +const STATION_PREPARE = path.join(REPO_ROOT, "scripts", "prepare-dgx-station-host.sh"); +const EXPECTED_PACKAGE_SPECS = [ + "dkms=1:3.4.0-1ubuntu1", + "nvidia-driver-pinning-610=610-2ubuntu1", + "nvidia-driver-open=610.43.02-1ubuntu1", + "containerd.io=2.2.6-1~ubuntu.24.04~noble", + "docker-buildx-plugin=0.35.0-1~ubuntu.24.04~noble", + "docker-ce=5:29.6.1-1~ubuntu.24.04~noble", + "docker-ce-cli=5:29.6.1-1~ubuntu.24.04~noble", + "libnvidia-container-tools=1.19.1-1", + "libnvidia-container1=1.19.1-1", + "nvidia-container-toolkit=1.19.1-1", + "nvidia-container-toolkit-base=1.19.1-1", +]; +const DOCKER_CE_SPEC = "docker-ce=5:29.6.1-1~ubuntu.24.04~noble"; +const DKMS_SPEC = "dkms=1:3.4.0-1ubuntu1"; + +function runSourced(body: string, extraEnv: NodeJS.ProcessEnv = {}) { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-station-package-transaction-")); + const result = spawnSync( + "bash", + ["--noprofile", "--norc", "-c", `source "$SCRIPT_UNDER_TEST" >/dev/null\n${body}`], + { + cwd: REPO_ROOT, + encoding: "utf-8", + env: { + HOME: home, + PATH: TEST_SYSTEM_PATH, + SCRIPT_UNDER_TEST: STATION_PREPARE, + ...extraEnv, + }, + timeout: 15_000, + killSignal: "SIGKILL", + }, + ); + return { result, output: `${result.stdout}${result.stderr}` }; +} + +function validateSimulation(plan: string, specs = [DOCKER_CE_SPEC]) { + return runSourced( + ` +installed_version() { :; } +validate_apt_simulation "$APT_PLAN" ${specs.map((spec) => `'${spec}'`).join(" ")} +`, + { APT_PLAN: plan }, + ); +} + +function aptProtocol(...actions: string[]) { + return ["VERSION 3", "APT::Architecture=arm64", "", ...actions].join("\n"); +} + +function validatePreinstallPlan(targets: string, plan: string) { + return runSourced( + ` +printf '%s' "$APT_TARGETS" >"$HOME/targets" +validate_apt_preinstall_plan "$HOME/targets" <<<"$APT_PLAN" +`, + { APT_PLAN: plan, APT_TARGETS: targets }, + ); +} + +describe("DGX Station package transaction", () => { + it("passes the complete pinned tuple when every package is missing", () => { + const { result, output } = runSourced(` +configure_repositories() { printf 'CONFIGURE_REPOSITORIES\n'; } +apt-cache() { printf 'APT_CACHE %s\n' "$*" >>"$HOME/apt-cache-calls"; } +apt-get() { + printf 'APT_GET %s\n' "$*" + if [[ "$1" == "-s" ]]; then + local spec name version + for spec in "$@"; do + [[ "$spec" == [a-z0-9]*=* ]] || continue + name="\${spec%%=*}" + version="\${spec#*=}" + printf 'Inst %s (%s fixture [arm64])\n' "$name" "$version" + printf 'Conf %s (%s fixture [arm64])\n' "$name" "$version" + done + fi +} +check_no_workloads() { printf 'RECHECK_ALL_WORKLOADS\n'; } +require_docker_restart_quiescence() { printf 'RECHECK_DOCKER_RESTART\n'; } +package_state() { printf 'missing\n'; } +package_is_exact() { return 0; } +create_apt_transaction_guard() { + APT_TRANSACTION_GUARD_DIR=/run/nemoclaw-apt-transaction.TEST + APT_TRANSACTION_HOOK="$APT_TRANSACTION_GUARD_DIR/verify-plan" +} +cleanup_apt_transaction_guard() { + printf 'CLEANUP_GUARD\n' + APT_TRANSACTION_GUARD_DIR="" + APT_TRANSACTION_HOOK="" +} +sudo() { + printf 'SUDO %s\n' "$*" + if [[ "$1" == "env" && "$*" == *" apt-get -s install "* ]]; then + while [[ "$1" != "apt-get" ]]; do shift; done + "$@" + fi +} +install_packages +cat "$HOME/apt-cache-calls" +`); + + expect(result.status, output).toBe(0); + const expectedTuple = EXPECTED_PACKAGE_SPECS.join(" "); + const aptCommands = output + .split("\n") + .filter((line) => + /^(APT_CACHE show |APT_GET -s install |SUDO env .* apt-get install )/.test(line), + ) + .sort(); + expect(aptCommands).toEqual( + [ + ...EXPECTED_PACKAGE_SPECS.map((spec) => `APT_CACHE show ${spec}`), + `APT_GET -s install --no-install-recommends --no-remove -o DPkg::Pre-Install-Pkgs::=/run/nemoclaw-apt-transaction.TEST/verify-plan -o DPkg::Tools::options::/run/nemoclaw-apt-transaction.TEST/verify-plan::Version=3 ${expectedTuple}`, + `SUDO env DEBIAN_FRONTEND=noninteractive LC_ALL=C apt-get install -y --no-install-recommends --no-remove -o DPkg::Pre-Install-Pkgs::=/run/nemoclaw-apt-transaction.TEST/verify-plan -o DPkg::Tools::options::/run/nemoclaw-apt-transaction.TEST/verify-plan::Version=3 ${expectedTuple}`, + ].sort(), + ); + expect(output).toContain( + "SUDO env DEBIAN_FRONTEND=noninteractive LC_ALL=C apt-get -s install --no-install-recommends --no-remove", + ); + expect(output).toContain("RECHECK_ALL_WORKLOADS"); + expect(output).toContain("CLEANUP_GUARD"); + expect(output).toContain("pinned_packages=installed"); + }); + + it("excludes retained exact packages from every APT transaction command", () => { + const retainedSpec = "docker-ce=5:29.6.1-1~ubuntu.24.04~noble"; + const missingSpecs = EXPECTED_PACKAGE_SPECS.filter((spec) => spec !== retainedSpec); + const { result, output } = runSourced(` +configure_repositories() { :; } +apt-cache() { printf 'APT_CACHE %s\n' "$*" >>"$HOME/apt-cache-calls"; } +apt-get() { + printf 'APT_GET %s\n' "$*" + if [[ "$1" == "-s" ]]; then + local spec name version + for spec in "$@"; do + [[ "$spec" == [a-z0-9]*=* ]] || continue + name="\${spec%%=*}" + version="\${spec#*=}" + printf 'Inst %s (%s fixture [arm64])\n' "$name" "$version" + printf 'Conf %s (%s fixture [arm64])\n' "$name" "$version" + done + fi +} +check_no_workloads() { :; } +require_docker_restart_quiescence() { :; } +package_state() { + if [[ "$1" == '${retainedSpec}' ]]; then printf 'exact\n'; else printf 'missing\n'; fi +} +package_is_exact() { return 0; } +create_apt_transaction_guard() { + APT_TRANSACTION_GUARD_DIR=/run/nemoclaw-apt-transaction.TEST + APT_TRANSACTION_HOOK="$APT_TRANSACTION_GUARD_DIR/verify-plan" +} +cleanup_apt_transaction_guard() { + APT_TRANSACTION_GUARD_DIR="" + APT_TRANSACTION_HOOK="" +} +sudo() { + printf 'SUDO %s\n' "$*" + if [[ "$1" == "env" && "$*" == *" apt-get -s install "* ]]; then + while [[ "$1" != "apt-get" ]]; do shift; done + "$@" + fi +} +install_packages +cat "$HOME/apt-cache-calls" +`); + + expect(result.status, output).toBe(0); + const expectedTuple = missingSpecs.join(" "); + const aptCommands = output + .split("\n") + .filter((line) => + /^(APT_CACHE show |APT_GET -s install |SUDO env .* apt-get install )/.test(line), + ) + .sort(); + expect(aptCommands).toEqual( + [ + ...missingSpecs.map((spec) => `APT_CACHE show ${spec}`), + `APT_GET -s install --no-install-recommends --no-remove -o DPkg::Pre-Install-Pkgs::=/run/nemoclaw-apt-transaction.TEST/verify-plan -o DPkg::Tools::options::/run/nemoclaw-apt-transaction.TEST/verify-plan::Version=3 ${expectedTuple}`, + `SUDO env DEBIAN_FRONTEND=noninteractive LC_ALL=C apt-get install -y --no-install-recommends --no-remove -o DPkg::Pre-Install-Pkgs::=/run/nemoclaw-apt-transaction.TEST/verify-plan -o DPkg::Tools::options::/run/nemoclaw-apt-transaction.TEST/verify-plan::Version=3 ${expectedTuple}`, + ].sort(), + ); + expect(aptCommands.join("\n")).not.toContain(retainedSpec); + }); + + it("rejects a simulated change to a retained dependency", () => { + const { result, output } = runSourced(` +configure_repositories() { :; } +apt-cache() { :; } +apt-get() { + if [[ "$1" == "-s" ]]; then + printf '%s\n' \ + 'Inst docker-ce (5:29.6.1-1~ubuntu.24.04~noble fixture [arm64])' \ + 'Inst libc6 [2.39-0ubuntu8] (2.39-0ubuntu9 fixture [arm64])' + fi +} +package_state() { + if [[ "$1" == docker-ce=* ]]; then printf 'missing\n'; else printf 'exact\n'; fi +} +installed_version() { if [[ "$1" == "libc6" ]]; then printf '2.39-0ubuntu8'; fi; } +package_is_exact() { return 0; } +create_apt_transaction_guard() { + APT_TRANSACTION_GUARD_DIR=/run/nemoclaw-apt-transaction.TEST + APT_TRANSACTION_HOOK="$APT_TRANSACTION_GUARD_DIR/verify-plan" +} +sudo() { + printf 'SUDO %s\n' "$*" + if [[ "$1" == "env" && "$*" == *" apt-get -s install "* ]]; then + while [[ "$1" != "apt-get" ]]; do shift; done + "$@" + fi +} +install_packages +`); + + expect(result.status, output).not.toBe(0); + expect(output).toContain( + "APT simulation proposed changing retained package libc6=2.39-0ubuntu8", + ); + expect(output).not.toContain("apt-get install -y"); + }); + + it("rejects unsafe simulation actions before the privileged install", () => { + const expected = "5:29.6.1-1~ubuntu.24.04~noble"; + const scenarios = [ + { + plan: `Inst docker-ce (${expected} fixture [arm64])\nRemv libc6 [2.39-0ubuntu8]`, + message: "APT simulation proposed a package removal", + }, + { + plan: "Inst docker-ce (5:29.5.0-1~ubuntu.24.04~noble fixture [arm64])", + message: "APT simulation selected docker-ce=5:29.5.0-1~ubuntu.24.04~noble", + }, + { + plan: "Inst pigz (2.8-1 fixture [arm64])", + message: `APT simulation did not include required package ${DOCKER_CE_SPEC}`, + }, + { + plan: `Inst docker-ce (${expected} fixture [arm64])\nConf libc6 (2.39-0ubuntu8 fixture [arm64])`, + message: "APT simulation proposed configuration without an approved install", + }, + ]; + + for (const scenario of scenarios) { + const { result, output } = validateSimulation(scenario.plan); + expect(result.status, `${scenario.plan}\n${output}`).not.toBe(0); + expect(output).toContain(scenario.message); + } + }); + + it("allows the approved DKMS transition and genuinely new dependencies in simulation", () => { + const transition = validateSimulation( + [ + "Inst dkms [3.0.11-1ubuntu13] (1:3.4.0-1ubuntu1 fixture [all])", + "Conf dkms (1:3.4.0-1ubuntu1 fixture [all])", + ].join("\n"), + [DKMS_SPEC], + ); + expect(transition.result.status, transition.output).toBe(0); + + const dependency = validateSimulation( + [ + "Inst docker-ce (5:29.6.1-1~ubuntu.24.04~noble fixture [arm64])", + "Inst pigz (2.8-1 fixture [arm64])", + "Conf docker-ce (5:29.6.1-1~ubuntu.24.04~noble fixture [arm64])", + "Conf pigz (2.8-1 fixture [arm64])", + ].join("\n"), + ); + expect(dependency.result.status, dependency.output).toBe(0); + }); + + it("accepts only missing packages, new dependencies, and the approved transition in the actual plan", () => { + const missingWithDependency = validatePreinstallPlan( + "docker-ce|5:29.6.1-1~ubuntu.24.04~noble||arm64\n", + aptProtocol( + "docker-ce - - none < 5:29.6.1-1~ubuntu.24.04~noble arm64 no /var/cache/apt/archives/docker-ce.deb", + "pigz - - none < 2.8-1 arm64 no /var/cache/apt/archives/pigz.deb", + "docker-ce - - none < 5:29.6.1-1~ubuntu.24.04~noble arm64 no **CONFIGURE**", + "pigz - - none < 2.8-1 arm64 no **CONFIGURE**", + ), + ); + expect(missingWithDependency.result.status, missingWithDependency.output).toBe(0); + + const transition = validatePreinstallPlan( + "dkms|1:3.4.0-1ubuntu1|3.0.11-1ubuntu13|arm64\n", + aptProtocol( + "dkms 3.0.11-1ubuntu13 all foreign < 1:3.4.0-1ubuntu1 all foreign /var/cache/apt/archives/dkms.deb", + "dkms 3.0.11-1ubuntu13 all foreign < 1:3.4.0-1ubuntu1 all foreign **CONFIGURE**", + ), + ); + expect(transition.result.status, transition.output).toBe(0); + }); + + it("rejects unsafe VERSION 3 actions in the actual pre-install plan", () => { + const targets = "docker-ce|5:29.6.1-1~ubuntu.24.04~noble||arm64\n"; + const targetAction = + "docker-ce - - none < 5:29.6.1-1~ubuntu.24.04~noble arm64 no /var/cache/apt/archives/docker-ce.deb"; + const scenarios = [ + { + plan: aptProtocol(targetAction).replace("VERSION 3", "VERSION 2"), + message: "APT pre-install protocol must be VERSION 3", + }, + { + plan: aptProtocol( + targetAction, + "libc6 2.39-0ubuntu8 arm64 same < 2.39-0ubuntu9 arm64 same /var/cache/apt/archives/libc6.deb", + ), + message: "APT proposed changing retained package libc6=2.39-0ubuntu8", + }, + { + plan: aptProtocol(targetAction, "obsolete 1.0 arm64 no > - - none **REMOVE**"), + message: "APT proposed removing obsolete", + }, + { + plan: aptProtocol( + "docker-ce - - none < 5:29.5.0-1~ubuntu.24.04~noble arm64 no /var/cache/apt/archives/docker-ce.deb", + ), + message: "APT selected docker-ce=5:29.5.0-1~ubuntu.24.04~noble", + }, + { + plan: aptProtocol("pigz - - none < 2.8-1 arm64 no /var/cache/apt/archives/pigz.deb"), + message: "APT omitted required target docker-ce=5:29.6.1-1~ubuntu.24.04~noble", + }, + { + plan: aptProtocol( + targetAction, + "libc6 2.39-0ubuntu8 arm64 same = 2.39-0ubuntu8 arm64 same **CONFIGURE**", + ), + message: "APT proposed configuring retained package libc6@arm64 without an archive action", + }, + { + plan: aptProtocol( + "docker-ce - - none < 5:29.6.1-1~ubuntu.24.04~noble amd64 no /var/cache/apt/archives/docker-ce.deb", + ), + message: "APT selected foreign architecture amd64 for docker-ce; expected arm64 or all", + }, + ]; + + for (const scenario of scenarios) { + const { result, output } = validatePreinstallPlan(targets, scenario.plan); + expect(result.status, `${scenario.plan}\n${output}`).not.toBe(0); + expect(output).toContain(scenario.message); + } + }); + + it("emits an executable root-hook payload bound to its target manifest", () => { + const { result, output } = runSourced( + ` +PACKAGE_TRANSACTION_SPECS=('${DOCKER_CE_SPEC}') +package_state() { printf 'missing\n'; } +assert_root_directory_safe() { :; } +assert_root_regular_file_safe() { :; } +sudo() { + case "$1" in + dpkg) + printf 'arm64\n' + ;; + mktemp) + mkdir -p "$HOME/generated-guard" + printf '/run/nemoclaw-apt-transaction.GENERATED\n' + ;; + tee) + cat >"$HOME/generated-guard/\${2##*/}" + ;; + chmod) + command chmod "$2" "$HOME/generated-guard/\${3##*/}" + ;; + esac +} +create_apt_transaction_guard +"$HOME/generated-guard/verify-plan" <<<"$APT_PLAN" +printf 'GENERATED_HOOK_ACCEPTED\n' +`, + { + APT_PLAN: aptProtocol( + "docker-ce - - none < 5:29.6.1-1~ubuntu.24.04~noble arm64 no /var/cache/apt/archives/docker-ce.deb", + "pigz - - none < 2.8-1 arm64 no /var/cache/apt/archives/pigz.deb", + ), + }, + ); + + expect(result.status, output).toBe(0); + expect(output).toContain("GENERATED_HOOK_ACCEPTED"); + }); + + it("cleans the root-owned transaction guard when the caller exits", () => { + const { result, output } = runSourced(` +sudo() { printf 'SUDO %s\n' "$*"; } +setup_log() { :; } +run_apply() { + APT_TRANSACTION_GUARD_DIR=/run/nemoclaw-apt-transaction.EXITTEST + APT_TRANSACTION_HOOK="$APT_TRANSACTION_GUARD_DIR/verify-plan" +} +main --apply +`); + + expect(result.status, output).toBe(0); + expect(output).toContain("SUDO rm -rf -- /run/nemoclaw-apt-transaction.EXITTEST"); + }); +}); diff --git a/test/install-station-platform-identity.test.ts b/test/install-station-platform-identity.test.ts index fb46862f011..0f62f7e7ed2 100644 --- a/test/install-station-platform-identity.test.ts +++ b/test/install-station-platform-identity.test.ts @@ -32,9 +32,14 @@ function runStationPrepare(body: string, extraEnv: Record = {}) return { result, output: `${result.stdout}${result.stderr}` }; } -function writePciIdentityFixture(vendor = "0x10de", device = "0x31c2", pciClass = "0x030200") { +function writePciIdentityFixture( + vendor = "0x10de", + device = "0x31c2", + pciClass = "0x030200", + busId = "0000:01:00.0", +) { const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-station-pci-")); - const pciDevice = path.join(root, "0000:01:00.0"); + const pciDevice = path.join(root, busId); fs.mkdirSync(pciDevice); fs.writeFileSync(path.join(pciDevice, "vendor"), `${vendor}\n`); fs.writeFileSync(path.join(pciDevice, "device"), `${device}\n`); @@ -77,6 +82,49 @@ describe("DGX Station platform identity", () => { expect(result.status, output).toBe(0); }); + it("selects the GB300 by PCI identity when an auxiliary GPU has the same name", () => { + const pciRoot = writePciIdentityFixture(); + const { result, output } = runStationPrepare( + ` +station_pci_devices_path() { printf '%s' "$PCI_ROOT"; } +nvidia-smi() { + printf '%s\n' \ + '00000000:02:00.0, NVIDIA GB300, 595.71.05, 1, 0' \ + '00000000:01:00.0, NVIDIA GB300, 595.71.05, 0, 0' +} +STATION_HOST_PROFILE=stock-dgx-os +verify_gpu +`, + { PCI_ROOT: pciRoot }, + ); + + expect(result.status, output).toBe(0); + expect(output).toContain( + "gpu_bdf=0000:02:00.0 gpu=NVIDIA GB300 role=auxiliary validation=skipped", + ); + expect(output).toContain( + "gpu_bdf=0000:01:00.0 gpu=NVIDIA GB300 role=inference driver=595.71.05 ecc_corrected=0 ecc_uncorrected=0", + ); + }); + + it("qualifies the loaded driver from the PCI-identified GB300 instead of GPU index zero", () => { + const pciRoot = writePciIdentityFixture(); + const { result, output } = runStationPrepare( + ` +station_pci_devices_path() { printf '%s' "$PCI_ROOT"; } +nvidia-smi() { + printf '%s\n' \ + '00000000:02:00.0, 620.1' \ + '00000000:01:00.0, 610.43.02' +} +driver_loaded_exact +`, + { PCI_ROOT: pciRoot }, + ); + + expect(result.status, output).toBe(0); + }); + it.each([ ["wrong vendor", writePciIdentityFixture("0x1234")], ["wrong device", writePciIdentityFixture("0x10de", "0x31c1")], diff --git a/test/test-boundary-guards.test.ts b/test/test-boundary-guards.test.ts index 3268bac69c9..35f10961f3f 100644 --- a/test/test-boundary-guards.test.ts +++ b/test/test-boundary-guards.test.ts @@ -754,7 +754,9 @@ describe("Vitest project membership boundary", () => { ["test/install-preflight-docker-bootstrap.test.ts", "installer-integration"], ["test/install-preflight.test.ts", "installer-integration"], ["test/install-station-dgx-os.test.ts", "installer-integration"], + ["test/install-station-docker-repository.test.ts", "installer-integration"], ["test/install-station-host-preparation.test.ts", "installer-integration"], + ["test/install-station-package-transaction.test.ts", "installer-integration"], ["test/package-contract/example.test.js", "package-contract"], ["test/e2e/support/example.test.js", "e2e-support"], ["test/e2e/live/example.spec.ts", "e2e-live"], diff --git a/vitest.config.ts b/vitest.config.ts index ef042778a24..64474df7ee0 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -125,7 +125,9 @@ export default defineConfig({ "test/install-preflight.test.ts", "test/install-preflight-docker-bootstrap.test.ts", "test/install-station-dgx-os.test.ts", + "test/install-station-docker-repository.test.ts", "test/install-station-host-preparation.test.ts", + "test/install-station-package-transaction.test.ts", "test/install-openshell-version-check.test.ts", ], }, @@ -145,7 +147,9 @@ export default defineConfig({ "test/install-preflight.test.ts", "test/install-preflight-docker-bootstrap.test.ts", "test/install-station-dgx-os.test.ts", + "test/install-station-docker-repository.test.ts", "test/install-station-host-preparation.test.ts", + "test/install-station-package-transaction.test.ts", "test/install-openshell-version-check.test.ts", ], // Slow tests that spawn real bash install.sh processes. Explicit