From 672de50c3dc08df68df0a1643427f2dd1cdc5c37 Mon Sep 17 00:00:00 2001 From: San Dang Date: Fri, 17 Jul 2026 14:50:54 +0700 Subject: [PATCH 1/7] fix(installer): retain Station prerequisite version drift --- docs/get-started/prerequisites.mdx | 12 +- docs/get-started/quickstart.mdx | 16 +- scripts/prepare-dgx-station-host.sh | 143 ++++++----- test/install-station-host-preparation.test.ts | 241 ++++++++++++++---- 4 files changed, 284 insertions(+), 128 deletions(-) diff --git a/docs/get-started/prerequisites.mdx b/docs/get-started/prerequisites.mdx index ca69d5ba405..a77fae70dae 100644 --- a/docs/get-started/prerequisites.mdx +++ b/docs/get-started/prerequisites.mdx @@ -41,11 +41,12 @@ If the group change is not active in the current shell, the installer exits with If you choose the native Linux Ollama install path, the onboard wizard also requires `zstd` for Ollama archive extraction. The installer also requires `strings` from `binutils` to verify the OpenShell binary before it continues with OpenShell install work. -On a DGX Station GB300 running the generic Ubuntu 24.04 ARM64 image, accepting express install prepares the host with NVIDIA open driver `610.43.02`, Docker CE `29.6.1` with Buildx, and NVIDIA Container Toolkit `1.19.1`. +On a DGX Station GB300 running the generic Ubuntu 24.04 ARM64 image, accepting express install uses NVIDIA open driver `610.43.02`, Docker CE `29.6.1` with Buildx, and NVIDIA Container Toolkit `1.19.1` as the reviewed versions for missing prerequisites. DGX OS, NVIDIA BaseOS images, and other Station generations are outside this automatic preparation boundary and stop before host preparation. On those systems, set `NEMOCLAW_PROVIDER` or `NEMOCLAW_NO_EXPRESS=1` explicitly to continue without Station host automation. -The preparation probes package and runtime state first, reuses exact matches, and installs only missing pinned packages, including the NVIDIA Container Toolkit libraries and `nvidia-ctk` CLI. -It permits only the reviewed factory transition from `dkms` `3.0.11-1ubuntu13` to `1:3.4.0-1ubuntu1`. +The preparation probes package and runtime state first, retains installed packages, and installs reviewed pinned versions only for missing packages, including the NVIDIA Container Toolkit libraries and `nvidia-ctk` CLI. +When an installed prerequisite or the loaded NVIDIA driver differs from the reviewed version, preparation prints the actual and expected versions as a warning and continues with the installed version. +When repository setup is required, a different installed `cuda-keyring` version remains usable only after `dpkg -V` verifies its package files and the CUDA repository signing fingerprint matches the expected NVIDIA fingerprint. After reboot, preparation enables NVIDIA's packaged CDI refresh path and service, requires the `nvidia.com/gpu=all` device, and verifies it with a real container launch. If the packaged refresh fails or does not produce that device, preparation prints service diagnostics and stops for administrator repair. It does not bypass the packaged lifecycle with direct CDI generation. @@ -56,8 +57,9 @@ It requires Secure Boot to be disabled, matching headers for the running kernel, It also stops when systemd reports a failed unit unless the unit matches an exact, condition-qualified state from the generic Station image: the pinned OEM `cloud-init` telemetry failure, a network-wait failure while current network health is established, masked `fwupd`, or an SSSD socket on a host without SSSD configuration. Any other failed unit blocks preparation for administrator review. It does not install a host CUDA toolkit or Docker Compose. -If any other existing prerequisite version differs, preparation stops instead of changing it automatically. -After changing pinned packages, the installer exits with status `10`; reboot, sign in, and run the printed command, which pins the exact accepted NemoClaw commit before resuming express setup. +Package and driver version warnings do not relax the generic Ubuntu 24.04 ARM64 platform, Station GB300 identity, Secure Boot, kernel, capacity, network, workload, systemd, repository-key, APT simulation, rollback, or file-integrity checks. +The ECC 0/0 requirement, packaged CDI lifecycle, `nvidia.com/gpu=all` device, Docker CDI launch, and `--gpus all` launch also remain mandatory. +After installing missing pinned packages, the installer exits with status `10`; reboot, sign in, and run the printed command, which pins the exact accepted NemoClaw commit before resuming express setup. DGX Station remains Deferred. diff --git a/docs/get-started/quickstart.mdx b/docs/get-started/quickstart.mdx index 95bc49ed200..f9be5f718ec 100644 --- a/docs/get-started/quickstart.mdx +++ b/docs/get-started/quickstart.mdx @@ -123,10 +123,13 @@ Use these details when your first-run path needs more control. On that Station configuration, accepting the express prompt selects the pinned `nemotron-3-ultra-550b-a55b` managed-vLLM recipe and completes onboarding without more provider, model, policy, or sandbox-name choices. DGX OS, NVIDIA BaseOS images, and other Station generations stop before host preparation. Set `NEMOCLAW_PROVIDER` or `NEMOCLAW_NO_EXPRESS=1` explicitly to continue on an unqualified Station without host automation. - It probes the pinned driver, Docker, Buildx, and NVIDIA Container Toolkit versions, reuses exact matches, installs missing pins, and permits only the reviewed factory `dkms` transition. + It uses NVIDIA open driver `610.43.02`, Docker CE `29.6.1` with Buildx, and NVIDIA Container Toolkit `1.19.1` as the reviewed versions for missing prerequisites. + Preparation retains installed packages, installs reviewed pinned versions only for missing packages, and warns with actual and expected versions when an installed prerequisite or the loaded NVIDIA driver differs. + A different installed `cuda-keyring` version remains usable only when `dpkg -V` verifies its package files and the CUDA repository signing fingerprint matches the expected NVIDIA fingerprint. + Driver-version drift is advisory, but the Station GB300 identity, ECC 0/0, packaged CDI lifecycle, `nvidia.com/gpu=all` device, Docker CDI launch, and `--gpus all` launch checks remain mandatory. It establishes NVIDIA CDI through the packaged refresh service, then proves both CDI and `--gpus all` with real container launches. If the packaged refresh fails or does not advertise `nvidia.com/gpu=all`, preparation prints service diagnostics and stops for administrator repair instead of generating CDI configuration directly. - After it changes pinned packages, the installer exits with status `10` at the required reboot boundary; reboot, sign in, and run the printed exact-commit command to resume the accepted recipe without another prompt. + After it installs missing pinned packages, the installer exits with status `10` at the required reboot boundary; reboot, sign in, and run the printed exact-commit command to resume the accepted recipe without another prompt. This automation does not change Station's Deferred support status; physical end-to-end validation remains open. Pass `--station-deepseek` to use DeepSeek V4 Flash for a Station demo instead. The flag selects the interactive express prompt and requires terminal access. @@ -198,14 +201,17 @@ Use these details when your first-run path needs more control. After you confirm the interactive express prompt, the installer switches the remaining onboarding to non-interactive mode, allows `sudo` password prompts for required host changes, and selects the managed local inference path for that platform. DGX Spark uses managed vLLM with `qwen3.6-35b-a3b-nvfp4` by default. DGX Station express install explicitly selects `nemotron-3-ultra-550b-a55b` instead of the Station managed-vLLM profile default, `deepseek-v4-flash`, and discloses the approximately `352 GB` model download before confirmation. - Before onboarding, the Station path requires Station GB300 with the generic Ubuntu 24.04 ARM64 image and checks for NVIDIA open driver `610.43.02`, Docker CE `29.6.1` with Buildx, and NVIDIA Container Toolkit `1.19.1`. + Before onboarding, the Station path requires Station GB300 with the generic Ubuntu 24.04 ARM64 image and uses NVIDIA open driver `610.43.02`, Docker CE `29.6.1` with Buildx, and NVIDIA Container Toolkit `1.19.1` as the reviewed versions for missing prerequisites. DGX OS, NVIDIA BaseOS images, and other Station generations are outside this automatic preparation boundary. Set `NEMOCLAW_PROVIDER` or `NEMOCLAW_NO_EXPRESS=1` to continue without Station host automation. - Preparation reuses exact versions, installs missing pinned packages, permits only the reviewed `dkms` transition from `3.0.11-1ubuntu13` to `1:3.4.0-1ubuntu1`, and refuses every other mismatched version. + Preparation retains installed packages, installs reviewed pinned versions only for missing packages, and warns with actual and expected versions when an installed prerequisite or the loaded NVIDIA driver differs. + A different installed `cuda-keyring` version remains usable only when `dpkg -V` verifies its package files and the CUDA repository signing fingerprint matches the expected NVIDIA fingerprint. + Version warnings do not relax the generic Ubuntu 24.04 ARM64 image, Station GB300 DMI and GPU identity, Secure Boot, kernel, capacity, network, workload, systemd, repository-key, APT simulation, rollback, or file-integrity checks. + Driver-version drift is advisory, but the ECC 0/0 requirement, packaged CDI lifecycle, `nvidia.com/gpu=all` device, Docker CDI launch, and `--gpus all` launch checks remain mandatory. It requires the packaged NVIDIA CDI refresh service to advertise `nvidia.com/gpu=all`, and verifies CDI and `--gpus all` with real container launches. If the packaged refresh fails or omits that device, preparation prints service diagnostics and stops for administrator repair instead of generating CDI configuration directly. If NVIDIA Docker runtime registration or a post-change acceptance probe fails, preparation restores the prior Docker daemon configuration. - Changing pinned packages exits with status `10` for a reboot; after you sign in and run the printed exact-commit command, the accepted express recipe resumes without another prompt. + Installing missing pinned packages exits with status `10` for a reboot; after you sign in and run the printed exact-commit command, the accepted express recipe resumes without another prompt. This automation does not change Station's Deferred support status; physical end-to-end validation remains open. To select DeepSeek V4 Flash while retaining the one-confirmation Station express flow, run `curl -fsSL https://www.nvidia.com/nemoclaw.sh | bash -s -- --station-deepseek`. The `--station-deepseek` flag requires an interactive terminal; in a `curl | bash` pipeline, `/dev/tty` must be available. diff --git a/scripts/prepare-dgx-station-host.sh b/scripts/prepare-dgx-station-host.sh index ae14f99bc70..fac2f348009 100755 --- a/scripts/prepare-dgx-station-host.sh +++ b/scripts/prepare-dgx-station-host.sh @@ -5,7 +5,7 @@ set -Eeuo pipefail umask 077 -readonly SCRIPT_VERSION="2026-07-16.5" +readonly SCRIPT_VERSION="2026-07-17.1" readonly REBOOT_REQUIRED_EXIT=10 readonly MIN_FREE_KIB=$((20 * 1024 * 1024)) # The qualified generic image currently ships this OEM telemetry bootcmd. Its @@ -26,7 +26,6 @@ readonly DOCKER_KEY_FINGERPRINT="9DC858229FC7DD38854AE2D88D81803C0EBFCD88" readonly DRIVER_VERSION="610.43.02" readonly DOCKER_VERSION="29.6.1" readonly TOOLKIT_VERSION="1.19.1" -readonly FACTORY_DKMS_VERSION="3.0.11-1ubuntu13" readonly TARGET_DKMS_VERSION="1:3.4.0-1ubuntu1" # Keep this as a plain Ubuntu image: NVIDIA Container Toolkit injects the host # driver utility when CDI or --gpus is requested. This intentionally exercises @@ -82,7 +81,7 @@ usage() { Usage: prepare-dgx-station-host.sh --check|--apply|--verify --check Read-only eligibility and current-state report. - --apply Install exact prerequisites or finish post-reboot runtime setup. + --apply Install missing reviewed prerequisites or finish post-reboot runtime setup. --verify Read-only host verification plus ephemeral GPU container tests. Exit 10 from --apply means an operator-controlled reboot is required. After @@ -197,42 +196,22 @@ package_is_exact() { [[ "$actual" == "$expected" ]] } -package_state() { +package_is_installed() { local spec=$1 - local name expected actual + local name name="$(package_name "$spec")" - expected="$(package_expected_version "$spec")" - actual="$(installed_version "$name")" - if [[ -z "$actual" ]]; then - printf 'missing\n' - elif [[ "$actual" == "$expected" ]]; then - printf 'exact\n' - elif [[ "$name" == "dkms" && "$actual" == "$FACTORY_DKMS_VERSION" && "$expected" == "$TARGET_DKMS_VERSION" ]]; then - printf 'approved-transition\n' - else - printf 'mismatch\n' - fi + [[ -n "$(installed_version "$name")" ]] } -assert_no_package_mismatches() { - local spec state name expected actual mismatch=0 +report_package_version_drift() { + local spec name expected actual for spec in "${PACKAGE_SPECS[@]}"; do - state="$(package_state "$spec")" - if [[ "$state" == "approved-transition" ]]; then - name="$(package_name "$spec")" - expected="$(package_expected_version "$spec")" - actual="$(installed_version "$name")" - info "package=${name} status=approved_transition actual=${actual} expected=${expected}" - continue - fi - [[ "$state" == "mismatch" ]] || continue name="$(package_name "$spec")" expected="$(package_expected_version "$spec")" actual="$(installed_version "$name")" - warn "package=${name} status=mismatch actual=${actual} expected=${expected}" - mismatch=1 + [[ -n "$actual" && "$actual" != "$expected" ]] || continue + warn "package=${name} status=version_drift actual=${actual} expected=${expected}; retaining installed version" done - ((mismatch == 0)) || fatal "Existing Station prerequisite versions differ from the validated pins or approved factory transition; refusing to change them automatically" } all_packages_exact() { @@ -243,6 +222,14 @@ all_packages_exact() { return 0 } +all_packages_installed() { + local spec + for spec in "${PACKAGE_SPECS[@]}"; do + package_is_installed "$spec" || return 1 + done + return 0 +} + setup_log() { local log_dir="${HOME}/station-bootstrap-logs" mkdir -p "$log_dir" @@ -379,11 +366,27 @@ check_no_workloads() { info "workloads=none port_8000=free" } -driver_loaded_exact() { +loaded_driver_version() { local loaded - command -v nvidia-smi >/dev/null 2>&1 || return 1 - loaded="$(nvidia-smi --query-gpu=driver_version --format=csv,noheader 2>/dev/null | head -n1 | tr -d '[:space:]')" - [[ "$loaded" == "$DRIVER_VERSION" ]] + command -v nvidia-smi >/dev/null 2>&1 || return 0 + loaded="$(nvidia-smi --query-gpu=driver_version --format=csv,noheader 2>/dev/null | head -n1 | tr -d '[:space:]')" \ + || return 0 + printf '%s\n' "$loaded" +} + +driver_is_loaded() { + [[ -n "$(loaded_driver_version)" ]] +} + +driver_loaded_exact() { + [[ "$(loaded_driver_version)" == "$DRIVER_VERSION" ]] +} + +report_driver_version_drift() { + local actual + actual="$(loaded_driver_version)" + [[ -n "$actual" && "$actual" != "$DRIVER_VERSION" ]] || return 0 + warn "driver status=version_drift actual=${actual} expected=${DRIVER_VERSION}; retaining loaded version" } assert_station_state_dir_safe() { @@ -448,10 +451,8 @@ print_package_status() { info "package=${name} status=exact version=${actual}" elif [[ -z "$actual" ]]; then info "package=${name} status=missing expected=${expected}" - elif [[ "$name" == "dkms" && "$actual" == "$FACTORY_DKMS_VERSION" ]]; then - info "package=${name} status=approved_transition actual=${actual} expected=${expected}" else - warn "package=${name} status=mismatch actual=${actual} expected=${expected}" + warn "package=${name} status=version_drift actual=${actual} expected=${expected}; retaining installed version" fi done } @@ -548,13 +549,15 @@ ensure_cuda_keyring() { sudo dpkg -i "$cuda_deb" package_is_exact "cuda-keyring=${CUDA_KEYRING_PACKAGE_VERSION}" \ || fatal "Installed cuda-keyring does not match ${CUDA_KEYRING_PACKAGE_VERSION}" - elif [[ "$actual" == "$CUDA_KEYRING_PACKAGE_VERSION" ]]; then + else verification="$(dpkg -V cuda-keyring 2>&1)" \ || fatal "Unable to verify the installed cuda-keyring package" [[ -z "$verification" ]] || fatal "Installed cuda-keyring files differ from the package manifest: ${verification}" - info "cuda_keyring=exact version=${actual}" - else - fatal "Existing cuda-keyring version ${actual} differs from validated pin ${CUDA_KEYRING_PACKAGE_VERSION}; refusing to upgrade or downgrade it automatically" + if [[ "$actual" == "$CUDA_KEYRING_PACKAGE_VERSION" ]]; then + info "cuda_keyring=exact version=${actual}" + else + warn "package=cuda-keyring status=version_drift actual=${actual} expected=${CUDA_KEYRING_PACKAGE_VERSION}; retaining installed version" + fi fi assert_root_regular_file_safe /usr/share/keyrings/cuda-archive-keyring.gpg 0644 "CUDA repository keyring" @@ -605,9 +608,16 @@ configure_repositories() { info "repository_keys=verified" } -validate_package_availability() { +missing_package_specs() { local spec for spec in "${PACKAGE_SPECS[@]}"; do + package_is_installed "$spec" || printf '%s\n' "$spec" + done +} + +validate_package_availability() { + local spec + for spec in "$@"; do apt-cache show "$spec" >/dev/null 2>&1 || fatal "Exact package version is unavailable: ${spec}" done info "exact_package_versions=available" @@ -615,7 +625,7 @@ validate_package_availability() { simulate_install() { local simulation - simulation="$(apt-get -s install --no-install-recommends "${PACKAGE_SPECS[@]}")" \ + simulation="$(apt-get -s install --no-install-recommends "$@")" \ || fatal "APT simulation failed" printf '%s\n' "$simulation" if grep -Eq '^(Remv |Purg )' <<<"$simulation"; then @@ -625,18 +635,24 @@ simulate_install() { } install_packages() { + local spec + local -a missing_specs=() + while IFS= read -r spec; do + missing_specs+=("$spec") + done < <(missing_package_specs) + ((${#missing_specs[@]} > 0)) || fatal "Package installation was requested without a missing prerequisite" + configure_repositories info "Refreshing package metadata" sudo apt-get update - validate_package_availability - simulate_install + validate_package_availability "${missing_specs[@]}" + simulate_install "${missing_specs[@]}" check_no_workloads - info "Installing pinned Station prerequisites" + info "Installing missing pinned Station prerequisites" sudo env DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - "${PACKAGE_SPECS[@]}" + "${missing_specs[@]}" - local spec - for spec in "${PACKAGE_SPECS[@]}"; do + for spec in "${missing_specs[@]}"; do package_is_exact "$spec" || fatal "Installed package does not match ${spec}" done info "pinned_packages=installed" @@ -819,7 +835,7 @@ finish_runtime() { verify_apply_state() { local spec for spec in "${PACKAGE_SPECS[@]}"; do - package_is_exact "$spec" || fatal "Package verification failed: ${spec}" + package_is_installed "$spec" || fatal "Package verification failed: ${spec}" done verify_gpu systemctl is-active --quiet nvidia-persistenced.service || fatal "nvidia-persistenced.service is not active" @@ -843,7 +859,6 @@ verify_gpu() { corrected="${corrected//[[:space:]]/}" uncorrected="${uncorrected//[[:space:]]/}" [[ "$name" == *"GB300"* ]] || fatal "Expected NVIDIA GB300, found ${name}" - [[ "$driver" == "$DRIVER_VERSION" ]] || fatal "Expected driver ${DRIVER_VERSION}, found ${driver}" [[ "$corrected" == "0" && "$uncorrected" == "0" ]] \ || fatal "ECC must be 0/0, found corrected=${corrected} uncorrected=${uncorrected}" info "gpu=${name} driver=${driver} ecc_corrected=${corrected} ecc_uncorrected=${uncorrected}" @@ -852,7 +867,7 @@ verify_gpu() { verify_host() { local spec user_name=${SUDO_USER:-$USER} for spec in "${PACKAGE_SPECS[@]}"; do - package_is_exact "$spec" || fatal "Package verification failed: ${spec}" + package_is_installed "$spec" || fatal "Package verification failed: ${spec}" done verify_gpu systemctl is-active --quiet nvidia-persistenced.service || fatal "nvidia-persistenced.service is not active" @@ -873,14 +888,15 @@ verify_host() { run_check() { common_preflight print_package_status - if all_packages_exact; then + report_driver_version_drift + if all_packages_installed; then if install_boot_marker_matches_current_boot; then warn "Package installation completed in the current boot; reboot is required" info "CHECK_RESULT=REBOOT_REQUIRED" - elif driver_loaded_exact; then + elif driver_is_loaded; then info "CHECK_RESULT=PACKAGES_AND_DRIVER_PRESENT" else - warn "Exact packages are installed but driver ${DRIVER_VERSION} is not loaded; reboot is required" + warn "Station prerequisite packages are installed but the NVIDIA driver is not loaded; reboot is required" info "CHECK_RESULT=REBOOT_REQUIRED" fi else @@ -901,17 +917,18 @@ run_apply() { require_command sudo acquire_sudo common_preflight + report_package_version_drift + report_driver_version_drift if [[ -e /var/run/reboot-required ]]; then - if all_packages_exact && ! driver_loaded_exact; then + if all_packages_installed && ! driver_is_loaded; then warn "A reboot is required before runtime setup can continue" exit "$REBOOT_REQUIRED_EXIT" fi fatal "An unrelated reboot is already pending" fi - if ! all_packages_exact; then - assert_no_package_mismatches + if ! all_packages_installed; then install_packages ensure_docker_group check_no_workloads @@ -929,8 +946,8 @@ run_apply() { exit "$REBOOT_REQUIRED_EXIT" fi - driver_loaded_exact || { - warn "Pinned packages are installed but driver ${DRIVER_VERSION} is not loaded" + driver_is_loaded || { + warn "Station prerequisite packages are installed but the NVIDIA driver is not loaded; reboot is required" info "APPLY_RESULT=REBOOT_REQUIRED" info "Run: sudo reboot" exit "$REBOOT_REQUIRED_EXIT" @@ -953,8 +970,10 @@ run_verify() { require_command docker require_command nvidia-ctk require_command nvidia-smi - all_packages_exact || fatal "Pinned prerequisite packages are incomplete; run --apply" - driver_loaded_exact || fatal "Pinned driver is not loaded; reboot, then run --apply" + all_packages_installed || fatal "Station prerequisite packages are incomplete; run --apply" + driver_is_loaded || fatal "NVIDIA driver is not loaded; reboot, then run --apply" + report_package_version_drift + report_driver_version_drift verify_host } diff --git a/test/install-station-host-preparation.test.ts b/test/install-station-host-preparation.test.ts index 67228d531e7..e662fb627dc 100644 --- a/test/install-station-host-preparation.test.ts +++ b/test/install-station-host-preparation.test.ts @@ -42,13 +42,7 @@ describe("DGX Station host preparation", () => { it("keeps documented Station pins and Deferred status aligned", () => { const helper = fs.readFileSync(STATION_PREPARE, "utf-8"); const docs = STATION_DOCS.map((doc) => fs.readFileSync(doc, "utf-8")); - const pinnedValues = [ - "DRIVER_VERSION", - "DOCKER_VERSION", - "TOOLKIT_VERSION", - "FACTORY_DKMS_VERSION", - "TARGET_DKMS_VERSION", - ].map((name) => { + const pinnedValues = ["DRIVER_VERSION", "DOCKER_VERSION", "TOOLKIT_VERSION"].map((name) => { const value = helper.match(new RegExp(`readonly ${name}="([^"]+)"`))?.[1]; expect(value, `${name} must remain declared in the Station helper`).toBeTruthy(); return value as string; @@ -81,23 +75,33 @@ run_gpus_test_sudo }); it.each([ - ["", "missing"], - ["5:29.6.1-1~ubuntu.24.04~noble", "exact"], - ["5:30.0.0-1~ubuntu.24.04~noble", "mismatch"], - ])("classifies an installed package version as %s -> %s", (actual, expected) => { + ["", false, false], + ["5:29.6.1-1~ubuntu.24.04~noble", true, true], + ["5:30.0.0-1~ubuntu.24.04~noble", true, false], + ])("distinguishes package presence from an exact reviewed version: %s", (actual, installed, exact) => { const { result, output } = runSourced( STATION_PREPARE, ` installed_version() { if [[ "$1" == "docker-ce" ]]; then printf '%s' "$PACKAGE_ACTUAL"; fi } -package_state 'docker-ce=5:29.6.1-1~ubuntu.24.04~noble' +if package_is_installed 'docker-ce=5:29.6.1-1~ubuntu.24.04~noble'; then + printf 'installed=0\n' +else + printf 'installed=1\n' +fi +if package_is_exact 'docker-ce=5:29.6.1-1~ubuntu.24.04~noble'; then + printf 'exact=0\n' +else + printf 'exact=1\n' +fi `, { PACKAGE_ACTUAL: actual }, ); expect(result.status, output).toBe(0); - expect(result.stdout.trim()).toBe(expected); + expect(output).toContain(`installed=${installed ? 0 : 1}`); + expect(output).toContain(`exact=${exact ? 0 : 1}`); }); it.each([ @@ -114,50 +118,42 @@ package_state 'docker-ce=5:29.6.1-1~ubuntu.24.04~noble' expect(result.status === 0).toBe(accepted); }); - it("allows only the reviewed factory DKMS transition", () => { - const approved = runSourced( + it.each([ + "3.0.11-1ubuntu13", + "3.2.0-1", + ])("retains installed DKMS version drift as advisory: %s", (actual) => { + const { result, output } = runSourced( STATION_PREPARE, ` installed_version() { if [[ "$1" == "dkms" ]]; then printf '%s' "$DKMS_ACTUAL"; fi } -package_state 'dkms=1:3.4.0-1ubuntu1' -assert_no_package_mismatches +report_package_version_drift `, - { DKMS_ACTUAL: "3.0.11-1ubuntu13" }, + { DKMS_ACTUAL: actual }, ); - expect(approved.result.status, approved.output).toBe(0); - expect(approved.output).toContain("approved-transition"); - expect(approved.output).toContain("status=approved_transition"); - const arbitrary = runSourced( - STATION_PREPARE, - ` -installed_version() { - if [[ "$1" == "dkms" ]]; then printf '%s' "$DKMS_ACTUAL"; fi -} -assert_no_package_mismatches -`, - { DKMS_ACTUAL: "3.2.0-1" }, + expect(result.status, output).toBe(0); + expect(output).toContain( + `package=dkms status=version_drift actual=${actual} expected=1:3.4.0-1ubuntu1; retaining installed version`, ); - expect(arbitrary.result.status, arbitrary.output).not.toBe(0); - expect(arbitrary.output).toMatch(/dkms status=mismatch/); }); - it("refuses to change an existing mismatched prerequisite", () => { + it("warns with actual and expected versions for an installed prerequisite drift", () => { const { result, output } = runSourced( STATION_PREPARE, ` installed_version() { if [[ "$1" == "docker-ce" ]]; then printf '5:30.0.0-1~ubuntu.24.04~noble'; fi } -assert_no_package_mismatches +report_package_version_drift `, ); - expect(result.status, output).not.toBe(0); - expect(output).toMatch(/docker-ce status=mismatch/); - expect(output).toMatch(/refusing to change them automatically/); + expect(result.status, output).toBe(0); + expect(output).toContain( + "package=docker-ce status=version_drift actual=5:30.0.0-1~ubuntu.24.04~noble expected=5:29.6.1-1~ubuntu.24.04~noble; retaining installed version", + ); }); it("allows only condition-qualified factory failures and blocks other failed units", () => { @@ -355,9 +351,10 @@ check_failed_units common_preflight() { :; } require_command() { :; } acquire_sudo() { :; } -all_packages_exact() { return 0; } +all_packages_installed() { return 0; } install_boot_marker_matches_current_boot() { return 1; } -driver_loaded_exact() { return 0; } +driver_is_loaded() { return 0; } +report_package_version_drift() { :; } install_packages() { printf 'INSTALL_PACKAGES\n'; } finish_runtime() { printf 'FINISH_RUNTIME\n'; } verify_apply_state() { printf 'VERIFY_APPLY_STATE\n'; } @@ -372,14 +369,14 @@ run_apply expect(output).toContain("APPLY_RESULT=COMPLETE"); }); - it("applies the reviewed factory DKMS transition and returns the reboot-required contract", () => { + it("retains factory DKMS drift while installing missing prerequisites", () => { const { result, output } = runSourced( STATION_PREPARE, ` common_preflight() { :; } require_command() { :; } acquire_sudo() { :; } -all_packages_exact() { return 1; } +all_packages_installed() { return 1; } installed_version() { if [[ "$1" == "dkms" ]]; then printf '3.0.11-1ubuntu13'; fi } @@ -393,7 +390,9 @@ run_apply ); expect(result.status, output).toBe(10); - expect(output).toContain("package=dkms status=approved_transition"); + expect(output).toContain( + "package=dkms status=version_drift actual=3.0.11-1ubuntu13 expected=1:3.4.0-1ubuntu1; retaining installed version", + ); expect(output).toContain("INSTALL_PACKAGES"); expect(output).toContain("ENSURE_DOCKER_GROUP"); expect(output).toContain("RECHECK_ALL_WORKLOADS"); @@ -404,17 +403,40 @@ run_apply expect(output).toContain("APPLY_RESULT=REBOOT_REQUIRED"); }); - it("installs the exact NVIDIA Container Toolkit package contract", () => { + it("passes only missing pinned packages to APT and retains installed drift", () => { const { result, output } = runSourced( STATION_PREPARE, ` +DOCKER_INSTALLED=0 +installed_version() { + local name="$1" spec + if [[ "$name" == "docker-ce" ]]; then + if [[ "$DOCKER_INSTALLED" == "1" ]]; then + printf '5:29.6.1-1~ubuntu.24.04~noble' + fi + return + fi + if [[ "$name" == "dkms" ]]; then + printf '3.0.11-1ubuntu13' + return + fi + for spec in "\${PACKAGE_SPECS[@]}"; do + if [[ "$(package_name "$spec")" == "$name" ]]; then + package_expected_version "$spec" + return + fi + done +} configure_repositories() { printf 'CONFIGURE_REPOSITORIES\n'; } -validate_package_availability() { printf 'VALIDATE_PACKAGES\n'; } -simulate_install() { printf 'SIMULATE_INSTALL\n'; } +apt-cache() { printf 'APT_CACHE %s\n' "$*" >>"$HOME/apt-cache-calls"; } +apt-get() { printf 'APT_GET %s\n' "$*"; } check_no_workloads() { printf 'RECHECK_ALL_WORKLOADS\n'; } -package_is_exact() { return 0; } -sudo() { printf 'SUDO %s\n' "$*"; } +sudo() { + printf 'SUDO %s\n' "$*" + if [[ "$*" == *"apt-get install"* ]]; then DOCKER_INSTALLED=1; fi +} install_packages +cat "$HOME/apt-cache-calls" `, ); @@ -422,6 +444,27 @@ install_packages expect(output).toContain("apt-get update"); expect(output).toContain("apt-get install -y --no-install-recommends"); expect(output).toContain("RECHECK_ALL_WORKLOADS"); + expect(output).toContain("APT_CACHE show docker-ce=5:29.6.1-1~ubuntu.24.04~noble"); + expect(output).toContain( + "APT_GET -s install --no-install-recommends docker-ce=5:29.6.1-1~ubuntu.24.04~noble", + ); + expect(output).toContain( + "SUDO env DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends docker-ce=5:29.6.1-1~ubuntu.24.04~noble", + ); + expect(output).not.toContain("apt-get install -y --no-install-recommends dkms="); + expect(output).toContain("pinned_packages=installed"); + }); + + it("includes every NVIDIA Container Toolkit package when it is missing", () => { + const { result, output } = runSourced( + STATION_PREPARE, + ` +installed_version() { :; } +missing_package_specs +`, + ); + + expect(result.status, output).toBe(0); for (const spec of [ "libnvidia-container-tools=1.19.1-1", "libnvidia-container1=1.19.1-1", @@ -430,7 +473,6 @@ install_packages ]) { expect(output).toContain(spec); } - expect(output).toContain("pinned_packages=installed"); }); it("does not refresh CDI when the GPU launch probe already passes", () => { @@ -545,18 +587,41 @@ check_no_workloads expect(output).toMatch(/container state cannot be verified safely/); }); - it("refuses an installed CUDA keyring version that differs from the pin", () => { + it("retains a verified CUDA keyring version drift and verifies its signing fingerprint", () => { + const { result, output } = runSourced( + STATION_PREPARE, + ` +assert_root_directory_safe() { :; } +assert_root_regular_file_safe() { :; } +installed_version() { printf '2.0-1'; } +dpkg() { :; } +verify_key_fingerprint() { printf 'VERIFIED_FINGERPRINT\n'; } +ensure_cuda_keyring "$HOME/cuda-keyring.deb" +`, + ); + + expect(result.status, output).toBe(0); + expect(output).toContain( + "package=cuda-keyring status=version_drift actual=2.0-1 expected=1.1-1; retaining installed version", + ); + expect(output).toContain("VERIFIED_FINGERPRINT"); + }); + + it("rejects CUDA keyring drift when package-file verification reports changes", () => { const { result, output } = runSourced( STATION_PREPARE, ` assert_root_directory_safe() { :; } installed_version() { printf '2.0-1'; } +dpkg() { printf '??5?????? c /usr/share/keyrings/cuda-archive-keyring.gpg\n'; } +verify_key_fingerprint() { printf 'VERIFIED_FINGERPRINT\n'; } ensure_cuda_keyring "$HOME/cuda-keyring.deb" `, ); expect(result.status, output).not.toBe(0); - expect(output).toMatch(/refusing to upgrade or downgrade it automatically/); + expect(output).toMatch(/files differ from the package manifest/); + expect(output).not.toContain("VERIFIED_FINGERPRINT"); }); it("reuses an exact verified CUDA keyring without downloading it again", () => { @@ -654,9 +719,10 @@ assert_root_directory_safe /etc/apt/keyrings test_directory common_preflight() { :; } require_command() { :; } acquire_sudo() { :; } -all_packages_exact() { return 0; } +all_packages_installed() { return 0; } install_boot_marker_matches_current_boot() { return 1; } -driver_loaded_exact() { return 0; } +driver_is_loaded() { return 0; } +report_package_version_drift() { :; } finish_runtime() { DOCKER_GROUP_ADDED=1; printf 'FINISH_RUNTIME\n'; } verify_apply_state() { printf 'VERIFY_APPLY_STATE\n'; } run_apply @@ -912,20 +978,83 @@ main "$READ_MODE" expect(fs.existsSync(path.join(home, "station-bootstrap-logs"))).toBe(false); }); - it("fails verification when exact packages are present but the driver is not loaded", () => { + it("warns once for package and driver drift during a Station check", () => { + const { result, output } = runSourced( + STATION_PREPARE, + ` +common_preflight() { :; } +installed_version() { + local name="$1" spec + if [[ "$name" == "docker-ce" ]]; then + printf '5:30.0.0-1~ubuntu.24.04~noble' + return + fi + for spec in "\${PACKAGE_SPECS[@]}"; do + if [[ "$(package_name "$spec")" == "$name" ]]; then + package_expected_version "$spec" + return + fi + done +} +loaded_driver_version() { printf '620.1'; } +install_boot_marker_matches_current_boot() { return 1; } +run_check +`, + ); + + expect(result.status, output).toBe(0); + expect( + output.match( + /package=docker-ce status=version_drift actual=5:30\.0\.0-1~ubuntu\.24\.04~noble expected=5:29\.6\.1-1~ubuntu\.24\.04~noble/g, + ), + ).toHaveLength(1); + expect( + output.match(/driver status=version_drift actual=620\.1 expected=610\.43\.02/g), + ).toHaveLength(1); + expect(output).toContain("CHECK_RESULT=PACKAGES_AND_DRIVER_PRESENT"); + }); + + it("treats loaded driver version drift as advisory without relaxing ECC checks", () => { + const advisory = runSourced( + STATION_PREPARE, + ` +loaded_driver_version() { printf '620.1'; } +nvidia-smi() { printf 'NVIDIA GB300, 620.1, 0, 0\n'; } +report_driver_version_drift +verify_gpu +`, + ); + expect(advisory.result.status, advisory.output).toBe(0); + expect(advisory.output).toContain( + "driver status=version_drift actual=620.1 expected=610.43.02; retaining loaded version", + ); + expect(advisory.output).toContain("gpu=NVIDIA GB300 driver=620.1 ecc_corrected=0"); + + const eccFailure = runSourced( + STATION_PREPARE, + ` +nvidia-smi() { printf 'NVIDIA GB300, 620.1, 1, 0\n'; } +verify_gpu +`, + ); + expect(eccFailure.result.status, eccFailure.output).not.toBe(0); + expect(eccFailure.output).toMatch(/ECC must be 0\/0/); + }); + + it("fails verification when prerequisite packages are present but the driver is not loaded", () => { const { result, output } = runSourced( STATION_PREPARE, ` common_preflight() { :; } require_command() { :; } -all_packages_exact() { return 0; } -driver_loaded_exact() { return 1; } +all_packages_installed() { return 0; } +driver_is_loaded() { return 1; } run_verify `, ); expect(result.status, output).not.toBe(0); - expect(output).toMatch(/Pinned driver is not loaded/); + expect(output).toMatch(/NVIDIA driver is not loaded/); }); it("rejects a symlinked Station bootstrap state directory", () => { From 84f5bb4278c717d2d37d0b7989105525022dcc19 Mon Sep 17 00:00:00 2001 From: San Dang Date: Fri, 17 Jul 2026 15:02:00 +0700 Subject: [PATCH 2/7] fix(installer): accept verified Docker ASCII source Signed-off-by: San Dang --- docs/get-started/prerequisites.mdx | 5 + docs/get-started/quickstart.mdx | 8 + scripts/checks/vitest-project-overlap.ts | 1 + scripts/install.sh | 4 +- scripts/prepare-dgx-station-host.sh | 49 ++++- .../install-station-docker-repository.test.ts | 182 ++++++++++++++++++ test/install-station-host-preparation.test.ts | 6 + test/test-boundary-guards.test.ts | 1 + vitest.config.ts | 2 + 9 files changed, 250 insertions(+), 8 deletions(-) create mode 100644 test/install-station-docker-repository.test.ts diff --git a/docs/get-started/prerequisites.mdx b/docs/get-started/prerequisites.mdx index a77fae70dae..a79e37cdd55 100644 --- a/docs/get-started/prerequisites.mdx +++ b/docs/get-started/prerequisites.mdx @@ -47,6 +47,11 @@ On those systems, set `NEMOCLAW_PROVIDER` or `NEMOCLAW_NO_EXPRESS=1` explicitly The preparation probes package and runtime state first, retains installed packages, and installs reviewed pinned versions only for missing packages, including the NVIDIA Container Toolkit libraries and `nvidia-ctk` CLI. When an installed prerequisite or the loaded NVIDIA driver differs from the reviewed version, preparation prints the actual and expected versions as a warning and continues with the installed version. When repository setup is required, a different installed `cuda-keyring` version remains usable only after `dpkg -V` verifies its package files and the CUDA repository signing fingerprint matches the expected NVIDIA fingerprint. +For new Docker repository setup, preparation writes the reviewed source with `arch=arm64`, the `https://download.docker.com/linux/ubuntu` URL, the `noble` suite, the `stable` component, and the verified dearmored key at `/etc/apt/keyrings/docker.gpg`. +An existing source that instead uses `/etc/apt/keyrings/docker.asc` is retained only when the source is a root-owned regular file with mode `0644` and exactly matches the reviewed architecture, URL, suite, component, and signing-key path. +The installed ASCII key must also be a root-owned regular file with mode `0644` and be byte-identical to the fresh download after preparation verifies its pinned SHA-256 and official Docker fingerprint. +Any other Docker source difference, extra line, symbolic link, unsafe file, or key mismatch stops preparation. +The Docker `.asc` compatibility path does not relax the generic byte-for-byte exact-file guard for other managed repository files. After reboot, preparation enables NVIDIA's packaged CDI refresh path and service, requires the `nvidia.com/gpu=all` device, and verifies it with a real container launch. If the packaged refresh fails or does not produce that device, preparation prints service diagnostics and stops for administrator repair. It does not bypass the packaged lifecycle with direct CDI generation. diff --git a/docs/get-started/quickstart.mdx b/docs/get-started/quickstart.mdx index f9be5f718ec..b9c0162056b 100644 --- a/docs/get-started/quickstart.mdx +++ b/docs/get-started/quickstart.mdx @@ -126,6 +126,10 @@ Use these details when your first-run path needs more control. It uses NVIDIA open driver `610.43.02`, Docker CE `29.6.1` with Buildx, and NVIDIA Container Toolkit `1.19.1` as the reviewed versions for missing prerequisites. Preparation retains installed packages, installs reviewed pinned versions only for missing packages, and warns with actual and expected versions when an installed prerequisite or the loaded NVIDIA driver differs. A different installed `cuda-keyring` version remains usable only when `dpkg -V` verifies its package files and the CUDA repository signing fingerprint matches the expected NVIDIA fingerprint. + New Docker repository setup uses the reviewed `arch=arm64`, `https://download.docker.com/linux/ubuntu`, `noble`, and `stable` source with the verified dearmored `/etc/apt/keyrings/docker.gpg` key. + An existing source using `/etc/apt/keyrings/docker.asc` is retained only when its safe root-owned regular source file exactly matches the reviewed architecture, URL, suite, component, and signing-key path. + Its safe installed ASCII key must be byte-identical to the fresh download after preparation verifies its pinned SHA-256 and official Docker fingerprint. + Other source differences, extra lines, symbolic links, unsafe files, and key mismatches remain fatal, and other managed repository files keep the generic exact-file guard. Driver-version drift is advisory, but the Station GB300 identity, ECC 0/0, packaged CDI lifecycle, `nvidia.com/gpu=all` device, Docker CDI launch, and `--gpus all` launch checks remain mandatory. It establishes NVIDIA CDI through the packaged refresh service, then proves both CDI and `--gpus all` with real container launches. If the packaged refresh fails or does not advertise `nvidia.com/gpu=all`, preparation prints service diagnostics and stops for administrator repair instead of generating CDI configuration directly. @@ -206,6 +210,10 @@ Use these details when your first-run path needs more control. Set `NEMOCLAW_PROVIDER` or `NEMOCLAW_NO_EXPRESS=1` to continue without Station host automation. Preparation retains installed packages, installs reviewed pinned versions only for missing packages, and warns with actual and expected versions when an installed prerequisite or the loaded NVIDIA driver differs. A different installed `cuda-keyring` version remains usable only when `dpkg -V` verifies its package files and the CUDA repository signing fingerprint matches the expected NVIDIA fingerprint. + New Docker repository setup uses the reviewed `arch=arm64`, `https://download.docker.com/linux/ubuntu`, `noble`, and `stable` source with the verified dearmored `/etc/apt/keyrings/docker.gpg` key. + An existing source using `/etc/apt/keyrings/docker.asc` is retained only when its root-owned regular source file has mode `0644` and exactly matches the reviewed architecture, URL, suite, component, and signing-key path. + Its installed ASCII key must also be a root-owned regular file with mode `0644` and be byte-identical to the fresh download after preparation verifies its pinned SHA-256 and official Docker fingerprint. + Other source differences, extra lines, symbolic links, unsafe files, and key mismatches remain fatal, and other managed repository files keep the generic byte-for-byte exact-file guard. Version warnings do not relax the generic Ubuntu 24.04 ARM64 image, Station GB300 DMI and GPU identity, Secure Boot, kernel, capacity, network, workload, systemd, repository-key, APT simulation, rollback, or file-integrity checks. Driver-version drift is advisory, but the ECC 0/0 requirement, packaged CDI lifecycle, `nvidia.com/gpu=all` device, Docker CDI launch, and `--gpus all` launch checks remain mandatory. It requires the packaged NVIDIA CDI refresh service to advertise `nvidia.com/gpu=all`, and verifies CDI and `--gpus all` with real container launches. diff --git a/scripts/checks/vitest-project-overlap.ts b/scripts/checks/vitest-project-overlap.ts index 9e5415c7eed..02f312e691f 100644 --- a/scripts/checks/vitest-project-overlap.ts +++ b/scripts/checks/vitest-project-overlap.ts @@ -46,6 +46,7 @@ const INSTALLER_INTEGRATION_TESTS = new Set([ "test/install-openshell-version-check.test.ts", "test/install-preflight-docker-bootstrap.test.ts", "test/install-preflight.test.ts", + "test/install-station-docker-repository.test.ts", "test/install-station-host-preparation.test.ts", ]); diff --git a/scripts/install.sh b/scripts/install.sh index ed5222324cf..d2c35471069 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -3222,7 +3222,7 @@ run_station_host_preparation() { ensure_station_express_host() { [[ "${_SELECTED_EXPRESS_PLATFORM:-}" == "DGX Station" ]] || return 0 - info "Checking pinned DGX Station host prerequisites. Exact matches are reused." + info "Checking reviewed DGX Station host prerequisites. Installed versions are retained with warnings; missing packages use reviewed versions." local status=0 run_station_host_preparation || status=$? case "$status" in @@ -3287,7 +3287,7 @@ describe_express_install() { inference_summary="managed local vLLM with NVIDIA Nemotron 3 Ultra 550B" inference_disclosure="Managed vLLM pulls the pinned Station image and approximately 352 GB model, then runs a local inference container." fi - printf " Station host setup reuses exact prerequisite versions, applies the reviewed factory DKMS transition when present, installs missing pinned driver, Docker, and NVIDIA Container Toolkit packages, and may require one reboot.\n" + printf " Station host setup retains installed prerequisite versions with actual-versus-expected warnings, applies the reviewed factory DKMS transition when present, installs reviewed driver, Docker, and NVIDIA Container Toolkit versions only when packages are missing, and may require one reboot after installing those packages.\n" printf " Host setup may add this trusted local account to the docker group, which grants root-equivalent control. This flow is only for trusted single-user development hosts; shared or managed hosts require an organization-approved Docker access path.\n" printf " DGX Station remains Deferred; this recipe has not completed end-to-end validation on physical hardware.\n" sandbox_summary="${NEMOCLAW_SANDBOX_NAME:-my-assistant}" diff --git a/scripts/prepare-dgx-station-host.sh b/scripts/prepare-dgx-station-host.sh index fac2f348009..b6d357f03d6 100755 --- a/scripts/prepare-dgx-station-host.sh +++ b/scripts/prepare-dgx-station-host.sh @@ -5,7 +5,7 @@ set -Eeuo pipefail umask 077 -readonly SCRIPT_VERSION="2026-07-17.1" +readonly SCRIPT_VERSION="2026-07-17.2" readonly REBOOT_REQUIRED_EXIT=10 readonly MIN_FREE_KIB=$((20 * 1024 * 1024)) # The qualified generic image currently ships this OEM telemetry bootcmd. Its @@ -581,13 +581,48 @@ install_exact_file_or_reuse() { info "${label}=installed path=${target}" } +ensure_docker_repository_source() { + local docker_asc=$1 docker_gpg=$2 docker_gpg_list=$3 docker_asc_list=$4 + local source_target=/etc/apt/sources.list.d/docker.list + local gpg_key_target=/etc/apt/keyrings/docker.gpg + local asc_key_target=/etc/apt/keyrings/docker.asc + + sudo test ! -L "$source_target" \ + || fatal "Docker repository source must not be a symbolic link: ${source_target}" + if ! sudo test -e "$source_target"; then + install_exact_file_or_reuse "$docker_gpg" "$gpg_key_target" 0644 docker_repository_key + install_exact_file_or_reuse "$docker_gpg_list" "$source_target" 0644 docker_repository_source + return 0 + fi + + assert_root_regular_file_safe "$source_target" 0644 "Docker repository source" + if sudo cmp -s "$docker_gpg_list" "$source_target"; then + assert_root_regular_file_safe "$gpg_key_target" 0644 "Docker repository key" + sudo cmp -s "$docker_gpg" "$gpg_key_target" \ + || fatal "Existing Docker repository key differs from the verified dearmored key: ${gpg_key_target}" + info "docker_repository_source=exact path=${source_target}" + return 0 + fi + + if sudo cmp -s "$docker_asc_list" "$source_target"; then + assert_root_regular_file_safe "$asc_key_target" 0644 "Docker repository ASCII key" + sudo cmp -s "$docker_asc" "$asc_key_target" \ + || fatal "Existing Docker repository ASCII key differs from the verified key: ${asc_key_target}" + info "docker_repository_source=verified_compatible path=${source_target}" + return 0 + fi + + fatal "Existing Docker repository source differs from the validated .gpg and .asc forms; refusing to overwrite ${source_target}" +} + configure_repositories() { - local tmp cuda_deb docker_asc docker_gpg docker_list + local tmp cuda_deb docker_asc docker_gpg docker_gpg_list docker_asc_list tmp="$(mktemp -d)" cuda_deb="${tmp}/cuda-keyring.deb" docker_asc="${tmp}/docker.asc" docker_gpg="${tmp}/docker.gpg" - docker_list="${tmp}/docker.list" + docker_gpg_list="${tmp}/docker-gpg.list" + docker_asc_list="${tmp}/docker-asc.list" info "Downloading and verifying official repository keys" ensure_cuda_keyring "$cuda_deb" @@ -598,11 +633,13 @@ configure_repositories() { gpg --batch --yes --dearmor --output "$docker_gpg" "$docker_asc" ensure_root_directory_safe /etc/apt/keyrings /etc/apt 0755 "Docker repository key directory" assert_root_directory_safe /etc/apt/sources.list.d "Docker repository source directory" - install_exact_file_or_reuse "$docker_gpg" /etc/apt/keyrings/docker.gpg 0644 docker_repository_key printf '%s\n' \ 'deb [arch=arm64 signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu noble stable' \ - >"$docker_list" - install_exact_file_or_reuse "$docker_list" /etc/apt/sources.list.d/docker.list 0644 docker_repository_source + >"$docker_gpg_list" + printf '%s\n' \ + 'deb [arch=arm64 signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu noble stable' \ + >"$docker_asc_list" + ensure_docker_repository_source "$docker_asc" "$docker_gpg" "$docker_gpg_list" "$docker_asc_list" rm -rf "$tmp" info "repository_keys=verified" diff --git a/test/install-station-docker-repository.test.ts b/test/install-station-docker-repository.test.ts new file mode 100644 index 00000000000..31a98abce14 --- /dev/null +++ b/test/install-station-docker-repository.test.ts @@ -0,0 +1,182 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { TEST_SYSTEM_PATH } from "./helpers/installer-sourced-env"; + +const REPO_ROOT = path.resolve(import.meta.dirname, ".."); +const STATION_PREPARE = path.join(REPO_ROOT, "scripts", "prepare-dgx-station-host.sh"); + +function runSourced(body: string, extraEnv: Record = {}) { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-station-docker-repository-")); + const result = spawnSync( + "bash", + ["--noprofile", "--norc", "-c", `source "$SCRIPT_UNDER_TEST" >/dev/null\n${body}`], + { + cwd: REPO_ROOT, + encoding: "utf-8", + env: { + HOME: home, + PATH: TEST_SYSTEM_PATH, + SCRIPT_UNDER_TEST: STATION_PREPARE, + ...extraEnv, + }, + timeout: 15_000, + killSignal: "SIGKILL", + }, + ); + return { result, output: `${result.stdout}${result.stderr}` }; +} + +const DOCKER_REPOSITORY_FIXTURE = ` +prepare_docker_repository_fixture() { + mkdir -p "$HOME/root/etc/apt/keyrings" "$HOME/root/etc/apt/sources.list.d" + printf 'verified ascii key\n' >"$HOME/docker.asc" + printf 'verified dearmored key\n' >"$HOME/docker.gpg" + printf '%s\n' \\ + 'deb [arch=arm64 signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu noble stable' \\ + >"$HOME/docker-gpg.list" + printf '%s\n' \\ + 'deb [arch=arm64 signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu noble stable' \\ + >"$HOME/docker-asc.list" +} +assert_root_regular_file_safe() { printf 'ASSERT_SAFE %s\n' "$1"; } +sudo() { + if [[ "$*" == 'test ! -L /etc/apt/sources.list.d/docker.list' ]]; then + test ! -L "$HOME/root/etc/apt/sources.list.d/docker.list" + return + fi + if [[ "$*" == 'test -e /etc/apt/sources.list.d/docker.list' ]]; then + test -e "$HOME/root/etc/apt/sources.list.d/docker.list" + return + fi + if [[ "$1" == 'cmp' && "$2" == '-s' ]]; then + case "$4" in + /etc/apt/sources.list.d/docker.list) + cmp -s "$3" "$HOME/root/etc/apt/sources.list.d/docker.list" + ;; + /etc/apt/keyrings/docker.gpg) + cmp -s "$3" "$HOME/root/etc/apt/keyrings/docker.gpg" + ;; + /etc/apt/keyrings/docker.asc) + cmp -s "$3" "$HOME/root/etc/apt/keyrings/docker.asc" + ;; + *) return 1 ;; + esac + return + fi + return 1 +} +`; + +const VERIFY_REPOSITORY = ` +ensure_docker_repository_source \ + "$HOME/docker.asc" \ + "$HOME/docker.gpg" \ + "$HOME/docker-gpg.list" \ + "$HOME/docker-asc.list" +`; + +describe("DGX Station Docker repository compatibility", () => { + it("reuses the exact .gpg source with its verified key", () => { + const { result, output } = runSourced(` +${DOCKER_REPOSITORY_FIXTURE} +prepare_docker_repository_fixture +cp "$HOME/docker.gpg" "$HOME/root/etc/apt/keyrings/docker.gpg" +cp "$HOME/docker-gpg.list" "$HOME/root/etc/apt/sources.list.d/docker.list" +${VERIFY_REPOSITORY} +`); + + expect(result.status, output).toBe(0); + expect(output).toContain("ASSERT_SAFE /etc/apt/sources.list.d/docker.list"); + expect(output).toContain("ASSERT_SAFE /etc/apt/keyrings/docker.gpg"); + expect(output).toContain("docker_repository_source=exact"); + }); + + it("reuses the equivalent .asc source with its verified key", () => { + const { result, output } = runSourced(` +${DOCKER_REPOSITORY_FIXTURE} +prepare_docker_repository_fixture +cp "$HOME/docker.asc" "$HOME/root/etc/apt/keyrings/docker.asc" +cp "$HOME/docker-asc.list" "$HOME/root/etc/apt/sources.list.d/docker.list" +${VERIFY_REPOSITORY} +`); + + expect(result.status, output).toBe(0); + expect(output).toContain("ASSERT_SAFE /etc/apt/sources.list.d/docker.list"); + expect(output).toContain("ASSERT_SAFE /etc/apt/keyrings/docker.asc"); + expect(output).toContain("docker_repository_source=verified_compatible"); + }); + + it("rejects an .asc source when its installed key differs", () => { + const { result, output } = runSourced(` +${DOCKER_REPOSITORY_FIXTURE} +prepare_docker_repository_fixture +printf 'different ascii key\n' >"$HOME/root/etc/apt/keyrings/docker.asc" +cp "$HOME/docker-asc.list" "$HOME/root/etc/apt/sources.list.d/docker.list" +${VERIFY_REPOSITORY} +`); + + expect(result.status, output).not.toBe(0); + expect(output).toMatch(/ASCII key differs from the verified key/); + expect(output).not.toContain("docker_repository_source=verified_compatible"); + }); + + it.each([ + [ + "a changed URL", + "deb [arch=arm64 signed-by=/etc/apt/keyrings/docker.asc] https://mirror.invalid/linux/ubuntu noble stable\n", + ], + [ + "an extra source line", + "deb [arch=arm64 signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu noble stable\ndeb https://mirror.invalid/linux/ubuntu noble stable\n", + ], + ])("rejects a source with %s", (_case, sourceContent) => { + const { result, output } = runSourced( + ` +${DOCKER_REPOSITORY_FIXTURE} +prepare_docker_repository_fixture +cp "$HOME/docker.asc" "$HOME/root/etc/apt/keyrings/docker.asc" +printf '%s' "$SOURCE_CONTENT" >"$HOME/root/etc/apt/sources.list.d/docker.list" +${VERIFY_REPOSITORY} +`, + { SOURCE_CONTENT: sourceContent }, + ); + + expect(result.status, output).not.toBe(0); + expect(output).toMatch(/differs from the validated \.gpg and \.asc forms/); + }); + + it("rejects a symlinked source", () => { + const { result, output } = runSourced(` +${DOCKER_REPOSITORY_FIXTURE} +prepare_docker_repository_fixture +cp "$HOME/docker.asc" "$HOME/root/etc/apt/keyrings/docker.asc" +ln -s "$HOME/docker-asc.list" "$HOME/root/etc/apt/sources.list.d/docker.list" +${VERIFY_REPOSITORY} +`); + + expect(result.status, output).not.toBe(0); + expect(output).toMatch(/Docker repository source must not be a symbolic link/); + }); + + it("uses the dearmored key for a new source", () => { + const { result, output } = runSourced(` +${DOCKER_REPOSITORY_FIXTURE} +prepare_docker_repository_fixture +install_exact_file_or_reuse() { printf 'INSTALL %s -> %s\n' "$1" "$2"; } +${VERIFY_REPOSITORY} +`); + + expect(result.status, output).toBe(0); + expect(output).toMatch(/INSTALL .+\/docker\.gpg -> \/etc\/apt\/keyrings\/docker\.gpg/); + expect(output).toMatch( + /INSTALL .+\/docker-gpg\.list -> \/etc\/apt\/sources\.list\.d\/docker\.list/, + ); + expect(output).not.toContain("docker.asc -> /etc/apt/keyrings"); + }); +}); diff --git a/test/install-station-host-preparation.test.ts b/test/install-station-host-preparation.test.ts index e662fb627dc..bdaf21204f9 100644 --- a/test/install-station-host-preparation.test.ts +++ b/test/install-station-host-preparation.test.ts @@ -329,6 +329,12 @@ describe_express_install 'DGX Station' expect(output).toContain( "shared or managed hosts require an organization-approved Docker access path", ); + expect(output).toContain( + "retains installed prerequisite versions with actual-versus-expected warnings", + ); + expect(output).toContain( + "installs reviewed driver, Docker, and NVIDIA Container Toolkit versions only when packages are missing", + ); }); it("fails closed when failed-service inspection is unavailable", () => { diff --git a/test/test-boundary-guards.test.ts b/test/test-boundary-guards.test.ts index 6f0a5115aeb..e1da712444d 100644 --- a/test/test-boundary-guards.test.ts +++ b/test/test-boundary-guards.test.ts @@ -696,6 +696,7 @@ describe("Vitest project membership boundary", () => { ["test/install-openshell-version-check.test.ts", "installer-integration"], ["test/install-preflight-docker-bootstrap.test.ts", "installer-integration"], ["test/install-preflight.test.ts", "installer-integration"], + ["test/install-station-docker-repository.test.ts", "installer-integration"], ["test/install-station-host-preparation.test.ts", "installer-integration"], ["test/package-contract/example.test.js", "package-contract"], ["test/e2e/support/example.test.js", "e2e-support"], diff --git a/vitest.config.ts b/vitest.config.ts index 6d5509c6b1d..3e3e7d6f71a 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -124,6 +124,7 @@ export default defineConfig({ "test/install-clone-ref.test.ts", "test/install-preflight.test.ts", "test/install-preflight-docker-bootstrap.test.ts", + "test/install-station-docker-repository.test.ts", "test/install-station-host-preparation.test.ts", "test/install-openshell-version-check.test.ts", ], @@ -143,6 +144,7 @@ export default defineConfig({ "test/install-clone-ref.test.ts", "test/install-preflight.test.ts", "test/install-preflight-docker-bootstrap.test.ts", + "test/install-station-docker-repository.test.ts", "test/install-station-host-preparation.test.ts", "test/install-openshell-version-check.test.ts", ], From 0d9d37e703e9659b7a8f5a1bbcf2fc636d23ba54 Mon Sep 17 00:00:00 2001 From: San Dang Date: Fri, 17 Jul 2026 15:09:53 +0700 Subject: [PATCH 3/7] test(installer): update Station express disclosure Signed-off-by: San Dang --- test/install-express-prompt.test.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/install-express-prompt.test.ts b/test/install-express-prompt.test.ts index 32e900bf63f..a7b70620c79 100644 --- a/test/install-express-prompt.test.ts +++ b/test/install-express-prompt.test.ts @@ -257,7 +257,10 @@ detect_express_platform ); expect(output).toMatch(/approximately 352 GB model/); expect(output).toMatch( - /installs missing pinned driver, Docker, and NVIDIA Container Toolkit packages/, + /retains installed prerequisite versions with actual-versus-expected warnings/, + ); + expect(output).toMatch( + /installs reviewed driver, Docker, and NVIDIA Container Toolkit versions only when packages are missing/, ); expect(output).toMatch(/DGX Station remains Deferred/); expect(output).toMatch(/Using express install for DGX Station/); From bbbb52fcde2580bee14a8dc36b60aa6c2bccaff3 Mon Sep 17 00:00:00 2001 From: San Dang Date: Fri, 17 Jul 2026 15:15:52 +0700 Subject: [PATCH 4/7] fix(installer): summarize Station preparation output Signed-off-by: San Dang --- docs/get-started/prerequisites.mdx | 2 + docs/get-started/quickstart.mdx | 4 ++ scripts/install.sh | 20 ++++++- test/install-station-host-preparation.test.ts | 60 ++++++++++++++++++- 4 files changed, 83 insertions(+), 3 deletions(-) diff --git a/docs/get-started/prerequisites.mdx b/docs/get-started/prerequisites.mdx index a79e37cdd55..bcdf121a7b6 100644 --- a/docs/get-started/prerequisites.mdx +++ b/docs/get-started/prerequisites.mdx @@ -46,6 +46,8 @@ DGX OS, NVIDIA BaseOS images, and other Station generations are outside this aut On those systems, set `NEMOCLAW_PROVIDER` or `NEMOCLAW_NO_EXPRESS=1` explicitly to continue without Station host automation. The preparation probes package and runtime state first, retains installed packages, and installs reviewed pinned versions only for missing packages, including the NVIDIA Container Toolkit libraries and `nvidia-ctk` CLI. When an installed prerequisite or the loaded NVIDIA driver differs from the reviewed version, preparation prints the actual and expected versions as a warning and continues with the installed version. +During express install, the terminal shows the Station host preparation log path, package and driver version-drift warnings with actual and expected versions, and errors while keeping the complete platform, systemd, Docker pull, NVIDIA-SMI, and status output in that log. +Running `scripts/prepare-dgx-station-host.sh` directly remains verbose. When repository setup is required, a different installed `cuda-keyring` version remains usable only after `dpkg -V` verifies its package files and the CUDA repository signing fingerprint matches the expected NVIDIA fingerprint. For new Docker repository setup, preparation writes the reviewed source with `arch=arm64`, the `https://download.docker.com/linux/ubuntu` URL, the `noble` suite, the `stable` component, and the verified dearmored key at `/etc/apt/keyrings/docker.gpg`. An existing source that instead uses `/etc/apt/keyrings/docker.asc` is retained only when the source is a root-owned regular file with mode `0644` and exactly matches the reviewed architecture, URL, suite, component, and signing-key path. diff --git a/docs/get-started/quickstart.mdx b/docs/get-started/quickstart.mdx index b9c0162056b..3915e9275e0 100644 --- a/docs/get-started/quickstart.mdx +++ b/docs/get-started/quickstart.mdx @@ -125,6 +125,7 @@ Use these details when your first-run path needs more control. Set `NEMOCLAW_PROVIDER` or `NEMOCLAW_NO_EXPRESS=1` explicitly to continue on an unqualified Station without host automation. It uses NVIDIA open driver `610.43.02`, Docker CE `29.6.1` with Buildx, and NVIDIA Container Toolkit `1.19.1` as the reviewed versions for missing prerequisites. Preparation retains installed packages, installs reviewed pinned versions only for missing packages, and warns with actual and expected versions when an installed prerequisite or the loaded NVIDIA driver differs. + Express install shows the Station host preparation log path, these version-drift warnings, and errors while keeping detailed helper output in that log. A different installed `cuda-keyring` version remains usable only when `dpkg -V` verifies its package files and the CUDA repository signing fingerprint matches the expected NVIDIA fingerprint. New Docker repository setup uses the reviewed `arch=arm64`, `https://download.docker.com/linux/ubuntu`, `noble`, and `stable` source with the verified dearmored `/etc/apt/keyrings/docker.gpg` key. An existing source using `/etc/apt/keyrings/docker.asc` is retained only when its safe root-owned regular source file exactly matches the reviewed architecture, URL, suite, component, and signing-key path. @@ -209,6 +210,9 @@ Use these details when your first-run path needs more control. DGX OS, NVIDIA BaseOS images, and other Station generations are outside this automatic preparation boundary. Set `NEMOCLAW_PROVIDER` or `NEMOCLAW_NO_EXPRESS=1` to continue without Station host automation. Preparation retains installed packages, installs reviewed pinned versions only for missing packages, and warns with actual and expected versions when an installed prerequisite or the loaded NVIDIA driver differs. + During express install, the terminal shows the Station host preparation log path, package and driver version-drift warnings with actual and expected versions, and errors. + The full platform, systemd, Docker pull, NVIDIA-SMI, and status output remains in that log. + Running `scripts/prepare-dgx-station-host.sh` directly retains the verbose output. A different installed `cuda-keyring` version remains usable only when `dpkg -V` verifies its package files and the CUDA repository signing fingerprint matches the expected NVIDIA fingerprint. New Docker repository setup uses the reviewed `arch=arm64`, `https://download.docker.com/linux/ubuntu`, `noble`, and `stable` source with the verified dearmored `/etc/apt/keyrings/docker.gpg` key. An existing source using `/etc/apt/keyrings/docker.asc` is retained only when its root-owned regular source file has mode `0644` and exactly matches the reviewed architecture, URL, suite, component, and signing-key path. diff --git a/scripts/install.sh b/scripts/install.sh index d2c35471069..c4abb97dff5 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -3216,7 +3216,25 @@ run_station_host_preparation() { # fail-closed check so Station preparation cannot drift from that ref. local helper="${SCRIPT_DIR}/prepare-dgx-station-host.sh" [[ -f "$helper" ]] || error "DGX Station host preparation helper is missing: ${helper}" - bash "$helper" --apply + bash "$helper" --apply 2>&1 | filter_station_host_preparation_output +} + +filter_station_host_preparation_output() { + local line detail + while IFS= read -r line; do + case "$line" in + *" version="*" log="*) + info "DGX Station host preparation log: ${line##* log=}" + ;; + *" WARNING: package="*" status=version_drift "* | *" WARNING: driver status=version_drift "*) + detail="${line#* WARNING: }" + warn "$detail" + ;; + *" ERROR: "*) + printf '%s\n' "$line" >&2 + ;; + esac + done } ensure_station_express_host() { diff --git a/test/install-station-host-preparation.test.ts b/test/install-station-host-preparation.test.ts index bdaf21204f9..a50c0fb54bd 100644 --- a/test/install-station-host-preparation.test.ts +++ b/test/install-station-host-preparation.test.ts @@ -337,6 +337,60 @@ describe_express_install 'DGX Station' ); }); + it("keeps Station preparation details in the log while showing actionable installer output", () => { + const { result, output } = runSourced( + INSTALLER_PAYLOAD, + ` +printf '%s\n' \\ + '[station-prepare] 2026-07-17T07:59:07Z version=2026-07-17.2 mode=--apply log=/tmp/station-prepare.log' \\ + '[station-prepare] 2026-07-17T07:59:07Z platform=Dell Pro Max with Station GB300 os=Ubuntu 24.04.4 LTS' \\ + '[station-prepare] 2026-07-17T07:59:08Z WARNING: condition-qualified generic-image failed unit: cloud-init.service' \\ + '[station-prepare] 2026-07-17T07:59:08Z WARNING: package=dkms status=version_drift actual=1:3.4.1-1ubuntu1 expected=1:3.4.0-1ubuntu1; retaining installed version' \\ + '[station-prepare] 2026-07-17T07:59:09Z WARNING: driver status=version_drift actual=611.0 expected=610.43.02; retaining loaded version' \\ + 'NVIDIA-SMI 610.43.02' \\ + '[station-prepare] 2026-07-17T07:59:20Z STATION_HOST_READY' \\ + '[station-prepare] 2026-07-17T07:59:20Z APPLY_RESULT=COMPLETE' \\ + '[station-prepare] 2026-07-17T07:59:20Z ERROR: example failure' \\ + | filter_station_host_preparation_output +`, + ); + + expect(result.status, output).toBe(0); + expect(output).toContain("DGX Station host preparation log: /tmp/station-prepare.log"); + expect(output).toContain( + "package=dkms status=version_drift actual=1:3.4.1-1ubuntu1 expected=1:3.4.0-1ubuntu1; retaining installed version", + ); + expect(output).toContain( + "driver status=version_drift actual=611.0 expected=610.43.02; retaining loaded version", + ); + expect(output).toContain("ERROR: example failure"); + expect(output).not.toMatch(/platform=Dell Pro Max|cloud-init\.service|NVIDIA-SMI/); + }); + + it("preserves the Station helper exit status while filtering installer output", () => { + const { result, output } = runSourced( + INSTALLER_PAYLOAD, + ` +bash() { + printf '%s\n' \\ + '[station-prepare] 2026-07-17T07:59:07Z version=2026-07-17.2 mode=--apply log=/tmp/station-prepare.log' \\ + '[station-prepare] 2026-07-17T07:59:08Z runtime_setup=complete' + return 10 +} +if run_station_host_preparation; then + printf 'STATUS=0\n' +else + printf 'STATUS=%s\n' "$?" +fi +`, + ); + + expect(result.status, output).toBe(0); + expect(output).toContain("STATUS=10"); + expect(output).toContain("DGX Station host preparation log: /tmp/station-prepare.log"); + expect(output).not.toContain("runtime_setup=complete"); + }); + it("fails closed when failed-service inspection is unavailable", () => { const { result, output } = runSourced( STATION_PREPARE, @@ -1125,6 +1179,7 @@ PAYLOAD #!/usr/bin/env bash set -euo pipefail [ "\${1:-}" = "--apply" ] +printf '[station-prepare] 2026-07-17T07:59:07Z version=2026-07-17.2 mode=--apply log=/tmp/station-prepare.log\\n' printf 'PREPARE_STATION\\n' HELPER chmod +x "$target/scripts/install.sh" "$target/scripts/prepare-dgx-station-host.sh" @@ -1154,11 +1209,12 @@ exit 0 killSignal: "SIGKILL", }); const output = `${result.stdout}${result.stderr}`; + const preparationLogIndex = output.indexOf("DGX Station host preparation log"); expect(result.status, output).toBe(0); expect(output).toContain("DGX Station host prerequisites are ready"); - expect(output.indexOf("PREPARE_STATION")).toBeGreaterThanOrEqual(0); - expect(output.indexOf("PREPARE_STATION")).toBeLessThan(output.indexOf("ENSURE_DOCKER")); + expect(preparationLogIndex).toBeGreaterThanOrEqual(0); + expect(preparationLogIndex).toBeLessThan(output.indexOf("ENSURE_DOCKER")); expect(output.indexOf("ENSURE_DOCKER")).toBeLessThan(output.indexOf("ENSURE_BUILD_DEPS")); }); From 45719e0ca72957763c56fc32d93b6eeab2a45d27 Mon Sep 17 00:00:00 2001 From: San Dang Date: Fri, 17 Jul 2026 15:19:41 +0700 Subject: [PATCH 5/7] test(installer): assert exact missing package commands Signed-off-by: San Dang --- test/install-station-host-preparation.test.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/test/install-station-host-preparation.test.ts b/test/install-station-host-preparation.test.ts index a50c0fb54bd..fc5b9a498d7 100644 --- a/test/install-station-host-preparation.test.ts +++ b/test/install-station-host-preparation.test.ts @@ -501,17 +501,17 @@ cat "$HOME/apt-cache-calls" ); expect(result.status, output).toBe(0); - expect(output).toContain("apt-get update"); - expect(output).toContain("apt-get install -y --no-install-recommends"); - expect(output).toContain("RECHECK_ALL_WORKLOADS"); - expect(output).toContain("APT_CACHE show docker-ce=5:29.6.1-1~ubuntu.24.04~noble"); - expect(output).toContain( + const aptCommands = output + .split("\n") + .filter((line) => + /^(APT_CACHE show |APT_GET -s install |SUDO env .* apt-get install )/.test(line), + ) + .sort(); + expect(aptCommands).toEqual([ + "APT_CACHE show docker-ce=5:29.6.1-1~ubuntu.24.04~noble", "APT_GET -s install --no-install-recommends docker-ce=5:29.6.1-1~ubuntu.24.04~noble", - ); - expect(output).toContain( "SUDO env DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends docker-ce=5:29.6.1-1~ubuntu.24.04~noble", - ); - expect(output).not.toContain("apt-get install -y --no-install-recommends dkms="); + ]); expect(output).toContain("pinned_packages=installed"); }); From c6642b6d961728b7bcf1892e396419353e54a091 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Fri, 17 Jul 2026 17:25:42 -0700 Subject: [PATCH 6/7] fix(installer): bind Station package transaction Co-authored-by: San Dang Signed-off-by: Apurv Kumaria --- scripts/prepare-dgx-station-host.sh | 277 +++++++++++++- ...nstall-station-package-transaction.test.ts | 344 +++++++++++++++++- .../install-station-platform-identity.test.ts | 6 +- 3 files changed, 607 insertions(+), 20 deletions(-) diff --git a/scripts/prepare-dgx-station-host.sh b/scripts/prepare-dgx-station-host.sh index 2770ba69dee..ad0fad67fd0 100755 --- a/scripts/prepare-dgx-station-host.sh +++ b/scripts/prepare-dgx-station-host.sh @@ -179,6 +179,9 @@ LOG_FILE="" DOCKER_GROUP_ADDED=0 CDI_LIFECYCLE_READY=0 NETWORK_VALIDATED=0 +PACKAGE_TRANSACTION_SPECS=() +APT_TRANSACTION_GUARD_DIR="" +APT_TRANSACTION_HOOK="" info() { printf '[station-prepare] %s %s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$*" @@ -872,35 +875,286 @@ configure_repositories() { info "repository_keys=verified" } +collect_package_transaction_specs() { + local spec state + PACKAGE_TRANSACTION_SPECS=() + for spec in "${PACKAGE_SPECS[@]}"; do + state="$(package_state "$spec")" + case "$state" in + missing | approved-transition) PACKAGE_TRANSACTION_SPECS+=("$spec") ;; + exact) ;; + *) fatal "Package transaction contains an unapproved prerequisite state: ${spec} (${state})" ;; + esac + done + ((${#PACKAGE_TRANSACTION_SPECS[@]} > 0)) \ + || fatal "Package transaction has no missing or approved-transition prerequisites" +} + validate_package_availability() { local spec - for spec in "${PACKAGE_SPECS[@]}"; do + for spec in "$@"; do apt-cache show "$spec" >/dev/null 2>&1 || fatal "Exact package version is unavailable: ${spec}" done info "exact_package_versions=available" } +apt_guard_fatal() { + printf 'APT transaction guard: %s\n' "$*" >&2 + return 1 +} + +validate_apt_preinstall_plan() { + local targets_file=$1 line package old_version old_arch old_multiarch direction + local new_version new_arch new_multiarch action extra record_key + local target expected allowed_old native_arch target_line target_expected target_allowed_old target_native_arch + local manifest_native_arch="" + local configured='|' changed='|' seen_targets='|' target_names='|' + [[ -r "$targets_file" && -f "$targets_file" && ! -L "$targets_file" ]] \ + || apt_guard_fatal "target manifest is unavailable or unsafe: ${targets_file}" || return + + while IFS='|' read -r target expected allowed_old native_arch extra; do + [[ -n "$target" && -n "$expected" && -n "$native_arch" && -z "$extra" ]] \ + || apt_guard_fatal "target manifest contains an invalid record" || return + [[ "$target" =~ ^[a-z0-9][a-z0-9+.-]+$ ]] \ + || apt_guard_fatal "target manifest contains an invalid package name: ${target}" || return + [[ "$expected" != *[[:space:]]* && "$expected" != *"|"* && + "$allowed_old" != *[[:space:]]* && "$allowed_old" != *"|"* && + "$native_arch" =~ ^[a-z0-9][a-z0-9-]*$ ]] \ + || apt_guard_fatal "target manifest contains an invalid version for ${target}" || return + [[ "$target_names" != *"|${target}|"* ]] \ + || apt_guard_fatal "target manifest repeats package ${target}" || return + if [[ -z "$manifest_native_arch" ]]; then + manifest_native_arch=$native_arch + else + [[ "$native_arch" == "$manifest_native_arch" ]] \ + || apt_guard_fatal "target manifest mixes native architectures" || return + fi + target_names="${target_names}${target}|" + done <"$targets_file" + [[ "$target_names" != "|" ]] || apt_guard_fatal "target manifest is empty" || return + + IFS= read -r line || apt_guard_fatal "APT omitted the pre-install protocol header" || return + [[ "$line" == "VERSION 3" ]] \ + || apt_guard_fatal "APT pre-install protocol must be VERSION 3, found: ${line}" || return + while IFS= read -r line; do + [[ -n "$line" ]] || break + done + [[ -z "$line" ]] || apt_guard_fatal "APT pre-install protocol omitted its record separator" || return + + while read -r package old_version old_arch old_multiarch direction new_version new_arch new_multiarch action extra; do + [[ -n "$package" && -n "$old_version" && -n "$old_arch" && -n "$old_multiarch" && + -n "$direction" && -n "$new_version" && -n "$new_arch" && -n "$new_multiarch" && + -n "$action" && -z "$extra" ]] \ + || apt_guard_fatal "APT emitted a malformed package action" || return + [[ "$package" =~ ^[a-z0-9][a-z0-9+.-]+$ ]] \ + || apt_guard_fatal "APT emitted an invalid package name: ${package}" || return + [[ "$old_arch" == "-" || "$old_arch" =~ ^[a-z0-9][a-z0-9-]*$ ]] \ + || apt_guard_fatal "APT emitted an invalid old architecture for ${package}: ${old_arch}" || return + [[ "$new_arch" == "-" || "$new_arch" =~ ^[a-z0-9][a-z0-9-]*$ ]] \ + || apt_guard_fatal "APT emitted an invalid new architecture for ${package}: ${new_arch}" || return + [[ "$old_multiarch" =~ ^(same|foreign|allowed|none|no|-)$ && + "$new_multiarch" =~ ^(same|foreign|allowed|none|no|-)$ ]] \ + || apt_guard_fatal "APT emitted an invalid Multi-Arch field for ${package}" || return + record_key="${package}@${new_arch}" + case "$action" in + "**REMOVE**") apt_guard_fatal "APT proposed removing ${package}" || return ;; + "**CONFIGURE**") + configured="${configured}${record_key}|" + ;; + /*) + [[ "$changed" != *"|${record_key}|"* ]] \ + || apt_guard_fatal "APT proposed duplicate archive actions for ${record_key}" || return + changed="${changed}${record_key}|" + target="" + expected="" + allowed_old="" + native_arch="" + while IFS='|' read -r target_line target_expected target_allowed_old target_native_arch; do + if [[ "$target_line" == "$package" ]]; then + target=$target_line + expected=$target_expected + allowed_old=$target_allowed_old + native_arch=$target_native_arch + break + fi + done <"$targets_file" + if [[ -z "$native_arch" ]]; then + native_arch=$manifest_native_arch + fi + [[ "$new_arch" == "$native_arch" || "$new_arch" == "all" ]] \ + || apt_guard_fatal "APT selected foreign architecture ${new_arch} for ${package}; expected ${native_arch} or all" || return + if [[ -n "$target" ]]; then + [[ "$new_version" == "$expected" ]] \ + || apt_guard_fatal "APT selected ${package}=${new_version}; expected ${expected}" || return + if [[ -n "$allowed_old" ]]; then + [[ "$old_version" == "$allowed_old" && "$direction" == "<" && + ("$old_arch" == "$native_arch" || "$old_arch" == "all") ]] \ + || apt_guard_fatal "APT changed approved transition ${package} from ${old_version}; expected ${allowed_old}" || return + else + [[ "$old_version" == "-" && "$old_arch" == "-" && "$direction" == "<" ]] \ + || apt_guard_fatal "APT proposed changing retained target ${package}=${old_version}" || return + fi + seen_targets="${seen_targets}${package}|" + else + [[ "$old_version" == "-" && "$old_arch" == "-" && "$direction" == "<" ]] \ + || apt_guard_fatal "APT proposed changing retained package ${package}=${old_version}" || return + fi + ;; + *) apt_guard_fatal "APT emitted an unsupported action for ${package}: ${action}" || return ;; + esac + done + + while IFS='|' read -r target expected allowed_old native_arch; do + [[ "$seen_targets" == *"|${target}|"* ]] \ + || apt_guard_fatal "APT omitted required target ${target}=${expected}" || return + done <"$targets_file" + while [[ "$configured" != "|" ]]; do + configured="${configured#|}" + package="${configured%%|*}" + configured="|${configured#*|}" + [[ "$changed" == *"|${package}|"* ]] \ + || apt_guard_fatal "APT proposed configuring retained package ${package} without an archive action" || return + done +} + +cleanup_apt_transaction_guard() { + local guard_dir=${APT_TRANSACTION_GUARD_DIR:-} + APT_TRANSACTION_GUARD_DIR="" + APT_TRANSACTION_HOOK="" + [[ -n "$guard_dir" ]] || return 0 + if [[ ! "$guard_dir" =~ ^/run/nemoclaw-apt-transaction\.[A-Za-z0-9]+$ ]]; then + warn "refusing to clean unexpected APT transaction guard path: ${guard_dir}" + return 0 + fi + sudo rm -rf -- "$guard_dir" \ + || warn "could not remove APT transaction guard directory: ${guard_dir}" +} + +create_apt_transaction_guard() { + local spec state name expected allowed_old native_arch targets="" hook_path targets_path + native_arch="$(sudo dpkg --print-architecture)" + [[ "$native_arch" =~ ^[a-z0-9][a-z0-9-]*$ ]] \ + || fatal "Could not determine the native package architecture" + for spec in "${PACKAGE_TRANSACTION_SPECS[@]}"; do + state="$(package_state "$spec")" + name="$(package_name "$spec")" + expected="$(package_expected_version "$spec")" + case "$state" in + missing) allowed_old="" ;; + approved-transition) allowed_old="$(installed_version "$name")" ;; + *) fatal "Cannot authorize APT target ${spec} from state ${state}" ;; + esac + targets="${targets}${name}|${expected}|${allowed_old}|${native_arch}"$'\n' + done + + APT_TRANSACTION_GUARD_DIR="$(sudo mktemp -d /run/nemoclaw-apt-transaction.XXXXXXXXXX)" \ + || fatal "Could not create the root-owned APT transaction guard directory" + [[ "$APT_TRANSACTION_GUARD_DIR" =~ ^/run/nemoclaw-apt-transaction\.[A-Za-z0-9]+$ ]] \ + || fatal "APT transaction guard returned an unexpected path: ${APT_TRANSACTION_GUARD_DIR}" + hook_path="${APT_TRANSACTION_GUARD_DIR}/verify-plan" + targets_path="${APT_TRANSACTION_GUARD_DIR}/targets" + { + printf '%s\n' '#!/usr/bin/env bash' 'set -euo pipefail' + declare -f apt_guard_fatal + declare -f validate_apt_preinstall_plan + # The generated hook expands its own path at execution time. + # shellcheck disable=SC2016 + printf '%s\n' 'validate_apt_preinstall_plan "${0%/*}/targets"' + } | sudo tee "$hook_path" >/dev/null + printf '%s' "$targets" | sudo tee "$targets_path" >/dev/null + sudo chmod 0700 "$hook_path" + sudo chmod 0600 "$targets_path" + assert_root_directory_safe "$APT_TRANSACTION_GUARD_DIR" "APT transaction guard directory" + assert_root_regular_file_safe "$hook_path" 0700 "APT transaction guard" + assert_root_regular_file_safe "$targets_path" 0600 "APT transaction target manifest" + APT_TRANSACTION_HOOK=$hook_path +} + +validate_apt_simulation() { + local simulation=$1 + shift + local spec target_spec name expected actual line action version before_version + local simulated_targets='|' simulated_changes='|' + + while IFS= read -r line; do + [[ "$line" =~ ^(Inst|Conf|Remv|Purg)[[:space:]] ]] || continue + read -r action name _ <<<"$line" + case "$action" in + Remv | Purg) fatal "APT simulation proposed a package removal: ${line}" ;; + Conf) + [[ "$simulated_changes" == *"|${name}|"* ]] \ + || fatal "APT simulation proposed configuration without an approved install: ${line}" + ;; + Inst) + [[ "$simulated_changes" != *"|${name}|"* ]] \ + || fatal "APT simulation proposed a duplicate package change: ${line}" + simulated_changes="${simulated_changes}${name}|" + target_spec="" + for spec in "$@"; do + if [[ "$(package_name "$spec")" == "$name" ]]; then + target_spec=$spec + break + fi + done + if [[ -n "$target_spec" ]]; then + [[ "$line" == *"("* ]] || fatal "APT simulation omitted the target version: ${line}" + version="${line#*(}" + version="${version%%[[:space:]]*}" + version="${version%)}" + expected="$(package_expected_version "$target_spec")" + [[ "$version" == "$expected" ]] \ + || fatal "APT simulation selected ${name}=${version}; expected ${expected}" + simulated_targets="${simulated_targets}${name}|" + continue + fi + + # A target package may require a new dependency, but APT must not + # upgrade, downgrade, or reinstall any package retained from the host. + actual="$(installed_version "$name")" + before_version="${line%%(*}" + [[ -z "$actual" && "$before_version" != *"["* ]] \ + || fatal "APT simulation proposed changing retained package ${name}=${actual}: ${line}" + ;; + esac + done <<<"$simulation" + + for spec in "$@"; do + name="$(package_name "$spec")" + [[ "$simulated_targets" == *"|${name}|"* ]] \ + || fatal "APT simulation did not include required package ${spec}" + done +} + simulate_install() { local simulation - simulation="$(apt-get -s install --no-install-recommends "${PACKAGE_SPECS[@]}")" \ + [[ "$APT_TRANSACTION_HOOK" =~ ^/run/nemoclaw-apt-transaction\.[A-Za-z0-9]+/verify-plan$ ]] \ + || fatal "APT transaction guard is not ready" + simulation="$(sudo env DEBIAN_FRONTEND=noninteractive LC_ALL=C \ + apt-get -s install --no-install-recommends --no-remove \ + -o "DPkg::Pre-Install-Pkgs::=${APT_TRANSACTION_HOOK}" \ + -o "DPkg::Tools::options::${APT_TRANSACTION_HOOK}::Version=3" "$@")" \ || fatal "APT simulation failed" printf '%s\n' "$simulation" - if grep -Eq '^(Remv |Purg )' <<<"$simulation"; then - fatal "APT simulation proposed a package removal" - fi - info "apt_simulation=no_removals" + validate_apt_simulation "$simulation" "$@" + info "apt_simulation=missing_only retained_packages=unchanged" } install_packages() { + collect_package_transaction_specs configure_repositories info "Refreshing package metadata" sudo apt-get update - validate_package_availability - simulate_install + validate_package_availability "${PACKAGE_TRANSACTION_SPECS[@]}" + create_apt_transaction_guard + simulate_install "${PACKAGE_TRANSACTION_SPECS[@]}" check_no_workloads - info "Installing pinned Station prerequisites" - sudo env DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - "${PACKAGE_SPECS[@]}" + info "Installing missing pinned Station prerequisites" + sudo env DEBIAN_FRONTEND=noninteractive LC_ALL=C \ + apt-get install -y --no-install-recommends --no-remove \ + -o "DPkg::Pre-Install-Pkgs::=${APT_TRANSACTION_HOOK}" \ + -o "DPkg::Tools::options::${APT_TRANSACTION_HOOK}::Version=3" \ + "${PACKAGE_TRANSACTION_SPECS[@]}" + cleanup_apt_transaction_guard local spec for spec in "${PACKAGE_SPECS[@]}"; do @@ -1339,6 +1593,7 @@ main() { info "version=${SCRIPT_VERSION} mode=${MODE} log=disabled_read_only" fi trap 'on_error "$LINENO"' ERR + trap 'cleanup_apt_transaction_guard' EXIT case "$MODE" in --check) run_check ;; --apply) run_apply ;; diff --git a/test/install-station-package-transaction.test.ts b/test/install-station-package-transaction.test.ts index 49d409504f6..1682a5b0021 100644 --- a/test/install-station-package-transaction.test.ts +++ b/test/install-station-package-transaction.test.ts @@ -23,8 +23,10 @@ const EXPECTED_PACKAGE_SPECS = [ "nvidia-container-toolkit=1.19.1-1", "nvidia-container-toolkit-base=1.19.1-1", ]; +const DOCKER_CE_SPEC = "docker-ce=5:29.6.1-1~ubuntu.24.04~noble"; +const DKMS_SPEC = "dkms=1:3.4.0-1ubuntu1"; -function runSourced(body: string) { +function runSourced(body: string, extraEnv: NodeJS.ProcessEnv = {}) { const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-station-package-transaction-")); const result = spawnSync( "bash", @@ -36,6 +38,7 @@ function runSourced(body: string) { HOME: home, PATH: TEST_SYSTEM_PATH, SCRIPT_UNDER_TEST: STATION_PREPARE, + ...extraEnv, }, timeout: 15_000, killSignal: "SIGKILL", @@ -44,15 +47,67 @@ function runSourced(body: string) { return { result, output: `${result.stdout}${result.stderr}` }; } +function validateSimulation(plan: string, specs = [DOCKER_CE_SPEC]) { + return runSourced( + ` +installed_version() { :; } +validate_apt_simulation "$APT_PLAN" ${specs.map((spec) => `'${spec}'`).join(" ")} +`, + { APT_PLAN: plan }, + ); +} + +function aptProtocol(...actions: string[]) { + return ["VERSION 3", "APT::Architecture=arm64", "", ...actions].join("\n"); +} + +function validatePreinstallPlan(targets: string, plan: string) { + return runSourced( + ` +printf '%s' "$APT_TARGETS" >"$HOME/targets" +validate_apt_preinstall_plan "$HOME/targets" <<<"$APT_PLAN" +`, + { APT_PLAN: plan, APT_TARGETS: targets }, + ); +} + describe("DGX Station package transaction", () => { - it("passes the complete pinned tuple to availability, simulation, and install", () => { + it("passes the complete pinned tuple when every package is missing", () => { const { result, output } = runSourced(` configure_repositories() { printf 'CONFIGURE_REPOSITORIES\n'; } apt-cache() { printf 'APT_CACHE %s\n' "$*" >>"$HOME/apt-cache-calls"; } -apt-get() { printf 'APT_GET %s\n' "$*"; } +apt-get() { + printf 'APT_GET %s\n' "$*" + if [[ "$1" == "-s" ]]; then + local spec name version + for spec in "$@"; do + [[ "$spec" == [a-z0-9]*=* ]] || continue + name="\${spec%%=*}" + version="\${spec#*=}" + printf 'Inst %s (%s fixture [arm64])\n' "$name" "$version" + printf 'Conf %s (%s fixture [arm64])\n' "$name" "$version" + done + fi +} check_no_workloads() { printf 'RECHECK_ALL_WORKLOADS\n'; } +package_state() { printf 'missing\n'; } package_is_exact() { return 0; } -sudo() { printf 'SUDO %s\n' "$*"; } +create_apt_transaction_guard() { + APT_TRANSACTION_GUARD_DIR=/run/nemoclaw-apt-transaction.TEST + APT_TRANSACTION_HOOK="$APT_TRANSACTION_GUARD_DIR/verify-plan" +} +cleanup_apt_transaction_guard() { + printf 'CLEANUP_GUARD\n' + APT_TRANSACTION_GUARD_DIR="" + APT_TRANSACTION_HOOK="" +} +sudo() { + printf 'SUDO %s\n' "$*" + if [[ "$1" == "env" && "$*" == *" apt-get -s install "* ]]; then + while [[ "$1" != "apt-get" ]]; do shift; done + "$@" + fi +} install_packages cat "$HOME/apt-cache-calls" `); @@ -68,11 +123,288 @@ cat "$HOME/apt-cache-calls" expect(aptCommands).toEqual( [ ...EXPECTED_PACKAGE_SPECS.map((spec) => `APT_CACHE show ${spec}`), - `APT_GET -s install --no-install-recommends ${expectedTuple}`, - `SUDO env DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends ${expectedTuple}`, + `APT_GET -s install --no-install-recommends --no-remove -o DPkg::Pre-Install-Pkgs::=/run/nemoclaw-apt-transaction.TEST/verify-plan -o DPkg::Tools::options::/run/nemoclaw-apt-transaction.TEST/verify-plan::Version=3 ${expectedTuple}`, + `SUDO env DEBIAN_FRONTEND=noninteractive LC_ALL=C apt-get install -y --no-install-recommends --no-remove -o DPkg::Pre-Install-Pkgs::=/run/nemoclaw-apt-transaction.TEST/verify-plan -o DPkg::Tools::options::/run/nemoclaw-apt-transaction.TEST/verify-plan::Version=3 ${expectedTuple}`, ].sort(), ); + expect(output).toContain( + "SUDO env DEBIAN_FRONTEND=noninteractive LC_ALL=C apt-get -s install --no-install-recommends --no-remove", + ); expect(output).toContain("RECHECK_ALL_WORKLOADS"); + expect(output).toContain("CLEANUP_GUARD"); expect(output).toContain("pinned_packages=installed"); }); + + it("excludes retained exact packages from every APT transaction command", () => { + const retainedSpec = "docker-ce=5:29.6.1-1~ubuntu.24.04~noble"; + const missingSpecs = EXPECTED_PACKAGE_SPECS.filter((spec) => spec !== retainedSpec); + const { result, output } = runSourced(` +configure_repositories() { :; } +apt-cache() { printf 'APT_CACHE %s\n' "$*" >>"$HOME/apt-cache-calls"; } +apt-get() { + printf 'APT_GET %s\n' "$*" + if [[ "$1" == "-s" ]]; then + local spec name version + for spec in "$@"; do + [[ "$spec" == [a-z0-9]*=* ]] || continue + name="\${spec%%=*}" + version="\${spec#*=}" + printf 'Inst %s (%s fixture [arm64])\n' "$name" "$version" + printf 'Conf %s (%s fixture [arm64])\n' "$name" "$version" + done + fi +} +check_no_workloads() { :; } +package_state() { + if [[ "$1" == '${retainedSpec}' ]]; then printf 'exact\n'; else printf 'missing\n'; fi +} +package_is_exact() { return 0; } +create_apt_transaction_guard() { + APT_TRANSACTION_GUARD_DIR=/run/nemoclaw-apt-transaction.TEST + APT_TRANSACTION_HOOK="$APT_TRANSACTION_GUARD_DIR/verify-plan" +} +cleanup_apt_transaction_guard() { + APT_TRANSACTION_GUARD_DIR="" + APT_TRANSACTION_HOOK="" +} +sudo() { + printf 'SUDO %s\n' "$*" + if [[ "$1" == "env" && "$*" == *" apt-get -s install "* ]]; then + while [[ "$1" != "apt-get" ]]; do shift; done + "$@" + fi +} +install_packages +cat "$HOME/apt-cache-calls" +`); + + expect(result.status, output).toBe(0); + const expectedTuple = missingSpecs.join(" "); + const aptCommands = output + .split("\n") + .filter((line) => + /^(APT_CACHE show |APT_GET -s install |SUDO env .* apt-get install )/.test(line), + ) + .sort(); + expect(aptCommands).toEqual( + [ + ...missingSpecs.map((spec) => `APT_CACHE show ${spec}`), + `APT_GET -s install --no-install-recommends --no-remove -o DPkg::Pre-Install-Pkgs::=/run/nemoclaw-apt-transaction.TEST/verify-plan -o DPkg::Tools::options::/run/nemoclaw-apt-transaction.TEST/verify-plan::Version=3 ${expectedTuple}`, + `SUDO env DEBIAN_FRONTEND=noninteractive LC_ALL=C apt-get install -y --no-install-recommends --no-remove -o DPkg::Pre-Install-Pkgs::=/run/nemoclaw-apt-transaction.TEST/verify-plan -o DPkg::Tools::options::/run/nemoclaw-apt-transaction.TEST/verify-plan::Version=3 ${expectedTuple}`, + ].sort(), + ); + expect(aptCommands.join("\n")).not.toContain(retainedSpec); + }); + + it("rejects a simulated change to a retained dependency", () => { + const { result, output } = runSourced(` +configure_repositories() { :; } +apt-cache() { :; } +apt-get() { + if [[ "$1" == "-s" ]]; then + printf '%s\n' \ + 'Inst docker-ce (5:29.6.1-1~ubuntu.24.04~noble fixture [arm64])' \ + 'Inst libc6 [2.39-0ubuntu8] (2.39-0ubuntu9 fixture [arm64])' + fi +} +package_state() { + if [[ "$1" == docker-ce=* ]]; then printf 'missing\n'; else printf 'exact\n'; fi +} +installed_version() { if [[ "$1" == "libc6" ]]; then printf '2.39-0ubuntu8'; fi; } +package_is_exact() { return 0; } +create_apt_transaction_guard() { + APT_TRANSACTION_GUARD_DIR=/run/nemoclaw-apt-transaction.TEST + APT_TRANSACTION_HOOK="$APT_TRANSACTION_GUARD_DIR/verify-plan" +} +sudo() { + printf 'SUDO %s\n' "$*" + if [[ "$1" == "env" && "$*" == *" apt-get -s install "* ]]; then + while [[ "$1" != "apt-get" ]]; do shift; done + "$@" + fi +} +install_packages +`); + + expect(result.status, output).not.toBe(0); + expect(output).toContain( + "APT simulation proposed changing retained package libc6=2.39-0ubuntu8", + ); + expect(output).not.toContain("apt-get install -y"); + }); + + it("rejects unsafe simulation actions before the privileged install", () => { + const expected = "5:29.6.1-1~ubuntu.24.04~noble"; + const scenarios = [ + { + plan: `Inst docker-ce (${expected} fixture [arm64])\nRemv libc6 [2.39-0ubuntu8]`, + message: "APT simulation proposed a package removal", + }, + { + plan: "Inst docker-ce (5:29.5.0-1~ubuntu.24.04~noble fixture [arm64])", + message: "APT simulation selected docker-ce=5:29.5.0-1~ubuntu.24.04~noble", + }, + { + plan: "Inst pigz (2.8-1 fixture [arm64])", + message: `APT simulation did not include required package ${DOCKER_CE_SPEC}`, + }, + { + plan: `Inst docker-ce (${expected} fixture [arm64])\nConf libc6 (2.39-0ubuntu8 fixture [arm64])`, + message: "APT simulation proposed configuration without an approved install", + }, + ]; + + for (const scenario of scenarios) { + const { result, output } = validateSimulation(scenario.plan); + expect(result.status, `${scenario.plan}\n${output}`).not.toBe(0); + expect(output).toContain(scenario.message); + } + }); + + it("allows the approved DKMS transition and genuinely new dependencies in simulation", () => { + const transition = validateSimulation( + [ + "Inst dkms [3.0.11-1ubuntu13] (1:3.4.0-1ubuntu1 fixture [all])", + "Conf dkms (1:3.4.0-1ubuntu1 fixture [all])", + ].join("\n"), + [DKMS_SPEC], + ); + expect(transition.result.status, transition.output).toBe(0); + + const dependency = validateSimulation( + [ + "Inst docker-ce (5:29.6.1-1~ubuntu.24.04~noble fixture [arm64])", + "Inst pigz (2.8-1 fixture [arm64])", + "Conf docker-ce (5:29.6.1-1~ubuntu.24.04~noble fixture [arm64])", + "Conf pigz (2.8-1 fixture [arm64])", + ].join("\n"), + ); + expect(dependency.result.status, dependency.output).toBe(0); + }); + + it("accepts only missing packages, new dependencies, and the approved transition in the actual plan", () => { + const missingWithDependency = validatePreinstallPlan( + "docker-ce|5:29.6.1-1~ubuntu.24.04~noble||arm64\n", + aptProtocol( + "docker-ce - - none < 5:29.6.1-1~ubuntu.24.04~noble arm64 no /var/cache/apt/archives/docker-ce.deb", + "pigz - - none < 2.8-1 arm64 no /var/cache/apt/archives/pigz.deb", + "docker-ce - - none < 5:29.6.1-1~ubuntu.24.04~noble arm64 no **CONFIGURE**", + "pigz - - none < 2.8-1 arm64 no **CONFIGURE**", + ), + ); + expect(missingWithDependency.result.status, missingWithDependency.output).toBe(0); + + const transition = validatePreinstallPlan( + "dkms|1:3.4.0-1ubuntu1|3.0.11-1ubuntu13|arm64\n", + aptProtocol( + "dkms 3.0.11-1ubuntu13 all foreign < 1:3.4.0-1ubuntu1 all foreign /var/cache/apt/archives/dkms.deb", + "dkms 3.0.11-1ubuntu13 all foreign < 1:3.4.0-1ubuntu1 all foreign **CONFIGURE**", + ), + ); + expect(transition.result.status, transition.output).toBe(0); + }); + + it("rejects unsafe VERSION 3 actions in the actual pre-install plan", () => { + const targets = "docker-ce|5:29.6.1-1~ubuntu.24.04~noble||arm64\n"; + const targetAction = + "docker-ce - - none < 5:29.6.1-1~ubuntu.24.04~noble arm64 no /var/cache/apt/archives/docker-ce.deb"; + const scenarios = [ + { + plan: aptProtocol(targetAction).replace("VERSION 3", "VERSION 2"), + message: "APT pre-install protocol must be VERSION 3", + }, + { + plan: aptProtocol( + targetAction, + "libc6 2.39-0ubuntu8 arm64 same < 2.39-0ubuntu9 arm64 same /var/cache/apt/archives/libc6.deb", + ), + message: "APT proposed changing retained package libc6=2.39-0ubuntu8", + }, + { + plan: aptProtocol(targetAction, "obsolete 1.0 arm64 no > - - none **REMOVE**"), + message: "APT proposed removing obsolete", + }, + { + plan: aptProtocol( + "docker-ce - - none < 5:29.5.0-1~ubuntu.24.04~noble arm64 no /var/cache/apt/archives/docker-ce.deb", + ), + message: "APT selected docker-ce=5:29.5.0-1~ubuntu.24.04~noble", + }, + { + plan: aptProtocol("pigz - - none < 2.8-1 arm64 no /var/cache/apt/archives/pigz.deb"), + message: "APT omitted required target docker-ce=5:29.6.1-1~ubuntu.24.04~noble", + }, + { + plan: aptProtocol( + targetAction, + "libc6 2.39-0ubuntu8 arm64 same = 2.39-0ubuntu8 arm64 same **CONFIGURE**", + ), + message: "APT proposed configuring retained package libc6@arm64 without an archive action", + }, + { + plan: aptProtocol( + "docker-ce - - none < 5:29.6.1-1~ubuntu.24.04~noble amd64 no /var/cache/apt/archives/docker-ce.deb", + ), + message: "APT selected foreign architecture amd64 for docker-ce; expected arm64 or all", + }, + ]; + + for (const scenario of scenarios) { + const { result, output } = validatePreinstallPlan(targets, scenario.plan); + expect(result.status, `${scenario.plan}\n${output}`).not.toBe(0); + expect(output).toContain(scenario.message); + } + }); + + it("emits an executable root-hook payload bound to its target manifest", () => { + const { result, output } = runSourced( + ` +PACKAGE_TRANSACTION_SPECS=('${DOCKER_CE_SPEC}') +package_state() { printf 'missing\n'; } +assert_root_directory_safe() { :; } +assert_root_regular_file_safe() { :; } +sudo() { + case "$1" in + dpkg) + printf 'arm64\n' + ;; + mktemp) + mkdir -p "$HOME/generated-guard" + printf '/run/nemoclaw-apt-transaction.GENERATED\n' + ;; + tee) + cat >"$HOME/generated-guard/\${2##*/}" + ;; + chmod) + command chmod "$2" "$HOME/generated-guard/\${3##*/}" + ;; + esac +} +create_apt_transaction_guard +bash "$HOME/generated-guard/verify-plan" <<<"$APT_PLAN" +printf 'GENERATED_HOOK_ACCEPTED\n' +`, + { + APT_PLAN: aptProtocol( + "docker-ce - - none < 5:29.6.1-1~ubuntu.24.04~noble arm64 no /var/cache/apt/archives/docker-ce.deb", + "pigz - - none < 2.8-1 arm64 no /var/cache/apt/archives/pigz.deb", + ), + }, + ); + + expect(result.status, output).toBe(0); + expect(output).toContain("GENERATED_HOOK_ACCEPTED"); + }); + + it("cleans the root-owned transaction guard when the caller exits", () => { + const { result, output } = runSourced(` +APT_TRANSACTION_GUARD_DIR=/run/nemoclaw-apt-transaction.EXITTEST +APT_TRANSACTION_HOOK="$APT_TRANSACTION_GUARD_DIR/verify-plan" +sudo() { printf 'SUDO %s\n' "$*"; } +trap 'cleanup_apt_transaction_guard' EXIT +`); + + expect(result.status, output).toBe(0); + expect(output).toContain("SUDO rm -rf -- /run/nemoclaw-apt-transaction.EXITTEST"); + }); }); diff --git a/test/install-station-platform-identity.test.ts b/test/install-station-platform-identity.test.ts index b6b312a3324..4dfd82a3aa2 100644 --- a/test/install-station-platform-identity.test.ts +++ b/test/install-station-platform-identity.test.ts @@ -111,14 +111,14 @@ describe("DGX Station platform identity", () => { expect(result.status, output).toBe(0); }); - it("selects the GB300 by PCI identity when an auxiliary RTX is listed first", () => { + it("selects the GB300 by PCI identity when an auxiliary GPU has the same name", () => { const pciRoot = writePciIdentityFixture(); const { result, output } = runStationPrepare( ` station_pci_devices_path() { printf '%s' "$PCI_ROOT"; } nvidia-smi() { printf '%s\n' \ - '00000000:02:00.0, NVIDIA RTX PRO, 595.71.05, N/A, N/A' \ + '00000000:02:00.0, NVIDIA GB300, 595.71.05, 1, 0' \ '00000000:01:00.0, NVIDIA GB300, 595.71.05, 0, 0' } STATION_HOST_PROFILE=stock-dgx-os @@ -129,7 +129,7 @@ verify_gpu expect(result.status, output).toBe(0); expect(output).toContain( - "gpu_bdf=0000:02:00.0 gpu=NVIDIA RTX PRO role=auxiliary validation=skipped", + "gpu_bdf=0000:02:00.0 gpu=NVIDIA GB300 role=auxiliary validation=skipped", ); expect(output).toContain( "gpu_bdf=0000:01:00.0 gpu=NVIDIA GB300 driver=595.71.05 ecc_corrected=0 ecc_uncorrected=0", From cfdcd4884c7c69d37601b9b4af441439dbe598c4 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sat, 18 Jul 2026 12:45:42 -0700 Subject: [PATCH 7/7] test(installer): observe Station helper arguments Signed-off-by: Carlos Villela --- test/install-station-dgx-os.test.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/test/install-station-dgx-os.test.ts b/test/install-station-dgx-os.test.ts index e0972bb111c..c6bc00f2751 100644 --- a/test/install-station-dgx-os.test.ts +++ b/test/install-station-dgx-os.test.ts @@ -522,16 +522,17 @@ describe("DGX Station forced metadata installer handoff", () => { INSTALLER_PAYLOAD, ` SCRIPT_DIR="$HOME" -touch "$SCRIPT_DIR/prepare-dgx-station-host.sh" -bash() { printf 'HELPER_ARGS=%s\n' "$*"; } +cat >"$SCRIPT_DIR/prepare-dgx-station-host.sh" <<'HELPER' +printf '%s\n' "$*" >"$HOME/helper-args" +HELPER FORCE_STATION_INSTALL=1 run_station_host_preparation `, ); expect(result.status, output).toBe(0); - expect(output).toContain( - `HELPER_ARGS=${path.join(home, "prepare-dgx-station-host.sh")} --apply --force-station-install`, + expect(fs.readFileSync(path.join(home, "helper-args"), "utf8")).toBe( + "--apply --force-station-install\n", ); });