fix(installer): retain qualified Station DKMS revision

docs/get-started/dgx-station-preparation.mdx

@@ -54,6 +54,10 @@ The installer records the override in the printed relogin command when Docker-gr
 On the generic Ubuntu path, accepting express install prepares the host with NVIDIA open driver `610.43.02`, Docker CE `29.6.1` with Buildx, and NVIDIA Container Toolkit `1.19.1`.
 Preparation probes package and runtime state first, reuses exact matches, and installs only missing pinned packages, including the NVIDIA Container Toolkit libraries and `nvidia-ctk` CLI.
 It permits only the reviewed factory transition from `dkms` `3.0.11-1ubuntu13` to `1:3.4.0-1ubuntu1`.
+It also accepts an installed `dkms` `1:3.4.1-1ubuntu1` as a retained-compatible forward revision, warns that it differs from the validated `1:3.4.0-1ubuntu1` pin, excludes it from APT transactions, and continues runtime validation.
+NemoClaw DGX Station maintainers own this retained-version allowlist and qualify it against the complete generic-Ubuntu package tuple.
+Any package pin change invalidates retention until maintainers requalify the tuple.
+Maintainers remove a retained revision when runtime validation no longer passes.
 
 After reboot, preparation enables NVIDIA's packaged CDI refresh path and service, requires the `nvidia.com/gpu=all` device, and verifies it with a real container launch.
 If the packaged refresh fails or does not produce that device, preparation prints service diagnostics and stops for administrator repair.

scripts/prepare-dgx-station-host.sh

@@ -35,6 +35,13 @@ readonly DOCKER_VERSION="29.6.1"
 readonly TOOLKIT_VERSION="1.19.1"
 readonly FACTORY_DKMS_VERSION="3.0.11-1ubuntu13"
 readonly TARGET_DKMS_VERSION="1:3.4.0-1ubuntu1"
+# NemoClaw DGX Station maintainers own this allowlist. The qualified tuple below
+# binds each retained revision to the complete generic-Ubuntu package contract.
+# Update both only after requalification; remove an entry when runtime
+# validation no longer passes. Any PACKAGE_SPECS change invalidates retention.
+readonly -a RETAINED_DKMS_VERSIONS=(
+  "1:3.4.1-1ubuntu1"
+)
 # Keep this as a plain Ubuntu image: NVIDIA Container Toolkit injects the host
 # driver utility when CDI or --gpus is requested. This intentionally exercises
 # the documented runtime contract instead of relying on a CUDA image payload:
@@ -62,6 +69,20 @@ readonly -a PACKAGE_SPECS=(
   "nvidia-container-toolkit-base=1.19.1-1"
 )
 
+readonly -a RETAINED_DKMS_QUALIFIED_PACKAGE_SPECS=(
+  "dkms=1:3.4.0-1ubuntu1"
+  "nvidia-driver-pinning-610=610-2ubuntu1"
+  "nvidia-driver-open=610.43.02-1ubuntu1"
+  "containerd.io=2.2.6-1~ubuntu.24.04~noble"
+  "docker-buildx-plugin=0.35.0-1~ubuntu.24.04~noble"
+  "docker-ce=5:29.6.1-1~ubuntu.24.04~noble"
+  "docker-ce-cli=5:29.6.1-1~ubuntu.24.04~noble"
+  "libnvidia-container-tools=1.19.1-1"
+  "libnvidia-container1=1.19.1-1"
+  "nvidia-container-toolkit=1.19.1-1"
+  "nvidia-container-toolkit-base=1.19.1-1"
+)
+
 readonly -a BASEOS_PACKAGE_SPECS=(
   "dgx-release=7.5.0"
   "dgx-repo=25.10-2"
@@ -556,6 +577,23 @@ package_is_exact() {
   [[ "$actual" == "$expected" ]]
 }
 
+retained_dkms_policy_is_current() {
+  local index
+  ((${#PACKAGE_SPECS[@]} == ${#RETAINED_DKMS_QUALIFIED_PACKAGE_SPECS[@]})) || return 1
+  for index in "${!PACKAGE_SPECS[@]}"; do
+    [[ "${PACKAGE_SPECS[$index]}" == "${RETAINED_DKMS_QUALIFIED_PACKAGE_SPECS[$index]}" ]] || return 1
+  done
+}
+
+dkms_version_is_retained() {
+  local actual=$1 retained
+  retained_dkms_policy_is_current || return 1
+  for retained in "${RETAINED_DKMS_VERSIONS[@]}"; do
+    [[ "$actual" == "$retained" ]] && return 0
+  done
+  return 1
+}
+
 package_state() {
   local spec=$1
   local name expected actual
@@ -566,13 +604,32 @@ package_state() {
     printf 'missing\n'
   elif [[ "$actual" == "$expected" ]]; then
     printf 'exact\n'
+  elif [[ "$name" == "dkms" && "$expected" == "$TARGET_DKMS_VERSION" ]] && dkms_version_is_retained "$actual"; then
+    printf 'retained-compatible\n'
   elif [[ "$name" == "dkms" && "$actual" == "$FACTORY_DKMS_VERSION" && "$expected" == "$TARGET_DKMS_VERSION" ]]; then
     printf 'approved-transition\n'
   else
     printf 'mismatch\n'
   fi
 }
 
+warn_retained_package_version() {
+  local spec=$1 state name expected actual
+  state="$(package_state "$spec")"
+  [[ "$state" == "retained-compatible" ]] || return 0
+  name="$(package_name "$spec")"
+  expected="$(package_expected_version "$spec")"
+  actual="$(installed_version "$name")"
+  warn "package=${name} status=retained_compatible actual=${actual} validated=${expected} decision=retain"
+}
+
+warn_retained_package_versions() {
+  local spec
+  for spec in "${PACKAGE_SPECS[@]}"; do
+    warn_retained_package_version "$spec"
+  done
+}
+
 assert_no_package_mismatches() {
   local spec state name expected actual mismatch=0
   for spec in "${PACKAGE_SPECS[@]}"; do
@@ -591,13 +648,19 @@ assert_no_package_mismatches() {
     warn "package=${name} status=mismatch actual=${actual} expected=${expected}"
     mismatch=1
   done
-  ((mismatch == 0)) || fatal "Existing Station prerequisite versions differ from the validated pins or approved factory transition; refusing to change them automatically"
+  ((mismatch == 0)) || fatal "Existing Station prerequisite versions differ from the validated pins, retained-compatible versions, or approved factory transition; refusing to change them automatically"
 }
 
-all_packages_exact() {
+package_is_ready() {
+  local state
+  state="$(package_state "$1")"
+  [[ "$state" == "exact" || "$state" == "retained-compatible" ]]
+}
+
+all_packages_ready() {
   local spec
   for spec in "${PACKAGE_SPECS[@]}"; do
-    package_is_exact "$spec" || return 1
+    package_is_ready "$spec" || return 1
   done
   return 0
 }
@@ -887,7 +950,7 @@ check_failed_units() {
     return 0
   fi
   for unit in "${units[@]}"; do
-    if is_driver_transitional_unit "$unit" && all_packages_exact && ! driver_loaded_exact; then
+    if is_driver_transitional_unit "$unit" && all_packages_ready && ! driver_loaded_exact; then
       warn "driver unit failure allowed only until post-reboot verification: ${unit}"
     elif is_preparation_critical_unit "$unit"; then
       warn "failed preparation-critical unit: ${unit}"
@@ -1040,6 +1103,8 @@ print_package_status() {
       info "package=${name} status=missing expected=${expected}"
     elif [[ "$name" == "dkms" && "$actual" == "$FACTORY_DKMS_VERSION" ]]; then
       info "package=${name} status=approved_transition actual=${actual} expected=${expected}"
+    elif [[ "$name" == "dkms" && "$expected" == "$TARGET_DKMS_VERSION" ]] && dkms_version_is_retained "$actual"; then
+      warn_retained_package_version "$spec"
     else
       warn "package=${name} status=mismatch actual=${actual} expected=${expected}"
     fi
@@ -1251,7 +1316,7 @@ collect_package_transaction_specs() {
     state="$(package_state "$spec")"
     case "$state" in
       missing | approved-transition) PACKAGE_TRANSACTION_SPECS+=("$spec") ;;
-      exact) ;;
+      exact | retained-compatible) ;;
       *) fatal "Package transaction contains an unapproved prerequisite state: ${spec} (${state})" ;;
     esac
   done
@@ -1528,9 +1593,9 @@ install_packages() {
 
   local spec
   for spec in "${PACKAGE_SPECS[@]}"; do
-    package_is_exact "$spec" || fatal "Installed package does not match ${spec}"
+    package_is_ready "$spec" || fatal "Installed package is outside the accepted state for ${spec}"
   done
-  info "pinned_packages=installed"
+  info "prerequisite_packages=ready"
 }
 
 ensure_docker_group() {
@@ -1860,7 +1925,7 @@ verify_dgx_os_runtime_user() {
 verify_apply_state() {
   local spec
   for spec in "${PACKAGE_SPECS[@]}"; do
-    package_is_exact "$spec" || fatal "Package verification failed: ${spec}"
+    package_is_ready "$spec" || fatal "Package verification failed: ${spec}"
   done
   verify_gpu
   systemctl is-active --quiet nvidia-persistenced.service || fatal "nvidia-persistenced.service is not active"
@@ -1928,7 +1993,7 @@ verify_gpu() {
 verify_host() {
   local spec user_name=${SUDO_USER:-$USER}
   for spec in "${PACKAGE_SPECS[@]}"; do
-    package_is_exact "$spec" || fatal "Package verification failed: ${spec}"
+    package_is_ready "$spec" || fatal "Package verification failed: ${spec}"
   done
   verify_gpu
   systemctl is-active --quiet nvidia-persistenced.service || fatal "nvidia-persistenced.service is not active"
@@ -1954,14 +2019,14 @@ run_check() {
     return 0
   fi
   print_package_status
-  if all_packages_exact; then
+  if all_packages_ready; then
     if install_boot_marker_matches_current_boot; then
       warn "Package installation completed in the current boot; reboot is required"
       info "CHECK_RESULT=REBOOT_REQUIRED"
     elif driver_loaded_exact; then
       info "CHECK_RESULT=PACKAGES_AND_DRIVER_PRESENT"
     else
-      warn "Exact packages are installed but driver ${DRIVER_VERSION} is not loaded; reboot is required"
+      warn "Accepted prerequisite packages are installed but driver ${DRIVER_VERSION} is not loaded; reboot is required"
       info "CHECK_RESULT=REBOOT_REQUIRED"
     fi
   else
@@ -2003,16 +2068,17 @@ run_apply() {
   require_command grep
   require_command readlink
   require_command sha256sum
+  warn_retained_package_versions
 
   if reboot_required; then
-    if all_packages_exact && ! driver_loaded_exact; then
+    if all_packages_ready && ! driver_loaded_exact; then
       warn "A reboot is required before runtime setup can continue"
       exit_reboot_required
     fi
     fatal "An unrelated reboot is already pending"
   fi
 
-  if ! all_packages_exact; then
+  if ! all_packages_ready; then
     assert_no_package_mismatches
     install_packages
     ensure_docker_group
@@ -2029,7 +2095,7 @@ run_apply() {
   fi
 
   driver_loaded_exact || {
-    warn "Pinned packages are installed but driver ${DRIVER_VERSION} is not loaded"
+    warn "Accepted prerequisite packages are installed but driver ${DRIVER_VERSION} is not loaded"
     exit_reboot_required
   }
 
@@ -2055,7 +2121,8 @@ run_verify() {
     verify_dgx_os_runtime_user
     return 0
   fi
-  all_packages_exact || fatal "Pinned prerequisite packages are incomplete; run --apply"
+  warn_retained_package_versions
+  all_packages_ready || fatal "Accepted prerequisite packages are incomplete; run --apply"
   driver_loaded_exact || fatal "Pinned driver is not loaded; reboot, then run --apply"
   verify_host
 }

test/install-station-container-coexistence.test.ts

@@ -110,7 +110,7 @@ check_package_managers_idle() { :; }
 check_failed_units() { :; }
 check_agent_and_inference_conflicts() { :; }
 driver_loaded_exact() { return 0; }
-package_is_exact() { return 0; }
+package_is_ready() { return 0; }
 verify_gpu() { :; }
 systemctl() {
   case "$*" in
@@ -185,7 +185,7 @@ require_docker_mutation_quiescence "refreshing NVIDIA CDI configuration"
       name: "pending prerequisite",
       setup: `
 reboot_required() { return 0; }
-all_packages_exact() { return 0; }
+all_packages_ready() { return 0; }
 driver_loaded_exact() { return 1; }
 `,
       expectedGate: "REBOOT_HANDOFF_BLOCKED check=1",
@@ -194,7 +194,7 @@ driver_loaded_exact() { return 1; }
       name: "post-install",
       setup: `
 reboot_required() { return 1; }
-all_packages_exact() { return 1; }
+all_packages_ready() { return 1; }
 require_docker_restart_quiescence() {
   local checks
   checks="$(cat "$HOME/reboot-gate-checks" 2>/dev/null || printf '0')"
@@ -211,7 +211,7 @@ require_docker_restart_quiescence() {
       name: "same-boot marker",
       setup: `
 reboot_required() { return 1; }
-all_packages_exact() { return 0; }
+all_packages_ready() { return 0; }
 install_boot_marker_matches_current_boot() { return 0; }
 `,
       expectedGate: "REBOOT_HANDOFF_BLOCKED check=1",
@@ -220,7 +220,7 @@ install_boot_marker_matches_current_boot() { return 0; }
       name: "unloaded driver",
       setup: `
 reboot_required() { return 1; }
-all_packages_exact() { return 0; }
+all_packages_ready() { return 0; }
 install_boot_marker_matches_current_boot() { return 1; }
 driver_loaded_exact() { return 1; }
 `,
@@ -230,7 +230,7 @@ driver_loaded_exact() { return 1; }
       name: "Docker group",
       setup: `
 reboot_required() { return 1; }
-all_packages_exact() { return 0; }
+all_packages_ready() { return 0; }
 install_boot_marker_matches_current_boot() { return 1; }
 driver_loaded_exact() { return 0; }
 finish_runtime() { DOCKER_GROUP_ADDED=1; }

test/install-station-host-preparation.test.ts

@@ -360,7 +360,7 @@ check_failed_units
 common_preflight() { :; }
 require_command() { :; }
 acquire_sudo() { :; }
-all_packages_exact() { return 0; }
+all_packages_ready() { return 0; }
 install_boot_marker_matches_current_boot() { return 1; }
 driver_loaded_exact() { return 0; }
 install_packages() { printf 'INSTALL_PACKAGES\n'; }
@@ -384,7 +384,7 @@ run_apply
 common_preflight() { :; }
 require_command() { :; }
 acquire_sudo() { :; }
-all_packages_exact() { return 1; }
+all_packages_ready() { return 1; }
 installed_version() {
   if [[ "$1" == "dkms" ]]; then printf '3.0.11-1ubuntu13'; fi
 }
@@ -418,7 +418,7 @@ validate_package_availability() { printf 'VALIDATE_PACKAGES\n'; }
 simulate_install() { printf 'SIMULATE_INSTALL\n'; }
 require_docker_restart_quiescence() { printf 'RECHECK_RESTART_QUIESCENCE\n'; }
 package_state() { printf 'missing\n'; }
-package_is_exact() { return 0; }
+package_is_ready() { return 0; }
 create_apt_transaction_guard() {
   APT_TRANSACTION_GUARD_DIR=/run/nemoclaw-apt-transaction.TEST
   APT_TRANSACTION_HOOK="/bin/bash $APT_TRANSACTION_GUARD_DIR/verify-plan"
@@ -441,7 +441,7 @@ install_packages
     ]) {
       expect(output).toContain(spec);
     }
-    expect(output).toContain("pinned_packages=installed");
+    expect(output).toContain("prerequisite_packages=ready");
   });
   it("does not refresh CDI when the GPU launch probe already passes", () => {
     const { result, output } = runSourced(
@@ -622,7 +622,7 @@ assert_root_directory_safe /etc/apt/keyrings test_directory
 common_preflight() { :; }
 require_command() { :; }
 acquire_sudo() { :; }
-all_packages_exact() { return 0; }
+all_packages_ready() { return 0; }
 install_boot_marker_matches_current_boot() { return 1; }
 driver_loaded_exact() { return 0; }
 finish_runtime() { DOCKER_GROUP_ADDED=1; printf 'FINISH_RUNTIME\n'; }
@@ -894,7 +894,7 @@ main "$READ_MODE"
       `
 common_preflight() { :; }
 require_command() { :; }
-all_packages_exact() { return 0; }
+all_packages_ready() { return 0; }
 driver_loaded_exact() { return 1; }
 run_verify
 `,