Fix and rescope the L40S runner stability qualification

.github/workflows/perf-smoke-runner-stability.yaml

@@ -3,11 +3,15 @@
 #
 # SPDX-License-Identifier: BSD-3-Clause
 
-# One-time staging qualification for the L40S runner pool. The workflow waits
-# for the normal gate and baseline seeder from the same push to finish, then fans
-# the commit out to five independent allocations. Each allocation collects three
-# FPS samples per bucket without publishing baselines. The final CPU job combines
-# all 15 samples and fails unless the pool meets the predeclared stability policy.
+# One-time staging qualification for the L40S runner pool. The workflow waits for
+# the commit's normal gate run to pass and for the L40 pool to go quiet, then fans
+# the commit out to five independent allocations so per-runner and run-to-run FPS
+# spread can be measured separately. Qualification only reports a verdict; it
+# never writes baselines.
+#
+# Scope is the Newton-touching buckets. The PhysX-only buckets already produced
+# clean evidence on 2026-07-28; these five crashed on a Newton import and are the
+# ones still lacking a verdict. Halving the matrix also halves the pool time.
 
 name: Performance Smoke - L40S Runner Stability
 
@@ -20,76 +24,69 @@ on:
 
 permissions:
   actions: read
-  # Reusable workflows cannot elevate their caller's token. The seeder requests
-  # write access for its normal publish mode, while this caller forces dry-run.
+  # The seeder this workflow calls declares contents: write, and a caller cannot
+  # grant a reusable workflow more than it holds. The stability allocations run
+  # with dry_run: true, so nothing is actually written.
   contents: write
 
 concurrency:
   group: perf-smoke-runner-stability-${{ github.ref }}
   cancel-in-progress: false
 
+env:
+  # Every backend_key whose benchmark loads Newton. Covers five task/backend
+  # buckets, since `newton` applies to both Cartpole and Velocity-Flat-G1.
+  STABILITY_BACKENDS: "newton,newton_rtx_renderer,newton_newton_renderer,physx_newton_renderer"
+
 jobs:
   wait_for_quiet_pool:
-    name: Wait for sibling performance workflows
-    if: ${{ github.event_name == 'push' }}
+    name: Wait for initial gate and quiet pool
     runs-on: ubuntu-latest
-    timeout-minutes: 350
+    timeout-minutes: 180
     permissions:
       actions: read
       contents: read
     steps:
-    - name: Wait for the gate and seeder from this push
+    - name: Wait for the initial gate and competing L40 work
       env:
         GH_TOKEN: ${{ github.token }}
       run: |
         set -euo pipefail
-        REQUIRED_WORKFLOWS=(
-          "Performance Smoke Test"
-          "Performance Smoke - Seed Baselines"
-        )
         L40_POOL_WORKFLOWS=(
           "Performance Smoke Test"
-          "Performance Smoke - Seed Baselines"
           "Perf Smoke — Publish CI Image"
           "Perf Smoke — Auto Era Roll"
         )
-        DEADLINE=$((SECONDS + 20400))
+        DEADLINE=$((SECONDS + 10200))
         QUIET_POLLS=0
         while (( SECONDS < DEADLINE )); do
           RUNS="$(gh run list \
             --repo "${GITHUB_REPOSITORY}" \
             --limit 100 \
-            --json headSha,workflowName,status)"
-          PREREQUISITES_READY=true
-          for workflow_name in "${REQUIRED_WORKFLOWS[@]}"; do
-            MATCH_COUNT="$(jq --arg name "${workflow_name}" \
-              --arg sha "${GITHUB_SHA}" \
-              '[.[] | select(.workflowName == $name and .headSha == $sha)] | length' <<<"${RUNS}")"
-            ACTIVE_COUNT="$(jq --arg name "${workflow_name}" \
-              --arg sha "${GITHUB_SHA}" \
-              '[.[] | select(.workflowName == $name and .headSha == $sha and .status != "completed")] | length' \
-              <<<"${RUNS}")"
-            if [ "${MATCH_COUNT}" -eq 0 ] || [ "${ACTIVE_COUNT}" -ne 0 ]; then
-              PREREQUISITES_READY=false
-              break
-            fi
-          done
+            --json conclusion,headSha,workflowName,status)"
+          GATE_CONCLUSION="$(jq -r --arg sha "${GITHUB_SHA}" \
+            '[.[] | select(.workflowName == "Performance Smoke Test" and .headSha == $sha and .status == "completed")]
+            | first | .conclusion // ""' <<<"${RUNS}")"
+          if [ -n "${GATE_CONCLUSION}" ] && [ "${GATE_CONCLUSION}" != "success" ]; then
+            echo "::error::Initial pinned-image gate concluded ${GATE_CONCLUSION}."
+            exit 1
+          fi
           ACTIVE_POOL_RUNS=0
           for workflow_name in "${L40_POOL_WORKFLOWS[@]}"; do
             ACTIVE_COUNT="$(jq --arg name "${workflow_name}" \
               '[.[] | select(.workflowName == $name and .status != "completed")] | length' <<<"${RUNS}")"
             ACTIVE_POOL_RUNS=$((ACTIVE_POOL_RUNS + ACTIVE_COUNT))
           done
-          if [ "${PREREQUISITES_READY}" = true ] && [ "${ACTIVE_POOL_RUNS}" -eq 0 ]; then
+          if [ "${GATE_CONCLUSION}" = "success" ] && [ "${ACTIVE_POOL_RUNS}" -eq 0 ]; then
             QUIET_POLLS=$((QUIET_POLLS + 1))
             if [ "${QUIET_POLLS}" -ge 2 ]; then
-              echo "Sibling workflows completed and the L40 pool stayed quiet; starting qualification."
+              echo "Initial gate passed and the L40 pool stayed quiet; starting qualification."
               exit 0
             fi
             echo "L40 pool is quiet; confirming for one more polling interval..."
           else
             QUIET_POLLS=0
-            echo "Waiting for sibling workflows and other L40 workloads to finish..."
+            echo "Waiting for the initial gate and other L40 workloads to finish..."
           fi
           sleep 60
         done
@@ -99,7 +96,7 @@ jobs:
   stability_sample:
     name: Runner allocation ${{ matrix.allocation }}
     needs: [wait_for_quiet_pool]
-    if: ${{ always() && (needs.wait_for_quiet_pool.result == 'success' || needs.wait_for_quiet_pool.result == 'skipped') }}
+    if: ${{ needs.wait_for_quiet_pool.result == 'success' }}
     strategy:
       fail-fast: false
       max-parallel: 5
@@ -114,6 +111,9 @@ jobs:
       commit_count: "1"
       samples_per_commit: "3"
       tasks: "__ALL_TASKS__"
+      # Literal because `with:` cannot read the `env` context. Kept in sync with
+      # STABILITY_BACKENDS by test_staging_workflow_qualifies_only_newton_buckets.
+      backends: "newton,newton_rtx_renderer,newton_newton_renderer,physx_newton_renderer"
       target_branch: perf-smoke/develop-staging
       strict_ancestry: true
       dry_run: true
@@ -168,6 +168,7 @@ jobs:
         done
         python3 tools/perf_smoke_test/runner_stability.py \
           "${RECORD_ARGS[@]}" \
+          --backends "${STABILITY_BACKENDS}" \
           --gpu_model l40s \
           --expected_target_branch perf-smoke/develop-staging \
           --expected_commit "${GITHUB_SHA}" \

tools/perf_smoke_test/aggregate.py

@@ -30,6 +30,7 @@
     update_baselines_git,
 )
 from contracts import BenchResult  # noqa: E402
+from environment_skew import DependencySkew, detect_dependency_skew  # noqa: E402
 from gate_config import BASELINE_PUSH_RETRIES, MIN_BASELINE_SAMPLES, load_gate_config  # noqa: E402
 from gate_types import FpsMeanThreshold, OracleVerdict  # noqa: E402
 from gpu_identity import canonical_gpu_model, gpu_model_config_keys  # noqa: E402
@@ -160,6 +161,18 @@ def _runtime_context(bench_result: BenchResult) -> tuple[str, str]:
     return str(gpu_name or "N/A"), runtime or "N/A"
 
 
+def _skewed_rows(rows: list[tuple]) -> list[tuple[str, str, DependencySkew]]:
+    """Return ``(task_id, backend, skew)`` for failures caused by a stale CI image."""
+    skewed = []
+    for result, bench_result in rows:
+        if result.verdict != OracleVerdict.HARD_FAILURE:
+            continue
+        skew = detect_dependency_skew(bench_result.stdout_tail)
+        if skew is not None:
+            skewed.append((result.task_id, result.backend, skew))
+    return skewed
+
+
 def _row_explanation(result) -> str:
     """Explain one verdict in reviewer-facing language."""
     if result.verdict == OracleVerdict.HARD_FAILURE:
@@ -227,16 +240,41 @@ def _build_technical_table(rows: list[tuple]) -> str:
     return "\n".join(lines)
 
 
+def _build_stale_image_section(skewed: list[tuple[str, str, DependencySkew]]) -> str:
+    """Explain that a stale CI image, not the PR, caused these failures."""
+    packages = sorted({skew.package for _, _, skew in skewed})
+    affected = "\n".join(f"- `{task_id}` ({backend}): {skew.describe()}" for task_id, backend, skew in skewed)
+    return (
+        "### Stale CI image\n\n"
+        f"The prebuilt CI image does not match this PR's pinned {' and '.join(packages)} version, so the "
+        "tasks below crashed before producing any FPS. This reflects the image, not the change under review, "
+        "so these results are advisory and do not fail the check.\n\n"
+        f"{affected}\n\n"
+        "This resolves itself once the CI image is rebuilt for the current dependency pins. Re-run the gate "
+        "after the next image publish to get real numbers for these tasks."
+    )
+
+
 def _build_summary_markdown(rows: list[tuple], *, blocking: bool) -> str:
     """Build reviewer-first Markdown for the sticky PR comment."""
     counts = {verdict: 0 for verdict in OracleVerdict}
     for result, _ in rows:
         counts[result.verdict] += 1
 
+    skewed = _skewed_rows(rows)
+    skewed_keys = {(task_id, backend) for task_id, backend, _ in skewed}
+    unexplained_failures = sum(
+        1
+        for result, _ in rows
+        if result.verdict == OracleVerdict.HARD_FAILURE and (result.task_id, result.backend) not in skewed_keys
+    )
+
     if not rows:
         overall = "❌ No benchmark results were produced"
-    elif counts[OracleVerdict.HARD_FAILURE]:
+    elif unexplained_failures:
         overall = "❌ One or more benchmarks failed before producing usable performance data"
+    elif skewed:
+        overall = "⚠️ The CI image is stale for this PR, so some tasks could not be measured"
     elif counts[OracleVerdict.BLOCK]:
         overall = "🚫 One or more blocking-level performance regressions were detected"
     elif counts[OracleVerdict.WARN]:
@@ -269,10 +307,14 @@ def _build_summary_markdown(rows: list[tuple], *, blocking: bool) -> str:
     else:
         gpu_name, runtime = "N/A", "N/A"
 
-    return "\n\n".join(
+    sections = [
+        f"### Overall result\n\n**{overall}**\n\n{count_summary}\n\n{mode}",
+        f"### Run context\n\n- **GPU:** {gpu_name}\n- **Runtime:** {runtime}",
+    ]
+    if skewed:
+        sections.append(_build_stale_image_section(skewed))
+    sections.extend(
         (
-            f"### Overall result\n\n**{overall}**\n\n{count_summary}\n\n{mode}",
-            f"### Run context\n\n- **GPU:** {gpu_name}\n- **Runtime:** {runtime}",
             "### How to read this\n\n"
             "Start with **BLOCK** and **HARD FAILURE**, then review any **WARN** rows.\n\n"
             "- **✅ PASS:** no meaningful slowdown was detected.\n"
@@ -292,6 +334,7 @@ def _build_summary_markdown(rows: list[tuple], *, blocking: bool) -> str:
             f"{_build_technical_table(rows)}\n\n</details>",
         )
     )
+    return "\n\n".join(sections)
 
 
 def _write_github_output(**values) -> None:
@@ -407,7 +450,14 @@ def main() -> int:
         if oracle_result.verdict == OracleVerdict.BLOCK:
             has_block = True
         elif oracle_result.verdict == OracleVerdict.HARD_FAILURE:
-            has_hard_failure = True
+            # A crash caused by the image lacking a symbol this source pins is a
+            # property of the image, not of the change under test, so it is
+            # reported loudly but never fails the PR.
+            skew = detect_dependency_skew(bench_result.stdout_tail)
+            if skew is None:
+                has_hard_failure = True
+            else:
+                print(f"[aggregate] {task_id}/{backend}: stale CI image; {skew.describe()}")
 
         if (
             allow_update
@@ -526,11 +576,13 @@ def main() -> int:
     if baseline_update_failed:
         return 1
 
-    if blocking:
-        if has_block:
-            return 1
-        if has_hard_failure:
-            return 2
+    # Benchmark execution health is never advisory. A crash, missing result, or
+    # invalid benchmark must fail even while FPS regressions are being rolled out
+    # in advisory mode.
+    if has_hard_failure:
+        return 2
+    if blocking and has_block:
+        return 1
     return 0
 
 

tools/perf_smoke_test/environment_skew.py

@@ -0,0 +1,85 @@
+# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md).
+# All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""Detection of source-versus-image dependency skew in a failed benchmark.
+
+The gate bind-mounts Isaac Lab source over a prebuilt CI image, but the image
+supplies the installed third-party packages (Newton, Warp, Isaac Sim). Between a
+dependency-pin change landing on ``develop`` and the next image publish, a PR's
+source can reference a symbol the installed package does not have yet, which
+crashes every affected task before any FPS is measured.
+
+That crash says nothing about the PR's performance, so it must not read as a
+performance failure. This module recognizes the crash signature so the gate can
+report a stale image and stay advisory for the affected tasks instead.
+
+Only packages the *image* installs are eligible. A missing symbol in Isaac Lab's
+own source is a genuine defect in the change under test and still fails.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+# Packages installed into the CI image rather than bind-mounted from the PR.
+IMAGE_PROVIDED_PACKAGES: frozenset[str] = frozenset(
+    {
+        "carb",
+        "isaacsim",
+        "mujoco",
+        "mujoco_warp",
+        "newton",
+        "omni",
+        "pxr",
+        "warp",
+    }
+)
+
+# Python spells "this name is not in the installed package" three ways.
+_MISSING_NAME_PATTERNS: tuple[re.Pattern[str], ...] = (
+    re.compile(r"ImportError: cannot import name ['\"](?P<symbol>\w+)['\"] from ['\"](?P<module>[\w.]+)['\"]"),
+    re.compile(r"ModuleNotFoundError: No module named ['\"](?P<module>[\w.]+)['\"]"),
+    re.compile(r"AttributeError: module ['\"](?P<module>[\w.]+)['\"] has no attribute ['\"](?P<symbol>\w+)['\"]"),
+)
+
+
+@dataclass(frozen=True)
+class DependencySkew:
+    """One detected mismatch between the PR's source and the image's packages."""
+
+    package: str
+    module: str
+    symbol: str | None
+
+    def describe(self) -> str:
+        """Return a one-line reviewer-facing description of the mismatch."""
+        if self.symbol:
+            return f"`{self.module}` in the CI image has no `{self.symbol}`"
+        return f"`{self.module}` is not installed in the CI image"
+
+
+def detect_dependency_skew(log_text: str | None) -> DependencySkew | None:
+    """Return the dependency skew a benchmark log indicates, if any.
+
+    Args:
+        log_text: Captured benchmark output, typically ``BenchResult.stdout_tail``.
+
+    Returns:
+        The detected mismatch, or ``None`` when the log shows no missing symbol
+        from an image-provided package.
+    """
+    if not log_text:
+        return None
+    for pattern in _MISSING_NAME_PATTERNS:
+        match = pattern.search(log_text)
+        if match is None:
+            continue
+        module = match.group("module")
+        package = module.split(".", 1)[0]
+        if package not in IMAGE_PROVIDED_PACKAGES:
+            continue
+        return DependencySkew(package=package, module=module, symbol=match.groupdict().get("symbol"))
+    return None