fix(workflows): add packages: read permission to update-codeql-version.yml

.github/scripts/pin-codeql-library-versions.sh

@@ -0,0 +1,97 @@
+#!/usr/bin/env bash
+# Pins every `codeql/<name>: '*'` dependency in this repo's qlpack.yml files to
+# the exact library version shipped in the official CodeQL Bundle for a given
+# CLI release.
+#
+# Why this exists: `codeql pack upgrade` resolves an unconstrained `'*'`
+# dependency to the *latest-ever-published* version in the configured
+# registry (GHCR) - completely independent of whatever CodeQL CLI version is
+# pinned in `.codeqlversion`. That mismatch can jump `codeql/<lang>-all`
+# several major versions ahead of what the pinned CLI actually ships/tests
+# against, which can silently break analyses (see CONTRIBUTING.md's
+# "Updating the pinned CodeQL CLI/library version" section for a worked
+# example). Pinning these `codeql/*` deps to the exact bundle-paired version
+# before running `codeql pack upgrade` makes that resolution deterministic
+# and keeps every pack's declared library dependency in lockstep with the
+# CLI version this repo says it supports.
+#
+# Usage: pin-codeql-library-versions.sh <cli-version>
+#   e.g. pin-codeql-library-versions.sh 2.21.4
+#
+# Requires: gh (authenticated), tar, sed, find. Run from the repo root.
+set -euo pipefail
+
+if [[ $# -ne 1 ]]; then
+  echo "Usage: $0 <cli-version>" >&2
+  exit 1
+fi
+
+VERSION="$1"
+BUNDLE_TAG="codeql-bundle-v${VERSION}"
+WORKDIR="$(mktemp -d)"
+trap 'rm -rf "$WORKDIR"' EXIT
+
+echo "Determining codeql/* library versions bundled with CodeQL CLI ${VERSION}..."
+echo "(source of truth: github/codeql-action release '${BUNDLE_TAG}', asset codeql-bundle-linux64.tar.gz)"
+
+if ! gh release download "$BUNDLE_TAG" --repo github/codeql-action \
+  --pattern "codeql-bundle-linux64.tar.gz" --dir "$WORKDIR" --clobber; then
+  echo "::error::Could not download the CodeQL Bundle for tag '${BUNDLE_TAG}' from github/codeql-action releases." >&2
+  echo "Every CLI release published to github/codeql-cli-binaries should have a matching 'codeql-bundle-v<version>' release in github/codeql-action - check that the tag exists." >&2
+  exit 1
+fi
+
+# List every bundled `codeql/<name>` qlpack and its exact version WITHOUT
+# extracting any file contents - the bundle is large (~500MB) and we only
+# need the directory listing (name + version are encoded in the path).
+VERSIONS_FILE="$WORKDIR/versions.txt"
+tar tzf "$WORKDIR/codeql-bundle-linux64.tar.gz" \
+  | grep -E '^codeql/qlpacks/codeql/[^/]+/[^/]+/$' \
+  | sed -E 's#^codeql/qlpacks/codeql/([^/]+)/([^/]+)/$#\1 \2#' \
+  | sort -u > "$VERSIONS_FILE"
+
+echo "Discovered $(wc -l < "$VERSIONS_FILE") bundled codeql/* packages for CLI ${VERSION}."
+
+# Every real qlpack.yml in the repo, excluding gitignored local build
+# artifacts (.codeql/ pack caches, the /codeql cloned-repo checkout dir).
+mapfile -t QLPACK_FILES < <(find . -name qlpack.yml -not -path "*/.codeql/*" -not -path "./codeql/*")
+
+declare -A PINNED_COUNT=()
+while read -r pkg ver; do
+  [[ -z "$pkg" ]] && continue
+  count=0
+  for file in "${QLPACK_FILES[@]}"; do
+    # Only touch lines declaring an unconstrained `codeql/<pkg>: '*'` (or
+    # unquoted / double-quoted `*`) dependency; any trailing comment is left
+    # untouched since the substitution only replaces the matched quote-star-quote.
+    if grep -qE "^[[:space:]]*codeql/${pkg}:[[:space:]]*[\"']?\\*[\"']?[[:space:]]*(#.*)?\$" "$file"; then
+      sed -i -E "s#^([[:space:]]*codeql/${pkg}:[[:space:]]*)([\"']?)\\*\\2#\\1\\2${ver}\\2#" "$file"
+      count=$((count + 1))
+    fi
+  done
+  if [[ $count -gt 0 ]]; then
+    PINNED_COUNT["$pkg@$ver"]=$count
+  fi
+done < "$VERSIONS_FILE"
+
+echo
+echo "Pinned codeql/* dependencies:"
+for key in "${!PINNED_COUNT[@]}"; do
+  echo "  - ${key} (${PINNED_COUNT[$key]} file(s))"
+done | sort
+
+# Surface any remaining unconstrained codeql/* dependency that this script
+# did NOT pin (e.g. it isn't one of the standard per-language bundle
+# packages) so it doesn't silently keep resolving to registry-latest.
+echo
+echo "codeql/* dependencies left unpinned (not found in the CodeQL Bundle):"
+REMAINING=0
+for file in "${QLPACK_FILES[@]}"; do
+  if grep -qE "^[[:space:]]*codeql/[A-Za-z0-9_.-]+:[[:space:]]*[\"']?\*[\"']?[[:space:]]*(#.*)?\$" "$file"; then
+    grep -nE "^[[:space:]]*codeql/[A-Za-z0-9_.-]+:[[:space:]]*[\"']?\*[\"']?[[:space:]]*(#.*)?\$" "$file" | sed "s#^#  ${file}:#"
+    REMAINING=1
+  fi
+done
+if [[ "$REMAINING" -eq 0 ]]; then
+  echo "  (none)"
+fi

.github/workflows/update-codeql-version.yml

@@ -23,6 +23,7 @@ jobs:
     runs-on: ubuntu-latest
     permissions:
       contents: read # PR creation uses a scoped GitHub App token (SECLABS_APP_ID/SECLABS_APP_KEY), not GITHUB_TOKEN
+      packages: read # `codeql pack upgrade` resolves codeql/* library deps from GHCR (see ci.yml)
 
     steps:
       - name: Checkout
@@ -48,12 +49,23 @@ jobs:
       - name: Setup CodeQL
         uses: ./.github/actions/install-codeql
 
+      - name: Pin codeql/* library dependencies to the CodeQL Bundle version
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          .github/scripts/pin-codeql-library-versions.sh "${{ steps.version.outputs.version }}"
+
       - name: Upgrade every pack's dependencies
         env:
           GITHUB_TOKEN: ${{ github.token }}
         run: |
           set -euo pipefail
-          for dir in $(find . -name qlpack.yml -exec dirname {} \;); do
+          # ql/hotspots is excluded: it's a standalone local dev tool (see
+          # ql/hotspots/README.md and .github/workflows/hotspots.yml), not one of
+          # the per-language src/lib/ext/ext-library-sources packs that ci.yml and
+          # publish.yml operate on. Its `codeql/ql: '*'` dependency isn't a real
+          # resolvable GHCR package, so it always fails `codeql pack upgrade`.
+          for dir in $(find . -name qlpack.yml -not -path "./ql/hotspots/*" -exec dirname {} \;); do
             echo "::group::codeql pack upgrade $dir"
             codeql pack upgrade "$dir"
             echo "::endgroup::"
@@ -86,8 +98,13 @@ jobs:
 
             This PR:
             - Updates `.codeqlversion` to `${{ steps.version.outputs.version }}`.
+            - Pins every `codeql/<lang>-all` / `codeql/<lang>-queries` dependency across all
+              `qlpack.yml` files to the exact version shipped in the official CodeQL Bundle
+              for this CLI release (see `.github/scripts/pin-codeql-library-versions.sh`) -
+              this keeps `codeql pack upgrade` from jumping those libraries to
+              registry-latest instead of the version this CLI actually ships/tests against.
             - Runs `codeql pack upgrade <dir>` for every pack directory to refresh its
-              `codeql-pack.lock.yml` against the new CLI.
+              `codeql-pack.lock.yml` against the new CLI and pinned library versions.
 
             **This PR does not publish anything by itself** - no pack `version:` field is
             bumped here, so `publish.yml`'s version-diff trigger won't fire for it yet.

.gitignore

@@ -11,6 +11,9 @@
 */*.class
 # Cloned repository of codeql
 /codeql
+# Downloaded/extracted CodeQL CLI (see .github/actions/install-codeql), cached
+# by actions/cache between CI runs - never meant to be committed.
+/codeql_home
 
 # Test files / folders
 test.ql

CONTRIBUTING.md

@@ -66,18 +66,56 @@ Every query pack in this repository is compiled and tested against a specific ve
 
 The pinning is codified per language across:
 - [`.codeqlversion`][codeqlversion] (repo root): the CodeQL **CLI** version CI installs and compiles/tests against.
-- `<language>/src/codeql-pack.lock.yml` **and** `<language>/lib/codeql-pack.lock.yml`: the exact resolved
-  (locked) version of `codeql/<language>-all` (and, for `src`, `codeql/<language>-queries` where used),
-  generated by `codeql pack install`/`codeql pack upgrade` against the CLI above. The `qlpack.yml` files
-  themselves only declare an unpinned `'*'` range, so the lock files, not the `qlpack.yml`, are the real
-  source of truth. **`src` and `lib` each have their own independently-resolved lock file** - nothing
-  keeps them in sync automatically, so it's possible (if `codeql pack upgrade` is run against one
-  directory but not the other) for them to drift apart. Always upgrade both when bumping
+- `<language>/{src,lib}/qlpack.yml`: **every `codeql/<language>-all` / `codeql/<language>-queries`
+  dependency is pinned to an exact version** (e.g. `codeql/go-all: '4.2.6'`), not left as an
+  unconstrained `'*'` range. This is deliberate, not incidental — see the warning box below for why.
+  Internal `githubsecuritylab/*` cross-pack dependencies (e.g. `<language>/lib` depending on
+  `<language>/ext`) are unaffected and still use whatever range/pin a maintainer set by hand.
+- `<language>/src/codeql-pack.lock.yml` **and** `<language>/lib/codeql-pack.lock.yml`: the exact
+  resolved (locked) version tree generated by `codeql pack install`/`codeql pack upgrade` against the
+  CLI and the pinned `qlpack.yml` dependency above. **`src` and `lib` each have their own
+  independently-resolved lock file** - nothing keeps them in sync automatically, so it's possible (if
+  `codeql pack upgrade` is run against one directory but not the other, or one `qlpack.yml` pin is
+  hand-edited without the other) for them to drift apart. Always upgrade both when bumping
   `.codeqlversion` (the [automated workflow](#updating-the-pinned-codeql-clilibrary-version) does this
   for every pack directory in one pass); the table below only shows `src` for brevity, but the
   auto-generated table in every publish summary (see [Cutting a release](#cutting-a-release)) checks
   both and will flag drift between them.
 
+> [!WARNING]
+> **Why `codeql/*` dependencies are pinned to an exact version instead of left as `'*'`:** they used
+> to be unconstrained (`codeql/go-all: '*'`, etc.). The problem: `codeql pack upgrade` resolves an
+> unconstrained `'*'` dependency to the **latest-ever-published** version in the configured registry
+> (GHCR) - completely independent of whatever CodeQL CLI version is pinned in `.codeqlversion`. In
+> practice this let a routine `.codeqlversion` bump silently jump `codeql/go-all` from the version
+> actually bundled/tested with the target CLI (e.g. `4.2.6`, bundled with CLI `v2.21.4`) to whatever
+> was newest in the registry at that moment (e.g. `7.2.0`) - a library several major versions ahead of
+> anything that CLI version ships or has ever been tested against, which can silently break analyses
+> or fail outright with errors like `'codeql/namebinding' not found in the registry`.
+>
+> The fix: [`.github/scripts/pin-codeql-library-versions.sh`][pin-codeql-library-versions-script] runs
+> before `codeql pack upgrade` (as part of
+> [`update-codeql-version.yml`](#updating-the-pinned-codeql-clilibrary-version)) and rewrites every
+> `codeql/<pkg>: '*'` dependency to the *exact* version shipped in the **official CodeQL Bundle** for
+> the target CLI release - i.e. the same library versions GitHub itself builds, tests, and ships
+> together with that CLI. It determines these by downloading the bundle release asset
+> (`codeql-bundle-linux64.tar.gz`, tag `codeql-bundle-v<version>`) from
+> [github/codeql-action releases](https://github.com/github/codeql-action/releases) and listing its
+> `codeql/qlpacks/codeql/<pkg>/<version>/` directory entries (via `tar tzf`, no extraction needed) -
+> this is the CLI-native source of truth, not a web scrape of any documentation page. (Two other
+> approaches were considered and rejected: the [`gh-codeql`](https://github.com/github/gh-codeql)
+> extension's `gh codeql set-version` only installs the bare CLI without any bundled library packs, so
+> it can't answer this question; and `codeql resolve packs`/`codeql pack upgrade` themselves are what's
+> *being* fixed, so they can't be used to validate their own input.) Any `codeql/*` dependency the
+> script can't find in the bundle (there's exactly one, see the note on [`ql/hotspots`](#ql-hotspots)
+> below) is left untouched and surfaced as a warning rather than silently skipped.
+>
+> Every `qlpack.yml` becomes pinned this way starting with the next CLI bump run through
+> [`update-codeql-version.yml`](#updating-the-pinned-codeql-clilibrary-version) - if you're reading
+> this shortly after this pinning behavior was introduced and a pack's `qlpack.yml` still shows
+> `codeql/<pkg>: '*'`, that just means its dependencies haven't been re-resolved since, not that the
+> convention doesn't apply to it.
+
 CodeQL CLI: `v2.21.1` (released 2025-04-16) ([Bundle](https://github.com/github/codeql-action/releases/tag/codeql-bundle-v2.21.1) / [Binary](https://github.com/github/codeql-cli-binaries/releases/tag/v2.21.1))
 
 | Language | Query pack | Standard library (`*-all`) | Upstream query pack (`*-queries`) | Lock file |
@@ -159,13 +197,21 @@ coding agent) in the loop for the hard part — fixing whatever the new CLI brea
    when to actually take the upgrade (and deal with any breakage) is a deliberate call, not
    something to run unattended.
 2. **Dependency refresh** — run [`update-codeql-version.yml`][update-codeql-version-workflow]
-   (`workflow_dispatch`, input the new CLI version, e.g. `2.22.0`). It updates `.codeqlversion`,
-   runs `codeql pack upgrade <dir>` for every pack directory to refresh each
-   `codeql-pack.lock.yml`, and opens a PR (via the same GitHub App token as
+   (`workflow_dispatch`, input the new CLI version, e.g. `2.22.0`). It updates `.codeqlversion`, then:
+   1. Pins every `codeql/<pkg>: '*'` dependency across every `qlpack.yml` to the exact version shipped
+      in the official CodeQL Bundle for that CLI release (see the warning box under
+      [Supported CodeQL versions](#supported-codeql-versions) above for why this step exists), via
+      [`pin-codeql-library-versions.sh`][pin-codeql-library-versions-script].
+   2. Runs `codeql pack upgrade <dir>` for every pack directory (except
+      [`ql/hotspots`](#ql-hotspots), see below) to refresh each `codeql-pack.lock.yml` against the
+      newly-pinned dependencies.
+
+   It then opens a PR (via the same GitHub App token as
    [`update-release.yml`][update-release-workflow], so CI actually runs on it — a plain
-   `GITHUB_TOKEN`-authored PR would not trigger downstream workflows). This is the automated
-   version of [#118][pr-118]'s original proposal, extended to also own the `.codeqlversion` bump
-   itself (not just the `codeql pack upgrade` loop) and to use a token that triggers CI.
+   `GITHUB_TOKEN`-authored PR would not trigger downstream workflows). This is the automated version
+   of [#118][pr-118]'s original proposal, extended to also own the `.codeqlversion` bump and the
+   exact-version pinning (not just a bare `codeql pack upgrade` loop) and to use a token that triggers
+   CI.
 3. **Fix breakage and finish the checklist** — this PR does **not** publish anything by itself (no
    pack `version:` field is touched), so there's no rush, but it still needs:
    - [ ] Fix any compilation/test errors CI surfaces from upstream API changes (usually the
@@ -177,6 +223,16 @@ coding agent) in the loop for the hard part — fixing whatever the new CLI brea
          [Cutting a release](#cutting-a-release) below to bump every pack's `version:` in
          lockstep and trigger the real batch publish.
 
+> [!NOTE]
+> <a id="ql-hotspots"></a>**Why `ql/hotspots` is excluded from the `codeql pack upgrade` loop:**
+> `ql/hotspots` is a standalone local dev tool (a QL-4-QL hotspot query generator, see
+> `ql/hotspots/README.md` and `.github/workflows/hotspots.yml`) that patches a freshly-cloned
+> `github/codeql` checkout — it's not one of the per-language `src`/`lib`/`ext`/`ext-library-sources`
+> packs `ci.yml`/`publish.yml` operate on. Its `qlpack.yml` declares `codeql/ql: '*'`, but
+> `codeql/ql` isn't a real package published to the registry or shipped in the CodeQL Bundle, so
+> `codeql pack upgrade`/the pinning script can never resolve it. This is a pre-existing, unrelated
+> quirk of that tool, not something the version-bump automation needs to (or can) fix.
+
 > [!WARNING]
 > The `.codeqlversion` bump and the pack version bumps don't have to land in the same PR, but
 > splitting them is risky: [#124][pr-124] refreshed `.codeqlversion` and every language's
@@ -282,6 +338,7 @@ Please do get in touch (privacy@github.com) if you have any questions about this
 [update-release-workflow]: ./.github/workflows/update-release.yml
 [update-codeql-version-workflow]: ./.github/workflows/update-codeql-version.yml
 [detect-codeql-release-workflow]: ./.github/workflows/detect-codeql-release.yml
+[pin-codeql-library-versions-script]: ./.github/scripts/pin-codeql-library-versions.sh
 [codeql-cli-binaries]: https://github.com/github/codeql-cli-binaries/releases
 [release-config]: ./.release.yml
 [patch-release-me]: https://github.com/42ByteLabs/patch-release-me