ci: scan changed files for malicious code and disguised content

.github/scripts/fetch-scanners.sh

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+#
+# Downloads the scanners the security scan needs, verifying each checksum before
+# use, and prints the directory they landed in.
+#
+#   eval "$(.github/scripts/fetch-scanners.sh <destination-directory>)"
+#
+# CI only, and x86_64 Linux only. Locally the same versions come from the nix dev
+# shell (nix/yara-x.nix, nix/magika.nix) - keep the versions here in step with
+# those files.
+#
+# This replaces apt + pipx, which cost ~16s of a 26s job: apt-get update 5.4s,
+# apt-get install yara 2.9s, pipx install magika 7.4s.
+
+set -euo pipefail
+
+# Release assets are maintainer-mutable - a tag can be deleted and re-uploaded -
+# so the digest is what protects this job, not the tag. Never use a floating tag
+# such as magika's `cli-latest`.
+readonly YARA_X_VERSION="1.19.0"
+readonly YARA_X_SHA256="a97d78189e3548797ac45b7b4a5fd8975783861875c594f772ec9b8bb5fa4d72"
+
+# magika versions its CLI separately from its Python package, and the CLI number
+# is what `magika --version` reports.
+readonly MAGIKA_VERSION="1.1.0"
+readonly MAGIKA_SHA256="6b4c1010c84d1f4f06205ccef4597f1690bcd7744f46d841eee26426bc100485"
+
+dest="${1:?usage: fetch-scanners.sh <destination-directory>}"
+
+if [[ "$(uname -s)-$(uname -m)" != "Linux-x86_64" ]]; then
+    echo "This script only handles Linux x86_64 (CI). Use 'nix develop' locally." >&2
+    exit 2
+fi
+
+tmp="$(mktemp -d)"
+trap 'rm -rf "$tmp"' EXIT
+
+fetch() {
+    local url="$1" out="$2" sha="$3"
+    curl --fail --silent --show-error --location --output "$out" "$url"
+    printf '%s  %s\n' "$sha" "$out" | sha256sum -c - >/dev/null
+}
+
+mkdir -p "$dest"
+
+fetch "https://github.com/VirusTotal/yara-x/releases/download/v${YARA_X_VERSION}/yara-x-v${YARA_X_VERSION}-x86_64-unknown-linux-gnu.tar.gz" \
+    "$tmp/yara-x.tar.gz" "$YARA_X_SHA256"
+# Holds a bare `yr`.
+tar xzf "$tmp/yara-x.tar.gz" -C "$dest" yr
+
+fetch "https://github.com/google/magika/releases/download/cli/v${MAGIKA_VERSION}/magika-cli-x86_64-unknown-linux-gnu.tar.xz" \
+    "$tmp/magika.tar.xz" "$MAGIKA_SHA256"
+# Nests the binary one directory deep, unlike the yara-x archive.
+tar xJf "$tmp/magika.tar.xz" -C "$dest" --strip-components=1
+
+"$dest/yr" --version | grep -q "$YARA_X_VERSION" \
+    || { echo "yr is not version $YARA_X_VERSION" >&2; exit 1; }
+"$dest/magika" --version | grep -q "$MAGIKA_VERSION" \
+    || { echo "magika is not version $MAGIKA_VERSION" >&2; exit 1; }
+
+echo "Fetched yara-x $YARA_X_VERSION and magika $MAGIKA_VERSION into $dest" >&2

.github/scripts/fetch-yara-rules.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+#
+# Fetches the GuardDog YARA rule pack from a pinned release, verifying its
+# checksum before extracting anything.
+#
+#   .github/scripts/fetch-yara-rules.sh <destination-directory>
+#
+# The rules ship inside the wheel, so none of GuardDog's dependencies are
+# installed and Python is not needed to evaluate them.
+
+set -euo pipefail
+
+# Bump all three together, then re-measure the allowlist per .github/yara/README.md.
+# From https://pypi.org/pypi/guarddog/<version>/json - PyPI URLs embed a content
+# hash, so they are stable per file.
+readonly VERSION="3.1.0"
+readonly WHEEL_URL="https://files.pythonhosted.org/packages/ca/34/989428df4a2221dc6873944b23efa1e91bf10d49f4a1fcca9b1ceb6ecf12/guarddog-3.1.0-py3-none-any.whl"
+readonly WHEEL_SHA256="80572d0dfccb9028a78c0a61e66d54b95ea71b335b105b54c97955786fc00846"
+
+# Well under 3.1.0's 54 means the wheel layout changed and the globs missed.
+readonly MIN_RULES=40
+
+dest="${1:?usage: fetch-yara-rules.sh <destination-directory>}"
+tmp="$(mktemp -d)"
+trap 'rm -rf "$tmp"' EXIT
+
+curl --fail --silent --show-error --location --output "$tmp/wheel.zip" "$WHEEL_URL"
+printf '%s  %s\n' "$WHEEL_SHA256" "$tmp/wheel.zip" | shasum -a 256 -c - >/dev/null
+
+mkdir -p "$dest"
+# -j flattens paths: three rules `include` the .meta files by bare name, so those
+# must land beside the .yar files.
+unzip -q -o -j "$tmp/wheel.zip" \
+    'guarddog/analyzer/sourcecode/*.yar' \
+    'guarddog/analyzer/sourcecode/*.meta' \
+    -d "$dest"
+
+rule_count="$(find "$dest" -name '*.yar' | wc -l | tr -d ' ')"
+if (( rule_count < MIN_RULES )); then
+    echo "Extracted only $rule_count rules, expected at least $MIN_RULES." >&2
+    exit 1
+fi
+
+echo "Fetched $rule_count YARA rules from guarddog $VERSION into $dest" >&2

.github/scripts/find-type-mismatches.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+"""Report files whose content is executable but whose extension claims an asset.
+
+Reads magika JSONL on stdin, writes `path<TAB>label<TAB>extension` per mismatch.
+Used by scan-changed-files.sh; see .github/yara/README.md.
+"""
+
+import json
+import sys
+
+ASSET_EXTENSIONS = frozenset(
+    """
+    woff woff2 ttf otf eot png jpg jpeg gif bmp ico webp avif tiff pdf
+    zip gz bz2 xz 7z rar tar wasm so dylib dll exe o a lib bin dat db
+    sqlite mp3 mp4 wav mov mkv pack idx class pyc pyo jar img iso
+    """.split()
+)
+
+EXECUTABLE_GROUPS = frozenset({"code", "executable"})
+
+
+def is_mismatch(path: str, detected: dict) -> bool:
+    name = path.rsplit("/", 1)[-1]
+    extension = name.rsplit(".", 1)[-1].lower() if "." in name else ""
+    expected = {e.lower() for e in detected.get("extensions", ())}
+    # The last condition spares assets that are executable by nature: a real
+    # .wasm is code, and magika lists "wasm" as an expected extension for it.
+    return (
+        detected.get("group") in EXECUTABLE_GROUPS
+        and extension in ASSET_EXTENSIONS
+        and extension not in expected
+    )
+
+
+def main() -> int:
+    for line in sys.stdin:
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            entry = json.loads(line)
+        except ValueError:
+            # Format drift, not a clean file.
+            print(f"unparseable magika output: {line[:120]}", file=sys.stderr)
+            return 2
+
+        result = entry.get("result", {})
+        if result.get("status") != "ok":
+            # Unclassifiable, so this file leaves the check unexamined. Say so
+            # rather than letting it drop out silently.
+            print(
+                f"magika could not classify {entry.get('path', '?')}: "
+                f"{result.get('status', 'unknown')}",
+                file=sys.stderr,
+            )
+            continue
+
+        path = entry.get("path", "")
+        detected = result.get("value", {}).get("output", {})
+        if is_mismatch(path, detected):
+            label = detected.get("label", "unknown")
+            print(f"{path}\t{label}\t{path.rsplit('.', 1)[-1]}")
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/scripts/scan-changed-files.sh

@@ -0,0 +1,160 @@
+#!/usr/bin/env bash
+#
+# Scans changed files for malicious code, for executable content disguised as a
+# binary asset, and for edits to files that run on repo open or build.
+#
+#   RULES_DIR=<dir of .yar files> BASE_SHA=<sha> HEAD_SHA=<sha> scan-changed-files.sh
+#
+# Exit 1 on a blocking finding, 2 when it cannot scan properly, 0 otherwise.
+
+set -euo pipefail
+
+# Fetch the pack with fetch-yara-rules.sh; see .github/yara/README.md.
+RULES_DIR="${RULES_DIR:-}"
+ALLOW_MISSING_SCANNERS="${ALLOW_MISSING_SCANNERS:-0}"
+
+readonly SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+BLOCKING_RULES_FILE="${BLOCKING_RULES_FILE:-$SCRIPT_DIR/../yara/blocking-rules.txt}"
+
+# yara-x spins up a thread pool per run, which costs more than it saves on a
+# small list. Measured on a 3-file scan: 33ms default versus 27ms single-threaded,
+# while at 930 files single-threaded is 330ms against 116ms. Switch at a point
+# comfortably above this repo's p90 diff of 17 files.
+readonly SINGLE_THREAD_BELOW=50
+
+# Runs on repo open or build without being invoked. Annotated, never blocked:
+# editing these is routine here.
+readonly AUTO_EXEC_PATTERN='^\.(vscode|devcontainer|githooks|cursor|claude|idea|github)/|\.code-workspace$|(^|/)build\.rs$|^\.cargo/config\.toml$|^(Makefile\.toml|justfile|flake\.nix|shell\.nix)$|\.(bat|cmd|ps1)$'
+
+log()  { printf '%s\n' "$*" >&2; }
+fail() { printf '::error::%s\n' "$*"; }
+warn() { printf '::warning::%s\n' "$*"; }
+die()  { log "$*"; exit 2; }
+
+# Returns 1 when the caller opted out of a missing scanner, so its section skips.
+have_scanner() {
+    command -v "$1" >/dev/null 2>&1 && return 0
+    [[ "$ALLOW_MISSING_SCANNERS" == "1" ]] || die "$1 is not installed; set ALLOW_MISSING_SCANNERS=1 to skip it."
+    log "$1 is not installed; skipping its checks."
+    return 1
+}
+
+[[ -n "$RULES_DIR" ]]    || die "RULES_DIR must point at a directory of .yar files."
+[[ -n "${BASE_SHA:-}" ]] || die "BASE_SHA must be set."
+[[ -n "${HEAD_SHA:-}" ]] || die "HEAD_SHA must be set."
+
+# A commit missing from the clone makes the diff empty, which would pass, so
+# refuse. Both ends need this: a head sha goes missing when a fork PR's branch is
+# force-pushed between event dispatch and checkout.
+for sha in "$BASE_SHA" "$HEAD_SHA"; do
+    git cat-file -e "${sha}^{commit}" 2>/dev/null \
+        || die "Commit ${sha} is not in this clone; needs actions/checkout with fetch-depth: 0."
+done
+
+workdir="$(mktemp -d)"
+trap 'rm -rf "$workdir"' EXIT
+
+# -z is required: without it core.quotePath quotes any path holding a byte above
+# 0x80, the literal fails the -f test below, and a homoglyph-named payload is
+# dropped from the scan silently. Staged through a file because a process
+# substitution's exit status is invisible to mapfile.
+git diff --name-only --diff-filter=ACMR -z "$BASE_SHA" "$HEAD_SHA" > "$workdir/changed" \
+    || die "git diff ${BASE_SHA}..${HEAD_SHA} failed."
+mapfile -d '' -t changed < "$workdir/changed"
+
+files=()
+for path in "${changed[@]}"; do
+    if [[ -f "$path" ]]; then
+        # Both scanners take their targets as a newline-delimited list, so a path
+        # containing a newline cannot be expressed and would be read as two
+        # entries. Git permits such paths, so refuse rather than mis-scan.
+        [[ "$path" == *$'\n'* ]] && die "Path contains a newline, which cannot be scanned safely: ${path@Q}"
+        files+=("$path")
+    else
+        # ACMR excludes deletions and a rename reports only its destination, so
+        # anything else here is unexpected rather than routine.
+        log "Skipping ${path}: not a regular file."
+    fi
+done
+
+if (( ${#files[@]} == 0 )); then
+    log "No files to scan."
+    exit 0
+fi
+
+log "Scanning ${#files[@]} changed file(s)."
+printf '%s\n' "${files[@]}" > "$workdir/list"
+blocking_findings=0
+
+# ----------------------------------------------------------------- yara-x ---
+if have_scanner yr; then
+    # Rules off the allowlist still report, they just do not fail the build, so a
+    # version bump cannot add an unmeasured gate.
+    declare -A is_blocking=()
+    while read -r rule || [[ -n "$rule" ]]; do   # tolerate a missing final newline
+        is_blocking["$rule"]=1
+    done < <(grep -vE '^[[:space:]]*(#|$)' "$BLOCKING_RULES_FILE")
+
+    (( ${#is_blocking[@]} > 0 )) || die "No blocking rules listed in $BLOCKING_RULES_FILE."
+
+    # --include-dir because three rules `include` the .meta files by bare name and
+    # yr resolves those against the working directory, not the including file.
+    compiled="$workdir/rules.yarc"
+    yr compile -w --include-dir "$RULES_DIR" "$RULES_DIR"/*.yar -o "$compiled" 2>"$workdir/compile.err" \
+        || die "Failed to compile YARA rules: $(cat "$workdir/compile.err")"
+
+    # An allowlisted name that no longer exists means a bump renamed the rule, and
+    # the renamed one would silently drop to advisory.
+    mapfile -t pack_rules < <(sed -n 's/^rule[[:space:]]\{1,\}\([A-Za-z0-9_]\{1,\}\).*/\1/p' "$RULES_DIR"/*.yar | sort -u)
+    for rule in "${!is_blocking[@]}"; do
+        printf '%s\n' "${pack_rules[@]}" | grep -qxF "$rule" \
+            || die "Blocking rule '$rule' is not in the pack; re-measure and update $BLOCKING_RULES_FILE."
+    done
+    log "${#is_blocking[@]} rule(s) are blocking; the rest are advisory."
+
+    threads=()
+    (( ${#files[@]} < SINGLE_THREAD_BELOW )) && threads=(--threads 1)
+
+    # yr exits 0 even when it could not read a listed file, reporting only on
+    # stderr, so stderr is the failure signal here rather than the exit status.
+    yr scan -w "${threads[@]}" --compiled-rules --scan-list "$compiled" "$workdir/list" \
+        > "$workdir/matches" 2> "$workdir/scan.err" \
+        || die "yr scan failed: $(cat "$workdir/scan.err")"
+    [[ -s "$workdir/scan.err" ]] && die "yr could not scan every file: $(cat "$workdir/scan.err")"
+
+    while read -r rule path; do
+        [[ -n "$rule" ]] || continue
+        if [[ -n "${is_blocking[$rule]:-}" ]]; then
+            fail "$path: yara rule $rule matched"
+            blocking_findings=1
+        else
+            warn "$path: yara rule $rule matched (advisory only)"
+        fi
+    done < "$workdir/matches"
+fi
+
+# ----------------------------------------------------------------- magika ---
+if have_scanner magika; then
+    # Status checked separately from the pipeline below, whose status would be
+    # python's: a magika crash would otherwise yield no findings and pass.
+    magika --jsonl -- "${files[@]}" > "$workdir/detected" 2> "$workdir/magika.err" \
+        || die "magika failed: $(cat "$workdir/magika.err")"
+    [[ -s "$workdir/magika.err" ]] && log "magika stderr: $(cat "$workdir/magika.err")"
+
+    mismatches=$(python3 "$SCRIPT_DIR/find-type-mismatches.py" < "$workdir/detected") \
+        || die "Could not interpret magika output."
+
+    while IFS=$'\t' read -r path label extension; do
+        [[ -n "$path" ]] || continue
+        fail "$path: content is $label but the .$extension extension declares a binary asset"
+        blocking_findings=1
+    done <<< "$mismatches"
+fi
+
+# --------------------------------------------------------- auto-exec paths ---
+for path in "${files[@]}"; do
+    [[ "$path" =~ $AUTO_EXEC_PATTERN ]] \
+        && warn "$path: runs automatically when the repo is opened or built - review as code"
+done
+
+exit "$blocking_findings"