ci: scan changed files for malicious code and disguised content

.github/scripts/fetch-yara-rules.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+#
+# Fetches the GuardDog YARA rule pack from a pinned release, verifying its
+# checksum before extracting anything.
+#
+#   .github/scripts/fetch-yara-rules.sh <destination-directory>
+#
+# The rules ship inside the wheel, so none of GuardDog's dependencies are
+# installed and Python is not needed to evaluate them.
+
+set -euo pipefail
+
+# Bump all three together, then re-measure the allowlist per .github/yara/README.md.
+# From https://pypi.org/pypi/guarddog/<version>/json - PyPI URLs embed a content
+# hash, so they are stable per file.
+readonly VERSION="3.1.0"
+readonly WHEEL_URL="https://files.pythonhosted.org/packages/ca/34/989428df4a2221dc6873944b23efa1e91bf10d49f4a1fcca9b1ceb6ecf12/guarddog-3.1.0-py3-none-any.whl"
+readonly WHEEL_SHA256="80572d0dfccb9028a78c0a61e66d54b95ea71b335b105b54c97955786fc00846"
+
+# Well under 3.1.0's 54 means the wheel layout changed and the globs missed.
+readonly MIN_RULES=40
+
+dest="${1:?usage: fetch-yara-rules.sh <destination-directory>}"
+tmp="$(mktemp -d)"
+trap 'rm -rf "$tmp"' EXIT
+
+curl --fail --silent --show-error --location --output "$tmp/wheel.zip" "$WHEEL_URL"
+printf '%s  %s\n' "$WHEEL_SHA256" "$tmp/wheel.zip" | shasum -a 256 -c - >/dev/null
+
+mkdir -p "$dest"
+# -j flattens paths: three rules `include` the .meta files by bare name, so those
+# must land beside the .yar files.
+unzip -q -o -j "$tmp/wheel.zip" \
+    'guarddog/analyzer/sourcecode/*.yar' \
+    'guarddog/analyzer/sourcecode/*.meta' \
+    -d "$dest"
+
+rule_count="$(find "$dest" -name '*.yar' | wc -l | tr -d ' ')"
+if (( rule_count < MIN_RULES )); then
+    echo "Extracted only $rule_count rules, expected at least $MIN_RULES." >&2
+    exit 1
+fi
+
+echo "Fetched $rule_count YARA rules from guarddog $VERSION into $dest" >&2

.github/scripts/find-type-mismatches.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+"""Report files whose content is executable but whose extension claims an asset.
+
+Reads magika JSONL on stdin, writes `path<TAB>label<TAB>extension` per mismatch.
+Used by scan-changed-files.sh; see .github/yara/README.md.
+"""
+
+import json
+import sys
+
+ASSET_EXTENSIONS = frozenset(
+    """
+    woff woff2 ttf otf eot png jpg jpeg gif bmp ico webp avif tiff pdf
+    zip gz bz2 xz 7z rar tar wasm so dylib dll exe o a lib bin dat db
+    sqlite mp3 mp4 wav mov mkv pack idx class pyc pyo jar img iso
+    """.split()
+)
+
+EXECUTABLE_GROUPS = frozenset({"code", "executable"})
+
+
+def is_mismatch(path: str, detected: dict) -> bool:
+    name = path.rsplit("/", 1)[-1]
+    extension = name.rsplit(".", 1)[-1].lower() if "." in name else ""
+    expected = {e.lower() for e in detected.get("extensions", ())}
+    # The last condition spares assets that are executable by nature: a real
+    # .wasm is code, and magika lists "wasm" as an expected extension for it.
+    return (
+        detected.get("group") in EXECUTABLE_GROUPS
+        and extension in ASSET_EXTENSIONS
+        and extension not in expected
+    )
+
+
+def main() -> int:
+    for line in sys.stdin:
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            entry = json.loads(line)
+        except ValueError:
+            # Format drift, not a clean file.
+            print(f"unparseable magika output: {line[:120]}", file=sys.stderr)
+            return 2
+
+        result = entry.get("result", {})
+        if result.get("status") != "ok":
+            continue
+
+        path = entry.get("path", "")
+        detected = result.get("value", {}).get("output", {})
+        if is_mismatch(path, detected):
+            label = detected.get("label", "unknown")
+            print(f"{path}\t{label}\t{path.rsplit('.', 1)[-1]}")
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/scripts/scan-changed-files.sh

@@ -0,0 +1,120 @@
+#!/usr/bin/env bash
+#
+# Scans changed files for malicious code, for executable content disguised as a
+# binary asset, and for edits to files that run on repo open or build.
+#
+#   RULES_DIR=<dir of .yar files> BASE_SHA=<sha> HEAD_SHA=<sha> scan-changed-files.sh
+#
+# Exit 1 on a blocking finding, 2 when it cannot scan properly, 0 otherwise.
+
+set -euo pipefail
+
+# Fetch the pack with fetch-yara-rules.sh; see .github/yara/README.md.
+RULES_DIR="${RULES_DIR:-}"
+BLOCKING_RULES_FILE="${BLOCKING_RULES_FILE:-.github/yara/blocking-rules.txt}"
+ALLOW_MISSING_SCANNERS="${ALLOW_MISSING_SCANNERS:-0}"
+
+readonly SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+
+# Runs on repo open or build without being invoked. Annotated, never blocked:
+# editing these is routine here.
+readonly AUTO_EXEC_PATTERN='^\.(vscode|devcontainer|githooks|cursor|claude|idea|github)/|\.code-workspace$|(^|/)build\.rs$|^\.cargo/config\.toml$|^(Makefile\.toml|justfile|flake\.nix|shell\.nix)$|\.(bat|cmd|ps1)$'
+
+log()  { printf '%s\n' "$*" >&2; }
+fail() { printf '::error::%s\n' "$*"; }
+warn() { printf '::warning::%s\n' "$*"; }
+die()  { log "$*"; exit 2; }
+
+# Returns 1 when the caller opted out of a missing scanner, so its section skips.
+have_scanner() {
+    command -v "$1" >/dev/null 2>&1 && return 0
+    [[ "$ALLOW_MISSING_SCANNERS" == "1" ]] || die "$1 is not installed; set ALLOW_MISSING_SCANNERS=1 to skip it."
+    log "$1 is not installed; skipping its checks."
+    return 1
+}
+
+[[ -n "$RULES_DIR" ]]    || die "RULES_DIR must point at a directory of .yar files."
+[[ -n "${BASE_SHA:-}" ]] || die "BASE_SHA must be set."
+[[ -n "${HEAD_SHA:-}" ]] || die "HEAD_SHA must be set."
+
+# Diffing against a base that is missing finds no files and would pass, so refuse.
+git cat-file -e "${BASE_SHA}^{commit}" 2>/dev/null \
+    || die "Base commit ${BASE_SHA} is not in this clone; needs actions/checkout with fetch-depth: 0."
+
+# -z is required: without it core.quotePath quotes any path holding a byte above
+# 0x80, the literal fails the -f test below, and a homoglyph-named payload is
+# dropped from the scan silently.
+mapfile -d '' -t changed < <(git diff --name-only --diff-filter=ACMR -z "$BASE_SHA" "$HEAD_SHA")
+
+files=()
+for path in "${changed[@]}"; do
+    [[ -f "$path" ]] && files+=("$path")   # renames and deletions are gone
+done
+
+if (( ${#files[@]} == 0 )); then
+    log "No files to scan."
+    exit 0
+fi
+
+log "Scanning ${#files[@]} changed file(s)."
+blocking_findings=0
+
+# ------------------------------------------------------------------- yara ---
+if have_scanner yarac && have_scanner yara; then
+    # Rules off the allowlist still report, they just do not fail the build, so a
+    # version bump cannot add an unmeasured gate.
+    declare -A is_blocking=()
+    while read -r rule; do
+        is_blocking["$rule"]=1
+    done < <(grep -vE '^[[:space:]]*(#|$)' "$BLOCKING_RULES_FILE")
+
+    (( ${#is_blocking[@]} > 0 )) || die "No blocking rules listed in $BLOCKING_RULES_FILE."
+    log "${#is_blocking[@]} rule(s) are blocking; the rest are advisory."
+
+    compiled="$(mktemp)"
+    trap 'rm -f "$compiled"' EXIT
+    yarac -w "$RULES_DIR"/*.yar "$compiled" || die "Failed to compile YARA rules from $RULES_DIR."
+
+    for path in "${files[@]}"; do
+        # One file per invocation: yara takes a single target, and given several it
+        # silently treats the extras as rule sources and still exits 0.
+        if ! matches=$(yara -w -C "$compiled" "$path" 2>&1); then
+            fail "$path: yara failed: $matches"
+            blocking_findings=1
+            continue
+        fi
+        while read -r rule _; do
+            [[ -n "$rule" ]] || continue
+            if [[ -n "${is_blocking[$rule]:-}" ]]; then
+                fail "$path: yara rule $rule matched"
+                blocking_findings=1
+            else
+                warn "$path: yara rule $rule matched (advisory only)"
+            fi
+        done <<< "$matches"
+    done
+fi
+
+# ----------------------------------------------------------------- magika ---
+if have_scanner magika; then
+    # Status checked here, not on the pipeline below whose status is python's: a
+    # magika crash would otherwise yield no findings and pass.
+    detected=$(magika --jsonl -- "${files[@]}" 2>&1) || die "magika failed: $detected"
+
+    mismatches=$(printf '%s\n' "$detected" | python3 "$SCRIPT_DIR/find-type-mismatches.py") \
+        || die "Could not interpret magika output."
+
+    while IFS=$'\t' read -r path label extension; do
+        [[ -n "$path" ]] || continue
+        fail "$path: content is $label but the .$extension extension declares a binary asset"
+        blocking_findings=1
+    done <<< "$mismatches"
+fi
+
+# --------------------------------------------------------- auto-exec paths ---
+for path in "${files[@]}"; do
+    [[ "$path" =~ $AUTO_EXEC_PATTERN ]] \
+        && warn "$path: runs automatically when the repo is opened or built - review as code"
+done
+
+exit "$blocking_findings"

.github/workflows/security-scan.yml

@@ -0,0 +1,75 @@
+name: Security Scan
+
+# Runs on every branch, not just PRs. The implant this job exists to catch was
+# force-pushed onto the head branches of already-open pull requests, and a push
+# to a branch with no open PR fires no `pull_request` event at all.
+on:
+  pull_request:
+  push:
+    branches:
+      - '**'
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  scan-changed-files:
+    name: "Scan changed files"
+    # A push covers every same-repo branch, including a push to the branch of an
+    # already-merged PR, which fires no pull_request event - that is how the
+    # implant this job exists to catch arrived. Fork PRs produce no push event
+    # here, so pull_request covers those; the guard keeps same-repo PRs from being
+    # scanned twice and annotated twice.
+    if: >-
+      github.event_name != 'pull_request' ||
+      github.event.pull_request.head.repo.full_name != github.repository
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+        with:
+          # Full history: the scan diffs against a base commit, and with a
+          # shallow clone that object is absent. The script treats a missing
+          # base as a hard error rather than scanning nothing and passing.
+          fetch-depth: 0
+          persist-credentials: false
+
+      - name: Install yara and magika
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends yara
+          pipx install magika==1.1.0
+
+      - name: Fetch YARA rules
+        run: .github/scripts/fetch-yara-rules.sh "$RUNNER_TEMP/yara-rules"
+
+      - name: Resolve the base commit
+        id: base
+        env:
+          PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          PUSH_BEFORE: ${{ github.event.before }}
+        run: |
+          if [ -n "$PR_BASE_SHA" ]; then
+            base="$PR_BASE_SHA"
+          elif [ -n "$PUSH_BEFORE" ] && git cat-file -e "${PUSH_BEFORE}^{commit}" 2>/dev/null; then
+            # Absent on branch creation, and set to all-zeroes on force-push to
+            # a ref whose old tip is already unreachable.
+            base="$PUSH_BEFORE"
+          else
+            base="$(git rev-parse HEAD~1)"
+          fi
+          echo "sha=$base" >> "$GITHUB_OUTPUT"
+
+      - name: Scan
+        env:
+          BASE_SHA: ${{ steps.base.outputs.sha }}
+          HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
+          RULES_DIR: ${{ runner.temp }}/yara-rules
+        run: .github/scripts/scan-changed-files.sh

.github/yara/README.md

@@ -0,0 +1,67 @@
+# YARA scanning of changed files
+
+`.github/workflows/security-scan.yml` runs a generic malicious-code rule pack over the files a
+push or pull request touches. The rules come from
+[DataDog/guarddog](https://github.com/DataDog/guarddog) (Apache-2.0) and describe behaviour —
+obfuscation, download-and-execute, silent process spawn, reverse shells, exfiltration, autostart
+persistence — rather than signatures for any single campaign.
+
+## The rules are not committed here
+
+`.github/scripts/fetch-yara-rules.sh` downloads them from a pinned GuardDog release and verifies
+the wheel's SHA-256 before extracting. That keeps 54 files of third-party content out of the
+repository and out of review, while still being reproducible: the version and checksum are pinned
+in that script, so a change to what CI enforces is a one-line diff rather than a 54-file one. This
+matches how `ci.yml` already pins `repro-env`.
+
+The rules ship inside the published wheel, so nothing GuardDog depends on gets installed and
+Python is not needed to evaluate them.
+
+Run it locally the same way CI does:
+
+```sh
+.github/scripts/fetch-yara-rules.sh /tmp/yara-rules
+RULES_DIR=/tmp/yara-rules \
+BASE_SHA=$(git merge-base origin/main HEAD) HEAD_SHA=$(git rev-parse HEAD) \
+  .github/scripts/scan-changed-files.sh
+```
+
+## Why not GuardDog's own scanner
+
+Every rule declares `path_include = "*.js,*.ts,..."` in its `meta:` block. That key is not a YARA
+construct: GuardDog's Python driver parses it and filters candidates with `fnmatch`, so it never
+opens a file whose extension is absent from the list. Selecting files by extension is the blind
+spot this scan exists to close — a payload named `.woff2` would simply be skipped. The `yara`
+binary treats unknown `meta:` keys as inert, so pointing it at a file list scans everything
+regardless of name.
+
+## blocking-rules.txt
+
+Only the rules listed there fail the build. Everything else in the pack still runs and still
+annotates the PR, but advisory only.
+
+The split is measured, not guessed: a rule qualifies as blocking only with zero false positives
+across both the whole tracked tree and every file version touched by the last 400 commits on
+`main`. Of 54 rules, 36 qualified; the 18 excluded are mostly `capability_*` rules, which flag the
+presence of a capability rather than misuse of it and so fire on ordinary code. Defaulting new
+rules to advisory means bumping the pinned version cannot silently introduce a gate nobody
+measured.
+
+## Bumping the version
+
+1. Edit `VERSION`, `WHEEL_URL` and `WHEEL_SHA256` in
+   `.github/scripts/fetch-yara-rules.sh`. All three come from
+   `https://pypi.org/pypi/guarddog/<version>/json`, from the `py3-none-any.whl` entry
+   under `.urls[]`.
+2. Re-measure, from a clean tree:
+
+   ```sh
+   .github/scripts/fetch-yara-rules.sh /tmp/yara-rules
+   yarac -w /tmp/yara-rules/*.yar /tmp/rules.yarc
+   git ls-files -z | xargs -0 -n1 yara -w -C /tmp/rules.yarc
+   ```
+
+   `yara` accepts many rule files but only one target path, and given several it silently treats
+   the extras as rule sources and still exits 0 — hence `-n1`.
+3. Any rule that fires has a false positive: drop it from `blocking-rules.txt` with a note, or fix
+   the offending file.