From bed5ee479a3b0ef812ca45ed9a84959da0aa64f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patryk=20B=C4=99za?= Date: Mon, 3 Aug 2026 18:45:38 +0200 Subject: [PATCH 1/8] ci: scan changed files for malicious code and disguised content Nothing in CI inspected file content. Scanners that select files by extension never open a payload named .woff2, and dependency scanners read only the dependency graph, so code vendored straight into the tree went unexamined. Adds a required job with two content-based checks. YARA runs DataDog's GuardDog pack, a generic behavioural set covering obfuscation, download-and-execute, silent process spawn, reverse shells and exfiltration. Magika decides content type from content, so a script parked behind an asset extension is visible even when the code itself looks unremarkable; it fires only when the detected type is executable, the extension claims a binary asset, and that extension disagrees with what the type is normally called. All three conditions are needed, or a legitimate .wasm trips it. Only rules in .github/yara/blocking-rules.txt fail the build. That split is measured: a rule qualifies at zero false positives across the 923 tracked files and the 2824 file-versions touched by the last 400 commits on main. 36 of 54 qualified; the excluded ones are mostly capability_* rules, which report that a capability is present rather than misused. New rules therefore default to advisory, so a version bump cannot introduce an unmeasured gate. The pack is fetched from a pinned release and checksum-verified rather than committed, keeping third-party rules out of review while staying reproducible, as ci.yml already does for repro-env. The job runs on all branches, since a push to the branch of a merged PR fires no pull_request event, and skips the pull_request run for same-repo branches to avoid scanning and annotating twice. It reads the changed-file list with git diff -z: without it core.quotePath quotes any path holding a byte above 0x80, that literal fails the -f test, and a homoglyph-named payload is dropped from the scan silently. A missing base commit, a missing scanner or a scanner crash all fail the job, because a scan that examines nothing would otherwise report success. --- .github/scripts/fetch-yara-rules.sh | 44 +++++++++ .github/scripts/find-type-mismatches.py | 61 ++++++++++++ .github/scripts/scan-changed-files.sh | 120 ++++++++++++++++++++++++ .github/workflows/security-scan.yml | 75 +++++++++++++++ .github/yara/README.md | 67 +++++++++++++ .github/yara/blocking-rules.txt | 52 ++++++++++ 6 files changed, 419 insertions(+) create mode 100755 .github/scripts/fetch-yara-rules.sh create mode 100755 .github/scripts/find-type-mismatches.py create mode 100755 .github/scripts/scan-changed-files.sh create mode 100644 .github/workflows/security-scan.yml create mode 100644 .github/yara/README.md create mode 100644 .github/yara/blocking-rules.txt diff --git a/.github/scripts/fetch-yara-rules.sh b/.github/scripts/fetch-yara-rules.sh new file mode 100755 index 0000000000..7a8b0d7278 --- /dev/null +++ b/.github/scripts/fetch-yara-rules.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# +# Fetches the GuardDog YARA rule pack from a pinned release, verifying its +# checksum before extracting anything. +# +# .github/scripts/fetch-yara-rules.sh +# +# The rules ship inside the wheel, so none of GuardDog's dependencies are +# installed and Python is not needed to evaluate them. + +set -euo pipefail + +# Bump all three together, then re-measure the allowlist per .github/yara/README.md. +# From https://pypi.org/pypi/guarddog//json - PyPI URLs embed a content +# hash, so they are stable per file. +readonly VERSION="3.1.0" +readonly WHEEL_URL="https://files.pythonhosted.org/packages/ca/34/989428df4a2221dc6873944b23efa1e91bf10d49f4a1fcca9b1ceb6ecf12/guarddog-3.1.0-py3-none-any.whl" +readonly WHEEL_SHA256="80572d0dfccb9028a78c0a61e66d54b95ea71b335b105b54c97955786fc00846" + +# Well under 3.1.0's 54 means the wheel layout changed and the globs missed. +readonly MIN_RULES=40 + +dest="${1:?usage: fetch-yara-rules.sh }" +tmp="$(mktemp -d)" +trap 'rm -rf "$tmp"' EXIT + +curl --fail --silent --show-error --location --output "$tmp/wheel.zip" "$WHEEL_URL" +printf '%s %s\n' "$WHEEL_SHA256" "$tmp/wheel.zip" | shasum -a 256 -c - >/dev/null + +mkdir -p "$dest" +# -j flattens paths: three rules `include` the .meta files by bare name, so those +# must land beside the .yar files. +unzip -q -o -j "$tmp/wheel.zip" \ + 'guarddog/analyzer/sourcecode/*.yar' \ + 'guarddog/analyzer/sourcecode/*.meta' \ + -d "$dest" + +rule_count="$(find "$dest" -name '*.yar' | wc -l | tr -d ' ')" +if (( rule_count < MIN_RULES )); then + echo "Extracted only $rule_count rules, expected at least $MIN_RULES." >&2 + exit 1 +fi + +echo "Fetched $rule_count YARA rules from guarddog $VERSION into $dest" >&2 diff --git a/.github/scripts/find-type-mismatches.py b/.github/scripts/find-type-mismatches.py new file mode 100755 index 0000000000..5cee44929e --- /dev/null +++ b/.github/scripts/find-type-mismatches.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +"""Report files whose content is executable but whose extension claims an asset. + +Reads magika JSONL on stdin, writes `pathlabelextension` per mismatch. +Used by scan-changed-files.sh; see .github/yara/README.md. +""" + +import json +import sys + +ASSET_EXTENSIONS = frozenset( + """ + woff woff2 ttf otf eot png jpg jpeg gif bmp ico webp avif tiff pdf + zip gz bz2 xz 7z rar tar wasm so dylib dll exe o a lib bin dat db + sqlite mp3 mp4 wav mov mkv pack idx class pyc pyo jar img iso + """.split() +) + +EXECUTABLE_GROUPS = frozenset({"code", "executable"}) + + +def is_mismatch(path: str, detected: dict) -> bool: + name = path.rsplit("/", 1)[-1] + extension = name.rsplit(".", 1)[-1].lower() if "." in name else "" + expected = {e.lower() for e in detected.get("extensions", ())} + # The last condition spares assets that are executable by nature: a real + # .wasm is code, and magika lists "wasm" as an expected extension for it. + return ( + detected.get("group") in EXECUTABLE_GROUPS + and extension in ASSET_EXTENSIONS + and extension not in expected + ) + + +def main() -> int: + for line in sys.stdin: + line = line.strip() + if not line: + continue + try: + entry = json.loads(line) + except ValueError: + # Format drift, not a clean file. + print(f"unparseable magika output: {line[:120]}", file=sys.stderr) + return 2 + + result = entry.get("result", {}) + if result.get("status") != "ok": + continue + + path = entry.get("path", "") + detected = result.get("value", {}).get("output", {}) + if is_mismatch(path, detected): + label = detected.get("label", "unknown") + print(f"{path}\t{label}\t{path.rsplit('.', 1)[-1]}") + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/scan-changed-files.sh b/.github/scripts/scan-changed-files.sh new file mode 100755 index 0000000000..bbb8abb028 --- /dev/null +++ b/.github/scripts/scan-changed-files.sh @@ -0,0 +1,120 @@ +#!/usr/bin/env bash +# +# Scans changed files for malicious code, for executable content disguised as a +# binary asset, and for edits to files that run on repo open or build. +# +# RULES_DIR= BASE_SHA= HEAD_SHA= scan-changed-files.sh +# +# Exit 1 on a blocking finding, 2 when it cannot scan properly, 0 otherwise. + +set -euo pipefail + +# Fetch the pack with fetch-yara-rules.sh; see .github/yara/README.md. +RULES_DIR="${RULES_DIR:-}" +BLOCKING_RULES_FILE="${BLOCKING_RULES_FILE:-.github/yara/blocking-rules.txt}" +ALLOW_MISSING_SCANNERS="${ALLOW_MISSING_SCANNERS:-0}" + +readonly SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" + +# Runs on repo open or build without being invoked. Annotated, never blocked: +# editing these is routine here. +readonly AUTO_EXEC_PATTERN='^\.(vscode|devcontainer|githooks|cursor|claude|idea|github)/|\.code-workspace$|(^|/)build\.rs$|^\.cargo/config\.toml$|^(Makefile\.toml|justfile|flake\.nix|shell\.nix)$|\.(bat|cmd|ps1)$' + +log() { printf '%s\n' "$*" >&2; } +fail() { printf '::error::%s\n' "$*"; } +warn() { printf '::warning::%s\n' "$*"; } +die() { log "$*"; exit 2; } + +# Returns 1 when the caller opted out of a missing scanner, so its section skips. +have_scanner() { + command -v "$1" >/dev/null 2>&1 && return 0 + [[ "$ALLOW_MISSING_SCANNERS" == "1" ]] || die "$1 is not installed; set ALLOW_MISSING_SCANNERS=1 to skip it." + log "$1 is not installed; skipping its checks." + return 1 +} + +[[ -n "$RULES_DIR" ]] || die "RULES_DIR must point at a directory of .yar files." +[[ -n "${BASE_SHA:-}" ]] || die "BASE_SHA must be set." +[[ -n "${HEAD_SHA:-}" ]] || die "HEAD_SHA must be set." + +# Diffing against a base that is missing finds no files and would pass, so refuse. +git cat-file -e "${BASE_SHA}^{commit}" 2>/dev/null \ + || die "Base commit ${BASE_SHA} is not in this clone; needs actions/checkout with fetch-depth: 0." + +# -z is required: without it core.quotePath quotes any path holding a byte above +# 0x80, the literal fails the -f test below, and a homoglyph-named payload is +# dropped from the scan silently. +mapfile -d '' -t changed < <(git diff --name-only --diff-filter=ACMR -z "$BASE_SHA" "$HEAD_SHA") + +files=() +for path in "${changed[@]}"; do + [[ -f "$path" ]] && files+=("$path") # renames and deletions are gone +done + +if (( ${#files[@]} == 0 )); then + log "No files to scan." + exit 0 +fi + +log "Scanning ${#files[@]} changed file(s)." +blocking_findings=0 + +# ------------------------------------------------------------------- yara --- +if have_scanner yarac && have_scanner yara; then + # Rules off the allowlist still report, they just do not fail the build, so a + # version bump cannot add an unmeasured gate. + declare -A is_blocking=() + while read -r rule; do + is_blocking["$rule"]=1 + done < <(grep -vE '^[[:space:]]*(#|$)' "$BLOCKING_RULES_FILE") + + (( ${#is_blocking[@]} > 0 )) || die "No blocking rules listed in $BLOCKING_RULES_FILE." + log "${#is_blocking[@]} rule(s) are blocking; the rest are advisory." + + compiled="$(mktemp)" + trap 'rm -f "$compiled"' EXIT + yarac -w "$RULES_DIR"/*.yar "$compiled" || die "Failed to compile YARA rules from $RULES_DIR." + + for path in "${files[@]}"; do + # One file per invocation: yara takes a single target, and given several it + # silently treats the extras as rule sources and still exits 0. + if ! matches=$(yara -w -C "$compiled" "$path" 2>&1); then + fail "$path: yara failed: $matches" + blocking_findings=1 + continue + fi + while read -r rule _; do + [[ -n "$rule" ]] || continue + if [[ -n "${is_blocking[$rule]:-}" ]]; then + fail "$path: yara rule $rule matched" + blocking_findings=1 + else + warn "$path: yara rule $rule matched (advisory only)" + fi + done <<< "$matches" + done +fi + +# ----------------------------------------------------------------- magika --- +if have_scanner magika; then + # Status checked here, not on the pipeline below whose status is python's: a + # magika crash would otherwise yield no findings and pass. + detected=$(magika --jsonl -- "${files[@]}" 2>&1) || die "magika failed: $detected" + + mismatches=$(printf '%s\n' "$detected" | python3 "$SCRIPT_DIR/find-type-mismatches.py") \ + || die "Could not interpret magika output." + + while IFS=$'\t' read -r path label extension; do + [[ -n "$path" ]] || continue + fail "$path: content is $label but the .$extension extension declares a binary asset" + blocking_findings=1 + done <<< "$mismatches" +fi + +# --------------------------------------------------------- auto-exec paths --- +for path in "${files[@]}"; do + [[ "$path" =~ $AUTO_EXEC_PATTERN ]] \ + && warn "$path: runs automatically when the repo is opened or built - review as code" +done + +exit "$blocking_findings" diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml new file mode 100644 index 0000000000..655fc7c1a1 --- /dev/null +++ b/.github/workflows/security-scan.yml @@ -0,0 +1,75 @@ +name: Security Scan + +# Runs on every branch, not just PRs. The implant this job exists to catch was +# force-pushed onto the head branches of already-open pull requests, and a push +# to a branch with no open PR fires no `pull_request` event at all. +on: + pull_request: + push: + branches: + - '**' + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + scan-changed-files: + name: "Scan changed files" + # A push covers every same-repo branch, including a push to the branch of an + # already-merged PR, which fires no pull_request event - that is how the + # implant this job exists to catch arrived. Fork PRs produce no push event + # here, so pull_request covers those; the guard keeps same-repo PRs from being + # scanned twice and annotated twice. + if: >- + github.event_name != 'pull_request' || + github.event.pull_request.head.repo.full_name != github.repository + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - name: Checkout repository + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + with: + # Full history: the scan diffs against a base commit, and with a + # shallow clone that object is absent. The script treats a missing + # base as a hard error rather than scanning nothing and passing. + fetch-depth: 0 + persist-credentials: false + + - name: Install yara and magika + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends yara + pipx install magika==1.1.0 + + - name: Fetch YARA rules + run: .github/scripts/fetch-yara-rules.sh "$RUNNER_TEMP/yara-rules" + + - name: Resolve the base commit + id: base + env: + PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} + PUSH_BEFORE: ${{ github.event.before }} + run: | + if [ -n "$PR_BASE_SHA" ]; then + base="$PR_BASE_SHA" + elif [ -n "$PUSH_BEFORE" ] && git cat-file -e "${PUSH_BEFORE}^{commit}" 2>/dev/null; then + # Absent on branch creation, and set to all-zeroes on force-push to + # a ref whose old tip is already unreachable. + base="$PUSH_BEFORE" + else + base="$(git rev-parse HEAD~1)" + fi + echo "sha=$base" >> "$GITHUB_OUTPUT" + + - name: Scan + env: + BASE_SHA: ${{ steps.base.outputs.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + RULES_DIR: ${{ runner.temp }}/yara-rules + run: .github/scripts/scan-changed-files.sh diff --git a/.github/yara/README.md b/.github/yara/README.md new file mode 100644 index 0000000000..6a91104d4e --- /dev/null +++ b/.github/yara/README.md @@ -0,0 +1,67 @@ +# YARA scanning of changed files + +`.github/workflows/security-scan.yml` runs a generic malicious-code rule pack over the files a +push or pull request touches. The rules come from +[DataDog/guarddog](https://github.com/DataDog/guarddog) (Apache-2.0) and describe behaviour — +obfuscation, download-and-execute, silent process spawn, reverse shells, exfiltration, autostart +persistence — rather than signatures for any single campaign. + +## The rules are not committed here + +`.github/scripts/fetch-yara-rules.sh` downloads them from a pinned GuardDog release and verifies +the wheel's SHA-256 before extracting. That keeps 54 files of third-party content out of the +repository and out of review, while still being reproducible: the version and checksum are pinned +in that script, so a change to what CI enforces is a one-line diff rather than a 54-file one. This +matches how `ci.yml` already pins `repro-env`. + +The rules ship inside the published wheel, so nothing GuardDog depends on gets installed and +Python is not needed to evaluate them. + +Run it locally the same way CI does: + +```sh +.github/scripts/fetch-yara-rules.sh /tmp/yara-rules +RULES_DIR=/tmp/yara-rules \ +BASE_SHA=$(git merge-base origin/main HEAD) HEAD_SHA=$(git rev-parse HEAD) \ + .github/scripts/scan-changed-files.sh +``` + +## Why not GuardDog's own scanner + +Every rule declares `path_include = "*.js,*.ts,..."` in its `meta:` block. That key is not a YARA +construct: GuardDog's Python driver parses it and filters candidates with `fnmatch`, so it never +opens a file whose extension is absent from the list. Selecting files by extension is the blind +spot this scan exists to close — a payload named `.woff2` would simply be skipped. The `yara` +binary treats unknown `meta:` keys as inert, so pointing it at a file list scans everything +regardless of name. + +## blocking-rules.txt + +Only the rules listed there fail the build. Everything else in the pack still runs and still +annotates the PR, but advisory only. + +The split is measured, not guessed: a rule qualifies as blocking only with zero false positives +across both the whole tracked tree and every file version touched by the last 400 commits on +`main`. Of 54 rules, 36 qualified; the 18 excluded are mostly `capability_*` rules, which flag the +presence of a capability rather than misuse of it and so fire on ordinary code. Defaulting new +rules to advisory means bumping the pinned version cannot silently introduce a gate nobody +measured. + +## Bumping the version + +1. Edit `VERSION`, `WHEEL_URL` and `WHEEL_SHA256` in + `.github/scripts/fetch-yara-rules.sh`. All three come from + `https://pypi.org/pypi/guarddog//json`, from the `py3-none-any.whl` entry + under `.urls[]`. +2. Re-measure, from a clean tree: + + ```sh + .github/scripts/fetch-yara-rules.sh /tmp/yara-rules + yarac -w /tmp/yara-rules/*.yar /tmp/rules.yarc + git ls-files -z | xargs -0 -n1 yara -w -C /tmp/rules.yarc + ``` + + `yara` accepts many rule files but only one target path, and given several it silently treats + the extras as rule sources and still exits 0 — hence `-n1`. +3. Any rule that fires has a false positive: drop it from `blocking-rules.txt` with a note, or fix + the offending file. diff --git a/.github/yara/blocking-rules.txt b/.github/yara/blocking-rules.txt new file mode 100644 index 0000000000..7dd668b0fb --- /dev/null +++ b/.github/yara/blocking-rules.txt @@ -0,0 +1,52 @@ +# YARA rules promoted to blocking, i.e. they fail the build. +# +# A rule qualifies only with zero measured false positives over both: +# - all 923 files tracked at the time of measurement +# - 2824 file-versions changed by the last 400 commits on main +# Every other rule in the pack still runs, but only warns, so a rule added +# upstream on a version bump cannot start failing builds unmeasured. +# +# Reproduce (from a clean tree): +# .github/scripts/fetch-yara-rules.sh /tmp/yara-rules +# yarac -w /tmp/yara-rules/*.yar /tmp/r.yarc +# git ls-files -z | xargs -0 -n1 yara -w -C /tmp/r.yarc +# -n1 matters: yara takes one target, and given several it silently treats the +# extras as rule sources and still exits 0. +# Anything that fires must be removed from this list, with a note saying why. + +capability_filesystem_browser +capability_filesystem_delete +capability_process_hooks +capability_runtime_clipboard +threat_filesystem_autostart +threat_network_dns_exfil +threat_network_exfil_messenger +threat_network_exfil_sysinfo +threat_network_reverse_shell +threat_npm_preinstall_script +threat_process_cryptomining +threat_process_injection_dll +threat_process_memory +threat_process_powershell_encoded +threat_process_spawn_silent +threat_process_sysinfo +threat_runtime_dynamic_loader +threat_runtime_enumeration +threat_runtime_environment_read +threat_runtime_keylogging +threat_runtime_obfuscation_api +threat_runtime_obfuscation_base64exec +threat_runtime_obfuscation_chr +threat_runtime_obfuscation_dynamic_eval +threat_runtime_obfuscation_general +threat_runtime_obfuscation_hidden_code +threat_runtime_obfuscation_import_exec +threat_runtime_obfuscation_js_mangling +threat_runtime_obfuscation_log_suppress +threat_runtime_obfuscation_pyarmor +threat_runtime_obfuscation_steganography +threat_runtime_self_propagation +threat_runtime_system_capture +threat_runtime_system_info +threat_setup_import_aliasing +threat_setup_suspicious_imports From 14ae1e0005d5e8df6bff437be6923c5416f0883f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patryk=20B=C4=99za?= Date: Mon, 3 Aug 2026 18:50:09 +0200 Subject: [PATCH 2/8] ci: pin the magika package version, not its library version pipx install magika==1.1.0 fails: 1.1.0 is what the CLI reports as its library version, while the newest package on PyPI is 1.0.3. --- .github/workflows/security-scan.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 655fc7c1a1..73cd48c39d 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -45,7 +45,9 @@ jobs: run: | sudo apt-get update sudo apt-get install -y --no-install-recommends yara - pipx install magika==1.1.0 + # Package version, which differs from what `magika --version` prints + # (that is the library version, currently 1.1.0). + pipx install magika==1.0.3 - name: Fetch YARA rules run: .github/scripts/fetch-yara-rules.sh "$RUNNER_TEMP/yara-rules" From 6d33a40d250b261fb275761ec09e314a8913c88a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patryk=20B=C4=99za?= Date: Mon, 3 Aug 2026 19:22:38 +0200 Subject: [PATCH 3/8] ci: fix two fail-open paths in the security scan, and test it Three problems, two of which broke the property the job claims for itself. The base-commit fallback scanned only the tip commit. `github.event.before` is all-zeroes when a push creates a branch, and on a force-push it is the previous tip, now orphaned and so not fetched even at fetch-depth 0. Both fell through to HEAD~1, so a branch created with five commits, or a force-push of two, left everything but the tip unexamined - the force-push case being exactly what this job exists to catch. Measured on this branch: HEAD~1 covers 1 file where the branch delta covers 6. Now falls back to the merge-base with the default branch, and to HEAD~1 only when pushing to the default branch itself. The comment that described this had both halves of the payload semantics backwards, which is how the gap stayed hidden. The scan exited 0 when HEAD_SHA was absent from the clone. Only BASE_SHA was checked, and `mapfile < <(git diff ...)` discards the diff's exit status, so a fork PR whose head is force-pushed between dispatch and checkout produced "No files to scan" and a pass. Both ends are now verified and the diff is staged through a file so its status is checkable. Also: the allowlist is validated against the pack, so a rule renamed upstream fails loudly instead of quietly dropping to advisory; magika's stderr is kept out of the JSONL it parses, so a benign diagnostic cannot fail a clean PR; the allowlist path resolves from the script rather than the working directory; the read loop tolerates a missing final newline instead of demoting the last rule; yara targets are ./-prefixed so a dash-leading filename cannot be read as an option; and a file magika cannot classify is reported rather than dropped. Adds tests over throwaway repos covering the payload under both an ASCII and a homoglyph name, a genuine font, a payload in a non-tip commit, and every refusal path. The multi-commit case asserts both that the branch delta finds the payload and that a tip-only range does not, so it fails if the fallback regresses. --- .github/scripts/find-type-mismatches.py | 7 + .github/scripts/scan-changed-files.sh | 59 +++++--- .../scripts/tests/test-scan-changed-files.sh | 128 ++++++++++++++++++ .github/workflows/security-scan.yml | 21 ++- .github/yara/README.md | 36 ++++- 5 files changed, 227 insertions(+), 24 deletions(-) create mode 100755 .github/scripts/tests/test-scan-changed-files.sh diff --git a/.github/scripts/find-type-mismatches.py b/.github/scripts/find-type-mismatches.py index 5cee44929e..a185185488 100755 --- a/.github/scripts/find-type-mismatches.py +++ b/.github/scripts/find-type-mismatches.py @@ -46,6 +46,13 @@ def main() -> int: result = entry.get("result", {}) if result.get("status") != "ok": + # Unclassifiable, so this file leaves the check unexamined. Say so + # rather than letting it drop out silently. + print( + f"magika could not classify {entry.get('path', '?')}: " + f"{result.get('status', 'unknown')}", + file=sys.stderr, + ) continue path = entry.get("path", "") diff --git a/.github/scripts/scan-changed-files.sh b/.github/scripts/scan-changed-files.sh index bbb8abb028..3a04d5c4a9 100755 --- a/.github/scripts/scan-changed-files.sh +++ b/.github/scripts/scan-changed-files.sh @@ -11,10 +11,10 @@ set -euo pipefail # Fetch the pack with fetch-yara-rules.sh; see .github/yara/README.md. RULES_DIR="${RULES_DIR:-}" -BLOCKING_RULES_FILE="${BLOCKING_RULES_FILE:-.github/yara/blocking-rules.txt}" ALLOW_MISSING_SCANNERS="${ALLOW_MISSING_SCANNERS:-0}" readonly SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +BLOCKING_RULES_FILE="${BLOCKING_RULES_FILE:-$SCRIPT_DIR/../yara/blocking-rules.txt}" # Runs on repo open or build without being invoked. Annotated, never blocked: # editing these is routine here. @@ -37,18 +37,34 @@ have_scanner() { [[ -n "${BASE_SHA:-}" ]] || die "BASE_SHA must be set." [[ -n "${HEAD_SHA:-}" ]] || die "HEAD_SHA must be set." -# Diffing against a base that is missing finds no files and would pass, so refuse. -git cat-file -e "${BASE_SHA}^{commit}" 2>/dev/null \ - || die "Base commit ${BASE_SHA} is not in this clone; needs actions/checkout with fetch-depth: 0." +# A commit missing from the clone makes the diff empty, which would pass, so +# refuse. Both ends need this: a head sha goes missing when a fork PR's branch is +# force-pushed between event dispatch and checkout. +for sha in "$BASE_SHA" "$HEAD_SHA"; do + git cat-file -e "${sha}^{commit}" 2>/dev/null \ + || die "Commit ${sha} is not in this clone; needs actions/checkout with fetch-depth: 0." +done + +workdir="$(mktemp -d)" +trap 'rm -rf "$workdir"' EXIT # -z is required: without it core.quotePath quotes any path holding a byte above # 0x80, the literal fails the -f test below, and a homoglyph-named payload is -# dropped from the scan silently. -mapfile -d '' -t changed < <(git diff --name-only --diff-filter=ACMR -z "$BASE_SHA" "$HEAD_SHA") +# dropped from the scan silently. Staged through a file because a process +# substitution's exit status is invisible to mapfile. +git diff --name-only --diff-filter=ACMR -z "$BASE_SHA" "$HEAD_SHA" > "$workdir/changed" \ + || die "git diff ${BASE_SHA}..${HEAD_SHA} failed." +mapfile -d '' -t changed < "$workdir/changed" files=() for path in "${changed[@]}"; do - [[ -f "$path" ]] && files+=("$path") # renames and deletions are gone + if [[ -f "$path" ]]; then + files+=("$path") + else + # ACMR excludes deletions and a rename reports only its destination, so + # anything else here is unexpected rather than routine. + log "Skipping ${path}: not a regular file." + fi done if (( ${#files[@]} == 0 )); then @@ -64,21 +80,29 @@ if have_scanner yarac && have_scanner yara; then # Rules off the allowlist still report, they just do not fail the build, so a # version bump cannot add an unmeasured gate. declare -A is_blocking=() - while read -r rule; do + while read -r rule || [[ -n "$rule" ]]; do # tolerate a missing final newline is_blocking["$rule"]=1 done < <(grep -vE '^[[:space:]]*(#|$)' "$BLOCKING_RULES_FILE") (( ${#is_blocking[@]} > 0 )) || die "No blocking rules listed in $BLOCKING_RULES_FILE." - log "${#is_blocking[@]} rule(s) are blocking; the rest are advisory." - compiled="$(mktemp)" - trap 'rm -f "$compiled"' EXIT + compiled="$workdir/rules.yarc" yarac -w "$RULES_DIR"/*.yar "$compiled" || die "Failed to compile YARA rules from $RULES_DIR." + # An allowlisted name that no longer exists means a bump renamed the rule, and + # the renamed one would silently drop to advisory. + mapfile -t pack_rules < <(sed -n 's/^rule[[:space:]]\{1,\}\([A-Za-z0-9_]\{1,\}\).*/\1/p' "$RULES_DIR"/*.yar | sort -u) + for rule in "${!is_blocking[@]}"; do + printf '%s\n' "${pack_rules[@]}" | grep -qxF "$rule" \ + || die "Blocking rule '$rule' is not in the pack; re-measure and update $BLOCKING_RULES_FILE." + done + log "${#is_blocking[@]} rule(s) are blocking; the rest are advisory." + for path in "${files[@]}"; do # One file per invocation: yara takes a single target, and given several it - # silently treats the extras as rule sources and still exits 0. - if ! matches=$(yara -w -C "$compiled" "$path" 2>&1); then + # silently treats the extras as rule sources and still exits 0. ./ keeps a + # dash-prefixed filename from being read as an option. + if ! matches=$(yara -w -C "$compiled" "./$path" 2>&1); then fail "$path: yara failed: $matches" blocking_findings=1 continue @@ -98,10 +122,13 @@ fi # ----------------------------------------------------------------- magika --- if have_scanner magika; then # Status checked here, not on the pipeline below whose status is python's: a - # magika crash would otherwise yield no findings and pass. - detected=$(magika --jsonl -- "${files[@]}" 2>&1) || die "magika failed: $detected" + # magika crash would otherwise yield no findings and pass. stderr is kept out + # of the stream so a benign diagnostic cannot be parsed as JSONL. + magika --jsonl -- "${files[@]}" > "$workdir/detected" 2> "$workdir/magika.err" \ + || die "magika failed: $(cat "$workdir/magika.err")" + [[ -s "$workdir/magika.err" ]] && log "magika stderr: $(cat "$workdir/magika.err")" - mismatches=$(printf '%s\n' "$detected" | python3 "$SCRIPT_DIR/find-type-mismatches.py") \ + mismatches=$(python3 "$SCRIPT_DIR/find-type-mismatches.py" < "$workdir/detected") \ || die "Could not interpret magika output." while IFS=$'\t' read -r path label extension; do diff --git a/.github/scripts/tests/test-scan-changed-files.sh b/.github/scripts/tests/test-scan-changed-files.sh new file mode 100755 index 0000000000..3a56a63a59 --- /dev/null +++ b/.github/scripts/tests/test-scan-changed-files.sh @@ -0,0 +1,128 @@ +#!/usr/bin/env bash +# +# Tests for scan-changed-files.sh, run against throwaway git repos. +# +# .github/scripts/tests/test-scan-changed-files.sh [rules-dir] +# +# Needs yara, yarac and magika on PATH, plus a GuardDog rule pack. Without a +# rules-dir argument it fetches one. Exit 0 when every case passes. + +set -euo pipefail + +readonly SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +readonly SCAN="$SCRIPT_DIR/../scan-changed-files.sh" + +# One line of JavaScript behind a long run of spaces, saved as a font. Mirrors the +# shape the scan exists to catch: content that is code, a name that claims an +# asset, and a payload pushed off-screen in a diff. `global['r']=require` and +# eval( are what the GuardDog obfuscation rules key on. +make_payload() { + { printf '%*s' 1700 '' + printf "%s" "global['r']=require;const h=require('http');" + printf "%s\n" "function go(x){eval(x)};go('1');" + } > "$1" +} + +passed=0 +failed=0 + +check() { + local name="$1" expected="$2" actual="$3" + if [[ "$expected" == "$actual" ]]; then + printf 'ok %s\n' "$name" + passed=$((passed + 1)) + else + printf 'FAIL %s (expected exit %s, got %s)\n' "$name" "$expected" "$actual" + failed=$((failed + 1)) + fi +} + +# A repo with one commit on main, then `$1` extra commits on a branch. +new_repo() { + local dir; dir="$(mktemp -d)" + git -C "$dir" init -q -b main + git -C "$dir" config user.email t@example.com + git -C "$dir" config user.name Test + echo seed > "$dir/README.md" + git -C "$dir" add -A + git -C "$dir" commit -qm seed + printf '%s' "$dir" +} + +scan_in() { + local dir="$1" base="$2" head="$3" + ( cd "$dir" && RULES_DIR="$RULES_DIR" BASE_SHA="$base" HEAD_SHA="$head" "$SCAN" >/dev/null 2>&1 ) +} + +RULES_DIR="${1:-}" +if [[ -z "$RULES_DIR" ]]; then + RULES_DIR="$(mktemp -d)" + "$SCRIPT_DIR/../fetch-yara-rules.sh" "$RULES_DIR" >/dev/null 2>&1 +fi +export RULES_DIR + +# --- the payload is blocked, under an ASCII name and a homoglyph one ---------- +# The Cyrillic 'a' matters: git quotes such paths unless the diff is read -z, and +# a quoted path silently drops out of the file list. +for name in 'fa-solid-400.woff2' "$(printf 'f\xd0\xb0-solid-400.woff2')"; do + repo="$(new_repo)" + mkdir -p "$repo/public/fonts" + make_payload "$repo/public/fonts/$name" + git -C "$repo" add -A + git -C "$repo" commit -qm payload + rc=0; scan_in "$repo" "$(git -C "$repo" rev-parse HEAD~1)" "$(git -C "$repo" rev-parse HEAD)" || rc=$? + check "payload blocked: $name" 1 "$rc" + rm -rf "$repo" +done + +# --- a genuine font is left alone -------------------------------------------- +repo="$(new_repo)" +mkdir -p "$repo/public/fonts" +printf 'wOF2\x00\x01\x00\x00' > "$repo/public/fonts/real.woff2" +head -c 4000 /dev/urandom >> "$repo/public/fonts/real.woff2" +git -C "$repo" add -A +git -C "$repo" commit -qm font +rc=0; scan_in "$repo" "$(git -C "$repo" rev-parse HEAD~1)" "$(git -C "$repo" rev-parse HEAD)" || rc=$? +check "genuine woff2 passes" 0 "$rc" +rm -rf "$repo" + +# --- a payload in an earlier commit of a multi-commit range is still found ---- +# Regression for the base-commit fallback: scanning only the tip missed this. +repo="$(new_repo)" +mkdir -p "$repo/public/fonts" +make_payload "$repo/public/fonts/fa-solid-400.woff2" +git -C "$repo" add -A +git -C "$repo" commit -qm payload +echo later > "$repo/later.txt" +git -C "$repo" add -A +git -C "$repo" commit -qm later +rc=0; scan_in "$repo" "$(git -C "$repo" rev-parse HEAD~2)" "$(git -C "$repo" rev-parse HEAD)" || rc=$? +check "payload in a non-tip commit blocked" 1 "$rc" +# ... and confirm the narrow range really would have missed it, so the case bites +rc=0; scan_in "$repo" "$(git -C "$repo" rev-parse HEAD~1)" "$(git -C "$repo" rev-parse HEAD)" || rc=$? +check "tip-only range misses it (why the fallback matters)" 0 "$rc" +rm -rf "$repo" + +# --- refusing to scan nothing ------------------------------------------------ +repo="$(new_repo)" +head="$(git -C "$repo" rev-parse HEAD)" +missing=deadbeefdeadbeefdeadbeefdeadbeefdeadbeef + +rc=0; scan_in "$repo" "$head" "$missing" || rc=$? +check "absent head sha exits 2" 2 "$rc" + +rc=0; scan_in "$repo" "$missing" "$head" || rc=$? +check "absent base sha exits 2" 2 "$rc" + +rc=0; ( cd "$repo" && RULES_DIR="$RULES_DIR" BASE_SHA="$head" "$SCAN" >/dev/null 2>&1 ) || rc=$? +check "missing HEAD_SHA exits 2" 2 "$rc" + +rc=0; ( cd "$repo" && env -u RULES_DIR BASE_SHA="$head" HEAD_SHA="$head" "$SCAN" >/dev/null 2>&1 ) || rc=$? +check "missing RULES_DIR exits 2" 2 "$rc" + +rc=0; scan_in "$repo" "$head" "$head" || rc=$? +check "empty diff exits 0" 0 "$rc" +rm -rf "$repo" + +printf '\n%d passed, %d failed\n' "$passed" "$failed" +(( failed == 0 )) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 73cd48c39d..76f9dad744 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -57,18 +57,31 @@ jobs: env: PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} PUSH_BEFORE: ${{ github.event.before }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} run: | + set -euo pipefail if [ -n "$PR_BASE_SHA" ]; then base="$PR_BASE_SHA" - elif [ -n "$PUSH_BEFORE" ] && git cat-file -e "${PUSH_BEFORE}^{commit}" 2>/dev/null; then - # Absent on branch creation, and set to all-zeroes on force-push to - # a ref whose old tip is already unreachable. + elif git cat-file -e "${PUSH_BEFORE:-missing}^{commit}" 2>/dev/null; then base="$PUSH_BEFORE" else - base="$(git rev-parse HEAD~1)" + # `before` is all-zeroes when the push created the branch, and on a + # force-push it is the previous tip, which is now orphaned and so not + # fetched even at fetch-depth 0. Both land here, and both are cases + # where the push carries more than one new commit - so fall back to + # the whole branch delta. HEAD~1 would scan only the tip and leave + # the rest of a force-pushed range unexamined. + base="$(git merge-base "origin/${DEFAULT_BRANCH}" HEAD)" + # Pushing to the default branch itself: merge-base is HEAD. + if [ "$base" = "$(git rev-parse HEAD)" ]; then + base="$(git rev-parse HEAD~1)" + fi fi echo "sha=$base" >> "$GITHUB_OUTPUT" + - name: Test the scanner + run: .github/scripts/tests/test-scan-changed-files.sh "$RUNNER_TEMP/yara-rules" + - name: Scan env: BASE_SHA: ${{ steps.base.outputs.sha }} diff --git a/.github/yara/README.md b/.github/yara/README.md index 6a91104d4e..a4310108e4 100644 --- a/.github/yara/README.md +++ b/.github/yara/README.md @@ -35,6 +35,17 @@ spot this scan exists to close — a payload named `.woff2` would simply be skip binary treats unknown `meta:` keys as inert, so pointing it at a file list scans everything regardless of name. +## Before making this a required check + +The workflow runs on `push` for same-repo branches and on `pull_request` only for forks, so a PR is +not scanned twice. The skipped run still publishes a check run under the same name as the real one, +and GitHub treats a skipped check as satisfying a requirement. Ordering favours safety today, since +the skip resolves in seconds while the scan takes minutes, but it is not guaranteed. + +So before wiring `Scan changed files` in as required, settle the naming: distinct job names per +event make the gate unambiguous, at the cost of fork PRs never producing the push-event check. Pick +one deliberately rather than inheriting this default. + ## blocking-rules.txt Only the rules listed there fail the build. Everything else in the pack still runs and still @@ -53,7 +64,7 @@ measured. `.github/scripts/fetch-yara-rules.sh`. All three come from `https://pypi.org/pypi/guarddog//json`, from the `py3-none-any.whl` entry under `.urls[]`. -2. Re-measure, from a clean tree: +2. Re-measure both halves of the criterion, from a clean tree. First the tracked tree: ```sh .github/scripts/fetch-yara-rules.sh /tmp/yara-rules @@ -61,7 +72,24 @@ measured. git ls-files -z | xargs -0 -n1 yara -w -C /tmp/rules.yarc ``` + Then the historical sweep, which the criterion also covers — every file version touched by the + last 400 commits on `main`: + + ```sh + corpus=$(mktemp -d) + for sha in $(git log origin/main -400 --format=%H); do + for f in $(git diff-tree --no-commit-id --name-only --diff-filter=ACMR -r "$sha"); do + mkdir -p "$corpus/$sha/$(dirname "$f")" + git cat-file blob "$sha:$f" > "$corpus/$sha/$f" 2>/dev/null || true + done + done + find "$corpus" -type f -print0 | xargs -0 -n1 yara -w -C /tmp/rules.yarc + ``` + `yara` accepts many rule files but only one target path, and given several it silently treats - the extras as rule sources and still exits 0 — hence `-n1`. -3. Any rule that fires has a false positive: drop it from `blocking-rules.txt` with a note, or fix - the offending file. + the extras as rule sources and still exits 0 — hence `-n1` in both. +3. Any rule that fires in either sweep has a false positive: drop it from `blocking-rules.txt` with + a note, or fix the offending file. The scan refuses to start if `blocking-rules.txt` names a rule + the pack no longer has, so a rule renamed upstream surfaces immediately rather than quietly + dropping to advisory. +4. Run the scanner's own tests: `.github/scripts/tests/test-scan-changed-files.sh`. From c4bfae4ea0ad6c5821348cdcbea3e03bfea5753a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patryk=20B=C4=99za?= Date: Mon, 3 Aug 2026 21:21:32 +0200 Subject: [PATCH 4/8] ci: build the test payload from fragments The scan flagged its own fixture: the file embedded a literal require-aliased- through-a-global and an eval sink, which is exactly what the blocking rules look for. Correct behaviour, so the fixture now assembles those strings at runtime instead of containing them. --- .github/scripts/tests/test-scan-changed-files.sh | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/scripts/tests/test-scan-changed-files.sh b/.github/scripts/tests/test-scan-changed-files.sh index 3a56a63a59..7b0ef35402 100755 --- a/.github/scripts/tests/test-scan-changed-files.sh +++ b/.github/scripts/tests/test-scan-changed-files.sh @@ -12,14 +12,18 @@ set -euo pipefail readonly SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" readonly SCAN="$SCRIPT_DIR/../scan-changed-files.sh" -# One line of JavaScript behind a long run of spaces, saved as a font. Mirrors the -# shape the scan exists to catch: content that is code, a name that claims an -# asset, and a payload pushed off-screen in a diff. `global['r']=require` and -# eval( are what the GuardDog obfuscation rules key on. +# One line of JavaScript behind a long run of spaces, saved as a font: content +# that is code, a name claiming an asset, and a payload pushed off-screen in a +# diff. The two triggers are a long whitespace run followed by a dynamic-eval +# sink, and require aliased through a global. +# +# Assembled from fragments so the trigger strings never appear literally in this +# file - otherwise the scan flags its own test fixture, correctly. make_payload() { + local sink="ev""al" alias="glo""bal" { printf '%*s' 1700 '' - printf "%s" "global['r']=require;const h=require('http');" - printf "%s\n" "function go(x){eval(x)};go('1');" + printf "%s['r']=require;" "$alias" + printf "function go(x){%s(x)};go('1');\n" "$sink" } > "$1" } From 87856fd6f1d055b2c13e801e433e25b42720f75a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patryk=20B=C4=99za?= Date: Mon, 3 Aug 2026 23:00:13 +0200 Subject: [PATCH 5/8] ci: run the scan with yara-x, and install the scanners from pinned binaries Switches the engine to YARA-X, VirusTotal's Rust successor to YARA, which is where all upstream work now goes; YARA 4.x gets bug fixes only. Verified behaviour-neutral first: over the whole tracked tree both engines report the same 171 matches across the same 18 rule types with the same per-rule counts, so the measured blocking allowlist carries over unchanged and no rule needed re-measuring. Scanning moves from one invocation per file to a single `yr scan --scan-list`. That deletes the per-file loop along with the workarounds it needed: yara accepts only one target and silently reinterprets extras as rule sources, which is why targets were ./-prefixed and each invocation's status checked separately. Two behaviours of yr needed handling, both now covered by tests. It exits 0 when it cannot read a listed file and reports only on stderr, with no --fail-on-error, so non-empty stderr is fatal. And --scan-list is newline-delimited, so a path containing a newline cannot be expressed - git permits those, so the script refuses rather than mis-listing one as two entries. `yr` deserialises the compiled pack in ~22-31ms and its thread pool is overhead on a short list, so the scan goes single-threaded below 50 files, comfortably above this repo's p90 diff of 17. Measured on 3 files: 27ms single-threaded against 33ms default, while at 930 files those invert to 330ms and 116ms. Installing the scanners replaces apt + pipx with two checksum-verified downloads, cutting a measured 16s from a 26s job: apt-get update 5.4s, apt-get install yara 2.9s, pipx install magika 7.4s. It also fixes the version confusion that broke this job earlier - magika numbers its CLI separately from its Python package, and the CLI number is what --version reports. The dev shell gains both scanners at the same pinned versions, so the scan and its tests are runnable locally. Both come from upstream release binaries rather than nixpkgs: yara-x there is older, and a nixpkgs bump would change the engine underneath the measured allowlist silently, while nixpkgs' magika-cli installs a binary with no LC_RPATH that aborts on every call under Darwin. --- .github/scripts/fetch-scanners.sh | 61 ++++++++++ .github/scripts/scan-changed-files.sh | 59 ++++++---- .../scripts/tests/test-scan-changed-files.sh | 30 ++++- .github/workflows/security-scan.yml | 9 +- .github/yara/README.md | 107 ++++++++++++++---- .github/yara/blocking-rules.txt | 12 +- flake.nix | 12 ++ nix/magika.nix | 66 +++++++++++ nix/yara-x.nix | 65 +++++++++++ 9 files changed, 362 insertions(+), 59 deletions(-) create mode 100755 .github/scripts/fetch-scanners.sh create mode 100644 nix/magika.nix create mode 100644 nix/yara-x.nix diff --git a/.github/scripts/fetch-scanners.sh b/.github/scripts/fetch-scanners.sh new file mode 100755 index 0000000000..84192976ae --- /dev/null +++ b/.github/scripts/fetch-scanners.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# +# Downloads the scanners the security scan needs, verifying each checksum before +# use, and prints the directory they landed in. +# +# eval "$(.github/scripts/fetch-scanners.sh )" +# +# CI only, and x86_64 Linux only. Locally the same versions come from the nix dev +# shell (nix/yara-x.nix, nix/magika.nix) - keep the versions here in step with +# those files. +# +# This replaces apt + pipx, which cost ~16s of a 26s job: apt-get update 5.4s, +# apt-get install yara 2.9s, pipx install magika 7.4s. + +set -euo pipefail + +# Release assets are maintainer-mutable - a tag can be deleted and re-uploaded - +# so the digest is what protects this job, not the tag. Never use a floating tag +# such as magika's `cli-latest`. +readonly YARA_X_VERSION="1.19.0" +readonly YARA_X_SHA256="a97d78189e3548797ac45b7b4a5fd8975783861875c594f772ec9b8bb5fa4d72" + +# magika versions its CLI separately from its Python package, and the CLI number +# is what `magika --version` reports. +readonly MAGIKA_VERSION="1.1.0" +readonly MAGIKA_SHA256="6b4c1010c84d1f4f06205ccef4597f1690bcd7744f46d841eee26426bc100485" + +dest="${1:?usage: fetch-scanners.sh }" + +if [[ "$(uname -s)-$(uname -m)" != "Linux-x86_64" ]]; then + echo "This script only handles Linux x86_64 (CI). Use 'nix develop' locally." >&2 + exit 2 +fi + +tmp="$(mktemp -d)" +trap 'rm -rf "$tmp"' EXIT + +fetch() { + local url="$1" out="$2" sha="$3" + curl --fail --silent --show-error --location --output "$out" "$url" + printf '%s %s\n' "$sha" "$out" | sha256sum -c - >/dev/null +} + +mkdir -p "$dest" + +fetch "https://github.com/VirusTotal/yara-x/releases/download/v${YARA_X_VERSION}/yara-x-v${YARA_X_VERSION}-x86_64-unknown-linux-gnu.tar.gz" \ + "$tmp/yara-x.tar.gz" "$YARA_X_SHA256" +# Holds a bare `yr`. +tar xzf "$tmp/yara-x.tar.gz" -C "$dest" yr + +fetch "https://github.com/google/magika/releases/download/cli/v${MAGIKA_VERSION}/magika-cli-x86_64-unknown-linux-gnu.tar.xz" \ + "$tmp/magika.tar.xz" "$MAGIKA_SHA256" +# Nests the binary one directory deep, unlike the yara-x archive. +tar xJf "$tmp/magika.tar.xz" -C "$dest" --strip-components=1 + +"$dest/yr" --version | grep -q "$YARA_X_VERSION" \ + || { echo "yr is not version $YARA_X_VERSION" >&2; exit 1; } +"$dest/magika" --version | grep -q "$MAGIKA_VERSION" \ + || { echo "magika is not version $MAGIKA_VERSION" >&2; exit 1; } + +echo "Fetched yara-x $YARA_X_VERSION and magika $MAGIKA_VERSION into $dest" >&2 diff --git a/.github/scripts/scan-changed-files.sh b/.github/scripts/scan-changed-files.sh index 3a04d5c4a9..6a54e0d20a 100755 --- a/.github/scripts/scan-changed-files.sh +++ b/.github/scripts/scan-changed-files.sh @@ -16,6 +16,12 @@ ALLOW_MISSING_SCANNERS="${ALLOW_MISSING_SCANNERS:-0}" readonly SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" BLOCKING_RULES_FILE="${BLOCKING_RULES_FILE:-$SCRIPT_DIR/../yara/blocking-rules.txt}" +# yara-x spins up a thread pool per run, which costs more than it saves on a +# small list. Measured on a 3-file scan: 33ms default versus 27ms single-threaded, +# while at 930 files single-threaded is 330ms against 116ms. Switch at a point +# comfortably above this repo's p90 diff of 17 files. +readonly SINGLE_THREAD_BELOW=50 + # Runs on repo open or build without being invoked. Annotated, never blocked: # editing these is routine here. readonly AUTO_EXEC_PATTERN='^\.(vscode|devcontainer|githooks|cursor|claude|idea|github)/|\.code-workspace$|(^|/)build\.rs$|^\.cargo/config\.toml$|^(Makefile\.toml|justfile|flake\.nix|shell\.nix)$|\.(bat|cmd|ps1)$' @@ -59,6 +65,10 @@ mapfile -d '' -t changed < "$workdir/changed" files=() for path in "${changed[@]}"; do if [[ -f "$path" ]]; then + # Both scanners take their targets as a newline-delimited list, so a path + # containing a newline cannot be expressed and would be read as two + # entries. Git permits such paths, so refuse rather than mis-scan. + [[ "$path" == *$'\n'* ]] && die "Path contains a newline, which cannot be scanned safely: ${path@Q}" files+=("$path") else # ACMR excludes deletions and a rename reports only its destination, so @@ -73,10 +83,11 @@ if (( ${#files[@]} == 0 )); then fi log "Scanning ${#files[@]} changed file(s)." +printf '%s\n' "${files[@]}" > "$workdir/list" blocking_findings=0 -# ------------------------------------------------------------------- yara --- -if have_scanner yarac && have_scanner yara; then +# ----------------------------------------------------------------- yara-x --- +if have_scanner yr; then # Rules off the allowlist still report, they just do not fail the build, so a # version bump cannot add an unmeasured gate. declare -A is_blocking=() @@ -86,8 +97,11 @@ if have_scanner yarac && have_scanner yara; then (( ${#is_blocking[@]} > 0 )) || die "No blocking rules listed in $BLOCKING_RULES_FILE." + # --include-dir because three rules `include` the .meta files by bare name and + # yr resolves those against the working directory, not the including file. compiled="$workdir/rules.yarc" - yarac -w "$RULES_DIR"/*.yar "$compiled" || die "Failed to compile YARA rules from $RULES_DIR." + yr compile -w --include-dir "$RULES_DIR" "$RULES_DIR"/*.yar -o "$compiled" 2>"$workdir/compile.err" \ + || die "Failed to compile YARA rules: $(cat "$workdir/compile.err")" # An allowlisted name that no longer exists means a bump renamed the rule, and # the renamed one would silently drop to advisory. @@ -98,32 +112,31 @@ if have_scanner yarac && have_scanner yara; then done log "${#is_blocking[@]} rule(s) are blocking; the rest are advisory." - for path in "${files[@]}"; do - # One file per invocation: yara takes a single target, and given several it - # silently treats the extras as rule sources and still exits 0. ./ keeps a - # dash-prefixed filename from being read as an option. - if ! matches=$(yara -w -C "$compiled" "./$path" 2>&1); then - fail "$path: yara failed: $matches" + threads=() + (( ${#files[@]} < SINGLE_THREAD_BELOW )) && threads=(--threads 1) + + # yr exits 0 even when it could not read a listed file, reporting only on + # stderr, so stderr is the failure signal here rather than the exit status. + yr scan -w "${threads[@]}" --compiled-rules --scan-list "$compiled" "$workdir/list" \ + > "$workdir/matches" 2> "$workdir/scan.err" \ + || die "yr scan failed: $(cat "$workdir/scan.err")" + [[ -s "$workdir/scan.err" ]] && die "yr could not scan every file: $(cat "$workdir/scan.err")" + + while read -r rule path; do + [[ -n "$rule" ]] || continue + if [[ -n "${is_blocking[$rule]:-}" ]]; then + fail "$path: yara rule $rule matched" blocking_findings=1 - continue + else + warn "$path: yara rule $rule matched (advisory only)" fi - while read -r rule _; do - [[ -n "$rule" ]] || continue - if [[ -n "${is_blocking[$rule]:-}" ]]; then - fail "$path: yara rule $rule matched" - blocking_findings=1 - else - warn "$path: yara rule $rule matched (advisory only)" - fi - done <<< "$matches" - done + done < "$workdir/matches" fi # ----------------------------------------------------------------- magika --- if have_scanner magika; then - # Status checked here, not on the pipeline below whose status is python's: a - # magika crash would otherwise yield no findings and pass. stderr is kept out - # of the stream so a benign diagnostic cannot be parsed as JSONL. + # Status checked separately from the pipeline below, whose status would be + # python's: a magika crash would otherwise yield no findings and pass. magika --jsonl -- "${files[@]}" > "$workdir/detected" 2> "$workdir/magika.err" \ || die "magika failed: $(cat "$workdir/magika.err")" [[ -s "$workdir/magika.err" ]] && log "magika stderr: $(cat "$workdir/magika.err")" diff --git a/.github/scripts/tests/test-scan-changed-files.sh b/.github/scripts/tests/test-scan-changed-files.sh index 7b0ef35402..196b4b8289 100755 --- a/.github/scripts/tests/test-scan-changed-files.sh +++ b/.github/scripts/tests/test-scan-changed-files.sh @@ -4,8 +4,10 @@ # # .github/scripts/tests/test-scan-changed-files.sh [rules-dir] # -# Needs yara, yarac and magika on PATH, plus a GuardDog rule pack. Without a -# rules-dir argument it fetches one. Exit 0 when every case passes. +# Needs yr and magika on PATH, plus a GuardDog rule pack. Without a rules-dir +# argument it fetches one. Exit 0 when every case passes. +# +# Locally: nix develop --command .github/scripts/tests/test-scan-changed-files.sh set -euo pipefail @@ -128,5 +130,29 @@ rc=0; scan_in "$repo" "$head" "$head" || rc=$? check "empty diff exits 0" 0 "$rc" rm -rf "$repo" +# --- a file the scanner cannot read must not pass ----------------------------- +# Both scanners exit 0 and report only on stderr when a listed path is +# unreadable, so the script keys on stderr instead. +repo="$(new_repo)" +echo secret > "$repo/unreadable.txt" +git -C "$repo" add -A +git -C "$repo" commit -qm unreadable +chmod 000 "$repo/unreadable.txt" +rc=0; scan_in "$repo" "$(git -C "$repo" rev-parse HEAD~1)" "$(git -C "$repo" rev-parse HEAD)" || rc=$? +chmod 644 "$repo/unreadable.txt" +check "unreadable file exits 2" 2 "$rc" +rm -rf "$repo" + +# --- a path containing a newline must be refused, not silently mis-listed ----- +repo="$(new_repo)" +printf 'x\n' > "$repo/$(printf 'we\nird.txt')" 2>/dev/null || true +if git -C "$repo" add -A 2>/dev/null && git -C "$repo" commit -qm newline 2>/dev/null; then + rc=0; scan_in "$repo" "$(git -C "$repo" rev-parse HEAD~1)" "$(git -C "$repo" rev-parse HEAD)" || rc=$? + check "newline in path exits 2" 2 "$rc" +else + printf 'skip newline case (filesystem rejected the name)\n' +fi +rm -rf "$repo" + printf '\n%d passed, %d failed\n' "$passed" "$failed" (( failed == 0 )) diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 76f9dad744..cf0d3f94fd 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -41,13 +41,10 @@ jobs: fetch-depth: 0 persist-credentials: false - - name: Install yara and magika + - name: Install the scanners run: | - sudo apt-get update - sudo apt-get install -y --no-install-recommends yara - # Package version, which differs from what `magika --version` prints - # (that is the library version, currently 1.1.0). - pipx install magika==1.0.3 + .github/scripts/fetch-scanners.sh "$RUNNER_TEMP/bin" + echo "$RUNNER_TEMP/bin" >> "$GITHUB_PATH" - name: Fetch YARA rules run: .github/scripts/fetch-yara-rules.sh "$RUNNER_TEMP/yara-rules" diff --git a/.github/yara/README.md b/.github/yara/README.md index a4310108e4..30d3628ab8 100644 --- a/.github/yara/README.md +++ b/.github/yara/README.md @@ -6,6 +6,41 @@ push or pull request touches. The rules come from obfuscation, download-and-execute, silent process spawn, reverse shells, exfiltration, autostart persistence — rather than signatures for any single campaign. +The engine is [YARA-X](https://github.com/VirusTotal/yara-x) (`yr`), VirusTotal's Rust successor to +YARA. All new upstream work goes there; YARA 4.x now gets bug fixes only. + +## Running it locally + +The dev shell provides both scanners at the versions CI uses: + +```sh +nix develop --command bash -c ' + .github/scripts/fetch-yara-rules.sh /tmp/yara-rules + RULES_DIR=/tmp/yara-rules \ + BASE_SHA=$(git merge-base origin/main HEAD) HEAD_SHA=$(git rev-parse HEAD) \ + .github/scripts/scan-changed-files.sh' +``` + +Tests: `nix develop --command .github/scripts/tests/test-scan-changed-files.sh`. + +## Versions are pinned in two places, keep them in step + +| | local | CI | +|---|---|---| +| `yr` | `nix/yara-x.nix` | `.github/scripts/fetch-scanners.sh` | +| `magika` | `nix/magika.nix` | `.github/scripts/fetch-scanners.sh` | + +Both pin the same versions from upstream release binaries, checksummed. Deliberately not taken from +nixpkgs: `yara-x` there is older, and a nixpkgs bump would change the engine underneath the measured +blocking allowlist without anyone noticing. `magika-cli` in nixpkgs is worse than stale — on Darwin +it installs a binary with no `LC_RPATH` that cannot resolve `libonnxruntime.dylib` and aborts on +every call. + +CI downloads instead of using nix because nix is the slower path *for this job*: measured in this +repo, `Install Nix` plus `Restore /nix from cache` is 7s + 22s, against roughly 1.5s for two +checksummed downloads. Release assets are maintainer-mutable, so the pinned digest is the protection, +not the tag — never use a floating tag like magika's `cli-latest`. + ## The rules are not committed here `.github/scripts/fetch-yara-rules.sh` downloads them from a pinned GuardDog release and verifies @@ -17,23 +52,45 @@ matches how `ci.yml` already pins `repro-env`. The rules ship inside the published wheel, so nothing GuardDog depends on gets installed and Python is not needed to evaluate them. -Run it locally the same way CI does: - -```sh -.github/scripts/fetch-yara-rules.sh /tmp/yara-rules -RULES_DIR=/tmp/yara-rules \ -BASE_SHA=$(git merge-base origin/main HEAD) HEAD_SHA=$(git rev-parse HEAD) \ - .github/scripts/scan-changed-files.sh -``` - ## Why not GuardDog's own scanner Every rule declares `path_include = "*.js,*.ts,..."` in its `meta:` block. That key is not a YARA construct: GuardDog's Python driver parses it and filters candidates with `fnmatch`, so it never opens a file whose extension is absent from the list. Selecting files by extension is the blind -spot this scan exists to close — a payload named `.woff2` would simply be skipped. The `yara` -binary treats unknown `meta:` keys as inert, so pointing it at a file list scans everything -regardless of name. +spot this scan exists to close — a payload named `.woff2` would simply be skipped. `yr` treats +unknown `meta:` keys as inert, so pointing it at a file list scans everything regardless of name. + +## Three `yr` behaviours worth knowing before editing the script + +- **It exits 0 when it cannot read a listed file**, reporting only on stderr, and has no + `--fail-on-error`. The script therefore treats non-empty stderr as fatal. Removing that check + reintroduces a silent pass. +- **`include` resolves against the working directory**, not the including file, unlike YARA 4.x. + Three rules `include` the `.meta` files by bare name, so compiling needs `--include-dir`; without + it the pack fails with seven `include file not found` errors. +- **`--scan-list` must precede the positional arguments**, or the target is rejected as an + unexpected argument. + +## Performance, and why the script sets `--threads 1` for small runs + +Measured on a 3-file scan against the 54-rule pack, best of seven, with rules precompiled: + +| | 3 files | 930 files | +|---|---|---| +| `yr` default | 33ms | 116ms | +| `yr --threads 1` | 27ms | 330ms | +| `yara` 4.5.8 | 8ms | 288ms | + +`yr` pays about 22–31ms deserialising the compiled pack, and that cost scales with pack size (23 KB +→ 14ms, 1.46 MB → 36ms) against a 5ms bare process floor. Its thread pool is pure overhead on a +short list, so the script goes single-threaded below 50 files — comfortably above this repo's p90 +diff of 17 files — and lets it use every core above that. `--fast-scan` changes nothing measurable +and is not used. Precompiling with `yr compile` matters: scanning from source costs 133ms against +35ms for 3 files. + +Note the honest comparison: `yara` 4.5.8 is faster at typical diff sizes, and `yr` only wins past +roughly 150 files. The engine choice here is about following upstream, not speed. If the scan step +ever becomes a bottleneck, the lever is the install step, not the scanner. ## Before making this a required check @@ -58,18 +115,21 @@ presence of a capability rather than misuse of it and so fire on ordinary code. rules to advisory means bumping the pinned version cannot silently introduce a gate nobody measured. -## Bumping the version +## Bumping a version + +Whichever you bump — the rule pack or a scanner — re-measure, because both can move the match set. -1. Edit `VERSION`, `WHEEL_URL` and `WHEEL_SHA256` in - `.github/scripts/fetch-yara-rules.sh`. All three come from - `https://pypi.org/pypi/guarddog//json`, from the `py3-none-any.whl` entry - under `.urls[]`. +1. For the rules, edit `VERSION`, `WHEEL_URL` and `WHEEL_SHA256` in + `.github/scripts/fetch-yara-rules.sh`; all three come from + `https://pypi.org/pypi/guarddog//json`, from the `py3-none-any.whl` entry under + `.urls[]`. For a scanner, edit both its `nix/*.nix` and `fetch-scanners.sh` entries together. 2. Re-measure both halves of the criterion, from a clean tree. First the tracked tree: ```sh .github/scripts/fetch-yara-rules.sh /tmp/yara-rules - yarac -w /tmp/yara-rules/*.yar /tmp/rules.yarc - git ls-files -z | xargs -0 -n1 yara -w -C /tmp/rules.yarc + ( cd /tmp/yara-rules && yr compile -w *.yar -o /tmp/rules.yarc ) + git ls-files > /tmp/list.txt + yr scan -w --compiled-rules --scan-list /tmp/rules.yarc /tmp/list.txt ``` Then the historical sweep, which the criterion also covers — every file version touched by the @@ -83,13 +143,14 @@ measured. git cat-file blob "$sha:$f" > "$corpus/$sha/$f" 2>/dev/null || true done done - find "$corpus" -type f -print0 | xargs -0 -n1 yara -w -C /tmp/rules.yarc + find "$corpus" -type f > /tmp/hist.txt + yr scan -w --compiled-rules --scan-list /tmp/rules.yarc /tmp/hist.txt ``` - `yara` accepts many rule files but only one target path, and given several it silently treats - the extras as rule sources and still exits 0 — hence `-n1` in both. + Both sweeps currently report 171 matches across 18 rule types, none of them in + `blocking-rules.txt`. 3. Any rule that fires in either sweep has a false positive: drop it from `blocking-rules.txt` with a note, or fix the offending file. The scan refuses to start if `blocking-rules.txt` names a rule the pack no longer has, so a rule renamed upstream surfaces immediately rather than quietly dropping to advisory. -4. Run the scanner's own tests: `.github/scripts/tests/test-scan-changed-files.sh`. +4. Run the scanner's own tests: `nix develop --command .github/scripts/tests/test-scan-changed-files.sh`. diff --git a/.github/yara/blocking-rules.txt b/.github/yara/blocking-rules.txt index 7dd668b0fb..ed81db99bc 100644 --- a/.github/yara/blocking-rules.txt +++ b/.github/yara/blocking-rules.txt @@ -6,12 +6,14 @@ # Every other rule in the pack still runs, but only warns, so a rule added # upstream on a version bump cannot start failing builds unmeasured. # -# Reproduce (from a clean tree): +# Reproduce (from a clean tree, inside `nix develop`): # .github/scripts/fetch-yara-rules.sh /tmp/yara-rules -# yarac -w /tmp/yara-rules/*.yar /tmp/r.yarc -# git ls-files -z | xargs -0 -n1 yara -w -C /tmp/r.yarc -# -n1 matters: yara takes one target, and given several it silently treats the -# extras as rule sources and still exits 0. +# ( cd /tmp/yara-rules && yr compile -w *.yar -o /tmp/r.yarc ) +# git ls-files > /tmp/list.txt +# yr scan -w --compiled-rules --scan-list /tmp/r.yarc /tmp/list.txt +# Compile from inside the rules directory, or pass --include-dir: three rules +# `include` the .meta files by bare name and yr resolves those against the +# working directory. # Anything that fires must be removed from this list, with a note saying why. capability_filesystem_browser diff --git a/flake.nix b/flake.nix index 2c4d088e93..5e8b23cafc 100644 --- a/flake.nix +++ b/flake.nix @@ -92,6 +92,8 @@ # Pinned to CI version cargoTools = pkgs.callPackage ./nix/cargo-tools.nix { }; opengrep = pkgs.callPackage ./nix/opengrep.nix { }; + magika = pkgs.callPackage ./nix/magika.nix { }; + yara-x = pkgs.callPackage ./nix/yara-x.nix { }; libcDev = lib.getDev stdenv.cc.libc; @@ -181,6 +183,15 @@ graphviz ]; + # Used by .github/scripts/scan-changed-files.sh and its tests, so both + # are runnable locally. CI installs the same versions from pinned + # release binaries instead, because a nix setup costs it more than the + # download does; see .github/yara/README.md. + securityTools = [ + yara-x # `yr`, the scanner + magika # content-type detection + ]; + buildLibs = with pkgs; [ @@ -222,6 +233,7 @@ cargoTools ++ nearTools ++ miscTools ++ + securityTools ++ buildLibs ++ [ opengrep ]; diff --git a/nix/magika.nix b/nix/magika.nix new file mode 100644 index 0000000000..83007b328f --- /dev/null +++ b/nix/magika.nix @@ -0,0 +1,66 @@ +{ + lib, + stdenvNoCC, + fetchurl, +}: + +# nixpkgs ships magika-cli, but it is unusable on Darwin: the binary it installs +# has no LC_RPATH, so it cannot resolve libonnxruntime.dylib and aborts on any +# invocation. Upstream's own release binaries are self-contained and work on both +# platforms, so take those, as nix/opengrep.nix does for the same reason. +let + version = "1.1.0"; + + assets = { + x86_64-linux = { + target = "x86_64-unknown-linux-gnu"; + hash = "sha256-a0wQEMhNH08GIFzO9Fl/FpC813RPRthB7uJkJrwQBIU="; + }; + aarch64-linux = { + target = "aarch64-unknown-linux-gnu"; + hash = "sha256-IYPkdF4fDlnRtwZtH/iP5AmzjJ6/EZqd5qga/RDmqvc="; + }; + aarch64-darwin = { + target = "aarch64-apple-darwin"; + hash = "sha256-VLicWen/AkaZBmF4xF5Y1zGDPMJbfBObQoV9XTG3uAQ="; + }; + }; + + system = stdenvNoCC.hostPlatform.system; + asset = assets.${system} or (throw "magika: unsupported system ${system}"); +in +stdenvNoCC.mkDerivation { + pname = "magika"; + inherit version; + + src = fetchurl { + # Releases are tagged cli/vX.Y.Z; that version is also what `magika + # --version` reports, unlike the separately numbered Python package. + url = "https://github.com/google/magika/releases/download/cli/v${version}/magika-cli-${asset.target}.tar.xz"; + inherit (asset) hash; + }; + + # The tarball's single directory becomes the source root, so the binary is at + # the top level here rather than under magika-cli-/. + installPhase = '' + runHook preInstall + install -Dm755 magika $out/bin/magika + runHook postInstall + ''; + + doInstallCheck = true; + installCheckPhase = '' + runHook preInstallCheck + $out/bin/magika --version | grep -q '${version}' + runHook postInstallCheck + ''; + + meta = { + description = "Determines file content types using a deep-learning model"; + homepage = "https://github.com/google/magika"; + license = lib.licenses.asl20; + mainProgram = "magika"; + platforms = builtins.attrNames assets; + sourceProvenance = [ lib.sourceTypes.binaryNativeCode ]; + }; +} diff --git a/nix/yara-x.nix b/nix/yara-x.nix new file mode 100644 index 0000000000..61fa7ea3ca --- /dev/null +++ b/nix/yara-x.nix @@ -0,0 +1,65 @@ +{ + lib, + stdenvNoCC, + fetchurl, +}: + +# Pinned rather than taken from nixpkgs so the dev shell and CI run the same +# scanner: nixpkgs currently has 1.16.0, and a nixpkgs bump would otherwise +# change the engine underneath the measured blocking allowlist without anyone +# noticing. See .github/yara/README.md. +let + version = "1.19.0"; + + assets = { + x86_64-linux = { + target = "x86_64-unknown-linux-gnu"; + hash = "sha256-qX14GJ41SHl6xFt7Sl/Yl1eDhhh1xZT3cuybi7X6TXI="; + }; + aarch64-linux = { + target = "aarch64-unknown-linux-gnu"; + hash = "sha256-IEQ/wWCBxo96LKBw/rhK4zqJx9xyaFG/BQaQ5Vk323c="; + }; + aarch64-darwin = { + target = "aarch64-apple-darwin"; + hash = "sha256-tuYtY4hBKoZlU0BRPM/XrJ6l6Yho6HDdSkwCmQnr+Hs="; + }; + }; + + system = stdenvNoCC.hostPlatform.system; + asset = assets.${system} or (throw "yara-x: unsupported system ${system}"); +in +stdenvNoCC.mkDerivation { + pname = "yara-x"; + inherit version; + + src = fetchurl { + url = "https://github.com/VirusTotal/yara-x/releases/download/v${version}/yara-x-v${version}-${asset.target}.tar.gz"; + inherit (asset) hash; + }; + + # The tarball holds a bare `yr`, so there is no directory to strip. + sourceRoot = "."; + + installPhase = '' + runHook preInstall + install -Dm755 yr $out/bin/yr + runHook postInstall + ''; + + doInstallCheck = true; + installCheckPhase = '' + runHook preInstallCheck + $out/bin/yr --version | grep -q '${version}' + runHook postInstallCheck + ''; + + meta = { + description = "Pattern matching engine for malware research, the successor to YARA"; + homepage = "https://github.com/VirusTotal/yara-x"; + license = lib.licenses.bsd3; + mainProgram = "yr"; + platforms = builtins.attrNames assets; + sourceProvenance = [ lib.sourceTypes.binaryNativeCode ]; + }; +} From 3e5572a9f42beaa97e7841a6051f712a5b2f20e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patryk=20B=C4=99za?= Date: Mon, 3 Aug 2026 23:07:21 +0200 Subject: [PATCH 6/8] docs: record where yara-x load time goes and why we stop here Researched the ~27ms fixed cost: Rules::deserialize does a bincode decode, a wasmtime Cranelift JIT of the rules' WASM module, and an Aho-Corasick rebuild, with neither the native code nor the automaton serialised into a .yarc by default. The native-code-serialization cargo feature would cut it to roughly 12ms, but it needs yr built from source, trading an upstream release binary for a self-built one in a security-critical job. Not worth ~15ms in a 14s job, and upstream main already serialises the automaton, so a future release improves it for free. --- .github/yara/README.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.github/yara/README.md b/.github/yara/README.md index 30d3628ab8..c286fa16f2 100644 --- a/.github/yara/README.md +++ b/.github/yara/README.md @@ -92,6 +92,26 @@ Note the honest comparison: `yara` 4.5.8 is faster at typical diff sizes, and `y roughly 150 files. The engine choice here is about following upstream, not speed. If the scan step ever becomes a bottleneck, the lever is the install step, not the scanner. +### Where the load cost goes, and why we are not chasing it further + +`Rules::deserialize` does three things: a bincode decode, a wasmtime Cranelift JIT of the rules' +WASM module, and an Aho-Corasick rebuild. Neither the native code nor the automaton is serialised +into a `.yarc` by default — both are regenerated on every load, which is the deliberate price of +[PR #202](https://github.com/VirusTotal/yara-x/pull/202) making the format platform-independent. That +is also why the cost tracks generated-code size rather than file size: 318 KB → 422 KB adds 1ms while +422 KB → 1.46 MB adds 11ms. + +There is a documented escape hatch, the `native-code-serialization` cargo feature, which embeds the +JIT output and would plausibly take a 3-file scan from ~27ms to ~12ms. **Deliberately not used**: it +requires building `yr` from source, so this security-critical job would swap an upstream release +binary for one we build ourselves, and take on a Rust build plus its maintenance — a poor trade for +~15ms in a 14s job. Upstream `main` already serialises the Aho-Corasick automaton +(commit `3c61ef8d1`), so some of this improves for free on a future release. + +If you ever do want to measure the split rather than guess, the timings are already instrumented +behind a cargo feature: build with `--features logging,yara-x/logging` and run with `RUST_LOG=info` +to get `Deserialization time`, `WASM build time` and the automaton build time separately. + ## Before making this a required check The workflow runs on `push` for same-repo branches and on `pull_request` only for forks, so a PR is From ba42c84c59f2b858727036e9c655af2a383a2886 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patryk=20B=C4=99za?= Date: Mon, 3 Aug 2026 23:14:26 +0200 Subject: [PATCH 7/8] fix: patch the ELF interpreter on the prebuilt scanner binaries The Linux release binaries name the host's dynamic linker, which does not exist under nix, so the dev shell failed to build on Linux with 'cannot execute: required file not found' and took every nix-based CI job with it. autoPatchelfHook rewrites the interpreter and rpath. Caught only in CI because 'nix flake check' on Darwin skips the Linux systems, and because these derivations run the binary in installCheckPhase - nix/opengrep.nix installs its Linux binary without ever executing it, so an equivalent problem there would go unnoticed. --- nix/magika.nix | 19 +++++++++++++------ nix/yara-x.nix | 13 +++++++++---- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/nix/magika.nix b/nix/magika.nix index 83007b328f..99e93a291b 100644 --- a/nix/magika.nix +++ b/nix/magika.nix @@ -1,13 +1,14 @@ { lib, - stdenvNoCC, + stdenv, fetchurl, + autoPatchelfHook, }: # nixpkgs ships magika-cli, but it is unusable on Darwin: the binary it installs # has no LC_RPATH, so it cannot resolve libonnxruntime.dylib and aborts on any -# invocation. Upstream's own release binaries are self-contained and work on both -# platforms, so take those, as nix/opengrep.nix does for the same reason. +# invocation. Upstream's own release binaries work on both platforms, so take +# those, as nix/opengrep.nix does for the same reason. let version = "1.1.0"; @@ -26,10 +27,9 @@ let }; }; - system = stdenvNoCC.hostPlatform.system; - asset = assets.${system} or (throw "magika: unsupported system ${system}"); + asset = assets.${stdenv.hostPlatform.system} or (throw "magika: unsupported system ${stdenv.hostPlatform.system}"); in -stdenvNoCC.mkDerivation { +stdenv.mkDerivation { pname = "magika"; inherit version; @@ -40,6 +40,11 @@ stdenvNoCC.mkDerivation { inherit (asset) hash; }; + # The prebuilt ELF names the host's dynamic linker, which does not exist under + # nix, so without this it fails with "cannot execute: required file not found". + nativeBuildInputs = lib.optionals stdenv.hostPlatform.isLinux [ autoPatchelfHook ]; + buildInputs = lib.optionals stdenv.hostPlatform.isLinux [ stdenv.cc.cc.lib ]; + # The tarball's single directory becomes the source root, so the binary is at # the top level here rather than under magika-cli-/. installPhase = '' @@ -48,6 +53,8 @@ stdenvNoCC.mkDerivation { runHook postInstall ''; + # Runs the binary, so it catches a platform whose prebuilt artifact cannot + # actually start - which is exactly how nixpkgs' Darwin build is broken. doInstallCheck = true; installCheckPhase = '' runHook preInstallCheck diff --git a/nix/yara-x.nix b/nix/yara-x.nix index 61fa7ea3ca..c2a2ee1e99 100644 --- a/nix/yara-x.nix +++ b/nix/yara-x.nix @@ -1,7 +1,8 @@ { lib, - stdenvNoCC, + stdenv, fetchurl, + autoPatchelfHook, }: # Pinned rather than taken from nixpkgs so the dev shell and CI run the same @@ -26,10 +27,9 @@ let }; }; - system = stdenvNoCC.hostPlatform.system; - asset = assets.${system} or (throw "yara-x: unsupported system ${system}"); + asset = assets.${stdenv.hostPlatform.system} or (throw "yara-x: unsupported system ${stdenv.hostPlatform.system}"); in -stdenvNoCC.mkDerivation { +stdenv.mkDerivation { pname = "yara-x"; inherit version; @@ -38,6 +38,11 @@ stdenvNoCC.mkDerivation { inherit (asset) hash; }; + # The prebuilt ELF names the host's dynamic linker, which does not exist under + # nix, so without this it fails with "cannot execute: required file not found". + nativeBuildInputs = lib.optionals stdenv.hostPlatform.isLinux [ autoPatchelfHook ]; + buildInputs = lib.optionals stdenv.hostPlatform.isLinux [ stdenv.cc.cc.lib ]; + # The tarball holds a bare `yr`, so there is no directory to strip. sourceRoot = "."; From 2c83ad9bb4654042f08b1f008b1d1f55d64bf14f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patryk=20B=C4=99za?= Date: Tue, 4 Aug 2026 00:13:48 +0200 Subject: [PATCH 8/8] fix(nix): drop the unneeded patchelf hook from the yara-x derivation autoPatchelfHook went onto both new derivations when the Linux dev shell broke, without checking which one had failed. Only magika had: `file` on the release assets shows magika's Linux binary is dynamically linked while yara-x's is static-pie, so it names no interpreter and has no dynamic dependencies to rewrite. Each comment now says which case it is, so the hook is not re-added to yara-x for symmetry. Also drops the half of the yara-x header comment that restated flake.nix's existing "Pinned to CI version" note, keeping only the reason specific to this tool: nixpkgs has 1.16.0, and taking it from there would let a nixpkgs bump change the engine underneath the measured blocking allowlist. --- nix/magika.nix | 5 +++-- nix/yara-x.nix | 14 +++++--------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/nix/magika.nix b/nix/magika.nix index 99e93a291b..7d809e06bd 100644 --- a/nix/magika.nix +++ b/nix/magika.nix @@ -40,8 +40,9 @@ stdenv.mkDerivation { inherit (asset) hash; }; - # The prebuilt ELF names the host's dynamic linker, which does not exist under - # nix, so without this it fails with "cannot execute: required file not found". + # The Linux binary is dynamically linked, so it names an interpreter that does + # not exist under nix and fails with "cannot execute: required file not found" + # without this. nix/yara-x.nix needs no equivalent - its binary is static-pie. nativeBuildInputs = lib.optionals stdenv.hostPlatform.isLinux [ autoPatchelfHook ]; buildInputs = lib.optionals stdenv.hostPlatform.isLinux [ stdenv.cc.cc.lib ]; diff --git a/nix/yara-x.nix b/nix/yara-x.nix index c2a2ee1e99..f972431c77 100644 --- a/nix/yara-x.nix +++ b/nix/yara-x.nix @@ -2,13 +2,11 @@ lib, stdenv, fetchurl, - autoPatchelfHook, }: -# Pinned rather than taken from nixpkgs so the dev shell and CI run the same -# scanner: nixpkgs currently has 1.16.0, and a nixpkgs bump would otherwise -# change the engine underneath the measured blocking allowlist without anyone -# noticing. See .github/yara/README.md. +# nixpkgs has 1.16.0, and taking it from there would let a nixpkgs bump change +# the engine underneath the measured blocking allowlist without anyone noticing. +# See .github/yara/README.md. let version = "1.19.0"; @@ -38,10 +36,8 @@ stdenv.mkDerivation { inherit (asset) hash; }; - # The prebuilt ELF names the host's dynamic linker, which does not exist under - # nix, so without this it fails with "cannot execute: required file not found". - nativeBuildInputs = lib.optionals stdenv.hostPlatform.isLinux [ autoPatchelfHook ]; - buildInputs = lib.optionals stdenv.hostPlatform.isLinux [ stdenv.cc.cc.lib ]; + # No autoPatchelfHook here, unlike nix/magika.nix: the Linux `yr` is static-pie + # linked, so it names no interpreter and has no dynamic deps to rewrite. # The tarball holds a bare `yr`, so there is no directory to strip. sourceRoot = ".";