diff --git a/proof/region/v1/PROTOCOL.md b/proof/region/v1/PROTOCOL.md index 880000cf..81edb8c2 100644 --- a/proof/region/v1/PROTOCOL.md +++ b/proof/region/v1/PROTOCOL.md @@ -32,7 +32,7 @@ MPFI source closure ещё не является provenance исполнения receipt, cross-path dependency overlap и diversity не представлены admitted типом; structural coordinates не восполняют это отсутствие. -## Wire и identity +## Бинарный формат и идентичность Для wire-artifact-ов из `region_proof_protocol.py` все целые беззнаковые и записаны big-endian как `u8`, `u32be` или `u64be`; `digest` — ровно 32 @@ -155,7 +155,7 @@ Arb controller связывает его с наблюдёнными BUILD/RUN границы доверия; receipt не заявляет отсутствие ambient inputs за пределами этой границы. Альтернативный JSON/TOML definition запрещён протоколом. -## Source lock и integrity observations +## Фиксация источников и наблюдения целостности `SourceReleaseLockV1` фиксирует bytes и структурный состав архива. Поле `.integrity` содержит один `SourceIntegrityPolicyV1`; это точная граница @@ -195,7 +195,7 @@ Lab Colors фиксирует exact HTTPS URL, длину и SHA-256 получ `MpfiSourceLockV1` использует те же единичные GMP/MPFR source declarations, что и Arb, однако имеет отдельную aggregate identity и отдельный typed admission. -## Diagnostic execution boundary +## Диагностическая граница исполнения `proof/region/v1/executor.py` — общий для enclosure engines leaf без импорта Arb/MPFI, formula или comparator semantics. Он же единолично кодирует @@ -234,7 +234,60 @@ observation. Право на Arb receipt получает не executor, а от `SourceBoundArbControllerV1`, который владеет всей цепью BUILD → RUN и не принимает backend, capability либо diagnostic observation от вызывающего. -## Source-bound Arb replay +## Общая граница BUILD + +`proof/region/v1/build/input.py` принимает уже нормализованные lane entries, +кодирует один канонический USTAR и владеет точными input bytes. Он не +импортирует и не перепроверяет source capability: это ответственность +потребляющего lane. `SealedInputV1` структурно неизменяем, связывает +целостность байтов с opaque caller digest и не утверждает recipe либо engine +semantics. Resource bounds передаёт lane: общий encoder не вводит собственный +fixture-specific cap. + +`proof/region/v1/build/transport.py` владеет immutable Docker policy, +одноразовым probe→build lease, bounded stdin/stdout observation, cleanup и +двумя свежими попытками. Доказательные координаты разделены по причинам: + +1. transport policy identity связывает все поля точной policy; +2. native command contract identity связывает один типизированный grammar для + probe, build и cleanup и один immutable child-launch context + (environment, cwd, umask, stdio topology, FD и session behavior); фактический argv и + Popen kwargs строятся только этими значениями; +3. daemon observation identity связывает только два raw probe stdout; +4. Docker capability identity связывает policy, command contract и exact CLI + path, daemon observation и наблюдённые host uid/gid. + +`BuildSessionV1` и каждый `DockerBuildRequestV1` сохраняют только ту же +capability, те же input bytes и output cap. Request не содержит host path, +CID file или имя контейнера: native adapter сам создаёт свежий приватный CID +path. Native cleanup поддерживается только в fresh one-job VM workflow Arb: +другой субъект с тем же effective UID либо Docker-daemon authority там не +сосуществует. Права `0700` закрывают лишь cross-UID pathname access и не +аутентифицируют same-UID writer. В этой объявленной operational boundary для +cleanup допускается только полный ID, который Docker записал в CID path; перед +`rm --force ` adapter сверяет, что `docker container inspect` вернул тот же +ID. Имя контейнера и fallback-координата в cleanup не участвуют. Вне этой +границы CID path не является доказательством ownership. Чужая либо не +полученная текущим probe capability отвергается до process spawn; ambient +path/user повторно не считываются. Разрешение принадлежит создавшему process: +fork и конкурентное повторное использование отвергаются до блокировки. После +возврата Popen handle `BaseException` до повторного выброса исходного +interruption запускает детерминированные попытки остановить и reap CLI, закрыть +streams и очистить допущенный container. Во время самого Popen construction +handle может ещё отсутствовать: тогда возможна только best-effort попытка CID +cleanup, без ложного заявления о reap CLI. `TwoBuildObservationV1` хранит обе успешные попытки и только +классифицирует их байты как identical или different, не называя пару +универсальным доказательством воспроизводимости. При отказе после создания +валидной session сохраняется весь уже завершённый causal prefix. Context-free +contract violation, обнаруженный до создания session (например, невалидная +session или сбой создания приватного CID-root через `mkdtemp`), может вернуть `BuildRejectedV1` без +`session` и `completed_processes`. +Transport не знает formula, ELF, comparator или +source provenance: lane отдельно перепроверяет semantic input binding перед +каждым process и передаёт output admission. Arb объявляет собственную exact +policy; MPFI обязан объявить другую, а не заимствовать Arb semantics. + +## Воспроизведение Arb, связанное с источником `SourceBoundArbControllerV1` сначала повторно парсит source lock и job, повторно допускает exact owned archive/build-input bytes и строит из regular @@ -244,7 +297,8 @@ object дважды передаётся через bounded stdin; каждый bounded tmpfs, а executable возвращает через stdout. Semantic host bind mounts, host output path и повторное открытие результата отсутствуют. Эта граница доказывает точный controller-observed byte stream, а не непрерывность inode -между host и Docker daemon; сам daemon остаётся явно доверенным V1 input. +между host и Docker daemon. Raw daemon observation входит в capability, но сам +daemon остаётся явно доверенным input объявленной границы. Успешный replay хранится одним token-closed `ContentResolvedEvaluatorReplayV1`, который повторно выводит три причинные @@ -253,9 +307,9 @@ identity без зеркальных промежуточных dataclass: 1. source identity связывает lock, три admitted archive closures, build inputs и formula support. Job сюда не входит: одинаковый evaluator build не меняет source identity от конкретного RUN; -2. build identity связывает source identity, versioned transport/isolation - policy, trust boundary, pinned OCI toolchain, один sealed bundle object, два - exact transfer и два byte-identical executable stdout. Comparator verifier +2. build identity связывает source identity, versioned Docker capability, + pipeline policy, trust boundary, один sealed bundle object, два exact + transfer и два byte-identical executable stdout. Comparator verifier строит свежий canonical manifest из SHA-256 retained preimage bytes и сверяет все его поля и identity с build observation; это проверка retained причинных данных, а не заявление о независимом втором выводе preimages; @@ -489,7 +543,7 @@ release не содержит. Family mint manifest: единственный range `[0, 2^24)` и point count `2^24`. Совпадение только point count или reduced-domain candidate этот gate не проходят. -## Ошибки admission +## Ошибки допуска `ProtocolReasonV1` — закрытая сумма: diff --git a/proof/region/v1/arb/build-inner.sh b/proof/region/v1/arb/build-inner.sh new file mode 100644 index 00000000..b16e0afd --- /dev/null +++ b/proof/region/v1/arb/build-inner.sh @@ -0,0 +1,156 @@ +#!/bin/sh +# Internal Arb recipe. The source-bound transport dispatches this file only +# after establishing its clean child environment; it is not a standalone API. +set -eu + +if [ "$#" -ne 0 ]; then + printf '%s\n' 'arb build takes no arguments' >&2 + exit 64 +fi + +umask 022 + +readonly inputs=/build/snapshot/inputs +readonly workspace=/build/snapshot/workspace +readonly build=/build/work + +require_regular() { + if [ ! -f "$1" ] || [ -L "$1" ]; then + printf 'missing regular build input: %s\n' "$1" >&2 + exit 66 + fi +} + +require_directory() { + if [ ! -d "$1" ] || [ -L "$1" ]; then + printf 'missing normalized source directory: %s\n' "$1" >&2 + exit 66 + fi +} + +require_empty_directory() { + if [ ! -d "$1" ] || [ -L "$1" ]; then + printf 'missing build directory: %s\n' "$1" >&2 + exit 66 + fi + if [ -n "$(find "$1" -mindepth 1 -maxdepth 1 -print -quit)" ]; then + printf 'build directory is not empty: %s\n' "$1" >&2 + exit 65 + fi +} + +require_directory "$inputs/gmp-6.3.0" +require_directory "$inputs/mpfr-4.2.2" +require_directory "$inputs/flint-3.6.0" +require_regular "$inputs/formula.generated.c" +printf '%s %s\n' \ + '9958f20c8ca598625db0593a45f8f8bc79e4b2f22b53263b6c32d78a5e1d2693' \ + "$inputs/formula.generated.c" \ + | /usr/bin/sha256sum --check --strict - +for source in main.c wire.c hash.c interval.c region.c; do + require_regular "$workspace/proof/region/v1/arb/evaluator/$source" +done +require_regular "$workspace/proof/region/v1/arb/evaluator/formula.h" +for header in wire.h hash.h interval.h region.h; do + require_regular "$workspace/proof/region/v1/arb/evaluator/$header" +done +require_empty_directory "$build" + +/usr/bin/mkdir "$build/prefix" "$build/gmp" "$build/mpfr" "$build/flint" "$build/tmp" + +# GCC 15 changed its implicit dialect to GNU C23, where GMP 6.3.0's locked +# no-prototype configure probes have different semantics. GNU C17 is the last +# default those probes targeted; changing it requires a source/toolchain slice +# and a fresh live build, not reliance on a compiler's moving default. +readonly common_cflags='-O2 -g0 -fno-ident -fno-fast-math -ffp-contract=off -fno-lto -std=gnu17 -march=x86-64 -mtune=generic -ffile-prefix-map=/build=. -fdebug-prefix-map=/build=.' +readonly common_ldflags='-Wl,--build-id=none -fno-lto' +readonly prefix="$build/prefix" + +cd "$build/gmp" +ABI=64 CC=/usr/local/bin/gcc CFLAGS="$common_cflags" LDFLAGS="$common_ldflags" \ + "$inputs/gmp-6.3.0/configure" \ + --build=x86_64-pc-linux-gnu \ + --host=x86_64-pc-linux-gnu \ + --prefix="$prefix" \ + --disable-shared \ + --enable-static \ + --disable-assembly \ + --disable-cxx +/usr/bin/make -j1 +/usr/bin/make check -j1 +/usr/bin/make install + +cd "$build/mpfr" +CC=/usr/local/bin/gcc CFLAGS="$common_cflags" LDFLAGS="$common_ldflags" \ + "$inputs/mpfr-4.2.2/configure" \ + --build=x86_64-pc-linux-gnu \ + --host=x86_64-pc-linux-gnu \ + --prefix="$prefix" \ + --with-gmp="$prefix" \ + --disable-shared \ + --enable-static \ + --enable-formally-proven-code +/usr/bin/make -j1 +/usr/bin/make check -j1 +/usr/bin/make install + +cd "$build/flint" +CC=/usr/local/bin/gcc CFLAGS="$common_cflags" LDFLAGS="$common_ldflags" \ + "$inputs/flint-3.6.0/configure" \ + --build=x86_64-pc-linux-gnu \ + --host=x86_64-pc-linux-gnu \ + --prefix="$prefix" \ + --with-gmp="$prefix" \ + --with-mpfr="$prefix" \ + --disable-shared \ + --enable-static \ + --disable-assembly \ + --disable-lto \ + --enable-assert +/usr/bin/make -j1 +/usr/bin/make check -j1 +/usr/bin/make install + +cd "$workspace/proof/region/v1/arb/evaluator" +/usr/local/bin/gcc \ + -O2 -g0 -fno-ident -fno-fast-math -ffp-contract=off -fno-lto \ + -march=x86-64 -mtune=generic \ + -ffile-prefix-map=/build=. -fdebug-prefix-map=/build=. \ + -std=c17 -Wall -Wextra -Werror -pedantic \ + -I. -I"$prefix/include" \ + main.c wire.c hash.c interval.c region.c "$inputs/formula.generated.c" \ + -static -Wl,--build-id=none -fno-lto \ + "$prefix/lib/libflint.a" "$prefix/lib/libmpfr.a" "$prefix/lib/libgmp.a" \ + -lm -lpthread \ + -o "$build/arb-evaluator-v1" + +if ! /usr/bin/readelf -l "$build/arb-evaluator-v1" > "$build/program-headers"; then + printf '%s\n' 'cannot inspect evaluator program headers' >&2 + exit 70 +fi +if /usr/bin/grep -q INTERP "$build/program-headers"; then + printf '%s\n' 'evaluator unexpectedly contains PT_INTERP' >&2 + exit 70 +else + grep_status=$? + if [ "$grep_status" -ne 1 ]; then + printf '%s\n' 'cannot search evaluator program headers' >&2 + exit 70 + fi +fi +if ! /usr/bin/readelf -d "$build/arb-evaluator-v1" > "$build/dynamic-section"; then + printf '%s\n' 'cannot inspect evaluator dynamic section' >&2 + exit 70 +fi +if /usr/bin/grep -q NEEDED "$build/dynamic-section"; then + printf '%s\n' 'evaluator unexpectedly contains DT_NEEDED' >&2 + exit 70 +else + grep_status=$? + if [ "$grep_status" -ne 1 ]; then + printf '%s\n' 'cannot search evaluator dynamic section' >&2 + exit 70 + fi +fi + +/usr/bin/sha256sum "$build/arb-evaluator-v1" diff --git a/proof/region/v1/arb/build.sh b/proof/region/v1/arb/build.sh index 7b4462c0..88cae183 100755 --- a/proof/region/v1/arb/build.sh +++ b/proof/region/v1/arb/build.sh @@ -1,8 +1,8 @@ #!/bin/sh -# Build the offline Arb evaluator from one admitted controller stream. -# Acquisition and origin verification intentionally happen before this -# network-free boundary; this recipe never resolves a tool or dependency online. - +# Source-owned Arb dispatcher. The trusted Docker transport invokes this file +# from the fixed bundle path with a clean environment. Keep the outer shell +# limited to builtins: path resolution must happen only in the clean child. +# shellcheck disable=SC2016 set -eu if [ "$#" -ne 0 ]; then @@ -10,167 +10,24 @@ if [ "$#" -ne 0 ]; then exit 64 fi -# Configure and Make observe many ambient variables. Re-exec once from an empty -# environment so a persistent CI host cannot silently change the binary. -if [ "${LC_BUILD_ENV_V1-}" != 1 ]; then - exec /usr/bin/env -i \ - LC_BUILD_ENV_V1=1 \ - PATH=/usr/local/bin:/usr/bin:/bin \ - LC_ALL=C \ - LANG=C \ - TZ=UTC \ - HOME=/nonexistent \ - TMPDIR=/build/work/tmp \ - SOURCE_DATE_EPOCH=0 \ - ZERO_AR_DATE=1 \ - ARFLAGS=crD \ - /bin/sh "$0" -fi -unset LC_BUILD_ENV_V1 - -umask 022 - -readonly inputs=/build/snapshot/inputs -readonly workspace=/build/snapshot/workspace -readonly build=/build/work - -require_regular() { - if [ ! -f "$1" ] || [ -L "$1" ]; then - printf 'missing regular build input: %s\n' "$1" >&2 - exit 66 - fi -} - -require_directory() { - if [ ! -d "$1" ] || [ -L "$1" ]; then - printf 'missing normalized source directory: %s\n' "$1" >&2 - exit 66 - fi -} - -require_empty_directory() { - if [ ! -d "$1" ] || [ -L "$1" ]; then - printf 'missing build directory: %s\n' "$1" >&2 - exit 66 - fi - if [ -n "$(find "$1" -mindepth 1 -maxdepth 1 -print -quit)" ]; then - printf 'build directory is not empty: %s\n' "$1" >&2 - exit 65 - fi -} - -require_directory "$inputs/gmp-6.3.0" -require_directory "$inputs/mpfr-4.2.2" -require_directory "$inputs/flint-3.6.0" -require_regular "$inputs/formula.generated.c" -printf '%s %s\n' \ - '9958f20c8ca598625db0593a45f8f8bc79e4b2f22b53263b6c32d78a5e1d2693' \ - "$inputs/formula.generated.c" \ - | /usr/bin/sha256sum --check --strict - -for source in main.c wire.c hash.c interval.c region.c; do - require_regular "$workspace/proof/region/v1/arb/evaluator/$source" -done -require_regular "$workspace/proof/region/v1/arb/evaluator/formula.h" -for header in wire.h hash.h interval.h region.h; do - require_regular "$workspace/proof/region/v1/arb/evaluator/$header" -done -require_empty_directory "$build" - -/usr/bin/mkdir "$build/prefix" "$build/gmp" "$build/mpfr" "$build/flint" "$build/tmp" - -# GCC 15 changed its implicit dialect to GNU C23, where GMP 6.3.0's locked -# no-prototype configure probes have different semantics. GNU C17 is the last -# default those probes targeted; changing it requires a source/toolchain slice -# and a fresh live build, not reliance on a compiler's moving default. -readonly common_cflags='-O2 -g0 -fno-ident -fno-fast-math -ffp-contract=off -fno-lto -std=gnu17 -march=x86-64 -mtune=generic -ffile-prefix-map=/build=. -fdebug-prefix-map=/build=.' -readonly common_ldflags='-Wl,--build-id=none -fno-lto' -readonly prefix="$build/prefix" - -cd "$build/gmp" -ABI=64 CC=/usr/local/bin/gcc CFLAGS="$common_cflags" LDFLAGS="$common_ldflags" \ - "$inputs/gmp-6.3.0/configure" \ - --build=x86_64-pc-linux-gnu \ - --host=x86_64-pc-linux-gnu \ - --prefix="$prefix" \ - --disable-shared \ - --enable-static \ - --disable-assembly \ - --disable-cxx -/usr/bin/make -j1 -/usr/bin/make check -j1 -/usr/bin/make install - -cd "$build/mpfr" -CC=/usr/local/bin/gcc CFLAGS="$common_cflags" LDFLAGS="$common_ldflags" \ - "$inputs/mpfr-4.2.2/configure" \ - --build=x86_64-pc-linux-gnu \ - --host=x86_64-pc-linux-gnu \ - --prefix="$prefix" \ - --with-gmp="$prefix" \ - --disable-shared \ - --enable-static \ - --enable-formally-proven-code -/usr/bin/make -j1 -/usr/bin/make check -j1 -/usr/bin/make install - -cd "$build/flint" -CC=/usr/local/bin/gcc CFLAGS="$common_cflags" LDFLAGS="$common_ldflags" \ - "$inputs/flint-3.6.0/configure" \ - --build=x86_64-pc-linux-gnu \ - --host=x86_64-pc-linux-gnu \ - --prefix="$prefix" \ - --with-gmp="$prefix" \ - --with-mpfr="$prefix" \ - --disable-shared \ - --enable-static \ - --disable-assembly \ - --disable-lto \ - --enable-assert -/usr/bin/make -j1 -/usr/bin/make check -j1 -/usr/bin/make install - -cd "$workspace/proof/region/v1/arb/evaluator" -/usr/local/bin/gcc \ - -O2 -g0 -fno-ident -fno-fast-math -ffp-contract=off -fno-lto \ - -march=x86-64 -mtune=generic \ - -ffile-prefix-map=/build=. -fdebug-prefix-map=/build=. \ - -std=c17 -Wall -Wextra -Werror -pedantic \ - -I. -I"$prefix/include" \ - main.c wire.c hash.c interval.c region.c "$inputs/formula.generated.c" \ - -static -Wl,--build-id=none -fno-lto \ - "$prefix/lib/libflint.a" "$prefix/lib/libmpfr.a" "$prefix/lib/libgmp.a" \ - -lm -lpthread \ - -o "$build/arb-evaluator-v1" - -if ! /usr/bin/readelf -l "$build/arb-evaluator-v1" > "$build/program-headers"; then - printf '%s\n' 'cannot inspect evaluator program headers' >&2 - exit 70 -fi -if /usr/bin/grep -q INTERP "$build/program-headers"; then - printf '%s\n' 'evaluator unexpectedly contains PT_INTERP' >&2 - exit 70 -else - grep_status=$? - if [ "$grep_status" -ne 1 ]; then - printf '%s\n' 'cannot search evaluator program headers' >&2 - exit 70 - fi -fi -if ! /usr/bin/readelf -d "$build/arb-evaluator-v1" > "$build/dynamic-section"; then - printf '%s\n' 'cannot inspect evaluator dynamic section' >&2 - exit 70 -fi -if /usr/bin/grep -q NEEDED "$build/dynamic-section"; then - printf '%s\n' 'evaluator unexpectedly contains DT_NEEDED' >&2 - exit 70 -else - grep_status=$? - if [ "$grep_status" -ne 1 ]; then - printf '%s\n' 'cannot search evaluator dynamic section' >&2 - exit 70 - fi -fi - -/usr/bin/sha256sum "$build/arb-evaluator-v1" +exec /usr/bin/env -i \ + PATH=/usr/local/bin:/usr/bin:/bin \ + LC_ALL=C \ + LANG=C \ + TZ=UTC \ + HOME=/nonexistent \ + TMPDIR=/build/work/tmp \ + SOURCE_DATE_EPOCH=0 \ + ZERO_AR_DATE=1 \ + ARFLAGS=crD \ + /bin/sh -c ' + set -eu + script_path=$(/usr/bin/readlink -f -- "$1") + script_dir=$(/usr/bin/dirname -- "$script_path") + inner="$script_dir/build-inner.sh" + if [ ! -f "$inner" ] || [ -L "$inner" ]; then + printf "%s\\n" "missing regular Arb inner build recipe" >&2 + exit 66 + fi + exec /bin/sh "$inner" + ' /bin/sh "$0" diff --git a/proof/region/v1/arb/pipeline.py b/proof/region/v1/arb/pipeline.py index bb09eae5..d937ec05 100644 --- a/proof/region/v1/arb/pipeline.py +++ b/proof/region/v1/arb/pipeline.py @@ -12,21 +12,14 @@ import hashlib import io -import json -import os -import platform -import selectors -import signal -import stat -import subprocess import tarfile -import tempfile -import time -from dataclasses import dataclass, field, fields +from dataclasses import dataclass, fields from enum import StrEnum from functools import cached_property -from pathlib import Path -from typing import NoReturn, Protocol, TypeAlias +from typing import NoReturn, TypeAlias + +from build import input as build_input +from build import transport as build_transport import executor import provenance @@ -43,6 +36,12 @@ FORMULA_SPEC_PATH_V1 = "crates/labcolors-core/contracts/contextual-region-formula-v1.lcir" FORMULA_GENERATOR_PATH_V1 = "proof/region/v1/arb/evaluator/formula.py" BUILD_RECIPE_PATH_V1 = "proof/region/v1/arb/build.sh" +INNER_BUILD_RECIPE_PATH_V1 = "proof/region/v1/arb/build-inner.sh" +# One ordered recipe coordinate is shared by BUILD and test-observation +# preimages. Keeping the dispatcher and the actual make/check body together +# prevents a recipe-only mutation from masquerading as evaluator source. +BUILD_RECIPE_PATHS_V1 = (BUILD_RECIPE_PATH_V1, INNER_BUILD_RECIPE_PATH_V1) +BUILD_RECIPE_PATH_SET_V1 = frozenset(BUILD_RECIPE_PATHS_V1) FORMULA_SPEC_SHA256_V1 = "a6f77ac462f226453b1c27bbd8637b62780b9a640c317a6f50028dacd1de8540" GENERATED_FORMULA_SHA256_V1 = "9958f20c8ca598625db0593a45f8f8bc79e4b2f22b53263b6c32d78a5e1d2693" @@ -53,7 +52,8 @@ _PINNED_BUILD_SOURCE_SHA256_V1 = { FORMULA_SPEC_PATH_V1: FORMULA_SPEC_SHA256_V1, GENERATED_FORMULA_PATH_V1: GENERATED_FORMULA_SHA256_V1, - BUILD_RECIPE_PATH_V1: "92d6de1a321d5e097e122eeda68111d75283089b0c75adc0d359d46494a65390", + BUILD_RECIPE_PATH_V1: "067db346c2d13b393104964fdd8298d3204b77e13bae213d9b1f3a0bf820f94f", + INNER_BUILD_RECIPE_PATH_V1: "b70ee1a6b0259745d5f9326c14f12f83ab026b5f5bf57a01f794edd1c5aa1d00", FORMULA_GENERATOR_PATH_V1: "16629cc3a2ef745ae244ae4762f8946a6546972886f96beeb9ee4920b043040c", "proof/region/v1/arb/evaluator/formula.h": "46fd5ad1b68b728efcd990a71d1dcc273b75e3391d8c06ef2fd0ac6a4d7dfdbd", "proof/region/v1/arb/evaluator/hash.c": "c28e6281208f09ca15fa74aea0091f27726ed68efc3480c34a7db33b8ca3567e", @@ -72,11 +72,13 @@ for path in sorted(_PINNED_BUILD_SOURCE_SHA256_V1) ) -BUILD_STDOUT_LIMIT_V1 = 16 * 1024 * 1024 -BUILD_STDERR_LIMIT_V1 = 16 * 1024 * 1024 -BUILD_TIMEOUT_NS_V1 = 2 * 60 * 60 * 1_000_000_000 -DOCKER_PROBE_OUTPUT_LIMIT_V1 = 1024 * 1024 -DOCKER_PROBE_TIMEOUT_NS_V1 = 30 * 1_000_000_000 +# The generic transport owns universal observer ceilings. This lane binds to +# those coordinates rather than recreating a coincident copy of the policy. +BUILD_STDOUT_LIMIT_V1 = build_transport.BUILD_STDOUT_LIMIT_V1 +BUILD_STDERR_LIMIT_V1 = build_transport.BUILD_STDERR_LIMIT_V1 +BUILD_TIMEOUT_NS_V1 = build_transport.BUILD_TIMEOUT_NS_V1 +DOCKER_PROBE_OUTPUT_LIMIT_V1 = build_transport.DOCKER_PROBE_OUTPUT_LIMIT_V1 +DOCKER_PROBE_TIMEOUT_NS_V1 = build_transport.DOCKER_PROBE_TIMEOUT_NS_V1 MAX_BUILD_SOURCE_FILE_BYTES_V1 = 16 * 1024 * 1024 MAX_BUILD_SOURCE_TOTAL_BYTES_V1 = 32 * 1024 * 1024 @@ -128,18 +130,13 @@ _FLINT_RELEASE_ONLY_ID_LABEL_V1 = ( b"labcolors.proof-region.flint-project-pinned-release-only.v1\0" ) -_PIPELINE_POLICY_ID_LABEL_V1 = b"labcolors.proof-region.arb-pipeline-policy.v1\0" -_BUILD_INPUT_BUNDLE_ID_LABEL_V1 = ( - b"labcolors.proof-region.arb-build-input-bundle.v1\0" +_PIPELINE_POLICY_ID_LABEL_V2 = b"labcolors.proof-region.arb-pipeline-policy.v2\0" +_BUILD_INPUT_BUNDLE_ID_LABEL_V2 = ( + b"labcolors.proof-region.arb-build-input-bundle.v2\0" ) _BUILD_SOURCES_TOKEN = object() _COMPARATOR_TOKEN = object() _BUILD_OBSERVATION_TOKEN = object() -_BUILD_INPUT_BUNDLE_TOKEN = object() -_BUILD_INPUT_PROGRESS_TOKEN = object() -_BUILD_INPUT_TRANSFER_TOKEN = object() -_DOCKER_COMMAND_EXITED_TOKEN = object() -_DOCKER_BUILD_EXITED_TOKEN = object() def _blob(value: bytes) -> bytes: @@ -346,205 +343,87 @@ def admit_build_sources_v1( ) -@dataclass(frozen=True, init=False) -class SealedBuildInputBundleV1: - """One controller-owned immutable byte object reused by both BUILDs.""" +ARB_BUILD_TRANSPORT_POLICY_V1 = build_transport.DockerBuildPolicyV1( + OCI_IMAGE_REFERENCE_V1, + OCI_PLATFORM_V1, + "labcolors-arb-build-v1", + _BUILD_BOOTSTRAP_V1, + "labcolors-arb-build-bootstrap-v1", + (_BUILD_TMPFS_SPEC_V1, _BUILD_STATE_TMPFS_SPEC_V1), + build_transport.DockerUserModeV1.HOST_EFFECTIVE_IDS, + BUILD_STDOUT_LIMIT_V1, + BUILD_STDERR_LIMIT_V1, + BUILD_TIMEOUT_NS_V1, + DOCKER_PROBE_OUTPUT_LIMIT_V1, + DOCKER_PROBE_TIMEOUT_NS_V1, +) - source_identity: bytes - build_input_identity: bytes - sha256: bytes - length: int - identity: bytes - _contents: bytes = field(repr=False, compare=False) - def __init__( - self, - source_identity: bytes, - build_input_identity: bytes, - contents: bytes, - *, - _token: object, - ) -> None: - if _token is not _BUILD_INPUT_BUNDLE_TOKEN: - raise TypeError( - "SealedBuildInputBundleV1 is created only by the build controller" - ) - if not _valid_digest(source_identity) or not _valid_digest( - build_input_identity - ): - raise TypeError("invalid build input coordinates") - if type(contents) is not bytes or not contents: - raise TypeError("build input bundle must be owned nonempty bytes") - digest = hashlib.sha256(contents).digest() - identity = _identity( - _BUILD_INPUT_BUNDLE_ID_LABEL_V1, - ( - source_identity, - build_input_identity, - len(contents).to_bytes(8, "big"), - digest, - hashlib.sha256(_BUILD_BOOTSTRAP_V1.encode("utf-8")).digest(), - ), - ) - for name, value in ( - ("source_identity", source_identity), - ("build_input_identity", build_input_identity), - ("sha256", digest), - ("length", len(contents)), - ("identity", identity), - ("_contents", contents), - ): - object.__setattr__(self, name, value) +def _arb_input_binding_identity_v2( + source_identity: bytes, + build_input_identity: bytes, + contents: bytes, + exact_policy: build_transport.DockerBuildPolicyV1, +) -> bytes: + if ( + not _valid_digest(source_identity) + or not _valid_digest(build_input_identity) + or type(contents) is not bytes + or not contents + or not build_transport.docker_policy_is_valid_v1(exact_policy) + ): + raise TypeError("invalid Arb build input binding coordinates") + digest = hashlib.sha256(contents).digest() + return _identity( + _BUILD_INPUT_BUNDLE_ID_LABEL_V2, + ( + source_identity, + build_input_identity, + len(contents).to_bytes(8, "big"), + digest, + # This inner identity fixes only the stream-to-tree program. V1 + # never reads shell $0; argv0 instead remains in the outer + # transport identity. A bootstrap that consumes $0 needs a new + # binding schema rather than silently widening this preimage. + hashlib.sha256( + exact_policy.bootstrap.encode("utf-8") + ).digest(), + ), + ) + +def arb_input_is_bound_v1( + request: object, + exact_policy: object, + value: object, +) -> bool: + """Recompute Arb semantics independently of generic byte integrity.""" -def sealed_build_input_bundle_is_well_bound_v1(value: object) -> bool: - if type(value) is not SealedBuildInputBundleV1: + if ( + type(request) is not PipelineRequestV1 + or type(value) is not build_input.SealedInputV1 + or not build_input.sealed_input_is_intact_v1(value) + ): return False try: - digest = hashlib.sha256(value._contents).digest() - identity = _identity( - _BUILD_INPUT_BUNDLE_ID_LABEL_V1, - ( - value.source_identity, - value.build_input_identity, - len(value._contents).to_bytes(8, "big"), - digest, - hashlib.sha256(_BUILD_BOOTSTRAP_V1.encode("utf-8")).digest(), - ), - ) - return ( - _valid_digest(value.source_identity) - and _valid_digest(value.build_input_identity) - and type(value._contents) is bytes - and bool(value._contents) - and value.length == len(value._contents) - and value.sha256 == digest - and value.identity == identity + return value.binding_identity == _arb_input_binding_identity_v2( + request.admitted_sources.identity, + request.build_sources.build_input_identity, + value.contents, + exact_policy, ) except Exception: return False -def _canonical_tar_v1( - entries: tuple[tuple[str, int, bytes], ...], -) -> bytes: - if type(entries) is not tuple or not entries: - raise TypeError("build bundle entries must be a canonical nonempty set") - paths = tuple(path for path, _mode, _contents in entries) - if paths != tuple(sorted(paths)) or len(set(paths)) != len(entries): - raise TypeError("build bundle entries must be a canonical nonempty set") - folded_paths: set[str] = set() - directories: set[str] = set() - for path, _mode, _contents in entries: - _logical_path(path) - folded = path.lower() - if folded in folded_paths: - raise TypeError("build bundle paths must be case-distinct") - folded_paths.add(folded) - parts = path.split("/")[:-1] - for length in range(1, len(parts) + 1): - directories.add("/".join(parts[:length])) - if directories.intersection(paths): - raise TypeError("build bundle file cannot also be a directory") - output = io.BytesIO() - with tarfile.open(fileobj=output, mode="w", format=tarfile.USTAR_FORMAT) as archive: - for path in sorted(directories, key=lambda value: (value.count("/"), value)): - member = tarfile.TarInfo(path) - member.type = tarfile.DIRTYPE - member.mode = 0o755 - member.uid = 0 - member.gid = 0 - member.uname = "" - member.gname = "" - member.mtime = 0 - member.size = 0 - archive.addfile(member) - for path, mode, contents in entries: - _logical_path(path) - if ( - type(mode) is not int - or mode not in (0o644, 0o755) - or type(contents) is not bytes - ): - raise TypeError("invalid build bundle entry") - member = tarfile.TarInfo(path) - member.type = tarfile.REGTYPE - member.mode = mode - member.uid = 0 - member.gid = 0 - member.uname = "" - member.gname = "" - member.mtime = 0 - member.size = len(contents) - archive.addfile(member, io.BytesIO(contents)) - return output.getvalue() - - -def _normalized_source_entries_v1( - lock: provenance.SourceReleaseLockV1, - admitted: provenance.SafeSourceArchiveV1, -) -> tuple[tuple[str, int, bytes], ...]: - replayed, raw_tar = provenance.replay_admitted_source_archive_v1( - lock, - admitted, - ) - if ( - replayed.source_lock_identity != admitted.source_lock_identity - or replayed.archive_sha256 != admitted.archive_sha256 - or replayed.tree_identity != admitted.tree_identity - or replayed.regular_file_count != admitted.regular_file_count - or replayed.regular_file_bytes != admitted.regular_file_bytes - or replayed.files != admitted.files - ): - raise TypeError("admitted source coordinates changed before bundle sealing") - expected = {item.path: item for item in replayed.files} - values: list[tuple[str, int, bytes]] = [] - seen: set[str] = set() - with tarfile.open(fileobj=io.BytesIO(raw_tar), mode="r:") as archive: - for member in archive: - if member.isdir(): - continue - if not member.isreg() or not member.name.startswith(lock.root_prefix): - raise TypeError("admitted source replay contains a foreign member") - relative = member.name[len(lock.root_prefix) :] - coordinate = expected.get(relative) - if coordinate is None or relative in seen: - raise TypeError("admitted source replay changed its file set") - stream = archive.extractfile(member) - if stream is None: - raise TypeError("admitted source replay lost a regular file") - chunks: list[bytes] = [] - length = 0 - hasher = hashlib.sha256() - while True: - chunk = stream.read(provenance.READ_CHUNK_BYTES) - if not chunk: - break - length += len(chunk) - if length > coordinate.length: - raise TypeError("admitted source replay exceeded locked length") - chunks.append(chunk) - hasher.update(chunk) - if length != coordinate.length or hasher.digest() != coordinate.sha256: - raise TypeError("admitted source replay changed locked contents") - values.append( - ( - f"inputs/{lock.root_prefix[:-1]}/{relative}", - coordinate.mode, - b"".join(chunks), - ) - ) - seen.add(relative) - if seen != set(expected): - raise TypeError("admitted source replay is incomplete") - return tuple(sorted(values)) - - def _seal_build_input_bundle_v1( request: "PipelineRequestV1", -) -> SealedBuildInputBundleV1: + exact_policy: build_transport.DockerBuildPolicyV1, +) -> build_input.SealedInputV1: if type(request) is not PipelineRequestV1: raise TypeError("request must be PipelineRequestV1") + if not build_transport.docker_policy_is_valid_v1(exact_policy): + raise TypeError("exact_policy must be canonical DockerBuildPolicyV1") source_entries = tuple( entry for lock, admitted in zip( @@ -565,12 +444,33 @@ def _seal_build_input_bundle_v1( for item in request.build_sources.files if item.path not in (FORMULA_SPEC_PATH_V1, FORMULA_GENERATOR_PATH_V1) ) - contents = _canonical_tar_v1(tuple(sorted(source_entries + workspace_entries))) - return SealedBuildInputBundleV1( - request.admitted_sources.identity, - request.build_sources.build_input_identity, + contents = build_input.canonical_ustar_v1( + tuple(sorted(source_entries + workspace_entries)), + build_input.CanonicalInputLimitsV1( + len(source_entries) + len(workspace_entries) + + sum( + path.count("/") + for path, _mode, _contents in source_entries + workspace_entries + ), + max( + MAX_BUILD_SOURCE_FILE_BYTES_V1, + *( + lock.regular_file_bytes + for lock in request.source_lock.sources + ), + ), + MAX_BUILD_SOURCE_TOTAL_BYTES_V1 + + sum(lock.regular_file_bytes for lock in request.source_lock.sources), + ), + ) + return build_input.seal_input_v1( + _arb_input_binding_identity_v2( + request.admitted_sources.identity, + request.build_sources.build_input_identity, + contents, + exact_policy, + ), contents, - _token=_BUILD_INPUT_BUNDLE_TOKEN, ) @@ -578,29 +478,23 @@ class HostTrustBoundaryV1(StrEnum): UNSEALED_LINUX_X64_DOCKER_HOST = "unsealed-linux-x64-docker-host" -def pipeline_policy_identity_v1( +def pipeline_policy_identity_v2( host_trust: HostTrustBoundaryV1, + exact_policy: build_transport.DockerBuildPolicyV1, ) -> bytes: if type(host_trust) is not HostTrustBoundaryV1: raise TypeError("host_trust must be HostTrustBoundaryV1") + if not build_transport.docker_policy_is_valid_v1(exact_policy): + raise TypeError("exact_policy must be canonical DockerBuildPolicyV1") return _identity( - _PIPELINE_POLICY_ID_LABEL_V1, + _PIPELINE_POLICY_ID_LABEL_V2, ( - OCI_IMAGE_REFERENCE_V1.encode("ascii"), - OCI_PLATFORM_V1.encode("ascii"), + build_transport.transport_policy_identity_v1(exact_policy), + build_transport.native_command_contract_identity_v1(), host_trust.value.encode("ascii"), b"build-observation=diagnostic-unsealed-v1", - b"network=none", - b"rootfs=readonly", - b"scratch-tmpfs=" + _BUILD_TMPFS_SPEC_V1.encode("ascii"), - b"build-state-tmpfs=" + _BUILD_STATE_TMPFS_SPEC_V1.encode("ascii"), - b"cap-drop=all", - b"no-new-privileges=true", b"inputs=one-controller-sealed-normalized-tree-ustar", - b"transport=bounded-docker-stdin-v1", b"container-admission=exact-length-and-sha256-before-extraction", - b"output=bounded-docker-stdout-v1", - hashlib.sha256(_BUILD_BOOTSTRAP_V1.encode("utf-8")).digest(), b"fresh-container-count=2", ), ) @@ -622,6 +516,89 @@ def __str__(self) -> str: return f"{self.reason.value}: {self.field}" +def _normalized_source_entries_v1( + lock: provenance.SourceReleaseLockV1, + admitted: provenance.SafeSourceArchiveV1, +) -> tuple[tuple[str, int, bytes], ...]: + """Replay Arb-owned source authority into generic canonical-tree entries.""" + + def reject(field_name: str) -> NoReturn: + raise PipelineInputErrorV1( + PipelineInputReasonV1.FOREIGN_SOURCE_CAPABILITY, + field_name, + ) + + if type(lock) is not provenance.SourceReleaseLockV1: + reject("lock") + if type(admitted) is not provenance.SafeSourceArchiveV1: + reject("admitted") + try: + replayed, raw_tar = provenance.replay_admitted_source_archive_v1( + lock, + admitted, + ) + except Exception: + reject("admitted") + if ( + replayed.source_lock_identity != admitted.source_lock_identity + or replayed.archive_sha256 != admitted.archive_sha256 + or replayed.tree_identity != admitted.tree_identity + or replayed.regular_file_count != admitted.regular_file_count + or replayed.regular_file_bytes != admitted.regular_file_bytes + or replayed.files != admitted.files + ): + reject("admitted") + expected = {item.path: item for item in replayed.files} + values: list[tuple[str, int, bytes]] = [] + seen: set[str] = set() + try: + with tarfile.open(fileobj=io.BytesIO(raw_tar), mode="r:") as archive: + for member in archive: + if member.isdir(): + continue + if not member.isreg() or not member.name.startswith(lock.root_prefix): + reject("member") + relative = member.name[len(lock.root_prefix) :] + coordinate = expected.get(relative) + if coordinate is None or relative in seen: + reject("file set") + stream = archive.extractfile(member) + if stream is None: + reject("regular file") + chunks: list[bytes] = [] + length = 0 + hasher = hashlib.sha256() + while True: + chunk = stream.read(provenance.READ_CHUNK_BYTES) + if not chunk: + break + length += len(chunk) + if length > coordinate.length: + reject("file length") + chunks.append(chunk) + hasher.update(chunk) + if ( + length != coordinate.length + or hasher.digest() != coordinate.sha256 + ): + reject("file contents") + values.append( + ( + f"inputs/{lock.root_prefix[:-1]}/{relative}", + coordinate.mode, + b"".join(chunks), + ) + ) + seen.add(relative) + except PipelineInputErrorV1: + raise + except (OSError, tarfile.TarError, ValueError): + reject("archive") + if seen != set(expected): + reject("incomplete archive") + return tuple(sorted(values)) + + @dataclass(frozen=True) class FlintSourceContentPartitionV1: """Structural FLINT archive partition, not an origin assertion. @@ -762,6 +739,89 @@ def _comparator_preimage_v1(label: bytes, chunks: tuple[bytes, ...]) -> bytes: ) +def _comparator_preimage_v2(label: bytes, chunks: tuple[bytes, ...]) -> bytes: + """Encode one V2 comparator preimage without accepting a V1 label.""" + + if ( + type(label) is not bytes + or not label.startswith(b"labcolors.proof-region.arb-comparator.") + or not label.endswith(b".v2\0") + or type(chunks) is not tuple + or not chunks + or any(type(chunk) is not bytes for chunk in chunks) + ): + raise TypeError("invalid V2 comparator preimage coordinates") + return label + b"\x02" + len(chunks).to_bytes(4, "big") + b"".join( + _blob(chunk) for chunk in chunks + ) + + +def comparator_build_preimage_v2( + build_sources: AdmittedBuildSourcesV1, + docker_capability_identity: bytes, + pipeline_policy_identity: bytes, + build_processes: tuple[ + build_transport.DockerBuildExitedV1, + build_transport.DockerBuildExitedV1, + ], + binary_sha256: bytes, + rebuild_sha256s: tuple[bytes, bytes], + binary_length: int, +) -> bytes: + """Single replay schema for the BUILD coordinate in the comparator.""" + + if type(build_sources) is not AdmittedBuildSourcesV1: + raise TypeError("build_sources must be AdmittedBuildSourcesV1") + for name, value in ( + ("docker_capability_identity", docker_capability_identity), + ("pipeline_policy_identity", pipeline_policy_identity), + ("binary_sha256", binary_sha256), + ): + if not _valid_digest(value): + raise TypeError(f"invalid {name}") + if ( + type(build_processes) is not tuple + or len(build_processes) != 2 + or any( + type(item) is not build_transport.DockerBuildExitedV1 + for item in build_processes + ) + or type(rebuild_sha256s) is not tuple + or rebuild_sha256s != (binary_sha256, binary_sha256) + or type(binary_length) is not int + or binary_length <= 0 + ): + raise TypeError("invalid comparator BUILD observation") + process_bytes = tuple( + build_transport.build_process_bytes_v1(item) for item in build_processes + ) + return _comparator_preimage_v2( + b"labcolors.proof-region.arb-comparator.build-identity.v2\0", + ( + _encoded_build_file_set_v1( + b"labcolors.proof-region.arb-comparator.build-recipes.v1\0", + tuple( + item + for item in build_sources.files + if item.path in BUILD_RECIPE_PATH_SET_V1 + ), + ), + build_sources.build_input_identity, + build_sources.formula_support_identity, + docker_capability_identity, + pipeline_policy_identity, + b"build-observation=diagnostic-unsealed-v1", + len(build_processes).to_bytes(4, "big"), + *process_bytes, + binary_sha256, + rebuild_sha256s[0], + rebuild_sha256s[1], + binary_length.to_bytes(8, "big"), + binary_sha256, + ), + ) + + def _encoded_build_file_set_v1( label: bytes, files_value: tuple[BuildSourceFileV1, ...], @@ -972,457 +1032,43 @@ def __post_init__(self) -> None: ) -class DockerBlockerReasonV1(StrEnum): - HOST_NOT_LINUX_AMD64 = "host_not_linux_amd64" - DOCKER_UNAVAILABLE = "docker_unavailable" - IMAGE_UNAVAILABLE = "image_unavailable" - IMAGE_IDENTITY_MISMATCH = "image_identity_mismatch" - BACKEND_CONTRACT = "backend_contract" - - -@dataclass(frozen=True) -class DockerUnsupportedV1: - reason: DockerBlockerReasonV1 - detail: str - - def __post_init__(self) -> None: - if type(self.reason) is not DockerBlockerReasonV1: - raise TypeError("invalid Docker blocker reason") - if type(self.detail) is not str or not self.detail or len(self.detail) > 4096: - raise TypeError("invalid Docker blocker detail") - - -@dataclass(frozen=True) -class DockerSupportedV1: - image_reference: str - platform: str - daemon_observation_sha256: bytes - - def __post_init__(self) -> None: - if self.image_reference != OCI_IMAGE_REFERENCE_V1: - raise TypeError("wrong OCI image reference") - if self.platform != OCI_PLATFORM_V1: - raise TypeError("wrong OCI platform") - if not _valid_digest(self.daemon_observation_sha256): - raise TypeError("invalid Docker daemon observation digest") - - -DockerCapabilityReportV1: TypeAlias = DockerSupportedV1 | DockerUnsupportedV1 - - -def _absolute_path(value: object, field_name: str) -> Path: - if not isinstance(value, Path) or not value.is_absolute(): - raise TypeError(f"{field_name} must be an absolute Path") - if any(character in str(value) for character in (",", "\n", "\r", "\0")): - raise TypeError(f"{field_name} is not Docker-mount-safe") - return value - - -_CONTAINER_NAME_PREFIX_V1 = "labcolors-arb-build-v1-" - - -def _container_name(value: object) -> str: - if ( - type(value) is not str - or not value.startswith(_CONTAINER_NAME_PREFIX_V1) - or len(value) > 128 - or any(character not in "abcdefghijklmnopqrstuvwxyz0123456789-" for character in value) - ): - raise TypeError("invalid controller-owned Docker container name") - return value - - -@dataclass(frozen=True) -class DockerBuildRequestV1: - attempt: int - input_bundle: SealedBuildInputBundleV1 - max_executable_bytes: int - cid_file: Path - container_name: str - - def __post_init__(self) -> None: - if type(self.attempt) is not int or self.attempt not in (1, 2): - raise TypeError("attempt must be 1 or 2") - if not sealed_build_input_bundle_is_well_bound_v1(self.input_bundle): - raise TypeError("input_bundle must be controller sealed and well bound") - if ( - type(self.max_executable_bytes) is not int - or self.max_executable_bytes <= 0 - or self.max_executable_bytes > BUILD_STDOUT_LIMIT_V1 - ): - raise TypeError("invalid executable output limit") - _absolute_path(self.cid_file, "cid_file") - _container_name(self.container_name) - - -def _bounded_bytes(value: object, maximum: int, field_name: str) -> bytes: - if type(value) is not bytes or len(value) > maximum: - raise TypeError(f"invalid {field_name}") - return value - - -@dataclass(frozen=True, init=False) -class BuildInputTransferProgressV1: - bundle_identity: bytes - expected_length: int - expected_sha256: bytes - written_length: int - written_sha256: bytes - - def __init__( - self, - bundle_identity: bytes, - expected_length: int, - expected_sha256: bytes, - written_length: int, - written_sha256: bytes, - *, - _token: object, - ) -> None: - if _token is not _BUILD_INPUT_PROGRESS_TOKEN: - raise TypeError("build input progress is controller-observed") - if not _valid_digest(bundle_identity) or not _valid_digest(expected_sha256): - raise TypeError("invalid build input progress coordinates") - if ( - type(expected_length) is not int - or expected_length <= 0 - or type(written_length) is not int - or written_length < 0 - or written_length > expected_length - or type(written_sha256) is not bytes - or len(written_sha256) != 32 - ): - raise TypeError("invalid build input progress") - for name, value in ( - ("bundle_identity", bundle_identity), - ("expected_length", expected_length), - ("expected_sha256", expected_sha256), - ("written_length", written_length), - ("written_sha256", written_sha256), - ): - object.__setattr__(self, name, value) - - -def _build_input_progress_v1( - bundle: SealedBuildInputBundleV1, - written_length: int, - written_sha256: bytes, -) -> BuildInputTransferProgressV1: - if not sealed_build_input_bundle_is_well_bound_v1(bundle): - raise TypeError("build input bundle is not well bound") - if ( - type(written_length) is not int - or written_length < 0 - or written_length > bundle.length - or type(written_sha256) is not bytes - or written_sha256 - != hashlib.sha256(bundle._contents[:written_length]).digest() - ): - raise TypeError("build input progress does not match the sealed bytes") - return BuildInputTransferProgressV1( - bundle.identity, - bundle.length, - bundle.sha256, - written_length, - written_sha256, - _token=_BUILD_INPUT_PROGRESS_TOKEN, - ) - - -@dataclass(frozen=True, init=False) -class BuildInputTransferV1: - bundle_identity: bytes - expected_length: int - expected_sha256: bytes - written_length: int - written_sha256: bytes - - def __init__( - self, - progress: BuildInputTransferProgressV1, - *, - _token: object, - ) -> None: - if _token is not _BUILD_INPUT_TRANSFER_TOKEN: - raise TypeError("build input transfer is controller-observed") - if ( - type(progress) is not BuildInputTransferProgressV1 - or progress.written_length != progress.expected_length - or progress.written_sha256 != progress.expected_sha256 - ): - raise TypeError("completed build input transfer must be exact") - for name in ( - "bundle_identity", - "expected_length", - "expected_sha256", - "written_length", - "written_sha256", - ): - object.__setattr__(self, name, getattr(progress, name)) - - -def _completed_build_input_transfer_v1( - bundle: SealedBuildInputBundleV1, - written_length: int, - written_sha256: bytes, -) -> BuildInputTransferV1: - progress = _build_input_progress_v1(bundle, written_length, written_sha256) - return BuildInputTransferV1( - progress, - _token=_BUILD_INPUT_TRANSFER_TOKEN, - ) - - -@dataclass(frozen=True, init=False) -class _DockerCommandExitedV1: - returncode: int - stdout: bytes - stderr: bytes - - def __init__( - self, - returncode: int, - stdout: bytes, - stderr: bytes, - *, - _token: object, - ) -> None: - if _token is not _DOCKER_COMMAND_EXITED_TOKEN: - raise TypeError("Docker command exit is controller-observed") - if type(returncode) is not int: - raise TypeError("invalid Docker returncode") - _bounded_bytes(stdout, BUILD_STDOUT_LIMIT_V1, "stdout") - _bounded_bytes(stderr, BUILD_STDERR_LIMIT_V1, "stderr") - object.__setattr__(self, "returncode", returncode) - object.__setattr__(self, "stdout", stdout) - object.__setattr__(self, "stderr", stderr) - - -def _docker_command_exited_v1( - returncode: int, - stdout: bytes, - stderr: bytes, -) -> _DockerCommandExitedV1: - return _DockerCommandExitedV1( - returncode, - stdout, - stderr, - _token=_DOCKER_COMMAND_EXITED_TOKEN, - ) - - -@dataclass(frozen=True, init=False) -class DockerBuildExitedV1: - returncode: int - stdout: bytes - stderr: bytes - input_transfer: BuildInputTransferV1 - - def __init__( - self, - returncode: int, - stdout: bytes, - stderr: bytes, - input_transfer: BuildInputTransferV1, - *, - _token: object, - ) -> None: - if _token is not _DOCKER_BUILD_EXITED_TOKEN: - raise TypeError("Docker build exit is controller-observed") - if type(returncode) is not int: - raise TypeError("invalid Docker returncode") - _bounded_bytes(stdout, BUILD_STDOUT_LIMIT_V1, "stdout") - _bounded_bytes(stderr, BUILD_STDERR_LIMIT_V1, "stderr") - if type(input_transfer) is not BuildInputTransferV1: - raise TypeError("invalid Docker build input transfer") - object.__setattr__(self, "returncode", returncode) - object.__setattr__(self, "stdout", stdout) - object.__setattr__(self, "stderr", stderr) - object.__setattr__(self, "input_transfer", input_transfer) - - -def _docker_build_exited_v1( - returncode: int, - stdout: bytes, - stderr: bytes, - input_transfer: BuildInputTransferV1, -) -> DockerBuildExitedV1: - return DockerBuildExitedV1( - returncode, - stdout, - stderr, - input_transfer, - _token=_DOCKER_BUILD_EXITED_TOKEN, - ) - - -@dataclass(frozen=True) -class DockerBuildTimedOutV1: - stdout: bytes - stderr: bytes - input_progress: BuildInputTransferProgressV1 | None = None - - def __post_init__(self) -> None: - _bounded_bytes(self.stdout, BUILD_STDOUT_LIMIT_V1, "stdout") - _bounded_bytes(self.stderr, BUILD_STDERR_LIMIT_V1, "stderr") - if self.input_progress is not None and type( - self.input_progress - ) is not BuildInputTransferProgressV1: - raise TypeError("invalid timed-out build input progress") - - -class DockerOutputStreamV1(StrEnum): - STDOUT = "stdout" - STDERR = "stderr" - - -@dataclass(frozen=True) -class DockerBuildOutputLimitV1: - stream: DockerOutputStreamV1 - stdout: bytes - stderr: bytes - input_progress: BuildInputTransferProgressV1 | None = None - - def __post_init__(self) -> None: - if type(self.stream) is not DockerOutputStreamV1: - raise TypeError("invalid Docker output stream") - _bounded_bytes(self.stdout, BUILD_STDOUT_LIMIT_V1, "stdout") - _bounded_bytes(self.stderr, BUILD_STDERR_LIMIT_V1, "stderr") - if self.input_progress is not None and type( - self.input_progress - ) is not BuildInputTransferProgressV1: - raise TypeError("invalid output-limited build input progress") - - -@dataclass(frozen=True) -class DockerBuildObserverFailureV1: - detail: str - - def __post_init__(self) -> None: - if type(self.detail) is not str or not self.detail or len(self.detail) > 4096: - raise TypeError("invalid Docker observer failure") - - -@dataclass(frozen=True) -class DockerBuildInputRejectedV1: - input_progress: BuildInputTransferProgressV1 - stdout: bytes - stderr: bytes - - def __post_init__(self) -> None: - if type(self.input_progress) is not BuildInputTransferProgressV1: - raise TypeError("invalid partial build input progress") - _bounded_bytes(self.stdout, BUILD_STDOUT_LIMIT_V1, "stdout") - _bounded_bytes(self.stderr, BUILD_STDERR_LIMIT_V1, "stderr") - - @property - def written_length(self) -> int: - return self.input_progress.written_length - - @property - def written_sha256(self) -> bytes: - return self.input_progress.written_sha256 - - -class DockerCleanupTriggerV1(StrEnum): - PROCESS_EXIT = "process_exit" - INPUT_TRANSFER = "input_transfer" - TIMEOUT = "timeout" - OUTPUT_LIMIT = "output_limit" - OBSERVER_FAILURE = "observer_failure" - - -@dataclass(frozen=True) -class DockerBuildCleanupFailureV1: - trigger: DockerCleanupTriggerV1 - detail: str - stdout: bytes - stderr: bytes - input_progress: BuildInputTransferProgressV1 | None = None - - def __post_init__(self) -> None: - if type(self.trigger) is not DockerCleanupTriggerV1: - raise TypeError("invalid Docker cleanup trigger") - if type(self.detail) is not str or not self.detail or len(self.detail) > 4096: - raise TypeError("invalid Docker cleanup failure") - _bounded_bytes(self.stdout, BUILD_STDOUT_LIMIT_V1, "stdout") - _bounded_bytes(self.stderr, BUILD_STDERR_LIMIT_V1, "stderr") - if self.input_progress is not None and type( - self.input_progress - ) is not BuildInputTransferProgressV1: - raise TypeError("invalid cleanup build input progress") - - -DockerBuildProcessObservationV1: TypeAlias = ( - DockerBuildExitedV1 - | DockerBuildTimedOutV1 - | DockerBuildOutputLimitV1 - | DockerBuildObserverFailureV1 - | DockerBuildInputRejectedV1 - | DockerBuildCleanupFailureV1 -) - -_DockerCommandObservationV1: TypeAlias = ( - _DockerCommandExitedV1 | DockerBuildProcessObservationV1 -) - - -class DockerBuildBackendV1(Protocol): - def probe(self) -> DockerCapabilityReportV1: ... - - def run_build( - self, - request: DockerBuildRequestV1, - ) -> DockerBuildProcessObservationV1: ... - - -def build_process_bytes_v1(process: DockerBuildExitedV1) -> bytes: - if ( - type(process) is not DockerBuildExitedV1 - or type(process.input_transfer) is not BuildInputTransferV1 - ): - raise TypeError("only successful typed build observations are encodable") - return b"".join( - ( - process.returncode.to_bytes(4, "big", signed=True), - len(process.stdout).to_bytes(8, "big"), - hashlib.sha256(process.stdout).digest(), - len(process.stderr).to_bytes(8, "big"), - hashlib.sha256(process.stderr).digest(), - process.input_transfer.bundle_identity, - process.input_transfer.expected_length.to_bytes(8, "big"), - process.input_transfer.expected_sha256, - process.input_transfer.written_length.to_bytes(8, "big"), - process.input_transfer.written_sha256, - ) - ) - - -def _derive_arb_comparator_for_build_v1( +def derive_arb_comparator_for_build_v1( request: PipelineRequestV1, - docker_report: DockerSupportedV1, + docker_capability: build_transport.DockerSupportedV1, binary: bytes, rebuild_sha256s: tuple[bytes, bytes], - build_processes: tuple[DockerBuildExitedV1, DockerBuildExitedV1], + build_processes: tuple[ + build_transport.DockerBuildExitedV1, + build_transport.DockerBuildExitedV1, + ], ) -> DiagnosticArbComparatorV1: """Derive all ten coordinates without accepting a caller digest/resolver.""" if type(request) is not PipelineRequestV1: raise TypeError("request must be PipelineRequestV1") - if type(docker_report) is not DockerSupportedV1: - raise TypeError("docker_report must be DockerSupportedV1") + if type(docker_capability) is not build_transport.DockerSupportedV1: + raise TypeError("docker_capability must be DockerSupportedV1") if type(binary) is not bytes or not binary: raise TypeError("binary must be exact nonempty bytes") binary_sha256 = hashlib.sha256(binary).digest() if ( type(build_processes) is not tuple or len(build_processes) != 2 - or any(type(item) is not DockerBuildExitedV1 for item in build_processes) + or any( + type(item) is not build_transport.DockerBuildExitedV1 + for item in build_processes + ) or any(item.returncode != 0 for item in build_processes) or rebuild_sha256s != (binary_sha256, binary_sha256) ): raise TypeError("comparator derivation requires two equal successful builds") - pipeline_policy_identity = pipeline_policy_identity_v1(request.host_trust) + docker_capability_identity = build_transport.docker_capability_identity_v1( + docker_capability + ) + pipeline_policy_identity = pipeline_policy_identity_v2( + request.host_trust, + docker_capability.policy, + ) flint_lock = request.source_lock.sources[2] flint_source = request.admitted_sources.sources[2] if type(flint_lock.integrity) is not provenance.GitContentRelationPolicyV1: @@ -1515,14 +1161,16 @@ def _derive_arb_comparator_for_build_v1( wrapper_files = tuple( item for item in request.build_sources.files if item.path in wrapper_paths ) + recipe_files = tuple( + item + for item in request.build_sources.files + if item.path in BUILD_RECIPE_PATH_SET_V1 + ) evaluator_files = tuple( item for item in request.build_sources.files - if item.path not in ( - FORMULA_SPEC_PATH_V1, - FORMULA_GENERATOR_PATH_V1, - BUILD_RECIPE_PATH_V1, - ) + if item.path not in (FORMULA_SPEC_PATH_V1, FORMULA_GENERATOR_PATH_V1) + and item.path not in BUILD_RECIPE_PATH_SET_V1 and item.path not in wrapper_paths ) wrapper_source = _encoded_build_file_set_v1( @@ -1534,32 +1182,26 @@ def _derive_arb_comparator_for_build_v1( evaluator_files, ) - process_bytes = tuple(build_process_bytes_v1(item) for item in build_processes) - build_identity = _comparator_preimage_v1( - b"labcolors.proof-region.arb-comparator.build-identity.v1\0", - ( - request.build_sources.contents(BUILD_RECIPE_PATH_V1), - request.build_sources.build_input_identity, - request.build_sources.formula_support_identity, - OCI_IMAGE_REFERENCE_V1.encode("ascii"), - OCI_PLATFORM_V1.encode("ascii"), - docker_report.daemon_observation_sha256, - pipeline_policy_identity, - b"build-observation=diagnostic-unsealed-v1", - len(build_processes).to_bytes(4, "big"), - *process_bytes, - binary_sha256, - rebuild_sha256s[0], - rebuild_sha256s[1], - len(binary).to_bytes(8, "big"), - binary_sha256, - ), + process_bytes = tuple( + build_transport.build_process_bytes_v1(item) for item in build_processes + ) + build_identity = comparator_build_preimage_v2( + request.build_sources, + docker_capability_identity, + pipeline_policy_identity, + build_processes, + binary_sha256, + rebuild_sha256s, + len(binary), ) test_observation = _comparator_preimage_v1( b"labcolors.proof-region.arb-comparator.test-observation.v1\0", ( b"kind:aggregate-outer-process-observation-no-per-test-records", - request.build_sources.contents(BUILD_RECIPE_PATH_V1), + _encoded_build_file_set_v1( + b"labcolors.proof-region.arb-comparator.build-recipes.v1\0", + recipe_files, + ), len(build_processes).to_bytes(4, "big"), *process_bytes, ), @@ -1654,637 +1296,12 @@ def _derive_arb_comparator_for_build_v1( ) -class NativeDockerBuildBackendV1: - """Docker adapter whose probe observes only Linux x64 and its daemon.""" - - def __init__( - self, - docker_path: Path, - *, - platform_name: str | None = None, - machine_name: str | None = None, - monotonic_ns: object = time.monotonic_ns, - ) -> None: - if not isinstance(docker_path, Path) or not docker_path.is_absolute(): - raise TypeError("docker_path must be an absolute Path") - self._docker_path = docker_path - self._platform_name = ( - platform.system().lower() if platform_name is None else platform_name - ) - self._machine_name = platform.machine() if machine_name is None else machine_name - self._monotonic_ns = monotonic_ns - - @staticmethod - def _environment() -> dict[str, str]: - return { - "HOME": "/nonexistent", - "PATH": "/usr/bin:/bin", - "DOCKER_CONFIG": "/nonexistent", - } - - def probe(self) -> DockerCapabilityReportV1: - if self._platform_name != "linux" or self._machine_name.lower() not in ( - "x86_64", - "amd64", - ): - return DockerUnsupportedV1( - DockerBlockerReasonV1.HOST_NOT_LINUX_AMD64, - "controlled build requires a Linux amd64 Docker host", - ) - try: - metadata = self._docker_path.lstat() - except OSError: - return DockerUnsupportedV1( - DockerBlockerReasonV1.DOCKER_UNAVAILABLE, - "exact Docker CLI path is unavailable", - ) - if not stat.S_ISREG(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode): - return DockerUnsupportedV1( - DockerBlockerReasonV1.DOCKER_UNAVAILABLE, - "Docker CLI must be one regular non-symlink path", - ) - commands = ( - ( - str(self._docker_path), - "version", - "--format", - "{{json .Server}}", - ), - ( - str(self._docker_path), - "image", - "inspect", - OCI_IMAGE_REFERENCE_V1, - ), - ) - outputs: list[bytes] = [] - for index, command in enumerate(commands): - result = self._observe_command( - command, - stdout_limit=DOCKER_PROBE_OUTPUT_LIMIT_V1, - stderr_limit=DOCKER_PROBE_OUTPUT_LIMIT_V1, - timeout_ns=DOCKER_PROBE_TIMEOUT_NS_V1, - cid_file=None, - ) - if ( - type(result) is not _DockerCommandExitedV1 - or result.returncode != 0 - or not result.stdout - or result.stderr - ): - return DockerUnsupportedV1( - DockerBlockerReasonV1.DOCKER_UNAVAILABLE - if index == 0 - else DockerBlockerReasonV1.IMAGE_UNAVAILABLE, - "Docker daemon probe failed" - if index == 0 - else "pinned image is not locally inspectable", - ) - outputs.append(result.stdout) - try: - inspected = json.loads(outputs[1]) - if type(inspected) is not list or len(inspected) != 1: - raise ValueError("wrong image inspection cardinality") - image = inspected[0] - if type(image) is not dict: - raise ValueError("wrong image inspection shape") - repo_digests = image.get("RepoDigests") - if ( - image.get("Os") != "linux" - or image.get("Architecture") not in ("amd64", "x86_64") - or type(repo_digests) is not list - or OCI_IMAGE_REFERENCE_V1 not in repo_digests - ): - raise ValueError("foreign image coordinate") - except (ValueError, TypeError, json.JSONDecodeError): - return DockerUnsupportedV1( - DockerBlockerReasonV1.IMAGE_IDENTITY_MISMATCH, - "local image does not match pinned linux/amd64 manifest", - ) - daemon_digest = _identity( - b"labcolors.proof-region.docker-daemon-observation.v1\0", - tuple(outputs), - ) - return DockerSupportedV1( - OCI_IMAGE_REFERENCE_V1, - OCI_PLATFORM_V1, - daemon_digest, - ) - - def command_for(self, request: DockerBuildRequestV1) -> tuple[str, ...]: - if type(request) is not DockerBuildRequestV1: - raise TypeError("request must be DockerBuildRequestV1") - command = [ - str(self._docker_path), - "run", - "--rm", - "--interactive", - "--pull", - "never", - "--platform", - OCI_PLATFORM_V1, - "--network", - "none", - "--read-only", - "--tmpfs", - _BUILD_TMPFS_SPEC_V1, - "--tmpfs", - _BUILD_STATE_TMPFS_SPEC_V1, - "--cap-drop", - "ALL", - "--security-opt", - "no-new-privileges:true", - "--name", - request.container_name, - "--hostname", - "labcolors-arb-build-v1", - "--user", - f"{os.getuid()}:{os.getgid()}", - "--workdir", - "/", - "--cidfile", - str(request.cid_file), - ] - command.extend( - ( - "--entrypoint", - "/usr/bin/env", - OCI_IMAGE_REFERENCE_V1, - "-i", - "PATH=/usr/local/bin:/usr/bin:/bin", - "LC_ALL=C", - "LANG=C", - "TZ=UTC", - "HOME=/nonexistent", - "/bin/sh", - "-c", - _BUILD_BOOTSTRAP_V1, - "labcolors-arb-build-bootstrap-v1", - str(request.input_bundle.length), - request.input_bundle.sha256.hex(), - ) - ) - return tuple(command) - - def run_build( - self, - request: DockerBuildRequestV1, - ) -> DockerBuildProcessObservationV1: - if type(request) is not DockerBuildRequestV1: - raise TypeError("request must be DockerBuildRequestV1") - return self._observe_command( - self.command_for(request), - stdout_limit=request.max_executable_bytes, - stderr_limit=BUILD_STDERR_LIMIT_V1, - timeout_ns=BUILD_TIMEOUT_NS_V1, - cid_file=request.cid_file, - container_name=request.container_name, - input_bundle=request.input_bundle, - ) - - def _observe_command( - self, - command: tuple[str, ...], - *, - stdout_limit: int, - stderr_limit: int, - timeout_ns: int, - cid_file: Path | None, - container_name: str | None = None, - input_bundle: SealedBuildInputBundleV1 | None = None, - ) -> _DockerCommandObservationV1: - if ( - type(command) is not tuple - or not command - or any(type(item) is not str or not item or "\0" in item for item in command) - ): - raise TypeError("command must be a nonempty string tuple") - if ( - type(stdout_limit) is not int - or stdout_limit <= 0 - or stdout_limit > BUILD_STDOUT_LIMIT_V1 - or type(stderr_limit) is not int - or stderr_limit <= 0 - or stderr_limit > BUILD_STDERR_LIMIT_V1 - or type(timeout_ns) is not int - or timeout_ns <= 0 - or timeout_ns > BUILD_TIMEOUT_NS_V1 - ): - raise TypeError("invalid Docker observation limits") - if (cid_file is None) != (container_name is None): - raise TypeError("Docker cleanup requires both CID file and exact name") - if cid_file is not None: - _absolute_path(cid_file, "cid_file") - _container_name(container_name) - if input_bundle is not None and type(input_bundle) is not SealedBuildInputBundleV1: - raise TypeError("input_bundle must be controller sealed") - if input_bundle is not None and not sealed_build_input_bundle_is_well_bound_v1( - input_bundle - ): - return DockerBuildObserverFailureV1("build input bundle is not well bound") - try: - process = subprocess.Popen( - command, - stdin=subprocess.PIPE if input_bundle is not None else subprocess.DEVNULL, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - cwd="/", - env=self._environment(), - close_fds=True, - start_new_session=True, - ) - except OSError: - return DockerBuildObserverFailureV1("cannot start Docker CLI") - if ( - process.stdout is None - or process.stderr is None - or (input_bundle is not None and process.stdin is None) - ): - input_progress = ( - _build_input_progress_v1( - input_bundle, - 0, - hashlib.sha256(b"").digest(), - ) - if input_bundle is not None - else None - ) - stop_detail = self._stop_process(process) - cleanup_detail = ( - self._cleanup_container(cid_file, container_name) - if cid_file is not None and container_name is not None - else None - ) - if stop_detail is not None or cleanup_detail is not None: - return DockerBuildCleanupFailureV1( - DockerCleanupTriggerV1.OBSERVER_FAILURE, - stop_detail or cleanup_detail or "Docker cleanup failed", - b"", - b"", - input_progress, - ) - return DockerBuildObserverFailureV1("Docker pipes unavailable") - - stdout = bytearray() - stderr = bytearray() - selector = selectors.DefaultSelector() - terminal: DockerOutputStreamV1 | None = None - timed_out = False - observer_failed = False - input_failed = False - written = 0 - input_hasher = hashlib.sha256() - bundle_view = ( - memoryview(input_bundle._contents) if input_bundle is not None else None - ) - try: - streams = ( - ( - process.stdout.fileno(), - DockerOutputStreamV1.STDOUT, - stdout, - stdout_limit, - ), - ( - process.stderr.fileno(), - DockerOutputStreamV1.STDERR, - stderr, - stderr_limit, - ), - ) - for descriptor, stream, target, maximum in streams: - os.set_blocking(descriptor, False) - selector.register( - descriptor, - selectors.EVENT_READ, - ("read", stream, target, maximum), - ) - if process.stdin is not None: - input_descriptor = process.stdin.fileno() - os.set_blocking(input_descriptor, False) - selector.register( - input_descriptor, - selectors.EVENT_WRITE, - ("write",), - ) - start = self._clock() - deadline = start + timeout_ns - while selector.get_map() or process.poll() is None: - now = self._clock() - if now >= deadline: - timed_out = True - break - timeout = min((deadline - now) / 1_000_000_000, 0.1) - for key, _events in selector.select(timeout): - if key.data[0] == "read": - _kind, stream, target, maximum = key.data - try: - chunk = os.read( - key.fd, - min(64 * 1024, maximum + 1 - len(target)), - ) - except BlockingIOError: - continue - if not chunk: - selector.unregister(key.fd) - continue - target.extend(chunk) - if len(target) > maximum: - del target[maximum:] - terminal = stream - break - continue - if input_bundle is None or bundle_view is None: - observer_failed = True - break - try: - count = os.write( - key.fd, - bundle_view[written : written + 64 * 1024], - ) - except BlockingIOError: - continue - except BrokenPipeError: - input_failed = True - break - if count <= 0: - input_failed = True - break - input_hasher.update(bundle_view[written : written + count]) - written += count - if written == input_bundle.length: - selector.unregister(key.fd) - if process.stdin is not None: - process.stdin.close() - if terminal is not None or input_failed or observer_failed: - break - except Exception: - observer_failed = True - finally: - try: - try: - selector.close() - except OSError: - observer_failed = True - finally: - try: - if process.stdin is not None and not process.stdin.closed: - try: - process.stdin.close() - except OSError: - observer_failed = True - finally: - if bundle_view is not None: - bundle_view.release() - - input_progress: BuildInputTransferProgressV1 | None = None - if input_bundle is not None: - try: - input_progress = _build_input_progress_v1( - input_bundle, - written, - input_hasher.digest(), - ) - except Exception: - observer_failed = True - - stop_detail: str | None = None - if ( - timed_out - or terminal is not None - or observer_failed - or input_failed - ): - stop_detail = self._stop_process(process) - elif process.poll() is None: - timed_out = True - stop_detail = self._stop_process(process) - process.stdout.close() - process.stderr.close() - cleanup_detail = ( - self._cleanup_container(cid_file, container_name) - if cid_file is not None and container_name is not None - else None - ) - if stop_detail is not None or cleanup_detail is not None: - trigger = DockerCleanupTriggerV1.PROCESS_EXIT - if observer_failed: - trigger = DockerCleanupTriggerV1.OBSERVER_FAILURE - elif terminal is not None: - trigger = DockerCleanupTriggerV1.OUTPUT_LIMIT - elif timed_out: - trigger = DockerCleanupTriggerV1.TIMEOUT - elif input_failed: - trigger = DockerCleanupTriggerV1.INPUT_TRANSFER - return DockerBuildCleanupFailureV1( - trigger, - stop_detail or cleanup_detail or "Docker cleanup failed", - bytes(stdout), - bytes(stderr), - input_progress, - ) - if input_failed: - if input_progress is None: - return DockerBuildObserverFailureV1( - "build input progress could not be retained" - ) - return DockerBuildInputRejectedV1( - input_progress, - bytes(stdout), - bytes(stderr), - ) - if observer_failed: - return DockerBuildObserverFailureV1("Docker output observation failed") - if terminal is not None: - return DockerBuildOutputLimitV1( - terminal, - bytes(stdout), - bytes(stderr), - input_progress, - ) - if timed_out: - return DockerBuildTimedOutV1( - bytes(stdout), - bytes(stderr), - input_progress, - ) - if type(process.returncode) is not int: - return DockerBuildObserverFailureV1("Docker returncode unavailable") - if input_bundle is not None: - if ( - input_progress is None - or written != input_bundle.length - or input_hasher.digest() != input_bundle.sha256 - ): - return DockerBuildObserverFailureV1( - "completed build input transfer invariant failed" - ) - input_transfer = _completed_build_input_transfer_v1( - input_bundle, - written, - input_hasher.digest(), - ) - return _docker_build_exited_v1( - process.returncode, - bytes(stdout), - bytes(stderr), - input_transfer, - ) - return _docker_command_exited_v1( - process.returncode, - bytes(stdout), - bytes(stderr), - ) - - def _clock(self) -> int: - value = self._monotonic_ns() - if type(value) is not int or value < 0: - raise RuntimeError("invalid monotonic clock") - return value - - def _stop_process( - self, - process: subprocess.Popen[bytes], - ) -> str | None: - failed = False - try: - os.killpg(process.pid, signal.SIGKILL) - except ProcessLookupError: - pass - except OSError: - try: - process.kill() - except ProcessLookupError: - pass - except OSError: - failed = True - try: - process.wait(timeout=30) - except subprocess.TimeoutExpired: - failed = True - if process.poll() is None: - failed = True - return "Docker CLI process could not be terminated" if failed else None - - @staticmethod - def _admitted_container_id(cid_file: Path) -> str | None: - try: - descriptor = os.open( - cid_file, - os.O_RDONLY - | getattr(os, "O_CLOEXEC", 0) - | getattr(os, "O_NOFOLLOW", 0), - ) - except OSError: - return None - try: - metadata = os.fstat(descriptor) - if ( - not stat.S_ISREG(metadata.st_mode) - or metadata.st_nlink != 1 - or metadata.st_size not in (64, 65) - ): - return None - raw = os.read(descriptor, 66) - except OSError: - return None - finally: - os.close(descriptor) - if len(raw) == 65 and raw.endswith(b"\n"): - raw = raw[:-1] - if len(raw) != 64 or any( - byte not in b"0123456789abcdef" for byte in raw - ): - return None - return raw.decode("ascii") - - def _observe_cleanup_command( - self, - command: tuple[str, ...], - ) -> _DockerCommandObservationV1: - return self._observe_command( - command, - stdout_limit=DOCKER_PROBE_OUTPUT_LIMIT_V1, - stderr_limit=DOCKER_PROBE_OUTPUT_LIMIT_V1, - timeout_ns=DOCKER_PROBE_TIMEOUT_NS_V1, - cid_file=None, - ) - - def _cleanup_container(self, cid_file: Path, container_name: str) -> str | None: - _absolute_path(cid_file, "cid_file") - _container_name(container_name) - container_id = self._admitted_container_id(cid_file) - removal_coordinates = ( - (container_id, container_name) - if container_id is not None - else (container_name,) - ) - try: - for coordinate in removal_coordinates: - self._observe_cleanup_command( - ( - str(self._docker_path), - "container", - "rm", - "--force", - coordinate, - ) - ) - filters = [f"name=^/{container_name}$"] - if container_id is not None: - filters.append(f"id={container_id}") - for filter_value in filters: - observation = self._observe_cleanup_command( - ( - str(self._docker_path), - "container", - "ls", - "--all", - "--quiet", - "--no-trunc", - "--filter", - filter_value, - ) - ) - if ( - type(observation) is not _DockerCommandExitedV1 - or observation.returncode != 0 - or observation.stdout - or observation.stderr - ): - return "Docker container absence could not be verified" - except Exception: - return "Docker container cleanup observer raised" - return None - - -class BuildFailureReasonV1(StrEnum): - BACKEND_CONTRACT = "backend_contract" - PROCESS_FAILED = "process_failed" - CLEANUP_FAILED = "cleanup_failed" - INPUT_TRANSFER_FAILED = "input_transfer_failed" - INVALID_OUTPUT = "invalid_output" - - @dataclass(frozen=True) class PipelineBlockedV1: - reason: DockerBlockerReasonV1 + reason: build_transport.DockerBlockerReasonV1 detail: str -@dataclass(frozen=True) -class BuildRejectedV1: - attempt: int - reason: BuildFailureReasonV1 - process: DockerBuildProcessObservationV1 | None = None - - -@dataclass(frozen=True) -class NonReproducibleBuildV1: - first_sha256: bytes - second_sha256: bytes - - class ExecutionFailureReasonV1(StrEnum): UNSUPPORTED = "unsupported" PROCESS_FAILED = "process_failed" @@ -2322,20 +1339,21 @@ class DiagnosticBuildObservationV1: build_input_identity: bytes formula_support_identity: bytes pipeline_policy_identity: bytes - docker_daemon_observation_sha256: bytes - oci_image_reference: str - oci_platform: str + docker_capability: build_transport.DockerSupportedV1 binary_sha256: bytes rebuild_sha256s: tuple[bytes, bytes] host_trust: HostTrustBoundaryV1 input_bundle_identity: bytes input_bundle_sha256: bytes input_bundle_length: int - build_processes: tuple[DockerBuildExitedV1, DockerBuildExitedV1] + build_processes: tuple[ + build_transport.DockerBuildExitedV1, + build_transport.DockerBuildExitedV1, + ] comparator: DiagnosticArbComparatorV1 _binary: bytes _rebuild_binaries: tuple[bytes, bytes] - _input_bundle: SealedBuildInputBundleV1 + _input_bundle: build_input.SealedInputV1 def __init__( self, @@ -2347,19 +1365,20 @@ def __init__( build_input_identity: bytes, formula_support_identity: bytes, pipeline_policy_identity: bytes, - docker_daemon_observation_sha256: bytes, - oci_image_reference: str, - oci_platform: str, + docker_capability: build_transport.DockerSupportedV1, binary_sha256: bytes, rebuild_sha256s: tuple[bytes, bytes], host_trust: HostTrustBoundaryV1, input_bundle_identity: bytes, input_bundle_sha256: bytes, input_bundle_length: int, - build_processes: tuple[DockerBuildExitedV1, DockerBuildExitedV1], + build_processes: tuple[ + build_transport.DockerBuildExitedV1, + build_transport.DockerBuildExitedV1, + ], comparator: DiagnosticArbComparatorV1, rebuild_binaries: tuple[bytes, bytes], - input_bundle: SealedBuildInputBundleV1, + input_bundle: build_input.SealedInputV1, *, _token: object, ) -> None: @@ -2375,7 +1394,6 @@ def __init__( ("build_input_identity", build_input_identity), ("formula_support_identity", formula_support_identity), ("pipeline_policy_identity", pipeline_policy_identity), - ("docker_daemon_observation_sha256", docker_daemon_observation_sha256), ("binary_sha256", binary_sha256), ("input_bundle_identity", input_bundle_identity), ("input_bundle_sha256", input_bundle_sha256), @@ -2389,34 +1407,50 @@ def __init__( or flint_project_pinned_release_only_file_count <= 0 ): raise TypeError("FLINT source partition must be nonempty") - if oci_image_reference != OCI_IMAGE_REFERENCE_V1 or oci_platform != OCI_PLATFORM_V1: - raise TypeError("diagnostic build does not bind the pinned OCI manifest/platform") + if type(docker_capability) is not build_transport.DockerSupportedV1: + raise TypeError("diagnostic build requires DockerSupportedV1") + canonical_capability = build_transport.DockerSupportedV1( + *tuple(docker_capability) + ) + if tuple(canonical_capability) != tuple(docker_capability): + raise TypeError("diagnostic build does not bind the exact Arb capability") if ( type(rebuild_sha256s) is not tuple or len(rebuild_sha256s) != 2 or any(not _valid_digest(item) for item in rebuild_sha256s) or rebuild_sha256s != (binary_sha256, binary_sha256) ): - raise TypeError("invalid reproducible-build digests") + raise TypeError("invalid observed two-build digests") if type(host_trust) is not HostTrustBoundaryV1: raise TypeError("invalid host trust boundary") if type(input_bundle_length) is not int or input_bundle_length <= 0: raise TypeError("invalid build input bundle length") if ( - not sealed_build_input_bundle_is_well_bound_v1(input_bundle) - or input_bundle.identity != input_bundle_identity + not build_input.sealed_input_is_intact_v1(input_bundle) + or input_bundle.binding_identity != input_bundle_identity or input_bundle.sha256 != input_bundle_sha256 or input_bundle.length != input_bundle_length - or input_bundle.build_input_identity != build_input_identity - or input_bundle.source_identity != structural_source_identity + or input_bundle.binding_identity + != _arb_input_binding_identity_v2( + structural_source_identity, + build_input_identity, + input_bundle.contents, + canonical_capability.policy, + ) ): raise TypeError("diagnostic build lost its sealed input bundle") - if pipeline_policy_identity != pipeline_policy_identity_v1(host_trust): + if pipeline_policy_identity != pipeline_policy_identity_v2( + host_trust, + canonical_capability.policy, + ): raise TypeError("pipeline policy is not the fixed diagnostic policy") if ( type(build_processes) is not tuple or len(build_processes) != 2 - or any(type(item) is not DockerBuildExitedV1 for item in build_processes) + or any( + type(item) is not build_transport.DockerBuildExitedV1 + for item in build_processes + ) or any(item.returncode != 0 for item in build_processes) ): raise TypeError("invalid build process observations") @@ -2464,12 +1498,7 @@ def __init__( ("build_input_identity", build_input_identity), ("formula_support_identity", formula_support_identity), ("pipeline_policy_identity", pipeline_policy_identity), - ( - "docker_daemon_observation_sha256", - docker_daemon_observation_sha256, - ), - ("oci_image_reference", oci_image_reference), - ("oci_platform", oci_platform), + ("docker_capability", docker_capability), ("binary_sha256", binary_sha256), ("rebuild_sha256s", rebuild_sha256s), ("host_trust", host_trust), @@ -2493,23 +1522,38 @@ def rebuild_binaries(self) -> tuple[bytes, bytes]: return self._rebuild_binaries @property - def input_transfers(self) -> tuple[BuildInputTransferV1, BuildInputTransferV1]: + def input_transfers( + self, + ) -> tuple[ + build_transport.BuildInputTransferV1, + build_transport.BuildInputTransferV1, + ]: first = self.build_processes[0].input_transfer second = self.build_processes[1].input_transfer - if type(first) is not BuildInputTransferV1 or type(second) is not BuildInputTransferV1: + if ( + type(first) is not build_transport.BuildInputTransferV1 + or type(second) is not build_transport.BuildInputTransferV1 + ): raise RuntimeError("sealed build observation lost its input transfer") return first, second @property - def input_bundle(self) -> SealedBuildInputBundleV1: + def input_bundle(self) -> build_input.SealedInputV1: + if ( + not build_input.sealed_input_is_intact_v1(self._input_bundle) + or self._input_bundle.binding_identity != self.input_bundle_identity + or self._input_bundle.sha256 != self.input_bundle_sha256 + or self._input_bundle.length != self.input_bundle_length + ): + raise RuntimeError("diagnostic build lost its exact input bytes") return self._input_bundle BuildResultV1: TypeAlias = ( DiagnosticBuildObservationV1 | PipelineBlockedV1 - | BuildRejectedV1 - | NonReproducibleBuildV1 + | build_transport.BuildRejectedV1 + | build_transport.TwoBuildObservationV1 ) @@ -2517,32 +1561,40 @@ class ControlledPipelineV1: def __init__( self, *, - build_backend: DockerBuildBackendV1, + build_backend: build_transport.DockerBuildBackendV1, ) -> None: - self._build_backend = build_backend + self._transport = build_transport.ControlledBuildTransportV1( + policy=ARB_BUILD_TRANSPORT_POLICY_V1, + backend=build_backend, + ) + + @staticmethod + def _admit_arb_output_v1(binary: bytes) -> bool: + try: + executor.require_static_x86_64_elf_v1(binary) + except executor.ExecutionRequestErrorV1: + return False + return True def build(self, request: PipelineRequestV1) -> BuildResultV1: """Observe two fresh equal builds without requiring a RUN capability.""" if type(request) is not PipelineRequestV1: raise PipelineInputErrorV1(PipelineInputReasonV1.WRONG_TYPE, "request") - try: - docker_report = self._build_backend.probe() - except Exception: - return PipelineBlockedV1( - DockerBlockerReasonV1.BACKEND_CONTRACT, - "Docker capability probe raised", - ) - if type(docker_report) is DockerUnsupportedV1: - return PipelineBlockedV1(docker_report.reason, docker_report.detail) - if type(docker_report) is not DockerSupportedV1: + probe_result = self._transport.probe() + if type(probe_result) is build_transport.DockerUnsupportedV1: + return PipelineBlockedV1(probe_result.reason, probe_result.detail) + if type(probe_result) is not build_transport.DockerSupportedV1: return PipelineBlockedV1( - DockerBlockerReasonV1.BACKEND_CONTRACT, + build_transport.DockerBlockerReasonV1.BACKEND_CONTRACT, "Docker capability report is not typed", ) - + docker_capability = probe_result try: - input_bundle = _seal_build_input_bundle_v1(request) + input_bundle = _seal_build_input_bundle_v1( + request, + docker_capability.policy, + ) except ( OSError, TypeError, @@ -2550,29 +1602,54 @@ def build(self, request: PipelineRequestV1) -> BuildResultV1: tarfile.TarError, BuildSourceAdmissionErrorV1, provenance.ProvenanceErrorV1, + build_input.InputErrorV1, + ): + return build_transport.BuildRejectedV1( + 1, + build_transport.BuildFailureReasonV1.CONTRACT_VIOLATION, + ) + built = self._transport.build( + docker_capability, + input_bundle, + request.execution_limits.max_executable_bytes, + input_admission=lambda value: arb_input_is_bound_v1( + request, + docker_capability.policy, + value, + ), + output_admission=self._admit_arb_output_v1, + ) + if type(built) is build_transport.BuildRejectedV1: + return built + if type(built) is not build_transport.TwoBuildObservationV1: + return build_transport.BuildRejectedV1( + 1, + build_transport.BuildFailureReasonV1.CONTRACT_VIOLATION, + ) + if not build_transport.two_build_observation_matches_v1( + built, + built.session, ): - return BuildRejectedV1( + return build_transport.BuildRejectedV1( + 1, + build_transport.BuildFailureReasonV1.CONTRACT_VIOLATION, + ) + if built.relation is build_transport.BuildByteRelationV1.DIFFERENT: + return built + if built.relation is not build_transport.BuildByteRelationV1.IDENTICAL: + return build_transport.BuildRejectedV1( 1, - BuildFailureReasonV1.BACKEND_CONTRACT, + build_transport.BuildFailureReasonV1.CONTRACT_VIOLATION, ) - builds: list[tuple[bytes, DockerBuildExitedV1]] = [] - for attempt in (1, 2): - built = self._build_once(request, attempt, input_bundle) - if type(built) is BuildRejectedV1: - return built - builds.append(built) - first, second = builds - first_digest = hashlib.sha256(first[0]).digest() - second_digest = hashlib.sha256(second[0]).digest() - if first[0] != second[0]: - return NonReproducibleBuildV1(first_digest, second_digest) - - binary = first[0] - rebuild_sha256s = (first_digest, second_digest) - build_processes = (first[1], second[1]) - comparator = _derive_arb_comparator_for_build_v1( + + binary = built.outputs[0] + rebuild_sha256s = tuple( + hashlib.sha256(item).digest() for item in built.outputs + ) + build_processes = built.processes + comparator = derive_arb_comparator_for_build_v1( request, - docker_report, + docker_capability, binary, rebuild_sha256s, build_processes, @@ -2589,116 +1666,20 @@ def build(self, request: PipelineRequestV1) -> BuildResultV1: flint_partition.project_pinned_release_only_file_count, request.build_sources.build_input_identity, request.build_sources.formula_support_identity, - pipeline_policy_identity_v1(request.host_trust), - docker_report.daemon_observation_sha256, - docker_report.image_reference, - docker_report.platform, - first_digest, + pipeline_policy_identity_v2( + request.host_trust, + docker_capability.policy, + ), + docker_capability, + rebuild_sha256s[0], rebuild_sha256s, request.host_trust, - input_bundle.identity, + input_bundle.binding_identity, input_bundle.sha256, input_bundle.length, build_processes, comparator, - (first[0], second[0]), + built.outputs, input_bundle, _token=_BUILD_OBSERVATION_TOKEN, ) - - def _build_once( - self, - request: PipelineRequestV1, - attempt: int, - input_bundle: SealedBuildInputBundleV1, - ) -> tuple[bytes, DockerBuildExitedV1] | BuildRejectedV1: - if ( - not sealed_build_input_bundle_is_well_bound_v1(input_bundle) - or input_bundle.source_identity != request.admitted_sources.identity - or input_bundle.build_input_identity - != request.build_sources.build_input_identity - ): - return BuildRejectedV1( - attempt, - BuildFailureReasonV1.BACKEND_CONTRACT, - ) - try: - with tempfile.TemporaryDirectory(prefix=f"labcolors-arb-build-v1-{attempt}-") as temporary: - root = Path(temporary).resolve() - build_request = DockerBuildRequestV1( - attempt, - input_bundle, - request.execution_limits.max_executable_bytes, - root / "container.cid", - _CONTAINER_NAME_PREFIX_V1 - + hashlib.sha256( - os.fsencode(root) + bytes((attempt,)) - ).hexdigest(), - ) - try: - process = self._build_backend.run_build(build_request) - except Exception: - return BuildRejectedV1( - attempt, - BuildFailureReasonV1.BACKEND_CONTRACT, - ) - known_process_types = ( - DockerBuildExitedV1, - DockerBuildTimedOutV1, - DockerBuildOutputLimitV1, - DockerBuildObserverFailureV1, - DockerBuildInputRejectedV1, - DockerBuildCleanupFailureV1, - ) - if type(process) not in known_process_types: - return BuildRejectedV1( - attempt, - BuildFailureReasonV1.BACKEND_CONTRACT, - ) - if type(process) is DockerBuildCleanupFailureV1: - return BuildRejectedV1( - attempt, - BuildFailureReasonV1.CLEANUP_FAILED, - process, - ) - if type(process) is DockerBuildInputRejectedV1: - return BuildRejectedV1( - attempt, - BuildFailureReasonV1.INPUT_TRANSFER_FAILED, - process, - ) - if type(process) is not DockerBuildExitedV1 or process.returncode != 0: - return BuildRejectedV1( - attempt, - BuildFailureReasonV1.PROCESS_FAILED, - process, - ) - transfer = process.input_transfer - if ( - type(transfer) is not BuildInputTransferV1 - or transfer.bundle_identity != input_bundle.identity - or transfer.expected_length != input_bundle.length - or transfer.expected_sha256 != input_bundle.sha256 - or transfer.written_length != input_bundle.length - or transfer.written_sha256 != input_bundle.sha256 - ): - return BuildRejectedV1( - attempt, - BuildFailureReasonV1.BACKEND_CONTRACT, - process, - ) - binary = process.stdout - try: - executor.require_static_x86_64_elf_v1(binary) - except executor.ExecutionRequestErrorV1: - return BuildRejectedV1( - attempt, - BuildFailureReasonV1.INVALID_OUTPUT, - process, - ) - return binary, process - except OSError: - return BuildRejectedV1( - attempt, - BuildFailureReasonV1.BACKEND_CONTRACT, - ) diff --git a/proof/region/v1/arb/receipt.py b/proof/region/v1/arb/receipt.py index d9ac938e..689a37df 100644 --- a/proof/region/v1/arb/receipt.py +++ b/proof/region/v1/arb/receipt.py @@ -17,6 +17,8 @@ from pathlib import Path from typing import TypeAlias +from build import transport as build_transport + import executor import pipeline import provenance @@ -25,15 +27,15 @@ _EVIDENCE_TOKEN = object() _RECEIPT_TOKEN = object() -_NATIVE_BUILD_BACKEND_TYPE = pipeline.NativeDockerBuildBackendV1 +_NATIVE_BUILD_BACKEND_TYPE = build_transport.NativeDockerBuildBackendV1 _NATIVE_RUN_BACKEND_TYPE = executor.NativeLinuxBackendV1 _SOURCE_ID_LABEL_V1 = b"labcolors.proof-region.arb-source-replay.v1\0" -_BUILD_ID_LABEL_V1 = b"labcolors.proof-region.arb-build-replay.v1\0" +_BUILD_ID_LABEL_V2 = b"labcolors.proof-region.arb-build-replay.v2\0" _RUN_ID_LABEL_V1 = b"labcolors.proof-region.arb-run-replay.v1\0" _EVIDENCE_ID_LABEL_V1 = b"labcolors.proof-region.arb-evaluator-replay.v1\0" -_SOURCE_BOUND_POLICY_ID_LABEL_V1 = ( - b"labcolors.proof-region.arb-source-bound-policy.v1\0" +_SOURCE_BOUND_POLICY_ID_LABEL_V2 = ( + b"labcolors.proof-region.arb-source-bound-policy.v2\0" ) @@ -46,23 +48,30 @@ def _identity(label: bytes, chunks: tuple[bytes, ...]) -> bytes: return hashlib.sha256(label + len(payload).to_bytes(8, "big") + payload).digest() -def source_bound_policy_identity_v1() -> bytes: - """Identity of the exact V1 observation rules and trust boundary.""" +def source_bound_policy_identity_v2( + capability: build_transport.DockerSupportedV1, + host_trust: pipeline.HostTrustBoundaryV1, +) -> bytes: + """Identity of the exact observation rules and observed BUILD capability.""" + + capability_identity = build_transport.docker_capability_identity_v1(capability) return _identity( - _SOURCE_BOUND_POLICY_ID_LABEL_V1, + _SOURCE_BOUND_POLICY_ID_LABEL_V2, ( - pipeline.pipeline_policy_identity_v1( - pipeline.HostTrustBoundaryV1.UNSEALED_LINUX_X64_DOCKER_HOST + pipeline.pipeline_policy_identity_v2( + host_trust, + capability.policy, ), + capability_identity, executor.SANDBOX_POLICY_RELEASE_V1.encode("ascii"), b"authority=one-shot-native-controller", b"source=lock-plus-owned-archive-and-build-input-replay", b"build=one-sealed-bundle-two-fresh-byte-equal-attempts", b"run=retained-executable-object-one-contained-process", - b"identity=immutable-coordinates-total-rejection-v1", + b"identity=immutable-coordinates-total-rejection-v2", b"claim=provenance-only-no-numerical-semantics", - b"trust=unsealed-linux-x64-host-and-docker-daemon", + b"trust=unsealed-linux-x64-host-native-docker-cli-and-daemon", ), ) @@ -106,6 +115,33 @@ def _comparator_replays_v1( ) -> bool: try: comparator = build.comparator + capability_identity = build_transport.docker_capability_identity_v1( + build.docker_capability + ) + expected_pipeline_policy = pipeline.pipeline_policy_identity_v2( + request.host_trust, + build.docker_capability.policy, + ) + expected_build_preimage = pipeline.comparator_build_preimage_v2( + request.build_sources, + capability_identity, + expected_pipeline_policy, + build.build_processes, + build.binary_sha256, + build.rebuild_sha256s, + len(build.binary), + ) + # Re-derive every named comparator coordinate from the retained + # request and BUILD/RUN observation. Re-hashing a supplied comparator + # only proves internal consistency; it cannot prove that its wrapper, + # evaluator, or test-observation coordinates describe this DAG. + expected_comparator = pipeline.derive_arb_comparator_for_build_v1( + request, + build.docker_capability, + build.binary, + build.rebuild_sha256s, + build.build_processes, + ) if ( type(comparator) is not pipeline.DiagnosticArbComparatorV1 or comparator.structural_source_identity @@ -113,6 +149,12 @@ def _comparator_replays_v1( or comparator.build_input_identity != request.build_sources.build_input_identity or comparator.pipeline_policy_identity != build.pipeline_policy_identity + or comparator.pipeline_policy_identity != expected_pipeline_policy + or comparator.preimages.build_identity != expected_build_preimage + or comparator.preimages != expected_comparator.preimages + or comparator.manifest.manifest != expected_comparator.manifest.manifest + or comparator.manifest.identity != expected_comparator.manifest.identity + or comparator.identity != expected_comparator.identity or comparator.binary_sha256 != build.binary_sha256 or comparator.rebuild_sha256s != build.rebuild_sha256s ): @@ -152,7 +194,7 @@ def _comparator_replays_v1( return False -def _build_identity_v1( +def _build_identity_v2( request: pipeline.PipelineRequestV1, source_identity: bytes, build: pipeline.DiagnosticBuildObservationV1, @@ -162,6 +204,9 @@ def _build_identity_v1( bundle = build.input_bundle processes = build.build_processes binaries = build.rebuild_binaries + capability_identity = build_transport.docker_capability_identity_v1( + build.docker_capability + ) flint_partition = pipeline.flint_source_content_partition_v1( request.source_lock, request.admitted_sources, @@ -180,15 +225,25 @@ def _build_identity_v1( or build.formula_support_identity != request.build_sources.formula_support_identity or build.pipeline_policy_identity - != pipeline.pipeline_policy_identity_v1(request.host_trust) + != pipeline.pipeline_policy_identity_v2( + request.host_trust, + build.docker_capability.policy, + ) or build.host_trust is not request.host_trust - or not pipeline.sealed_build_input_bundle_is_well_bound_v1(bundle) - or build.input_bundle_identity != bundle.identity + or not pipeline.arb_input_is_bound_v1( + request, + build.docker_capability.policy, + bundle, + ) + or build.input_bundle_identity != bundle.binding_identity or build.input_bundle_sha256 != bundle.sha256 or build.input_bundle_length != bundle.length or type(processes) is not tuple or len(processes) != 2 - or any(type(item) is not pipeline.DockerBuildExitedV1 for item in processes) + or any( + type(item) is not build_transport.DockerBuildExitedV1 + for item in processes + ) or type(binaries) is not tuple or len(binaries) != 2 or binaries[0] is not processes[0].stdout @@ -204,8 +259,8 @@ def _build_identity_v1( transfer = process.input_transfer if ( process.returncode != 0 - or type(transfer) is not pipeline.BuildInputTransferV1 - or transfer.bundle_identity != bundle.identity + or type(transfer) is not build_transport.BuildInputTransferV1 + or transfer.bundle_identity != bundle.binding_identity or transfer.expected_length != bundle.length or transfer.expected_sha256 != bundle.sha256 or transfer.written_length != bundle.length @@ -213,19 +268,17 @@ def _build_identity_v1( ): raise TypeError("BUILD transfer did not consume the sealed bundle") return _identity( - _BUILD_ID_LABEL_V1, + _BUILD_ID_LABEL_V2, ( source_identity, build.pipeline_policy_identity, build.host_trust.value.encode("ascii"), - build.docker_daemon_observation_sha256, - build.oci_image_reference.encode("ascii"), - build.oci_platform.encode("ascii"), - bundle.identity, + capability_identity, + bundle.binding_identity, bundle.sha256, bundle.length.to_bytes(8, "big"), - pipeline.build_process_bytes_v1(processes[0]), - pipeline.build_process_bytes_v1(processes[1]), + build_transport.build_process_bytes_v1(processes[0]), + build_transport.build_process_bytes_v1(processes[1]), build.binary_sha256, len(build.binary).to_bytes(8, "big"), build.comparator.identity, @@ -365,7 +418,7 @@ def __init__( if _token is not _EVIDENCE_TOKEN: raise TypeError("ContentResolvedEvaluatorReplayV1 is controller-derived") source_identity = _source_identity_v1(request) - build_identity = _build_identity_v1(request, source_identity, build) + build_identity = _build_identity_v2(request, source_identity, build) run_identity = _run_identity_v1( request, build, @@ -409,7 +462,7 @@ def replay_evidence_is_well_bound_v1(value: object) -> bool: if type(value) is not ContentResolvedEvaluatorReplayV1: return False source_identity = _source_identity_v1(value.request) - build_identity = _build_identity_v1( + build_identity = _build_identity_v2( value.request, source_identity, value.build, @@ -470,7 +523,11 @@ def __init__( ): raise TypeError("SourceBoundEvaluatorReceiptV1 is controller-sealed") if ( - claim.provenance_policy_identity != source_bound_policy_identity_v1() + claim.provenance_policy_identity + != source_bound_policy_identity_v2( + evidence.build.docker_capability, + evidence.request.host_trust, + ) or claim.run_claim_identity != evidence.run_claim.identity or claim.replay_evidence_identity != evidence.identity ): @@ -524,8 +581,8 @@ def __post_init__(self) -> None: SourceBoundEvaluatorReceiptV1 | SourceBoundRejectedV1 | pipeline.PipelineBlockedV1 - | pipeline.BuildRejectedV1 - | pipeline.NonReproducibleBuildV1 + | build_transport.BuildRejectedV1 + | build_transport.TwoBuildObservationV1 | pipeline.ExecutionRejectedV1 | pipeline.TranscriptRejectedV1 ) @@ -631,7 +688,10 @@ def execute(self, request: pipeline.PipelineRequestV1) -> SourceBoundResultV1: "exact source, build input, or job replay failed", ) - build_backend = _NATIVE_BUILD_BACKEND_TYPE(self._docker_path) + build_backend = _NATIVE_BUILD_BACKEND_TYPE( + self._docker_path, + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + ) if type(build_backend) is not _NATIVE_BUILD_BACKEND_TYPE: return SourceBoundRejectedV1( SourceBoundFailureReasonV1.REPLAY_BINDING_FAILED, @@ -761,7 +821,10 @@ def execute(self, request: pipeline.PipelineRequestV1) -> SourceBoundResultV1: _token=_EVIDENCE_TOKEN, ) claim = protocol.EvaluatorProvenanceClaimV1( - source_bound_policy_identity_v1(), + source_bound_policy_identity_v2( + built.docker_capability, + replay_request.host_trust, + ), run_claim.identity, evidence.identity, ) diff --git a/proof/region/v1/arb/tests/gate.py b/proof/region/v1/arb/tests/gate.py index 5c8c2fa5..4cff9a81 100644 --- a/proof/region/v1/arb/tests/gate.py +++ b/proof/region/v1/arb/tests/gate.py @@ -15,8 +15,9 @@ REPO = Path(__file__).resolve().parents[5] sys.path.insert(0, str(REPO)) EXPECTED_TEST_INVENTORY_SHA256 = ( - "9df49e5bc78ab7cf2386570f500c80f3759f21385f597f2720aa27b1e9700a76" + "43c97cac58e82df841be8da440c6c2351c4ab5cde578dd95dac921162f608b11" ) +EXPECTED_TEST_COUNT = 185 _EVALUATOR_REASON = "set LABCOLORS_ARB_EVALUATOR to the controlled C17 binary" EXPECTED_SKIPS = frozenset( { diff --git a/proof/region/v1/arb/tests/test_build_identity_v2.py b/proof/region/v1/arb/tests/test_build_identity_v2.py new file mode 100644 index 00000000..267510da --- /dev/null +++ b/proof/region/v1/arb/tests/test_build_identity_v2.py @@ -0,0 +1,557 @@ +#!/usr/bin/env python3 +"""Causal BUILD policy and capability identity contract.""" + +from __future__ import annotations + +import ast +import hashlib +import inspect +import json +import sys +import tempfile +import unittest +from dataclasses import fields as dataclass_fields +from pathlib import Path +from unittest import mock + + +PROOF = Path(__file__).resolve().parents[2] +ARB = PROOF / "arb" +sys.path[:0] = [str(PROOF), str(ARB), str(ARB / "tests")] + +from build import transport as build_transport # noqa: E402 + +import pipeline # noqa: E402 +import receipt # noqa: E402 +from test_pipeline import ( # noqa: E402 + _BuildBackend, + _docker_capability, + _request, + _static_elf, +) + + +_POLICY_FIELDS = ( + "image_reference", + "platform", + "hostname", + "bootstrap", + "bootstrap_argv0", + "tmpfs_specs", + "user_mode", + "stdout_limit", + "stderr_limit", + "build_timeout_ns", + "probe_output_limit", + "probe_timeout_ns", +) + + +def _called_names(function: object) -> set[str]: + tree = ast.parse(inspect.getsource(function)) + names: set[str] = set() + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + if isinstance(node.func, ast.Attribute): + names.add(node.func.attr) + elif isinstance(node.func, ast.Name): + names.add(node.func.id) + return names + + +def _policy_with( + policy: build_transport.DockerBuildPolicyV1, + **changes: object, +) -> build_transport.DockerBuildPolicyV1: + values = {name: getattr(policy, name) for name in _POLICY_FIELDS} + values.update(changes) + return build_transport.DockerBuildPolicyV1( + *(values[name] for name in _POLICY_FIELDS) + ) + + +def _policy_mutants( + policy: build_transport.DockerBuildPolicyV1, +) -> tuple[tuple[str, build_transport.DockerBuildPolicyV1], ...]: + first_tmpfs, *remaining_tmpfs = policy.tmpfs_specs + tmpfs_parts = first_tmpfs.split(",") + size_index = next( + index for index, part in enumerate(tmpfs_parts) if part.startswith("size=") + ) + size_value = int(tmpfs_parts[size_index].removeprefix("size=")) + tmpfs_parts[size_index] = f"size={size_value - 1}" + changed_tmpfs = ",".join(tmpfs_parts) + numeric_fields = ( + "stdout_limit", + "stderr_limit", + "build_timeout_ns", + "probe_output_limit", + "probe_timeout_ns", + ) + mutants: list[tuple[str, build_transport.DockerBuildPolicyV1]] = [ + ( + "image_reference", + _policy_with( + policy, + image_reference="gcc@sha256:" + "ab" * 32, + ), + ), + ("hostname", _policy_with(policy, hostname="labcolors-build-mutant")), + ("bootstrap", _policy_with(policy, bootstrap=policy.bootstrap + "\n:")), + ( + "bootstrap_argv0", + _policy_with(policy, bootstrap_argv0="labcolors-mutant-bootstrap"), + ), + ( + "tmpfs_specs", + _policy_with( + policy, + tmpfs_specs=(changed_tmpfs, *remaining_tmpfs), + ), + ), + ] + for name in numeric_fields: + value = getattr(policy, name) + mutants.append((name, _policy_with(policy, **{name: value - 1}))) + return tuple(mutants) + + +def _arb_input_binding_oracle_v2( + source_identity: bytes, + build_input_identity: bytes, + contents: bytes, + bootstrap: str, +) -> bytes: + """Independent frozen formula for the V2 Arb input binding.""" + + chunks = ( + source_identity, + build_input_identity, + len(contents).to_bytes(8, "big"), + hashlib.sha256(contents).digest(), + hashlib.sha256(bootstrap.encode("utf-8")).digest(), + ) + payload = b"".join( + len(chunk).to_bytes(8, "big") + chunk for chunk in chunks + ) + return hashlib.sha256( + b"labcolors.proof-region.arb-build-input-bundle.v2\0" + + len(payload).to_bytes(8, "big") + + payload + ).digest() + + +def _capability( + docker_path: Path, + policy: build_transport.DockerBuildPolicyV1, + *, + host_user: tuple[int, int] = (501, 20), + daemon_marker: str = "daemon-a", +) -> build_transport.DockerSupportedV1: + backend = build_transport.NativeDockerBuildBackendV1( + docker_path, + policy, + platform_name="linux", + machine_name="x86_64", + host_user=host_user, + ) + image_observation = json.dumps( + [ + { + "Os": "linux", + "Architecture": "amd64", + "RepoDigests": [policy.image_reference], + } + ], + sort_keys=True, + separators=(",", ":"), + ).encode("ascii") + observations = ( + build_transport._docker_command_exited_v1( + 0, + json.dumps( + {"daemon": daemon_marker}, + sort_keys=True, + separators=(",", ":"), + ).encode("ascii"), + b"", + ), + build_transport._docker_command_exited_v1(0, image_observation, b""), + ) + with mock.patch.object(backend, "_observe_command", side_effect=observations): + capability = backend.probe() + if type(capability) is not build_transport.DockerSupportedV1: + raise AssertionError(capability) + return capability + + +def _observed_build_coordinates( + policy: build_transport.DockerBuildPolicyV1, + capability: build_transport.DockerSupportedV1, +) -> tuple[bytes, bytes, bytes, bytes, tuple[bytes, bytes], bytes, bytes]: + request = _request() + binary = _static_elf(b"identity-v2-invariant-output") + with mock.patch.object(pipeline, "ARB_BUILD_TRANSPORT_POLICY_V1", policy): + result = pipeline.ControlledPipelineV1( + build_backend=_BuildBackend((binary, binary), probe=capability) + ).build(request) + if type(result) is not pipeline.DiagnosticBuildObservationV1: + raise AssertionError(result) + source_identity = receipt._source_identity_v1(request) + receipt_build_identity = receipt._build_identity_v2( + request, + source_identity, + result, + ) + source_bound_policy = receipt.source_bound_policy_identity_v2( + result.docker_capability, + request.host_trust, + ) + process_encodings = tuple( + build_transport.build_process_bytes_v1(process) + for process in result.build_processes + ) + return ( + build_transport.docker_capability_identity_v1(result.docker_capability), + result.comparator.preimages.build_identity, + receipt_build_identity, + source_identity, + process_encodings, + result.input_bundle.contents, + source_bound_policy, + ) + + +class BuildIdentityV2Tests(unittest.TestCase): + def test_v2_surface_replaces_v1_aliases_and_preimage_labels(self) -> None: + self.assertFalse(hasattr(pipeline, "pipeline_policy_identity_v1")) + self.assertFalse(hasattr(receipt, "source_bound_policy_identity_v1")) + self.assertFalse(hasattr(receipt, "_build_identity_v1")) + self.assertTrue(callable(pipeline.pipeline_policy_identity_v2)) + self.assertTrue(callable(receipt.source_bound_policy_identity_v2)) + self.assertTrue(callable(receipt._build_identity_v2)) + + pipeline_source = (ARB / "pipeline.py").read_text(encoding="utf-8") + receipt_source = (ARB / "receipt.py").read_text(encoding="utf-8") + for stale in ( + "labcolors.proof-region.arb-pipeline-policy.v1", + "labcolors.proof-region.arb-comparator.build-identity.v1", + ): + with self.subTest(stale=stale): + self.assertNotIn(stale, pipeline_source) + for stale in ( + "labcolors.proof-region.arb-build-replay.v1", + "labcolors.proof-region.arb-source-bound-policy.v1", + ): + with self.subTest(stale=stale): + self.assertNotIn(stale, receipt_source) + + def test_pipeline_policy_consumes_both_owned_transport_identities(self) -> None: + trust = pipeline.HostTrustBoundaryV1.UNSEALED_LINUX_X64_DOCKER_HOST + policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 + transport_identity = build_transport.transport_policy_identity_v1(policy) + command_identity = build_transport.native_command_contract_identity_v1() + pipeline_identity = pipeline.pipeline_policy_identity_v2(trust, policy) + + for name, value in ( + ("transport", transport_identity), + ("command", command_identity), + ("pipeline", pipeline_identity), + ): + with self.subTest(identity=name): + self.assertIs(type(value), bytes) + self.assertEqual(len(value), hashlib.sha256().digest_size) + self.assertNotEqual(value, bytes(hashlib.sha256().digest_size)) + self.assertNotEqual(transport_identity, command_identity) + self.assertNotEqual(transport_identity, pipeline_identity) + self.assertNotEqual(command_identity, pipeline_identity) + + calls = _called_names(pipeline.pipeline_policy_identity_v2) + self.assertIn("transport_policy_identity_v1", calls) + self.assertIn("native_command_contract_identity_v1", calls) + for surrogate in (tuple(policy), list(policy), object()): + with self.subTest(surrogate=type(surrogate).__name__): + with self.assertRaises(TypeError): + build_transport.transport_policy_identity_v1(surrogate) + with self.assertRaises(TypeError): + pipeline.pipeline_policy_identity_v2(trust, surrogate) + + def test_generic_sealing_preserves_the_frozen_arb_binding_formula(self) -> None: + """Moving byte storage cannot silently change an unchanged protocol ID.""" + + request = _request() + baseline_policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 + baseline = pipeline._seal_build_input_bundle_v1(request, baseline_policy) + expected = _arb_input_binding_oracle_v2( + request.admitted_sources.identity, + request.build_sources.build_input_identity, + baseline.contents, + baseline_policy.bootstrap, + ) + self.assertEqual(baseline.binding_identity, expected) + + changed_policy = _policy_with( + baseline_policy, + bootstrap=baseline_policy.bootstrap + "\n:", + ) + with mock.patch.object( + pipeline, + "ARB_BUILD_TRANSPORT_POLICY_V1", + changed_policy, + ): + changed = pipeline._seal_build_input_bundle_v1(request, changed_policy) + self.assertTrue( + pipeline.arb_input_is_bound_v1( + request, + changed_policy, + changed, + ) + ) + expected_changed = _arb_input_binding_oracle_v2( + request.admitted_sources.identity, + request.build_sources.build_input_identity, + changed.contents, + changed_policy.bootstrap, + ) + self.assertEqual(changed.contents, baseline.contents) + self.assertEqual(changed.binding_identity, expected_changed) + self.assertNotEqual(changed.binding_identity, baseline.binding_identity) + + # The fixed V1 bootstrap consumes $1 and $2 only. Its shell argv0 is + # bound by the outer transport identity, so it cannot make identical + # source-to-tree bytes a second inner binding. + argv0_changed_policy = _policy_with( + baseline_policy, + bootstrap_argv0="labcolors-other-bootstrap-argv0", + ) + argv0_changed = pipeline._seal_build_input_bundle_v1( + request, + argv0_changed_policy, + ) + self.assertEqual(argv0_changed.binding_identity, baseline.binding_identity) + self.assertTrue( + pipeline.arb_input_is_bound_v1( + request, + argv0_changed_policy, + baseline, + ) + ) + + def test_input_binding_follows_probed_capability_not_module_global(self) -> None: + """A reentrant backend cannot swap a post-probe sealing dependency.""" + + bound_policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 + foreign_policy = _policy_with( + bound_policy, + bootstrap=bound_policy.bootstrap + "\n:", + ) + request = _request() + binary = _static_elf(b"capability-bound-input") + + class _GlobalSwitchingBackend(_BuildBackend): + def __init__(self) -> None: + super().__init__( + (binary, binary), + probe=_docker_capability(bound_policy), + ) + self._attempts = 0 + + def probe(self) -> build_transport.DockerCapabilityReportV1: + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 = foreign_policy + return super().probe() + + def run_build( + self, + value: build_transport.DockerBuildRequestV1, + ) -> build_transport.DockerBuildProcessObservationV1: + observed = super().run_build(value) + self._attempts += 1 + if self._attempts == 2: + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 = bound_policy + return observed + + original_policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 + try: + result = pipeline.ControlledPipelineV1( + build_backend=_GlobalSwitchingBackend(), + ).build(request) + finally: + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 = original_policy + + self.assertIs(type(result), pipeline.DiagnosticBuildObservationV1) + expected = _arb_input_binding_oracle_v2( + request.admitted_sources.identity, + request.build_sources.build_input_identity, + result.input_bundle.contents, + bound_policy.bootstrap, + ) + self.assertEqual(result.input_bundle.binding_identity, expected) + self.assertTrue( + pipeline.arb_input_is_bound_v1( + request, + bound_policy, + result.input_bundle, + ) + ) + expected_receipt_identity = receipt._build_identity_v2( + request, + receipt._source_identity_v1(request), + result, + ) + with mock.patch.object( + pipeline, + "ARB_BUILD_TRANSPORT_POLICY_V1", + foreign_policy, + ): + self.assertTrue( + pipeline.arb_input_is_bound_v1( + request, + bound_policy, + result.input_bundle, + ) + ) + self.assertEqual( + receipt._build_identity_v2( + request, + receipt._source_identity_v1(request), + result, + ), + expected_receipt_identity, + ) + + def test_every_admitted_policy_mutation_changes_transport_and_pipeline_identity(self) -> None: + trust = pipeline.HostTrustBoundaryV1.UNSEALED_LINUX_X64_DOCKER_HOST + policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 + baseline_transport = build_transport.transport_policy_identity_v1(policy) + baseline_pipeline = pipeline.pipeline_policy_identity_v2(trust, policy) + mutants = _policy_mutants(policy) + + self.assertEqual( + {name for name, _mutant in mutants}, + set(_POLICY_FIELDS) - {"platform", "user_mode"}, + ) + for name, mutant in mutants: + with self.subTest(field=name): + self.assertTrue(build_transport.docker_policy_is_valid_v1(mutant)) + self.assertNotEqual( + build_transport.transport_policy_identity_v1(mutant), + baseline_transport, + ) + self.assertNotEqual( + pipeline.pipeline_policy_identity_v2(trust, mutant), + baseline_pipeline, + ) + + # These coordinates currently have singleton admitted domains. Their + # only meaningful mutants are invalid inputs, not a second policy. + self.assertEqual(tuple(build_transport.DockerUserModeV1), (policy.user_mode,)) + with self.assertRaises(TypeError): + _policy_with(policy, platform="linux/arm64") + with self.assertRaises(TypeError): + _policy_with(policy, user_mode="host_effective_ids") + + def test_diagnostic_build_owns_one_capability_and_replayers_consume_its_identity(self) -> None: + field_names = tuple( + field.name for field in dataclass_fields(pipeline.DiagnosticBuildObservationV1) + ) + self.assertEqual(field_names.count("docker_capability"), 1) + for mirror in ( + "docker_daemon_observation_sha256", + "oci_image_reference", + "oci_platform", + "docker_path", + "host_user", + ): + with self.subTest(mirror=mirror): + self.assertNotIn(mirror, field_names) + + comparator_calls = _called_names( + pipeline.derive_arb_comparator_for_build_v1 + ) + comparator_replay_calls = _called_names(receipt._comparator_replays_v1) + receipt_build_calls = _called_names(receipt._build_identity_v2) + source_bound_calls = _called_names( + receipt.source_bound_policy_identity_v2 + ) + self.assertIn("docker_capability_identity_v1", comparator_calls) + self.assertIn("docker_capability_identity_v1", comparator_replay_calls) + self.assertIn("docker_capability_identity_v1", receipt_build_calls) + self.assertIn("docker_capability_identity_v1", source_bound_calls) + + def test_path_uid_daemon_and_hostname_flow_to_downstream_build_identity_only(self) -> None: + baseline_policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 + hostname_policy = _policy_with( + baseline_policy, + hostname="labcolors-build-other-host", + ) + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary).resolve() + first_path = root / "docker-a" + second_path = root / "docker-b" + first_path.write_bytes(b"same-docker-cli-fixture") + second_path.write_bytes(b"same-docker-cli-fixture") + first_path.chmod(0o755) + second_path.chmod(0o755) + + baseline = _observed_build_coordinates( + baseline_policy, + _capability(first_path, baseline_policy), + ) + variants = { + "path": _observed_build_coordinates( + baseline_policy, + _capability(second_path, baseline_policy), + ), + "uid": _observed_build_coordinates( + baseline_policy, + _capability(first_path, baseline_policy, host_user=(502, 20)), + ), + "daemon": _observed_build_coordinates( + baseline_policy, + _capability( + first_path, + baseline_policy, + daemon_marker="daemon-b", + ), + ), + "hostname": _observed_build_coordinates( + hostname_policy, + _capability(first_path, hostname_policy), + ), + } + + ( + baseline_capability, + baseline_comparator_build, + baseline_receipt_build, + baseline_source, + baseline_processes, + baseline_bundle, + baseline_source_bound_policy, + ) = baseline + for name, variant in variants.items(): + with self.subTest(mutation=name): + ( + capability_identity, + comparator_build, + receipt_build, + source_identity, + process_encodings, + bundle_bytes, + source_bound_policy, + ) = variant + self.assertNotEqual(capability_identity, baseline_capability) + self.assertNotEqual(comparator_build, baseline_comparator_build) + self.assertNotEqual(receipt_build, baseline_receipt_build) + self.assertNotEqual( + source_bound_policy, + baseline_source_bound_policy, + ) + self.assertEqual(source_identity, baseline_source) + self.assertEqual(process_encodings, baseline_processes) + self.assertEqual(bundle_bytes, baseline_bundle) + + +if __name__ == "__main__": + unittest.main() diff --git a/proof/region/v1/arb/tests/test_build_recipe.py b/proof/region/v1/arb/tests/test_build_recipe.py index 98ea1e03..3a861446 100644 --- a/proof/region/v1/arb/tests/test_build_recipe.py +++ b/proof/region/v1/arb/tests/test_build_recipe.py @@ -6,6 +6,7 @@ import hashlib import os import subprocess +import tempfile import unittest from pathlib import Path @@ -14,6 +15,7 @@ ARB = Path(__file__).resolve().parents[1] BUILD = ARB / "build.sh" +INNER_BUILD = ARB / "build-inner.sh" WORKFLOW = ARB.parents[3] / ".github" / "workflows" / "arb.yml" RECIPE_REJECTION_TIMEOUT_SECONDS = 5 @@ -145,11 +147,12 @@ def test_replacement(self) -> None: ) def test_recipe_is_offline_static_and_platform_explicit(self) -> None: - source = BUILD.read_text(encoding="utf-8") + source = INNER_BUILD.read_text(encoding="utf-8") + entrypoint = BUILD.read_text(encoding="utf-8") + self.assertIn("/usr/bin/env -i", entrypoint) + self.assertNotIn("LC_BUILD_ENV_V1", entrypoint) for required in ( - "/usr/bin/env -i", - "LC_BUILD_ENV_V1=1", 'require_directory "$inputs/gmp-6.3.0"', 'require_directory "$inputs/mpfr-4.2.2"', 'require_directory "$inputs/flint-3.6.0"', @@ -204,6 +207,58 @@ def test_recipe_is_offline_static_and_platform_explicit(self) -> None: self.assertNotIn("readelf -l \"$build/arb-evaluator-v1\" |", source) self.assertNotIn("readelf -d \"$build/arb-evaluator-v1\" 2>&1 |", source) + def test_source_owned_dispatch_cleans_child_environment(self) -> None: + entrypoint = BUILD.read_text(encoding="utf-8") + recipe = INNER_BUILD.read_text(encoding="utf-8") + self.assertIn("/usr/bin/env -i", entrypoint) + self.assertIn("/usr/bin/readlink -f", entrypoint) + self.assertIn('inner="$script_dir/build-inner.sh"', entrypoint) + self.assertIn('/bin/sh "$inner"', entrypoint) + self.assertLess( + entrypoint.index("exec /usr/bin/env -i"), + entrypoint.index("script_path="), + ) + self.assertNotIn("LC_BUILD_ENV_V1", entrypoint) + self.assertNotIn("LC_BUILD_ENV_V1", recipe) + self.assertNotIn("/usr/bin/env -i", recipe) + + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + entrypoint_copy = root / "build.sh" + inner = root / "build-inner.sh" + observed = root / "environment" + entrypoint_copy.write_text(entrypoint, encoding="utf-8") + entrypoint_copy.chmod(0o755) + inner.write_text( + "#!/bin/sh\n" + f"/usr/bin/env | /usr/bin/sort > '{observed}'\n", + encoding="utf-8", + ) + result = subprocess.run( + [str(entrypoint_copy)], + check=False, + capture_output=True, + text=True, + env={ + "LC_BUILD_ENV_V1": "1", + "MAKEFLAGS": "--jobserver-auth=spoof", + "PYTHONPATH": "/host-controlled", + "CONFIG_SITE": "/host-controlled/site", + "PATH": "/host-controlled/bin", + }, + ) + self.assertEqual(result.returncode, 0, result.stderr) + environment = observed.read_text(encoding="utf-8") + self.assertIn("PATH=/usr/local/bin:/usr/bin:/bin\n", environment) + for forbidden in ( + "LC_BUILD_ENV_V1=1", + "MAKEFLAGS=--jobserver-auth=spoof", + "PYTHONPATH=/host-controlled", + "CONFIG_SITE=/host-controlled/site", + "PATH=/host-controlled/bin", + ): + self.assertNotIn(forbidden, environment) + def test_recipe_rejects_ambient_or_incomplete_invocation_before_build(self) -> None: self.assertTrue(os.access(BUILD, os.X_OK), BUILD) result = subprocess.run( diff --git a/proof/region/v1/arb/tests/test_pipeline.py b/proof/region/v1/arb/tests/test_pipeline.py index 95934f3c..a228d956 100644 --- a/proof/region/v1/arb/tests/test_pipeline.py +++ b/proof/region/v1/arb/tests/test_pipeline.py @@ -7,6 +7,7 @@ import hashlib import io import inspect +import json import os import stat import struct @@ -28,6 +29,8 @@ sys.path.insert(0, str(PROOF)) sys.path.insert(0, str(ARB)) +from build import input as build_input # noqa: E402 +from build import transport as build_transport # noqa: E402 import executor # noqa: E402 import pipeline # noqa: E402 import provenance # noqa: E402 @@ -250,62 +253,117 @@ def _request(**changes: object) -> pipeline.PipelineRequestV1: return pipeline.PipelineRequestV1(**values) +def _docker_capability( + policy: build_transport.DockerBuildPolicyV1 | None = None, + *, + docker_path: Path = Path("/usr/bin/docker"), + host_user: tuple[int, int] = (501, 20), + daemon_marker: bytes = b"docker-daemon-fixture", +) -> build_transport.DockerSupportedV1: + owned_policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 if policy is None else policy + return build_transport.DockerSupportedV1( + owned_policy, + build_transport.DockerDaemonObservationV1( + daemon_marker, + b"docker-image-inspection-fixture", + ), + build_transport.native_command_coordinate_v1(docker_path), + host_user, + ) + + +def _probe_native_backend( + backend: build_transport.NativeDockerBuildBackendV1, + policy: build_transport.DockerBuildPolicyV1, +) -> build_transport.DockerSupportedV1: + image_observation = json.dumps( + [ + { + "Os": "linux", + "Architecture": "amd64", + "RepoDigests": [policy.image_reference], + } + ], + separators=(",", ":"), + ).encode("ascii") + with mock.patch.object( + backend, + "_observe_command", + side_effect=( + build_transport._docker_command_exited_v1( + 0, + b'{"Version":"fixture"}', + b"", + ), + build_transport._docker_command_exited_v1( + 0, + image_observation, + b"", + ), + ), + ): + capability = backend.probe() + if type(capability) is not build_transport.DockerSupportedV1: + raise AssertionError(capability) + return capability + + class _BuildBackend: def __init__( self, outputs: tuple[bytes, ...], *, - probe: pipeline.DockerCapabilityReportV1 | None = None, + probe: build_transport.DockerCapabilityReportV1 | None = None, reject_input: bool = False, omit_transfer: bool = False, foreign_transfer: bool = False, reported_stderr: bytes = b"", ) -> None: self.outputs = list(outputs) - self.probe_result = probe or pipeline.DockerSupportedV1( - pipeline.OCI_IMAGE_REFERENCE_V1, - pipeline.OCI_PLATFORM_V1, - _digest("docker-daemon"), - ) + self.probe_result = probe or _docker_capability() self.reject_input = reject_input self.omit_transfer = omit_transfer self.foreign_transfer = foreign_transfer self.reported_stderr = reported_stderr - self.requests: list[pipeline.DockerBuildRequestV1] = [] + self.requests: list[build_transport.DockerBuildRequestV1] = [] - def probe(self) -> pipeline.DockerCapabilityReportV1: + def probe(self) -> build_transport.DockerCapabilityReportV1: return self.probe_result def run_build( self, - request: pipeline.DockerBuildRequestV1, - ) -> pipeline.DockerBuildProcessObservationV1: + request: build_transport.DockerBuildRequestV1, + ) -> build_transport.DockerBuildProcessObservationV1: self.requests.append(request) output = self.outputs.pop(0) if self.reject_input: - return pipeline.DockerBuildInputRejectedV1( - pipeline._build_input_progress_v1( + return build_transport.DockerBuildInputRejectedV1( + build_transport._build_input_progress_v1( request.input_bundle, 1, - hashlib.sha256(request.input_bundle._contents[:1]).digest(), + hashlib.sha256(request.input_bundle.contents[:1]).digest(), ), b"", b"", ) if self.omit_transfer: - return pipeline._docker_command_exited_v1(0, output, self.reported_stderr) - transfer = pipeline._completed_build_input_transfer_v1( + return build_transport._docker_command_exited_v1(0, output, self.reported_stderr) + transfer = build_transport._completed_build_input_transfer_v1( request.input_bundle, request.input_bundle.length, request.input_bundle.sha256, ) if self.foreign_transfer: - object.__setattr__( - transfer, - "bundle_identity", + foreign_input = build_input.seal_input_v1( _digest("foreign-bundle"), + request.input_bundle.contents, ) - return pipeline._docker_build_exited_v1( + transfer = build_transport._completed_build_input_transfer_v1( + foreign_input, + foreign_input.length, + foreign_input.sha256, + ) + return build_transport._docker_build_exited_v1( 0, output, self.reported_stderr, @@ -523,8 +581,7 @@ def test_wrapper_and_evaluator_file_sets_are_exact_and_disjoint(self) -> None: excluded = wrapper_paths | { pipeline.FORMULA_SPEC_PATH_V1, pipeline.FORMULA_GENERATOR_PATH_V1, - pipeline.BUILD_RECIPE_PATH_V1, - } + } | pipeline.BUILD_RECIPE_PATH_SET_V1 wrapper_files = tuple(item for item in files if item.path in wrapper_paths) evaluator_files = tuple(item for item in files if item.path not in excluded) @@ -544,6 +601,44 @@ def test_wrapper_and_evaluator_file_sets_are_exact_and_disjoint(self) -> None: ), ) + def test_inner_recipe_mutation_is_not_evaluator_source(self) -> None: + original = self._result() + files = _build_sources().files + inner_index = next( + index + for index, item in enumerate(files) + if item.path == pipeline.INNER_BUILD_RECIPE_PATH_V1 + ) + mutated_files = files[:inner_index] + ( + replace(files[inner_index], contents=files[inner_index].contents + b"\n"), + ) + files[inner_index + 1 :] + mutated_sources = pipeline.AdmittedBuildSourcesV1( + mutated_files, + pipeline._build_sources_identity(mutated_files), + _token=pipeline._BUILD_SOURCES_TOKEN, + ) + mutated_request = replace(_request(), build_sources=mutated_sources) + mutated = pipeline.derive_arb_comparator_for_build_v1( + mutated_request, + original.docker_capability, + original._binary, + original.rebuild_sha256s, + original.build_processes, + ) + + self.assertEqual( + mutated.preimages.evaluator_source, + original.comparator.preimages.evaluator_source, + ) + self.assertNotEqual( + mutated.preimages.build_identity, + original.comparator.preimages.build_identity, + ) + self.assertNotEqual( + mutated.preimages.test_observation, + original.comparator.preimages.test_observation, + ) + def test_build_stdout_cannot_supply_a_foreign_manifest_or_coordinate(self) -> None: foreign = _foreign_comparator() report = b"manifest=" + foreign.identity.hex().encode("ascii") @@ -602,27 +697,26 @@ def test_host_trust_claims_only_backend_observable_facts(self) -> None: def test_pipeline_policy_identity_binds_the_stream_bootstrap(self) -> None: trust = pipeline.HostTrustBoundaryV1.UNSEALED_LINUX_X64_DOCKER_HOST - original = pipeline.pipeline_policy_identity_v1(trust) - - with mock.patch.object( - pipeline, - "_BUILD_BOOTSTRAP_V1", - pipeline._BUILD_BOOTSTRAP_V1 + "\nexit 1", - ): - changed = pipeline.pipeline_policy_identity_v1(trust) + policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 + coordinates = list(policy) + coordinates[3] = policy.bootstrap + "\nexit 1" + changed_policy = build_transport.DockerBuildPolicyV1(*coordinates) + original = pipeline.pipeline_policy_identity_v2(trust, policy) + changed = pipeline.pipeline_policy_identity_v2(trust, changed_policy) self.assertNotEqual(original, changed) def test_pipeline_policy_identity_binds_the_private_tmpfs_policy(self) -> None: trust = pipeline.HostTrustBoundaryV1.UNSEALED_LINUX_X64_DOCKER_HOST - original = pipeline.pipeline_policy_identity_v1(trust) - - with mock.patch.object( - pipeline, - "_BUILD_TMPFS_SPEC_V1", - "/tmp:rw,exec,suid,dev,mode=1777", - ): - changed = pipeline.pipeline_policy_identity_v1(trust) + policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 + coordinates = list(policy) + coordinates[5] = ( + "/tmp:rw,exec,suid,dev,size=536870912,mode=1777", + policy.tmpfs_specs[1], + ) + changed_policy = build_transport.DockerBuildPolicyV1(*coordinates) + original = pipeline.pipeline_policy_identity_v2(trust, policy) + changed = pipeline.pipeline_policy_identity_v2(trust, changed_policy) self.assertNotEqual(original, changed) @@ -649,6 +743,12 @@ def test_two_fresh_equal_builds_retain_one_input_and_exact_outputs(self) -> None self.assertIs(type(result), pipeline.DiagnosticBuildObservationV1) self.assertEqual(len(build.requests), 2) self.assertEqual(tuple(item.attempt for item in build.requests), (1, 2)) + self.assertTrue( + all( + item.capability is result.docker_capability + for item in build.requests + ) + ) self.assertIs(build.requests[0].input_bundle, build.requests[1].input_bundle) self.assertEqual(result.binary, binary) self.assertEqual(result.binary_sha256, hashlib.sha256(binary).digest()) @@ -697,7 +797,10 @@ def test_two_fresh_equal_builds_retain_one_input_and_exact_outputs(self) -> None ) self.assertEqual( result.pipeline_policy_identity, - pipeline.pipeline_policy_identity_v1(result.host_trust), + pipeline.pipeline_policy_identity_v2( + result.host_trust, + result.docker_capability.policy, + ), ) self.assertFalse(hasattr(result, "build_observer_kind")) self.assertFalse(hasattr(result, "build_source_identity")) @@ -707,8 +810,16 @@ def test_two_fresh_equal_builds_retain_one_input_and_exact_outputs(self) -> None result.host_trust, pipeline.HostTrustBoundaryV1.UNSEALED_LINUX_X64_DOCKER_HOST, ) - self.assertEqual(result.oci_image_reference, pipeline.OCI_IMAGE_REFERENCE_V1) - self.assertEqual(result.oci_platform, pipeline.OCI_PLATFORM_V1) + self.assertEqual( + result.docker_capability.policy.image_reference, + pipeline.OCI_IMAGE_REFERENCE_V1, + ) + self.assertEqual( + result.docker_capability.policy.platform, + pipeline.OCI_PLATFORM_V1, + ) + self.assertFalse(hasattr(result, "oci_image_reference")) + self.assertFalse(hasattr(result, "oci_platform")) self.assertFalse(hasattr(result, "slsa_level")) self.assertFalse(hasattr(result, "fresh_vm")) @@ -720,32 +831,32 @@ def test_builds_must_be_byte_identical(self) -> None: build_backend=_BuildBackend((first, second)), ).build(_request()) - self.assertEqual( - result, - pipeline.NonReproducibleBuildV1( - hashlib.sha256(first).digest(), - hashlib.sha256(second).digest(), - ), + self.assertIs(type(result), build_transport.TwoBuildObservationV1) + self.assertIs( + result.relation, + build_transport.BuildByteRelationV1.DIFFERENT, ) + self.assertEqual(result.first_sha256, hashlib.sha256(first).digest()) + self.assertEqual(result.second_sha256, hashlib.sha256(second).digest()) def test_input_transport_or_invalid_binary_is_typed_failure(self) -> None: binary = _static_elf() cases = ( ( _BuildBackend((binary,), reject_input=True), - pipeline.BuildFailureReasonV1.INPUT_TRANSFER_FAILED, + build_transport.BuildFailureReasonV1.INPUT_TRANSFER_FAILED, ), ( _BuildBackend((binary,), omit_transfer=True), - pipeline.BuildFailureReasonV1.BACKEND_CONTRACT, + build_transport.BuildFailureReasonV1.CONTRACT_VIOLATION, ), ( _BuildBackend((binary,), foreign_transfer=True), - pipeline.BuildFailureReasonV1.BACKEND_CONTRACT, + build_transport.BuildFailureReasonV1.CONTRACT_VIOLATION, ), ( _BuildBackend((b"not-an-elf",)), - pipeline.BuildFailureReasonV1.INVALID_OUTPUT, + build_transport.BuildFailureReasonV1.INVALID_OUTPUT, ), ) for backend, reason in cases: @@ -753,7 +864,7 @@ def test_input_transport_or_invalid_binary_is_typed_failure(self) -> None: result = pipeline.ControlledPipelineV1( build_backend=backend, ).build(_request()) - self.assertIs(type(result), pipeline.BuildRejectedV1) + self.assertIs(type(result), build_transport.BuildRejectedV1) self.assertEqual(result.attempt, 1) self.assertEqual(result.reason, reason) @@ -829,29 +940,29 @@ def test_native_observer_promotion_is_not_representable_in_v1(self) -> None: def test_mutable_exact_native_build_backend_cannot_upgrade_fabricated_build(self) -> None: binary = _static_elf(b"self-mutating-build") - backend = pipeline.NativeDockerBuildBackendV1( + backend = build_transport.NativeDockerBuildBackendV1( Path("/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, platform_name="linux", machine_name="x86_64", ) - def probe(_self: object) -> pipeline.DockerCapabilityReportV1: - return pipeline.DockerSupportedV1( - pipeline.OCI_IMAGE_REFERENCE_V1, - pipeline.OCI_PLATFORM_V1, - _digest("fabricated-daemon"), + def probe(_self: object) -> build_transport.DockerCapabilityReportV1: + return _docker_capability( + docker_path=Path("/bin/true"), + daemon_marker=b"fabricated-daemon", ) def run_build( _self: object, - request: pipeline.DockerBuildRequestV1, - ) -> pipeline.DockerBuildProcessObservationV1: - transfer = pipeline._completed_build_input_transfer_v1( + request: build_transport.DockerBuildRequestV1, + ) -> build_transport.DockerBuildProcessObservationV1: + transfer = build_transport._completed_build_input_transfer_v1( request.input_bundle, request.input_bundle.length, request.input_bundle.sha256, ) - return pipeline._docker_build_exited_v1(0, binary, b"", transfer) + return build_transport._docker_build_exited_v1(0, binary, b"", transfer) backend.probe = MethodType(probe, backend) backend.run_build = MethodType(run_build, backend) @@ -865,59 +976,82 @@ def run_build( class DockerCommandContractTests(unittest.TestCase): - def test_command_is_exact_digest_offline_read_only_and_capability_free(self) -> None: - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary).resolve() - request = pipeline.DockerBuildRequestV1( - 1, - pipeline._seal_build_input_bundle_v1(_request()), - _limits().max_executable_bytes, - root / "container.cid", - "labcolors-arb-build-v1-test", - ) - backend = pipeline.NativeDockerBuildBackendV1( - Path("/usr/bin/docker"), - platform_name="linux", - machine_name="x86_64", - ) - - command = backend.command_for(request) - - joined = " ".join(command) - self.assertEqual(command[0], "/usr/bin/docker") - self.assertIn(pipeline.OCI_IMAGE_REFERENCE_V1, command) - self.assertNotIn("gcc:latest", joined) - for fragment in ( - "--pull never", - "--platform linux/amd64", - "--network none", - "--read-only", - "--interactive", - "--cap-drop ALL", - "--security-opt no-new-privileges:true", - "--name labcolors-arb-build-v1-test", - "--rm", - ): - with self.subTest(fragment=fragment): - self.assertIn(fragment, joined) - for forbidden in ("--privileged", "--network host", ":latest"): - self.assertNotIn(forbidden, joined) + def test_command_is_exact_digest_offline_read_only_and_capability_bound(self) -> None: + backend = build_transport.NativeDockerBuildBackendV1( + Path("/usr/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + platform_name="linux", + machine_name="x86_64", + host_user=(501, 20), + ) + capability = _probe_native_backend( + backend, + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + ) + request = build_transport.DockerBuildRequestV1( + 1, + capability, + pipeline._seal_build_input_bundle_v1( + _request(), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + ), + _limits().max_executable_bytes, + ) + lease = backend._next_run_lease_v1(capability) + try: + command = backend._command_for_v1(request, lease) + + joined = " ".join(command) + self.assertEqual(command[0], "/usr/bin/true") + self.assertIn(pipeline.OCI_IMAGE_REFERENCE_V1, command) + self.assertNotIn("gcc:latest", joined) + for fragment in ( + "--pull never", + "--platform linux/amd64", + "--network none", + "--read-only", + "--interactive", + "--cap-drop ALL", + "--security-opt no-new-privileges:true", + f"--cidfile {lease.cid_file}", + "--rm", + ): + with self.subTest(fragment=fragment): + self.assertIn(fragment, joined) + for forbidden in ( + "--name", + "--privileged", + "--network host", + ":latest", + ): + self.assertNotIn(forbidden, joined) + finally: + backend._release_run_lease_v1(lease) def test_command_exposes_only_a_private_non_executable_standard_tmp(self) -> None: - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary).resolve() - request = pipeline.DockerBuildRequestV1( - 1, - pipeline._seal_build_input_bundle_v1(_request()), - _limits().max_executable_bytes, - root / "container.cid", - "labcolors-arb-build-v1-test", - ) - command = pipeline.NativeDockerBuildBackendV1( - Path("/usr/bin/docker"), - platform_name="linux", - machine_name="x86_64", - ).command_for(request) + backend = build_transport.NativeDockerBuildBackendV1( + Path("/usr/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + platform_name="linux", + machine_name="x86_64", + host_user=(501, 20), + ) + capability = _probe_native_backend( + backend, + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + ) + request = build_transport.DockerBuildRequestV1( + 1, + capability, + pipeline._seal_build_input_bundle_v1( + _request(), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + ), + _limits().max_executable_bytes, + ) + lease = backend._next_run_lease_v1(capability) + self.addCleanup(backend._release_run_lease_v1, lease) + command = backend._command_for_v1(request, lease) tmpfs_indexes = tuple( index for index, item in enumerate(command) if item == "--tmpfs" @@ -938,23 +1072,26 @@ def test_command_exposes_only_a_private_non_executable_standard_tmp(self) -> Non self.assertNotIn("--volume", command) def test_native_probe_fails_closed_without_linux_or_exact_docker(self) -> None: - non_linux = pipeline.NativeDockerBuildBackendV1( + non_linux = build_transport.NativeDockerBuildBackendV1( Path("/usr/bin/docker"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, platform_name="darwin", machine_name="arm64", ).probe() - missing = pipeline.NativeDockerBuildBackendV1( + missing = build_transport.NativeDockerBuildBackendV1( Path("/definitely/missing/docker"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, platform_name="linux", machine_name="x86_64", ).probe() - self.assertEqual(non_linux.reason, pipeline.DockerBlockerReasonV1.HOST_NOT_LINUX_AMD64) - self.assertEqual(missing.reason, pipeline.DockerBlockerReasonV1.DOCKER_UNAVAILABLE) + self.assertEqual(non_linux.reason, build_transport.DockerBlockerReasonV1.HOST_NOT_LINUX_AMD64) + self.assertEqual(missing.reason, build_transport.DockerBlockerReasonV1.DOCKER_UNAVAILABLE) def test_native_command_observer_caps_probe_output_before_allocation(self) -> None: - backend = pipeline.NativeDockerBuildBackendV1( + backend = build_transport.NativeDockerBuildBackendV1( Path("/bin/sh"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, platform_name="linux", machine_name="x86_64", ) @@ -968,74 +1105,260 @@ def test_native_command_observer_caps_probe_output_before_allocation(self) -> No stdout_limit=8, stderr_limit=8, timeout_ns=5_000_000_000, - cid_file=None, ) self.assertEqual( result, - pipeline.DockerBuildOutputLimitV1( - pipeline.DockerOutputStreamV1.STDOUT, + build_transport.DockerBuildOutputLimitV1( + build_transport.DockerOutputStreamV1.STDOUT, b"x" * 8, b"", ), ) - def test_cleanup_falls_back_to_exact_name_for_absent_or_invalid_cidfile(self) -> None: - backend = pipeline.NativeDockerBuildBackendV1( - Path("/bin/sh"), + def test_cleanup_uses_only_a_docker_issued_id_from_its_private_lease(self) -> None: + backend = build_transport.NativeDockerBuildBackendV1( + Path("/usr/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, platform_name="linux", machine_name="x86_64", + host_user=(501, 20), ) - name = "labcolors-arb-build-v1-cleanup-test" - for cid_contents in (None, b"partial-or-foreign"): - with self.subTest(cid_contents=cid_contents): - with tempfile.TemporaryDirectory() as temporary: - cid_file = Path(temporary) / "container.cid" - if cid_contents is not None: - cid_file.write_bytes(cid_contents) - observations = ( - pipeline._docker_command_exited_v1(1, b"", b"not found"), - pipeline._docker_command_exited_v1(0, b"", b""), - ) + capability = _probe_native_backend( + backend, + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + ) + lease = backend._next_run_lease_v1(capability) + lease.cid_file.write_text("b" * 64 + "\n", encoding="ascii") + backend._mark_run_lease_launched_v1(lease) + foreign_id = "a" * 64 + observations = ( + build_transport._docker_command_exited_v1(0, b"b" * 64 + b"\n", b""), + build_transport._docker_command_exited_v1(0, b"b" * 64 + b"\n", b""), + build_transport._docker_command_exited_v1(0, b"", b""), + ) + try: + with mock.patch.object( + backend, + "_observe_cleanup_command", + side_effect=observations, + ) as observe: + detail = backend._cleanup_container(lease) + + self.assertIsNone(detail) + commands = tuple(call.args[1] for call in observe.call_args_list) + self.assertEqual(len(commands), 3) + self.assertEqual(commands[0][-1], "b" * 64) + self.assertEqual(commands[1][-1], "b" * 64) + self.assertIn("id=" + "b" * 64, commands[2]) + self.assertNotIn( + foreign_id, + " ".join(" ".join(command) for command in commands), + ) + self.assertNotIn("name=", " ".join(" ".join(command) for command in commands)) + finally: + backend._release_run_lease_v1(lease) + + def test_absent_cidfile_never_falls_back_to_a_name_or_docker_io(self) -> None: + backend = build_transport.NativeDockerBuildBackendV1( + Path("/usr/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + platform_name="linux", + machine_name="x86_64", + host_user=(501, 20), + ) + capability = _probe_native_backend( + backend, + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + ) + lease = backend._next_run_lease_v1(capability) + try: + with mock.patch.object(backend, "_observe_cleanup_command") as observe: + self.assertIsNone(backend._cleanup_container(lease)) + observe.assert_not_called() + backend._mark_run_lease_launched_v1(lease) + with mock.patch.object(backend, "_observe_cleanup_command") as observe: + self.assertEqual( + backend._cleanup_container(lease), + "Docker-issued cleanup ID is unavailable", + ) + observe.assert_not_called() + finally: + backend._release_run_lease_v1(lease) + + def test_malformed_or_aliased_cid_never_reaches_destructive_cleanup(self) -> None: + backend = build_transport.NativeDockerBuildBackendV1( + Path("/usr/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + platform_name="linux", + machine_name="x86_64", + host_user=(501, 20), + ) + capability = _probe_native_backend( + backend, + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + ) + for raw, alias in ((b"B" * 64, False), (b"c" * 64, True)): + with self.subTest(alias=alias, raw=raw[:1]): + lease = backend._next_run_lease_v1(capability) + try: + lease.cid_file.write_bytes(raw) + if alias: + os.link(lease.cid_file, lease.cid_file.parent / "cid-alias") + backend._mark_run_lease_launched_v1(lease) with mock.patch.object( backend, "_observe_cleanup_command", - side_effect=observations, ) as observe: - detail = backend._cleanup_container(cid_file, name) + self.assertEqual( + backend._cleanup_container(lease), + "Docker-issued cleanup ID is unavailable", + ) + observe.assert_not_called() + finally: + backend._release_run_lease_v1(lease) + + def test_foreign_name_matching_a_stale_id_never_reaches_rm(self) -> None: + backend = build_transport.NativeDockerBuildBackendV1( + Path("/usr/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + platform_name="linux", + machine_name="x86_64", + host_user=(501, 20), + ) + capability = _probe_native_backend( + backend, + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + ) + lease = backend._next_run_lease_v1(capability) + container_id = "d" * 64 + foreign_id = "e" * 64 + try: + lease.cid_file.write_text(container_id + "\n", encoding="ascii") + backend._mark_run_lease_launched_v1(lease) + with mock.patch.object( + backend, + "_observe_cleanup_command", + return_value=build_transport._docker_command_exited_v1( + 0, + foreign_id.encode("ascii") + b"\n", + b"", + ), + ) as observe: + self.assertEqual( + backend._cleanup_container(lease), + "Docker-issued cleanup ID did not resolve exactly", + ) + commands = tuple(call.args[1] for call in observe.call_args_list) + self.assertEqual(len(commands), 1) + self.assertIn("inspect", commands[0]) + self.assertNotIn("rm", commands[0]) + finally: + backend._release_run_lease_v1(lease) + + def test_cleanup_uses_capability_captured_by_the_run_lease(self) -> None: + backend = build_transport.NativeDockerBuildBackendV1( + Path("/usr/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + platform_name="linux", + machine_name="x86_64", + host_user=(501, 20), + ) + capability = _probe_native_backend( + backend, + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + ) + lease = backend._next_run_lease_v1(capability) + try: + lease.cid_file.write_text("f" * 64 + "\n", encoding="ascii") + backend._mark_run_lease_launched_v1(lease) + backend._probed_capability = None + observations = ( + build_transport._docker_command_exited_v1( + 0, + b"f" * 64 + b"\n", + b"", + ), + build_transport._docker_command_exited_v1( + 0, + b"f" * 64 + b"\n", + b"", + ), + build_transport._docker_command_exited_v1(0, b"", b""), + ) + with mock.patch.object( + backend, + "_observe_cleanup_command", + side_effect=observations, + ) as observe: + self.assertIsNone(backend._cleanup_container(lease)) + self.assertEqual( + tuple(call.args[0] for call in observe.call_args_list), + (capability, capability, capability), + ) + finally: + backend._release_run_lease_v1(lease) + + def test_cleanup_rejects_another_adapter_lease_before_docker_io(self) -> None: + backend = build_transport.NativeDockerBuildBackendV1( + Path("/usr/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + platform_name="linux", + machine_name="x86_64", + host_user=(501, 20), + ) + _probe_native_backend(backend, pipeline.ARB_BUILD_TRANSPORT_POLICY_V1) + foreign_backend = build_transport.NativeDockerBuildBackendV1( + Path("/usr/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + platform_name="linux", + machine_name="x86_64", + host_user=(501, 20), + ) + foreign_capability = _probe_native_backend( + foreign_backend, + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + ) + foreign_lease = foreign_backend._next_run_lease_v1(foreign_capability) + try: + with mock.patch.object(backend, "_observe_cleanup_command") as observe: + with self.assertRaises(TypeError): + backend._cleanup_container(foreign_lease) - self.assertIsNone(detail) - commands = tuple(call.args[0] for call in observe.call_args_list) - self.assertEqual(commands[0][-1], name) - self.assertIn(f"name=^/{name}$", commands[1]) - self.assertNotIn("partial-or-foreign", " ".join(commands[0])) + observe.assert_not_called() + finally: + foreign_backend._release_run_lease_v1(foreign_lease) def test_unverified_container_removal_is_typed_cleanup_failure(self) -> None: - backend = pipeline.NativeDockerBuildBackendV1( + backend = build_transport.NativeDockerBuildBackendV1( Path("/bin/sh"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, platform_name="linux", machine_name="x86_64", ) - with tempfile.TemporaryDirectory() as temporary: - cid_file = Path(temporary).resolve() / "container.cid" - with mock.patch.object( - backend, - "_cleanup_container", - return_value="container absence could not be verified", - ): - result = backend._observe_command( - (sys.executable, "-c", "pass"), - stdout_limit=8, - stderr_limit=8, - timeout_ns=5_000_000_000, - cid_file=cid_file, - container_name="labcolors-arb-build-v1-cleanup-failure", - ) + capability = _docker_capability( + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + docker_path=Path("/bin/sh"), + ) + lease = backend._next_run_lease_v1(capability) + self.addCleanup(backend._release_run_lease_v1, lease) + with mock.patch.object( + backend, + "_cleanup_container", + return_value="container absence could not be verified", + ): + result = backend._observe_command( + (sys.executable, "-c", "pass"), + stdout_limit=8, + stderr_limit=8, + timeout_ns=5_000_000_000, + lease=lease, + ) - self.assertIs(type(result), pipeline.DockerBuildCleanupFailureV1) + self.assertIs(type(result), build_transport.DockerBuildCleanupFailureV1) self.assertEqual( result.trigger, - pipeline.DockerCleanupTriggerV1.PROCESS_EXIT, + build_transport.DockerCleanupTriggerV1.PROCESS_EXIT, ) diff --git a/proof/region/v1/arb/tests/test_receipt.py b/proof/region/v1/arb/tests/test_receipt.py index 80beade1..2716e69e 100644 --- a/proof/region/v1/arb/tests/test_receipt.py +++ b/proof/region/v1/arb/tests/test_receipt.py @@ -22,10 +22,13 @@ TESTS = ARB / "tests" sys.path[:0] = [str(PROOF), str(ARB), str(TESTS)] +from build import transport as build_transport # noqa: E402 + import executor # noqa: E402 import pipeline # noqa: E402 import provenance # noqa: E402 import receipt # noqa: E402 +import region_proof_protocol as protocol # noqa: E402 from region_proof_protocol import ( # noqa: E402 BoundaryUnprovenWitnessV1, DecisionTranscriptV1, @@ -34,6 +37,7 @@ ) from test_pipeline import ( # noqa: E402 _BuildBackend, + _docker_capability, _foreign_comparator, _job, _request, @@ -127,13 +131,13 @@ def _controller( ) return controller, ( mock.patch.object( - pipeline.NativeDockerBuildBackendV1, + build_transport.NativeDockerBuildBackendV1, "probe", autospec=True, side_effect=lambda _self: build_backend.probe(), ), mock.patch.object( - pipeline.NativeDockerBuildBackendV1, + build_transport.NativeDockerBuildBackendV1, "run_build", autospec=True, side_effect=lambda _self, request: build_backend.run_build(request), @@ -219,11 +223,29 @@ def _replace_invocation( class SourceBoundReceiptTests(unittest.TestCase): def test_source_bound_policy_identity_binds_immutable_coordinates(self) -> None: + capability = _docker_capability() + request = _request() + # This golden belongs to the exact observed capability fixture; changing + # its daemon, CLI path or host user must deliberately rederive it. self.assertEqual( - receipt.source_bound_policy_identity_v1().hex(), - "a7cf0c142397a1e8ce3f9bb9dd4168120cdf78814745d1d4596d08a8e88a6b1b", + receipt.source_bound_policy_identity_v2( + capability, + request.host_trust, + ).hex(), + "f223e1a1569ca5cf6251fd012af8a789a75aedd830e3ccb8f13db77d7ac67bd4", ) + def test_source_bound_policy_identity_consumes_explicit_trust_coordinate(self) -> None: + capability = _docker_capability() + trust = object() + with mock.patch.object( + receipt.pipeline, + "pipeline_policy_identity_v2", + return_value=_digest("pipeline-policy"), + ) as policy_identity: + receipt.source_bound_policy_identity_v2(capability, trust) + policy_identity.assert_called_once_with(trust, capability.policy) + def test_identity_rejection_remains_typed_at_the_receipt_boundary(self) -> None: invocation_rejection = executor.ExecutionIdentityRejectedV1( executor.ExecutionIdentityReasonV1.REQUEST_NOT_ADMITTED, @@ -275,7 +297,10 @@ def test_only_controller_execution_can_seal_a_receipt(self) -> None: self.assertEqual(result.evidence.identity, result.claim.replay_evidence_identity) self.assertEqual( result.claim.provenance_policy_identity, - receipt.source_bound_policy_identity_v1(), + receipt.source_bound_policy_identity_v2( + result.evidence.build.docker_capability, + result.evidence.request.host_trust, + ), ) self.assertTrue(receipt.replay_evidence_is_well_bound_v1(result.evidence)) self.assertEqual(len(backend.requests), 1) @@ -314,8 +339,13 @@ def test_receipt_uses_only_versioned_public_cross_module_verifiers(self) -> None self.assertNotIn("pipeline._sealed_build_input_bundle_is_well_bound_v1", source) self.assertNotIn("pipeline._build_process_bytes_v1", source) self.assertNotIn("executor._execution_identity_v1", source) - self.assertTrue(hasattr(pipeline, "sealed_build_input_bundle_is_well_bound_v1")) - self.assertTrue(hasattr(pipeline, "build_process_bytes_v1")) + self.assertNotIn("sealed_build_input_bundle_is_well_bound_v1", source) + self.assertIn("pipeline.arb_input_is_bound_v1", source) + self.assertIn("pipeline.derive_arb_comparator_for_build_v1", source) + self.assertIn("build_transport.build_process_bytes_v1", source) + self.assertTrue(hasattr(pipeline, "arb_input_is_bound_v1")) + self.assertTrue(hasattr(pipeline, "derive_arb_comparator_for_build_v1")) + self.assertTrue(hasattr(build_transport, "build_process_bytes_v1")) self.assertTrue(hasattr(executor, "invocation_identity_v1")) self.assertTrue(hasattr(executor, "platform_identity_v1")) @@ -323,7 +353,10 @@ def test_reference_does_not_describe_shipped_arb_receipt_as_future(self) -> None documentation = (PROOF / "PROTOCOL.md").read_text(encoding="utf-8") prose = " ".join(documentation.split()) - self.assertIn("## Source-bound Arb replay", documentation) + self.assertIn( + "## Воспроизведение Arb, связанное с источником", + documentation, + ) self.assertIn("SourceBoundEvaluatorReceiptV1", documentation) for stale_claim in ( "заявленные результаты будущих Arb/MPFI processes", @@ -372,8 +405,8 @@ def test_job_first_binds_at_run_not_source_or_build(self) -> None: self.assertEqual(first_source, second_source) self.assertEqual(first_build.input_bundle_identity, second_build.input_bundle_identity) self.assertEqual( - receipt._build_identity_v1(request, first_source, first_build), - receipt._build_identity_v1( + receipt._build_identity_v2(request, first_source, first_build), + receipt._build_identity_v2( different_request, second_source, second_build, @@ -402,7 +435,6 @@ def test_root_and_build_coordinates_are_recomputed(self) -> None: "pipeline_policy_identity", "flint_commit_content_identity", "flint_project_pinned_release_only_identity", - "docker_daemon_observation_sha256", "binary_sha256", "input_bundle_identity", "input_bundle_sha256", @@ -414,6 +446,22 @@ def test_root_and_build_coordinates_are_recomputed(self) -> None: _tamper(dag, "build", build) ) ) + different_capability = _docker_capability( + daemon_marker=b"different-docker-daemon" + ) + self.assertFalse( + receipt.replay_evidence_is_well_bound_v1( + _tamper( + dag, + "build", + _tamper( + dag.build, + "docker_capability", + different_capability, + ), + ) + ) + ) for field_name in ( "flint_commit_content_file_count", "flint_project_pinned_release_only_file_count", @@ -465,41 +513,57 @@ def test_source_process_transfer_and_comparator_mutations_fail(self) -> None: ) first = dag.build.build_processes[0] - for field_name in ("bundle_identity", "expected_sha256", "written_sha256"): - with self.subTest(transfer=field_name): - transfer = _tamper( - first.input_transfer, - field_name, - _digest(field_name), - ) - process = _tamper(first, "input_transfer", transfer) - build = _tamper( - dag.build, - "build_processes", - (process, dag.build.build_processes[1]), - ) - self.assertFalse( - receipt.replay_evidence_is_well_bound_v1( - _tamper(dag, "build", build) - ) - ) - for field_name in ("expected_length", "written_length"): - transfer = _tamper( - first.input_transfer, - field_name, + self.assertFalse(hasattr(first.input_transfer, "__dict__")) + self.assertFalse(hasattr(first, "__dict__")) + forged_transfer = tuple.__new__( + type(first.input_transfer), + ( + first.input_transfer.bundle_identity, first.input_transfer.expected_length + 1, + first.input_transfer.expected_sha256, + first.input_transfer.written_length, + first.input_transfer.written_sha256, + ), + ) + forged_process = tuple.__new__( + type(first), + ( + first.returncode, + first.stdout, + first.stderr, + forged_transfer, + ), + ) + forged_build = _tamper( + dag.build, + "build_processes", + (forged_process, dag.build.build_processes[1]), + ) + self.assertFalse( + receipt.replay_evidence_is_well_bound_v1( + _tamper(dag, "build", forged_build) ) - process = _tamper(first, "input_transfer", transfer) - build = _tamper( - dag.build, - "build_processes", - (process, dag.build.build_processes[1]), - ) - self.assertFalse( - receipt.replay_evidence_is_well_bound_v1( - _tamper(dag, "build", build) - ) + ) + + forged_process = tuple.__new__( + type(first), + ( + first.returncode + 1, + first.stdout, + first.stderr, + first.input_transfer, + ), + ) + forged_build = _tamper( + dag.build, + "build_processes", + (forged_process, dag.build.build_processes[1]), + ) + self.assertFalse( + receipt.replay_evidence_is_well_bound_v1( + _tamper(dag, "build", forged_build) ) + ) preimages = _tamper( dag.build.comparator.preimages, @@ -530,6 +594,43 @@ def test_source_process_transfer_and_comparator_mutations_fail(self) -> None: receipt.replay_evidence_is_well_bound_v1(_tamper(dag, "build", build)) ) + def test_each_self_consistent_comparator_coordinate_is_rederived(self) -> None: + result, _backend = _execute() + dag = result.evidence + original = dag.build.comparator + for field in original.preimages.__dataclass_fields__: + with self.subTest(field=field): + preimages = _tamper( + original.preimages, + field, + getattr(original.preimages, field) + b"\x00mutation", + ) + names = tuple(original.preimages.__dataclass_fields__) + coordinates = tuple( + hashlib.sha256(getattr(preimages, name)).digest() + for name in names + ) + manifest = protocol.ComparatorManifestV2( + original.manifest.manifest.kind, + *coordinates, + ) + by_digest = { + coordinate: getattr(preimages, name) + for name, coordinate in zip(names, coordinates, strict=True) + } + resolved = protocol.ContentResolvedComparatorManifestV2.admit( + manifest, + by_digest.get, + ) + comparator = _tamper(original, "preimages", preimages) + comparator = _tamper(comparator, "manifest", resolved) + forged_build = _tamper(dag.build, "comparator", comparator) + self.assertFalse( + receipt.replay_evidence_is_well_bound_v1( + _tamper(dag, "build", forged_build) + ) + ) + def test_source_replay_rejects_a_self_consistent_forged_manifest(self) -> None: request = _request() lock = request.source_lock.sources[2] diff --git a/proof/region/v1/arb/tests/test_transport.py b/proof/region/v1/arb/tests/test_transport.py index 63db7075..1bd572a1 100644 --- a/proof/region/v1/arb/tests/test_transport.py +++ b/proof/region/v1/arb/tests/test_transport.py @@ -3,15 +3,18 @@ from __future__ import annotations -import dataclasses +import ast +from collections.abc import Callable import hashlib import io import inspect +import json import os import subprocess import sys import tarfile import tempfile +import textwrap import unittest from pathlib import Path from unittest import mock @@ -24,66 +27,258 @@ sys.path.insert(0, str(ARB)) sys.path.insert(0, str(TESTS)) +from build import input as build_input # noqa: E402 +from build import transport as build_transport # noqa: E402 import pipeline # noqa: E402 -from test_pipeline import _request # noqa: E402 +from test_pipeline import ( # noqa: E402 + _docker_capability, + _probe_native_backend, + _request, +) BUILD_RECIPE = ARB / "build.sh" +INNER_BUILD_RECIPE = ARB / "build-inner.sh" NATIVE_GATE = ARB / "tests" / "native_gate.py" +_TEST_CANONICAL_LIMITS = build_input.CanonicalInputLimitsV1(64, 1024, 4096) def _digest(label: str) -> bytes: return hashlib.sha256(label.encode("ascii")).digest() -def _bundle(length: int = 1024 * 1024) -> pipeline.SealedBuildInputBundleV1: +def _bundle(length: int = 1024 * 1024) -> build_input.SealedInputV1: contents = (b"0123456789abcdef" * ((length + 15) // 16))[:length] - return pipeline.SealedBuildInputBundleV1( - _digest("source"), - _digest("build-input"), - contents, - _token=pipeline._BUILD_INPUT_BUNDLE_TOKEN, - ) + return build_input.seal_input_v1(_digest("opaque-binding"), contents) -def _backend() -> pipeline.NativeDockerBuildBackendV1: - return pipeline.NativeDockerBuildBackendV1( +def _backend() -> build_transport.NativeDockerBuildBackendV1: + return build_transport.NativeDockerBuildBackendV1( Path("/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + platform_name="linux", + machine_name="x86_64", + ) + + +def _native_backend_with_request( + bundle: build_input.SealedInputV1 | None = None, +) -> tuple[ + build_transport.NativeDockerBuildBackendV1, + build_transport.DockerBuildRequestV1, +]: + """One fixture owns the native request shape used by cleanup tests.""" + + backend = build_transport.NativeDockerBuildBackendV1( + Path("/usr/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, platform_name="linux", machine_name="x86_64", + host_user=(501, 20), ) + capability = _probe_native_backend( + backend, + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + ) + return backend, build_transport.DockerBuildRequestV1( + 1, + capability, + _bundle(1024) if bundle is None else bundle, + 1024, + ) + + +def _report_released_cid_root( + backend: build_transport.NativeDockerBuildBackendV1, + detail: object = "forced CID-root cleanup failure", +) -> Callable[[object], object]: + """Keep failure fixtures honest: a reported cleanup failure follows release.""" + + release = backend._release_run_lease_v1 + + def release_then_report(lease: object) -> object: + if release(lease) is not None: + raise AssertionError("native CID-root fixture lease did not release") + return detail + + return release_then_report def _observe( source: str, - bundle: pipeline.SealedBuildInputBundleV1, + bundle: build_input.SealedInputV1, *, stdout_limit: int = 2 * 1024 * 1024, stderr_limit: int = 2 * 1024 * 1024, timeout_ns: int = 5_000_000_000, -) -> pipeline.DockerBuildProcessObservationV1: +) -> build_transport.DockerBuildProcessObservationV1: return _backend()._observe_command( (sys.executable, "-c", source), stdout_limit=stdout_limit, stderr_limit=stderr_limit, timeout_ns=timeout_ns, - cid_file=None, input_bundle=bundle, ) class CanonicalBuildBundleTests(unittest.TestCase): + def test_public_probe_starts_with_a_typed_terminal_outcome(self) -> None: + """A backend cannot leave the public probe in a non-report state.""" + + source = inspect.getsource(build_transport.ControlledBuildTransportV1.probe) + tree = ast.parse(textwrap.dedent(source)) + function = tree.body[0] + if not isinstance(function, ast.FunctionDef): + self.fail("public probe source must remain one function definition") + typed_initializers = [ + node + for node in function.body + if isinstance(node, ast.AnnAssign) + and isinstance(node.target, ast.Name) + and node.target.id == "outcome" + and isinstance(node.annotation, ast.Name) + and node.annotation.id == "DockerCapabilityReportV1" + and isinstance(node.value, ast.Call) + and isinstance(node.value.func, ast.Name) + and node.value.func.id == "DockerUnsupportedV1" + ] + self.assertEqual(len(typed_initializers), 1) + self.assertNotIn('raise RuntimeError("probe outcome was not produced")', source) + + def test_probe_releases_its_transient_lease_after_backend_failure(self) -> None: + report = _docker_capability(pipeline.ARB_BUILD_TRANSPORT_POLICY_V1) + + class FlakyProbeBackend: + def __init__(self) -> None: + self.calls = 0 + + def probe(self) -> object: + self.calls += 1 + if self.calls == 1: + raise ValueError("forced backend failure") + return report + + backend = FlakyProbeBackend() + controller = build_transport.ControlledBuildTransportV1( + policy=pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + backend=backend, + ) + first = controller.probe() + second = controller.probe() + + self.assertIs(type(first), build_transport.DockerUnsupportedV1) + self.assertEqual( + first.reason, + build_transport.DockerBlockerReasonV1.BACKEND_CONTRACT, + ) + self.assertIs(second, report) + self.assertEqual(backend.calls, 2) + + def test_public_input_constructors_use_typed_value_errors(self) -> None: + def reject( + constructor: Callable[[], object], + reason: build_input.InputReasonV1, + field: str, + ) -> None: + with self.assertRaises(build_input.InputErrorV1) as caught: + constructor() + self.assertEqual(caught.exception.reason, reason) + self.assertEqual(caught.exception.field, field) + + def limits( + max_members: object = 1, + max_file_bytes: object = 1, + max_payload_bytes: object = 1, + max_encoded_bytes: object = None, + ) -> object: + return build_input.CanonicalInputLimitsV1( + max_members, + max_file_bytes, + max_payload_bytes, + max_encoded_bytes, + ) + + def seal( + binding_identity: object = _digest("binding"), + contents: object = b"x", + ) -> object: + return build_input.seal_input_v1(binding_identity, contents) + + limit_fields = ( + "max_members", + "max_file_bytes", + "max_payload_bytes", + "max_encoded_bytes", + ) + self.assertIs( + type(build_input.CanonicalInputLimitsV1(1, 1, 1, None)), + build_input.CanonicalInputLimitsV1, + ) + for field, value in ( + ("max_members", True), + ("max_file_bytes", 1.0), + ("max_payload_bytes", object()), + ("max_encoded_bytes", b"1"), + ): + with self.subTest(kind="wrong_type", field=field): + reject( + lambda field=field, value=value: limits(**{field: value}), + build_input.InputReasonV1.WRONG_TYPE, + field, + ) + for field in limit_fields: + for value in (0, 1 << 64): + with self.subTest(kind="invalid_limit", field=field, value=value): + reject( + lambda field=field, value=value: limits(**{field: value}), + build_input.InputReasonV1.INVALID_VALUE, + field, + ) + reject( + lambda: limits(max_payload_bytes=(1 << 64) - 1), + build_input.InputReasonV1.INVALID_VALUE, + "max_encoded_bytes", + ) + + for field, constructor in ( + ("binding_identity", lambda: seal(bytearray(_digest("binding")))), + ("contents", lambda: seal(contents=bytearray(b"x"))), + ): + with self.subTest(kind="wrong_type", field=field): + reject(constructor, build_input.InputReasonV1.WRONG_TYPE, field) + for field, constructor in ( + ("binding_identity", lambda: seal(bytes(32))), + ("binding_identity", lambda: seal(b"x" * 31)), + ("contents", lambda: seal(contents=b"")), + ): + with self.subTest(kind="invalid_value", field=field): + reject(constructor, build_input.InputReasonV1.INVALID_VALUE, field) + + for constructor in ( + lambda: build_input.CanonicalInputLimitsV1(1, 1), + lambda: build_input.seal_input_v1(_digest("binding")), + lambda: build_input.SealedInputV1( + _digest("binding"), + b"x", + _token=object(), + ), + ): + with self.subTest(kind="private_or_call_shape"): + with self.assertRaises(TypeError): + constructor() + def test_bundle_is_reproducible_normalized_ustar_with_no_host_authority(self) -> None: request = _request() - first = pipeline._seal_build_input_bundle_v1(request) - second = pipeline._seal_build_input_bundle_v1(request) + policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 + first = pipeline._seal_build_input_bundle_v1(request, policy) + second = pipeline._seal_build_input_bundle_v1(request, policy) self.assertIsNot(first, second) - self.assertIs(first._contents, first._contents) - self.assertEqual(first._contents, second._contents) + self.assertIs(first.contents, first.contents) + self.assertEqual(first.contents, second.contents) self.assertEqual(first.sha256, second.sha256) - self.assertEqual(first.identity, second.identity) - self.assertTrue(pipeline.sealed_build_input_bundle_is_well_bound_v1(first)) + self.assertEqual(first.binding_identity, second.binding_identity) + self.assertTrue(pipeline.arb_input_is_bound_v1(request, policy, first)) source_entries = tuple( entry @@ -117,7 +312,7 @@ def test_bundle_is_reproducible_normalized_ustar_with_no_host_authority(self) -> sorted(expected_directories, key=lambda value: (value.count("/"), value)) ) + tuple(sorted(expected_files)) - with tarfile.open(fileobj=io.BytesIO(first._contents), mode="r:") as archive: + with tarfile.open(fileobj=io.BytesIO(first.contents), mode="r:") as archive: members = tuple(archive) self.assertFalse(archive.pax_headers) self.assertEqual(tuple(member.name for member in members), expected_order) @@ -145,36 +340,149 @@ def test_bundle_is_reproducible_normalized_ustar_with_no_host_authority(self) -> self.assertEqual(stream.read(), body) def test_canonical_encoder_rejects_reorder_collision_and_unencodable_path(self) -> None: + def reject( + values: object, + reason: build_input.InputReasonV1, + field: str, + limits: build_input.CanonicalInputLimitsV1 = _TEST_CANONICAL_LIMITS, + ) -> None: + with self.assertRaises(build_input.InputErrorV1) as caught: + build_input.canonical_ustar_v1(values, limits) + self.assertEqual(caught.exception.reason, reason) + self.assertEqual(caught.exception.field, field) + entries = (("a/b", 0o644, b"x"), ("c", 0o755, b"y")) - encoded = pipeline._canonical_tar_v1(entries) + encoded = build_input.canonical_ustar_v1(entries, _TEST_CANONICAL_LIMITS) self.assertEqual( hashlib.sha256(encoded).hexdigest(), "11bc313cba907e89535876eb8ce46194472367007053ab58b723338676f99427", ) - with self.assertRaises(TypeError): - pipeline._canonical_tar_v1(tuple(reversed(entries))) - with self.assertRaises(TypeError): - pipeline._canonical_tar_v1((("a", 0o644, b"x"), ("a/b", 0o644, b"y"))) - with self.assertRaises(TypeError): - pipeline._canonical_tar_v1((("A", 0o644, b"x"), ("a", 0o644, b"y"))) - with self.assertRaises((TypeError, ValueError)): - pipeline._canonical_tar_v1((("a" * 256, 0o644, b"x"),)) + for hostile, reason, field in ( + ( + tuple(reversed(entries)), + build_input.InputReasonV1.NONCANONICAL_SET, + "entries", + ), + ( + (("a", 0o644, b"x"), ("a/b", 0o644, b"y")), + build_input.InputReasonV1.NONCANONICAL_SET, + "a", + ), + ( + (("A", 0o644, b"x"), ("a", 0o644, b"y")), + build_input.InputReasonV1.NONCANONICAL_SET, + "a", + ), + ( + (("a" * 256, 0o644, b"x"),), + build_input.InputReasonV1.INVALID_PATH, + "a" * 256, + ), + ( + ((1, 0o644, b"x"),), + build_input.InputReasonV1.INVALID_PATH, + "path", + ), + ( + ((["a"], 0o644, b"x"),), + build_input.InputReasonV1.INVALID_PATH, + "path", + ), + ( + (("a" * 101, 0o644, b"x"),), + build_input.InputReasonV1.INVALID_PATH, + "a" * 101, + ), + ( + (("A", 0o644, b"x"), ("a/b", 0o644, b"y")), + build_input.InputReasonV1.NONCANONICAL_SET, + "A", + ), + ): + with self.subTest(hostile=repr(hostile)): + reject(hostile, reason, field) + + resource_cases = ( + ( + (("a", 0o644, b"x"), ("b", 0o644, b"y")), + build_input.CanonicalInputLimitsV1(1, 1, 2), + "max_members", + ), + ( + (("a", 0o644, b"xy"),), + build_input.CanonicalInputLimitsV1(1, 1, 2), + "max_file_bytes", + ), + ( + (("a", 0o644, b"x"), ("b", 0o644, b"y")), + build_input.CanonicalInputLimitsV1(2, 1, 1), + "max_payload_bytes", + ), + ( + (("a/b", 0o644, b"x"),), + build_input.CanonicalInputLimitsV1(1, 1, 1), + "max_members", + ), + ( + (("a", 0o644, b"x"),), + build_input.CanonicalInputLimitsV1(1, 1, 1, 10_239), + "max_encoded_bytes", + ), + ) + for values, limits, field in resource_cases: + with self.subTest(resource=field): + reject( + values, + build_input.InputReasonV1.RESOURCE_LIMIT, + field, + limits, + ) + exact_cap = build_input.CanonicalInputLimitsV1(1, 1, 1, 10_240) + self.assertEqual( + len(build_input.canonical_ustar_v1((("a", 0o644, b"x"),), exact_cap)), + exact_cap.max_encoded_bytes, + ) + + def test_long_implicit_directory_uses_the_ustar_trailing_separator(self) -> None: + directory = "d" * 155 + entries = ((f"{directory}/f", 0o644, b"x"),) + limits = build_input.CanonicalInputLimitsV1(2, 1, 1) + + encoded = build_input.canonical_ustar_v1(entries, limits) + + with tarfile.open(fileobj=io.BytesIO(encoded), mode="r:") as archive: + self.assertEqual( + tuple(member.name for member in archive), + (directory, f"{directory}/f"), + ) + with self.assertRaises(build_input.InputErrorV1) as caught: + build_input.canonical_ustar_v1( + ((f"{'d' * 156}/f", 0o644, b"x"),), + limits, + ) + self.assertEqual(caught.exception.reason, build_input.InputReasonV1.INVALID_PATH) + self.assertEqual(caught.exception.field, f"{'d' * 156}/f") def test_omission_or_content_mutation_changes_bundle_identity(self) -> None: entries = (("a", 0o644, b"x"), ("b", 0o644, b"y")) - original = pipeline._canonical_tar_v1(entries) - omitted = pipeline._canonical_tar_v1(entries[:1]) - mutated = pipeline._canonical_tar_v1( - (("a", 0o644, b"x"), ("b", 0o644, b"z")) + original = build_input.canonical_ustar_v1(entries, _TEST_CANONICAL_LIMITS) + omitted = build_input.canonical_ustar_v1( + entries[:1], + _TEST_CANONICAL_LIMITS, + ) + mutated = build_input.canonical_ustar_v1( + (("a", 0o644, b"x"), ("b", 0o644, b"z")), + _TEST_CANONICAL_LIMITS, ) identities = { - pipeline.SealedBuildInputBundleV1( - _digest("source"), - _digest("build-input"), - body, - _token=pipeline._BUILD_INPUT_BUNDLE_TOKEN, - ).identity + ( + sealed.binding_identity, + sealed.sha256, + ) for body in (original, omitted, mutated) + for sealed in ( + build_input.seal_input_v1(_digest("opaque-binding"), body), + ) } self.assertEqual(len(identities), 3) @@ -184,7 +492,7 @@ def test_replayed_source_coordinates_must_match_the_admitted_capability(self) -> original = admitted.tree_identity object.__setattr__(admitted, "tree_identity", _digest("mutated-tree")) try: - with self.assertRaises(TypeError): + with self.assertRaises(pipeline.PipelineInputErrorV1): pipeline._normalized_source_entries_v1( request.source_lock.sources[0], admitted, @@ -195,17 +503,72 @@ def test_replayed_source_coordinates_must_match_the_admitted_capability(self) -> def test_transport_authorities_cannot_be_directly_forged(self) -> None: bundle = _bundle(1024) with self.assertRaises(TypeError): - pipeline.BuildInputTransferProgressV1( - bundle.identity, + build_transport.BuildInputTransferProgressV1( + bundle.binding_identity, bundle.length, bundle.sha256, bundle.length, bundle.sha256, ) with self.assertRaises(TypeError): - pipeline.BuildInputTransferV1(object()) + build_transport.BuildInputTransferV1(object()) + with self.assertRaises(TypeError): + build_transport.DockerBuildExitedV1(0, b"binary", b"", object()) with self.assertRaises(TypeError): - pipeline.DockerBuildExitedV1(0, b"binary", b"", object()) + build_transport.DockerBuildPolicyV1( + "gcc@sha256:bad@sha256:" + "0" * 64, + *pipeline.ARB_BUILD_TRANSPORT_POLICY_V1[1:], + ) + report = _docker_capability() + self.assertFalse(hasattr(report, "__dict__")) + with self.assertRaises((AttributeError, TypeError)): + object.__setattr__(report, "host_user", (0, 0)) + forged_policy = tuple.__new__(build_transport.DockerBuildPolicyV1, ()) + with self.assertRaises(TypeError): + build_transport.ControlledBuildTransportV1( + policy=forged_policy, + backend=object(), + ) + + class ForgedProbeBackend: + def probe(self) -> object: + return tuple.__new__(build_transport.DockerSupportedV1, ()) + + probed = build_transport.ControlledBuildTransportV1( + policy=pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + backend=ForgedProbeBackend(), + ).probe() + self.assertIs(type(probed), build_transport.DockerUnsupportedV1) + self.assertEqual( + probed.reason, + build_transport.DockerBlockerReasonV1.BACKEND_CONTRACT, + ) + + class ForgedProcessBackend: + def probe(self) -> object: + return report + + def run_build(self, _request: object) -> object: + return tuple.__new__(build_transport.DockerBuildExitedV1, ()) + + controller = build_transport.ControlledBuildTransportV1( + policy=pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + backend=ForgedProcessBackend(), + ) + observed_report = controller.probe() + self.assertIs(type(observed_report), build_transport.DockerSupportedV1) + rejected = controller.build( + observed_report, + bundle, + 1, + input_admission=lambda _value: True, + output_admission=lambda _value: True, + ) + self.assertIs(type(rejected), build_transport.BuildRejectedV1) + self.assertEqual( + rejected.reason, + build_transport.BuildFailureReasonV1.CONTRACT_VIOLATION, + ) class BuildInputObserverTests(unittest.TestCase): @@ -216,7 +579,7 @@ def test_positive_partial_writes_are_normal_and_commit_exact_transfer(self) -> N def partial_write(descriptor: int, contents: object) -> int: return real_write(descriptor, contents[:997]) - with mock.patch.object(pipeline.os, "write", side_effect=partial_write): + with mock.patch.object(build_transport.os, "write", side_effect=partial_write): result = _observe( "import hashlib,sys; d=sys.stdin.buffer.read(); " "sys.stdout.buffer.write(hashlib.sha256(d).digest()); " @@ -224,10 +587,10 @@ def partial_write(descriptor: int, contents: object) -> int: bundle, ) - self.assertIs(type(result), pipeline.DockerBuildExitedV1, result) + self.assertIs(type(result), build_transport.DockerBuildExitedV1, result) self.assertEqual(result.stdout, bundle.sha256) self.assertEqual(result.stderr, b"observed") - self.assertEqual(result.input_transfer.bundle_identity, bundle.identity) + self.assertEqual(result.input_transfer.bundle_identity, bundle.binding_identity) self.assertEqual(result.input_transfer.expected_length, bundle.length) self.assertEqual(result.input_transfer.expected_sha256, bundle.sha256) self.assertEqual(result.input_transfer.written_length, bundle.length) @@ -235,18 +598,18 @@ def partial_write(descriptor: int, contents: object) -> int: def test_zero_write_and_epipe_are_typed_with_exact_partial_progress(self) -> None: bundle = _bundle() - with mock.patch.object(pipeline.os, "write", return_value=0): + with mock.patch.object(build_transport.os, "write", return_value=0): zero = _observe("import sys; sys.stdin.buffer.read()", bundle) - self.assertIs(type(zero), pipeline.DockerBuildInputRejectedV1, zero) + self.assertIs(type(zero), build_transport.DockerBuildInputRejectedV1, zero) self.assertEqual(zero.written_length, 0) self.assertEqual(zero.written_sha256, hashlib.sha256(b"").digest()) closed = _observe("import os,time; os.close(0); time.sleep(1)", bundle) - self.assertIs(type(closed), pipeline.DockerBuildInputRejectedV1, closed) + self.assertIs(type(closed), build_transport.DockerBuildInputRejectedV1, closed) self.assertLess(closed.written_length, bundle.length) self.assertEqual( closed.written_sha256, - hashlib.sha256(bundle._contents[: closed.written_length]).digest(), + hashlib.sha256(bundle.contents[: closed.written_length]).digest(), ) def test_final_stdin_close_failure_is_a_typed_observer_failure(self) -> None: @@ -273,14 +636,14 @@ def spawn(*args: object, **kwargs: object) -> subprocess.Popen[bytes]: process.stdin = CloseFailsOnce(process.stdin) return process - with mock.patch.object(pipeline.subprocess, "Popen", side_effect=spawn): + with mock.patch.object(build_transport.subprocess, "Popen", side_effect=spawn): result = _observe( "import time; time.sleep(1)", _bundle(2 * 1024 * 1024), timeout_ns=100_000_000, ) - self.assertIs(type(result), pipeline.DockerBuildObserverFailureV1, result) + self.assertIs(type(result), build_transport.DockerBuildObserverFailureV1, result) def test_full_duplex_backpressure_does_not_deadlock_or_drop_bytes(self) -> None: bundle = _bundle(512 * 1024) @@ -293,7 +656,7 @@ def test_full_duplex_backpressure_does_not_deadlock_or_drop_bytes(self) -> None: " os.write(2,b'e'*len(d))\n", bundle, ) - self.assertIs(type(result), pipeline.DockerBuildExitedV1, result) + self.assertIs(type(result), build_transport.DockerBuildExitedV1, result) self.assertEqual(len(result.stdout), bundle.length) self.assertEqual(len(result.stderr), bundle.length) self.assertEqual(result.input_transfer.written_sha256, bundle.sha256) @@ -305,8 +668,8 @@ def test_timeout_and_output_limit_preserve_input_progress(self) -> None: bundle, timeout_ns=100_000_000, ) - self.assertIs(type(timed), pipeline.DockerBuildTimedOutV1, timed) - self.assertIs(type(timed.input_progress), pipeline.BuildInputTransferProgressV1) + self.assertIs(type(timed), build_transport.DockerBuildTimedOutV1, timed) + self.assertIs(type(timed.input_progress), build_transport.BuildInputTransferProgressV1) self.assertGreater(timed.input_progress.written_length, 0) self.assertLess(timed.input_progress.written_length, bundle.length) @@ -315,76 +678,580 @@ def test_timeout_and_output_limit_preserve_input_progress(self) -> None: bundle, stdout_limit=8, ) - self.assertIs(type(limited), pipeline.DockerBuildOutputLimitV1, limited) - self.assertEqual(limited.stream, pipeline.DockerOutputStreamV1.STDOUT) + self.assertIs(type(limited), build_transport.DockerBuildOutputLimitV1, limited) + self.assertEqual(limited.stream, build_transport.DockerOutputStreamV1.STDOUT) self.assertEqual(limited.stdout, b"x" * 8) - self.assertIs(type(limited.input_progress), pipeline.BuildInputTransferProgressV1) + self.assertIs(type(limited.input_progress), build_transport.BuildInputTransferProgressV1) def test_cleanup_failure_preserves_input_trigger_and_progress(self) -> None: bundle = _bundle() backend = _backend() - with tempfile.TemporaryDirectory() as temporary, mock.patch.object( - backend, - "_cleanup_container", - return_value="forced cleanup failure", - ): - result = backend._observe_command( - (sys.executable, "-c", "import os,time; os.close(0); time.sleep(1)"), - stdout_limit=1024, - stderr_limit=1024, - timeout_ns=5_000_000_000, - cid_file=Path(temporary).resolve() / "container.cid", - container_name="labcolors-arb-build-v1-transport-test", - input_bundle=bundle, + lease = backend._next_run_lease_v1( + _docker_capability( + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + docker_path=Path("/bin/true"), ) - self.assertIs(type(result), pipeline.DockerBuildCleanupFailureV1, result) - self.assertEqual(result.trigger, pipeline.DockerCleanupTriggerV1.INPUT_TRANSFER) + ) + try: + with mock.patch.object( + backend, + "_cleanup_container", + return_value="forced cleanup failure", + ): + result = backend._observe_command( + (sys.executable, "-c", "import os,time; os.close(0); time.sleep(1)"), + stdout_limit=1024, + stderr_limit=1024, + timeout_ns=5_000_000_000, + lease=lease, + input_bundle=bundle, + ) + finally: + backend._release_run_lease_v1(lease) + self.assertIs(type(result), build_transport.DockerBuildCleanupFailureV1, result) + self.assertEqual(result.trigger, build_transport.DockerCleanupTriggerV1.INPUT_TRANSFER) self.assertEqual(result.detail, "forced cleanup failure") - self.assertIs(type(result.input_progress), pipeline.BuildInputTransferProgressV1) + self.assertIs(type(result.input_progress), build_transport.BuildInputTransferProgressV1) self.assertLess(result.input_progress.written_length, bundle.length) class SealedBuildTransportContractTests(unittest.TestCase): + def test_diagnostic_details_have_one_strict_admission_law(self) -> None: + constructors = ( + ( + "unsupported", + lambda detail: build_transport.DockerUnsupportedV1( + build_transport.DockerBlockerReasonV1.BACKEND_CONTRACT, + detail, + ), + ), + ( + "observer-failure", + lambda detail: build_transport.DockerBuildObserverFailureV1( + detail, + b"", + b"", + ), + ), + ( + "cleanup-record", + lambda detail: build_transport.CleanupFailureRecordV1( + build_transport.CleanupResourceV1.DOCKER_CID_ROOT, + detail, + ), + ), + ) + + class DetailSubclass(str): + pass + + for constructor_name, constructor in constructors: + with self.subTest(constructor=constructor_name, detail="valid"): + self.assertIsNotNone(constructor("valid diagnostic detail")) + for name, invalid_detail in ( + ("empty", ""), + ("subclass", DetailSubclass("detail")), + ( + "too-long", + "x" * (build_transport._DIAGNOSTIC_DETAIL_TEXT_LIMIT_V1 + 1), + ), + ("wrong-type", object()), + ): + with self.subTest(constructor=constructor_name, detail=name): + with self.assertRaises(TypeError): + constructor(invalid_detail) + + def test_successful_probe_keeps_machine_readable_stdout_despite_cli_warning(self) -> None: + policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 + with tempfile.TemporaryDirectory() as temporary: + docker_path = Path(temporary) / "docker" + docker_path.write_bytes(b"fixture") + docker_path.chmod(0o755) + backend = build_transport.NativeDockerBuildBackendV1( + docker_path, + policy, + platform_name="linux", + machine_name="x86_64", + host_user=(501, 20), + ) + image = json.dumps( + [ + { + "Os": "linux", + "Architecture": "amd64", + "RepoDigests": [policy.image_reference], + } + ], + separators=(",", ":"), + ).encode("ascii") + with mock.patch.object( + backend, + "_observe_command", + side_effect=( + build_transport._docker_command_exited_v1( + 0, + b'{"Version":"fixture"}', + b"warning: CLI hint\n", + ), + build_transport._docker_command_exited_v1( + 0, + image, + b"warning: local metadata\n", + ), + ), + ): + capability = backend.probe() + + self.assertIs(type(capability), build_transport.DockerSupportedV1) + self.assertEqual( + capability.daemon_observation.server_stdout, + b'{"Version":"fixture"}', + ) + def test_controller_owns_one_sealed_bundle_for_both_builds(self) -> None: - build_source = inspect.getsource(pipeline.ControlledPipelineV1.build) - self.assertEqual(build_source.count("_seal_build_input_bundle_v1("), 1) - self.assertIn("for attempt in (1, 2)", build_source) + pipeline_source = inspect.getsource(pipeline.ControlledPipelineV1.build) + transport_source = inspect.getsource( + build_transport.ControlledBuildTransportV1.build + ) + self.assertEqual(pipeline_source.count("_seal_build_input_bundle_v1("), 1) + self.assertIn("for attempt in (1, 2)", transport_source) - def test_docker_request_has_no_semantic_host_path_authority(self) -> None: - fields = {item.name for item in dataclasses.fields(pipeline.DockerBuildRequestV1)} + def test_docker_request_carries_only_semantic_build_coordinates(self) -> None: + fields = set(inspect.signature(build_transport.DockerBuildRequestV1).parameters) self.assertEqual( fields, - {"attempt", "input_bundle", "max_executable_bytes", "cid_file", "container_name"}, + { + "attempt", + "capability", + "input_bundle", + "max_output_bytes", + }, ) - command = pipeline.NativeDockerBuildBackendV1( - Path("/usr/bin/docker"), + backend = build_transport.NativeDockerBuildBackendV1( + Path("/usr/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, platform_name="linux", machine_name="x86_64", - ).command_for( - pipeline.DockerBuildRequestV1( + host_user=(501, 20), + ) + capability = _probe_native_backend( + backend, + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + ) + request = build_transport.DockerBuildRequestV1( + 1, + capability, + _bundle(1024), + 1024, + ) + self.assertFalse(hasattr(request, "cid_file")) + self.assertFalse(hasattr(request, "container_name")) + # The field assertion above proves no cleanup coordinate is modeled. + # This call separately guards the fixed-arity boundary against extras. + with self.assertRaises(TypeError): + build_transport.DockerBuildRequestV1( 1, + capability, _bundle(1024), 1024, - Path("/tmp/container.cid"), - "labcolors-arb-build-v1-command-test", + Path("/tmp/foreign.cid"), + "labcolors-arb-build-v1-foreign", ) + + def test_native_adapter_mints_private_docker_issued_cleanup_authority(self) -> None: + backend, request = _native_backend_with_request() + capability = request.capability + lease = backend._next_run_lease_v1(capability) + try: + command = backend._command_for_v1(request, lease) + + self.assertIn("--cidfile", command) + self.assertNotIn("--name", command) + self.assertFalse(hasattr(lease, "container_name")) + self.assertTrue(lease.cid_file.is_absolute()) + self.assertFalse(lease.cid_file.exists()) + finally: + backend._release_run_lease_v1(lease) + + def test_native_adapter_rejects_malformed_nominal_observations_typed(self) -> None: + raw_observations = ( + ("unknown", object()), + *( + (kind.__name__, tuple.__new__(kind, ())) + for kind in ( + build_transport.DockerBuildExitedV1, + build_transport.DockerBuildTimedOutV1, + build_transport.DockerBuildOutputLimitV1, + build_transport.DockerBuildObserverFailureV1, + build_transport.DockerBuildInputRejectedV1, + build_transport.DockerBuildCleanupFailureV1, + ) + ), + ( + "DockerBuildObserverFailureV1/invalid-progress", + tuple.__new__( + build_transport.DockerBuildObserverFailureV1, + ("forged", b"untrusted stdout", b"untrusted stderr", object()), + ), + ), + ) + + def observe(raw: object, *, cleanup_fails: bool) -> object: + backend, request = _native_backend_with_request() + if not cleanup_fails: + with mock.patch.object( + backend, + "_observe_command", + return_value=raw, + ): + return backend.run_build(request) + + with mock.patch.object( + backend, + "_observe_command", + return_value=raw, + ), mock.patch.object( + backend, + "_release_run_lease_v1", + side_effect=_report_released_cid_root(backend), + ): + return backend.run_build(request) + + for name, raw in raw_observations: + with self.subTest(observation=name, cleanup_fails=False): + without_cleanup_failure = observe(raw, cleanup_fails=False) + self.assertIs( + type(without_cleanup_failure), + build_transport.DockerBuildObserverFailureV1, + ) + self.assertEqual( + without_cleanup_failure.detail, + "native Docker build observation is not canonical", + ) + self.assertEqual(without_cleanup_failure.stdout, b"") + self.assertEqual(without_cleanup_failure.stderr, b"") + self.assertIsNone(without_cleanup_failure.input_progress) + with self.subTest(observation=name, cleanup_fails=True): + with_cleanup_failure = observe(raw, cleanup_fails=True) + self.assertIs( + type(with_cleanup_failure), + build_transport.DockerBuildObserverFailureV1, + ) + self.assertEqual( + with_cleanup_failure.detail, + "native Docker build observation is not canonical; " + "forced CID-root cleanup failure", + ) + self.assertEqual(with_cleanup_failure.stdout, b"") + self.assertEqual(with_cleanup_failure.stderr, b"") + self.assertIsNone(with_cleanup_failure.input_progress) + + def test_native_adapter_rejects_a_preexisting_cid_root_claim(self) -> None: + bundle = _bundle(1024) + backend, request = _native_backend_with_request(bundle) + raw = build_transport.DockerBuildCleanupFailureV1( + build_transport.DockerCleanupTriggerV1.PROCESS_EXIT, + ( + build_transport.CleanupFailureRecordV1( + build_transport.CleanupResourceV1.DOCKER_CID_ROOT, + "forged prior CID-root cleanup failure", + ), + ), + b"retained stdout", + b"retained stderr", + build_transport._build_input_progress_v1( + bundle, + bundle.length, + bundle.sha256, + ), + ) + with mock.patch.object( + backend, + "_observe_command", + return_value=raw, + ): + without_cleanup_failure = backend.run_build(request) + + self.assertIs( + type(without_cleanup_failure), + build_transport.DockerBuildObserverFailureV1, + ) + self.assertEqual( + without_cleanup_failure.detail, + "native Docker build observation already contains a CID-root " + "cleanup failure", + ) + self.assertEqual(without_cleanup_failure.stdout, b"retained stdout") + self.assertEqual(without_cleanup_failure.stderr, b"retained stderr") + self.assertIs(without_cleanup_failure.input_progress, raw.input_progress) + + with mock.patch.object( + backend, + "_observe_command", + return_value=raw, + ), mock.patch.object( + backend, + "_release_run_lease_v1", + side_effect=_report_released_cid_root(backend), + ): + with_cleanup_failure = backend.run_build(request) + + self.assertIs( + type(with_cleanup_failure), + build_transport.DockerBuildCleanupFailureV1, + ) + self.assertEqual( + with_cleanup_failure.trigger, + build_transport.DockerCleanupTriggerV1.OBSERVER_FAILURE, + ) + self.assertEqual( + with_cleanup_failure.failures, + ( + build_transport.CleanupFailureRecordV1( + build_transport.CleanupResourceV1.DOCKER_CID_ROOT, + "forced CID-root cleanup failure", + ), + ), ) - self.assertNotIn("--mount", command) - self.assertEqual(command.count("--tmpfs"), 2) - self.assertIn(pipeline._BUILD_TMPFS_SPEC_V1, command) - self.assertIn(pipeline._BUILD_STATE_TMPFS_SPEC_V1, command) - self.assertIn("--interactive", command) - self.assertIn("/usr/bin/env", command) - self.assertIn("-i", command) + self.assertEqual(with_cleanup_failure.stdout, b"retained stdout") + self.assertEqual(with_cleanup_failure.stderr, b"retained stderr") + self.assertIs(with_cleanup_failure.input_progress, raw.input_progress) + + def test_native_adapter_bounds_unknown_observation_cleanup_detail(self) -> None: + prefix = "native Docker build observation is not canonical; " + for name, detail, expected_detail in ( + ( + "retained", + "forced CID-root cleanup failure", + prefix + "forced CID-root cleanup failure", + ), + ( + "bounded", + "x" * build_transport._DIAGNOSTIC_DETAIL_TEXT_LIMIT_V1, + prefix + + "x" + * ( + build_transport._DIAGNOSTIC_DETAIL_TEXT_LIMIT_V1 + - len(prefix) + ), + ), + ): + with self.subTest(detail=name): + result = ( + build_transport.NativeDockerBuildBackendV1._with_cid_root_cleanup_failure_v1( + object(), + detail, + ) + ) + + self.assertIs(type(result), build_transport.DockerBuildObserverFailureV1) + self.assertEqual(result.detail, expected_detail) + self.assertEqual(result.stdout, b"") + self.assertEqual(result.stderr, b"") + self.assertIsNone(result.input_progress) + + def test_native_adapter_appends_cid_root_to_canonical_cleanup_prefix(self) -> None: + bundle = _bundle(1024) + backend, request = _native_backend_with_request(bundle) + progress = build_transport._build_input_progress_v1( + bundle, + bundle.length, + bundle.sha256, + ) + raw = build_transport.DockerBuildCleanupFailureV1( + build_transport.DockerCleanupTriggerV1.PROCESS_EXIT, + ( + build_transport.CleanupFailureRecordV1( + build_transport.CleanupResourceV1.DOCKER_CLI_PROCESS, + "prior CLI cleanup failure", + ), + build_transport.CleanupFailureRecordV1( + build_transport.CleanupResourceV1.DOCKER_CONTAINER, + "prior container cleanup failure", + ), + ), + b"retained stdout", + b"retained stderr", + progress, + ) + with mock.patch.object( + backend, + "_observe_command", + return_value=raw, + ), mock.patch.object( + backend, + "_release_run_lease_v1", + side_effect=_report_released_cid_root(backend), + ): + result = backend.run_build(request) + + self.assertIs(type(result), build_transport.DockerBuildCleanupFailureV1) + self.assertEqual(result.trigger, build_transport.DockerCleanupTriggerV1.PROCESS_EXIT) + self.assertEqual( + tuple(record.resource for record in result.failures), + ( + build_transport.CleanupResourceV1.DOCKER_CLI_PROCESS, + build_transport.CleanupResourceV1.DOCKER_CONTAINER, + build_transport.CleanupResourceV1.DOCKER_CID_ROOT, + ), + ) + self.assertEqual( + tuple(record.detail for record in result.failures), + ( + "prior CLI cleanup failure", + "prior container cleanup failure", + "forced CID-root cleanup failure", + ), + ) + self.assertEqual(result.stdout, b"retained stdout") + self.assertEqual(result.stderr, b"retained stderr") + self.assertIs(result.input_progress, progress) + + def test_native_adapter_reowns_invalid_cid_cleanup_details(self) -> None: + bundle = _bundle(1024) + backend, request = _native_backend_with_request(bundle) + transfer = build_transport._completed_build_input_transfer_v1( + bundle, + bundle.length, + bundle.sha256, + ) + raw = build_transport._docker_build_exited_v1( + 0, + b"built stdout", + b"built stderr", + transfer, + ) + for name, invalid_detail in ( + ("empty", ""), + ("wrong-type", object()), + ( + "too-long", + "x" * (build_transport._DIAGNOSTIC_DETAIL_TEXT_LIMIT_V1 + 1), + ), + ): + with self.subTest(detail=name): + + with mock.patch.object( + backend, + "_observe_command", + return_value=raw, + ), mock.patch.object( + backend, + "_release_run_lease_v1", + side_effect=_report_released_cid_root( + backend, + invalid_detail, + ), + ): + result = backend.run_build(request) + + self.assertIs(type(result), build_transport.DockerBuildCleanupFailureV1) + self.assertEqual( + result.detail, + "native Docker CID root cleanup detail is not canonical", + ) + self.assertEqual(result.stdout, b"built stdout") + self.assertEqual(result.stderr, b"built stderr") + self.assertEqual( + result.input_progress, + build_transport._build_input_progress_v1( + bundle, + bundle.length, + bundle.sha256, + ), + ) + + def test_native_adapter_keeps_dual_failure_typed_at_detail_limit(self) -> None: + backend, request = _native_backend_with_request() + raw = build_transport.DockerBuildObserverFailureV1( + "x" * build_transport._DIAGNOSTIC_DETAIL_TEXT_LIMIT_V1, + b"retained stdout", + b"retained stderr", + ) + with mock.patch.object( + backend, + "_observe_command", + return_value=raw, + ), mock.patch.object( + backend, + "_release_run_lease_v1", + side_effect=_report_released_cid_root(backend), + ): + result = backend.run_build(request) + + self.assertIs(type(result), build_transport.DockerBuildObserverFailureV1) + self.assertEqual( + result.detail, + "native Docker build observation and CID root cleanup both failed", + ) + self.assertEqual(result.stdout, b"retained stdout") + self.assertEqual(result.stderr, b"retained stderr") + + def test_native_adapter_releases_before_propagating_first_canonicalization_interruption( + self, + ) -> None: + backend = build_transport.NativeDockerBuildBackendV1( + Path("/usr/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + platform_name="linux", + machine_name="x86_64", + host_user=(501, 20), + ) + capability = _probe_native_backend( + backend, + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + ) + request = build_transport.DockerBuildRequestV1( + 1, + capability, + _bundle(1024), + 1024, + ) + release = backend._release_run_lease_v1 + released: list[None] = [] + + class FalseyInterrupt(BaseException): + def __bool__(self) -> bool: + return False + + original = FalseyInterrupt("first interruption") + + def release_then_record(lease: object) -> None: + self.assertIsNone(release(lease)) + released.append(None) + raise KeyboardInterrupt("later cleanup interruption") + + with mock.patch.object( + backend, + "_observe_command", + return_value=object(), + ), mock.patch.object( + build_transport, + "_canonical_process_observation_v1", + side_effect=original, + ), mock.patch.object( + backend, + "_release_run_lease_v1", + side_effect=release_then_record, + ): + with self.assertRaises(FalseyInterrupt) as raised: + backend.run_build(request) + + self.assertIs(raised.exception, original) + self.assertEqual(released, [None]) def test_recipe_is_transport_agnostic_and_bootstrap_owns_binary_stdout(self) -> None: - source = BUILD_RECIPE.read_text(encoding="utf-8") + source = INNER_BUILD_RECIPE.read_text(encoding="utf-8") self.assertIn("readonly inputs=/build/snapshot/inputs", source) self.assertIn("readonly workspace=/build/snapshot/workspace", source) self.assertIn("readonly build=/build/work", source) self.assertNotIn("/out", source) self.assertNotIn(">&3", source) self.assertIn("exec 3>&1", pipeline._BUILD_BOOTSTRAP_V1) + self.assertIn( + '/bin/sh "$snapshot/workspace/proof/region/v1/arb/build.sh"', + pipeline._BUILD_BOOTSTRAP_V1, + ) self.assertIn("/build/work/arb-evaluator-v1 >&3", pipeline._BUILD_BOOTSTRAP_V1) def test_native_gate_executes_the_one_shot_receipt_controller(self) -> None: diff --git a/proof/region/v1/build/__init__.py b/proof/region/v1/build/__init__.py new file mode 100644 index 00000000..dd2df699 --- /dev/null +++ b/proof/region/v1/build/__init__.py @@ -0,0 +1 @@ +"""Owned shared BUILD package; public contracts live in focused leaf modules.""" diff --git a/proof/region/v1/build/input.py b/proof/region/v1/build/input.py new file mode 100644 index 00000000..f09cf886 --- /dev/null +++ b/proof/region/v1/build/input.py @@ -0,0 +1,345 @@ +#!/usr/bin/env python3 +"""Canonical BUILD input bytes without engine or recipe semantics.""" + +from __future__ import annotations + +import hashlib +import io +import tarfile +from dataclasses import dataclass +from enum import StrEnum +from typing import NoReturn + + +_SEALED_INPUT_TOKEN = object() +_USTAR_BLOCK_BYTES = 512 +_USTAR_RECORD_BYTES = 20 * _USTAR_BLOCK_BYTES +_USTAR_EOF_BLOCKS = 2 + + +def _valid_digest(value: object) -> bool: + return type(value) is bytes and len(value) == 32 and value != bytes(32) + + +class InputReasonV1(StrEnum): + WRONG_TYPE = "wrong_type" + INVALID_VALUE = "invalid_value" + INVALID_PATH = "invalid_path" + INVALID_MODE = "invalid_mode" + NONCANONICAL_SET = "noncanonical_set" + RESOURCE_LIMIT = "resource_limit" + + +@dataclass(frozen=True) +class InputErrorV1(ValueError): + reason: InputReasonV1 + field: str + + def __str__(self) -> str: + return f"{self.reason.value}: {self.field}" + + +def _fail(reason: InputReasonV1, field_name: str) -> NoReturn: + raise InputErrorV1(reason, field_name) + + +def _positive_u64(value: object, field_name: str) -> int: + if type(value) is not int: + _fail(InputReasonV1.WRONG_TYPE, field_name) + if value <= 0 or value >= 1 << 64: + _fail(InputReasonV1.INVALID_VALUE, field_name) + return value + + +def _logical_path(value: object) -> str: + if type(value) is not str or not value or value.startswith("/") or "\\" in value: + _fail(InputReasonV1.INVALID_PATH, "path") + try: + encoded = value.encode("ascii") + except UnicodeEncodeError: + _fail(InputReasonV1.INVALID_PATH, "path") + if ( + len(encoded) > 4096 + or any(byte < 0x20 or byte == 0x7F for byte in encoded) + or any(part in ("", ".", "..") for part in value.split("/")) + ): + _fail(InputReasonV1.INVALID_PATH, "path") + return value + + +class CanonicalInputLimitsV1(tuple): + """Caller-owned resource bounds for one in-memory canonical archive.""" + + __slots__ = () + + def __new__( + cls, + max_members: int, + max_file_bytes: int, + max_payload_bytes: int, + max_encoded_bytes: int | None = None, + ) -> CanonicalInputLimitsV1: + max_members = _positive_u64(max_members, "max_members") + max_file_bytes = _positive_u64(max_file_bytes, "max_file_bytes") + max_payload_bytes = _positive_u64(max_payload_bytes, "max_payload_bytes") + if max_encoded_bytes is None: + # USTAR adds one header block per member, at most one partial data + # block per member, two EOF blocks, then pads to one record. This + # is derived from the caller's bounds, not a fixture-specific cap. + maximum_unpadded = ( + max_payload_bytes + + (2 * _USTAR_BLOCK_BYTES - 1) * max_members + + _USTAR_EOF_BLOCKS * _USTAR_BLOCK_BYTES + ) + max_encoded_bytes = _round_up( + maximum_unpadded, + _USTAR_RECORD_BYTES, + ) + max_encoded_bytes = _positive_u64(max_encoded_bytes, "max_encoded_bytes") + return tuple.__new__( + cls, + ( + max_members, + max_file_bytes, + max_payload_bytes, + max_encoded_bytes, + ), + ) + + @property + def max_members(self) -> int: + return self[0] + + @property + def max_file_bytes(self) -> int: + return self[1] + + @property + def max_payload_bytes(self) -> int: + return self[2] + + @property + def max_encoded_bytes(self) -> int: + return self[3] + + +def _limits_are_valid(value: object) -> bool: + if type(value) is not CanonicalInputLimitsV1: + return False + try: + return tuple(CanonicalInputLimitsV1(*tuple(value))) == tuple(value) + except Exception: + return False + + +def _ustar_path_is_encodable(path: str, *, directory: bool = False) -> bool: + encoded = path.encode("ascii") + if directory: + # tarfile writes DIRTYPE without a trailing separator as one with it; + # the USTAR prefix split must validate the exact emitted header name. + encoded += b"/" + if len(encoded) <= 100: + return True + return any( + 0 < index <= 155 and len(encoded) - index - 1 <= 100 + for index, byte in enumerate(encoded) + if byte == ord("/") + ) + + +def _round_up(value: int, quantum: int) -> int: + return ((value + quantum - 1) // quantum) * quantum + + +def _encoded_ustar_length( + entries: tuple[tuple[str, int, bytes], ...], + directory_count: int, +) -> int: + data_bytes = sum( + _round_up(len(contents), _USTAR_BLOCK_BYTES) + for _path, _mode, contents in entries + ) + raw_bytes = ( + (len(entries) + directory_count + _USTAR_EOF_BLOCKS) + * _USTAR_BLOCK_BYTES + + data_bytes + ) + return _round_up(raw_bytes, _USTAR_RECORD_BYTES) + + +class SealedInputV1(tuple): + """Owned exact bytes carrying only integrity and an opaque caller binding.""" + + __slots__ = () + + def __new__( + cls, + binding_identity: bytes, + contents: bytes, + *, + _token: object, + ) -> SealedInputV1: + if _token is not _SEALED_INPUT_TOKEN: + raise TypeError("SealedInputV1 is created only by seal_input_v1") + if not _valid_digest(binding_identity): + raise TypeError("binding_identity must be one opaque nonzero digest") + if type(contents) is not bytes or not contents: + raise TypeError("sealed input must own nonempty exact bytes") + return tuple.__new__( + cls, + ( + binding_identity, + hashlib.sha256(contents).digest(), + len(contents), + contents, + ), + ) + + @property + def binding_identity(self) -> bytes: + return self[0] + + @property + def sha256(self) -> bytes: + return self[1] + + @property + def length(self) -> int: + return self[2] + + @property + def contents(self) -> bytes: + return self[3] + + +def seal_input_v1(binding_identity: bytes, contents: bytes) -> SealedInputV1: + """Seal exact bytes while treating their semantic binding as opaque.""" + + if type(binding_identity) is not bytes: + _fail(InputReasonV1.WRONG_TYPE, "binding_identity") + if not _valid_digest(binding_identity): + _fail(InputReasonV1.INVALID_VALUE, "binding_identity") + if type(contents) is not bytes: + _fail(InputReasonV1.WRONG_TYPE, "contents") + if not contents: + _fail(InputReasonV1.INVALID_VALUE, "contents") + return SealedInputV1( + binding_identity, + contents, + _token=_SEALED_INPUT_TOKEN, + ) + + +def sealed_input_is_intact_v1(value: object) -> bool: + """Recheck byte integrity without interpreting the caller-owned binding.""" + + if type(value) is not SealedInputV1: + return False + try: + return ( + _valid_digest(value.binding_identity) + and type(value.contents) is bytes + and bool(value.contents) + and value.length == len(value.contents) + and value.sha256 == hashlib.sha256(value.contents).digest() + ) + except Exception: + return False + + +def canonical_ustar_v1( + entries: tuple[tuple[str, int, bytes], ...], + limits: CanonicalInputLimitsV1, +) -> bytes: + """Encode one canonical normalized USTAR file tree.""" + + if ( + type(entries) is not tuple + or not entries + or not _limits_are_valid(limits) + ): + _fail(InputReasonV1.NONCANONICAL_SET, "entries") + if len(entries) > limits.max_members: + _fail(InputReasonV1.RESOURCE_LIMIT, "max_members") + parsed: list[tuple[str, int, bytes]] = [] + total_bytes = 0 + for entry in entries: + if type(entry) is not tuple or len(entry) != 3: + _fail(InputReasonV1.WRONG_TYPE, "entries") + path, mode, contents = entry + path = _logical_path(path) + if not _ustar_path_is_encodable(path): + _fail(InputReasonV1.INVALID_PATH, path) + if type(mode) is not int or mode not in (0o644, 0o755): + _fail(InputReasonV1.INVALID_MODE, path) + if type(contents) is not bytes: + _fail(InputReasonV1.WRONG_TYPE, path) + total_bytes += len(contents) + if len(contents) > limits.max_file_bytes: + _fail(InputReasonV1.RESOURCE_LIMIT, "max_file_bytes") + if total_bytes > limits.max_payload_bytes: + _fail(InputReasonV1.RESOURCE_LIMIT, "max_payload_bytes") + parsed.append((path, mode, contents)) + owned = tuple(parsed) + paths = tuple(path for path, _mode, _contents in owned) + if paths != tuple(sorted(paths)) or len(set(paths)) != len(entries): + _fail(InputReasonV1.NONCANONICAL_SET, "entries") + directories: set[str] = set() + for path, _mode, _contents in owned: + parts = path.split("/")[:-1] + for length in range(1, len(parts) + 1): + directories.add("/".join(parts[:length])) + for path in directories: + if not _ustar_path_is_encodable(path, directory=True): + _fail(InputReasonV1.INVALID_PATH, path) + namespace: dict[str, tuple[str, str]] = {} + for kind, values in (("directory", tuple(sorted(directories))), ("file", paths)): + for path in values: + folded = path.lower() + prior = namespace.get(folded) + coordinate = (kind, path) + if prior is not None and prior != coordinate: + _fail(InputReasonV1.NONCANONICAL_SET, path) + namespace[folded] = coordinate + if ( + directories.intersection(paths) + or len(directories) + len(owned) > limits.max_members + ): + if directories.intersection(paths): + _fail(InputReasonV1.NONCANONICAL_SET, "file-directory collision") + _fail(InputReasonV1.RESOURCE_LIMIT, "max_members") + encoded_length = _encoded_ustar_length(owned, len(directories)) + if encoded_length > limits.max_encoded_bytes: + _fail(InputReasonV1.RESOURCE_LIMIT, "max_encoded_bytes") + + output = io.BytesIO() + try: + with tarfile.open(fileobj=output, mode="w", format=tarfile.USTAR_FORMAT) as archive: + for path in sorted(directories, key=lambda value: (value.count("/"), value)): + member = tarfile.TarInfo(path) + member.type = tarfile.DIRTYPE + member.mode = 0o755 + member.uid = 0 + member.gid = 0 + member.uname = "" + member.gname = "" + member.mtime = 0 + member.size = 0 + archive.addfile(member) + for path, mode, contents in owned: + member = tarfile.TarInfo(path) + member.type = tarfile.REGTYPE + member.mode = mode + member.uid = 0 + member.gid = 0 + member.uname = "" + member.gname = "" + member.mtime = 0 + member.size = len(contents) + archive.addfile(member, io.BytesIO(contents)) + except (OSError, OverflowError, tarfile.TarError, ValueError): + _fail(InputReasonV1.INVALID_PATH, "USTAR encoding") + encoded = output.getvalue() + if len(encoded) != encoded_length: + _fail(InputReasonV1.NONCANONICAL_SET, "USTAR size mismatch") + return encoded diff --git a/proof/region/v1/build/transport.py b/proof/region/v1/build/transport.py new file mode 100644 index 00000000..1a799761 --- /dev/null +++ b/proof/region/v1/build/transport.py @@ -0,0 +1,3875 @@ +#!/usr/bin/env python3 +"""Engine-neutral, causally observed Docker BUILD transport.""" + +from __future__ import annotations + +import hashlib +import json +import os +import platform +import re +import selectors +import shutil +import signal +import stat +import subprocess +import tempfile +import threading +import time +from enum import StrEnum +from pathlib import Path +from typing import Callable, Protocol, TypeAlias + +from . import input as _build_input + + +# These versioned observer bounds are not physical constants or a claim that +# every client build fits. Changing one requires a transport-version, +# streaming/resource review, and a targeted native high-water gate. A client +# policy may only tighten them. +BUILD_STDOUT_LIMIT_V1 = 16 * 1024 * 1024 +BUILD_STDERR_LIMIT_V1 = 16 * 1024 * 1024 +BUILD_TIMEOUT_NS_V1 = 2 * 60 * 60 * 1_000_000_000 +DOCKER_PROBE_OUTPUT_LIMIT_V1 = 1024 * 1024 +DOCKER_PROBE_TIMEOUT_NS_V1 = 30 * 1_000_000_000 + +# These are observer scheduling/termination mechanics, not successful-build +# evidence coordinates. CPU, RAM and PID containment belong to the declared +# disposable worker, outside this Docker transport. +_IO_CHUNK_BYTES_V1 = 64 * 1024 +_POLL_SLICE_SECONDS_V1 = 0.1 +_PROCESS_STOP_TIMEOUT_SECONDS_V1 = 30 +# Details enter receipts as bounded diagnostic evidence; this avoids allowing +# an adapter error string to become an unbounded transport payload. +_DIAGNOSTIC_DETAIL_TEXT_LIMIT_V1 = 4096 +_INVALID_CID_ROOT_CLEANUP_DETAIL_V1 = ( + "native Docker CID root cleanup detail is not canonical" +) +_OBSERVER_AND_CID_ROOT_CLEANUP_FAILURE_V1 = ( + "native Docker build observation and CID root cleanup both failed" +) +_PATH_TYPE = type(Path("/")) +_NATIVE_CID_ROOT_PREFIX_V1 = "labcolors-docker-cid-" + +_BUILD_INPUT_PROGRESS_TOKEN = object() +_BUILD_INPUT_TRANSFER_TOKEN = object() +_DOCKER_COMMAND_EXITED_TOKEN = object() +_DOCKER_BUILD_EXITED_TOKEN = object() +_NATIVE_RUN_LEASE_TOKEN = object() +_DOCKER_ISSUED_CONTAINER_ID_TOKEN = object() +_BUILD_SESSION_TOKEN = object() +_TWO_BUILD_OBSERVATION_TOKEN = object() + + +class _NativeOwnershipLostV1(RuntimeError): + """A post-fork copy must not act on its creator's native resources.""" + + +def _valid_digest(value: object) -> bool: + return type(value) is bytes and len(value) == 32 and value != bytes(32) + + +def _blob(value: bytes) -> bytes: + return len(value).to_bytes(8, "big") + value + + +def _identity(label: bytes, chunks: tuple[bytes, ...]) -> bytes: + payload = b"".join(_blob(chunk) for chunk in chunks) + return hashlib.sha256(label + len(payload).to_bytes(8, "big") + payload).digest() + + +def _retain_first_base_exception_v1( + retained: BaseException | None, + current: BaseException, +) -> BaseException: + return retained if retained is not None else current + + +def _canonical_diagnostic_detail_v1(value: object) -> str | None: + if ( + type(value) is str + and value + and len(value) <= _DIAGNOSTIC_DETAIL_TEXT_LIMIT_V1 + ): + return value + return None + + +def _canonical_cid_root_cleanup_detail_v1(value: object) -> str | None: + if value is None: + return None + detail = _canonical_diagnostic_detail_v1(value) + if detail is not None: + return detail + return _INVALID_CID_ROOT_CLEANUP_DETAIL_V1 + + +def _pinned_image_reference(value: object) -> bool: + if type(value) is not str or value.count("@sha256:") != 1: + return False + repository, digest = value.split("@sha256:", 1) + components = repository.split("/") + if any(not component for component in components): + return False + first, *path_components = components + if ":" in first: + domain, separator, port = first.rpartition(":") + if ( + not separator + or not domain + or not port + or any(character not in "0123456789" for character in port) + ): + return False + first = domain + repository_component = re.compile( + r"[a-z0-9]+(?:[._-]+[a-z0-9]+)*\Z" + ) + return ( + bool(repository) + and repository[0].isalnum() + and repository[-1].isalnum() + and repository == repository.lower() + and repository_component.fullmatch(first) is not None + and all( + repository_component.fullmatch(component) is not None + for component in path_components + ) + and len(digest) == 64 + and all(character in "0123456789abcdef" for character in digest) + ) + + +def _encoded_policy_text( + value: object, + maximum: int, + field_name: str, + *, + allow_newlines: bool = False, +) -> str: + if type(value) is not str or not value or "\0" in value: + raise TypeError(f"invalid {field_name}") + try: + encoded = value.encode("utf-8") + except UnicodeEncodeError as error: + raise TypeError(f"invalid {field_name}") from error + if ( + len(encoded) > maximum + or not allow_newlines and ("\n" in value or "\r" in value) + ): + raise TypeError(f"invalid {field_name}") + return value + + +class DockerUserModeV1(StrEnum): + """Declare which unsealed-host user coordinates Docker observes.""" + + HOST_EFFECTIVE_IDS = "host_effective_ids" + + +class DockerBuildPolicyV1(tuple): + """Deeply immutable coordinates for one bounded Docker build transport.""" + + __slots__ = () + + def __new__( + cls, + image_reference: str, + platform: str, + hostname: str, + bootstrap: str, + bootstrap_argv0: str, + tmpfs_specs: tuple[str, ...], + user_mode: DockerUserModeV1, + stdout_limit: int, + stderr_limit: int, + build_timeout_ns: int, + probe_output_limit: int, + probe_timeout_ns: int, + ) -> DockerBuildPolicyV1: + strings = tuple( + _encoded_policy_text( + value, + maximum, + field_name, + allow_newlines=field_name == "bootstrap", + ) + for field_name, value, maximum in ( + ("image_reference", image_reference, 512), + ("platform", platform, 64), + ("hostname", hostname, 64), + ("bootstrap", bootstrap, 64 * 1024), + ("bootstrap_argv0", bootstrap_argv0, 128), + ) + ) + ( + image_reference, + platform, + hostname, + bootstrap, + bootstrap_argv0, + ) = strings + if ( + platform != "linux/amd64" + or not _pinned_image_reference(image_reference) + ): + raise TypeError("policy requires one pinned linux/amd64 image") + if ( + any( + character not in "abcdefghijklmnopqrstuvwxyz0123456789-" + for character in hostname + ) + ): + raise TypeError("invalid Docker hostname") + if type(tmpfs_specs) is not tuple or not tmpfs_specs: + raise TypeError("invalid tmpfs_specs") + owned_tmpfs: list[str] = [] + for spec in tmpfs_specs: + parsed = _encoded_policy_text(spec, 4096, "tmpfs_specs") + if not parsed.startswith("/"): + raise TypeError("invalid tmpfs_specs") + owned_tmpfs.append(parsed) + tmpfs_specs = tuple(owned_tmpfs) + if len(set(tmpfs_specs)) != len(tmpfs_specs): + raise TypeError("invalid tmpfs_specs") + if type(user_mode) is not DockerUserModeV1: + raise TypeError("invalid Docker user mode") + limits = ( + (stdout_limit, BUILD_STDOUT_LIMIT_V1, "stdout_limit"), + (stderr_limit, BUILD_STDERR_LIMIT_V1, "stderr_limit"), + (build_timeout_ns, BUILD_TIMEOUT_NS_V1, "build_timeout_ns"), + (probe_output_limit, DOCKER_PROBE_OUTPUT_LIMIT_V1, "probe_output_limit"), + (probe_timeout_ns, DOCKER_PROBE_TIMEOUT_NS_V1, "probe_timeout_ns"), + ) + if any( + type(value) is not int or value <= 0 or value > maximum + for value, maximum, _name in limits + ): + raise TypeError("invalid Docker policy limit") + return tuple.__new__( + cls, + ( + image_reference, + platform, + hostname, + bootstrap, + bootstrap_argv0, + tmpfs_specs, + user_mode, + stdout_limit, + stderr_limit, + build_timeout_ns, + probe_output_limit, + probe_timeout_ns, + ), + ) + + @property + def image_reference(self) -> str: + return self[0] + + @property + def platform(self) -> str: + return self[1] + + @property + def hostname(self) -> str: + return self[2] + + @property + def bootstrap(self) -> str: + return self[3] + + @property + def bootstrap_argv0(self) -> str: + return self[4] + + @property + def tmpfs_specs(self) -> tuple[str, ...]: + return self[5] + + @property + def user_mode(self) -> DockerUserModeV1: + return self[6] + + @property + def stdout_limit(self) -> int: + return self[7] + + @property + def stderr_limit(self) -> int: + return self[8] + + @property + def build_timeout_ns(self) -> int: + return self[9] + + @property + def probe_output_limit(self) -> int: + return self[10] + + @property + def probe_timeout_ns(self) -> int: + return self[11] + + +def docker_policy_is_valid_v1(value: object) -> bool: + if type(value) is not DockerBuildPolicyV1: + return False + try: + return tuple(DockerBuildPolicyV1(*tuple(value))) == tuple(value) + except Exception: + return False + + +def transport_policy_identity_v1(policy: DockerBuildPolicyV1) -> bytes: + """Bind every declared transport-policy coordinate in constructor order.""" + + if not docker_policy_is_valid_v1(policy): + raise TypeError("policy must be canonical DockerBuildPolicyV1") + return _identity( + b"labcolors.proof-region.docker-transport-policy.v1\0", + ( + policy.image_reference.encode("utf-8"), + policy.platform.encode("utf-8"), + policy.hostname.encode("utf-8"), + policy.bootstrap.encode("utf-8"), + policy.bootstrap_argv0.encode("utf-8"), + len(policy.tmpfs_specs).to_bytes(4, "big"), + *(spec.encode("utf-8") for spec in policy.tmpfs_specs), + policy.user_mode.value.encode("ascii"), + policy.stdout_limit.to_bytes(8, "big"), + policy.stderr_limit.to_bytes(8, "big"), + policy.build_timeout_ns.to_bytes(8, "big"), + policy.probe_output_limit.to_bytes(8, "big"), + policy.probe_timeout_ns.to_bytes(8, "big"), + ), + ) + + +class _NativeCommandSlotV1(StrEnum): + CLI_PATH = "cli_path" + IMAGE_REFERENCE = "image_reference" + PLATFORM = "platform" + ORDERED_TMPFS_SPECS = "ordered_tmpfs_specs" + HOSTNAME = "hostname" + HOST_USER = "host_user" + CID_FILE = "cid_file" + BOOTSTRAP = "bootstrap" + BOOTSTRAP_ARGV0 = "bootstrap_argv0" + INPUT_LENGTH = "input_length" + INPUT_SHA256 = "input_sha256" + CONTAINER_COORDINATE = "container_coordinate" + CONTAINER_FILTER = "container_filter" + + +class _NativeCommandTokenV1(tuple): + """One tagged literal or named slot in the native command grammar.""" + + __slots__ = () + + def __new__( + cls, + literal: str | None = None, + slot: _NativeCommandSlotV1 | None = None, + ) -> _NativeCommandTokenV1: + if (literal is None) == (slot is None): + raise TypeError("command token must be exactly one literal or slot") + if literal is not None: + if type(literal) is not str or not literal or "\0" in literal: + raise TypeError("invalid native command literal") + return tuple.__new__(cls, (b"literal", literal)) + if type(slot) is not _NativeCommandSlotV1: + raise TypeError("invalid native command slot") + return tuple.__new__(cls, (b"slot", slot)) + + @property + def tag(self) -> bytes: + return self[0] + + @property + def value(self) -> str | _NativeCommandSlotV1: + return self[1] + + +def _literal_v1(value: str) -> _NativeCommandTokenV1: + return _NativeCommandTokenV1(literal=value) + + +def _slot_v1(value: _NativeCommandSlotV1) -> _NativeCommandTokenV1: + return _NativeCommandTokenV1(slot=value) + + +_NATIVE_COMMAND_TEMPLATES_V1: tuple[ + tuple[str, tuple[_NativeCommandTokenV1, ...]], ... +] = ( + ( + "version_probe", + ( + _slot_v1(_NativeCommandSlotV1.CLI_PATH), + _literal_v1("version"), + _literal_v1("--format"), + _literal_v1("{{json .Server}}"), + ), + ), + ( + "image_inspect", + ( + _slot_v1(_NativeCommandSlotV1.CLI_PATH), + _literal_v1("image"), + _literal_v1("inspect"), + _slot_v1(_NativeCommandSlotV1.IMAGE_REFERENCE), + ), + ), + ( + "build", + ( + _slot_v1(_NativeCommandSlotV1.CLI_PATH), + _literal_v1("run"), + _literal_v1("--rm"), + _literal_v1("--interactive"), + _literal_v1("--pull"), + _literal_v1("never"), + _literal_v1("--platform"), + _slot_v1(_NativeCommandSlotV1.PLATFORM), + _literal_v1("--network"), + _literal_v1("none"), + _literal_v1("--read-only"), + _literal_v1("--tmpfs"), + _slot_v1(_NativeCommandSlotV1.ORDERED_TMPFS_SPECS), + _literal_v1("--cap-drop"), + _literal_v1("ALL"), + _literal_v1("--security-opt"), + _literal_v1("no-new-privileges:true"), + _literal_v1("--hostname"), + _slot_v1(_NativeCommandSlotV1.HOSTNAME), + _literal_v1("--user"), + _slot_v1(_NativeCommandSlotV1.HOST_USER), + _literal_v1("--workdir"), + _literal_v1("/"), + _literal_v1("--cidfile"), + _slot_v1(_NativeCommandSlotV1.CID_FILE), + _literal_v1("--entrypoint"), + _literal_v1("/usr/bin/env"), + _slot_v1(_NativeCommandSlotV1.IMAGE_REFERENCE), + _literal_v1("-i"), + _literal_v1("PATH=/usr/local/bin:/usr/bin:/bin"), + _literal_v1("LC_ALL=C"), + _literal_v1("LANG=C"), + _literal_v1("TZ=UTC"), + _literal_v1("HOME=/nonexistent"), + _literal_v1("/bin/sh"), + _literal_v1("-c"), + _slot_v1(_NativeCommandSlotV1.BOOTSTRAP), + _slot_v1(_NativeCommandSlotV1.BOOTSTRAP_ARGV0), + _slot_v1(_NativeCommandSlotV1.INPUT_LENGTH), + _slot_v1(_NativeCommandSlotV1.INPUT_SHA256), + ), + ), + ( + "cleanup_rm", + ( + _slot_v1(_NativeCommandSlotV1.CLI_PATH), + _literal_v1("container"), + _literal_v1("rm"), + _literal_v1("--force"), + _slot_v1(_NativeCommandSlotV1.CONTAINER_COORDINATE), + ), + ), + ( + "cleanup_inspect", + ( + _slot_v1(_NativeCommandSlotV1.CLI_PATH), + _literal_v1("container"), + _literal_v1("inspect"), + _literal_v1("--format"), + _literal_v1("{{.Id}}"), + _slot_v1(_NativeCommandSlotV1.CONTAINER_COORDINATE), + ), + ), + ( + "cleanup_ls", + ( + _slot_v1(_NativeCommandSlotV1.CLI_PATH), + _literal_v1("container"), + _literal_v1("ls"), + _literal_v1("--all"), + _literal_v1("--quiet"), + _literal_v1("--no-trunc"), + _literal_v1("--filter"), + _slot_v1(_NativeCommandSlotV1.CONTAINER_FILTER), + ), + ), +) + + +class _NativeStdioModeV1(StrEnum): + PIPE = "pipe" + DEVNULL = "devnull" + + +def _native_stdio_value_v1(mode: _NativeStdioModeV1) -> int: + if type(mode) is not _NativeStdioModeV1: + raise TypeError("invalid native stdio mode") + if mode is _NativeStdioModeV1.PIPE: + return subprocess.PIPE + if mode is _NativeStdioModeV1.DEVNULL: + return subprocess.DEVNULL + raise TypeError("invalid native stdio mode") + + +class _NativeProcessContextV1(tuple): + """One fixed, identity-bound child-launch context for the Docker CLI.""" + + __slots__ = () + + def __new__( + cls, + environment: tuple[tuple[str, str], ...], + cwd: str, + umask: int, + close_fds: bool, + restore_signals: bool, + start_new_session: bool, + stdin_with_input: _NativeStdioModeV1, + stdin_without_input: _NativeStdioModeV1, + stdout: _NativeStdioModeV1, + stderr: _NativeStdioModeV1, + ) -> _NativeProcessContextV1: + if type(environment) is not tuple or not environment: + raise TypeError("invalid native process environment") + owned_environment: list[tuple[str, str]] = [] + for entry in environment: + if type(entry) is not tuple or len(entry) != 2: + raise TypeError("invalid native process environment") + name, value = entry + if ( + type(name) is not str + or not name + or re.fullmatch(r"[A-Z_][A-Z0-9_]*", name) is None + or type(value) is not str + or "\0" in value + or "\n" in value + or "\r" in value + ): + raise TypeError("invalid native process environment") + try: + value.encode("utf-8") + except UnicodeEncodeError as error: + raise TypeError("invalid native process environment") from error + owned_environment.append((name, value)) + canonical_environment = tuple(owned_environment) + if ( + canonical_environment != tuple(sorted(canonical_environment)) + or len({name for name, _value in canonical_environment}) + != len(canonical_environment) + ): + raise TypeError("native process environment must be ordered and unique") + if ( + type(cwd) is not str + or not cwd + or "\0" in cwd + or "\n" in cwd + or "\r" in cwd + or not os.path.isabs(cwd) + ): + raise TypeError("invalid native process cwd") + try: + cwd.encode("utf-8") + except UnicodeEncodeError as error: + raise TypeError("invalid native process cwd") from error + if type(umask) is not int or umask < 0 or umask > 0o777: + raise TypeError("invalid native process umask") + if any( + type(value) is not bool + for value in (close_fds, restore_signals, start_new_session) + ): + raise TypeError("invalid native process launch flags") + if any( + type(value) is not _NativeStdioModeV1 + for value in ( + stdin_with_input, + stdin_without_input, + stdout, + stderr, + ) + ): + raise TypeError("invalid native stdio topology") + return tuple.__new__( + cls, + ( + canonical_environment, + cwd, + umask, + close_fds, + restore_signals, + start_new_session, + stdin_with_input, + stdin_without_input, + stdout, + stderr, + ), + ) + + @property + def environment(self) -> tuple[tuple[str, str], ...]: + return self[0] + + @property + def cwd(self) -> str: + return self[1] + + @property + def umask(self) -> int: + return self[2] + + @property + def close_fds(self) -> bool: + return self[3] + + @property + def restore_signals(self) -> bool: + return self[4] + + @property + def start_new_session(self) -> bool: + return self[5] + + @property + def stdin_with_input(self) -> _NativeStdioModeV1: + return self[6] + + @property + def stdin_without_input(self) -> _NativeStdioModeV1: + return self[7] + + @property + def stdout(self) -> _NativeStdioModeV1: + return self[8] + + @property + def stderr(self) -> _NativeStdioModeV1: + return self[9] + + def identity_chunks_v1(self) -> tuple[bytes, ...]: + return ( + b"native-process-context.v1", + len(self.environment).to_bytes(4, "big"), + *( + chunk + for name, value in self.environment + for chunk in (name.encode("ascii"), value.encode("utf-8")) + ), + self.cwd.encode("utf-8"), + self.umask.to_bytes(4, "big"), + bytes((self.close_fds,)), + bytes((self.restore_signals,)), + bytes((self.start_new_session,)), + b"native-stdio-topology.v1", + b"stdin-with-input", + self.stdin_with_input.value.encode("ascii"), + b"stdin-without-input", + self.stdin_without_input.value.encode("ascii"), + b"stdout", + self.stdout.value.encode("ascii"), + b"stderr", + self.stderr.value.encode("ascii"), + ) + + def popen_kwargs_v1(self, receives_stdin: bool) -> dict[str, object]: + if type(receives_stdin) is not bool: + raise TypeError("receives_stdin must be bool") + return { + "stdin": _native_stdio_value_v1( + self.stdin_with_input + if receives_stdin + else self.stdin_without_input + ), + "stdout": _native_stdio_value_v1(self.stdout), + "stderr": _native_stdio_value_v1(self.stderr), + "cwd": self.cwd, + "env": dict(self.environment), + "close_fds": self.close_fds, + "restore_signals": self.restore_signals, + "start_new_session": self.start_new_session, + "umask": self.umask, + } + + +# The Docker CLI is part of an evidence-producing observation. Its launch +# cannot inherit locale, config, cwd, umask or session state from the host: +# this immutable value both renders Popen kwargs and enters the command +# identity, so a future change cannot silently alter what the observer ran. +_NATIVE_PROCESS_CONTEXT_V1 = _NativeProcessContextV1( + ( + ("DOCKER_CONFIG", "/nonexistent"), + ("HOME", "/nonexistent"), + ("LANG", "C"), + ("LC_ALL", "C"), + ("PATH", "/usr/bin:/bin"), + ("TZ", "UTC"), + ), + "/", + 0o077, + True, + True, + True, + _NativeStdioModeV1.PIPE, + _NativeStdioModeV1.DEVNULL, + _NativeStdioModeV1.PIPE, + _NativeStdioModeV1.PIPE, +) + + +def native_command_contract_identity_v1() -> bytes: + chunks: list[bytes] = [len(_NATIVE_COMMAND_TEMPLATES_V1).to_bytes(4, "big")] + for name, tokens in _NATIVE_COMMAND_TEMPLATES_V1: + chunks.extend((name.encode("ascii"), len(tokens).to_bytes(4, "big"))) + for token in tokens: + value = token.value + chunks.extend( + ( + token.tag, + ( + value.value.encode("ascii") + if type(value) is _NativeCommandSlotV1 + else value.encode("utf-8") + ), + ) + ) + chunks.extend(_NATIVE_PROCESS_CONTEXT_V1.identity_chunks_v1()) + return _identity( + b"labcolors.proof-region.native-command-contract.v1\0", + tuple(chunks), + ) + + +def _native_command_path_v1(value: object) -> tuple[Path, bytes]: + if type(value) is not _PATH_TYPE or not value.is_absolute(): + raise TypeError("native command path must be an absolute Path") + try: + encoded = os.fsencode(value) + except (TypeError, UnicodeEncodeError) as error: + raise TypeError("native command path is not filesystem-encodable") from error + if not encoded or b"\0" in encoded: + raise TypeError("invalid native command path bytes") + return value, encoded + + +class NativeCommandCoordinateV1(tuple): + """Exact filesystem command coordinate bound to the native argv grammar.""" + + __slots__ = () + + def __new__( + cls, + path_bytes: bytes, + command_contract_identity: bytes, + ) -> NativeCommandCoordinateV1: + if type(path_bytes) is not bytes or not path_bytes or b"\0" in path_bytes: + raise TypeError("invalid native command path bytes") + try: + path = Path(os.fsdecode(path_bytes)) + except (TypeError, UnicodeDecodeError) as error: + raise TypeError("invalid native command path bytes") from error + _owned_path, encoded = _native_command_path_v1(path) + if encoded != path_bytes: + raise TypeError("native command path bytes are not canonical") + if command_contract_identity != native_command_contract_identity_v1(): + raise TypeError("foreign native command contract") + return tuple.__new__(cls, (path_bytes, command_contract_identity)) + + @property + def path_bytes(self) -> bytes: + return self[0] + + @property + def path(self) -> Path: + return Path(os.fsdecode(self.path_bytes)) + + @property + def command_contract_identity(self) -> bytes: + return self[1] + + @property + def identity(self) -> bytes: + return _native_command_coordinate_identity_v1(self) + + +def native_command_coordinate_v1(path: Path) -> NativeCommandCoordinateV1: + _owned, encoded = _native_command_path_v1(path) + return NativeCommandCoordinateV1( + encoded, + native_command_contract_identity_v1(), + ) + + +def _native_command_coordinate_identity_v1( + coordinate: NativeCommandCoordinateV1, +) -> bytes: + if type(coordinate) is not NativeCommandCoordinateV1: + raise TypeError("coordinate must be NativeCommandCoordinateV1") + canonical = NativeCommandCoordinateV1(*tuple(coordinate)) + if tuple(canonical) != tuple(coordinate): + raise TypeError("coordinate is not canonical") + return _identity( + b"labcolors.proof-region.native-command-coordinate.v1\0", + ( + coordinate.command_contract_identity, + coordinate.path_bytes, + ), + ) + + +def _render_native_command_v1( + template_name: str, + command_coordinate: NativeCommandCoordinateV1, + values: dict[_NativeCommandSlotV1, tuple[str, ...]], +) -> tuple[str, ...]: + if type(template_name) is not str or type(values) is not dict: + raise TypeError("invalid native command expansion") + canonical_coordinate = NativeCommandCoordinateV1(*tuple(command_coordinate)) + templates = dict(_NATIVE_COMMAND_TEMPLATES_V1) + try: + tokens = templates[template_name] + except KeyError as error: + raise TypeError("unknown native command template") from error + owned_values = dict(values) + if _NativeCommandSlotV1.CLI_PATH in owned_values: + raise TypeError("native command path is owned by its coordinate") + owned_values[_NativeCommandSlotV1.CLI_PATH] = ( + os.fsdecode(canonical_coordinate.path_bytes), + ) + expected_slots = { + token.value + for token in tokens + if token.tag == b"slot" + } + if set(owned_values) != expected_slots: + raise TypeError("native command slots do not match its template") + command: list[str] = [] + for token in tokens: + if token.tag == b"literal": + command.append(token.value) + continue + slot = token.value + expanded = owned_values[slot] + if ( + type(expanded) is not tuple + or ( + slot is not _NativeCommandSlotV1.ORDERED_TMPFS_SPECS + and len(expanded) != 1 + ) + or any( + type(value) is not str or not value or "\0" in value + for value in expanded + ) + ): + raise TypeError("invalid native command slot expansion") + if slot is _NativeCommandSlotV1.ORDERED_TMPFS_SPECS: + if not expanded or not command: + raise TypeError("invalid ordered tmpfs template") + repeated_literal = command.pop() + for value in expanded: + command.extend((repeated_literal, value)) + continue + command.extend(expanded) + try: + tuple(os.fsencode(value) for value in command) + except (TypeError, UnicodeEncodeError) as error: + raise TypeError("native command contains an unencodable coordinate") from error + return tuple(command) + + +class DockerBlockerReasonV1(StrEnum): + HOST_NOT_LINUX_AMD64 = "host_not_linux_amd64" + HOST_USER_UNAVAILABLE = "host_user_unavailable" + DOCKER_UNAVAILABLE = "docker_unavailable" + IMAGE_UNAVAILABLE = "image_unavailable" + IMAGE_IDENTITY_MISMATCH = "image_identity_mismatch" + BACKEND_CONTRACT = "backend_contract" + + +class DockerUnsupportedV1(tuple): + __slots__ = () + + def __new__( + cls, + reason: DockerBlockerReasonV1, + detail: str, + ) -> DockerUnsupportedV1: + if type(reason) is not DockerBlockerReasonV1: + raise TypeError("invalid Docker blocker reason") + if _canonical_diagnostic_detail_v1(detail) is None: + raise TypeError("invalid Docker blocker detail") + return tuple.__new__(cls, (reason, detail)) + + @property + def reason(self) -> DockerBlockerReasonV1: + return self[0] + + @property + def detail(self) -> str: + return self[1] + + +class DockerDaemonObservationV1(tuple): + """Exact stdout bytes observed from the two admitted Docker probes.""" + + __slots__ = () + + def __new__( + cls, + server_stdout: bytes, + image_inspect_stdout: bytes, + ) -> DockerDaemonObservationV1: + for value, field_name in ( + (server_stdout, "server_stdout"), + (image_inspect_stdout, "image_inspect_stdout"), + ): + if ( + type(value) is not bytes + or not value + or len(value) > DOCKER_PROBE_OUTPUT_LIMIT_V1 + ): + raise TypeError(f"invalid Docker daemon {field_name}") + return tuple.__new__(cls, (server_stdout, image_inspect_stdout)) + + @property + def server_stdout(self) -> bytes: + return self[0] + + @property + def image_inspect_stdout(self) -> bytes: + return self[1] + + @property + def identity(self) -> bytes: + return _docker_daemon_observation_identity_v1(self) + + +def _docker_daemon_observation_identity_v1( + observation: DockerDaemonObservationV1, +) -> bytes: + if type(observation) is not DockerDaemonObservationV1: + raise TypeError("observation must be DockerDaemonObservationV1") + canonical = DockerDaemonObservationV1(*tuple(observation)) + if tuple(canonical) != tuple(observation): + raise TypeError("daemon observation is not canonical") + return _identity( + b"labcolors.proof-region.docker-daemon-observation.v1\0", + ( + observation.server_stdout, + observation.image_inspect_stdout, + ), + ) + + +def _host_user_identity_v1(host_user: tuple[int, int]) -> bytes: + owned = _host_user_coordinates(host_user) + return _identity( + b"labcolors.proof-region.host-user.v1\0", + ( + owned[0].to_bytes(4, "big"), + owned[1].to_bytes(4, "big"), + ), + ) + + +class DockerSupportedV1(tuple): + """Canonical capability observed for one exact native Docker coordinate.""" + + __slots__ = () + + def __new__( + cls, + policy: DockerBuildPolicyV1, + daemon_observation: DockerDaemonObservationV1, + command_coordinate: NativeCommandCoordinateV1, + host_user: tuple[int, int], + ) -> DockerSupportedV1: + if not docker_policy_is_valid_v1(policy): + raise TypeError("invalid Docker policy capability") + if type(daemon_observation) is not DockerDaemonObservationV1: + raise TypeError("invalid Docker daemon observation") + canonical_daemon = DockerDaemonObservationV1(*tuple(daemon_observation)) + if ( + tuple(canonical_daemon) != tuple(daemon_observation) + or len(canonical_daemon.server_stdout) > policy.probe_output_limit + or len(canonical_daemon.image_inspect_stdout) + > policy.probe_output_limit + ): + raise TypeError("Docker daemon observation is not canonical") + if type(command_coordinate) is not NativeCommandCoordinateV1: + raise TypeError("invalid native Docker command coordinate") + canonical_command = NativeCommandCoordinateV1(*tuple(command_coordinate)) + if tuple(canonical_command) != tuple(command_coordinate): + raise TypeError("native Docker command coordinate is not canonical") + owned_user = _host_user_coordinates(host_user) + return tuple.__new__( + cls, + (policy, daemon_observation, command_coordinate, owned_user), + ) + + @property + def policy(self) -> DockerBuildPolicyV1: + return self[0] + + @property + def daemon_observation(self) -> DockerDaemonObservationV1: + return self[1] + + @property + def command_coordinate(self) -> NativeCommandCoordinateV1: + return self[2] + + @property + def host_user(self) -> tuple[int, int]: + return self[3] + + @property + def policy_identity(self) -> bytes: + return transport_policy_identity_v1(self.policy) + + @property + def daemon_observation_identity(self) -> bytes: + return _docker_daemon_observation_identity_v1(self.daemon_observation) + + @property + def command_coordinate_identity(self) -> bytes: + return _native_command_coordinate_identity_v1(self.command_coordinate) + + @property + def host_user_identity(self) -> bytes: + return _host_user_identity_v1(self.host_user) + + @property + def identity(self) -> bytes: + return docker_capability_identity_v1(self) + + +def _docker_supported_is_valid_v1(value: object) -> bool: + if type(value) is not DockerSupportedV1: + return False + try: + return tuple(DockerSupportedV1(*tuple(value))) == tuple(value) + except Exception: + return False + + +def docker_capability_identity_v1(capability: DockerSupportedV1) -> bytes: + if not _docker_supported_is_valid_v1(capability): + raise TypeError("capability must be canonical DockerSupportedV1") + return _identity( + b"labcolors.proof-region.docker-capability.v1\0", + ( + capability.policy_identity, + capability.command_coordinate_identity, + capability.daemon_observation_identity, + capability.host_user[0].to_bytes(4, "big"), + capability.host_user[1].to_bytes(4, "big"), + ), + ) + + +DockerCapabilityReportV1: TypeAlias = DockerSupportedV1 | DockerUnsupportedV1 + + +def _docker_unsupported_is_valid_v1(value: object) -> bool: + if type(value) is not DockerUnsupportedV1: + return False + try: + return tuple(DockerUnsupportedV1(*tuple(value))) == tuple(value) + except Exception: + return False + + +def _absolute_path(value: object, field_name: str) -> Path: + if type(value) is not _PATH_TYPE or not value.is_absolute(): + raise TypeError(f"{field_name} must be an absolute Path") + try: + encoded = os.fsencode(value) + except (TypeError, UnicodeEncodeError) as error: + raise TypeError(f"{field_name} is not filesystem-encodable") from error + if ( + not encoded + or b"\0" in encoded + or any(character in str(value) for character in (",", "\n", "\r")) + ): + raise TypeError(f"{field_name} is not Docker-mount-safe") + return value + + +def _host_user_coordinates(value: object) -> tuple[int, int]: + if ( + type(value) is not tuple + or len(value) != 2 + or any(type(item) is not int or item < 0 or item >= 1 << 32 for item in value) + ): + raise TypeError("host_user must be one exact Linux uid/gid pair") + return value + + +class DockerBuildRequestV1(tuple): + """Semantic BUILD coordinates; the adapter owns all host resources.""" + + __slots__ = () + + def __new__( + cls, + attempt: int, + capability: DockerSupportedV1, + input_bundle: _build_input.SealedInputV1, + max_output_bytes: int, + ) -> DockerBuildRequestV1: + if type(attempt) is not int or attempt not in (1, 2): + raise TypeError("attempt must be 1 or 2") + if not _docker_supported_is_valid_v1(capability): + raise TypeError("capability must be canonical DockerSupportedV1") + if not _build_input.sealed_input_is_intact_v1(input_bundle): + raise TypeError("input_bundle must preserve exact sealed bytes") + if ( + type(max_output_bytes) is not int + or max_output_bytes <= 0 + or max_output_bytes > capability.policy.stdout_limit + ): + raise TypeError("invalid executable output limit") + return tuple.__new__( + cls, + ( + attempt, + capability, + input_bundle, + max_output_bytes, + ), + ) + + @property + def attempt(self) -> int: + return self[0] + + @property + def capability(self) -> DockerSupportedV1: + return self[1] + + @property + def input_bundle(self) -> _build_input.SealedInputV1: + return self[2] + + @property + def max_output_bytes(self) -> int: + return self[3] + + +def _docker_build_request_is_valid_v1( + value: object, + capability: DockerSupportedV1, +) -> bool: + if ( + type(value) is not DockerBuildRequestV1 + or not _docker_supported_is_valid_v1(capability) + ): + return False + try: + canonical = DockerBuildRequestV1(*tuple(value)) + return ( + tuple(canonical) == tuple(value) + and canonical.capability == capability + and _build_input.sealed_input_is_intact_v1(canonical.input_bundle) + ) + except Exception: + return False + + +class _NativeRunLeaseV1: + """Adapter-owned authority for one Docker-issued container ID file.""" + + __slots__ = ( + "_owner", + "_creator_pid", + "_capability", + "_root", + "_cid_file", + "_launched", + "_released", + ) + + def __init__( + self, + owner: object, + creator_pid: int, + capability: DockerSupportedV1, + root: Path, + cid_file: Path, + *, + _token: object, + ) -> None: + if ( + _token is not _NATIVE_RUN_LEASE_TOKEN + or type(owner) is not object + or type(creator_pid) is not int + or creator_pid <= 0 + or not _docker_supported_is_valid_v1(capability) + or type(root) is not _PATH_TYPE + or not root.is_absolute() + or type(cid_file) is not _PATH_TYPE + or not cid_file.is_absolute() + ): + raise TypeError("native run lease is adapter-owned") + self._owner = owner + self._creator_pid = creator_pid + self._capability = capability + self._root = root + self._cid_file = cid_file + self._launched = False + self._released = False + + @property + def owner(self) -> object: + return self._owner + + @property + def creator_pid(self) -> int: + return self._creator_pid + + @property + def capability(self) -> DockerSupportedV1: + return self._capability + + @property + def cid_file(self) -> Path: + return self._cid_file + + @property + def launched(self) -> bool: + return self._launched + + +class _DockerIssuedContainerIdV1(str): + """A full ID read from the adapter-private CID file Docker created.""" + + def __new__( + cls, + value: str, + *, + _token: object, + ) -> _DockerIssuedContainerIdV1: + if ( + _token is not _DOCKER_ISSUED_CONTAINER_ID_TOKEN + or type(value) is not str + or len(value) != 64 + or any(character not in "0123456789abcdef" for character in value) + ): + raise TypeError("invalid Docker-issued container ID") + return str.__new__(cls, value) + + +def _bounded_bytes(value: object, maximum: int, field_name: str) -> bytes: + if type(value) is not bytes or len(value) > maximum: + raise TypeError(f"invalid {field_name}") + return value + + +class BuildInputTransferProgressV1(tuple): + __slots__ = () + + def __new__( + cls, + bundle_identity: bytes, + expected_length: int, + expected_sha256: bytes, + written_length: int, + written_sha256: bytes, + *, + _token: object, + ) -> BuildInputTransferProgressV1: + if _token is not _BUILD_INPUT_PROGRESS_TOKEN: + raise TypeError("build input progress is controller-observed") + if not _valid_digest(bundle_identity) or not _valid_digest(expected_sha256): + raise TypeError("invalid build input progress coordinates") + if ( + type(expected_length) is not int + or expected_length <= 0 + or expected_length >= 1 << 64 + or type(written_length) is not int + or written_length < 0 + or written_length >= 1 << 64 + or written_length > expected_length + or type(written_sha256) is not bytes + or len(written_sha256) != 32 + ): + raise TypeError("invalid build input progress") + return tuple.__new__( + cls, + ( + bundle_identity, + expected_length, + expected_sha256, + written_length, + written_sha256, + ), + ) + + @property + def bundle_identity(self) -> bytes: + return self[0] + + @property + def expected_length(self) -> int: + return self[1] + + @property + def expected_sha256(self) -> bytes: + return self[2] + + @property + def written_length(self) -> int: + return self[3] + + @property + def written_sha256(self) -> bytes: + return self[4] + + +def _build_input_progress_v1( + bundle: _build_input.SealedInputV1, + written_length: int, + written_sha256: bytes, +) -> BuildInputTransferProgressV1: + if not _build_input.sealed_input_is_intact_v1(bundle): + raise TypeError("build input bytes are not intact") + if ( + type(written_length) is not int + or written_length < 0 + or written_length > bundle.length + or type(written_sha256) is not bytes + or written_sha256 + != hashlib.sha256(bundle.contents[:written_length]).digest() + ): + raise TypeError("build input progress does not match the sealed bytes") + return BuildInputTransferProgressV1( + bundle.binding_identity, + bundle.length, + bundle.sha256, + written_length, + written_sha256, + _token=_BUILD_INPUT_PROGRESS_TOKEN, + ) + + +def _input_progress_matches_v1( + value: object, + bundle: _build_input.SealedInputV1, +) -> bool: + if ( + type(value) is not BuildInputTransferProgressV1 + or not _build_input.sealed_input_is_intact_v1(bundle) + ): + return False + try: + canonical = _build_input_progress_v1( + bundle, + value.written_length, + value.written_sha256, + ) + return tuple(canonical) == tuple(value) + except Exception: + return False + + +class BuildInputTransferV1(tuple): + __slots__ = () + + def __new__( + cls, + progress: BuildInputTransferProgressV1, + *, + _token: object, + ) -> BuildInputTransferV1: + if _token is not _BUILD_INPUT_TRANSFER_TOKEN: + raise TypeError("build input transfer is controller-observed") + if ( + type(progress) is not BuildInputTransferProgressV1 + or progress.written_length != progress.expected_length + or progress.written_sha256 != progress.expected_sha256 + ): + raise TypeError("completed build input transfer must be exact") + return tuple.__new__(cls, tuple(progress)) + + @property + def bundle_identity(self) -> bytes: + return self[0] + + @property + def expected_length(self) -> int: + return self[1] + + @property + def expected_sha256(self) -> bytes: + return self[2] + + @property + def written_length(self) -> int: + return self[3] + + @property + def written_sha256(self) -> bytes: + return self[4] + + +def _input_transfer_is_structurally_valid_v1(value: object) -> bool: + if type(value) is not BuildInputTransferV1: + return False + try: + return ( + len(value) == 5 + and _valid_digest(value.bundle_identity) + and type(value.expected_length) is int + and 0 < value.expected_length < 1 << 64 + and _valid_digest(value.expected_sha256) + and value.written_length == value.expected_length + and value.written_sha256 == value.expected_sha256 + ) + except Exception: + return False + + +def _completed_build_input_transfer_v1( + bundle: _build_input.SealedInputV1, + written_length: int, + written_sha256: bytes, +) -> BuildInputTransferV1: + progress = _build_input_progress_v1(bundle, written_length, written_sha256) + return BuildInputTransferV1( + progress, + _token=_BUILD_INPUT_TRANSFER_TOKEN, + ) + + +class _DockerCommandExitedV1(tuple): + __slots__ = () + + def __new__( + cls, + returncode: int, + stdout: bytes, + stderr: bytes, + *, + _token: object, + ) -> _DockerCommandExitedV1: + if _token is not _DOCKER_COMMAND_EXITED_TOKEN: + raise TypeError("Docker command exit is controller-observed") + if type(returncode) is not int or not -(1 << 31) <= returncode < 1 << 31: + raise TypeError("invalid Docker returncode") + _bounded_bytes(stdout, BUILD_STDOUT_LIMIT_V1, "stdout") + _bounded_bytes(stderr, BUILD_STDERR_LIMIT_V1, "stderr") + return tuple.__new__(cls, (returncode, stdout, stderr)) + + @property + def returncode(self) -> int: + return self[0] + + @property + def stdout(self) -> bytes: + return self[1] + + @property + def stderr(self) -> bytes: + return self[2] + + +def _docker_command_exited_v1( + returncode: int, + stdout: bytes, + stderr: bytes, +) -> _DockerCommandExitedV1: + return _DockerCommandExitedV1( + returncode, + stdout, + stderr, + _token=_DOCKER_COMMAND_EXITED_TOKEN, + ) + + +class DockerBuildExitedV1(tuple): + __slots__ = () + + def __new__( + cls, + returncode: int, + stdout: bytes, + stderr: bytes, + input_transfer: BuildInputTransferV1, + *, + _token: object, + ) -> DockerBuildExitedV1: + if _token is not _DOCKER_BUILD_EXITED_TOKEN: + raise TypeError("Docker build exit is controller-observed") + if type(returncode) is not int or not -(1 << 31) <= returncode < 1 << 31: + raise TypeError("invalid Docker returncode") + _bounded_bytes(stdout, BUILD_STDOUT_LIMIT_V1, "stdout") + _bounded_bytes(stderr, BUILD_STDERR_LIMIT_V1, "stderr") + if not _input_transfer_is_structurally_valid_v1(input_transfer): + raise TypeError("invalid Docker build input transfer") + return tuple.__new__( + cls, + (returncode, stdout, stderr, input_transfer), + ) + + @property + def returncode(self) -> int: + return self[0] + + @property + def stdout(self) -> bytes: + return self[1] + + @property + def stderr(self) -> bytes: + return self[2] + + @property + def input_transfer(self) -> BuildInputTransferV1: + return self[3] + + +def _docker_build_exited_v1( + returncode: int, + stdout: bytes, + stderr: bytes, + input_transfer: BuildInputTransferV1, +) -> DockerBuildExitedV1: + return DockerBuildExitedV1( + returncode, + stdout, + stderr, + input_transfer, + _token=_DOCKER_BUILD_EXITED_TOKEN, + ) + + +def docker_build_exited_is_valid_v1( + value: object, + input_value: _build_input.SealedInputV1, + max_output_bytes: int, + max_stderr_bytes: int, +) -> bool: + if ( + type(value) is not DockerBuildExitedV1 + or not _build_input.sealed_input_is_intact_v1(input_value) + or type(max_output_bytes) is not int + or max_output_bytes <= 0 + or type(max_stderr_bytes) is not int + or max_stderr_bytes <= 0 + ): + return False + try: + return ( + len(value) == 4 + and type(value.returncode) is int + and -(1 << 31) <= value.returncode < 1 << 31 + and type(value.stdout) is bytes + and len(value.stdout) <= max_output_bytes + and type(value.stderr) is bytes + and len(value.stderr) <= max_stderr_bytes + and _input_transfer_is_structurally_valid_v1(value.input_transfer) + and value.input_transfer.bundle_identity + == input_value.binding_identity + and value.input_transfer.expected_length == input_value.length + and value.input_transfer.expected_sha256 == input_value.sha256 + and value.input_transfer.written_length == input_value.length + and value.input_transfer.written_sha256 == input_value.sha256 + ) + except Exception: + return False + + +class DockerBuildTimedOutV1(tuple): + __slots__ = () + + def __new__( + cls, + stdout: bytes, + stderr: bytes, + input_progress: BuildInputTransferProgressV1 | None = None, + ) -> DockerBuildTimedOutV1: + _bounded_bytes(stdout, BUILD_STDOUT_LIMIT_V1, "stdout") + _bounded_bytes(stderr, BUILD_STDERR_LIMIT_V1, "stderr") + if input_progress is not None and type( + input_progress + ) is not BuildInputTransferProgressV1: + raise TypeError("invalid timed-out build input progress") + return tuple.__new__(cls, (stdout, stderr, input_progress)) + + @property + def stdout(self) -> bytes: + return self[0] + + @property + def stderr(self) -> bytes: + return self[1] + + @property + def input_progress(self) -> BuildInputTransferProgressV1 | None: + return self[2] + + +class DockerOutputStreamV1(StrEnum): + STDOUT = "stdout" + STDERR = "stderr" + + +class DockerBuildOutputLimitV1(tuple): + __slots__ = () + + def __new__( + cls, + stream: DockerOutputStreamV1, + stdout: bytes, + stderr: bytes, + input_progress: BuildInputTransferProgressV1 | None = None, + ) -> DockerBuildOutputLimitV1: + if type(stream) is not DockerOutputStreamV1: + raise TypeError("invalid Docker output stream") + _bounded_bytes(stdout, BUILD_STDOUT_LIMIT_V1, "stdout") + _bounded_bytes(stderr, BUILD_STDERR_LIMIT_V1, "stderr") + if input_progress is not None and type( + input_progress + ) is not BuildInputTransferProgressV1: + raise TypeError("invalid output-limited build input progress") + return tuple.__new__(cls, (stream, stdout, stderr, input_progress)) + + @property + def stream(self) -> DockerOutputStreamV1: + return self[0] + + @property + def stdout(self) -> bytes: + return self[1] + + @property + def stderr(self) -> bytes: + return self[2] + + @property + def input_progress(self) -> BuildInputTransferProgressV1 | None: + return self[3] + + +class DockerBuildObserverFailureV1(tuple): + __slots__ = () + + def __new__( + cls, + detail: str, + stdout: bytes, + stderr: bytes, + input_progress: BuildInputTransferProgressV1 | None = None, + ) -> DockerBuildObserverFailureV1: + if _canonical_diagnostic_detail_v1(detail) is None: + raise TypeError("invalid Docker observer failure") + _bounded_bytes(stdout, BUILD_STDOUT_LIMIT_V1, "stdout") + _bounded_bytes(stderr, BUILD_STDERR_LIMIT_V1, "stderr") + if input_progress is not None and type( + input_progress + ) is not BuildInputTransferProgressV1: + raise TypeError("invalid observer-failure build input progress") + return tuple.__new__(cls, (detail, stdout, stderr, input_progress)) + + @property + def detail(self) -> str: + return self[0] + + @property + def stdout(self) -> bytes: + return self[1] + + @property + def stderr(self) -> bytes: + return self[2] + + @property + def input_progress(self) -> BuildInputTransferProgressV1 | None: + return self[3] + + +class DockerBuildInputRejectedV1(tuple): + __slots__ = () + + def __new__( + cls, + input_progress: BuildInputTransferProgressV1, + stdout: bytes, + stderr: bytes, + ) -> DockerBuildInputRejectedV1: + if type(input_progress) is not BuildInputTransferProgressV1: + raise TypeError("invalid partial build input progress") + _bounded_bytes(stdout, BUILD_STDOUT_LIMIT_V1, "stdout") + _bounded_bytes(stderr, BUILD_STDERR_LIMIT_V1, "stderr") + return tuple.__new__(cls, (input_progress, stdout, stderr)) + + @property + def input_progress(self) -> BuildInputTransferProgressV1: + return self[0] + + @property + def stdout(self) -> bytes: + return self[1] + + @property + def stderr(self) -> bytes: + return self[2] + + @property + def written_length(self) -> int: + return self.input_progress.written_length + + @property + def written_sha256(self) -> bytes: + return self.input_progress.written_sha256 + + +class DockerCleanupTriggerV1(StrEnum): + PROCESS_EXIT = "process_exit" + INPUT_TRANSFER = "input_transfer" + TIMEOUT = "timeout" + OUTPUT_LIMIT = "output_limit" + OBSERVER_FAILURE = "observer_failure" + + +class CleanupResourceV1(StrEnum): + DOCKER_CLI_PROCESS = "docker_cli_process" + DOCKER_CONTAINER = "docker_container" + DOCKER_CID_ROOT = "docker_cid_root" + + +class CleanupFailureRecordV1(tuple): + __slots__ = () + + def __new__( + cls, + resource: CleanupResourceV1, + detail: str, + ) -> CleanupFailureRecordV1: + if type(resource) is not CleanupResourceV1: + raise TypeError("invalid cleanup resource") + if _canonical_diagnostic_detail_v1(detail) is None: + raise TypeError("invalid cleanup failure detail") + return tuple.__new__(cls, (resource, detail)) + + @property + def resource(self) -> CleanupResourceV1: + return self[0] + + @property + def detail(self) -> str: + return self[1] + + +def _cleanup_failure_records_v1( + value: object, + allowed_order: tuple[CleanupResourceV1, ...], +) -> tuple[CleanupFailureRecordV1, ...]: + if type(value) is not tuple or not value: + raise TypeError("cleanup failures must be one nonempty tuple") + order = {resource: index for index, resource in enumerate(allowed_order)} + owned: list[CleanupFailureRecordV1] = [] + indexes: list[int] = [] + for record in value: + if type(record) is not CleanupFailureRecordV1: + raise TypeError("cleanup failure record is not canonical") + canonical = CleanupFailureRecordV1(*tuple(record)) + if tuple(canonical) != tuple(record) or canonical.resource not in order: + raise TypeError("cleanup failure record is not canonical") + owned.append(canonical) + indexes.append(order[canonical.resource]) + if indexes != sorted(set(indexes)): + raise TypeError("cleanup failure records are not in stable resource order") + return tuple(owned) + + +class DockerBuildCleanupFailureV1(tuple): + __slots__ = () + + def __new__( + cls, + trigger: DockerCleanupTriggerV1, + failures: tuple[CleanupFailureRecordV1, ...], + stdout: bytes, + stderr: bytes, + input_progress: BuildInputTransferProgressV1 | None = None, + ) -> DockerBuildCleanupFailureV1: + if type(trigger) is not DockerCleanupTriggerV1: + raise TypeError("invalid Docker cleanup trigger") + owned_failures = _cleanup_failure_records_v1( + failures, + ( + CleanupResourceV1.DOCKER_CLI_PROCESS, + CleanupResourceV1.DOCKER_CONTAINER, + CleanupResourceV1.DOCKER_CID_ROOT, + ), + ) + _bounded_bytes(stdout, BUILD_STDOUT_LIMIT_V1, "stdout") + _bounded_bytes(stderr, BUILD_STDERR_LIMIT_V1, "stderr") + if input_progress is not None and type( + input_progress + ) is not BuildInputTransferProgressV1: + raise TypeError("invalid cleanup build input progress") + return tuple.__new__( + cls, + (trigger, owned_failures, stdout, stderr, input_progress), + ) + + @property + def trigger(self) -> DockerCleanupTriggerV1: + return self[0] + + @property + def failures(self) -> tuple[CleanupFailureRecordV1, ...]: + return self[1] + + @property + def detail(self) -> str: + """Render all typed records for diagnostic-only consumers.""" + + return "; ".join(record.detail for record in self.failures) + + @property + def stdout(self) -> bytes: + return self[2] + + @property + def stderr(self) -> bytes: + return self[3] + + @property + def input_progress(self) -> BuildInputTransferProgressV1 | None: + return self[4] + + +DockerBuildProcessObservationV1: TypeAlias = ( + DockerBuildExitedV1 + | DockerBuildTimedOutV1 + | DockerBuildOutputLimitV1 + | DockerBuildObserverFailureV1 + | DockerBuildInputRejectedV1 + | DockerBuildCleanupFailureV1 +) + + +_DockerCommandObservationV1: TypeAlias = ( + _DockerCommandExitedV1 | DockerBuildProcessObservationV1 +) + + +def _canonical_progress_v1( + value: object, + input_value: _build_input.SealedInputV1, +) -> BuildInputTransferProgressV1 | None: + if value is None: + return None + if not _input_progress_matches_v1(value, input_value): + raise TypeError("build input progress is not canonical") + return _build_input_progress_v1( + input_value, + value.written_length, + value.written_sha256, + ) + + +def _canonical_process_observation_v1( + value: object, + input_value: _build_input.SealedInputV1, + max_output_bytes: int, + max_stderr_bytes: int, +) -> DockerBuildProcessObservationV1: + """Own a backend observation before classification or retention.""" + + if ( + not _build_input.sealed_input_is_intact_v1(input_value) + or type(max_output_bytes) is not int + or max_output_bytes <= 0 + or type(max_stderr_bytes) is not int + or max_stderr_bytes <= 0 + ): + raise TypeError("invalid process-observation boundary") + try: + if type(value) is DockerBuildExitedV1: + if not docker_build_exited_is_valid_v1( + value, + input_value, + max_output_bytes, + max_stderr_bytes, + ): + raise TypeError("invalid exited build observation") + transfer = _completed_build_input_transfer_v1( + input_value, + value.input_transfer.written_length, + value.input_transfer.written_sha256, + ) + canonical = _docker_build_exited_v1( + value.returncode, + bytes(value.stdout), + bytes(value.stderr), + transfer, + ) + if tuple(canonical) != tuple(value): + raise TypeError("exited build observation is not canonical") + return value + if type(value) is DockerBuildTimedOutV1: + progress = _canonical_progress_v1(value.input_progress, input_value) + if progress is None: + raise TypeError("build timeout did not retain input progress") + canonical = DockerBuildTimedOutV1( + _bounded_bytes(value.stdout, max_output_bytes, "stdout"), + _bounded_bytes(value.stderr, max_stderr_bytes, "stderr"), + progress, + ) + if tuple(canonical) != tuple(value): + raise TypeError("timeout observation is not canonical") + return value + if type(value) is DockerBuildOutputLimitV1: + progress = _canonical_progress_v1(value.input_progress, input_value) + if progress is None: + raise TypeError("build output limit did not retain input progress") + stdout = _bounded_bytes(value.stdout, max_output_bytes, "stdout") + stderr = _bounded_bytes(value.stderr, max_stderr_bytes, "stderr") + if ( + ( + value.stream is DockerOutputStreamV1.STDOUT + and len(stdout) != max_output_bytes + ) + or ( + value.stream is DockerOutputStreamV1.STDERR + and len(stderr) != max_stderr_bytes + ) + ): + raise TypeError("output-limit observation did not reach its cap") + canonical = DockerBuildOutputLimitV1( + value.stream, + stdout, + stderr, + progress, + ) + if tuple(canonical) != tuple(value): + raise TypeError("output-limit observation is not canonical") + return value + if type(value) is DockerBuildObserverFailureV1: + canonical = DockerBuildObserverFailureV1( + value.detail, + _bounded_bytes(value.stdout, max_output_bytes, "stdout"), + _bounded_bytes(value.stderr, max_stderr_bytes, "stderr"), + _canonical_progress_v1(value.input_progress, input_value), + ) + if tuple(canonical) != tuple(value): + raise TypeError("observer failure is not canonical") + return value + if type(value) is DockerBuildInputRejectedV1: + progress = _canonical_progress_v1(value.input_progress, input_value) + if progress is None or progress.written_length >= progress.expected_length: + raise TypeError("input rejection did not retain partial progress") + canonical = DockerBuildInputRejectedV1( + progress, + _bounded_bytes(value.stdout, max_output_bytes, "stdout"), + _bounded_bytes(value.stderr, max_stderr_bytes, "stderr"), + ) + if tuple(canonical) != tuple(value): + raise TypeError("input rejection is not canonical") + return value + if type(value) is DockerBuildCleanupFailureV1: + progress = _canonical_progress_v1(value.input_progress, input_value) + if progress is None: + raise TypeError("build cleanup failure did not retain input progress") + canonical = DockerBuildCleanupFailureV1( + value.trigger, + value.failures, + _bounded_bytes(value.stdout, max_output_bytes, "stdout"), + _bounded_bytes(value.stderr, max_stderr_bytes, "stderr"), + progress, + ) + if tuple(canonical) != tuple(value): + raise TypeError("cleanup failure is not canonical") + return value + except (AttributeError, IndexError, TypeError, ValueError) as error: + raise TypeError("backend process observation is not canonical") from error + raise TypeError("backend returned an unknown process observation") + + +class DockerBuildBackendV1(Protocol): + def probe(self) -> DockerCapabilityReportV1: ... + + def run_build( + self, + request: DockerBuildRequestV1, + ) -> DockerBuildProcessObservationV1: ... + + +def build_process_bytes_v1(process: DockerBuildExitedV1) -> bytes: + if type(process) is not DockerBuildExitedV1: + raise TypeError("only successful typed build observations are encodable") + try: + transfer = process.input_transfer + if ( + process.returncode != 0 + or not -(1 << 31) <= process.returncode < 1 << 31 + or type(process.stdout) is not bytes + or len(process.stdout) > BUILD_STDOUT_LIMIT_V1 + or type(process.stderr) is not bytes + or len(process.stderr) > BUILD_STDERR_LIMIT_V1 + or not _input_transfer_is_structurally_valid_v1(transfer) + ): + raise TypeError("successful build observation is not canonical") + except (AttributeError, IndexError, OverflowError, TypeError) as error: + raise TypeError( + "only successful canonical build observations are encodable" + ) from error + return b"".join( + ( + process.returncode.to_bytes(4, "big", signed=True), + len(process.stdout).to_bytes(8, "big"), + hashlib.sha256(process.stdout).digest(), + len(process.stderr).to_bytes(8, "big"), + hashlib.sha256(process.stderr).digest(), + transfer.bundle_identity, + transfer.expected_length.to_bytes(8, "big"), + transfer.expected_sha256, + transfer.written_length.to_bytes(8, "big"), + transfer.written_sha256, + ) + ) + + +class NativeDockerBuildBackendV1: + """Docker adapter whose probe observes only Linux x64 and its daemon.""" + + def __init__( + self, + docker_path: Path, + policy: DockerBuildPolicyV1, + *, + platform_name: str | None = None, + machine_name: str | None = None, + monotonic_ns: object = time.monotonic_ns, + host_user: tuple[int, int] | None = None, + ) -> None: + _absolute_path(docker_path, "docker_path") + if not docker_policy_is_valid_v1(policy): + raise TypeError("policy must be DockerBuildPolicyV1") + if policy.user_mode is not DockerUserModeV1.HOST_EFFECTIVE_IDS: + raise TypeError("unsupported Docker user policy") + observed_user = ( + None if host_user is None else _host_user_coordinates(host_user) + ) + observed_platform = ( + platform.system().lower() if platform_name is None else platform_name + ) + observed_machine = ( + platform.machine() if machine_name is None else machine_name + ) + self._command_coordinate = native_command_coordinate_v1(docker_path) + self._policy = DockerBuildPolicyV1(*tuple(policy)) + self._platform_name = _encoded_policy_text( + observed_platform, + 64, + "platform_name", + ) + self._machine_name = _encoded_policy_text( + observed_machine, + 64, + "machine_name", + ) + self._monotonic_ns = monotonic_ns + self._configured_host_user = observed_user + self._run_lease_owner = object() + self._owner_pid = os.getpid() + self._probed_capability: DockerSupportedV1 | None = None + + def _in_owner_process_v1(self) -> bool: + return os.getpid() == self._owner_pid + + def probe(self) -> DockerCapabilityReportV1: + self._probed_capability = None + if not self._in_owner_process_v1(): + return DockerUnsupportedV1( + DockerBlockerReasonV1.BACKEND_CONTRACT, + "native Docker capability belongs to its creator process", + ) + if self._platform_name != "linux" or self._machine_name.lower() not in ( + "x86_64", + "amd64", + ): + return DockerUnsupportedV1( + DockerBlockerReasonV1.HOST_NOT_LINUX_AMD64, + "controlled build requires a Linux amd64 Docker host", + ) + host_user = self._configured_host_user + if host_user is None: + try: + host_user = _host_user_coordinates((os.geteuid(), os.getegid())) + except (AttributeError, OSError, TypeError): + return DockerUnsupportedV1( + DockerBlockerReasonV1.HOST_USER_UNAVAILABLE, + "host effective uid/gid are unavailable", + ) + try: + metadata = self._command_coordinate.path.lstat() + except OSError: + return DockerUnsupportedV1( + DockerBlockerReasonV1.DOCKER_UNAVAILABLE, + "exact Docker CLI path is unavailable", + ) + if not stat.S_ISREG(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode): + return DockerUnsupportedV1( + DockerBlockerReasonV1.DOCKER_UNAVAILABLE, + "Docker CLI must be one regular non-symlink path", + ) + commands = ( + _render_native_command_v1( + "version_probe", + self._command_coordinate, + {}, + ), + _render_native_command_v1( + "image_inspect", + self._command_coordinate, + { + _NativeCommandSlotV1.IMAGE_REFERENCE: ( + self._policy.image_reference, + ), + }, + ), + ) + outputs: list[bytes] = [] + for index, command in enumerate(commands): + result = self._observe_command( + command, + stdout_limit=self._policy.probe_output_limit, + stderr_limit=self._policy.probe_output_limit, + timeout_ns=self._policy.probe_timeout_ns, + ) + # The versioned capability observes machine-readable stdout; + # successful Docker CLI warnings are diagnostic, not absence proof. + if ( + type(result) is not _DockerCommandExitedV1 + or result.returncode != 0 + or not result.stdout + ): + return DockerUnsupportedV1( + DockerBlockerReasonV1.DOCKER_UNAVAILABLE + if index == 0 + else DockerBlockerReasonV1.IMAGE_UNAVAILABLE, + "Docker daemon probe failed" + if index == 0 + else "pinned image is not locally inspectable", + ) + outputs.append(result.stdout) + try: + inspected = json.loads(outputs[1]) + if type(inspected) is not list or len(inspected) != 1: + raise ValueError("wrong image inspection cardinality") + image = inspected[0] + if type(image) is not dict: + raise ValueError("wrong image inspection shape") + repo_digests = image.get("RepoDigests") + if ( + image.get("Os") != "linux" + or image.get("Architecture") not in ("amd64", "x86_64") + or type(repo_digests) is not list + or self._policy.image_reference not in repo_digests + ): + raise ValueError("foreign image coordinate") + except (ValueError, TypeError, json.JSONDecodeError): + return DockerUnsupportedV1( + DockerBlockerReasonV1.IMAGE_IDENTITY_MISMATCH, + "local image does not match pinned linux/amd64 manifest", + ) + daemon_observation = DockerDaemonObservationV1( + outputs[0], + outputs[1], + ) + capability = DockerSupportedV1( + self._policy, + daemon_observation, + self._command_coordinate, + host_user, + ) + self._probed_capability = capability + return capability + + def _bound_request_capability_v1( + self, + request: DockerBuildRequestV1, + ) -> DockerSupportedV1: + if type(request) is not DockerBuildRequestV1: + raise TypeError("request must be DockerBuildRequestV1") + try: + capability = request.capability + except (AttributeError, IndexError) as error: + raise TypeError("request lost its Docker capability") from error + if ( + self._probed_capability is None + or not _docker_build_request_is_valid_v1(request, capability) + or capability is not self._probed_capability + ): + raise TypeError("request capability does not match this backend probe") + return capability + + def _next_run_lease_v1( + self, + capability: DockerSupportedV1, + ) -> _NativeRunLeaseV1: + if not self._in_owner_process_v1(): + raise RuntimeError("native Docker build belongs to its creator process") + if not _docker_supported_is_valid_v1(capability): + raise TypeError("run lease requires one canonical Docker capability") + root = Path( + tempfile.mkdtemp( + prefix=_NATIVE_CID_ROOT_PREFIX_V1, + dir="/tmp", + ) + ) + try: + metadata = root.lstat() + if ( + not root.is_absolute() + or not stat.S_ISDIR(metadata.st_mode) + or stat.S_ISLNK(metadata.st_mode) + or metadata.st_mode & 0o077 + ): + raise RuntimeError("native Docker CID root is not private") + cid_file = root / "cid" + if cid_file.exists() or cid_file.is_symlink(): + raise RuntimeError("fresh native Docker CID path already exists") + return _NativeRunLeaseV1( + self._run_lease_owner, + self._owner_pid, + capability, + root, + cid_file, + _token=_NATIVE_RUN_LEASE_TOKEN, + ) + except BaseException: + try: + shutil.rmtree(root) + except BaseException: + pass + raise + + def _owns_run_lease_v1(self, lease: object) -> bool: + if type(lease) is not _NativeRunLeaseV1: + return False + try: + return ( + lease.owner is self._run_lease_owner + and lease.creator_pid == self._owner_pid + and _docker_supported_is_valid_v1(lease.capability) + and lease.cid_file.is_absolute() + ) + except (AttributeError, TypeError): + return False + + def _lease_belongs_to_current_process_v1(self, lease: _NativeRunLeaseV1) -> bool: + return self._owns_run_lease_v1(lease) and os.getpid() == lease.creator_pid + + def _mark_run_lease_launched_v1(self, lease: _NativeRunLeaseV1) -> None: + if not self._lease_belongs_to_current_process_v1(lease): + raise RuntimeError("native Docker run lease belongs to another process") + if lease._released: + raise RuntimeError("native Docker run lease was already released") + lease._launched = True + + def _release_run_lease_v1(self, lease: _NativeRunLeaseV1) -> str | None: + if not self._owns_run_lease_v1(lease): + raise TypeError("native run lease does not belong to this adapter") + if os.getpid() != lease.creator_pid: + return None + if lease._released: + return None + try: + shutil.rmtree(lease._root) + except Exception: + return "native Docker CID root cleanup failed" + # This flag certifies completed removal, not merely an attempted one: + # an interrupted caller may safely retry with the same private lease. + lease._released = True + return None + + def _command_for_v1( + self, + request: DockerBuildRequestV1, + lease: _NativeRunLeaseV1, + ) -> tuple[str, ...]: + if not self._owns_run_lease_v1(lease): + raise TypeError("native run lease does not belong to this adapter") + capability = lease.capability + if ( + not _docker_build_request_is_valid_v1(request, capability) + or request.capability is not capability + ): + raise TypeError("request capability does not match this native run lease") + policy = capability.policy + return _render_native_command_v1( + "build", + capability.command_coordinate, + { + _NativeCommandSlotV1.PLATFORM: (policy.platform,), + _NativeCommandSlotV1.ORDERED_TMPFS_SPECS: policy.tmpfs_specs, + _NativeCommandSlotV1.HOSTNAME: (policy.hostname,), + _NativeCommandSlotV1.HOST_USER: ( + f"{capability.host_user[0]}:{capability.host_user[1]}", + ), + _NativeCommandSlotV1.CID_FILE: (str(lease.cid_file),), + _NativeCommandSlotV1.IMAGE_REFERENCE: (policy.image_reference,), + _NativeCommandSlotV1.BOOTSTRAP: (policy.bootstrap,), + _NativeCommandSlotV1.BOOTSTRAP_ARGV0: (policy.bootstrap_argv0,), + _NativeCommandSlotV1.INPUT_LENGTH: ( + str(request.input_bundle.length), + ), + _NativeCommandSlotV1.INPUT_SHA256: ( + request.input_bundle.sha256.hex(), + ), + }, + ) + + def run_build( + self, + request: DockerBuildRequestV1, + ) -> DockerBuildProcessObservationV1: + if not self._in_owner_process_v1(): + return DockerBuildObserverFailureV1( + "native Docker build belongs to its creator process", + b"", + b"", + ) + lease: _NativeRunLeaseV1 | None = None + observation: DockerBuildProcessObservationV1 | None = None + retained_base_exception: BaseException | None = None + try: + capability = self._bound_request_capability_v1(request) + lease = self._next_run_lease_v1(capability) + command = self._command_for_v1(request, lease) + if not self._lease_belongs_to_current_process_v1(lease): + return DockerBuildObserverFailureV1( + "native Docker build belongs to its creator process", + b"", + b"", + ) + policy = capability.policy + raw_observation = self._observe_command( + command, + stdout_limit=request.max_output_bytes, + stderr_limit=policy.stderr_limit, + timeout_ns=policy.build_timeout_ns, + lease=lease, + input_bundle=request.input_bundle, + ) + try: + canonical_observation = _canonical_process_observation_v1( + raw_observation, + request.input_bundle, + request.max_output_bytes, + policy.stderr_limit, + ) + except Exception: + observation = DockerBuildObserverFailureV1( + "native Docker build observation is not canonical", + b"", + b"", + ) + else: + if ( + type(canonical_observation) is DockerBuildCleanupFailureV1 + and any( + record.resource is CleanupResourceV1.DOCKER_CID_ROOT + for record in canonical_observation.failures + ) + ): + observation = DockerBuildObserverFailureV1( + "native Docker build observation already contains a " + "CID-root cleanup failure", + canonical_observation.stdout, + canonical_observation.stderr, + canonical_observation.input_progress, + ) + else: + observation = canonical_observation + except Exception: + observation = DockerBuildObserverFailureV1( + "native Docker build request could not be materialized", + b"", + b"", + ) + except BaseException as error: + retained_base_exception = error + finally: + release_detail: str | None = None + if lease is not None: + try: + release_detail = _canonical_cid_root_cleanup_detail_v1( + self._release_run_lease_v1(lease) + ) + except Exception: + release_detail = "native Docker CID root cleanup observer raised" + except BaseException as error: + retained_base_exception = _retain_first_base_exception_v1( + retained_base_exception, + error, + ) + release_detail = "native Docker CID root cleanup was interrupted" + if retained_base_exception is None and release_detail is not None: + observation = self._with_cid_root_cleanup_failure_v1( + observation, + release_detail, + ) + if retained_base_exception is not None: + raise retained_base_exception.with_traceback( + retained_base_exception.__traceback__ + ) + if observation is None: + return DockerBuildObserverFailureV1( + "native Docker build observation was unavailable", + b"", + b"", + ) + return observation + + @staticmethod + def _with_cid_root_cleanup_failure_v1( + observation: object, + detail: str, + ) -> DockerBuildProcessObservationV1: + """Retain a canonical prefix, otherwise report a typed observer failure.""" + + if observation is None: + return DockerBuildObserverFailureV1(detail, b"", b"") + if type(observation) is DockerBuildCleanupFailureV1: + return DockerBuildCleanupFailureV1( + observation.trigger, + observation.failures + + ( + CleanupFailureRecordV1( + CleanupResourceV1.DOCKER_CID_ROOT, + detail, + ), + ), + observation.stdout, + observation.stderr, + observation.input_progress, + ) + if type(observation) is DockerBuildExitedV1: + transfer = observation.input_transfer + progress = BuildInputTransferProgressV1( + transfer.bundle_identity, + transfer.expected_length, + transfer.expected_sha256, + transfer.written_length, + transfer.written_sha256, + _token=_BUILD_INPUT_PROGRESS_TOKEN, + ) + trigger = DockerCleanupTriggerV1.PROCESS_EXIT + elif type(observation) is DockerBuildTimedOutV1: + progress = observation.input_progress + trigger = DockerCleanupTriggerV1.TIMEOUT + elif type(observation) is DockerBuildOutputLimitV1: + progress = observation.input_progress + trigger = DockerCleanupTriggerV1.OUTPUT_LIMIT + elif type(observation) is DockerBuildInputRejectedV1: + progress = observation.input_progress + trigger = DockerCleanupTriggerV1.INPUT_TRANSFER + elif type(observation) is DockerBuildObserverFailureV1: + progress = observation.input_progress + trigger = DockerCleanupTriggerV1.OBSERVER_FAILURE + else: + fallback_detail = ( + "native Docker build observation is not canonical; " + detail + ) + # Keep the joined diagnostic typed and bounded even when cleanup + # supplies the full diagnostic budget. + fallback_detail = fallback_detail[:_DIAGNOSTIC_DETAIL_TEXT_LIMIT_V1] + return DockerBuildObserverFailureV1( + fallback_detail, + b"", + b"", + ) + if progress is None: + if type(observation) is DockerBuildObserverFailureV1: + try: + return DockerBuildObserverFailureV1( + observation.detail + "; " + detail, + observation.stdout, + observation.stderr, + ) + except Exception: + return DockerBuildObserverFailureV1( + _OBSERVER_AND_CID_ROOT_CLEANUP_FAILURE_V1, + observation.stdout, + observation.stderr, + ) + return DockerBuildObserverFailureV1( + detail, + observation.stdout, + observation.stderr, + ) + return DockerBuildCleanupFailureV1( + trigger, + ( + CleanupFailureRecordV1( + CleanupResourceV1.DOCKER_CID_ROOT, + detail, + ), + ), + observation.stdout, + observation.stderr, + progress, + ) + + def _observe_command( + self, + command: tuple[str, ...], + *, + stdout_limit: int, + stderr_limit: int, + timeout_ns: int, + lease: _NativeRunLeaseV1 | None = None, + input_bundle: _build_input.SealedInputV1 | None = None, + ) -> _DockerCommandObservationV1: + if ( + type(command) is not tuple + or not command + or any(type(item) is not str or not item or "\0" in item for item in command) + ): + raise TypeError("command must be a nonempty string tuple") + try: + tuple(os.fsencode(item) for item in command) + except (TypeError, UnicodeEncodeError) as error: + raise TypeError("command contains an unencodable coordinate") from error + if ( + type(stdout_limit) is not int + or stdout_limit <= 0 + or stdout_limit > BUILD_STDOUT_LIMIT_V1 + or type(stderr_limit) is not int + or stderr_limit <= 0 + or stderr_limit > BUILD_STDERR_LIMIT_V1 + or type(timeout_ns) is not int + or timeout_ns <= 0 + or timeout_ns > BUILD_TIMEOUT_NS_V1 + ): + raise TypeError("invalid Docker observation limits") + if lease is not None and not self._owns_run_lease_v1(lease): + raise TypeError("native run lease does not belong to this adapter") + if not self._in_owner_process_v1() or ( + lease is not None and not self._lease_belongs_to_current_process_v1(lease) + ): + return DockerBuildObserverFailureV1( + "native Docker observation belongs to its creator process", + b"", + b"", + ) + if input_bundle is not None and type(input_bundle) is not _build_input.SealedInputV1: + raise TypeError("input_bundle must be controller sealed") + if input_bundle is not None and not _build_input.sealed_input_is_intact_v1( + input_bundle + ): + return DockerBuildObserverFailureV1( + "build input bytes are not intact", + b"", + b"", + ) + # The one protected region starts before Popen. Once Popen returns + # its handle, every following Python bytecode has that handle under + # the finalizer; state allocation cannot create a post-spawn gap. + stdout: bytearray | bytes = b"" + stderr: bytearray | bytes = b"" + selector: selectors.BaseSelector | None = None + terminal: DockerOutputStreamV1 | None = None + timed_out = False + observer_failed = False + input_failed = False + written = 0 + input_hasher = hashlib.sha256() + bundle_view: memoryview | None = None + input_progress: BuildInputTransferProgressV1 | None = None + stop_detail: str | None = None + cleanup_detail: str | None = None + input_descriptor: int | None = None + stdout_descriptor: int | None = None + stderr_descriptor: int | None = None + retained_base_exception: BaseException | None = None + ownership_lost = False + process: subprocess.Popen[bytes] | None = None + try: + try: + process = subprocess.Popen( + command, + **_NATIVE_PROCESS_CONTEXT_V1.popen_kwargs_v1( + input_bundle is not None + ), + ) + except (OSError, UnicodeEncodeError): + return DockerBuildObserverFailureV1( + "cannot start Docker CLI", + b"", + b"", + ) + stdout = bytearray() + stderr = bytearray() + if lease is not None: + self._mark_run_lease_launched_v1(lease) + if ( + process.stdout is None + or process.stderr is None + or (input_bundle is not None and process.stdin is None) + ): + observer_failed = True + raise RuntimeError("Docker pipes unavailable") + stdout_descriptor = process.stdout.fileno() + stderr_descriptor = process.stderr.fileno() + if process.stdin is not None: + input_descriptor = process.stdin.fileno() + selector = selectors.DefaultSelector() + bundle_view = ( + memoryview(input_bundle.contents) + if input_bundle is not None + else None + ) + streams = ( + ( + stdout_descriptor, + DockerOutputStreamV1.STDOUT, + stdout, + stdout_limit, + ), + ( + stderr_descriptor, + DockerOutputStreamV1.STDERR, + stderr, + stderr_limit, + ), + ) + for descriptor, stream, target, maximum in streams: + os.set_blocking(descriptor, False) + selector.register( + descriptor, + selectors.EVENT_READ, + ("read", stream, target, maximum), + ) + if process.stdin is not None: + os.set_blocking(input_descriptor, False) + selector.register( + input_descriptor, + selectors.EVENT_WRITE, + ("write",), + ) + start = self._clock() + deadline = start + timeout_ns + while selector.get_map() or process.poll() is None: + now = self._clock() + if now >= deadline: + timed_out = True + break + timeout = min( + (deadline - now) / 1_000_000_000, + _POLL_SLICE_SECONDS_V1, + ) + for key, _events in selector.select(timeout): + if key.data[0] == "read": + _kind, stream, target, maximum = key.data + try: + chunk = os.read( + key.fd, + min( + _IO_CHUNK_BYTES_V1, + maximum + 1 - len(target), + ), + ) + except BlockingIOError: + continue + if not chunk: + selector.unregister(key.fd) + continue + target.extend(chunk) + if len(target) > maximum: + del target[maximum:] + terminal = stream + break + continue + if input_bundle is None or bundle_view is None: + observer_failed = True + break + try: + count = os.write( + key.fd, + bundle_view[written : written + _IO_CHUNK_BYTES_V1], + ) + except BlockingIOError: + continue + except BrokenPipeError: + input_failed = True + break + if count <= 0: + input_failed = True + break + input_hasher.update(bundle_view[written : written + count]) + written += count + if written == input_bundle.length: + selector.unregister(key.fd) + if process.stdin is not None: + process.stdin.close() + if terminal is not None or input_failed or observer_failed: + break + except _NativeOwnershipLostV1: + ownership_lost = True + observer_failed = True + except Exception: + observer_failed = True + except BaseException as error: + # Cancellation is not an excuse to leak a child or a container. It + # is re-raised only after every independently-owned resource got a + # best-effort deterministic release attempt. + retained_base_exception = error + finally: + if process is None: + # Popen can itself be interrupted after the daemon received a + # launch request but before Python returned a handle. There + # is no safe CLI PID to reap then, yet a CID cleanup attempt + # can still release a Docker container without replacing the + # caller's original interruption. + if retained_base_exception is not None and lease is not None: + try: + self._cleanup_container( + lease, + spawn_may_have_started=True, + ) + except BaseException: + pass + else: + if selector is not None: + try: + selector.close() + except Exception: + observer_failed = True + except BaseException as error: + retained_base_exception = _retain_first_base_exception_v1( + retained_base_exception, + error, + ) + observer_failed = True + if process.stdin is not None: + close_failed, close_interrupt = self._close_owned_stream( + process.stdin, + ) + if close_failed: + observer_failed = True + if close_interrupt is not None: + retained_base_exception = _retain_first_base_exception_v1( + retained_base_exception, + close_interrupt, + ) + if bundle_view is not None: + try: + bundle_view.release() + except Exception: + observer_failed = True + except BaseException as error: + retained_base_exception = _retain_first_base_exception_v1( + retained_base_exception, + error, + ) + observer_failed = True + if input_bundle is not None: + try: + input_progress = _build_input_progress_v1( + input_bundle, + written, + input_hasher.digest(), + ) + except Exception: + observer_failed = True + except BaseException as error: + retained_base_exception = _retain_first_base_exception_v1( + retained_base_exception, + error, + ) + observer_failed = True + ownership_lost = ownership_lost or not self._in_owner_process_v1() + if not ownership_lost: + try: + process_running = process.poll() is None + except Exception: + process_running = True + stop_detail = "Docker CLI process state could not be observed" + except BaseException as error: + retained_base_exception = _retain_first_base_exception_v1( + retained_base_exception, + error, + ) + process_running = True + stop_detail = "Docker CLI process state observation was interrupted" + if process_running: + if not ( + timed_out + or terminal is not None + or observer_failed + or input_failed + ): + timed_out = True + try: + observed_stop = self._stop_process(process) + except Exception: + observed_stop = "Docker CLI process termination raised" + except BaseException as error: + retained_base_exception = _retain_first_base_exception_v1( + retained_base_exception, + error, + ) + observed_stop = "Docker CLI process termination was interrupted" + fallback_stop = self._force_reap_after_interruption_v1(process) + stop_detail = stop_detail or observed_stop or fallback_stop + for stream in (process.stdout, process.stderr): + if stream is None: + continue + close_failed, close_interrupt = self._close_owned_stream( + stream, + ) + if close_failed: + observer_failed = True + if close_interrupt is not None: + retained_base_exception = _retain_first_base_exception_v1( + retained_base_exception, + close_interrupt, + ) + if lease is not None and not ownership_lost: + try: + cleanup_detail = self._cleanup_container( + lease, + spawn_may_have_started=True, + ) + except Exception: + cleanup_detail = "Docker container cleanup observer raised" + except BaseException as error: + retained_base_exception = _retain_first_base_exception_v1( + retained_base_exception, + error, + ) + cleanup_detail = "Docker container cleanup was interrupted" + if retained_base_exception is not None: + raise retained_base_exception.with_traceback( + retained_base_exception.__traceback__ + ) + if ownership_lost: + return DockerBuildObserverFailureV1( + "native Docker observation left its creator process", + bytes(stdout), + bytes(stderr), + input_progress, + ) + if stop_detail is not None or cleanup_detail is not None: + trigger = DockerCleanupTriggerV1.PROCESS_EXIT + if observer_failed: + trigger = DockerCleanupTriggerV1.OBSERVER_FAILURE + elif terminal is not None: + trigger = DockerCleanupTriggerV1.OUTPUT_LIMIT + elif timed_out: + trigger = DockerCleanupTriggerV1.TIMEOUT + elif input_failed: + trigger = DockerCleanupTriggerV1.INPUT_TRANSFER + failures: list[CleanupFailureRecordV1] = [] + if stop_detail is not None: + failures.append( + CleanupFailureRecordV1( + CleanupResourceV1.DOCKER_CLI_PROCESS, + stop_detail, + ) + ) + if cleanup_detail is not None: + failures.append( + CleanupFailureRecordV1( + CleanupResourceV1.DOCKER_CONTAINER, + cleanup_detail, + ) + ) + return DockerBuildCleanupFailureV1( + trigger, + tuple(failures), + bytes(stdout), + bytes(stderr), + input_progress, + ) + if input_failed: + if input_progress is None: + return DockerBuildObserverFailureV1( + "build input progress could not be retained", + bytes(stdout), + bytes(stderr), + input_progress, + ) + return DockerBuildInputRejectedV1( + input_progress, + bytes(stdout), + bytes(stderr), + ) + if observer_failed: + return DockerBuildObserverFailureV1( + "Docker output observation failed", + bytes(stdout), + bytes(stderr), + input_progress, + ) + if terminal is not None: + return DockerBuildOutputLimitV1( + terminal, + bytes(stdout), + bytes(stderr), + input_progress, + ) + if timed_out: + return DockerBuildTimedOutV1( + bytes(stdout), + bytes(stderr), + input_progress, + ) + if type(process.returncode) is not int: + return DockerBuildObserverFailureV1( + "Docker returncode unavailable", + bytes(stdout), + bytes(stderr), + input_progress, + ) + if input_bundle is not None: + if ( + input_progress is None + or written != input_bundle.length + or input_hasher.digest() != input_bundle.sha256 + ): + return DockerBuildObserverFailureV1( + "completed build input transfer invariant failed", + bytes(stdout), + bytes(stderr), + input_progress, + ) + input_transfer = _completed_build_input_transfer_v1( + input_bundle, + written, + input_hasher.digest(), + ) + return _docker_build_exited_v1( + process.returncode, + bytes(stdout), + bytes(stderr), + input_transfer, + ) + return _docker_command_exited_v1( + process.returncode, + bytes(stdout), + bytes(stderr), + ) + + @staticmethod + def _close_owned_stream( + stream: object, + ) -> tuple[bool, BaseException | None]: + """Release through the stream owner without aliasing its descriptor.""" + + close_failed = False + close_raised = False + retained_base_exception: BaseException | None = None + try: + closed = stream.closed is True + except Exception: + closed = False + close_failed = True + except BaseException as error: + closed = False + close_failed = True + retained_base_exception = error + if not closed: + try: + stream.close() + except Exception: + close_failed = True + close_raised = True + except BaseException as error: + close_failed = True + close_raised = True + retained_base_exception = _retain_first_base_exception_v1( + retained_base_exception, + error, + ) + if close_raised: + try: + still_open = stream.closed is False + except Exception: + still_open = False + except BaseException as error: + retained_base_exception = _retain_first_base_exception_v1( + retained_base_exception, + error, + ) + still_open = False + if still_open: + # close() may fail before it releases the resource, but a + # saved FD can already name another resource. Retrying the + # same owner is the only bounded release attempt that keeps + # ownership unambiguous. + try: + stream.close() + except Exception: + close_failed = True + except BaseException as error: + close_failed = True + retained_base_exception = _retain_first_base_exception_v1( + retained_base_exception, + error, + ) + return close_failed, retained_base_exception + + def _clock(self) -> int: + if not self._in_owner_process_v1(): + raise _NativeOwnershipLostV1( + "native Docker observation belongs to its creator process" + ) + value = self._monotonic_ns() + if not self._in_owner_process_v1(): + raise _NativeOwnershipLostV1( + "native Docker observation left its creator process" + ) + if type(value) is not int or value < 0: + raise RuntimeError("invalid monotonic clock") + return value + + def _stop_process( + self, + process: subprocess.Popen[bytes], + ) -> str | None: + failed = False + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + except OSError: + try: + process.kill() + except ProcessLookupError: + pass + except OSError: + failed = True + try: + process.wait(timeout=_PROCESS_STOP_TIMEOUT_SECONDS_V1) + except subprocess.TimeoutExpired: + failed = True + if process.poll() is None: + failed = True + return "Docker CLI process could not be terminated" if failed else None + + @staticmethod + def _force_reap_after_interruption_v1( + process: subprocess.Popen[bytes], + ) -> str | None: + """Use an independent, interruption-safe kill/reap path after stop fails.""" + + failed = False + try: + running = process.poll() is None + except BaseException: + running = True + failed = True + if running: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + except BaseException: + try: + process.kill() + except ProcessLookupError: + pass + except BaseException: + failed = True + try: + process.wait(timeout=_PROCESS_STOP_TIMEOUT_SECONDS_V1) + except BaseException: + failed = True + try: + if process.poll() is None: + failed = True + except BaseException: + failed = True + return "Docker CLI process could not be force-reaped" if failed else None + + @staticmethod + def _docker_issued_container_id_v1( + lease: _NativeRunLeaseV1, + ) -> _DockerIssuedContainerIdV1 | None: + """Admit only the exact ID written into this fresh private CID path.""" + + try: + descriptor = os.open( + lease.cid_file, + os.O_RDONLY + | getattr(os, "O_CLOEXEC", 0) + | getattr(os, "O_NOFOLLOW", 0), + ) + except OSError: + return None + try: + metadata = os.fstat(descriptor) + if ( + not stat.S_ISREG(metadata.st_mode) + or metadata.st_nlink != 1 + or metadata.st_size not in (64, 65) + ): + return None + raw = os.read(descriptor, 66) + except OSError: + return None + finally: + try: + os.close(descriptor) + except OSError: + pass + if len(raw) == 65 and raw.endswith(b"\n"): + raw = raw[:-1] + if len(raw) != 64 or any(byte not in b"0123456789abcdef" for byte in raw): + return None + return _DockerIssuedContainerIdV1( + raw.decode("ascii"), + _token=_DOCKER_ISSUED_CONTAINER_ID_TOKEN, + ) + + def _observe_cleanup_command( + self, + capability: DockerSupportedV1, + command: tuple[str, ...], + ) -> _DockerCommandObservationV1: + if not self._in_owner_process_v1(): + raise RuntimeError("native Docker cleanup belongs to its creator process") + if not _docker_supported_is_valid_v1(capability): + raise TypeError("Docker cleanup requires one observed capability") + policy = capability.policy + return self._observe_command( + command, + stdout_limit=policy.probe_output_limit, + stderr_limit=policy.probe_output_limit, + timeout_ns=policy.probe_timeout_ns, + ) + + def _container_is_absent_v1( + self, + capability: DockerSupportedV1, + container_id: _DockerIssuedContainerIdV1, + ) -> bool: + observation = self._observe_cleanup_command( + capability, + _render_native_command_v1( + "cleanup_ls", + capability.command_coordinate, + { + _NativeCommandSlotV1.CONTAINER_FILTER: ( + f"id={str(container_id)}", + ), + }, + ), + ) + return ( + type(observation) is _DockerCommandExitedV1 + and observation.returncode == 0 + and observation.stdout == b"" + and observation.stderr == b"" + ) + + def _cleanup_container( + self, + lease: _NativeRunLeaseV1, + *, + spawn_may_have_started: bool = False, + ) -> str | None: + if not self._owns_run_lease_v1(lease): + raise TypeError("native run lease does not belong to this adapter") + if type(spawn_may_have_started) is not bool: + raise TypeError("native Docker spawn state must be bool") + if not self._lease_belongs_to_current_process_v1(lease): + return "native Docker run lease belongs to another process" + if not lease.launched and not spawn_may_have_started: + return None + capability = lease.capability + container_id = self._docker_issued_container_id_v1(lease) + if container_id is None: + return "Docker-issued cleanup ID is unavailable" + try: + inspection = self._observe_cleanup_command( + capability, + _render_native_command_v1( + "cleanup_inspect", + capability.command_coordinate, + { + _NativeCommandSlotV1.CONTAINER_COORDINATE: ( + str(container_id), + ), + }, + ), + ) + if ( + type(inspection) is _DockerCommandExitedV1 + and inspection.returncode != 0 + ): + return ( + None + if self._container_is_absent_v1(capability, container_id) + else "Docker container absence could not be verified" + ) + if ( + type(inspection) is not _DockerCommandExitedV1 + or inspection.returncode != 0 + or inspection.stdout != container_id.encode("ascii") + b"\n" + or inspection.stderr + ): + return "Docker-issued cleanup ID did not resolve exactly" + self._observe_cleanup_command( + capability, + _render_native_command_v1( + "cleanup_rm", + capability.command_coordinate, + { + _NativeCommandSlotV1.CONTAINER_COORDINATE: ( + str(container_id), + ), + }, + ) + ) + if not self._container_is_absent_v1(capability, container_id): + return "Docker container absence could not be verified" + except Exception: + return "Docker container cleanup observer raised" + return None + + +class BuildFailureReasonV1(StrEnum): + CONTRACT_VIOLATION = "contract_violation" + PROCESS_FAILED = "process_failed" + CLEANUP_FAILED = "cleanup_failed" + INPUT_TRANSFER_FAILED = "input_transfer_failed" + TIMEOUT = "timeout" + OUTPUT_LIMIT = "output_limit" + OBSERVER_FAILURE = "observer_failure" + INVALID_OUTPUT = "invalid_output" + + +BuildAttemptObservationV1: TypeAlias = DockerBuildProcessObservationV1 + + +class BuildSessionV1(tuple): + """Owned coordinates shared by every attempt in one two-build session.""" + + __slots__ = () + + def __new__( + cls, + capability: DockerSupportedV1, + input_value: _build_input.SealedInputV1, + max_output_bytes: int, + *, + _token: object, + ) -> BuildSessionV1: + if ( + _token is not _BUILD_SESSION_TOKEN + or not _docker_supported_is_valid_v1(capability) + or not _build_input.sealed_input_is_intact_v1(input_value) + or type(max_output_bytes) is not int + or max_output_bytes <= 0 + or max_output_bytes > capability.policy.stdout_limit + ): + raise TypeError("invalid two-build session coordinates") + return tuple.__new__( + cls, + ( + capability, + input_value, + max_output_bytes, + ), + ) + + @property + def policy(self) -> DockerBuildPolicyV1: + return self.capability.policy + + @property + def capability(self) -> DockerSupportedV1: + return self[0] + + @property + def input_value(self) -> _build_input.SealedInputV1: + return self[1] + + @property + def max_output_bytes(self) -> int: + return self[2] + + +def _build_session_v1( + capability: DockerSupportedV1, + input_value: _build_input.SealedInputV1, + max_output_bytes: int, +) -> BuildSessionV1: + return BuildSessionV1( + capability, + input_value, + max_output_bytes, + _token=_BUILD_SESSION_TOKEN, + ) + + +def _build_session_is_valid_v1(value: object) -> bool: + if type(value) is not BuildSessionV1: + return False + try: + canonical = _build_session_v1( + value.capability, + value.input_value, + value.max_output_bytes, + ) + return tuple(canonical) == tuple(value) + except Exception: + return False + + +class BuildByteRelationV1(StrEnum): + IDENTICAL = "identical" + DIFFERENT = "different" + + +class TwoBuildObservationV1(tuple): + """Two fresh successful attempts and their observed byte relation.""" + + __slots__ = () + + def __new__( + cls, + session: BuildSessionV1, + processes: tuple[DockerBuildExitedV1, DockerBuildExitedV1], + *, + _token: object, + ) -> TwoBuildObservationV1: + if ( + _token is not _TWO_BUILD_OBSERVATION_TOKEN + or not _build_session_is_valid_v1(session) + or type(processes) is not tuple + or len(processes) != 2 + ): + raise TypeError("invalid two-build observation") + owned: list[DockerBuildExitedV1] = [] + for process in processes: + canonical = _canonical_process_observation_v1( + process, + session.input_value, + session.max_output_bytes, + session.policy.stderr_limit, + ) + if type(canonical) is not DockerBuildExitedV1 or canonical.returncode != 0: + raise TypeError("two-build observation requires successful exits") + owned.append(canonical) + owned_processes = (owned[0], owned[1]) + relation = ( + BuildByteRelationV1.IDENTICAL + if owned_processes[0].stdout == owned_processes[1].stdout + else BuildByteRelationV1.DIFFERENT + ) + return tuple.__new__(cls, (session, relation, owned_processes)) + + @property + def session(self) -> BuildSessionV1: + return self[0] + + @property + def relation(self) -> BuildByteRelationV1: + return self[1] + + @property + def policy(self) -> DockerBuildPolicyV1: + return self.session.policy + + @property + def capability(self) -> DockerSupportedV1: + return self.session.capability + + @property + def input_value(self) -> _build_input.SealedInputV1: + return self.session.input_value + + @property + def max_output_bytes(self) -> int: + return self.session.max_output_bytes + + @property + def processes( + self, + ) -> tuple[DockerBuildExitedV1, DockerBuildExitedV1]: + return self[2] + + @property + def outputs(self) -> tuple[bytes, bytes]: + return self.processes[0].stdout, self.processes[1].stdout + + @property + def first_sha256(self) -> bytes: + return hashlib.sha256(self.outputs[0]).digest() + + @property + def second_sha256(self) -> bytes: + return hashlib.sha256(self.outputs[1]).digest() + + +class BuildRejectedV1(tuple): + """Typed failed attempt retaining the successful causal prefix.""" + + __slots__ = () + + def __new__( + cls, + attempt: int, + reason: BuildFailureReasonV1, + process: BuildAttemptObservationV1 | None = None, + *, + session: BuildSessionV1 | None = None, + completed_processes: tuple[DockerBuildExitedV1, ...] = (), + ) -> BuildRejectedV1: + if ( + type(attempt) is not int + or attempt not in (1, 2) + or type(reason) is not BuildFailureReasonV1 + or type(completed_processes) is not tuple + ): + raise TypeError("invalid build rejection") + if session is None: + if ( + reason is not BuildFailureReasonV1.CONTRACT_VIOLATION + or process is not None + or completed_processes + ): + raise TypeError("context-free rejection must be a contract violation") + return tuple.__new__(cls, (attempt, reason, None, None, ())) + if ( + not _build_session_is_valid_v1(session) + or len(completed_processes) != attempt - 1 + ): + raise TypeError("build rejection lost its causal prefix") + owned_completed: list[DockerBuildExitedV1] = [] + for completed in completed_processes: + canonical = _canonical_process_observation_v1( + completed, + session.input_value, + session.max_output_bytes, + session.policy.stderr_limit, + ) + if type(canonical) is not DockerBuildExitedV1 or canonical.returncode != 0: + raise TypeError("causal prefix contains a failed attempt") + owned_completed.append(canonical) + if process is None: + owned_process: BuildAttemptObservationV1 | None = None + else: + owned_process = _canonical_process_observation_v1( + process, + session.input_value, + session.max_output_bytes, + session.policy.stderr_limit, + ) + expected_process_types: dict[BuildFailureReasonV1, tuple[type, ...]] = { + BuildFailureReasonV1.CONTRACT_VIOLATION: (), + BuildFailureReasonV1.PROCESS_FAILED: (DockerBuildExitedV1,), + BuildFailureReasonV1.CLEANUP_FAILED: (DockerBuildCleanupFailureV1,), + BuildFailureReasonV1.INPUT_TRANSFER_FAILED: ( + DockerBuildInputRejectedV1, + ), + BuildFailureReasonV1.TIMEOUT: (DockerBuildTimedOutV1,), + BuildFailureReasonV1.OUTPUT_LIMIT: (DockerBuildOutputLimitV1,), + BuildFailureReasonV1.OBSERVER_FAILURE: ( + DockerBuildObserverFailureV1, + ), + BuildFailureReasonV1.INVALID_OUTPUT: (DockerBuildExitedV1,), + } + expected = expected_process_types[reason] + if not expected: + if owned_process is not None: + raise TypeError("contract-violation rejection cannot retain authority") + elif type(owned_process) not in expected: + raise TypeError("build rejection reason and observation disagree") + if ( + ( + reason is BuildFailureReasonV1.PROCESS_FAILED + and owned_process.returncode == 0 + ) + or ( + reason is BuildFailureReasonV1.INVALID_OUTPUT + and owned_process.returncode != 0 + ) + ): + raise TypeError("build rejection exit status disagrees with reason") + return tuple.__new__( + cls, + ( + attempt, + reason, + owned_process, + session, + tuple(owned_completed), + ), + ) + + @property + def attempt(self) -> int: + return self[0] + + @property + def reason(self) -> BuildFailureReasonV1: + return self[1] + + @property + def process(self) -> BuildAttemptObservationV1 | None: + return self[2] + + @property + def session(self) -> BuildSessionV1 | None: + return self[3] + + @property + def completed_processes(self) -> tuple[DockerBuildExitedV1, ...]: + return self[4] + + +def two_build_observation_matches_v1( + value: object, + session: BuildSessionV1, +) -> bool: + if ( + type(value) is not TwoBuildObservationV1 + or not _build_session_is_valid_v1(session) + ): + return False + try: + replayed = TwoBuildObservationV1( + session, + value.processes, + _token=_TWO_BUILD_OBSERVATION_TOKEN, + ) + return tuple(replayed) == tuple(value) + except Exception: + return False + + +BuildTransportResultV1: TypeAlias = ( + TwoBuildObservationV1 | BuildRejectedV1 +) + + +class ControlledBuildTransportV1: + """Own two fresh attempts; callers own semantic input and output admission.""" + + def __init__( + self, + *, + policy: DockerBuildPolicyV1, + backend: DockerBuildBackendV1, + ) -> None: + if not docker_policy_is_valid_v1(policy): + raise TypeError("policy must be DockerBuildPolicyV1") + self._policy = DockerBuildPolicyV1(*tuple(policy)) + self._backend = backend + # Fork copies Python object state and may copy a locked mutex. This + # controller's capability is therefore valid only in its creator; + # every public operation checks PID before it can touch that mutex. + self._owner_pid = os.getpid() + self._probed_capability: DockerSupportedV1 | None = None + self._consumed = False + # Probe result and its one-shot BUILD right are one causal state. A + # lock makes the state transition indivisible across reentrant or + # concurrent callers without holding it during caller/backend IO. + self._lease_lock = threading.Lock() + self._probe_in_flight = False + + def _in_owner_process_v1(self) -> bool: + return os.getpid() == self._owner_pid + + def probe(self) -> DockerCapabilityReportV1: + if not self._in_owner_process_v1(): + return DockerUnsupportedV1( + DockerBlockerReasonV1.BACKEND_CONTRACT, + "build transport capability belongs to its creator process", + ) + with self._lease_lock: + if ( + self._consumed + or self._probed_capability is not None + or self._probe_in_flight + ): + return DockerUnsupportedV1( + DockerBlockerReasonV1.BACKEND_CONTRACT, + "build transport capability is one-shot", + ) + self._probe_in_flight = True + # Start at a typed rejection before foreign IO so every ordinary exit + # has a public report. A BaseException still propagates unchanged. + outcome: DockerCapabilityReportV1 = DockerUnsupportedV1( + DockerBlockerReasonV1.BACKEND_CONTRACT, + "Docker capability probe produced no canonical report", + ) + try: + report = self._backend.probe() + except Exception: + outcome = DockerUnsupportedV1( + DockerBlockerReasonV1.BACKEND_CONTRACT, + "Docker capability probe raised", + ) + else: + if type(report) is DockerUnsupportedV1: + if _docker_unsupported_is_valid_v1(report): + outcome = DockerUnsupportedV1(*tuple(report)) + else: + outcome = DockerUnsupportedV1( + DockerBlockerReasonV1.BACKEND_CONTRACT, + "Docker capability rejection is not canonical", + ) + elif ( + not _docker_supported_is_valid_v1(report) + or report.policy != self._policy + ): + outcome = DockerUnsupportedV1( + DockerBlockerReasonV1.BACKEND_CONTRACT, + "Docker capability report does not match build policy", + ) + else: + outcome = report + finally: + # BaseException must not permanently leave this controller in the + # transient PROBING state. It still propagates to the caller; the + # cleanup only revokes that incomplete external observation. + if not self._in_owner_process_v1(): + outcome = DockerUnsupportedV1( + DockerBlockerReasonV1.BACKEND_CONTRACT, + "build transport capability belongs to its creator process", + ) + else: + with self._lease_lock: + self._probe_in_flight = False + if type(outcome) is DockerSupportedV1: + self._probed_capability = outcome + return outcome + + def build( + self, + capability: DockerSupportedV1, + input_value: _build_input.SealedInputV1, + max_output_bytes: int, + *, + input_admission: Callable[[_build_input.SealedInputV1], bool], + output_admission: Callable[[bytes], bool], + ) -> BuildTransportResultV1: + if not self._in_owner_process_v1(): + return BuildRejectedV1(1, BuildFailureReasonV1.CONTRACT_VIOLATION) + if ( + not _docker_supported_is_valid_v1(capability) + or capability.policy != self._policy + or type(max_output_bytes) is not int + or max_output_bytes <= 0 + or max_output_bytes > capability.policy.stdout_limit + or not callable(input_admission) + or not callable(output_admission) + or not _build_input.sealed_input_is_intact_v1(input_value) + ): + return BuildRejectedV1(1, BuildFailureReasonV1.CONTRACT_VIOLATION) + if not self._in_owner_process_v1(): + return BuildRejectedV1(1, BuildFailureReasonV1.CONTRACT_VIOLATION) + with self._lease_lock: + if ( + self._consumed + or capability is not self._probed_capability + or not _docker_supported_is_valid_v1(capability) + or capability.policy != self._policy + ): + return BuildRejectedV1(1, BuildFailureReasonV1.CONTRACT_VIOLATION) + # A rejected declaration has not reached an owned execution attempt, + # so it must not burn the lease. Recheck after claim below because a + # callback is external and may be reentrant or mutate hostile input. + if not self._admitted(input_admission, input_value): + return BuildRejectedV1(1, BuildFailureReasonV1.CONTRACT_VIOLATION) + if not self._in_owner_process_v1(): + return BuildRejectedV1(1, BuildFailureReasonV1.CONTRACT_VIOLATION) + with self._lease_lock: + if ( + self._consumed + or capability is not self._probed_capability + or not _docker_supported_is_valid_v1(capability) + or capability.policy != self._policy + ): + return BuildRejectedV1(1, BuildFailureReasonV1.CONTRACT_VIOLATION) + self._consumed = True + if not _build_input.sealed_input_is_intact_v1(input_value): + return BuildRejectedV1(1, BuildFailureReasonV1.CONTRACT_VIOLATION) + try: + session = _build_session_v1( + capability, + input_value, + max_output_bytes, + ) + except Exception: + return BuildRejectedV1(1, BuildFailureReasonV1.CONTRACT_VIOLATION) + completed: list[DockerBuildExitedV1] = [] + for attempt in (1, 2): + if ( + not self._in_owner_process_v1() + or not _build_input.sealed_input_is_intact_v1(input_value) + or not self._admitted(input_admission, input_value) + or not self._in_owner_process_v1() + ): + return BuildRejectedV1( + attempt, + BuildFailureReasonV1.CONTRACT_VIOLATION, + session=session, + completed_processes=tuple(completed), + ) + built = self._build_once( + attempt, + session, + output_admission, + tuple(completed), + ) + if not self._in_owner_process_v1(): + return BuildRejectedV1( + attempt, + BuildFailureReasonV1.CONTRACT_VIOLATION, + session=session, + completed_processes=tuple(completed), + ) + if type(built) is BuildRejectedV1: + return built + completed.append(built) + return TwoBuildObservationV1( + session, + (completed[0], completed[1]), + _token=_TWO_BUILD_OBSERVATION_TOKEN, + ) + + @staticmethod + def _admitted( + admission: Callable[[object], bool], + value: object, + ) -> bool: + try: + return admission(value) is True + except Exception: + return False + + def _build_once( + self, + attempt: int, + session: BuildSessionV1, + output_admission: Callable[[bytes], bool], + completed_processes: tuple[DockerBuildExitedV1, ...], + ) -> DockerBuildExitedV1 | BuildRejectedV1: + if not _build_session_is_valid_v1(session): + return BuildRejectedV1( + attempt, + BuildFailureReasonV1.CONTRACT_VIOLATION, + ) + contract_rejection = BuildRejectedV1( + attempt, + BuildFailureReasonV1.CONTRACT_VIOLATION, + session=session, + completed_processes=completed_processes, + ) + try: + return self._observe_build_attempt_v1( + attempt, + session, + output_admission, + completed_processes, + ) + except Exception: + return contract_rejection + + def _observe_build_attempt_v1( + self, + attempt: int, + session: BuildSessionV1, + output_admission: Callable[[bytes], bool], + completed_processes: tuple[DockerBuildExitedV1, ...], + ) -> DockerBuildExitedV1 | BuildRejectedV1: + contract_rejection = BuildRejectedV1( + attempt, + BuildFailureReasonV1.CONTRACT_VIOLATION, + session=session, + completed_processes=completed_processes, + ) + request = DockerBuildRequestV1( + attempt, + session.capability, + session.input_value, + session.max_output_bytes, + ) + try: + observed = self._backend.run_build(request) + except Exception: + return contract_rejection + if not self._in_owner_process_v1(): + return contract_rejection + try: + process = _canonical_process_observation_v1( + observed, + session.input_value, + session.max_output_bytes, + session.policy.stderr_limit, + ) + except TypeError: + return contract_rejection + reason_by_type: dict[type, BuildFailureReasonV1] = { + DockerBuildCleanupFailureV1: BuildFailureReasonV1.CLEANUP_FAILED, + DockerBuildInputRejectedV1: BuildFailureReasonV1.INPUT_TRANSFER_FAILED, + DockerBuildTimedOutV1: BuildFailureReasonV1.TIMEOUT, + DockerBuildOutputLimitV1: BuildFailureReasonV1.OUTPUT_LIMIT, + DockerBuildObserverFailureV1: BuildFailureReasonV1.OBSERVER_FAILURE, + } + failure_reason = reason_by_type.get(type(process)) + if failure_reason is not None: + return BuildRejectedV1( + attempt, + failure_reason, + process, + session=session, + completed_processes=completed_processes, + ) + if type(process) is not DockerBuildExitedV1: + return contract_rejection + if not docker_build_exited_is_valid_v1( + process, + session.input_value, + session.max_output_bytes, + session.policy.stderr_limit, + ): + return contract_rejection + if process.returncode != 0: + return BuildRejectedV1( + attempt, + BuildFailureReasonV1.PROCESS_FAILED, + process, + session=session, + completed_processes=completed_processes, + ) + transfer = process.input_transfer + if ( + type(transfer) is not BuildInputTransferV1 + or transfer.bundle_identity != session.input_value.binding_identity + or transfer.expected_length != session.input_value.length + or transfer.expected_sha256 != session.input_value.sha256 + or transfer.written_length != session.input_value.length + or transfer.written_sha256 != session.input_value.sha256 + ): + return contract_rejection + if not self._admitted(output_admission, process.stdout): + return BuildRejectedV1( + attempt, + BuildFailureReasonV1.INVALID_OUTPUT, + process, + session=session, + completed_processes=completed_processes, + ) + if not self._in_owner_process_v1(): + return contract_rejection + return process diff --git a/proof/region/v1/tests/test_build.py b/proof/region/v1/tests/test_build.py new file mode 100644 index 00000000..fffbfa5e --- /dev/null +++ b/proof/region/v1/tests/test_build.py @@ -0,0 +1,2331 @@ +#!/usr/bin/env python3 +"""RED contract for an identity-preserving engine-neutral BUILD leaf.""" + +from __future__ import annotations + +import ast +import dis +import gc +import hashlib +import importlib +import os +import select +import subprocess +import sys +import threading +import unittest +from pathlib import Path +from unittest import mock + + +PROOF = Path(__file__).resolve().parents[1] +ARB = PROOF / "arb" +ARB_TESTS = ARB / "tests" +REPO = PROOF.parents[2] +sys.path[:0] = (str(REPO), str(PROOF), str(ARB), str(ARB_TESTS)) + +import pipeline # noqa: E402 +from proof.region.v1.arb.tests import gate as arb_gate # noqa: E402 +from test_pipeline import ( # noqa: E402 + _docker_capability, + _probe_native_backend, + _request, +) +from test_receipt import _execute # noqa: E402 + + +# The gate owns the inventory contract; this test reuses the same SSOT instead +# of maintaining a second literal that could drift from the executed gate. +ARB_INVENTORY_SHA256_V1 = arb_gate.EXPECTED_TEST_INVENTORY_SHA256 +ARB_ORDER_SHA256_V1 = ( + "6700241b8685179ecaed8eab062e65581ae183fa544b02b039b464d26ce53d7c" +) +ARB_TEST_COUNT_V1 = arb_gate.EXPECTED_TEST_COUNT + +MOVED_INPUT_SURFACE_V1 = ( + "CanonicalInputLimitsV1", + "SealedInputV1", + "seal_input_v1", + "sealed_input_is_intact_v1", + "canonical_ustar_v1", +) + +MOVED_TRANSPORT_SURFACE_V1 = ( + "DockerBuildPolicyV1", + "DockerUserModeV1", + "DockerBlockerReasonV1", + "DockerUnsupportedV1", + "DockerSupportedV1", + "DockerDaemonObservationV1", + "NativeCommandCoordinateV1", + "DockerBuildRequestV1", + "transport_policy_identity_v1", + "native_command_contract_identity_v1", + "native_command_coordinate_v1", + "docker_capability_identity_v1", + "BuildInputTransferProgressV1", + "BuildInputTransferV1", + "DockerBuildExitedV1", + "DockerBuildTimedOutV1", + "DockerOutputStreamV1", + "DockerBuildOutputLimitV1", + "DockerBuildObserverFailureV1", + "DockerBuildInputRejectedV1", + "DockerCleanupTriggerV1", + "CleanupResourceV1", + "CleanupFailureRecordV1", + "DockerBuildCleanupFailureV1", + "DockerBuildBackendV1", + "NativeDockerBuildBackendV1", + "ControlledBuildTransportV1", + "BuildFailureReasonV1", + "BuildRejectedV1", + "BuildByteRelationV1", + "TwoBuildObservationV1", + "build_process_bytes_v1", +) + +REMOVED_TRANSPORT_SURFACE_V1 = ( + "NonReproducibleBuildV1", + "ReproducibleBuildV1", + "docker_report_matches_policy_v1", +) + +FORBIDDEN_INPUT_IMPORTS_V1 = ( + "arb", + "mpfi", + "pipeline", + "formula", + "comparator", + "receipt", + "region_proof_protocol", + "provenance", +) + +# Both shared leaves must remain unaware of engine semantics; separate names +# keep the two contracts legible without making their import policy diverge. +FORBIDDEN_TRANSPORT_IMPORTS_V1 = FORBIDDEN_INPUT_IMPORTS_V1 + + +def _imported_modules(source: str) -> tuple[str, ...]: + modules: list[str] = [] + for node in ast.walk(ast.parse(source)): + if isinstance(node, ast.Import): + modules.extend(alias.name for alias in node.names) + elif isinstance(node, ast.ImportFrom): + modules.append(node.module or "") + return tuple(modules) + + +def _digest(label: str) -> bytes: + return hashlib.sha256(label.encode("ascii")).digest() + + +def _sealed_input() -> object: + build_input = importlib.import_module("build.input") + return build_input.seal_input_v1(_digest("generic-build-binding"), b"input") + + +def _docker_capability_fixture(policy: object) -> object: + return _docker_capability(policy) + + +def _completed_process( + transport: object, + input_value: object, + stdout: bytes, + *, + returncode: int = 0, + stderr: bytes = b"", +) -> object: + transfer = transport._completed_build_input_transfer_v1( + input_value, + input_value.length, + input_value.sha256, + ) + return transport._docker_build_exited_v1( + returncode, + stdout, + stderr, + transfer, + ) + + +def _initial_progress(transport: object, input_value: object) -> object: + return transport._build_input_progress_v1( + input_value, + 0, + hashlib.sha256(b"").digest(), + ) + + +def _forged_exact_type(value_type: type[object]) -> object: + if issubclass(value_type, tuple): + return tuple.__new__(value_type, ()) + return object.__new__(value_type) + + +class _ScriptedBuildBackend: + def __init__(self, report: object, observations: tuple[object, ...]) -> None: + self._report = report + self._observations = list(observations) + self.requests: list[object] = [] + + def probe(self) -> object: + return self._report + + def run_build(self, request: object) -> object: + self.requests.append(request) + return self._observations.pop(0) + + +def _racing_build_transport( + transport: object, + *, + policy: object, + backend: object, +) -> object: + """Force the former unlocked check→consume race without scheduler guesses.""" + + class TrackingLock: + def __init__(self) -> None: + self._lock = threading.Lock() + self._owner: int | None = None + + def __enter__(self) -> TrackingLock: + self._lock.acquire() + self._owner = threading.get_ident() + return self + + def __exit__( + self, + _exception_type: object, + _exception: object, + _traceback: object, + ) -> None: + self._owner = None + self._lock.release() + + def held_by_current_thread(self) -> bool: + return self._owner == threading.get_ident() + + class RacingController(transport.ControlledBuildTransportV1): + def __init__(self) -> None: + self._consume_barrier = threading.Barrier(2) + self._race_armed = False + super().__init__(policy=policy, backend=backend) + self._lease_lock = TrackingLock() + + def arm_consume_race(self) -> None: + self._race_armed = True + + def __getattribute__(self, name: str) -> object: + if ( + name == "_consumed" + and object.__getattribute__(self, "_race_armed") + and not object.__getattribute__( + self, + "_lease_lock", + ).held_by_current_thread() + ): + object.__getattribute__(self, "_consume_barrier").wait(timeout=2) + return super().__getattribute__(name) + + return RacingController() + + +def _controlled_build( + transport: object, + policy: object, + observations: tuple[object, ...], + *, + max_output_bytes: int = 64, + input_value: object | None = None, +) -> tuple[object, _ScriptedBuildBackend, object, object]: + if input_value is None: + input_value = _sealed_input() + capability = _docker_capability_fixture(policy) + backend = _ScriptedBuildBackend(capability, observations) + controller = transport.ControlledBuildTransportV1( + policy=policy, + backend=backend, + ) + owned_capability = controller.probe() + result = controller.build( + owned_capability, + input_value, + max_output_bytes, + input_admission=lambda _value: True, + output_admission=lambda _value: True, + ) + return result, backend, owned_capability, input_value + + +class ExistingArbGateTests(unittest.TestCase): + def test_existing_arb_suite_keeps_exact_count_order_and_inventory(self) -> None: + tests = tuple(arb_gate._iter_tests_v1(arb_gate.full_suite_v1())) + identifiers = tuple(test.id() for test in tests) + ordered_preimage = b"".join( + identifier.encode("utf-8") + b"\n" for identifier in identifiers + ) + + self.assertEqual(len(identifiers), ARB_TEST_COUNT_V1) + self.assertEqual(len(set(identifiers)), ARB_TEST_COUNT_V1) + self.assertEqual( + arb_gate.test_inventory_sha256_v1(arb_gate.full_suite_v1()), + ARB_INVENTORY_SHA256_V1, + ) + self.assertEqual( + hashlib.sha256(ordered_preimage).hexdigest(), + ARB_ORDER_SHA256_V1, + ) + + +class ArbBuildIdentityCharacterizationTests(unittest.TestCase): + def test_arb_v2_binding_propagates_to_downstream_identities(self) -> None: + transport = importlib.import_module("build.transport") + request = _request() + result, _backend = _execute() + observed = result.evidence.build + process_bytes = transport.build_process_bytes_v1( + observed.build_processes[0] + ) + + self.assertEqual(observed.input_bundle_length, 174_080) + self.assertEqual( + observed.input_bundle_sha256.hex(), + "5d6e789a721aeed1a8ff023f0af5389711f85f6fe95294d8290b20301235f4df", + ) + self.assertEqual( + observed.input_bundle_identity.hex(), + "a9194ab4318be3283dc37efed4390de9b15d8c5d65a5f8c10dd3c59e41ed9978", + ) + self.assertEqual( + pipeline.pipeline_policy_identity_v2( + request.host_trust, + observed.docker_capability.policy, + ).hex(), + "5ff9cac8af5fee7ffb05d18da33721842150dafe43edd6f0e356566c7be12144", + ) + self.assertEqual(len(process_bytes), 196) + self.assertEqual( + hashlib.sha256(process_bytes).hexdigest(), + "d0bc7878be513e2b78515f35dfe33b54b4e4f45de27dc462be69f75a9f215073", + ) + self.assertEqual( + result.comparator.identity.hex(), + "4758d9369c3fbe987e4431291739d61da03b8923d2b98ddd212b2fff96627a61", + ) + self.assertEqual( + result.evidence.source_identity.hex(), + "07d85ad695ec17104bdb34f6e9819d25be08afb3aa485918c44a363d7679f7c9", + ) + self.assertEqual( + result.evidence.build_identity.hex(), + "59643d08452643e9b747d832dab30cc642ba7fc98e2f6dbf588436fb7a5b08d7", + ) + self.assertEqual( + result.evidence.run_identity.hex(), + "1255c905f657e0c1e9aee75828c9cdf6e0f3906c758535b1c58ca0723eb3715e", + ) + self.assertEqual( + result.evidence.identity.hex(), + "829363f9e2fdb26356b3def25681c6afcaeb9637e5a0f81a539c7af6f6512737", + ) + self.assertEqual( + result.claim.identity.hex(), + "edf6fb6e23b9af06289c19217f5a3b9e3f81335af0fb1bbecb0094f496263c55", + ) + + +class SharedBuildExtractionTests(unittest.TestCase): + def test_build_namespace_has_two_focused_shared_leaves(self) -> None: + package = importlib.import_module("build") + build_input = importlib.import_module("build.input") + transport = importlib.import_module("build.transport") + namespace = PROOF / "build" + + self.assertEqual( + Path(package.__file__).resolve(), + (namespace / "__init__.py").resolve(), + ) + self.assertEqual( + tuple(Path(item).resolve() for item in package.__path__), + (namespace.resolve(),), + ) + self.assertFalse((PROOF / "build.py").exists()) + self.assertEqual( + Path(build_input.__file__).resolve(), + (namespace / "input.py").resolve(), + ) + self.assertEqual( + Path(transport.__file__).resolve(), + (namespace / "transport.py").resolve(), + ) + self.assertFalse((ARB / "build").exists()) + self.assertFalse((PROOF / "mpfi/build").exists()) + self.assertFalse(hasattr(transport, "input")) + self.assertFalse(hasattr(transport, "build_input")) + self.assertFalse(hasattr(build_input, "normalized_source_entries_v1")) + for name in MOVED_INPUT_SURFACE_V1: + with self.subTest(name=name): + self.assertTrue(hasattr(build_input, name)) + for name in MOVED_TRANSPORT_SURFACE_V1: + with self.subTest(name=name): + self.assertTrue(hasattr(transport, name)) + for name in REMOVED_TRANSPORT_SURFACE_V1: + with self.subTest(removed=name): + self.assertFalse(hasattr(transport, name)) + + def test_shared_leaves_import_no_engine_or_proof_semantics(self) -> None: + surfaces = ( + ( + importlib.import_module("build.input"), + FORBIDDEN_INPUT_IMPORTS_V1, + ), + ( + importlib.import_module("build.transport"), + FORBIDDEN_TRANSPORT_IMPORTS_V1, + ), + ) + for surface, forbidden_imports in surfaces: + source = Path(surface.__file__).read_text(encoding="utf-8") + for module in _imported_modules(source): + with self.subTest(surface=surface.__name__, module=module): + top_level = module.lstrip(".").split(".", 1)[0].lower() + self.assertNotIn(top_level, forbidden_imports, module) + + def test_observation_contract_uses_current_non_claiming_language(self) -> None: + transport_source = (PROOF / "build" / "transport.py").read_text( + encoding="utf-8" + ) + pipeline_source = (ARB / "pipeline.py").read_text(encoding="utf-8") + protocol_source = (PROOF / "PROTOCOL.md").read_text(encoding="utf-8") + self.assertNotIn( + "backend-contract rejection cannot retain authority", + transport_source, + ) + self.assertNotIn("invalid reproducible-build digests", pipeline_source) + self.assertIn("fresh one-job VM workflow Arb", protocol_source) + self.assertIn("same-UID writer", protocol_source) + self.assertIn("Popen construction", protocol_source) + self.assertNotIn("cleanup выполняет только по его точному имени", protocol_source) + + def test_arb_consumers_move_atomically_without_compatibility_reexports(self) -> None: + build_input = importlib.import_module("build.input") + transport = importlib.import_module("build.transport") + arb_pipeline = pipeline + arb_receipt = importlib.import_module("receipt") + request = _request() + bundle = arb_pipeline._seal_build_input_bundle_v1( + request, + arb_pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + ) + + self.assertIs(arb_pipeline.build_input, build_input) + self.assertIs(arb_pipeline.build_transport, transport) + self.assertIs(arb_receipt.build_transport, transport) + self.assertFalse(hasattr(arb_receipt, "build_input")) + self.assertIs(type(bundle), build_input.SealedInputV1) + for name in MOVED_INPUT_SURFACE_V1 + MOVED_TRANSPORT_SURFACE_V1: + with self.subTest(consumer=arb_pipeline.__name__, name=name): + self.assertFalse(hasattr(arb_pipeline, name)) + for name in MOVED_TRANSPORT_SURFACE_V1: + with self.subTest(consumer=arb_receipt.__name__, name=name): + self.assertFalse(hasattr(arb_receipt, name)) + for name in REMOVED_TRANSPORT_SURFACE_V1: + with self.subTest(consumer=arb_pipeline.__name__, removed=name): + self.assertFalse(hasattr(arb_pipeline, name)) + with self.subTest(consumer=arb_receipt.__name__, removed=name): + self.assertFalse(hasattr(arb_receipt, name)) + + def test_arb_policy_reuses_generic_observer_ceiling_ssot(self) -> None: + transport = importlib.import_module("build.transport") + pipeline_source = (ARB / "pipeline.py").read_text(encoding="utf-8") + for name in ( + "BUILD_STDOUT_LIMIT_V1", + "BUILD_STDERR_LIMIT_V1", + "BUILD_TIMEOUT_NS_V1", + "DOCKER_PROBE_OUTPUT_LIMIT_V1", + "DOCKER_PROBE_TIMEOUT_NS_V1", + ): + with self.subTest(name=name): + self.assertIs(getattr(pipeline, name), getattr(transport, name)) + self.assertIn(f"build_transport.{name}", pipeline_source) + + def test_shared_input_and_policy_are_deeply_immutable_coordinates(self) -> None: + build_input = importlib.import_module("build.input") + transport = importlib.import_module("build.transport") + sealed = build_input.seal_input_v1( + hashlib.sha256(b"binding").digest(), + b"exact bytes", + ) + policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 + + for value in (sealed, policy): + with self.subTest(value=type(value).__name__): + self.assertFalse(hasattr(value, "__dict__")) + with self.assertRaises((AttributeError, TypeError)): + value[0] = value[0] + with self.assertRaises((AttributeError, TypeError)): + object.__setattr__(value, "foreign", object()) + self.assertIs(type(sealed), build_input.SealedInputV1) + self.assertIs(type(policy), transport.DockerBuildPolicyV1) + + def test_forged_source_authorities_fail_in_the_arb_taxonomy(self) -> None: + build_input = importlib.import_module("build.input") + provenance = importlib.import_module("provenance") + request = _request() + lock = request.source_lock.sources[0] + admitted = request.admitted_sources.sources[0] + + for hostile_lock, hostile_admitted in ( + (object.__new__(provenance.SourceReleaseLockV1), admitted), + (lock, object.__new__(provenance.SafeSourceArchiveV1)), + ): + with self.subTest(authority=type(hostile_lock).__name__): + with self.assertRaises(pipeline.PipelineInputErrorV1) as raised: + pipeline._normalized_source_entries_v1( + hostile_lock, + hostile_admitted, + ) + self.assertEqual( + raised.exception.reason, + pipeline.PipelineInputReasonV1.FOREIGN_SOURCE_CAPABILITY, + ) + + +class SharedBuildTransportTargetTests(unittest.TestCase): + def test_stream_close_fallback_requires_current_stream_ownership(self) -> None: + transport = importlib.import_module("build.transport") + real_close = os.close + + descriptor = os.open(os.devnull, os.O_RDONLY) + + class ClosesThenRaises: + closed = False + replacement: int | None = None + close_calls = 0 + + def close(self) -> None: + self.close_calls += 1 + real_close(descriptor) + self.closed = True + self.replacement = os.open(os.devnull, os.O_RDONLY) + raise OSError("stream close released its descriptor") + + stream = ClosesThenRaises() + try: + with mock.patch.object( + transport.os, + "close", + side_effect=AssertionError("helper closed a numeric descriptor"), + ) as direct_close: + failed, interruption = ( + transport.NativeDockerBuildBackendV1._close_owned_stream( + stream, + ) + ) + + self.assertTrue(failed) + self.assertIsNone(interruption) + self.assertEqual(stream.replacement, descriptor) + self.assertEqual(stream.close_calls, 1) + direct_close.assert_not_called() + os.fstat(descriptor) + finally: + if stream.replacement is not None: + try: + real_close(stream.replacement) + except OSError: + pass + + descriptor = os.open(os.devnull, os.O_RDONLY) + + class RaisesBeforeClose: + closed = False + close_calls = 0 + + def close(self) -> None: + self.close_calls += 1 + if self.close_calls == 1: + raise OSError("stream close kept its descriptor") + real_close(descriptor) + self.closed = True + + stream = RaisesBeforeClose() + try: + with mock.patch.object( + transport.os, + "close", + side_effect=AssertionError("helper closed a numeric descriptor"), + ) as direct_close: + failed, interruption = ( + transport.NativeDockerBuildBackendV1._close_owned_stream( + stream, + ) + ) + + self.assertTrue(failed) + self.assertIsNone(interruption) + self.assertEqual(stream.close_calls, 2) + direct_close.assert_not_called() + with self.assertRaises(OSError): + os.fstat(descriptor) + finally: + try: + real_close(descriptor) + except OSError: + pass + + def test_session_property_is_pure_while_boundary_validator_rejects_forgery(self) -> None: + transport = importlib.import_module("build.transport") + build_input = importlib.import_module("build.input") + forged_input = tuple.__new__(build_input.SealedInputV1, ()) + session = tuple.__new__( + transport.BuildSessionV1, + ( + _docker_capability_fixture(pipeline.ARB_BUILD_TRANSPORT_POLICY_V1), + forged_input, + 64, + ), + ) + + self.assertIs(session.input_value, forged_input) + self.assertFalse(transport._build_session_is_valid_v1(session)) + + def test_overlapping_probe_is_rejected_without_a_second_backend_probe(self) -> None: + transport = importlib.import_module("build.transport") + policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 + capability = _docker_capability_fixture(policy) + + class BlockingBackend: + def __init__(self) -> None: + self.entered = threading.Event() + self.release = threading.Event() + self.calls = 0 + + def probe(self) -> object: + self.calls += 1 + self.entered.set() + self.release.wait(timeout=2) + return capability + + def run_build(self, _request: object) -> object: + raise AssertionError("probe-only test reached build") + + backend = BlockingBackend() + controller = transport.ControlledBuildTransportV1( + policy=policy, + backend=backend, + ) + first_results: list[object] = [] + second_results: list[object] = [] + second_done = threading.Event() + first = threading.Thread(target=lambda: first_results.append(controller.probe())) + + def second_probe() -> None: + try: + second_results.append(controller.probe()) + finally: + second_done.set() + + second = threading.Thread(target=second_probe) + first.start() + self.assertTrue(backend.entered.wait(timeout=1)) + second.start() + try: + self.assertTrue(second_done.wait(timeout=1)) + finally: + backend.release.set() + first.join(timeout=2) + second.join(timeout=2) + + self.assertFalse(first.is_alive()) + self.assertFalse(second.is_alive()) + self.assertEqual(backend.calls, 1) + self.assertEqual(first_results, [capability]) + self.assertEqual(len(second_results), 1) + self.assertIs(type(second_results[0]), transport.DockerUnsupportedV1) + self.assertEqual( + second_results[0].reason, + transport.DockerBlockerReasonV1.BACKEND_CONTRACT, + ) + + def test_one_probe_lease_cannot_start_two_concurrent_two_build_sessions(self) -> None: + transport = importlib.import_module("build.transport") + policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 + input_value = _sealed_input() + capability = _docker_capability_fixture(policy) + backend = _ScriptedBuildBackend( + capability, + tuple( + _completed_process(transport, input_value, b"same executable") + for _ in range(4) + ), + ) + controller = _racing_build_transport( + transport, + policy=policy, + backend=backend, + ) + owned_capability = controller.probe() + controller.arm_consume_race() + start = threading.Barrier(3) + results: list[object] = [] + failures: list[BaseException] = [] + + def build() -> None: + try: + start.wait(timeout=2) + results.append( + controller.build( + owned_capability, + input_value, + 64, + input_admission=lambda _value: True, + output_admission=lambda _value: True, + ) + ) + except BaseException as error: + failures.append(error) + + workers = tuple(threading.Thread(target=build) for _ in range(2)) + for worker in workers: + worker.start() + start.wait(timeout=2) + for worker in workers: + worker.join(timeout=3) + self.assertFalse(worker.is_alive()) + + self.assertEqual(failures, []) + self.assertEqual(len(results), 2) + self.assertEqual( + sum(type(result) is transport.TwoBuildObservationV1 for result in results), + 1, + ) + rejections = tuple( + result + for result in results + if type(result) is transport.BuildRejectedV1 + ) + self.assertEqual(len(rejections), 1) + self.assertEqual( + rejections[0].reason, + transport.BuildFailureReasonV1.CONTRACT_VIOLATION, + ) + self.assertEqual(len(backend.requests), 2) + + def test_rejected_preflight_preserves_the_unconsumed_build_lease(self) -> None: + transport = importlib.import_module("build.transport") + policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 + input_value = _sealed_input() + capability = _docker_capability_fixture(policy) + backend = _ScriptedBuildBackend( + capability, + ( + _completed_process(transport, input_value, b"same executable"), + _completed_process(transport, input_value, b"same executable"), + ), + ) + controller = transport.ControlledBuildTransportV1( + policy=policy, + backend=backend, + ) + owned_capability = controller.probe() + + rejected = controller.build( + owned_capability, + input_value, + 64, + input_admission=lambda _value: False, + output_admission=lambda _value: True, + ) + self.assertIs(type(rejected), transport.BuildRejectedV1) + self.assertEqual( + rejected.reason, + transport.BuildFailureReasonV1.CONTRACT_VIOLATION, + ) + self.assertEqual(backend.requests, []) + + admitted = controller.build( + owned_capability, + input_value, + 64, + input_admission=lambda _value: True, + output_admission=lambda _value: True, + ) + self.assertIs(type(admitted), transport.TwoBuildObservationV1) + self.assertEqual(len(backend.requests), 2) + + @unittest.skipUnless(hasattr(os, "fork"), "requires POSIX fork") + def test_forked_child_cannot_wait_on_or_duplicate_a_build_lease(self) -> None: + transport = importlib.import_module("build.transport") + policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 + input_value = _sealed_input() + capability = _docker_capability_fixture(policy) + backend = _ScriptedBuildBackend( + capability, + ( + _completed_process(transport, input_value, b"same executable"), + _completed_process(transport, input_value, b"same executable"), + ), + ) + controller = transport.ControlledBuildTransportV1( + policy=policy, + backend=backend, + ) + owned_capability = controller.probe() + read_fd, write_fd = os.pipe() + controller._lease_lock.acquire() + child_pid: int | None = None + child_reaped = False + try: + child_pid = os.fork() + if child_pid == 0: + os.close(read_fd) + try: + child_result = controller.build( + owned_capability, + input_value, + 64, + input_admission=lambda _value: True, + output_admission=lambda _value: True, + ) + os.write( + write_fd, + ( + f"{type(child_result).__name__}:" + f"{len(backend.requests)}" + ).encode("ascii"), + ) + finally: + os.close(write_fd) + os._exit(0) + os.close(write_fd) + ready, _write_ready, _errors = select.select([read_fd], [], [], 1) + self.assertEqual(ready, [read_fd]) + child_message = os.read(read_fd, 128).decode("ascii") + _waited_pid, status = os.waitpid(child_pid, 0) + child_reaped = True + finally: + controller._lease_lock.release() + if child_pid is not None and not child_reaped: + try: + os.kill(child_pid, 9) + except ProcessLookupError: + pass + try: + os.waitpid(child_pid, 0) + except ChildProcessError: + pass + try: + os.close(read_fd) + except OSError: + pass + + self.assertTrue(os.WIFEXITED(status)) + self.assertEqual(child_message, "BuildRejectedV1:0") + parent_result = controller.build( + owned_capability, + input_value, + 64, + input_admission=lambda _value: True, + output_admission=lambda _value: True, + ) + self.assertIs(type(parent_result), transport.TwoBuildObservationV1) + self.assertEqual(len(backend.requests), 2) + + def test_public_build_contract_violations_are_typed_before_backend(self) -> None: + build_input = importlib.import_module("build.input") + transport = importlib.import_module("build.transport") + policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 + cases = ( + ("wrong type", None, lambda _value: True), + ( + "forged sealed input", + tuple.__new__(build_input.SealedInputV1, ()), + lambda _value: True, + ), + ("lane admission", _sealed_input(), lambda _value: False), + ) + for name, hostile, admission in cases: + with self.subTest(case=name): + capability = _docker_capability_fixture(policy) + backend = _ScriptedBuildBackend(capability, ()) + controller = transport.ControlledBuildTransportV1( + policy=policy, + backend=backend, + ) + owned_capability = controller.probe() + result = controller.build( + owned_capability, + hostile, + 64, + input_admission=admission, + output_admission=lambda _value: True, + ) + self.assertIs(type(result), transport.BuildRejectedV1) + self.assertEqual( + result.reason, + transport.BuildFailureReasonV1.CONTRACT_VIOLATION, + ) + self.assertEqual(backend.requests, []) + + def test_capability_owns_injected_host_user_and_rejects_surrogate_coordinates(self) -> None: + transport = importlib.import_module("build.transport") + self.assertTrue(hasattr(transport, "DockerUserModeV1")) + user_mode = transport.DockerUserModeV1.HOST_EFFECTIVE_IDS + shipped = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 + coordinates = { + "image_reference": shipped.image_reference, + "platform": shipped.platform, + "hostname": shipped.hostname, + "bootstrap": shipped.bootstrap, + "bootstrap_argv0": shipped.bootstrap_argv0, + "tmpfs_specs": shipped.tmpfs_specs, + "user_mode": user_mode, + "stdout_limit": shipped.stdout_limit, + "stderr_limit": shipped.stderr_limit, + "build_timeout_ns": shipped.build_timeout_ns, + "probe_output_limit": shipped.probe_output_limit, + "probe_timeout_ns": shipped.probe_timeout_ns, + } + policy = transport.DockerBuildPolicyV1(**coordinates) + input_value = _sealed_input() + backends = tuple( + transport.NativeDockerBuildBackendV1( + Path("/usr/bin/true"), + policy, + host_user=(501, 20), + platform_name="linux", + machine_name="x86_64", + ) + for _ in range(2) + ) + capabilities = tuple( + _probe_native_backend(backend, policy) for backend in backends + ) + requests = tuple( + transport.DockerBuildRequestV1( + 1, + capability, + input_value, + 64, + ) + for capability in capabilities + ) + commands: list[tuple[str, ...]] = [] + + def observe( + command: tuple[str, ...], + **_kwargs: object, + ) -> object: + commands.append(command) + return _completed_process(transport, input_value, b"") + + with mock.patch.object( + transport.os, + "geteuid", + side_effect=AssertionError("command_for performed ambient uid IO"), + ), mock.patch.object( + transport.os, + "getegid", + side_effect=AssertionError("command_for performed ambient gid IO"), + ): + for backend, request in zip(backends, requests, strict=True): + with mock.patch.object( + backend, + "_observe_command", + side_effect=observe, + ): + self.assertIs( + type(backend.run_build(request)), + transport.DockerBuildExitedV1, + ) + self.assertEqual(len(commands), 2) + + def without_native_cid_path(command: tuple[str, ...]) -> tuple[str, ...]: + index = command.index("--cidfile") + return command[: index + 1] + command[index + 2 :] + + self.assertEqual( + without_native_cid_path(commands[0]), + without_native_cid_path(commands[1]), + ) + user_index = commands[0].index("--user") + self.assertEqual(commands[0][user_index + 1], "501:20") + + for field_name, value in ( + ("bootstrap", "\ud800"), + ("tmpfs_specs", ("/tmp/\ud800:rw",)), + ): + with self.subTest(field=field_name): + hostile = dict(coordinates) + hostile[field_name] = value + with self.assertRaises(TypeError): + transport.DockerBuildPolicyV1(**hostile) + with self.assertRaises(TypeError): + transport.NativeDockerBuildBackendV1( + Path("/tmp/\ud800"), + policy, + host_user=(501, 20), + platform_name="linux", + machine_name="x86_64", + ) + + def test_native_backend_defers_host_user_observation_to_supported_probe(self) -> None: + transport = importlib.import_module("build.transport") + policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 + + with mock.patch.object( + transport.os, + "geteuid", + side_effect=AttributeError("not available on this host"), + ), mock.patch.object( + transport.os, + "getegid", + side_effect=AttributeError("not available on this host"), + ): + unsupported_backend = transport.NativeDockerBuildBackendV1( + Path("/usr/bin/true"), + policy, + platform_name="windows", + machine_name="amd64", + ) + unsupported = unsupported_backend.probe() + + self.assertIs(type(unsupported), transport.DockerUnsupportedV1) + self.assertEqual( + unsupported.reason, + transport.DockerBlockerReasonV1.HOST_NOT_LINUX_AMD64, + ) + + supported_backend = transport.NativeDockerBuildBackendV1( + Path("/usr/bin/true"), + policy, + platform_name="linux", + machine_name="x86_64", + ) + with mock.patch.object( + transport.os, + "geteuid", + side_effect=AttributeError("not available on this host"), + ): + unavailable = supported_backend.probe() + + self.assertIs(type(unavailable), transport.DockerUnsupportedV1) + self.assertEqual( + unavailable.reason, + transport.DockerBlockerReasonV1.HOST_USER_UNAVAILABLE, + ) + + def test_native_probe_observes_unconfigured_host_user_each_time(self) -> None: + """Ambient uid/gid belong to a capability observation, never backend cache.""" + + transport = importlib.import_module("build.transport") + policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 + backend = transport.NativeDockerBuildBackendV1( + Path("/usr/bin/true"), + policy, + platform_name="linux", + machine_name="x86_64", + ) + + with mock.patch.object( + transport.os, + "geteuid", + side_effect=(501, 502), + ), mock.patch.object( + transport.os, + "getegid", + side_effect=(20, 21), + ): + first = _probe_native_backend(backend, policy) + second = _probe_native_backend(backend, policy) + + self.assertEqual(first.host_user, (501, 20)) + self.assertEqual(second.host_user, (502, 21)) + + def test_native_host_coordinates_are_exact_strings_and_oci_ports_are_ascii(self) -> None: + transport = importlib.import_module("build.transport") + policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 + + class StringSubclass(str): + pass + + for field_name, value in ( + ("platform_name", StringSubclass("linux")), + ("machine_name", StringSubclass("x86_64")), + ("platform_name", 7), + ("machine_name", object()), + ): + with self.subTest(field=field_name, value_type=type(value).__name__): + coordinates = { + "platform_name": "linux", + "machine_name": "x86_64", + } + coordinates[field_name] = value + with self.assertRaises(TypeError): + transport.NativeDockerBuildBackendV1( + Path("/usr/bin/docker"), + policy, + host_user=(501, 20), + **coordinates, + ) + + hostile_policy = { + "image_reference": ( + "registry.example:\u0661/toolchain@sha256:" + "a" * 64 + ), + "platform": policy.platform, + "hostname": policy.hostname, + "bootstrap": policy.bootstrap, + "bootstrap_argv0": policy.bootstrap_argv0, + "tmpfs_specs": policy.tmpfs_specs, + "user_mode": policy.user_mode, + "stdout_limit": policy.stdout_limit, + "stderr_limit": policy.stderr_limit, + "build_timeout_ns": policy.build_timeout_ns, + "probe_output_limit": policy.probe_output_limit, + "probe_timeout_ns": policy.probe_timeout_ns, + } + with self.assertRaises(TypeError): + transport.DockerBuildPolicyV1(**hostile_policy) + + def test_stream_close_failure_retries_owner_and_retains_evidence(self) -> None: + transport = importlib.import_module("build.transport") + backend = transport.NativeDockerBuildBackendV1( + Path("/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + host_user=(501, 20), + platform_name="linux", + machine_name="x86_64", + ) + real_popen = subprocess.Popen + spawned: list[subprocess.Popen[bytes]] = [] + wrapped_streams: list[object] = [] + + class CloseRaises: + def __init__(self, wrapped: object) -> None: + self.wrapped = wrapped + self.descriptor = wrapped.fileno() + self.close_calls = 0 + + @property + def closed(self) -> bool: + return self.wrapped.closed + + def fileno(self) -> int: + return self.descriptor + + def close(self) -> None: + self.close_calls += 1 + if self.close_calls == 1: + raise OSError("forced close failure") + self.wrapped.close() + + def spawn(*args: object, **kwargs: object) -> subprocess.Popen[bytes]: + process = real_popen(*args, **kwargs) + spawned.append(process) + wrapped = CloseRaises(process.stdout) + wrapped_streams.append(wrapped) + process.stdout = wrapped + return process + + input_value = _sealed_input() + command = ( + sys.executable, + "-c", + ( + "import sys; sys.stdin.buffer.read(); " + "sys.stdout.buffer.write(b'evidence')" + ), + ) + try: + with mock.patch.object( + transport.subprocess, + "Popen", + side_effect=spawn, + ): + result = backend._observe_command( + command, + stdout_limit=64, + stderr_limit=64, + timeout_ns=1_000_000_000, + input_bundle=input_value, + ) + finally: + for process in spawned: + if process.poll() is None: + process.kill() + process.wait(timeout=5) + for stream in (process.stdin, process.stderr): + if stream is not None and not stream.closed: + stream.close() + original = wrapped_streams[0].wrapped + if not original.closed: + original.close() + + self.assertIs(type(result), transport.DockerBuildObserverFailureV1) + self.assertEqual(result.stdout, b"evidence") + self.assertEqual(result.stderr, b"") + self.assertIsNotNone(result.input_progress) + self.assertEqual(result.input_progress.written_length, input_value.length) + self.assertEqual(result.input_progress.written_sha256, input_value.sha256) + self.assertTrue(wrapped_streams[0].wrapped.closed) + self.assertEqual(wrapped_streams[0].close_calls, 2) + + def test_post_popen_failures_always_close_streams_and_cleanup_once(self) -> None: + transport = importlib.import_module("build.transport") + real_popen = subprocess.Popen + + def exercise( + *, + selector_failure: bool, + ) -> tuple[object, bool, int]: + backend = transport.NativeDockerBuildBackendV1( + Path("/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + host_user=(501, 20), + platform_name="linux", + machine_name="x86_64", + ) + spawned: list[subprocess.Popen[bytes]] = [] + cleanup_calls: list[object] = [] + + def spawn(*args: object, **kwargs: object) -> subprocess.Popen[bytes]: + process = real_popen(*args, **kwargs) + spawned.append(process) + return process + + def cleanup(lease: object, **_kwargs: object) -> None: + cleanup_calls.append(lease) + return None + + def stop_raises(process: subprocess.Popen[bytes]) -> None: + process.kill() + process.wait(timeout=5) + raise RuntimeError("forced stop failure") + + selector_patch = ( + mock.patch.object( + transport.selectors, + "DefaultSelector", + side_effect=RuntimeError("forced selector failure"), + ) + if selector_failure + else mock.patch.object( + backend, + "_stop_process", + side_effect=stop_raises, + ) + ) + command = ( + sys.executable, + "-c", + "pass" if selector_failure else "import time; time.sleep(5)", + ) + lease = backend._next_run_lease_v1( + _docker_capability_fixture(pipeline.ARB_BUILD_TRANSPORT_POLICY_V1) + ) + self.addCleanup(backend._release_run_lease_v1, lease) + with mock.patch.object( + transport.subprocess, + "Popen", + side_effect=spawn, + ), mock.patch.object( + backend, + "_cleanup_container", + side_effect=cleanup, + ), selector_patch: + try: + result: object = backend._observe_command( + command, + stdout_limit=64, + stderr_limit=64, + timeout_ns=1 if not selector_failure else 1_000_000_000, + lease=lease, + ) + except Exception as error: + result = error + self.assertEqual(len(spawned), 1) + process = spawned[0] + streams_closed = bool( + process.stdout is not None + and process.stdout.closed + and process.stderr is not None + and process.stderr.closed + ) + if process.poll() is None: + process.kill() + process.wait(timeout=5) + for stream in (process.stdin, process.stdout, process.stderr): + if stream is not None and not stream.closed: + stream.close() + return result, streams_closed, len(cleanup_calls) + + selector_result, selector_closed, selector_cleanups = exercise( + selector_failure=True + ) + self.assertIs(type(selector_result), transport.DockerBuildObserverFailureV1) + self.assertTrue(selector_closed) + self.assertEqual(selector_cleanups, 1) + + stop_result, stop_closed, stop_cleanups = exercise(selector_failure=False) + self.assertIn( + type(stop_result), + ( + transport.DockerBuildObserverFailureV1, + transport.DockerBuildCleanupFailureV1, + ), + ) + self.assertTrue(stop_closed) + self.assertEqual(stop_cleanups, 1) + + def test_base_exception_during_stop_still_reaps_streams_and_container(self) -> None: + transport = importlib.import_module("build.transport") + backend = transport.NativeDockerBuildBackendV1( + Path("/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + host_user=(501, 20), + platform_name="linux", + machine_name="x86_64", + ) + real_popen = subprocess.Popen + spawned: list[subprocess.Popen[bytes]] = [] + cleanup_calls: list[object] = [] + + def spawn(*args: object, **kwargs: object) -> subprocess.Popen[bytes]: + process = real_popen(*args, **kwargs) + spawned.append(process) + return process + + def cleanup(lease: object, **_kwargs: object) -> None: + cleanup_calls.append(lease) + + lease = backend._next_run_lease_v1( + _docker_capability_fixture(pipeline.ARB_BUILD_TRANSPORT_POLICY_V1) + ) + self.addCleanup(backend._release_run_lease_v1, lease) + + with mock.patch.object( + transport.subprocess, + "Popen", + side_effect=spawn, + ), mock.patch.object( + backend, + "_stop_process", + side_effect=KeyboardInterrupt("interrupt during stop"), + ), mock.patch.object( + backend, + "_cleanup_container", + side_effect=cleanup, + ): + with self.assertRaises(KeyboardInterrupt): + backend._observe_command( + (sys.executable, "-c", "import time; time.sleep(5)"), + stdout_limit=64, + stderr_limit=64, + timeout_ns=1, + lease=lease, + ) + process = spawned[0] + running_before_test_cleanup = process.poll() is None + streams_closed = bool( + process.stdout is not None + and process.stdout.closed + and process.stderr is not None + and process.stderr.closed + ) + if running_before_test_cleanup: + process.kill() + process.wait(timeout=5) + for stream in (process.stdin, process.stdout, process.stderr): + if stream is not None and not stream.closed: + stream.close() + + self.assertFalse(running_before_test_cleanup) + self.assertTrue(streams_closed) + self.assertEqual(len(cleanup_calls), 1) + + def test_interrupt_after_spawn_still_reaps_and_attempts_cid_cleanup(self) -> None: + """A post-spawn interruption cannot bypass the native finalizer.""" + + transport = importlib.import_module("build.transport") + backend = transport.NativeDockerBuildBackendV1( + Path("/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + host_user=(501, 20), + platform_name="linux", + machine_name="x86_64", + ) + lease = backend._next_run_lease_v1( + _docker_capability_fixture(pipeline.ARB_BUILD_TRANSPORT_POLICY_V1) + ) + self.addCleanup(backend._release_run_lease_v1, lease) + real_popen = subprocess.Popen + spawned: list[subprocess.Popen[bytes]] = [] + + def spawn(*args: object, **kwargs: object) -> subprocess.Popen[bytes]: + process = real_popen(*args, **kwargs) + spawned.append(process) + return process + + with mock.patch.object( + transport.subprocess, + "Popen", + side_effect=spawn, + ), mock.patch.object( + backend, + "_mark_run_lease_launched_v1", + side_effect=KeyboardInterrupt("interrupt after spawn"), + ), mock.patch.object( + backend, + "_cleanup_container", + return_value=None, + ) as cleanup: + with self.assertRaisesRegex(KeyboardInterrupt, "after spawn"): + backend._observe_command( + (sys.executable, "-c", "import time; time.sleep(5)"), + stdout_limit=64, + stderr_limit=64, + timeout_ns=1, + lease=lease, + ) + + process = spawned[0] + self.assertIsNotNone(process.poll()) + cleanup.assert_called_once_with(lease, spawn_may_have_started=True) + + def test_interrupt_during_post_spawn_state_initialization_reaps_and_cleans(self) -> None: + """No allocation between Popen and the finalizer may leak a child.""" + + transport = importlib.import_module("build.transport") + backend = transport.NativeDockerBuildBackendV1( + Path("/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + host_user=(501, 20), + platform_name="linux", + machine_name="x86_64", + ) + lease = backend._next_run_lease_v1( + _docker_capability_fixture(pipeline.ARB_BUILD_TRANSPORT_POLICY_V1) + ) + self.addCleanup(backend._release_run_lease_v1, lease) + real_popen = subprocess.Popen + spawned: list[subprocess.Popen[bytes]] = [] + + def spawn(*args: object, **kwargs: object) -> subprocess.Popen[bytes]: + process = real_popen(*args, **kwargs) + spawned.append(process) + return process + + with mock.patch.object( + transport.subprocess, + "Popen", + side_effect=spawn, + ), mock.patch.object( + transport, + "bytearray", + side_effect=KeyboardInterrupt("interrupt during post-spawn allocation"), + create=True, + ), mock.patch.object( + backend, + "_cleanup_container", + return_value=None, + ) as cleanup: + with self.assertRaisesRegex( + KeyboardInterrupt, + "post-spawn allocation", + ): + backend._observe_command( + (sys.executable, "-c", "import time; time.sleep(5)"), + stdout_limit=64, + stderr_limit=64, + timeout_ns=1, + lease=lease, + ) + + process = spawned[0] + try: + self.assertIsNotNone(process.poll()) + cleanup.assert_called_once_with(lease, spawn_may_have_started=True) + finally: + if process.poll() is None: + process.kill() + process.wait(timeout=5) + for stream in (process.stdin, process.stdout, process.stderr): + if stream is not None and not stream.closed: + stream.close() + + def test_post_popen_handler_gap_cannot_bypass_finalizer(self) -> None: + """An interrupt at the first bytecode after Popen still owns its child.""" + + transport = importlib.import_module("build.transport") + backend = transport.NativeDockerBuildBackendV1( + Path("/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + host_user=(501, 20), + platform_name="linux", + machine_name="x86_64", + ) + lease = backend._next_run_lease_v1( + _docker_capability_fixture(pipeline.ARB_BUILD_TRANSPORT_POLICY_V1) + ) + self.addCleanup(backend._release_run_lease_v1, lease) + observe = backend._observe_command + instructions = tuple(dis.Bytecode(observe)) + process_store = next( + ( + index + for index, instruction in enumerate(instructions) + if ( + instruction.opname == "STORE_FAST" + and instruction.argval == "process" + and index > 0 + and instructions[index - 1].opname == "CALL_FUNCTION_EX" + ) + ), + None, + ) + if process_store is None: + self.fail( + "CPython bytecode no longer exposes CALL_FUNCTION_EX before " + f"STORE_FAST process (Python {sys.version})" + ) + interruption_offset = instructions[process_store + 1].offset + real_popen = subprocess.Popen + spawned: list[subprocess.Popen[bytes]] = [] + injected = False + + def spawn(*args: object, **kwargs: object) -> subprocess.Popen[bytes]: + process = real_popen(*args, **kwargs) + spawned.append(process) + return process + + def tracer(frame: object, event: str, _arg: object) -> object: + nonlocal injected + if getattr(frame, "f_code", None) is observe.__code__: + frame.f_trace_opcodes = True + if ( + not injected + and event == "opcode" + and frame.f_lasti == interruption_offset + ): + injected = True + raise KeyboardInterrupt("interrupt in post-Popen handler gap") + return tracer + + with mock.patch.object( + transport.subprocess, + "Popen", + side_effect=spawn, + ), mock.patch.object( + backend, + "_cleanup_container", + return_value=None, + ) as cleanup: + previous = sys.gettrace() + sys.settrace(tracer) + try: + with self.assertRaisesRegex(KeyboardInterrupt, "handler gap"): + observe( + (sys.executable, "-c", "import time; time.sleep(5)"), + stdout_limit=64, + stderr_limit=64, + timeout_ns=1, + lease=lease, + ) + finally: + sys.settrace(previous) + + self.assertTrue(injected) + process = spawned[0] + try: + self.assertIsNotNone(process.poll()) + cleanup.assert_called_once_with(lease, spawn_may_have_started=True) + finally: + if process.poll() is None: + process.kill() + process.wait(timeout=5) + for stream in (process.stdin, process.stdout, process.stderr): + if stream is not None and not stream.closed: + stream.close() + + def test_popen_construction_interrupt_attempts_cid_cleanup_without_a_handle(self) -> None: + """The pre-handle boundary retains the interruption and tries CID cleanup.""" + + transport = importlib.import_module("build.transport") + backend = transport.NativeDockerBuildBackendV1( + Path("/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + host_user=(501, 20), + platform_name="linux", + machine_name="x86_64", + ) + lease = backend._next_run_lease_v1( + _docker_capability_fixture(pipeline.ARB_BUILD_TRANSPORT_POLICY_V1) + ) + self.addCleanup(backend._release_run_lease_v1, lease) + + with mock.patch.object( + transport.subprocess, + "Popen", + side_effect=KeyboardInterrupt("interrupt during Popen construction"), + ), mock.patch.object( + backend, + "_cleanup_container", + return_value=None, + ) as cleanup: + with self.assertRaisesRegex(KeyboardInterrupt, "Popen construction"): + backend._observe_command( + (sys.executable, "-c", "pass"), + stdout_limit=64, + stderr_limit=64, + timeout_ns=1, + lease=lease, + ) + + cleanup.assert_called_once_with(lease, spawn_may_have_started=True) + + @unittest.skipUnless(hasattr(os, "fork"), "requires POSIX fork") + def test_forked_child_gc_cannot_delete_parent_cid_root(self) -> None: + transport = importlib.import_module("build.transport") + backend = transport.NativeDockerBuildBackendV1( + Path("/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + host_user=(501, 20), + platform_name="linux", + machine_name="x86_64", + ) + lease = backend._next_run_lease_v1( + _docker_capability_fixture(pipeline.ARB_BUILD_TRANSPORT_POLICY_V1) + ) + root = lease.cid_file.parent + try: + child = os.fork() + if child == 0: + del lease + gc.collect() + os._exit(0) + _pid, status = os.waitpid(child, 0) + self.assertEqual(os.waitstatus_to_exitcode(status), 0) + self.assertTrue(root.is_dir()) + finally: + backend._release_run_lease_v1(lease) + + def test_interrupted_cid_root_release_remains_retryable(self) -> None: + """A failed root release must not permanently consume its cleanup lease.""" + + transport = importlib.import_module("build.transport") + backend = transport.NativeDockerBuildBackendV1( + Path("/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + host_user=(501, 20), + platform_name="linux", + machine_name="x86_64", + ) + lease = backend._next_run_lease_v1( + _docker_capability_fixture(pipeline.ARB_BUILD_TRANSPORT_POLICY_V1) + ) + root = lease.cid_file.parent + real_rmtree = transport.shutil.rmtree + try: + with mock.patch.object( + transport.shutil, + "rmtree", + side_effect=KeyboardInterrupt("interrupt during CID-root release"), + ): + with self.assertRaisesRegex(KeyboardInterrupt, "CID-root release"): + backend._release_run_lease_v1(lease) + + self.assertTrue(root.is_dir()) + self.assertFalse(lease._released) + self.assertIsNone(backend._release_run_lease_v1(lease)) + self.assertFalse(root.exists()) + finally: + if root.exists(): + real_rmtree(root) + + def test_stop_interrupt_survives_container_cleanup_failure(self) -> None: + """A later cleanup error cannot replace the caller's interruption.""" + + transport = importlib.import_module("build.transport") + backend = transport.NativeDockerBuildBackendV1( + Path("/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + host_user=(501, 20), + platform_name="linux", + machine_name="x86_64", + ) + real_popen = subprocess.Popen + spawned: list[subprocess.Popen[bytes]] = [] + + def spawn(*args: object, **kwargs: object) -> subprocess.Popen[bytes]: + process = real_popen(*args, **kwargs) + spawned.append(process) + return process + + lease = backend._next_run_lease_v1( + _docker_capability_fixture(pipeline.ARB_BUILD_TRANSPORT_POLICY_V1) + ) + self.addCleanup(backend._release_run_lease_v1, lease) + + with mock.patch.object( + transport.subprocess, + "Popen", + side_effect=spawn, + ), mock.patch.object( + backend, + "_stop_process", + side_effect=KeyboardInterrupt("interrupt during stop"), + ), mock.patch.object( + backend, + "_cleanup_container", + side_effect=OSError("cleanup failed after interruption"), + ) as cleanup: + with self.assertRaisesRegex(KeyboardInterrupt, "interrupt during stop"): + backend._observe_command( + (sys.executable, "-c", "import time; time.sleep(5)"), + stdout_limit=64, + stderr_limit=64, + timeout_ns=1, + lease=lease, + ) + process = spawned[0] + running_before_test_cleanup = process.poll() is None + streams_closed = bool( + process.stdout is not None + and process.stdout.closed + and process.stderr is not None + and process.stderr.closed + ) + if running_before_test_cleanup: + process.kill() + process.wait(timeout=5) + for stream in (process.stdin, process.stdout, process.stderr): + if stream is not None and not stream.closed: + stream.close() + + self.assertFalse(running_before_test_cleanup) + self.assertTrue(streams_closed) + self.assertEqual(cleanup.call_count, 1) + + def test_stream_close_interrupt_still_closes_siblings_and_cleans_container(self) -> None: + transport = importlib.import_module("build.transport") + backend = transport.NativeDockerBuildBackendV1( + Path("/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + host_user=(501, 20), + platform_name="linux", + machine_name="x86_64", + ) + real_popen = subprocess.Popen + spawned: list[subprocess.Popen[bytes]] = [] + wrapped_stdout: list[object] = [] + cleanup_calls: list[object] = [] + + class CloseInterrupts: + def __init__(self, wrapped: object) -> None: + self.wrapped = wrapped + self.descriptor = wrapped.fileno() + self.close_calls = 0 + + @property + def closed(self) -> bool: + return self.wrapped.closed + + def fileno(self) -> int: + return self.descriptor + + def close(self) -> None: + self.close_calls += 1 + if self.close_calls == 1: + raise KeyboardInterrupt("interrupt during stdout close") + self.wrapped.close() + + def spawn(*args: object, **kwargs: object) -> subprocess.Popen[bytes]: + process = real_popen(*args, **kwargs) + spawned.append(process) + wrapper = CloseInterrupts(process.stdout) + wrapped_stdout.append(wrapper) + process.stdout = wrapper + return process + + def cleanup(lease: object, **_kwargs: object) -> None: + cleanup_calls.append(lease) + + lease = backend._next_run_lease_v1( + _docker_capability_fixture(pipeline.ARB_BUILD_TRANSPORT_POLICY_V1) + ) + self.addCleanup(backend._release_run_lease_v1, lease) + with mock.patch.object( + transport.subprocess, + "Popen", + side_effect=spawn, + ), mock.patch.object( + backend, + "_cleanup_container", + side_effect=cleanup, + ): + with self.assertRaisesRegex(KeyboardInterrupt, "stdout close"): + backend._observe_command( + (sys.executable, "-c", "pass"), + stdout_limit=64, + stderr_limit=64, + timeout_ns=1_000_000_000, + lease=lease, + ) + process = spawned[0] + stderr_closed = process.stderr is not None and process.stderr.closed + try: + os.fstat(wrapped_stdout[0].descriptor) + except OSError: + stdout_descriptor_closed = True + else: + stdout_descriptor_closed = False + if process.poll() is None: + process.kill() + process.wait(timeout=5) + for stream in (process.stdin, process.stderr): + if stream is not None and not stream.closed: + stream.close() + try: + wrapped_stdout[0].wrapped.close() + except OSError: + pass + + self.assertTrue(stderr_closed) + self.assertTrue(stdout_descriptor_closed) + self.assertEqual(wrapped_stdout[0].close_calls, 2) + self.assertEqual(cleanup_calls, [lease]) + + def test_persistent_stream_close_interrupt_keeps_release_failure_honest(self) -> None: + transport = importlib.import_module("build.transport") + backend = transport.NativeDockerBuildBackendV1( + Path("/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + host_user=(501, 20), + platform_name="linux", + machine_name="x86_64", + ) + real_popen = subprocess.Popen + spawned: list[subprocess.Popen[bytes]] = [] + wrapped_stdout: list[object] = [] + cleanup_calls: list[object] = [] + + class CloseAlwaysInterrupts: + def __init__(self, wrapped: object) -> None: + self.wrapped = wrapped + self.descriptor = wrapped.fileno() + self.close_calls = 0 + + @property + def closed(self) -> bool: + return False + + def fileno(self) -> int: + return self.descriptor + + def close(self) -> None: + self.close_calls += 1 + raise KeyboardInterrupt("persistent stdout close interrupt") + + def cleanup_spawned( + process: subprocess.Popen[bytes], + original_stdout: object, + ) -> None: + if process.poll() is None: + try: + process.kill() + except ProcessLookupError: + pass + process.wait(timeout=5) + for stream in (process.stdin, process.stderr, original_stdout): + if stream is not None and not stream.closed: + stream.close() + + def spawn(*args: object, **kwargs: object) -> subprocess.Popen[bytes]: + process = real_popen(*args, **kwargs) + # Fixture setup may fail after Popen; ownership starts with its + # original stream, before the hostile wrapper exists. + self.addCleanup(cleanup_spawned, process, process.stdout) + spawned.append(process) + wrapper = CloseAlwaysInterrupts(process.stdout) + wrapped_stdout.append(wrapper) + process.stdout = wrapper + return process + + def cleanup(lease: object, **_kwargs: object) -> None: + cleanup_calls.append(lease) + + lease = backend._next_run_lease_v1( + _docker_capability_fixture(pipeline.ARB_BUILD_TRANSPORT_POLICY_V1) + ) + self.addCleanup(backend._release_run_lease_v1, lease) + with mock.patch.object( + transport.subprocess, + "Popen", + side_effect=spawn, + ), mock.patch.object( + backend, + "_cleanup_container", + side_effect=cleanup, + ): + with self.assertRaisesRegex(KeyboardInterrupt, "persistent stdout"): + backend._observe_command( + (sys.executable, "-c", "pass"), + stdout_limit=64, + stderr_limit=64, + timeout_ns=1_000_000_000, + lease=lease, + ) + process = spawned[0] + stderr_closed = process.stderr is not None and process.stderr.closed + os.fstat(wrapped_stdout[0].descriptor) + + self.assertTrue(stderr_closed) + self.assertEqual(wrapped_stdout[0].close_calls, 2) + self.assertEqual(cleanup_calls, [lease]) + + def test_process_and_container_cleanup_failures_are_both_retained_in_order(self) -> None: + transport = importlib.import_module("build.transport") + backend = transport.NativeDockerBuildBackendV1( + Path("/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + host_user=(501, 20), + platform_name="linux", + machine_name="x86_64", + ) + real_popen = subprocess.Popen + + def stop(process: subprocess.Popen[bytes]) -> str: + process.kill() + process.wait(timeout=5) + return "process stop failed" + + lease = backend._next_run_lease_v1( + _docker_capability_fixture(pipeline.ARB_BUILD_TRANSPORT_POLICY_V1) + ) + self.addCleanup(backend._release_run_lease_v1, lease) + + with mock.patch.object( + transport.subprocess, + "Popen", + side_effect=real_popen, + ), mock.patch.object( + backend, + "_stop_process", + side_effect=stop, + ), mock.patch.object( + backend, + "_cleanup_container", + return_value="container cleanup failed", + ): + result = backend._observe_command( + (sys.executable, "-c", "import time; time.sleep(5)"), + stdout_limit=64, + stderr_limit=64, + timeout_ns=1, + lease=lease, + input_bundle=_sealed_input(), + ) + + self.assertIs(type(result), transport.DockerBuildCleanupFailureV1) + self.assertEqual( + tuple(record.resource for record in result.failures), + ( + transport.CleanupResourceV1.DOCKER_CLI_PROCESS, + transport.CleanupResourceV1.DOCKER_CONTAINER, + ), + ) + self.assertEqual( + tuple(record.detail for record in result.failures), + ("process stop failed", "container cleanup failed"), + ) + self.assertTrue( + all( + type(record) is transport.CleanupFailureRecordV1 + for record in result.failures + ) + ) + + def test_impossible_build_failures_without_input_progress_are_contract_violations(self) -> None: + transport = importlib.import_module("build.transport") + policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 + cleanup = transport.CleanupFailureRecordV1( + transport.CleanupResourceV1.DOCKER_CONTAINER, + "container cleanup failed", + ) + impossible = ( + transport.DockerBuildTimedOutV1(b"", b""), + transport.DockerBuildOutputLimitV1( + transport.DockerOutputStreamV1.STDOUT, + b"x" * 64, + b"", + ), + transport.DockerBuildCleanupFailureV1( + transport.DockerCleanupTriggerV1.TIMEOUT, + (cleanup,), + b"", + b"", + ), + ) + for observation in impossible: + with self.subTest(observation=type(observation).__name__): + result, _backend, _report, _input = _controlled_build( + transport, + policy, + (observation,), + ) + self.assertIs(type(result), transport.BuildRejectedV1) + self.assertEqual( + result.reason, + transport.BuildFailureReasonV1.CONTRACT_VIOLATION, + ) + self.assertIsNone(result.process) + + def test_controller_passes_only_semantic_build_request_to_its_backend(self) -> None: + transport = importlib.import_module("build.transport") + policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 + input_value = _sealed_input() + observations = ( + _completed_process(transport, input_value, b"first"), + _completed_process(transport, input_value, b"second"), + ) + result, backend, _capability, _input = _controlled_build( + transport, + policy, + observations, + input_value=input_value, + ) + + self.assertIs(type(result), transport.TwoBuildObservationV1) + self.assertEqual(len(backend.requests), 2) + for request in backend.requests: + self.assertEqual(len(tuple(request)), 4) + self.assertFalse(hasattr(request, "cid_file")) + self.assertFalse(hasattr(request, "container_name")) + + def test_backend_interrupt_propagates_without_controller_cleanup_authority(self) -> None: + transport = importlib.import_module("build.transport") + policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 + input_value = _sealed_input() + capability = _docker_capability_fixture(policy) + + class InterruptingBackend: + def __init__(self) -> None: + self.requests: list[object] = [] + + def probe(self) -> object: + return capability + + def run_build(self, request: object) -> object: + self.requests.append(request) + raise KeyboardInterrupt("interrupt during build observation") + + backend = InterruptingBackend() + controller = transport.ControlledBuildTransportV1( + policy=policy, + backend=backend, + ) + owned_capability = controller.probe() + with self.assertRaisesRegex(KeyboardInterrupt, "interrupt during build observation"): + controller.build( + owned_capability, + input_value, + 64, + input_admission=lambda _value: True, + output_admission=lambda _value: True, + ) + + self.assertEqual(len(backend.requests), 1) + self.assertEqual(len(tuple(backend.requests[0])), 4) + + def test_forged_backend_failures_canonicalize_to_contract_violation(self) -> None: + transport = importlib.import_module("build.transport") + policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 + for failure_type in ( + transport.DockerBuildTimedOutV1, + transport.DockerBuildOutputLimitV1, + transport.DockerBuildObserverFailureV1, + transport.DockerBuildInputRejectedV1, + transport.DockerBuildCleanupFailureV1, + ): + with self.subTest(failure=failure_type.__name__): + forged = _forged_exact_type(failure_type) + if hasattr(forged, "__dict__"): + object.__setattr__(forged, "foreign", object()) + result, _backend, _report, _input = _controlled_build( + transport, + policy, + (forged,), + ) + self.assertIs(type(result), transport.BuildRejectedV1) + self.assertEqual( + result.reason, + transport.BuildFailureReasonV1.CONTRACT_VIOLATION, + ) + self.assertIsNone(result.process) + self.assertEqual(result.completed_processes, ()) + + def test_observer_failure_evidence_is_bounded_and_progress_is_exact(self) -> None: + transport = importlib.import_module("build.transport") + policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 + input_value = _sealed_input() + foreign_input = importlib.import_module("build.input").seal_input_v1( + _digest("foreign-build-binding"), + input_value.contents, + ) + cases = ( + transport.DockerBuildObserverFailureV1( + "oversized stdout", + b"x" * 65, + b"", + _initial_progress(transport, input_value), + ), + transport.DockerBuildObserverFailureV1( + "foreign progress", + b"", + b"", + _initial_progress(transport, foreign_input), + ), + ) + for observation in cases: + with self.subTest(detail=observation.detail): + result, _backend, _report, _input = _controlled_build( + transport, + policy, + (observation,), + input_value=input_value, + ) + self.assertIs(type(result), transport.BuildRejectedV1) + self.assertEqual( + result.reason, + transport.BuildFailureReasonV1.CONTRACT_VIOLATION, + ) + self.assertIsNone(result.process) + + def test_native_run_relabel_keeps_progress_when_cid_cleanup_is_already_failed( + self, + ) -> None: + """Relabeling a canonical CID-root failure must not erase input evidence.""" + + transport = importlib.import_module("build.transport") + policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 + input_value = _sealed_input() + backend = transport.NativeDockerBuildBackendV1( + Path("/bin/true"), + policy, + host_user=(501, 20), + platform_name="linux", + machine_name="x86_64", + ) + capability = _docker_capability_fixture(policy) + backend._probed_capability = capability + request = transport.DockerBuildRequestV1(1, capability, input_value, 64) + progress = _initial_progress(transport, input_value) + raw = transport.DockerBuildCleanupFailureV1( + transport.DockerCleanupTriggerV1.TIMEOUT, + ( + transport.CleanupFailureRecordV1( + transport.CleanupResourceV1.DOCKER_CID_ROOT, + "CID-root cleanup failed while observing the build", + ), + ), + b"stdout", + b"stderr", + progress, + ) + lease = backend._next_run_lease_v1(capability) + release = backend._release_run_lease_v1 + + with mock.patch.object( + backend, + "_next_run_lease_v1", + return_value=lease, + ), mock.patch.object( + backend, + "_command_for_v1", + return_value=("docker", "build"), + ), mock.patch.object( + backend, + "_observe_command", + return_value=raw, + ), mock.patch.object( + backend, + "_release_run_lease_v1", + side_effect=release, + ): + result = backend.run_build(request) + + self.assertIs(type(result), transport.DockerBuildObserverFailureV1) + self.assertEqual(result.stdout, b"stdout") + self.assertEqual(result.stderr, b"stderr") + self.assertIsNotNone(result.input_progress) + self.assertEqual(result.input_progress.written_length, 0) + self.assertEqual( + result.input_progress.written_sha256, + hashlib.sha256(b"").digest(), + ) + + def test_noncanonical_cleanup_join_stays_bounded_and_typed(self) -> None: + """A full cleanup diagnostic must not turn the fallback into an exception.""" + + transport = importlib.import_module("build.transport") + detail = "x" * transport._DIAGNOSTIC_DETAIL_TEXT_LIMIT_V1 + result = transport.NativeDockerBuildBackendV1._with_cid_root_cleanup_failure_v1( + object(), + detail, + ) + + self.assertIs(type(result), transport.DockerBuildObserverFailureV1) + self.assertLessEqual( + len(result.detail), + transport._DIAGNOSTIC_DETAIL_TEXT_LIMIT_V1, + ) + self.assertTrue( + result.detail.startswith( + "native Docker build observation is not canonical; " + ) + ) + + def test_top_level_failure_reason_preserves_observer_outcome(self) -> None: + transport = importlib.import_module("build.transport") + policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 + expected_reasons = ( + transport.BuildFailureReasonV1.TIMEOUT, + transport.BuildFailureReasonV1.OUTPUT_LIMIT, + transport.BuildFailureReasonV1.OBSERVER_FAILURE, + transport.BuildFailureReasonV1.PROCESS_FAILED, + ) + input_value = _sealed_input() + progress = _initial_progress(transport, input_value) + observations = ( + transport.DockerBuildTimedOutV1(b"", b"", progress), + transport.DockerBuildOutputLimitV1( + transport.DockerOutputStreamV1.STDOUT, + b"x" * 64, + b"", + progress, + ), + transport.DockerBuildObserverFailureV1( + "observer failed", + b"observer stdout", + b"observer stderr", + progress, + ), + _completed_process( + transport, + input_value, + b"", + returncode=7, + ), + ) + for observation, reason in zip( + observations, + expected_reasons, + strict=True, + ): + with self.subTest(reason=reason): + result, _backend, _report, _input = _controlled_build( + transport, + policy, + (observation,), + input_value=input_value, + ) + self.assertIs(type(result), transport.BuildRejectedV1) + self.assertEqual(result.reason, reason) + + def test_second_attempt_failure_retains_first_completed_process(self) -> None: + transport = importlib.import_module("build.transport") + policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 + input_value = _sealed_input() + first = _completed_process(transport, input_value, b"first") + progress = _initial_progress(transport, input_value) + result, _backend, _report, _input = _controlled_build( + transport, + policy, + (first, transport.DockerBuildTimedOutV1(b"", b"", progress)), + input_value=input_value, + ) + self.assertIs(type(result), transport.BuildRejectedV1) + self.assertEqual(result.attempt, 2) + self.assertTrue(hasattr(result, "completed_processes")) + self.assertIs(type(result.completed_processes), tuple) + self.assertEqual(result.completed_processes, (first,)) + self.assertIs(result.completed_processes[0], first) + self.assertFalse(hasattr(result, "__dict__")) + with self.assertRaises((AttributeError, TypeError)): + object.__setattr__(result, "completed_processes", ()) + + def test_two_build_observation_derives_byte_relation_and_binds_session(self) -> None: + transport = importlib.import_module("build.transport") + self.assertTrue(hasattr(transport, "BuildByteRelationV1")) + self.assertTrue(hasattr(transport, "TwoBuildObservationV1")) + self.assertFalse(hasattr(transport, "ReproducibleBuildV1")) + self.assertFalse(hasattr(transport, "NonReproducibleBuildV1")) + policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 + for outputs, relation in ( + ((b"same", b"same"), transport.BuildByteRelationV1.IDENTICAL), + ((b"first", b"second"), transport.BuildByteRelationV1.DIFFERENT), + ): + with self.subTest(relation=relation): + input_value = _sealed_input() + processes = tuple( + _completed_process(transport, input_value, output) + for output in outputs + ) + result, _backend, capability, _input = _controlled_build( + transport, + policy, + processes, + max_output_bytes=64, + input_value=input_value, + ) + self.assertIs(type(result), transport.TwoBuildObservationV1) + self.assertEqual(result.relation, relation) + self.assertEqual(result.policy, policy) + self.assertEqual(result.capability, capability) + self.assertIs(result.input_value, input_value) + self.assertEqual(result.max_output_bytes, 64) + self.assertEqual(result.processes, processes) + self.assertEqual( + result.relation, + ( + transport.BuildByteRelationV1.IDENTICAL + if processes[0].stdout == processes[1].stdout + else transport.BuildByteRelationV1.DIFFERENT + ), + ) + + def test_build_process_encoding_is_total_and_keeps_exact_golden(self) -> None: + transport = importlib.import_module("build.transport") + result, _backend = _execute() + process = result.evidence.build.build_processes[0] + encoded = transport.build_process_bytes_v1(process) + self.assertEqual(len(encoded), 196) + self.assertEqual( + hashlib.sha256(encoded).hexdigest(), + "d0bc7878be513e2b78515f35dfe33b54b4e4f45de27dc462be69f75a9f215073", + ) + + forged = tuple.__new__(transport.DockerBuildExitedV1, ()) + with self.assertRaises(TypeError): + transport.build_process_bytes_v1(forged) + overflow = tuple.__new__( + transport.DockerBuildExitedV1, + ( + 1 << 40, + process.stdout, + process.stderr, + process.input_transfer, + ), + ) + with self.assertRaises(TypeError): + transport.build_process_bytes_v1(overflow) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/proof/region/v1/tests/test_build_identity.py b/proof/region/v1/tests/test_build_identity.py new file mode 100644 index 00000000..19032312 --- /dev/null +++ b/proof/region/v1/tests/test_build_identity.py @@ -0,0 +1,914 @@ +#!/usr/bin/env python3 +"""RED contract for orthogonal Docker BUILD capability identities.""" + +from __future__ import annotations + +import hashlib +import inspect +import json +import os +import subprocess +import sys +import unittest +from pathlib import Path +from unittest import mock + + +PROOF = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(PROOF)) + +from build import input as build_input # noqa: E402 +from build import transport # noqa: E402 + + +_POLICY_FIELDS_V1 = ( + "image_reference", + "platform", + "hostname", + "bootstrap", + "bootstrap_argv0", + "tmpfs_specs", + "user_mode", + "stdout_limit", + "stderr_limit", + "build_timeout_ns", + "probe_output_limit", + "probe_timeout_ns", +) + +def _literal(value: str) -> tuple[str, str]: + return "literal", value + + +def _slot(value: str) -> tuple[str, str]: + return "slot", value + + +_NATIVE_COMMAND_TEMPLATES_V1 = ( + ( + "version_probe", + ( + _slot("cli_path"), + _literal("version"), + _literal("--format"), + _literal("{{json .Server}}"), + ), + ), + ( + "image_inspect", + ( + _slot("cli_path"), + _literal("image"), + _literal("inspect"), + _slot("image_reference"), + ), + ), + ( + "build", + ( + _slot("cli_path"), + _literal("run"), + _literal("--rm"), + _literal("--interactive"), + _literal("--pull"), + _literal("never"), + _literal("--platform"), + _slot("platform"), + _literal("--network"), + _literal("none"), + _literal("--read-only"), + _literal("--tmpfs"), + _slot("ordered_tmpfs_specs"), + _literal("--cap-drop"), + _literal("ALL"), + _literal("--security-opt"), + _literal("no-new-privileges:true"), + _literal("--hostname"), + _slot("hostname"), + _literal("--user"), + _slot("host_user"), + _literal("--workdir"), + _literal("/"), + _literal("--cidfile"), + _slot("cid_file"), + _literal("--entrypoint"), + _literal("/usr/bin/env"), + _slot("image_reference"), + _literal("-i"), + _literal("PATH=/usr/local/bin:/usr/bin:/bin"), + _literal("LC_ALL=C"), + _literal("LANG=C"), + _literal("TZ=UTC"), + _literal("HOME=/nonexistent"), + _literal("/bin/sh"), + _literal("-c"), + _slot("bootstrap"), + _slot("bootstrap_argv0"), + _slot("input_length"), + _slot("input_sha256"), + ), + ), + ( + "cleanup_rm", + ( + _slot("cli_path"), + _literal("container"), + _literal("rm"), + _literal("--force"), + _slot("container_coordinate"), + ), + ), + ( + "cleanup_inspect", + ( + _slot("cli_path"), + _literal("container"), + _literal("inspect"), + _literal("--format"), + _literal("{{.Id}}"), + _slot("container_coordinate"), + ), + ), + ( + "cleanup_ls", + ( + _slot("cli_path"), + _literal("container"), + _literal("ls"), + _literal("--all"), + _literal("--quiet"), + _literal("--no-trunc"), + _literal("--filter"), + _slot("container_filter"), + ), + ), +) + +_NATIVE_PROCESS_ENVIRONMENT_V1 = ( + ("DOCKER_CONFIG", "/nonexistent"), + ("HOME", "/nonexistent"), + ("LANG", "C"), + ("LC_ALL", "C"), + ("PATH", "/usr/bin:/bin"), + ("TZ", "UTC"), +) +_NATIVE_PROCESS_CWD_V1 = "/" +_NATIVE_PROCESS_UMASK_V1 = 0o077 +_NATIVE_PROCESS_CLOSE_FDS_V1 = True +_NATIVE_PROCESS_RESTORE_SIGNALS_V1 = True +_NATIVE_PROCESS_START_NEW_SESSION_V1 = True +_NATIVE_PROCESS_STDIN_WITH_INPUT_V1 = "pipe" +_NATIVE_PROCESS_STDIN_WITHOUT_INPUT_V1 = "devnull" +_NATIVE_PROCESS_STDOUT_V1 = "pipe" +_NATIVE_PROCESS_STDERR_V1 = "pipe" + + +# This literal oracle intentionally does not call production encoders: changing +# a production preimage silently must turn a test failure, not rewrite its proof. +def _blob(value: bytes) -> bytes: + return len(value).to_bytes(8, "big") + value + + +def _identity(label: bytes, chunks: tuple[bytes, ...]) -> bytes: + payload = b"".join(_blob(chunk) for chunk in chunks) + return hashlib.sha256( + label + len(payload).to_bytes(8, "big") + payload + ).digest() + + +def _policy_coordinates(policy: object) -> dict[str, object]: + return {name: getattr(policy, name) for name in _POLICY_FIELDS_V1} + + +def _policy_chunks(coordinates: dict[str, object]) -> tuple[bytes, ...]: + tmpfs_specs = coordinates["tmpfs_specs"] + user_mode = coordinates["user_mode"] + if type(tmpfs_specs) is not tuple: + raise TypeError("tmpfs_specs must be an exact tuple") + return ( + coordinates["image_reference"].encode("utf-8"), + coordinates["platform"].encode("utf-8"), + coordinates["hostname"].encode("utf-8"), + coordinates["bootstrap"].encode("utf-8"), + coordinates["bootstrap_argv0"].encode("utf-8"), + len(tmpfs_specs).to_bytes(4, "big"), + *(item.encode("utf-8") for item in tmpfs_specs), + user_mode.value.encode("ascii"), + coordinates["stdout_limit"].to_bytes(8, "big"), + coordinates["stderr_limit"].to_bytes(8, "big"), + coordinates["build_timeout_ns"].to_bytes(8, "big"), + coordinates["probe_output_limit"].to_bytes(8, "big"), + coordinates["probe_timeout_ns"].to_bytes(8, "big"), + ) + + +def _expected_policy_identity(coordinates: dict[str, object]) -> bytes: + return _identity( + b"labcolors.proof-region.docker-transport-policy.v1\0", + _policy_chunks(coordinates), + ) + + +def _expected_command_contract_identity() -> bytes: + chunks: list[bytes] = [len(_NATIVE_COMMAND_TEMPLATES_V1).to_bytes(4, "big")] + for name, tokens in _NATIVE_COMMAND_TEMPLATES_V1: + chunks.extend((name.encode("ascii"), len(tokens).to_bytes(4, "big"))) + for tag, value in tokens: + chunks.extend((tag.encode("ascii"), value.encode("utf-8"))) + chunks.extend( + ( + b"native-process-context.v1", + len(_NATIVE_PROCESS_ENVIRONMENT_V1).to_bytes(4, "big"), + *( + item + for key, value in _NATIVE_PROCESS_ENVIRONMENT_V1 + for item in (key.encode("ascii"), value.encode("utf-8")) + ), + _NATIVE_PROCESS_CWD_V1.encode("ascii"), + _NATIVE_PROCESS_UMASK_V1.to_bytes(4, "big"), + bytes((_NATIVE_PROCESS_CLOSE_FDS_V1,)), + bytes((_NATIVE_PROCESS_RESTORE_SIGNALS_V1,)), + bytes((_NATIVE_PROCESS_START_NEW_SESSION_V1,)), + b"native-stdio-topology.v1", + b"stdin-with-input", + _NATIVE_PROCESS_STDIN_WITH_INPUT_V1.encode("ascii"), + b"stdin-without-input", + _NATIVE_PROCESS_STDIN_WITHOUT_INPUT_V1.encode("ascii"), + b"stdout", + _NATIVE_PROCESS_STDOUT_V1.encode("ascii"), + b"stderr", + _NATIVE_PROCESS_STDERR_V1.encode("ascii"), + ) + ) + return _identity( + b"labcolors.proof-region.native-command-contract.v1\0", + tuple(chunks), + ) + + +def _expected_command_coordinate(docker_path: Path) -> bytes: + return _identity( + b"labcolors.proof-region.native-command-coordinate.v1\0", + (_expected_command_contract_identity(), os.fsencode(docker_path)), + ) + + +def _expected_daemon_identity( + server_stdout: bytes, + image_inspect_stdout: bytes, +) -> bytes: + return _identity( + b"labcolors.proof-region.docker-daemon-observation.v1\0", + (server_stdout, image_inspect_stdout), + ) + + +def _expected_host_user_identity(host_user: tuple[int, int]) -> bytes: + return _identity( + b"labcolors.proof-region.host-user.v1\0", + ( + host_user[0].to_bytes(4, "big"), + host_user[1].to_bytes(4, "big"), + ), + ) + + +def _expected_capability_identity( + policy_identity: bytes, + daemon_identity: bytes, + command_coordinate: bytes, + host_user: tuple[int, int], +) -> bytes: + return _identity( + b"labcolors.proof-region.docker-capability.v1\0", + ( + policy_identity, + command_coordinate, + daemon_identity, + host_user[0].to_bytes(4, "big"), + host_user[1].to_bytes(4, "big"), + ), + ) + + +def _policy(**changes: object) -> object: + coordinates: dict[str, object] = { + "image_reference": ( + "registry.example/toolchain@sha256:" + "1" * 64 + ), + "platform": "linux/amd64", + "hostname": "lc-build", + "bootstrap": "set -eu\ncat", + "bootstrap_argv0": "labcolors-build-v1", + "tmpfs_specs": ( + "/work:rw,nosuid,nodev,noexec,size=1048576", + "/tmp:rw,nosuid,nodev,noexec,size=2097152", + ), + "user_mode": transport.DockerUserModeV1.HOST_EFFECTIVE_IDS, + "stdout_limit": 4096, + "stderr_limit": 2048, + "build_timeout_ns": 5_000_000_000, + "probe_output_limit": 1024, + "probe_timeout_ns": 1_000_000_000, + } + coordinates.update(changes) + return transport.DockerBuildPolicyV1(**coordinates) + + +def _daemon( + *, + server_stdout: bytes = b'{"Version":"identity-test"}\n', + image_inspect_stdout: bytes = b'[{"Id":"sha256:identity-test"}]\n', +) -> object: + return transport.DockerDaemonObservationV1( + server_stdout, + image_inspect_stdout, + ) + + +def _capability( + *, + policy: object | None = None, + daemon: object | None = None, + docker_path: Path = Path("/usr/bin/true"), + host_user: tuple[int, int] = (501, 20), +) -> object: + owned_policy = _policy() if policy is None else policy + owned_daemon = _daemon() if daemon is None else daemon + return transport.DockerSupportedV1( + owned_policy, + owned_daemon, + transport.native_command_coordinate_v1(docker_path), + host_user, + ) + + +def _sealed_input() -> object: + return build_input.seal_input_v1( + hashlib.sha256(b"build-identity-test-binding").digest(), + b"identity-test-input", + ) + + +def _assert_deeply_immutable( + case: unittest.TestCase, + value: object, +) -> None: + case.assertFalse(hasattr(value, "__dict__"), type(value).__name__) + with case.assertRaises((AttributeError, TypeError)): + value[0] = value[0] + with case.assertRaises((AttributeError, TypeError)): + object.__setattr__(value, "foreign", object()) + + +class _AlternateUserMode: + """Test-only value with the encoder surface of the closed production enum.""" + + def __init__(self, value: str) -> None: + self.value = value + + +class BuildIdentitySurfaceTests(unittest.TestCase): + def test_identity_surface_is_exact_and_has_no_legacy_report_aliases(self) -> None: + for name in ( + "DockerDaemonObservationV1", + "NativeCommandCoordinateV1", + "transport_policy_identity_v1", + "native_command_contract_identity_v1", + "native_command_coordinate_v1", + "docker_capability_identity_v1", + ): + with self.subTest(required=name): + self.assertTrue(hasattr(transport, name), name) + + self.assertEqual( + tuple(inspect.signature(transport.DockerDaemonObservationV1).parameters), + ("server_stdout", "image_inspect_stdout"), + ) + self.assertEqual( + tuple(inspect.signature(transport.DockerSupportedV1).parameters), + ( + "policy", + "daemon_observation", + "command_coordinate", + "host_user", + ), + ) + self.assertEqual( + tuple(inspect.signature(transport.DockerBuildRequestV1).parameters), + ( + "attempt", + "capability", + "input_bundle", + "max_output_bytes", + ), + ) + self.assertFalse(hasattr(transport, "docker_report_matches_policy_v1")) + self.assertNotIn( + "docker_report", + Path(transport.__file__).read_text(encoding="utf-8"), + ) + + capability = _capability() + request = transport.DockerBuildRequestV1( + 1, + capability, + _sealed_input(), + 64, + ) + for legacy in ( + "image_reference", + "platform", + "daemon_observation_sha256", + ): + with self.subTest(legacy_capability_property=legacy): + self.assertFalse(hasattr(capability, legacy)) + self.assertFalse(hasattr(request, "policy")) + self.assertFalse(hasattr(request, "cid_file")) + self.assertFalse(hasattr(request, "container_name")) + self.assertIs(request.capability, capability) + + with self.assertRaises(TypeError): + transport.DockerSupportedV1( + capability.policy.image_reference, + capability.policy.platform, + capability.daemon_observation.identity, + capability.host_user, + ) + + def test_policy_identity_binds_all_twelve_coordinates(self) -> None: + policy = _policy() + coordinates = _policy_coordinates(policy) + identity = transport.transport_policy_identity_v1(policy) + + self.assertEqual(identity, _expected_policy_identity(coordinates)) + self.assertIs(type(identity), bytes) + self.assertEqual(len(identity), 32) + + # V1 has one admitted platform and one admitted user mode. Their + # mutation cannot be represented as a valid policy, so the independent + # literal preimage and source guard prove that neither closed-domain + # coordinate silently disappears from the versioned identity. + source = inspect.getsource(transport.transport_policy_identity_v1) + for field_name in _POLICY_FIELDS_V1: + with self.subTest(source_coordinate=field_name): + self.assertIn(f".{field_name}", source) + + raw_mutations: dict[str, object] = { + "image_reference": ( + "registry.example/toolchain@sha256:" + "2" * 64 + ), + "platform": "linux/arm64", + "hostname": "lc-build-alt", + "bootstrap": "set -eu\nprintf changed", + "bootstrap_argv0": "labcolors-build-v1-alt", + "tmpfs_specs": coordinates["tmpfs_specs"] + ("/run:rw,size=4096",), + "user_mode": _AlternateUserMode("explicit_ids"), + "stdout_limit": coordinates["stdout_limit"] + 1, + "stderr_limit": coordinates["stderr_limit"] + 1, + "build_timeout_ns": coordinates["build_timeout_ns"] + 1, + "probe_output_limit": coordinates["probe_output_limit"] + 1, + "probe_timeout_ns": coordinates["probe_timeout_ns"] + 1, + } + for field_name, changed_value in raw_mutations.items(): + with self.subTest(preimage_coordinate=field_name): + changed = dict(coordinates) + changed[field_name] = changed_value + self.assertNotEqual( + _expected_policy_identity(changed), + _expected_policy_identity(coordinates), + ) + + valid_mutations = { + key: value + for key, value in raw_mutations.items() + if key not in ("platform", "user_mode") + } + for field_name, changed_value in valid_mutations.items(): + with self.subTest(runtime_coordinate=field_name): + changed_policy = _policy(**{field_name: changed_value}) + self.assertNotEqual( + transport.transport_policy_identity_v1(changed_policy), + identity, + ) + + def test_literal_oracle_rejects_non_tuple_without_asserts(self) -> None: + coordinates = _policy_coordinates(_policy()) + coordinates["tmpfs_specs"] = object() + + with self.assertRaises(TypeError): + _policy_chunks(coordinates) + + +class BuildCapabilityIdentityTests(unittest.TestCase): + def test_daemon_identity_owns_only_the_two_raw_probe_outputs(self) -> None: + server_stdout = b'{"Version":"26.1.4","Os":"linux"}\n' + image_stdout = b'[{"Os":"linux","Architecture":"amd64"}]\n' + daemon = transport.DockerDaemonObservationV1( + server_stdout, + image_stdout, + ) + + self.assertEqual(daemon.server_stdout, server_stdout) + self.assertEqual(daemon.image_inspect_stdout, image_stdout) + self.assertEqual( + daemon.identity, + _expected_daemon_identity(server_stdout, image_stdout), + ) + self.assertEqual( + transport.DockerDaemonObservationV1( + server_stdout, + image_stdout, + ).identity, + daemon.identity, + ) + self.assertNotEqual( + transport.DockerDaemonObservationV1( + server_stdout + b" ", + image_stdout, + ).identity, + daemon.identity, + ) + self.assertNotEqual( + transport.DockerDaemonObservationV1( + server_stdout, + image_stdout + b" ", + ).identity, + daemon.identity, + ) + _assert_deeply_immutable(self, daemon) + + def test_native_command_coordinate_binds_path_and_literal_template(self) -> None: + first_path = Path("/usr/bin/true") + second_path = Path("/usr/bin/false") + expected_contract = _expected_command_contract_identity() + first = transport.native_command_coordinate_v1(first_path) + second = transport.native_command_coordinate_v1(second_path) + + self.assertEqual( + transport.native_command_contract_identity_v1(), + expected_contract, + ) + self.assertEqual( + transport.native_command_contract_identity_v1(), + transport.native_command_contract_identity_v1(), + ) + self.assertIs(type(first), transport.NativeCommandCoordinateV1) + self.assertEqual(first.path, first_path) + self.assertEqual(first.path_bytes, os.fsencode(first_path)) + self.assertEqual(first.command_contract_identity, expected_contract) + self.assertEqual(first.identity, _expected_command_coordinate(first_path)) + self.assertEqual(second.path, second_path) + self.assertEqual(second.identity, _expected_command_coordinate(second_path)) + self.assertNotEqual(first.identity, second.identity) + _assert_deeply_immutable(self, first) + + def test_capability_identity_keeps_policy_daemon_path_and_user_orthogonal(self) -> None: + policy = _policy() + daemon = _daemon() + command_coordinate = transport.native_command_coordinate_v1( + Path("/usr/bin/true") + ) + host_user = (501, 20) + capability = transport.DockerSupportedV1( + policy, + daemon, + command_coordinate, + host_user, + ) + expected_policy_identity = transport.transport_policy_identity_v1(policy) + expected = _expected_capability_identity( + expected_policy_identity, + daemon.identity, + command_coordinate.identity, + host_user, + ) + + self.assertIs(capability.policy, policy) + self.assertIs(capability.daemon_observation, daemon) + self.assertIs(capability.command_coordinate, command_coordinate) + self.assertEqual(capability.host_user, host_user) + self.assertEqual(capability.policy_identity, expected_policy_identity) + self.assertEqual( + capability.daemon_observation_identity, + daemon.identity, + ) + self.assertEqual( + capability.command_coordinate_identity, + command_coordinate.identity, + ) + self.assertEqual( + capability.host_user_identity, + _expected_host_user_identity(host_user), + ) + self.assertEqual(capability.identity, expected) + self.assertEqual( + transport.docker_capability_identity_v1(capability), + expected, + ) + + variants = ( + _capability(policy=_policy(hostname="lc-build-other"), daemon=daemon), + _capability( + policy=policy, + daemon=_daemon(server_stdout=b'{"Version":"other"}\n'), + ), + _capability(policy=policy, daemon=daemon, docker_path=Path("/bin/sh")), + _capability(policy=policy, daemon=daemon, host_user=(502, 20)), + _capability(policy=policy, daemon=daemon, host_user=(501, 21)), + ) + for variant in variants: + with self.subTest(component=variant): + self.assertNotEqual(variant.identity, capability.identity) + + # Changing outer capability coordinates never contaminates the raw + # daemon-observation identity. + self.assertTrue( + all( + variant.daemon_observation_identity + == variant.daemon_observation.identity + for variant in variants + ) + ) + self.assertEqual( + variants[0].daemon_observation_identity, + daemon.identity, + ) + self.assertEqual( + variants[2].daemon_observation_identity, + daemon.identity, + ) + self.assertEqual( + variants[3].daemon_observation_identity, + daemon.identity, + ) + self.assertEqual( + variants[4].daemon_observation_identity, + daemon.identity, + ) + + for value in (policy, capability): + with self.subTest(immutable=type(value).__name__): + _assert_deeply_immutable(self, value) + + +class NativeCommandAndRequestTests(unittest.TestCase): + def test_native_process_context_is_one_identity_bound_launch_renderer(self) -> None: + expected_base = { + "stdout": subprocess.PIPE, + "stderr": subprocess.PIPE, + "cwd": _NATIVE_PROCESS_CWD_V1, + "env": dict(_NATIVE_PROCESS_ENVIRONMENT_V1), + "close_fds": _NATIVE_PROCESS_CLOSE_FDS_V1, + "restore_signals": _NATIVE_PROCESS_RESTORE_SIGNALS_V1, + "start_new_session": _NATIVE_PROCESS_START_NEW_SESSION_V1, + "umask": _NATIVE_PROCESS_UMASK_V1, + } + context = transport._NATIVE_PROCESS_CONTEXT_V1 + for receives_stdin, stdin in ( + (False, subprocess.DEVNULL), + (True, subprocess.PIPE), + ): + with self.subTest(receives_stdin=receives_stdin): + expected = {"stdin": stdin, **expected_base} + first = context.popen_kwargs_v1(receives_stdin) + second = context.popen_kwargs_v1(receives_stdin) + self.assertEqual(first, expected) + self.assertEqual(second, expected) + self.assertIsNot(first, second) + self.assertIsNot(first["env"], second["env"]) + + backend = transport.NativeDockerBuildBackendV1( + Path("/usr/bin/true"), + _policy(), + platform_name="linux", + machine_name="x86_64", + host_user=(501, 20), + ) + with mock.patch.object( + transport.subprocess, + "Popen", + side_effect=OSError("do not launch in identity test"), + ) as spawn: + for receives_stdin, stdin in ( + (False, subprocess.DEVNULL), + (True, subprocess.PIPE), + ): + with self.subTest(receives_stdin=receives_stdin): + result = backend._observe_command( + ("/usr/bin/true",), + stdout_limit=64, + stderr_limit=64, + timeout_ns=1_000_000_000, + input_bundle=_sealed_input() if receives_stdin else None, + ) + self.assertIs( + type(result), + transport.DockerBuildObserverFailureV1, + ) + expected = {"stdin": stdin, **expected_base} + self.assertEqual(spawn.call_args.kwargs, expected) + + def test_native_backend_requires_its_exact_probe_lease(self) -> None: + policy = _policy() + backend = transport.NativeDockerBuildBackendV1( + Path("/usr/bin/true"), + policy, + platform_name="linux", + machine_name="x86_64", + host_user=(501, 20), + ) + unobserved = _capability(policy=policy) + request = transport.DockerBuildRequestV1( + 1, + unobserved, + _sealed_input(), + 64, + ) + with self.assertRaises(TypeError): + backend._bound_request_capability_v1(request) + + server_stdout = b'{"Version":"identity-test"}\n' + image_stdout = json.dumps( + [ + { + "Os": "linux", + "Architecture": "amd64", + "RepoDigests": [policy.image_reference], + } + ], + separators=(",", ":"), + ).encode("ascii") + observed = ( + transport._docker_command_exited_v1(0, server_stdout, b""), + transport._docker_command_exited_v1(0, image_stdout, b""), + ) + with mock.patch.object(backend, "_observe_command", side_effect=observed): + capability = backend.probe() + self.assertIs(type(capability), transport.DockerSupportedV1) + + equal_but_foreign = transport.DockerSupportedV1(*tuple(capability)) + self.assertEqual(equal_but_foreign, capability) + self.assertIsNot(equal_but_foreign, capability) + cloned_request = transport.DockerBuildRequestV1( + 1, + equal_but_foreign, + _sealed_input(), + 64, + ) + with self.assertRaises(TypeError): + backend._bound_request_capability_v1(cloned_request) + + def test_native_adapter_expands_the_versioned_template_to_exact_argv(self) -> None: + policy = _policy() + docker_path = Path("/usr/bin/true") + backend = transport.NativeDockerBuildBackendV1( + docker_path, + policy, + platform_name="linux", + machine_name="x86_64", + host_user=(501, 20), + ) + server_stdout = b'{"Version":"identity-test"}\n' + image_stdout = json.dumps( + [ + { + "Os": "linux", + "Architecture": "amd64", + "RepoDigests": [policy.image_reference], + } + ], + separators=(",", ":"), + ).encode("ascii") + observed = ( + transport._docker_command_exited_v1(0, server_stdout, b""), + transport._docker_command_exited_v1(0, image_stdout, b""), + ) + with mock.patch.object( + backend, + "_observe_command", + side_effect=observed, + ): + capability = backend.probe() + self.assertIs(type(capability), transport.DockerSupportedV1) + + input_bundle = _sealed_input() + request = transport.DockerBuildRequestV1( + 1, + capability, + input_bundle, + 64, + ) + lease = backend._next_run_lease_v1(capability) + try: + command = backend._command_for_v1(request, lease) + expected = ( + str(docker_path), + "run", + "--rm", + "--interactive", + "--pull", + "never", + "--platform", + policy.platform, + "--network", + "none", + "--read-only", + "--tmpfs", + policy.tmpfs_specs[0], + "--tmpfs", + policy.tmpfs_specs[1], + "--cap-drop", + "ALL", + "--security-opt", + "no-new-privileges:true", + "--hostname", + policy.hostname, + "--user", + "501:20", + "--workdir", + "/", + "--cidfile", + str(lease.cid_file), + "--entrypoint", + "/usr/bin/env", + policy.image_reference, + "-i", + "PATH=/usr/local/bin:/usr/bin:/bin", + "LC_ALL=C", + "LANG=C", + "TZ=UTC", + "HOME=/nonexistent", + "/bin/sh", + "-c", + policy.bootstrap, + policy.bootstrap_argv0, + str(input_bundle.length), + input_bundle.sha256.hex(), + ) + self.assertEqual(command, expected) + self.assertEqual(command.count("--tmpfs"), len(policy.tmpfs_specs)) + self.assertLess( + command.index(policy.tmpfs_specs[0]), + command.index(policy.tmpfs_specs[1]), + ) + self.assertEqual( + transport.native_command_contract_identity_v1(), + _expected_command_contract_identity(), + ) + finally: + backend._release_run_lease_v1(lease) + + def test_foreign_capability_is_rejected_before_backend_run(self) -> None: + policy = _policy() + owned = _capability(policy=policy) + foreign = _capability(policy=policy, docker_path=Path("/bin/sh")) + + class Backend: + def __init__(self) -> None: + self.requests: list[object] = [] + + def probe(self) -> object: + return owned + + def run_build(self, request: object) -> object: + self.requests.append(request) + raise AssertionError("foreign capability reached backend") + + backend = Backend() + controller = transport.ControlledBuildTransportV1( + policy=policy, + backend=backend, + ) + self.assertIs(controller.probe(), owned) + result = controller.build( + foreign, + _sealed_input(), + 64, + input_admission=lambda _value: True, + output_admission=lambda _value: True, + ) + self.assertIs(type(result), transport.BuildRejectedV1) + self.assertEqual( + result.reason, + transport.BuildFailureReasonV1.CONTRACT_VIOLATION, + ) + self.assertEqual(backend.requests, []) + + def test_request_is_deeply_immutable_and_owns_capability_not_policy(self) -> None: + capability = _capability() + request = transport.DockerBuildRequestV1( + 1, + capability, + _sealed_input(), + 64, + ) + + self.assertIs(request.capability, capability) + self.assertFalse(hasattr(request, "policy")) + _assert_deeply_immutable(self, request) + _assert_deeply_immutable(self, request.capability) + + +if __name__ == "__main__": + unittest.main(verbosity=2)