From f6c7dbfb72ba7caf0b8873ec38f66efe236cfc9a Mon Sep 17 00:00:00 2001 From: Daniel from Labpics <63733699+lemone112@users.noreply.github.com> Date: Sat, 1 Aug 2026 06:59:52 +0300 Subject: [PATCH 01/20] =?UTF-8?q?Proof:=20=D0=B2=D1=8B=D0=BD=D0=B5=D1=81?= =?UTF-8?q?=D1=82=D0=B8=20=D0=BE=D0=B1=D1=89=D1=83=D1=8E=20BUILD-=D0=B3?= =?UTF-8?q?=D1=80=D0=B0=D0=BD=D0=B8=D1=86=D1=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- proof/region/v1/PROTOCOL.md | 51 +- proof/region/v1/arb/pipeline.py | 1876 +++------- proof/region/v1/arb/receipt.py | 103 +- proof/region/v1/arb/tests/gate.py | 2 +- .../v1/arb/tests/test_build_identity_v2.py | 389 ++ proof/region/v1/arb/tests/test_pipeline.py | 270 +- proof/region/v1/arb/tests/test_receipt.py | 90 +- proof/region/v1/arb/tests/test_transport.py | 311 +- proof/region/v1/build/__init__.py | 1 + proof/region/v1/build/input.py | 321 ++ proof/region/v1/build/transport.py | 3114 +++++++++++++++++ proof/region/v1/tests/test_build.py | 1189 +++++++ proof/region/v1/tests/test_build_identity.py | 809 +++++ 13 files changed, 6817 insertions(+), 1709 deletions(-) create mode 100644 proof/region/v1/arb/tests/test_build_identity_v2.py create mode 100644 proof/region/v1/build/__init__.py create mode 100644 proof/region/v1/build/input.py create mode 100644 proof/region/v1/build/transport.py create mode 100644 proof/region/v1/tests/test_build.py create mode 100644 proof/region/v1/tests/test_build_identity.py diff --git a/proof/region/v1/PROTOCOL.md b/proof/region/v1/PROTOCOL.md index 609982b4..12447821 100644 --- a/proof/region/v1/PROTOCOL.md +++ b/proof/region/v1/PROTOCOL.md @@ -32,7 +32,7 @@ MPFI source closure ещё не является provenance исполнения receipt, cross-path dependency overlap и diversity не представлены admitted типом; structural coordinates не восполняют это отсутствие. -## Wire и identity +## Бинарный формат и идентичность Для wire-artifact-ов из `region_proof_protocol.py` все целые беззнаковые и записаны big-endian как `u8`, `u32be` или `u64be`; `digest` — ровно 32 @@ -154,7 +154,7 @@ Arb controller связывает его с наблюдёнными BUILD/RUN границы доверия; receipt не заявляет отсутствие ambient inputs за пределами этой границы. Альтернативный JSON/TOML definition запрещён протоколом. -## Source lock и integrity observations +## Фиксация источников и наблюдения целостности `SourceReleaseLockV1` фиксирует bytes и структурный состав архива. Поле `.integrity` содержит один `SourceIntegrityPolicyV1`; это точная граница @@ -194,7 +194,7 @@ Lab Colors фиксирует exact HTTPS URL, длину и SHA-256 получ `MpfiSourceLockV1` использует те же единичные GMP/MPFR source declarations, что и Arb, однако имеет отдельную aggregate identity и отдельный typed admission. -## Diagnostic execution boundary +## Диагностическая граница исполнения `proof/region/v1/executor.py` — общий для enclosure engines leaf без импорта Arb/MPFI, formula или comparator semantics. Он же единолично кодирует @@ -233,7 +233,39 @@ observation. Право на Arb receipt получает не executor, а от `SourceBoundArbControllerV1`, который владеет всей цепью BUILD → RUN и не принимает backend, capability либо diagnostic observation от вызывающего. -## Source-bound Arb replay +## Общая граница BUILD + +`proof/region/v1/build/input.py` принимает уже нормализованные lane entries, +кодирует один канонический USTAR и владеет точными input bytes. Он не +импортирует и не перепроверяет source capability: это ответственность +потребляющего lane. `SealedInputV1` структурно неизменяем, связывает +целостность байтов с opaque caller digest и не утверждает recipe либо engine +semantics. Resource bounds передаёт lane: общий encoder не вводит собственный +fixture-specific cap. + +`proof/region/v1/build/transport.py` владеет immutable Docker policy, +одноразовым probe→build lease, bounded stdin/stdout observation, cleanup и +двумя свежими попытками. Доказательные координаты разделены по причинам: + +1. transport policy identity связывает все поля точной policy; +2. native command contract identity связывает один типизированный grammar для + probe, build и cleanup; фактический argv строится только этим grammar; +3. daemon observation identity связывает только два raw probe stdout; +4. Docker capability identity связывает policy, command contract и exact CLI + path, daemon observation и наблюдённые host uid/gid. + +`BuildSessionV1` и каждый `DockerBuildRequestV1` сохраняют ту же capability, +те же input bytes и output cap. Чужая либо не полученная текущим probe +capability отвергается до process spawn; ambient path/user повторно не +считываются. `TwoBuildObservationV1` хранит обе успешные попытки и только +классифицирует их байты как identical или different, не называя пару +универсальным доказательством воспроизводимости. При отказе сохраняется весь +уже завершённый causal prefix. Transport не знает formula, ELF, comparator или +source provenance: lane отдельно перепроверяет semantic input binding перед +каждым process и передаёт output admission. Arb объявляет собственную exact +policy; MPFI обязан объявить другую, а не заимствовать Arb semantics. + +## Воспроизведение Arb, связанное с источником `SourceBoundArbControllerV1` сначала повторно парсит source lock и job, повторно допускает exact owned archive/build-input bytes и строит из regular @@ -243,7 +275,8 @@ object дважды передаётся через bounded stdin; каждый bounded tmpfs, а executable возвращает через stdout. Semantic host bind mounts, host output path и повторное открытие результата отсутствуют. Эта граница доказывает точный controller-observed byte stream, а не непрерывность inode -между host и Docker daemon; сам daemon остаётся явно доверенным V1 input. +между host и Docker daemon. Raw daemon observation входит в capability, но сам +daemon остаётся явно доверенным input объявленной границы. Успешный replay хранится одним token-closed `ContentResolvedEvaluatorReplayV1`, который повторно выводит три причинные @@ -252,9 +285,9 @@ identity без зеркальных промежуточных dataclass: 1. source identity связывает lock, три admitted archive closures, build inputs и formula support. Job сюда не входит: одинаковый evaluator build не меняет source identity от конкретного RUN; -2. build identity связывает source identity, versioned transport/isolation - policy, trust boundary, pinned OCI toolchain, один sealed bundle object, два - exact transfer и два byte-identical executable stdout. Comparator verifier +2. build identity связывает source identity, versioned Docker capability, + pipeline policy, trust boundary, один sealed bundle object, два exact + transfer и два byte-identical executable stdout. Comparator verifier строит свежий canonical manifest из SHA-256 retained preimage bytes и сверяет все его поля и identity с build observation; это проверка retained причинных данных, а не заявление о независимом втором выводе preimages; @@ -488,7 +521,7 @@ release не содержит. Family mint manifest: единственный range `[0, 2^24)` и point count `2^24`. Совпадение только point count или reduced-domain candidate этот gate не проходят. -## Ошибки admission +## Ошибки допуска `ProtocolReasonV1` — закрытая сумма: diff --git a/proof/region/v1/arb/pipeline.py b/proof/region/v1/arb/pipeline.py index bb09eae5..299b189f 100644 --- a/proof/region/v1/arb/pipeline.py +++ b/proof/region/v1/arb/pipeline.py @@ -12,21 +12,14 @@ import hashlib import io -import json -import os -import platform -import selectors -import signal -import stat -import subprocess import tarfile -import tempfile -import time -from dataclasses import dataclass, field, fields +from dataclasses import dataclass, fields from enum import StrEnum from functools import cached_property -from pathlib import Path -from typing import NoReturn, Protocol, TypeAlias +from typing import NoReturn, TypeAlias + +from build import input as build_input +from build import transport as build_transport import executor import provenance @@ -128,18 +121,13 @@ _FLINT_RELEASE_ONLY_ID_LABEL_V1 = ( b"labcolors.proof-region.flint-project-pinned-release-only.v1\0" ) -_PIPELINE_POLICY_ID_LABEL_V1 = b"labcolors.proof-region.arb-pipeline-policy.v1\0" +_PIPELINE_POLICY_ID_LABEL_V2 = b"labcolors.proof-region.arb-pipeline-policy.v2\0" _BUILD_INPUT_BUNDLE_ID_LABEL_V1 = ( b"labcolors.proof-region.arb-build-input-bundle.v1\0" ) _BUILD_SOURCES_TOKEN = object() _COMPARATOR_TOKEN = object() _BUILD_OBSERVATION_TOKEN = object() -_BUILD_INPUT_BUNDLE_TOKEN = object() -_BUILD_INPUT_PROGRESS_TOKEN = object() -_BUILD_INPUT_TRANSFER_TOKEN = object() -_DOCKER_COMMAND_EXITED_TOKEN = object() -_DOCKER_BUILD_EXITED_TOKEN = object() def _blob(value: bytes) -> bytes: @@ -346,203 +334,75 @@ def admit_build_sources_v1( ) -@dataclass(frozen=True, init=False) -class SealedBuildInputBundleV1: - """One controller-owned immutable byte object reused by both BUILDs.""" +ARB_BUILD_TRANSPORT_POLICY_V1 = build_transport.DockerBuildPolicyV1( + OCI_IMAGE_REFERENCE_V1, + OCI_PLATFORM_V1, + "labcolors-arb-build-v1", + "labcolors-arb-build-v1-", + _BUILD_BOOTSTRAP_V1, + "labcolors-arb-build-bootstrap-v1", + (_BUILD_TMPFS_SPEC_V1, _BUILD_STATE_TMPFS_SPEC_V1), + build_transport.DockerUserModeV1.HOST_EFFECTIVE_IDS, + BUILD_STDOUT_LIMIT_V1, + BUILD_STDERR_LIMIT_V1, + BUILD_TIMEOUT_NS_V1, + DOCKER_PROBE_OUTPUT_LIMIT_V1, + DOCKER_PROBE_TIMEOUT_NS_V1, +) - source_identity: bytes - build_input_identity: bytes - sha256: bytes - length: int - identity: bytes - _contents: bytes = field(repr=False, compare=False) - def __init__( - self, - source_identity: bytes, - build_input_identity: bytes, - contents: bytes, - *, - _token: object, - ) -> None: - if _token is not _BUILD_INPUT_BUNDLE_TOKEN: - raise TypeError( - "SealedBuildInputBundleV1 is created only by the build controller" - ) - if not _valid_digest(source_identity) or not _valid_digest( - build_input_identity - ): - raise TypeError("invalid build input coordinates") - if type(contents) is not bytes or not contents: - raise TypeError("build input bundle must be owned nonempty bytes") - digest = hashlib.sha256(contents).digest() - identity = _identity( - _BUILD_INPUT_BUNDLE_ID_LABEL_V1, - ( - source_identity, - build_input_identity, - len(contents).to_bytes(8, "big"), - digest, - hashlib.sha256(_BUILD_BOOTSTRAP_V1.encode("utf-8")).digest(), - ), - ) - for name, value in ( - ("source_identity", source_identity), - ("build_input_identity", build_input_identity), - ("sha256", digest), - ("length", len(contents)), - ("identity", identity), - ("_contents", contents), - ): - object.__setattr__(self, name, value) +def _arb_input_binding_identity_v1( + source_identity: bytes, + build_input_identity: bytes, + contents: bytes, +) -> bytes: + if ( + not _valid_digest(source_identity) + or not _valid_digest(build_input_identity) + or type(contents) is not bytes + or not contents + ): + raise TypeError("invalid Arb build input binding coordinates") + digest = hashlib.sha256(contents).digest() + return _identity( + _BUILD_INPUT_BUNDLE_ID_LABEL_V1, + ( + source_identity, + build_input_identity, + len(contents).to_bytes(8, "big"), + digest, + hashlib.sha256( + ARB_BUILD_TRANSPORT_POLICY_V1.bootstrap.encode("utf-8") + ).digest(), + ), + ) -def sealed_build_input_bundle_is_well_bound_v1(value: object) -> bool: - if type(value) is not SealedBuildInputBundleV1: +def arb_input_is_bound_v1( + request: object, + value: object, +) -> bool: + """Recompute Arb semantics independently of generic byte integrity.""" + + if ( + type(request) is not PipelineRequestV1 + or type(value) is not build_input.SealedInputV1 + or not build_input.sealed_input_is_intact_v1(value) + ): return False try: - digest = hashlib.sha256(value._contents).digest() - identity = _identity( - _BUILD_INPUT_BUNDLE_ID_LABEL_V1, - ( - value.source_identity, - value.build_input_identity, - len(value._contents).to_bytes(8, "big"), - digest, - hashlib.sha256(_BUILD_BOOTSTRAP_V1.encode("utf-8")).digest(), - ), - ) - return ( - _valid_digest(value.source_identity) - and _valid_digest(value.build_input_identity) - and type(value._contents) is bytes - and bool(value._contents) - and value.length == len(value._contents) - and value.sha256 == digest - and value.identity == identity + return value.binding_identity == _arb_input_binding_identity_v1( + request.admitted_sources.identity, + request.build_sources.build_input_identity, + value.contents, ) except Exception: return False -def _canonical_tar_v1( - entries: tuple[tuple[str, int, bytes], ...], -) -> bytes: - if type(entries) is not tuple or not entries: - raise TypeError("build bundle entries must be a canonical nonempty set") - paths = tuple(path for path, _mode, _contents in entries) - if paths != tuple(sorted(paths)) or len(set(paths)) != len(entries): - raise TypeError("build bundle entries must be a canonical nonempty set") - folded_paths: set[str] = set() - directories: set[str] = set() - for path, _mode, _contents in entries: - _logical_path(path) - folded = path.lower() - if folded in folded_paths: - raise TypeError("build bundle paths must be case-distinct") - folded_paths.add(folded) - parts = path.split("/")[:-1] - for length in range(1, len(parts) + 1): - directories.add("/".join(parts[:length])) - if directories.intersection(paths): - raise TypeError("build bundle file cannot also be a directory") - output = io.BytesIO() - with tarfile.open(fileobj=output, mode="w", format=tarfile.USTAR_FORMAT) as archive: - for path in sorted(directories, key=lambda value: (value.count("/"), value)): - member = tarfile.TarInfo(path) - member.type = tarfile.DIRTYPE - member.mode = 0o755 - member.uid = 0 - member.gid = 0 - member.uname = "" - member.gname = "" - member.mtime = 0 - member.size = 0 - archive.addfile(member) - for path, mode, contents in entries: - _logical_path(path) - if ( - type(mode) is not int - or mode not in (0o644, 0o755) - or type(contents) is not bytes - ): - raise TypeError("invalid build bundle entry") - member = tarfile.TarInfo(path) - member.type = tarfile.REGTYPE - member.mode = mode - member.uid = 0 - member.gid = 0 - member.uname = "" - member.gname = "" - member.mtime = 0 - member.size = len(contents) - archive.addfile(member, io.BytesIO(contents)) - return output.getvalue() - - -def _normalized_source_entries_v1( - lock: provenance.SourceReleaseLockV1, - admitted: provenance.SafeSourceArchiveV1, -) -> tuple[tuple[str, int, bytes], ...]: - replayed, raw_tar = provenance.replay_admitted_source_archive_v1( - lock, - admitted, - ) - if ( - replayed.source_lock_identity != admitted.source_lock_identity - or replayed.archive_sha256 != admitted.archive_sha256 - or replayed.tree_identity != admitted.tree_identity - or replayed.regular_file_count != admitted.regular_file_count - or replayed.regular_file_bytes != admitted.regular_file_bytes - or replayed.files != admitted.files - ): - raise TypeError("admitted source coordinates changed before bundle sealing") - expected = {item.path: item for item in replayed.files} - values: list[tuple[str, int, bytes]] = [] - seen: set[str] = set() - with tarfile.open(fileobj=io.BytesIO(raw_tar), mode="r:") as archive: - for member in archive: - if member.isdir(): - continue - if not member.isreg() or not member.name.startswith(lock.root_prefix): - raise TypeError("admitted source replay contains a foreign member") - relative = member.name[len(lock.root_prefix) :] - coordinate = expected.get(relative) - if coordinate is None or relative in seen: - raise TypeError("admitted source replay changed its file set") - stream = archive.extractfile(member) - if stream is None: - raise TypeError("admitted source replay lost a regular file") - chunks: list[bytes] = [] - length = 0 - hasher = hashlib.sha256() - while True: - chunk = stream.read(provenance.READ_CHUNK_BYTES) - if not chunk: - break - length += len(chunk) - if length > coordinate.length: - raise TypeError("admitted source replay exceeded locked length") - chunks.append(chunk) - hasher.update(chunk) - if length != coordinate.length or hasher.digest() != coordinate.sha256: - raise TypeError("admitted source replay changed locked contents") - values.append( - ( - f"inputs/{lock.root_prefix[:-1]}/{relative}", - coordinate.mode, - b"".join(chunks), - ) - ) - seen.add(relative) - if seen != set(expected): - raise TypeError("admitted source replay is incomplete") - return tuple(sorted(values)) - - def _seal_build_input_bundle_v1( request: "PipelineRequestV1", -) -> SealedBuildInputBundleV1: +) -> build_input.SealedInputV1: if type(request) is not PipelineRequestV1: raise TypeError("request must be PipelineRequestV1") source_entries = tuple( @@ -565,12 +425,32 @@ def _seal_build_input_bundle_v1( for item in request.build_sources.files if item.path not in (FORMULA_SPEC_PATH_V1, FORMULA_GENERATOR_PATH_V1) ) - contents = _canonical_tar_v1(tuple(sorted(source_entries + workspace_entries))) - return SealedBuildInputBundleV1( - request.admitted_sources.identity, - request.build_sources.build_input_identity, + contents = build_input.canonical_ustar_v1( + tuple(sorted(source_entries + workspace_entries)), + build_input.CanonicalInputLimitsV1( + len(source_entries) + len(workspace_entries) + + sum( + path.count("/") + for path, _mode, _contents in source_entries + workspace_entries + ), + max( + MAX_BUILD_SOURCE_FILE_BYTES_V1, + *( + lock.regular_file_bytes + for lock in request.source_lock.sources + ), + ), + MAX_BUILD_SOURCE_TOTAL_BYTES_V1 + + sum(lock.regular_file_bytes for lock in request.source_lock.sources), + ), + ) + return build_input.seal_input_v1( + _arb_input_binding_identity_v1( + request.admitted_sources.identity, + request.build_sources.build_input_identity, + contents, + ), contents, - _token=_BUILD_INPUT_BUNDLE_TOKEN, ) @@ -578,29 +458,23 @@ class HostTrustBoundaryV1(StrEnum): UNSEALED_LINUX_X64_DOCKER_HOST = "unsealed-linux-x64-docker-host" -def pipeline_policy_identity_v1( +def pipeline_policy_identity_v2( host_trust: HostTrustBoundaryV1, + exact_policy: build_transport.DockerBuildPolicyV1, ) -> bytes: if type(host_trust) is not HostTrustBoundaryV1: raise TypeError("host_trust must be HostTrustBoundaryV1") + if not build_transport.docker_policy_is_valid_v1(exact_policy): + raise TypeError("exact_policy must be canonical DockerBuildPolicyV1") return _identity( - _PIPELINE_POLICY_ID_LABEL_V1, + _PIPELINE_POLICY_ID_LABEL_V2, ( - OCI_IMAGE_REFERENCE_V1.encode("ascii"), - OCI_PLATFORM_V1.encode("ascii"), + build_transport.transport_policy_identity_v1(exact_policy), + build_transport.native_command_contract_identity_v1(), host_trust.value.encode("ascii"), b"build-observation=diagnostic-unsealed-v1", - b"network=none", - b"rootfs=readonly", - b"scratch-tmpfs=" + _BUILD_TMPFS_SPEC_V1.encode("ascii"), - b"build-state-tmpfs=" + _BUILD_STATE_TMPFS_SPEC_V1.encode("ascii"), - b"cap-drop=all", - b"no-new-privileges=true", b"inputs=one-controller-sealed-normalized-tree-ustar", - b"transport=bounded-docker-stdin-v1", b"container-admission=exact-length-and-sha256-before-extraction", - b"output=bounded-docker-stdout-v1", - hashlib.sha256(_BUILD_BOOTSTRAP_V1.encode("utf-8")).digest(), b"fresh-container-count=2", ), ) @@ -622,6 +496,89 @@ def __str__(self) -> str: return f"{self.reason.value}: {self.field}" +def _normalized_source_entries_v1( + lock: provenance.SourceReleaseLockV1, + admitted: provenance.SafeSourceArchiveV1, +) -> tuple[tuple[str, int, bytes], ...]: + """Replay Arb-owned source authority into generic canonical-tree entries.""" + + def reject(field_name: str) -> NoReturn: + raise PipelineInputErrorV1( + PipelineInputReasonV1.FOREIGN_SOURCE_CAPABILITY, + field_name, + ) + + if type(lock) is not provenance.SourceReleaseLockV1: + reject("lock") + if type(admitted) is not provenance.SafeSourceArchiveV1: + reject("admitted") + try: + replayed, raw_tar = provenance.replay_admitted_source_archive_v1( + lock, + admitted, + ) + except Exception: + reject("admitted") + if ( + replayed.source_lock_identity != admitted.source_lock_identity + or replayed.archive_sha256 != admitted.archive_sha256 + or replayed.tree_identity != admitted.tree_identity + or replayed.regular_file_count != admitted.regular_file_count + or replayed.regular_file_bytes != admitted.regular_file_bytes + or replayed.files != admitted.files + ): + reject("admitted") + expected = {item.path: item for item in replayed.files} + values: list[tuple[str, int, bytes]] = [] + seen: set[str] = set() + try: + with tarfile.open(fileobj=io.BytesIO(raw_tar), mode="r:") as archive: + for member in archive: + if member.isdir(): + continue + if not member.isreg() or not member.name.startswith(lock.root_prefix): + reject("member") + relative = member.name[len(lock.root_prefix) :] + coordinate = expected.get(relative) + if coordinate is None or relative in seen: + reject("file set") + stream = archive.extractfile(member) + if stream is None: + reject("regular file") + chunks: list[bytes] = [] + length = 0 + hasher = hashlib.sha256() + while True: + chunk = stream.read(provenance.READ_CHUNK_BYTES) + if not chunk: + break + length += len(chunk) + if length > coordinate.length: + reject("file length") + chunks.append(chunk) + hasher.update(chunk) + if ( + length != coordinate.length + or hasher.digest() != coordinate.sha256 + ): + reject("file contents") + values.append( + ( + f"inputs/{lock.root_prefix[:-1]}/{relative}", + coordinate.mode, + b"".join(chunks), + ) + ) + seen.add(relative) + except PipelineInputErrorV1: + raise + except (OSError, tarfile.TarError, ValueError): + reject("archive") + if seen != set(expected): + reject("incomplete archive") + return tuple(sorted(values)) + + @dataclass(frozen=True) class FlintSourceContentPartitionV1: """Structural FLINT archive partition, not an origin assertion. @@ -762,6 +719,82 @@ def _comparator_preimage_v1(label: bytes, chunks: tuple[bytes, ...]) -> bytes: ) +def _comparator_preimage_v2(label: bytes, chunks: tuple[bytes, ...]) -> bytes: + """Encode one V2 comparator preimage without accepting a V1 label.""" + + if ( + type(label) is not bytes + or not label.startswith(b"labcolors.proof-region.arb-comparator.") + or not label.endswith(b".v2\0") + or type(chunks) is not tuple + or not chunks + or any(type(chunk) is not bytes for chunk in chunks) + ): + raise TypeError("invalid V2 comparator preimage coordinates") + return label + b"\x02" + len(chunks).to_bytes(4, "big") + b"".join( + _blob(chunk) for chunk in chunks + ) + + +def comparator_build_preimage_v2( + build_sources: AdmittedBuildSourcesV1, + docker_capability_identity: bytes, + pipeline_policy_identity: bytes, + build_processes: tuple[ + build_transport.DockerBuildExitedV1, + build_transport.DockerBuildExitedV1, + ], + binary_sha256: bytes, + rebuild_sha256s: tuple[bytes, bytes], + binary_length: int, +) -> bytes: + """Single replay schema for the BUILD coordinate in the comparator.""" + + if type(build_sources) is not AdmittedBuildSourcesV1: + raise TypeError("build_sources must be AdmittedBuildSourcesV1") + for name, value in ( + ("docker_capability_identity", docker_capability_identity), + ("pipeline_policy_identity", pipeline_policy_identity), + ("binary_sha256", binary_sha256), + ): + if not _valid_digest(value): + raise TypeError(f"invalid {name}") + if ( + type(build_processes) is not tuple + or len(build_processes) != 2 + or any( + type(item) is not build_transport.DockerBuildExitedV1 + for item in build_processes + ) + or type(rebuild_sha256s) is not tuple + or rebuild_sha256s != (binary_sha256, binary_sha256) + or type(binary_length) is not int + or binary_length <= 0 + ): + raise TypeError("invalid comparator BUILD observation") + process_bytes = tuple( + build_transport.build_process_bytes_v1(item) for item in build_processes + ) + return _comparator_preimage_v2( + b"labcolors.proof-region.arb-comparator.build-identity.v2\0", + ( + build_sources.contents(BUILD_RECIPE_PATH_V1), + build_sources.build_input_identity, + build_sources.formula_support_identity, + docker_capability_identity, + pipeline_policy_identity, + b"build-observation=diagnostic-unsealed-v1", + len(build_processes).to_bytes(4, "big"), + *process_bytes, + binary_sha256, + rebuild_sha256s[0], + rebuild_sha256s[1], + binary_length.to_bytes(8, "big"), + binary_sha256, + ), + ) + + def _encoded_build_file_set_v1( label: bytes, files_value: tuple[BuildSourceFileV1, ...], @@ -972,457 +1005,45 @@ def __post_init__(self) -> None: ) -class DockerBlockerReasonV1(StrEnum): - HOST_NOT_LINUX_AMD64 = "host_not_linux_amd64" - DOCKER_UNAVAILABLE = "docker_unavailable" - IMAGE_UNAVAILABLE = "image_unavailable" - IMAGE_IDENTITY_MISMATCH = "image_identity_mismatch" - BACKEND_CONTRACT = "backend_contract" - - -@dataclass(frozen=True) -class DockerUnsupportedV1: - reason: DockerBlockerReasonV1 - detail: str - - def __post_init__(self) -> None: - if type(self.reason) is not DockerBlockerReasonV1: - raise TypeError("invalid Docker blocker reason") - if type(self.detail) is not str or not self.detail or len(self.detail) > 4096: - raise TypeError("invalid Docker blocker detail") - - -@dataclass(frozen=True) -class DockerSupportedV1: - image_reference: str - platform: str - daemon_observation_sha256: bytes - - def __post_init__(self) -> None: - if self.image_reference != OCI_IMAGE_REFERENCE_V1: - raise TypeError("wrong OCI image reference") - if self.platform != OCI_PLATFORM_V1: - raise TypeError("wrong OCI platform") - if not _valid_digest(self.daemon_observation_sha256): - raise TypeError("invalid Docker daemon observation digest") - - -DockerCapabilityReportV1: TypeAlias = DockerSupportedV1 | DockerUnsupportedV1 - - -def _absolute_path(value: object, field_name: str) -> Path: - if not isinstance(value, Path) or not value.is_absolute(): - raise TypeError(f"{field_name} must be an absolute Path") - if any(character in str(value) for character in (",", "\n", "\r", "\0")): - raise TypeError(f"{field_name} is not Docker-mount-safe") - return value - - -_CONTAINER_NAME_PREFIX_V1 = "labcolors-arb-build-v1-" - - -def _container_name(value: object) -> str: - if ( - type(value) is not str - or not value.startswith(_CONTAINER_NAME_PREFIX_V1) - or len(value) > 128 - or any(character not in "abcdefghijklmnopqrstuvwxyz0123456789-" for character in value) - ): - raise TypeError("invalid controller-owned Docker container name") - return value - - -@dataclass(frozen=True) -class DockerBuildRequestV1: - attempt: int - input_bundle: SealedBuildInputBundleV1 - max_executable_bytes: int - cid_file: Path - container_name: str - - def __post_init__(self) -> None: - if type(self.attempt) is not int or self.attempt not in (1, 2): - raise TypeError("attempt must be 1 or 2") - if not sealed_build_input_bundle_is_well_bound_v1(self.input_bundle): - raise TypeError("input_bundle must be controller sealed and well bound") - if ( - type(self.max_executable_bytes) is not int - or self.max_executable_bytes <= 0 - or self.max_executable_bytes > BUILD_STDOUT_LIMIT_V1 - ): - raise TypeError("invalid executable output limit") - _absolute_path(self.cid_file, "cid_file") - _container_name(self.container_name) - - -def _bounded_bytes(value: object, maximum: int, field_name: str) -> bytes: - if type(value) is not bytes or len(value) > maximum: - raise TypeError(f"invalid {field_name}") - return value - - -@dataclass(frozen=True, init=False) -class BuildInputTransferProgressV1: - bundle_identity: bytes - expected_length: int - expected_sha256: bytes - written_length: int - written_sha256: bytes - - def __init__( - self, - bundle_identity: bytes, - expected_length: int, - expected_sha256: bytes, - written_length: int, - written_sha256: bytes, - *, - _token: object, - ) -> None: - if _token is not _BUILD_INPUT_PROGRESS_TOKEN: - raise TypeError("build input progress is controller-observed") - if not _valid_digest(bundle_identity) or not _valid_digest(expected_sha256): - raise TypeError("invalid build input progress coordinates") - if ( - type(expected_length) is not int - or expected_length <= 0 - or type(written_length) is not int - or written_length < 0 - or written_length > expected_length - or type(written_sha256) is not bytes - or len(written_sha256) != 32 - ): - raise TypeError("invalid build input progress") - for name, value in ( - ("bundle_identity", bundle_identity), - ("expected_length", expected_length), - ("expected_sha256", expected_sha256), - ("written_length", written_length), - ("written_sha256", written_sha256), - ): - object.__setattr__(self, name, value) - - -def _build_input_progress_v1( - bundle: SealedBuildInputBundleV1, - written_length: int, - written_sha256: bytes, -) -> BuildInputTransferProgressV1: - if not sealed_build_input_bundle_is_well_bound_v1(bundle): - raise TypeError("build input bundle is not well bound") - if ( - type(written_length) is not int - or written_length < 0 - or written_length > bundle.length - or type(written_sha256) is not bytes - or written_sha256 - != hashlib.sha256(bundle._contents[:written_length]).digest() - ): - raise TypeError("build input progress does not match the sealed bytes") - return BuildInputTransferProgressV1( - bundle.identity, - bundle.length, - bundle.sha256, - written_length, - written_sha256, - _token=_BUILD_INPUT_PROGRESS_TOKEN, - ) - - -@dataclass(frozen=True, init=False) -class BuildInputTransferV1: - bundle_identity: bytes - expected_length: int - expected_sha256: bytes - written_length: int - written_sha256: bytes - - def __init__( - self, - progress: BuildInputTransferProgressV1, - *, - _token: object, - ) -> None: - if _token is not _BUILD_INPUT_TRANSFER_TOKEN: - raise TypeError("build input transfer is controller-observed") - if ( - type(progress) is not BuildInputTransferProgressV1 - or progress.written_length != progress.expected_length - or progress.written_sha256 != progress.expected_sha256 - ): - raise TypeError("completed build input transfer must be exact") - for name in ( - "bundle_identity", - "expected_length", - "expected_sha256", - "written_length", - "written_sha256", - ): - object.__setattr__(self, name, getattr(progress, name)) - - -def _completed_build_input_transfer_v1( - bundle: SealedBuildInputBundleV1, - written_length: int, - written_sha256: bytes, -) -> BuildInputTransferV1: - progress = _build_input_progress_v1(bundle, written_length, written_sha256) - return BuildInputTransferV1( - progress, - _token=_BUILD_INPUT_TRANSFER_TOKEN, - ) - - -@dataclass(frozen=True, init=False) -class _DockerCommandExitedV1: - returncode: int - stdout: bytes - stderr: bytes - - def __init__( - self, - returncode: int, - stdout: bytes, - stderr: bytes, - *, - _token: object, - ) -> None: - if _token is not _DOCKER_COMMAND_EXITED_TOKEN: - raise TypeError("Docker command exit is controller-observed") - if type(returncode) is not int: - raise TypeError("invalid Docker returncode") - _bounded_bytes(stdout, BUILD_STDOUT_LIMIT_V1, "stdout") - _bounded_bytes(stderr, BUILD_STDERR_LIMIT_V1, "stderr") - object.__setattr__(self, "returncode", returncode) - object.__setattr__(self, "stdout", stdout) - object.__setattr__(self, "stderr", stderr) - - -def _docker_command_exited_v1( - returncode: int, - stdout: bytes, - stderr: bytes, -) -> _DockerCommandExitedV1: - return _DockerCommandExitedV1( - returncode, - stdout, - stderr, - _token=_DOCKER_COMMAND_EXITED_TOKEN, - ) - - -@dataclass(frozen=True, init=False) -class DockerBuildExitedV1: - returncode: int - stdout: bytes - stderr: bytes - input_transfer: BuildInputTransferV1 - - def __init__( - self, - returncode: int, - stdout: bytes, - stderr: bytes, - input_transfer: BuildInputTransferV1, - *, - _token: object, - ) -> None: - if _token is not _DOCKER_BUILD_EXITED_TOKEN: - raise TypeError("Docker build exit is controller-observed") - if type(returncode) is not int: - raise TypeError("invalid Docker returncode") - _bounded_bytes(stdout, BUILD_STDOUT_LIMIT_V1, "stdout") - _bounded_bytes(stderr, BUILD_STDERR_LIMIT_V1, "stderr") - if type(input_transfer) is not BuildInputTransferV1: - raise TypeError("invalid Docker build input transfer") - object.__setattr__(self, "returncode", returncode) - object.__setattr__(self, "stdout", stdout) - object.__setattr__(self, "stderr", stderr) - object.__setattr__(self, "input_transfer", input_transfer) - - -def _docker_build_exited_v1( - returncode: int, - stdout: bytes, - stderr: bytes, - input_transfer: BuildInputTransferV1, -) -> DockerBuildExitedV1: - return DockerBuildExitedV1( - returncode, - stdout, - stderr, - input_transfer, - _token=_DOCKER_BUILD_EXITED_TOKEN, - ) - - -@dataclass(frozen=True) -class DockerBuildTimedOutV1: - stdout: bytes - stderr: bytes - input_progress: BuildInputTransferProgressV1 | None = None - - def __post_init__(self) -> None: - _bounded_bytes(self.stdout, BUILD_STDOUT_LIMIT_V1, "stdout") - _bounded_bytes(self.stderr, BUILD_STDERR_LIMIT_V1, "stderr") - if self.input_progress is not None and type( - self.input_progress - ) is not BuildInputTransferProgressV1: - raise TypeError("invalid timed-out build input progress") - - -class DockerOutputStreamV1(StrEnum): - STDOUT = "stdout" - STDERR = "stderr" - - -@dataclass(frozen=True) -class DockerBuildOutputLimitV1: - stream: DockerOutputStreamV1 - stdout: bytes - stderr: bytes - input_progress: BuildInputTransferProgressV1 | None = None - - def __post_init__(self) -> None: - if type(self.stream) is not DockerOutputStreamV1: - raise TypeError("invalid Docker output stream") - _bounded_bytes(self.stdout, BUILD_STDOUT_LIMIT_V1, "stdout") - _bounded_bytes(self.stderr, BUILD_STDERR_LIMIT_V1, "stderr") - if self.input_progress is not None and type( - self.input_progress - ) is not BuildInputTransferProgressV1: - raise TypeError("invalid output-limited build input progress") - - -@dataclass(frozen=True) -class DockerBuildObserverFailureV1: - detail: str - - def __post_init__(self) -> None: - if type(self.detail) is not str or not self.detail or len(self.detail) > 4096: - raise TypeError("invalid Docker observer failure") - - -@dataclass(frozen=True) -class DockerBuildInputRejectedV1: - input_progress: BuildInputTransferProgressV1 - stdout: bytes - stderr: bytes - - def __post_init__(self) -> None: - if type(self.input_progress) is not BuildInputTransferProgressV1: - raise TypeError("invalid partial build input progress") - _bounded_bytes(self.stdout, BUILD_STDOUT_LIMIT_V1, "stdout") - _bounded_bytes(self.stderr, BUILD_STDERR_LIMIT_V1, "stderr") - - @property - def written_length(self) -> int: - return self.input_progress.written_length - - @property - def written_sha256(self) -> bytes: - return self.input_progress.written_sha256 - - -class DockerCleanupTriggerV1(StrEnum): - PROCESS_EXIT = "process_exit" - INPUT_TRANSFER = "input_transfer" - TIMEOUT = "timeout" - OUTPUT_LIMIT = "output_limit" - OBSERVER_FAILURE = "observer_failure" - - -@dataclass(frozen=True) -class DockerBuildCleanupFailureV1: - trigger: DockerCleanupTriggerV1 - detail: str - stdout: bytes - stderr: bytes - input_progress: BuildInputTransferProgressV1 | None = None - - def __post_init__(self) -> None: - if type(self.trigger) is not DockerCleanupTriggerV1: - raise TypeError("invalid Docker cleanup trigger") - if type(self.detail) is not str or not self.detail or len(self.detail) > 4096: - raise TypeError("invalid Docker cleanup failure") - _bounded_bytes(self.stdout, BUILD_STDOUT_LIMIT_V1, "stdout") - _bounded_bytes(self.stderr, BUILD_STDERR_LIMIT_V1, "stderr") - if self.input_progress is not None and type( - self.input_progress - ) is not BuildInputTransferProgressV1: - raise TypeError("invalid cleanup build input progress") - - -DockerBuildProcessObservationV1: TypeAlias = ( - DockerBuildExitedV1 - | DockerBuildTimedOutV1 - | DockerBuildOutputLimitV1 - | DockerBuildObserverFailureV1 - | DockerBuildInputRejectedV1 - | DockerBuildCleanupFailureV1 -) - -_DockerCommandObservationV1: TypeAlias = ( - _DockerCommandExitedV1 | DockerBuildProcessObservationV1 -) - - -class DockerBuildBackendV1(Protocol): - def probe(self) -> DockerCapabilityReportV1: ... - - def run_build( - self, - request: DockerBuildRequestV1, - ) -> DockerBuildProcessObservationV1: ... - - -def build_process_bytes_v1(process: DockerBuildExitedV1) -> bytes: - if ( - type(process) is not DockerBuildExitedV1 - or type(process.input_transfer) is not BuildInputTransferV1 - ): - raise TypeError("only successful typed build observations are encodable") - return b"".join( - ( - process.returncode.to_bytes(4, "big", signed=True), - len(process.stdout).to_bytes(8, "big"), - hashlib.sha256(process.stdout).digest(), - len(process.stderr).to_bytes(8, "big"), - hashlib.sha256(process.stderr).digest(), - process.input_transfer.bundle_identity, - process.input_transfer.expected_length.to_bytes(8, "big"), - process.input_transfer.expected_sha256, - process.input_transfer.written_length.to_bytes(8, "big"), - process.input_transfer.written_sha256, - ) - ) - - def _derive_arb_comparator_for_build_v1( request: PipelineRequestV1, - docker_report: DockerSupportedV1, + docker_capability: build_transport.DockerSupportedV1, binary: bytes, rebuild_sha256s: tuple[bytes, bytes], - build_processes: tuple[DockerBuildExitedV1, DockerBuildExitedV1], + build_processes: tuple[ + build_transport.DockerBuildExitedV1, + build_transport.DockerBuildExitedV1, + ], ) -> DiagnosticArbComparatorV1: """Derive all ten coordinates without accepting a caller digest/resolver.""" if type(request) is not PipelineRequestV1: raise TypeError("request must be PipelineRequestV1") - if type(docker_report) is not DockerSupportedV1: - raise TypeError("docker_report must be DockerSupportedV1") + if type(docker_capability) is not build_transport.DockerSupportedV1: + raise TypeError("docker_capability must be DockerSupportedV1") if type(binary) is not bytes or not binary: raise TypeError("binary must be exact nonempty bytes") binary_sha256 = hashlib.sha256(binary).digest() if ( type(build_processes) is not tuple or len(build_processes) != 2 - or any(type(item) is not DockerBuildExitedV1 for item in build_processes) + or any( + type(item) is not build_transport.DockerBuildExitedV1 + for item in build_processes + ) or any(item.returncode != 0 for item in build_processes) or rebuild_sha256s != (binary_sha256, binary_sha256) ): raise TypeError("comparator derivation requires two equal successful builds") - pipeline_policy_identity = pipeline_policy_identity_v1(request.host_trust) + if docker_capability.policy != ARB_BUILD_TRANSPORT_POLICY_V1: + raise TypeError("Docker capability does not bind the Arb transport policy") + docker_capability_identity = build_transport.docker_capability_identity_v1( + docker_capability + ) + pipeline_policy_identity = pipeline_policy_identity_v2( + request.host_trust, + docker_capability.policy, + ) flint_lock = request.source_lock.sources[2] flint_source = request.admitted_sources.sources[2] if type(flint_lock.integrity) is not provenance.GitContentRelationPolicyV1: @@ -1534,26 +1155,17 @@ def _derive_arb_comparator_for_build_v1( evaluator_files, ) - process_bytes = tuple(build_process_bytes_v1(item) for item in build_processes) - build_identity = _comparator_preimage_v1( - b"labcolors.proof-region.arb-comparator.build-identity.v1\0", - ( - request.build_sources.contents(BUILD_RECIPE_PATH_V1), - request.build_sources.build_input_identity, - request.build_sources.formula_support_identity, - OCI_IMAGE_REFERENCE_V1.encode("ascii"), - OCI_PLATFORM_V1.encode("ascii"), - docker_report.daemon_observation_sha256, - pipeline_policy_identity, - b"build-observation=diagnostic-unsealed-v1", - len(build_processes).to_bytes(4, "big"), - *process_bytes, - binary_sha256, - rebuild_sha256s[0], - rebuild_sha256s[1], - len(binary).to_bytes(8, "big"), - binary_sha256, - ), + process_bytes = tuple( + build_transport.build_process_bytes_v1(item) for item in build_processes + ) + build_identity = comparator_build_preimage_v2( + request.build_sources, + docker_capability_identity, + pipeline_policy_identity, + build_processes, + binary_sha256, + rebuild_sha256s, + len(binary), ) test_observation = _comparator_preimage_v1( b"labcolors.proof-region.arb-comparator.test-observation.v1\0", @@ -1654,637 +1266,12 @@ def _derive_arb_comparator_for_build_v1( ) -class NativeDockerBuildBackendV1: - """Docker adapter whose probe observes only Linux x64 and its daemon.""" - - def __init__( - self, - docker_path: Path, - *, - platform_name: str | None = None, - machine_name: str | None = None, - monotonic_ns: object = time.monotonic_ns, - ) -> None: - if not isinstance(docker_path, Path) or not docker_path.is_absolute(): - raise TypeError("docker_path must be an absolute Path") - self._docker_path = docker_path - self._platform_name = ( - platform.system().lower() if platform_name is None else platform_name - ) - self._machine_name = platform.machine() if machine_name is None else machine_name - self._monotonic_ns = monotonic_ns - - @staticmethod - def _environment() -> dict[str, str]: - return { - "HOME": "/nonexistent", - "PATH": "/usr/bin:/bin", - "DOCKER_CONFIG": "/nonexistent", - } - - def probe(self) -> DockerCapabilityReportV1: - if self._platform_name != "linux" or self._machine_name.lower() not in ( - "x86_64", - "amd64", - ): - return DockerUnsupportedV1( - DockerBlockerReasonV1.HOST_NOT_LINUX_AMD64, - "controlled build requires a Linux amd64 Docker host", - ) - try: - metadata = self._docker_path.lstat() - except OSError: - return DockerUnsupportedV1( - DockerBlockerReasonV1.DOCKER_UNAVAILABLE, - "exact Docker CLI path is unavailable", - ) - if not stat.S_ISREG(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode): - return DockerUnsupportedV1( - DockerBlockerReasonV1.DOCKER_UNAVAILABLE, - "Docker CLI must be one regular non-symlink path", - ) - commands = ( - ( - str(self._docker_path), - "version", - "--format", - "{{json .Server}}", - ), - ( - str(self._docker_path), - "image", - "inspect", - OCI_IMAGE_REFERENCE_V1, - ), - ) - outputs: list[bytes] = [] - for index, command in enumerate(commands): - result = self._observe_command( - command, - stdout_limit=DOCKER_PROBE_OUTPUT_LIMIT_V1, - stderr_limit=DOCKER_PROBE_OUTPUT_LIMIT_V1, - timeout_ns=DOCKER_PROBE_TIMEOUT_NS_V1, - cid_file=None, - ) - if ( - type(result) is not _DockerCommandExitedV1 - or result.returncode != 0 - or not result.stdout - or result.stderr - ): - return DockerUnsupportedV1( - DockerBlockerReasonV1.DOCKER_UNAVAILABLE - if index == 0 - else DockerBlockerReasonV1.IMAGE_UNAVAILABLE, - "Docker daemon probe failed" - if index == 0 - else "pinned image is not locally inspectable", - ) - outputs.append(result.stdout) - try: - inspected = json.loads(outputs[1]) - if type(inspected) is not list or len(inspected) != 1: - raise ValueError("wrong image inspection cardinality") - image = inspected[0] - if type(image) is not dict: - raise ValueError("wrong image inspection shape") - repo_digests = image.get("RepoDigests") - if ( - image.get("Os") != "linux" - or image.get("Architecture") not in ("amd64", "x86_64") - or type(repo_digests) is not list - or OCI_IMAGE_REFERENCE_V1 not in repo_digests - ): - raise ValueError("foreign image coordinate") - except (ValueError, TypeError, json.JSONDecodeError): - return DockerUnsupportedV1( - DockerBlockerReasonV1.IMAGE_IDENTITY_MISMATCH, - "local image does not match pinned linux/amd64 manifest", - ) - daemon_digest = _identity( - b"labcolors.proof-region.docker-daemon-observation.v1\0", - tuple(outputs), - ) - return DockerSupportedV1( - OCI_IMAGE_REFERENCE_V1, - OCI_PLATFORM_V1, - daemon_digest, - ) - - def command_for(self, request: DockerBuildRequestV1) -> tuple[str, ...]: - if type(request) is not DockerBuildRequestV1: - raise TypeError("request must be DockerBuildRequestV1") - command = [ - str(self._docker_path), - "run", - "--rm", - "--interactive", - "--pull", - "never", - "--platform", - OCI_PLATFORM_V1, - "--network", - "none", - "--read-only", - "--tmpfs", - _BUILD_TMPFS_SPEC_V1, - "--tmpfs", - _BUILD_STATE_TMPFS_SPEC_V1, - "--cap-drop", - "ALL", - "--security-opt", - "no-new-privileges:true", - "--name", - request.container_name, - "--hostname", - "labcolors-arb-build-v1", - "--user", - f"{os.getuid()}:{os.getgid()}", - "--workdir", - "/", - "--cidfile", - str(request.cid_file), - ] - command.extend( - ( - "--entrypoint", - "/usr/bin/env", - OCI_IMAGE_REFERENCE_V1, - "-i", - "PATH=/usr/local/bin:/usr/bin:/bin", - "LC_ALL=C", - "LANG=C", - "TZ=UTC", - "HOME=/nonexistent", - "/bin/sh", - "-c", - _BUILD_BOOTSTRAP_V1, - "labcolors-arb-build-bootstrap-v1", - str(request.input_bundle.length), - request.input_bundle.sha256.hex(), - ) - ) - return tuple(command) - - def run_build( - self, - request: DockerBuildRequestV1, - ) -> DockerBuildProcessObservationV1: - if type(request) is not DockerBuildRequestV1: - raise TypeError("request must be DockerBuildRequestV1") - return self._observe_command( - self.command_for(request), - stdout_limit=request.max_executable_bytes, - stderr_limit=BUILD_STDERR_LIMIT_V1, - timeout_ns=BUILD_TIMEOUT_NS_V1, - cid_file=request.cid_file, - container_name=request.container_name, - input_bundle=request.input_bundle, - ) - - def _observe_command( - self, - command: tuple[str, ...], - *, - stdout_limit: int, - stderr_limit: int, - timeout_ns: int, - cid_file: Path | None, - container_name: str | None = None, - input_bundle: SealedBuildInputBundleV1 | None = None, - ) -> _DockerCommandObservationV1: - if ( - type(command) is not tuple - or not command - or any(type(item) is not str or not item or "\0" in item for item in command) - ): - raise TypeError("command must be a nonempty string tuple") - if ( - type(stdout_limit) is not int - or stdout_limit <= 0 - or stdout_limit > BUILD_STDOUT_LIMIT_V1 - or type(stderr_limit) is not int - or stderr_limit <= 0 - or stderr_limit > BUILD_STDERR_LIMIT_V1 - or type(timeout_ns) is not int - or timeout_ns <= 0 - or timeout_ns > BUILD_TIMEOUT_NS_V1 - ): - raise TypeError("invalid Docker observation limits") - if (cid_file is None) != (container_name is None): - raise TypeError("Docker cleanup requires both CID file and exact name") - if cid_file is not None: - _absolute_path(cid_file, "cid_file") - _container_name(container_name) - if input_bundle is not None and type(input_bundle) is not SealedBuildInputBundleV1: - raise TypeError("input_bundle must be controller sealed") - if input_bundle is not None and not sealed_build_input_bundle_is_well_bound_v1( - input_bundle - ): - return DockerBuildObserverFailureV1("build input bundle is not well bound") - try: - process = subprocess.Popen( - command, - stdin=subprocess.PIPE if input_bundle is not None else subprocess.DEVNULL, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - cwd="/", - env=self._environment(), - close_fds=True, - start_new_session=True, - ) - except OSError: - return DockerBuildObserverFailureV1("cannot start Docker CLI") - if ( - process.stdout is None - or process.stderr is None - or (input_bundle is not None and process.stdin is None) - ): - input_progress = ( - _build_input_progress_v1( - input_bundle, - 0, - hashlib.sha256(b"").digest(), - ) - if input_bundle is not None - else None - ) - stop_detail = self._stop_process(process) - cleanup_detail = ( - self._cleanup_container(cid_file, container_name) - if cid_file is not None and container_name is not None - else None - ) - if stop_detail is not None or cleanup_detail is not None: - return DockerBuildCleanupFailureV1( - DockerCleanupTriggerV1.OBSERVER_FAILURE, - stop_detail or cleanup_detail or "Docker cleanup failed", - b"", - b"", - input_progress, - ) - return DockerBuildObserverFailureV1("Docker pipes unavailable") - - stdout = bytearray() - stderr = bytearray() - selector = selectors.DefaultSelector() - terminal: DockerOutputStreamV1 | None = None - timed_out = False - observer_failed = False - input_failed = False - written = 0 - input_hasher = hashlib.sha256() - bundle_view = ( - memoryview(input_bundle._contents) if input_bundle is not None else None - ) - try: - streams = ( - ( - process.stdout.fileno(), - DockerOutputStreamV1.STDOUT, - stdout, - stdout_limit, - ), - ( - process.stderr.fileno(), - DockerOutputStreamV1.STDERR, - stderr, - stderr_limit, - ), - ) - for descriptor, stream, target, maximum in streams: - os.set_blocking(descriptor, False) - selector.register( - descriptor, - selectors.EVENT_READ, - ("read", stream, target, maximum), - ) - if process.stdin is not None: - input_descriptor = process.stdin.fileno() - os.set_blocking(input_descriptor, False) - selector.register( - input_descriptor, - selectors.EVENT_WRITE, - ("write",), - ) - start = self._clock() - deadline = start + timeout_ns - while selector.get_map() or process.poll() is None: - now = self._clock() - if now >= deadline: - timed_out = True - break - timeout = min((deadline - now) / 1_000_000_000, 0.1) - for key, _events in selector.select(timeout): - if key.data[0] == "read": - _kind, stream, target, maximum = key.data - try: - chunk = os.read( - key.fd, - min(64 * 1024, maximum + 1 - len(target)), - ) - except BlockingIOError: - continue - if not chunk: - selector.unregister(key.fd) - continue - target.extend(chunk) - if len(target) > maximum: - del target[maximum:] - terminal = stream - break - continue - if input_bundle is None or bundle_view is None: - observer_failed = True - break - try: - count = os.write( - key.fd, - bundle_view[written : written + 64 * 1024], - ) - except BlockingIOError: - continue - except BrokenPipeError: - input_failed = True - break - if count <= 0: - input_failed = True - break - input_hasher.update(bundle_view[written : written + count]) - written += count - if written == input_bundle.length: - selector.unregister(key.fd) - if process.stdin is not None: - process.stdin.close() - if terminal is not None or input_failed or observer_failed: - break - except Exception: - observer_failed = True - finally: - try: - try: - selector.close() - except OSError: - observer_failed = True - finally: - try: - if process.stdin is not None and not process.stdin.closed: - try: - process.stdin.close() - except OSError: - observer_failed = True - finally: - if bundle_view is not None: - bundle_view.release() - - input_progress: BuildInputTransferProgressV1 | None = None - if input_bundle is not None: - try: - input_progress = _build_input_progress_v1( - input_bundle, - written, - input_hasher.digest(), - ) - except Exception: - observer_failed = True - - stop_detail: str | None = None - if ( - timed_out - or terminal is not None - or observer_failed - or input_failed - ): - stop_detail = self._stop_process(process) - elif process.poll() is None: - timed_out = True - stop_detail = self._stop_process(process) - process.stdout.close() - process.stderr.close() - cleanup_detail = ( - self._cleanup_container(cid_file, container_name) - if cid_file is not None and container_name is not None - else None - ) - if stop_detail is not None or cleanup_detail is not None: - trigger = DockerCleanupTriggerV1.PROCESS_EXIT - if observer_failed: - trigger = DockerCleanupTriggerV1.OBSERVER_FAILURE - elif terminal is not None: - trigger = DockerCleanupTriggerV1.OUTPUT_LIMIT - elif timed_out: - trigger = DockerCleanupTriggerV1.TIMEOUT - elif input_failed: - trigger = DockerCleanupTriggerV1.INPUT_TRANSFER - return DockerBuildCleanupFailureV1( - trigger, - stop_detail or cleanup_detail or "Docker cleanup failed", - bytes(stdout), - bytes(stderr), - input_progress, - ) - if input_failed: - if input_progress is None: - return DockerBuildObserverFailureV1( - "build input progress could not be retained" - ) - return DockerBuildInputRejectedV1( - input_progress, - bytes(stdout), - bytes(stderr), - ) - if observer_failed: - return DockerBuildObserverFailureV1("Docker output observation failed") - if terminal is not None: - return DockerBuildOutputLimitV1( - terminal, - bytes(stdout), - bytes(stderr), - input_progress, - ) - if timed_out: - return DockerBuildTimedOutV1( - bytes(stdout), - bytes(stderr), - input_progress, - ) - if type(process.returncode) is not int: - return DockerBuildObserverFailureV1("Docker returncode unavailable") - if input_bundle is not None: - if ( - input_progress is None - or written != input_bundle.length - or input_hasher.digest() != input_bundle.sha256 - ): - return DockerBuildObserverFailureV1( - "completed build input transfer invariant failed" - ) - input_transfer = _completed_build_input_transfer_v1( - input_bundle, - written, - input_hasher.digest(), - ) - return _docker_build_exited_v1( - process.returncode, - bytes(stdout), - bytes(stderr), - input_transfer, - ) - return _docker_command_exited_v1( - process.returncode, - bytes(stdout), - bytes(stderr), - ) - - def _clock(self) -> int: - value = self._monotonic_ns() - if type(value) is not int or value < 0: - raise RuntimeError("invalid monotonic clock") - return value - - def _stop_process( - self, - process: subprocess.Popen[bytes], - ) -> str | None: - failed = False - try: - os.killpg(process.pid, signal.SIGKILL) - except ProcessLookupError: - pass - except OSError: - try: - process.kill() - except ProcessLookupError: - pass - except OSError: - failed = True - try: - process.wait(timeout=30) - except subprocess.TimeoutExpired: - failed = True - if process.poll() is None: - failed = True - return "Docker CLI process could not be terminated" if failed else None - - @staticmethod - def _admitted_container_id(cid_file: Path) -> str | None: - try: - descriptor = os.open( - cid_file, - os.O_RDONLY - | getattr(os, "O_CLOEXEC", 0) - | getattr(os, "O_NOFOLLOW", 0), - ) - except OSError: - return None - try: - metadata = os.fstat(descriptor) - if ( - not stat.S_ISREG(metadata.st_mode) - or metadata.st_nlink != 1 - or metadata.st_size not in (64, 65) - ): - return None - raw = os.read(descriptor, 66) - except OSError: - return None - finally: - os.close(descriptor) - if len(raw) == 65 and raw.endswith(b"\n"): - raw = raw[:-1] - if len(raw) != 64 or any( - byte not in b"0123456789abcdef" for byte in raw - ): - return None - return raw.decode("ascii") - - def _observe_cleanup_command( - self, - command: tuple[str, ...], - ) -> _DockerCommandObservationV1: - return self._observe_command( - command, - stdout_limit=DOCKER_PROBE_OUTPUT_LIMIT_V1, - stderr_limit=DOCKER_PROBE_OUTPUT_LIMIT_V1, - timeout_ns=DOCKER_PROBE_TIMEOUT_NS_V1, - cid_file=None, - ) - - def _cleanup_container(self, cid_file: Path, container_name: str) -> str | None: - _absolute_path(cid_file, "cid_file") - _container_name(container_name) - container_id = self._admitted_container_id(cid_file) - removal_coordinates = ( - (container_id, container_name) - if container_id is not None - else (container_name,) - ) - try: - for coordinate in removal_coordinates: - self._observe_cleanup_command( - ( - str(self._docker_path), - "container", - "rm", - "--force", - coordinate, - ) - ) - filters = [f"name=^/{container_name}$"] - if container_id is not None: - filters.append(f"id={container_id}") - for filter_value in filters: - observation = self._observe_cleanup_command( - ( - str(self._docker_path), - "container", - "ls", - "--all", - "--quiet", - "--no-trunc", - "--filter", - filter_value, - ) - ) - if ( - type(observation) is not _DockerCommandExitedV1 - or observation.returncode != 0 - or observation.stdout - or observation.stderr - ): - return "Docker container absence could not be verified" - except Exception: - return "Docker container cleanup observer raised" - return None - - -class BuildFailureReasonV1(StrEnum): - BACKEND_CONTRACT = "backend_contract" - PROCESS_FAILED = "process_failed" - CLEANUP_FAILED = "cleanup_failed" - INPUT_TRANSFER_FAILED = "input_transfer_failed" - INVALID_OUTPUT = "invalid_output" - - @dataclass(frozen=True) class PipelineBlockedV1: - reason: DockerBlockerReasonV1 + reason: build_transport.DockerBlockerReasonV1 detail: str -@dataclass(frozen=True) -class BuildRejectedV1: - attempt: int - reason: BuildFailureReasonV1 - process: DockerBuildProcessObservationV1 | None = None - - -@dataclass(frozen=True) -class NonReproducibleBuildV1: - first_sha256: bytes - second_sha256: bytes - - class ExecutionFailureReasonV1(StrEnum): UNSUPPORTED = "unsupported" PROCESS_FAILED = "process_failed" @@ -2322,20 +1309,21 @@ class DiagnosticBuildObservationV1: build_input_identity: bytes formula_support_identity: bytes pipeline_policy_identity: bytes - docker_daemon_observation_sha256: bytes - oci_image_reference: str - oci_platform: str + docker_capability: build_transport.DockerSupportedV1 binary_sha256: bytes rebuild_sha256s: tuple[bytes, bytes] host_trust: HostTrustBoundaryV1 input_bundle_identity: bytes input_bundle_sha256: bytes input_bundle_length: int - build_processes: tuple[DockerBuildExitedV1, DockerBuildExitedV1] + build_processes: tuple[ + build_transport.DockerBuildExitedV1, + build_transport.DockerBuildExitedV1, + ] comparator: DiagnosticArbComparatorV1 _binary: bytes _rebuild_binaries: tuple[bytes, bytes] - _input_bundle: SealedBuildInputBundleV1 + _input_bundle: build_input.SealedInputV1 def __init__( self, @@ -2347,19 +1335,20 @@ def __init__( build_input_identity: bytes, formula_support_identity: bytes, pipeline_policy_identity: bytes, - docker_daemon_observation_sha256: bytes, - oci_image_reference: str, - oci_platform: str, + docker_capability: build_transport.DockerSupportedV1, binary_sha256: bytes, rebuild_sha256s: tuple[bytes, bytes], host_trust: HostTrustBoundaryV1, input_bundle_identity: bytes, input_bundle_sha256: bytes, input_bundle_length: int, - build_processes: tuple[DockerBuildExitedV1, DockerBuildExitedV1], + build_processes: tuple[ + build_transport.DockerBuildExitedV1, + build_transport.DockerBuildExitedV1, + ], comparator: DiagnosticArbComparatorV1, rebuild_binaries: tuple[bytes, bytes], - input_bundle: SealedBuildInputBundleV1, + input_bundle: build_input.SealedInputV1, *, _token: object, ) -> None: @@ -2375,7 +1364,6 @@ def __init__( ("build_input_identity", build_input_identity), ("formula_support_identity", formula_support_identity), ("pipeline_policy_identity", pipeline_policy_identity), - ("docker_daemon_observation_sha256", docker_daemon_observation_sha256), ("binary_sha256", binary_sha256), ("input_bundle_identity", input_bundle_identity), ("input_bundle_sha256", input_bundle_sha256), @@ -2389,34 +1377,52 @@ def __init__( or flint_project_pinned_release_only_file_count <= 0 ): raise TypeError("FLINT source partition must be nonempty") - if oci_image_reference != OCI_IMAGE_REFERENCE_V1 or oci_platform != OCI_PLATFORM_V1: - raise TypeError("diagnostic build does not bind the pinned OCI manifest/platform") + if type(docker_capability) is not build_transport.DockerSupportedV1: + raise TypeError("diagnostic build requires DockerSupportedV1") + canonical_capability = build_transport.DockerSupportedV1( + *tuple(docker_capability) + ) + if ( + tuple(canonical_capability) != tuple(docker_capability) + or canonical_capability.policy != ARB_BUILD_TRANSPORT_POLICY_V1 + ): + raise TypeError("diagnostic build does not bind the exact Arb capability") if ( type(rebuild_sha256s) is not tuple or len(rebuild_sha256s) != 2 or any(not _valid_digest(item) for item in rebuild_sha256s) or rebuild_sha256s != (binary_sha256, binary_sha256) ): - raise TypeError("invalid reproducible-build digests") + raise TypeError("invalid observed two-build digests") if type(host_trust) is not HostTrustBoundaryV1: raise TypeError("invalid host trust boundary") if type(input_bundle_length) is not int or input_bundle_length <= 0: raise TypeError("invalid build input bundle length") if ( - not sealed_build_input_bundle_is_well_bound_v1(input_bundle) - or input_bundle.identity != input_bundle_identity + not build_input.sealed_input_is_intact_v1(input_bundle) + or input_bundle.binding_identity != input_bundle_identity or input_bundle.sha256 != input_bundle_sha256 or input_bundle.length != input_bundle_length - or input_bundle.build_input_identity != build_input_identity - or input_bundle.source_identity != structural_source_identity + or input_bundle.binding_identity + != _arb_input_binding_identity_v1( + structural_source_identity, + build_input_identity, + input_bundle.contents, + ) ): raise TypeError("diagnostic build lost its sealed input bundle") - if pipeline_policy_identity != pipeline_policy_identity_v1(host_trust): + if pipeline_policy_identity != pipeline_policy_identity_v2( + host_trust, + canonical_capability.policy, + ): raise TypeError("pipeline policy is not the fixed diagnostic policy") if ( type(build_processes) is not tuple or len(build_processes) != 2 - or any(type(item) is not DockerBuildExitedV1 for item in build_processes) + or any( + type(item) is not build_transport.DockerBuildExitedV1 + for item in build_processes + ) or any(item.returncode != 0 for item in build_processes) ): raise TypeError("invalid build process observations") @@ -2464,12 +1470,7 @@ def __init__( ("build_input_identity", build_input_identity), ("formula_support_identity", formula_support_identity), ("pipeline_policy_identity", pipeline_policy_identity), - ( - "docker_daemon_observation_sha256", - docker_daemon_observation_sha256, - ), - ("oci_image_reference", oci_image_reference), - ("oci_platform", oci_platform), + ("docker_capability", docker_capability), ("binary_sha256", binary_sha256), ("rebuild_sha256s", rebuild_sha256s), ("host_trust", host_trust), @@ -2493,23 +1494,38 @@ def rebuild_binaries(self) -> tuple[bytes, bytes]: return self._rebuild_binaries @property - def input_transfers(self) -> tuple[BuildInputTransferV1, BuildInputTransferV1]: + def input_transfers( + self, + ) -> tuple[ + build_transport.BuildInputTransferV1, + build_transport.BuildInputTransferV1, + ]: first = self.build_processes[0].input_transfer second = self.build_processes[1].input_transfer - if type(first) is not BuildInputTransferV1 or type(second) is not BuildInputTransferV1: + if ( + type(first) is not build_transport.BuildInputTransferV1 + or type(second) is not build_transport.BuildInputTransferV1 + ): raise RuntimeError("sealed build observation lost its input transfer") return first, second @property - def input_bundle(self) -> SealedBuildInputBundleV1: + def input_bundle(self) -> build_input.SealedInputV1: + if ( + not build_input.sealed_input_is_intact_v1(self._input_bundle) + or self._input_bundle.binding_identity != self.input_bundle_identity + or self._input_bundle.sha256 != self.input_bundle_sha256 + or self._input_bundle.length != self.input_bundle_length + ): + raise RuntimeError("diagnostic build lost its exact input bytes") return self._input_bundle BuildResultV1: TypeAlias = ( DiagnosticBuildObservationV1 | PipelineBlockedV1 - | BuildRejectedV1 - | NonReproducibleBuildV1 + | build_transport.BuildRejectedV1 + | build_transport.TwoBuildObservationV1 ) @@ -2517,30 +1533,35 @@ class ControlledPipelineV1: def __init__( self, *, - build_backend: DockerBuildBackendV1, + build_backend: build_transport.DockerBuildBackendV1, ) -> None: - self._build_backend = build_backend + self._transport = build_transport.ControlledBuildTransportV1( + policy=ARB_BUILD_TRANSPORT_POLICY_V1, + backend=build_backend, + ) + + @staticmethod + def _admit_arb_output_v1(binary: bytes) -> bool: + try: + executor.require_static_x86_64_elf_v1(binary) + except executor.ExecutionRequestErrorV1: + return False + return True def build(self, request: PipelineRequestV1) -> BuildResultV1: """Observe two fresh equal builds without requiring a RUN capability.""" if type(request) is not PipelineRequestV1: raise PipelineInputErrorV1(PipelineInputReasonV1.WRONG_TYPE, "request") - try: - docker_report = self._build_backend.probe() - except Exception: - return PipelineBlockedV1( - DockerBlockerReasonV1.BACKEND_CONTRACT, - "Docker capability probe raised", - ) - if type(docker_report) is DockerUnsupportedV1: - return PipelineBlockedV1(docker_report.reason, docker_report.detail) - if type(docker_report) is not DockerSupportedV1: + probe_result = self._transport.probe() + if type(probe_result) is build_transport.DockerUnsupportedV1: + return PipelineBlockedV1(probe_result.reason, probe_result.detail) + if type(probe_result) is not build_transport.DockerSupportedV1: return PipelineBlockedV1( - DockerBlockerReasonV1.BACKEND_CONTRACT, + build_transport.DockerBlockerReasonV1.BACKEND_CONTRACT, "Docker capability report is not typed", ) - + docker_capability = probe_result try: input_bundle = _seal_build_input_bundle_v1(request) except ( @@ -2550,29 +1571,50 @@ def build(self, request: PipelineRequestV1) -> BuildResultV1: tarfile.TarError, BuildSourceAdmissionErrorV1, provenance.ProvenanceErrorV1, + build_input.InputErrorV1, ): - return BuildRejectedV1( + return build_transport.BuildRejectedV1( 1, - BuildFailureReasonV1.BACKEND_CONTRACT, + build_transport.BuildFailureReasonV1.CONTRACT_VIOLATION, ) - builds: list[tuple[bytes, DockerBuildExitedV1]] = [] - for attempt in (1, 2): - built = self._build_once(request, attempt, input_bundle) - if type(built) is BuildRejectedV1: - return built - builds.append(built) - first, second = builds - first_digest = hashlib.sha256(first[0]).digest() - second_digest = hashlib.sha256(second[0]).digest() - if first[0] != second[0]: - return NonReproducibleBuildV1(first_digest, second_digest) - - binary = first[0] - rebuild_sha256s = (first_digest, second_digest) - build_processes = (first[1], second[1]) + built = self._transport.build( + docker_capability, + input_bundle, + request.execution_limits.max_executable_bytes, + input_admission=lambda value: arb_input_is_bound_v1(request, value), + output_admission=self._admit_arb_output_v1, + ) + if type(built) is build_transport.BuildRejectedV1: + return built + if type(built) is not build_transport.TwoBuildObservationV1: + return build_transport.BuildRejectedV1( + 1, + build_transport.BuildFailureReasonV1.CONTRACT_VIOLATION, + ) + if not build_transport.two_build_observation_matches_v1( + built, + built.session, + ): + return build_transport.BuildRejectedV1( + 1, + build_transport.BuildFailureReasonV1.CONTRACT_VIOLATION, + ) + if built.relation is build_transport.BuildByteRelationV1.DIFFERENT: + return built + if built.relation is not build_transport.BuildByteRelationV1.IDENTICAL: + return build_transport.BuildRejectedV1( + 1, + build_transport.BuildFailureReasonV1.CONTRACT_VIOLATION, + ) + + binary = built.outputs[0] + rebuild_sha256s = tuple( + hashlib.sha256(item).digest() for item in built.outputs + ) + build_processes = built.processes comparator = _derive_arb_comparator_for_build_v1( request, - docker_report, + docker_capability, binary, rebuild_sha256s, build_processes, @@ -2589,116 +1631,20 @@ def build(self, request: PipelineRequestV1) -> BuildResultV1: flint_partition.project_pinned_release_only_file_count, request.build_sources.build_input_identity, request.build_sources.formula_support_identity, - pipeline_policy_identity_v1(request.host_trust), - docker_report.daemon_observation_sha256, - docker_report.image_reference, - docker_report.platform, - first_digest, + pipeline_policy_identity_v2( + request.host_trust, + docker_capability.policy, + ), + docker_capability, + rebuild_sha256s[0], rebuild_sha256s, request.host_trust, - input_bundle.identity, + input_bundle.binding_identity, input_bundle.sha256, input_bundle.length, build_processes, comparator, - (first[0], second[0]), + built.outputs, input_bundle, _token=_BUILD_OBSERVATION_TOKEN, ) - - def _build_once( - self, - request: PipelineRequestV1, - attempt: int, - input_bundle: SealedBuildInputBundleV1, - ) -> tuple[bytes, DockerBuildExitedV1] | BuildRejectedV1: - if ( - not sealed_build_input_bundle_is_well_bound_v1(input_bundle) - or input_bundle.source_identity != request.admitted_sources.identity - or input_bundle.build_input_identity - != request.build_sources.build_input_identity - ): - return BuildRejectedV1( - attempt, - BuildFailureReasonV1.BACKEND_CONTRACT, - ) - try: - with tempfile.TemporaryDirectory(prefix=f"labcolors-arb-build-v1-{attempt}-") as temporary: - root = Path(temporary).resolve() - build_request = DockerBuildRequestV1( - attempt, - input_bundle, - request.execution_limits.max_executable_bytes, - root / "container.cid", - _CONTAINER_NAME_PREFIX_V1 - + hashlib.sha256( - os.fsencode(root) + bytes((attempt,)) - ).hexdigest(), - ) - try: - process = self._build_backend.run_build(build_request) - except Exception: - return BuildRejectedV1( - attempt, - BuildFailureReasonV1.BACKEND_CONTRACT, - ) - known_process_types = ( - DockerBuildExitedV1, - DockerBuildTimedOutV1, - DockerBuildOutputLimitV1, - DockerBuildObserverFailureV1, - DockerBuildInputRejectedV1, - DockerBuildCleanupFailureV1, - ) - if type(process) not in known_process_types: - return BuildRejectedV1( - attempt, - BuildFailureReasonV1.BACKEND_CONTRACT, - ) - if type(process) is DockerBuildCleanupFailureV1: - return BuildRejectedV1( - attempt, - BuildFailureReasonV1.CLEANUP_FAILED, - process, - ) - if type(process) is DockerBuildInputRejectedV1: - return BuildRejectedV1( - attempt, - BuildFailureReasonV1.INPUT_TRANSFER_FAILED, - process, - ) - if type(process) is not DockerBuildExitedV1 or process.returncode != 0: - return BuildRejectedV1( - attempt, - BuildFailureReasonV1.PROCESS_FAILED, - process, - ) - transfer = process.input_transfer - if ( - type(transfer) is not BuildInputTransferV1 - or transfer.bundle_identity != input_bundle.identity - or transfer.expected_length != input_bundle.length - or transfer.expected_sha256 != input_bundle.sha256 - or transfer.written_length != input_bundle.length - or transfer.written_sha256 != input_bundle.sha256 - ): - return BuildRejectedV1( - attempt, - BuildFailureReasonV1.BACKEND_CONTRACT, - process, - ) - binary = process.stdout - try: - executor.require_static_x86_64_elf_v1(binary) - except executor.ExecutionRequestErrorV1: - return BuildRejectedV1( - attempt, - BuildFailureReasonV1.INVALID_OUTPUT, - process, - ) - return binary, process - except OSError: - return BuildRejectedV1( - attempt, - BuildFailureReasonV1.BACKEND_CONTRACT, - ) diff --git a/proof/region/v1/arb/receipt.py b/proof/region/v1/arb/receipt.py index d9ac938e..d5191f05 100644 --- a/proof/region/v1/arb/receipt.py +++ b/proof/region/v1/arb/receipt.py @@ -17,6 +17,8 @@ from pathlib import Path from typing import TypeAlias +from build import transport as build_transport + import executor import pipeline import provenance @@ -25,15 +27,15 @@ _EVIDENCE_TOKEN = object() _RECEIPT_TOKEN = object() -_NATIVE_BUILD_BACKEND_TYPE = pipeline.NativeDockerBuildBackendV1 +_NATIVE_BUILD_BACKEND_TYPE = build_transport.NativeDockerBuildBackendV1 _NATIVE_RUN_BACKEND_TYPE = executor.NativeLinuxBackendV1 _SOURCE_ID_LABEL_V1 = b"labcolors.proof-region.arb-source-replay.v1\0" -_BUILD_ID_LABEL_V1 = b"labcolors.proof-region.arb-build-replay.v1\0" +_BUILD_ID_LABEL_V2 = b"labcolors.proof-region.arb-build-replay.v2\0" _RUN_ID_LABEL_V1 = b"labcolors.proof-region.arb-run-replay.v1\0" _EVIDENCE_ID_LABEL_V1 = b"labcolors.proof-region.arb-evaluator-replay.v1\0" -_SOURCE_BOUND_POLICY_ID_LABEL_V1 = ( - b"labcolors.proof-region.arb-source-bound-policy.v1\0" +_SOURCE_BOUND_POLICY_ID_LABEL_V2 = ( + b"labcolors.proof-region.arb-source-bound-policy.v2\0" ) @@ -46,23 +48,29 @@ def _identity(label: bytes, chunks: tuple[bytes, ...]) -> bytes: return hashlib.sha256(label + len(payload).to_bytes(8, "big") + payload).digest() -def source_bound_policy_identity_v1() -> bytes: - """Identity of the exact V1 observation rules and trust boundary.""" +def source_bound_policy_identity_v2( + capability: build_transport.DockerSupportedV1, +) -> bytes: + """Identity of the exact observation rules and observed BUILD capability.""" + + capability_identity = build_transport.docker_capability_identity_v1(capability) return _identity( - _SOURCE_BOUND_POLICY_ID_LABEL_V1, + _SOURCE_BOUND_POLICY_ID_LABEL_V2, ( - pipeline.pipeline_policy_identity_v1( - pipeline.HostTrustBoundaryV1.UNSEALED_LINUX_X64_DOCKER_HOST + pipeline.pipeline_policy_identity_v2( + pipeline.HostTrustBoundaryV1.UNSEALED_LINUX_X64_DOCKER_HOST, + capability.policy, ), + capability_identity, executor.SANDBOX_POLICY_RELEASE_V1.encode("ascii"), b"authority=one-shot-native-controller", b"source=lock-plus-owned-archive-and-build-input-replay", b"build=one-sealed-bundle-two-fresh-byte-equal-attempts", b"run=retained-executable-object-one-contained-process", - b"identity=immutable-coordinates-total-rejection-v1", + b"identity=immutable-coordinates-total-rejection-v2", b"claim=provenance-only-no-numerical-semantics", - b"trust=unsealed-linux-x64-host-and-docker-daemon", + b"trust=unsealed-linux-x64-host-native-docker-cli-and-daemon", ), ) @@ -106,6 +114,22 @@ def _comparator_replays_v1( ) -> bool: try: comparator = build.comparator + capability_identity = build_transport.docker_capability_identity_v1( + build.docker_capability + ) + expected_pipeline_policy = pipeline.pipeline_policy_identity_v2( + request.host_trust, + build.docker_capability.policy, + ) + expected_build_preimage = pipeline.comparator_build_preimage_v2( + request.build_sources, + capability_identity, + expected_pipeline_policy, + build.build_processes, + build.binary_sha256, + build.rebuild_sha256s, + len(build.binary), + ) if ( type(comparator) is not pipeline.DiagnosticArbComparatorV1 or comparator.structural_source_identity @@ -113,6 +137,8 @@ def _comparator_replays_v1( or comparator.build_input_identity != request.build_sources.build_input_identity or comparator.pipeline_policy_identity != build.pipeline_policy_identity + or comparator.pipeline_policy_identity != expected_pipeline_policy + or comparator.preimages.build_identity != expected_build_preimage or comparator.binary_sha256 != build.binary_sha256 or comparator.rebuild_sha256s != build.rebuild_sha256s ): @@ -152,7 +178,7 @@ def _comparator_replays_v1( return False -def _build_identity_v1( +def _build_identity_v2( request: pipeline.PipelineRequestV1, source_identity: bytes, build: pipeline.DiagnosticBuildObservationV1, @@ -162,6 +188,9 @@ def _build_identity_v1( bundle = build.input_bundle processes = build.build_processes binaries = build.rebuild_binaries + capability_identity = build_transport.docker_capability_identity_v1( + build.docker_capability + ) flint_partition = pipeline.flint_source_content_partition_v1( request.source_lock, request.admitted_sources, @@ -180,15 +209,23 @@ def _build_identity_v1( or build.formula_support_identity != request.build_sources.formula_support_identity or build.pipeline_policy_identity - != pipeline.pipeline_policy_identity_v1(request.host_trust) + != pipeline.pipeline_policy_identity_v2( + request.host_trust, + build.docker_capability.policy, + ) + or build.docker_capability.policy + != pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 or build.host_trust is not request.host_trust - or not pipeline.sealed_build_input_bundle_is_well_bound_v1(bundle) - or build.input_bundle_identity != bundle.identity + or not pipeline.arb_input_is_bound_v1(request, bundle) + or build.input_bundle_identity != bundle.binding_identity or build.input_bundle_sha256 != bundle.sha256 or build.input_bundle_length != bundle.length or type(processes) is not tuple or len(processes) != 2 - or any(type(item) is not pipeline.DockerBuildExitedV1 for item in processes) + or any( + type(item) is not build_transport.DockerBuildExitedV1 + for item in processes + ) or type(binaries) is not tuple or len(binaries) != 2 or binaries[0] is not processes[0].stdout @@ -204,8 +241,8 @@ def _build_identity_v1( transfer = process.input_transfer if ( process.returncode != 0 - or type(transfer) is not pipeline.BuildInputTransferV1 - or transfer.bundle_identity != bundle.identity + or type(transfer) is not build_transport.BuildInputTransferV1 + or transfer.bundle_identity != bundle.binding_identity or transfer.expected_length != bundle.length or transfer.expected_sha256 != bundle.sha256 or transfer.written_length != bundle.length @@ -213,19 +250,17 @@ def _build_identity_v1( ): raise TypeError("BUILD transfer did not consume the sealed bundle") return _identity( - _BUILD_ID_LABEL_V1, + _BUILD_ID_LABEL_V2, ( source_identity, build.pipeline_policy_identity, build.host_trust.value.encode("ascii"), - build.docker_daemon_observation_sha256, - build.oci_image_reference.encode("ascii"), - build.oci_platform.encode("ascii"), - bundle.identity, + capability_identity, + bundle.binding_identity, bundle.sha256, bundle.length.to_bytes(8, "big"), - pipeline.build_process_bytes_v1(processes[0]), - pipeline.build_process_bytes_v1(processes[1]), + build_transport.build_process_bytes_v1(processes[0]), + build_transport.build_process_bytes_v1(processes[1]), build.binary_sha256, len(build.binary).to_bytes(8, "big"), build.comparator.identity, @@ -365,7 +400,7 @@ def __init__( if _token is not _EVIDENCE_TOKEN: raise TypeError("ContentResolvedEvaluatorReplayV1 is controller-derived") source_identity = _source_identity_v1(request) - build_identity = _build_identity_v1(request, source_identity, build) + build_identity = _build_identity_v2(request, source_identity, build) run_identity = _run_identity_v1( request, build, @@ -409,7 +444,7 @@ def replay_evidence_is_well_bound_v1(value: object) -> bool: if type(value) is not ContentResolvedEvaluatorReplayV1: return False source_identity = _source_identity_v1(value.request) - build_identity = _build_identity_v1( + build_identity = _build_identity_v2( value.request, source_identity, value.build, @@ -470,7 +505,8 @@ def __init__( ): raise TypeError("SourceBoundEvaluatorReceiptV1 is controller-sealed") if ( - claim.provenance_policy_identity != source_bound_policy_identity_v1() + claim.provenance_policy_identity + != source_bound_policy_identity_v2(evidence.build.docker_capability) or claim.run_claim_identity != evidence.run_claim.identity or claim.replay_evidence_identity != evidence.identity ): @@ -524,8 +560,8 @@ def __post_init__(self) -> None: SourceBoundEvaluatorReceiptV1 | SourceBoundRejectedV1 | pipeline.PipelineBlockedV1 - | pipeline.BuildRejectedV1 - | pipeline.NonReproducibleBuildV1 + | build_transport.BuildRejectedV1 + | build_transport.TwoBuildObservationV1 | pipeline.ExecutionRejectedV1 | pipeline.TranscriptRejectedV1 ) @@ -631,7 +667,10 @@ def execute(self, request: pipeline.PipelineRequestV1) -> SourceBoundResultV1: "exact source, build input, or job replay failed", ) - build_backend = _NATIVE_BUILD_BACKEND_TYPE(self._docker_path) + build_backend = _NATIVE_BUILD_BACKEND_TYPE( + self._docker_path, + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + ) if type(build_backend) is not _NATIVE_BUILD_BACKEND_TYPE: return SourceBoundRejectedV1( SourceBoundFailureReasonV1.REPLAY_BINDING_FAILED, @@ -761,7 +800,7 @@ def execute(self, request: pipeline.PipelineRequestV1) -> SourceBoundResultV1: _token=_EVIDENCE_TOKEN, ) claim = protocol.EvaluatorProvenanceClaimV1( - source_bound_policy_identity_v1(), + source_bound_policy_identity_v2(built.docker_capability), run_claim.identity, evidence.identity, ) diff --git a/proof/region/v1/arb/tests/gate.py b/proof/region/v1/arb/tests/gate.py index 5c8c2fa5..bdba130e 100644 --- a/proof/region/v1/arb/tests/gate.py +++ b/proof/region/v1/arb/tests/gate.py @@ -15,7 +15,7 @@ REPO = Path(__file__).resolve().parents[5] sys.path.insert(0, str(REPO)) EXPECTED_TEST_INVENTORY_SHA256 = ( - "9df49e5bc78ab7cf2386570f500c80f3759f21385f597f2720aa27b1e9700a76" + "383672bd1ac2a2d472fdba33d3ec4c770a897ecd13192ec41e7c12dc1e563219" ) _EVALUATOR_REASON = "set LABCOLORS_ARB_EVALUATOR to the controlled C17 binary" EXPECTED_SKIPS = frozenset( diff --git a/proof/region/v1/arb/tests/test_build_identity_v2.py b/proof/region/v1/arb/tests/test_build_identity_v2.py new file mode 100644 index 00000000..bc66e090 --- /dev/null +++ b/proof/region/v1/arb/tests/test_build_identity_v2.py @@ -0,0 +1,389 @@ +#!/usr/bin/env python3 +"""RED contract for causal BUILD policy and capability identities.""" + +from __future__ import annotations + +import ast +import hashlib +import inspect +import json +import sys +import tempfile +import unittest +from dataclasses import fields as dataclass_fields +from pathlib import Path +from unittest import mock + + +PROOF = Path(__file__).resolve().parents[2] +ARB = PROOF / "arb" +sys.path[:0] = [str(PROOF), str(ARB), str(ARB / "tests")] + +from build import transport as build_transport # noqa: E402 + +import pipeline # noqa: E402 +import receipt # noqa: E402 +from test_pipeline import _BuildBackend, _request, _static_elf # noqa: E402 + + +_POLICY_FIELDS = ( + "image_reference", + "platform", + "hostname", + "container_name_prefix", + "bootstrap", + "bootstrap_argv0", + "tmpfs_specs", + "user_mode", + "stdout_limit", + "stderr_limit", + "build_timeout_ns", + "probe_output_limit", + "probe_timeout_ns", +) + + +def _called_names(function: object) -> set[str]: + tree = ast.parse(inspect.getsource(function)) + names: set[str] = set() + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + if isinstance(node.func, ast.Attribute): + names.add(node.func.attr) + elif isinstance(node.func, ast.Name): + names.add(node.func.id) + return names + + +def _policy_with( + policy: build_transport.DockerBuildPolicyV1, + **changes: object, +) -> build_transport.DockerBuildPolicyV1: + values = {name: getattr(policy, name) for name in _POLICY_FIELDS} + values.update(changes) + return build_transport.DockerBuildPolicyV1( + *(values[name] for name in _POLICY_FIELDS) + ) + + +def _policy_mutants( + policy: build_transport.DockerBuildPolicyV1, +) -> tuple[tuple[str, build_transport.DockerBuildPolicyV1], ...]: + first_tmpfs, *remaining_tmpfs = policy.tmpfs_specs + tmpfs_parts = first_tmpfs.split(",") + size_index = next( + index for index, part in enumerate(tmpfs_parts) if part.startswith("size=") + ) + size_value = int(tmpfs_parts[size_index].removeprefix("size=")) + tmpfs_parts[size_index] = f"size={size_value - 1}" + changed_tmpfs = ",".join(tmpfs_parts) + numeric_fields = ( + "stdout_limit", + "stderr_limit", + "build_timeout_ns", + "probe_output_limit", + "probe_timeout_ns", + ) + mutants: list[tuple[str, build_transport.DockerBuildPolicyV1]] = [ + ( + "image_reference", + _policy_with( + policy, + image_reference="gcc@sha256:" + "ab" * 32, + ), + ), + ("hostname", _policy_with(policy, hostname="labcolors-build-mutant")), + ( + "container_name_prefix", + _policy_with(policy, container_name_prefix="labcolors-mutant-"), + ), + ("bootstrap", _policy_with(policy, bootstrap=policy.bootstrap + "\n:")), + ( + "bootstrap_argv0", + _policy_with(policy, bootstrap_argv0="labcolors-mutant-bootstrap"), + ), + ( + "tmpfs_specs", + _policy_with( + policy, + tmpfs_specs=(changed_tmpfs, *remaining_tmpfs), + ), + ), + ] + for name in numeric_fields: + value = getattr(policy, name) + mutants.append((name, _policy_with(policy, **{name: value - 1}))) + return tuple(mutants) + + +def _capability( + docker_path: Path, + policy: build_transport.DockerBuildPolicyV1, + *, + host_user: tuple[int, int] = (501, 20), + daemon_marker: str = "daemon-a", +) -> build_transport.DockerSupportedV1: + backend = build_transport.NativeDockerBuildBackendV1( + docker_path, + policy, + platform_name="linux", + machine_name="x86_64", + host_user=host_user, + ) + image_observation = json.dumps( + [ + { + "Os": "linux", + "Architecture": "amd64", + "RepoDigests": [policy.image_reference], + } + ], + sort_keys=True, + separators=(",", ":"), + ).encode("ascii") + observations = ( + build_transport._docker_command_exited_v1( + 0, + json.dumps( + {"daemon": daemon_marker}, + sort_keys=True, + separators=(",", ":"), + ).encode("ascii"), + b"", + ), + build_transport._docker_command_exited_v1(0, image_observation, b""), + ) + with mock.patch.object(backend, "_observe_command", side_effect=observations): + capability = backend.probe() + if type(capability) is not build_transport.DockerSupportedV1: + raise AssertionError(capability) + return capability + + +def _observed_build_coordinates( + policy: build_transport.DockerBuildPolicyV1, + capability: build_transport.DockerSupportedV1, +) -> tuple[bytes, bytes, bytes, bytes, tuple[bytes, bytes], bytes, bytes]: + request = _request() + binary = _static_elf(b"identity-v2-invariant-output") + with mock.patch.object(pipeline, "ARB_BUILD_TRANSPORT_POLICY_V1", policy): + result = pipeline.ControlledPipelineV1( + build_backend=_BuildBackend((binary, binary), probe=capability) + ).build(request) + if type(result) is not pipeline.DiagnosticBuildObservationV1: + raise AssertionError(result) + source_identity = receipt._source_identity_v1(request) + receipt_build_identity = receipt._build_identity_v2( + request, + source_identity, + result, + ) + source_bound_policy = receipt.source_bound_policy_identity_v2( + result.docker_capability + ) + process_encodings = tuple( + build_transport.build_process_bytes_v1(process) + for process in result.build_processes + ) + return ( + build_transport.docker_capability_identity_v1(result.docker_capability), + result.comparator.preimages.build_identity, + receipt_build_identity, + source_identity, + process_encodings, + result.input_bundle.contents, + source_bound_policy, + ) + + +class BuildIdentityV2Tests(unittest.TestCase): + def test_v2_surface_replaces_v1_aliases_and_preimage_labels(self) -> None: + self.assertFalse(hasattr(pipeline, "pipeline_policy_identity_v1")) + self.assertFalse(hasattr(receipt, "source_bound_policy_identity_v1")) + self.assertFalse(hasattr(receipt, "_build_identity_v1")) + self.assertTrue(callable(pipeline.pipeline_policy_identity_v2)) + self.assertTrue(callable(receipt.source_bound_policy_identity_v2)) + self.assertTrue(callable(receipt._build_identity_v2)) + + pipeline_source = (ARB / "pipeline.py").read_text(encoding="utf-8") + receipt_source = (ARB / "receipt.py").read_text(encoding="utf-8") + for stale in ( + "labcolors.proof-region.arb-pipeline-policy.v1", + "labcolors.proof-region.arb-comparator.build-identity.v1", + ): + with self.subTest(stale=stale): + self.assertNotIn(stale, pipeline_source) + for stale in ( + "labcolors.proof-region.arb-build-replay.v1", + "labcolors.proof-region.arb-source-bound-policy.v1", + ): + with self.subTest(stale=stale): + self.assertNotIn(stale, receipt_source) + + def test_pipeline_policy_consumes_both_owned_transport_identities(self) -> None: + trust = pipeline.HostTrustBoundaryV1.UNSEALED_LINUX_X64_DOCKER_HOST + policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 + transport_identity = build_transport.transport_policy_identity_v1(policy) + command_identity = build_transport.native_command_contract_identity_v1() + pipeline_identity = pipeline.pipeline_policy_identity_v2(trust, policy) + + for name, value in ( + ("transport", transport_identity), + ("command", command_identity), + ("pipeline", pipeline_identity), + ): + with self.subTest(identity=name): + self.assertIs(type(value), bytes) + self.assertEqual(len(value), hashlib.sha256().digest_size) + self.assertNotEqual(value, bytes(hashlib.sha256().digest_size)) + self.assertNotEqual(transport_identity, command_identity) + self.assertNotEqual(transport_identity, pipeline_identity) + self.assertNotEqual(command_identity, pipeline_identity) + + calls = _called_names(pipeline.pipeline_policy_identity_v2) + self.assertIn("transport_policy_identity_v1", calls) + self.assertIn("native_command_contract_identity_v1", calls) + for surrogate in (tuple(policy), list(policy), object()): + with self.subTest(surrogate=type(surrogate).__name__): + with self.assertRaises(TypeError): + build_transport.transport_policy_identity_v1(surrogate) + with self.assertRaises(TypeError): + pipeline.pipeline_policy_identity_v2(trust, surrogate) + + def test_every_admitted_policy_mutation_changes_transport_and_pipeline_identity(self) -> None: + trust = pipeline.HostTrustBoundaryV1.UNSEALED_LINUX_X64_DOCKER_HOST + policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 + baseline_transport = build_transport.transport_policy_identity_v1(policy) + baseline_pipeline = pipeline.pipeline_policy_identity_v2(trust, policy) + mutants = _policy_mutants(policy) + + self.assertEqual( + {name for name, _mutant in mutants}, + set(_POLICY_FIELDS) - {"platform", "user_mode"}, + ) + for name, mutant in mutants: + with self.subTest(field=name): + self.assertTrue(build_transport.docker_policy_is_valid_v1(mutant)) + self.assertNotEqual( + build_transport.transport_policy_identity_v1(mutant), + baseline_transport, + ) + self.assertNotEqual( + pipeline.pipeline_policy_identity_v2(trust, mutant), + baseline_pipeline, + ) + + # These coordinates currently have singleton admitted domains. Their + # only meaningful mutants are invalid inputs, not a second policy. + self.assertEqual(tuple(build_transport.DockerUserModeV1), (policy.user_mode,)) + with self.assertRaises(TypeError): + _policy_with(policy, platform="linux/arm64") + with self.assertRaises(TypeError): + _policy_with(policy, user_mode="host_effective_ids") + + def test_diagnostic_build_owns_one_capability_and_replayers_consume_its_identity(self) -> None: + field_names = tuple( + field.name for field in dataclass_fields(pipeline.DiagnosticBuildObservationV1) + ) + self.assertEqual(field_names.count("docker_capability"), 1) + for mirror in ( + "docker_daemon_observation_sha256", + "oci_image_reference", + "oci_platform", + "docker_path", + "host_user", + ): + with self.subTest(mirror=mirror): + self.assertNotIn(mirror, field_names) + + comparator_calls = _called_names( + pipeline._derive_arb_comparator_for_build_v1 + ) + comparator_replay_calls = _called_names(receipt._comparator_replays_v1) + receipt_build_calls = _called_names(receipt._build_identity_v2) + source_bound_calls = _called_names( + receipt.source_bound_policy_identity_v2 + ) + self.assertIn("docker_capability_identity_v1", comparator_calls) + self.assertIn("docker_capability_identity_v1", comparator_replay_calls) + self.assertIn("docker_capability_identity_v1", receipt_build_calls) + self.assertIn("docker_capability_identity_v1", source_bound_calls) + + def test_path_uid_daemon_and_hostname_flow_to_downstream_build_identity_only(self) -> None: + baseline_policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 + hostname_policy = _policy_with( + baseline_policy, + hostname="labcolors-build-other-host", + ) + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary).resolve() + first_path = root / "docker-a" + second_path = root / "docker-b" + first_path.write_bytes(b"same-docker-cli-fixture") + second_path.write_bytes(b"same-docker-cli-fixture") + first_path.chmod(0o755) + second_path.chmod(0o755) + + baseline = _observed_build_coordinates( + baseline_policy, + _capability(first_path, baseline_policy), + ) + variants = { + "path": _observed_build_coordinates( + baseline_policy, + _capability(second_path, baseline_policy), + ), + "uid": _observed_build_coordinates( + baseline_policy, + _capability(first_path, baseline_policy, host_user=(502, 20)), + ), + "daemon": _observed_build_coordinates( + baseline_policy, + _capability( + first_path, + baseline_policy, + daemon_marker="daemon-b", + ), + ), + "hostname": _observed_build_coordinates( + hostname_policy, + _capability(first_path, hostname_policy), + ), + } + + ( + baseline_capability, + baseline_comparator_build, + baseline_receipt_build, + baseline_source, + baseline_processes, + baseline_bundle, + baseline_source_bound_policy, + ) = baseline + for name, variant in variants.items(): + with self.subTest(mutation=name): + ( + capability_identity, + comparator_build, + receipt_build, + source_identity, + process_encodings, + bundle_bytes, + source_bound_policy, + ) = variant + if name != "hostname": + self.assertNotEqual(capability_identity, baseline_capability) + self.assertNotEqual(comparator_build, baseline_comparator_build) + self.assertNotEqual(receipt_build, baseline_receipt_build) + self.assertNotEqual( + source_bound_policy, + baseline_source_bound_policy, + ) + self.assertEqual(source_identity, baseline_source) + self.assertEqual(process_encodings, baseline_processes) + self.assertEqual(bundle_bytes, baseline_bundle) + + +if __name__ == "__main__": + unittest.main() diff --git a/proof/region/v1/arb/tests/test_pipeline.py b/proof/region/v1/arb/tests/test_pipeline.py index 95934f3c..7fd65557 100644 --- a/proof/region/v1/arb/tests/test_pipeline.py +++ b/proof/region/v1/arb/tests/test_pipeline.py @@ -7,6 +7,7 @@ import hashlib import io import inspect +import json import os import stat import struct @@ -28,6 +29,8 @@ sys.path.insert(0, str(PROOF)) sys.path.insert(0, str(ARB)) +from build import input as build_input # noqa: E402 +from build import transport as build_transport # noqa: E402 import executor # noqa: E402 import pipeline # noqa: E402 import provenance # noqa: E402 @@ -250,62 +253,117 @@ def _request(**changes: object) -> pipeline.PipelineRequestV1: return pipeline.PipelineRequestV1(**values) +def _docker_capability( + policy: build_transport.DockerBuildPolicyV1 | None = None, + *, + docker_path: Path = Path("/usr/bin/docker"), + host_user: tuple[int, int] = (501, 20), + daemon_marker: bytes = b"docker-daemon-fixture", +) -> build_transport.DockerSupportedV1: + owned_policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 if policy is None else policy + return build_transport.DockerSupportedV1( + owned_policy, + build_transport.DockerDaemonObservationV1( + daemon_marker, + b"docker-image-inspection-fixture", + ), + build_transport.native_command_coordinate_v1(docker_path), + host_user, + ) + + +def _probe_native_backend( + backend: build_transport.NativeDockerBuildBackendV1, + policy: build_transport.DockerBuildPolicyV1, +) -> build_transport.DockerSupportedV1: + image_observation = json.dumps( + [ + { + "Os": "linux", + "Architecture": "amd64", + "RepoDigests": [policy.image_reference], + } + ], + separators=(",", ":"), + ).encode("ascii") + with mock.patch.object( + backend, + "_observe_command", + side_effect=( + build_transport._docker_command_exited_v1( + 0, + b'{"Version":"fixture"}', + b"", + ), + build_transport._docker_command_exited_v1( + 0, + image_observation, + b"", + ), + ), + ): + capability = backend.probe() + if type(capability) is not build_transport.DockerSupportedV1: + raise AssertionError(capability) + return capability + + class _BuildBackend: def __init__( self, outputs: tuple[bytes, ...], *, - probe: pipeline.DockerCapabilityReportV1 | None = None, + probe: build_transport.DockerCapabilityReportV1 | None = None, reject_input: bool = False, omit_transfer: bool = False, foreign_transfer: bool = False, reported_stderr: bytes = b"", ) -> None: self.outputs = list(outputs) - self.probe_result = probe or pipeline.DockerSupportedV1( - pipeline.OCI_IMAGE_REFERENCE_V1, - pipeline.OCI_PLATFORM_V1, - _digest("docker-daemon"), - ) + self.probe_result = probe or _docker_capability() self.reject_input = reject_input self.omit_transfer = omit_transfer self.foreign_transfer = foreign_transfer self.reported_stderr = reported_stderr - self.requests: list[pipeline.DockerBuildRequestV1] = [] + self.requests: list[build_transport.DockerBuildRequestV1] = [] - def probe(self) -> pipeline.DockerCapabilityReportV1: + def probe(self) -> build_transport.DockerCapabilityReportV1: return self.probe_result def run_build( self, - request: pipeline.DockerBuildRequestV1, - ) -> pipeline.DockerBuildProcessObservationV1: + request: build_transport.DockerBuildRequestV1, + ) -> build_transport.DockerBuildProcessObservationV1: self.requests.append(request) output = self.outputs.pop(0) if self.reject_input: - return pipeline.DockerBuildInputRejectedV1( - pipeline._build_input_progress_v1( + return build_transport.DockerBuildInputRejectedV1( + build_transport._build_input_progress_v1( request.input_bundle, 1, - hashlib.sha256(request.input_bundle._contents[:1]).digest(), + hashlib.sha256(request.input_bundle.contents[:1]).digest(), ), b"", b"", ) if self.omit_transfer: - return pipeline._docker_command_exited_v1(0, output, self.reported_stderr) - transfer = pipeline._completed_build_input_transfer_v1( + return build_transport._docker_command_exited_v1(0, output, self.reported_stderr) + transfer = build_transport._completed_build_input_transfer_v1( request.input_bundle, request.input_bundle.length, request.input_bundle.sha256, ) if self.foreign_transfer: - object.__setattr__( - transfer, - "bundle_identity", + foreign_input = build_input.seal_input_v1( _digest("foreign-bundle"), + request.input_bundle.contents, + ) + transfer = build_transport._completed_build_input_transfer_v1( + foreign_input, + foreign_input.length, + foreign_input.sha256, ) - return pipeline._docker_build_exited_v1( + return build_transport._docker_build_exited_v1( 0, output, self.reported_stderr, @@ -602,27 +660,26 @@ def test_host_trust_claims_only_backend_observable_facts(self) -> None: def test_pipeline_policy_identity_binds_the_stream_bootstrap(self) -> None: trust = pipeline.HostTrustBoundaryV1.UNSEALED_LINUX_X64_DOCKER_HOST - original = pipeline.pipeline_policy_identity_v1(trust) - - with mock.patch.object( - pipeline, - "_BUILD_BOOTSTRAP_V1", - pipeline._BUILD_BOOTSTRAP_V1 + "\nexit 1", - ): - changed = pipeline.pipeline_policy_identity_v1(trust) + policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 + coordinates = list(policy) + coordinates[4] = policy.bootstrap + "\nexit 1" + changed_policy = build_transport.DockerBuildPolicyV1(*coordinates) + original = pipeline.pipeline_policy_identity_v2(trust, policy) + changed = pipeline.pipeline_policy_identity_v2(trust, changed_policy) self.assertNotEqual(original, changed) def test_pipeline_policy_identity_binds_the_private_tmpfs_policy(self) -> None: trust = pipeline.HostTrustBoundaryV1.UNSEALED_LINUX_X64_DOCKER_HOST - original = pipeline.pipeline_policy_identity_v1(trust) - - with mock.patch.object( - pipeline, - "_BUILD_TMPFS_SPEC_V1", - "/tmp:rw,exec,suid,dev,mode=1777", - ): - changed = pipeline.pipeline_policy_identity_v1(trust) + policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 + coordinates = list(policy) + coordinates[6] = ( + "/tmp:rw,exec,suid,dev,size=536870912,mode=1777", + policy.tmpfs_specs[1], + ) + changed_policy = build_transport.DockerBuildPolicyV1(*coordinates) + original = pipeline.pipeline_policy_identity_v2(trust, policy) + changed = pipeline.pipeline_policy_identity_v2(trust, changed_policy) self.assertNotEqual(original, changed) @@ -649,6 +706,12 @@ def test_two_fresh_equal_builds_retain_one_input_and_exact_outputs(self) -> None self.assertIs(type(result), pipeline.DiagnosticBuildObservationV1) self.assertEqual(len(build.requests), 2) self.assertEqual(tuple(item.attempt for item in build.requests), (1, 2)) + self.assertTrue( + all( + item.capability is result.docker_capability + for item in build.requests + ) + ) self.assertIs(build.requests[0].input_bundle, build.requests[1].input_bundle) self.assertEqual(result.binary, binary) self.assertEqual(result.binary_sha256, hashlib.sha256(binary).digest()) @@ -697,7 +760,10 @@ def test_two_fresh_equal_builds_retain_one_input_and_exact_outputs(self) -> None ) self.assertEqual( result.pipeline_policy_identity, - pipeline.pipeline_policy_identity_v1(result.host_trust), + pipeline.pipeline_policy_identity_v2( + result.host_trust, + result.docker_capability.policy, + ), ) self.assertFalse(hasattr(result, "build_observer_kind")) self.assertFalse(hasattr(result, "build_source_identity")) @@ -707,8 +773,16 @@ def test_two_fresh_equal_builds_retain_one_input_and_exact_outputs(self) -> None result.host_trust, pipeline.HostTrustBoundaryV1.UNSEALED_LINUX_X64_DOCKER_HOST, ) - self.assertEqual(result.oci_image_reference, pipeline.OCI_IMAGE_REFERENCE_V1) - self.assertEqual(result.oci_platform, pipeline.OCI_PLATFORM_V1) + self.assertEqual( + result.docker_capability.policy.image_reference, + pipeline.OCI_IMAGE_REFERENCE_V1, + ) + self.assertEqual( + result.docker_capability.policy.platform, + pipeline.OCI_PLATFORM_V1, + ) + self.assertFalse(hasattr(result, "oci_image_reference")) + self.assertFalse(hasattr(result, "oci_platform")) self.assertFalse(hasattr(result, "slsa_level")) self.assertFalse(hasattr(result, "fresh_vm")) @@ -720,32 +794,32 @@ def test_builds_must_be_byte_identical(self) -> None: build_backend=_BuildBackend((first, second)), ).build(_request()) - self.assertEqual( - result, - pipeline.NonReproducibleBuildV1( - hashlib.sha256(first).digest(), - hashlib.sha256(second).digest(), - ), + self.assertIs(type(result), build_transport.TwoBuildObservationV1) + self.assertIs( + result.relation, + build_transport.BuildByteRelationV1.DIFFERENT, ) + self.assertEqual(result.first_sha256, hashlib.sha256(first).digest()) + self.assertEqual(result.second_sha256, hashlib.sha256(second).digest()) def test_input_transport_or_invalid_binary_is_typed_failure(self) -> None: binary = _static_elf() cases = ( ( _BuildBackend((binary,), reject_input=True), - pipeline.BuildFailureReasonV1.INPUT_TRANSFER_FAILED, + build_transport.BuildFailureReasonV1.INPUT_TRANSFER_FAILED, ), ( _BuildBackend((binary,), omit_transfer=True), - pipeline.BuildFailureReasonV1.BACKEND_CONTRACT, + build_transport.BuildFailureReasonV1.CONTRACT_VIOLATION, ), ( _BuildBackend((binary,), foreign_transfer=True), - pipeline.BuildFailureReasonV1.BACKEND_CONTRACT, + build_transport.BuildFailureReasonV1.CONTRACT_VIOLATION, ), ( _BuildBackend((b"not-an-elf",)), - pipeline.BuildFailureReasonV1.INVALID_OUTPUT, + build_transport.BuildFailureReasonV1.INVALID_OUTPUT, ), ) for backend, reason in cases: @@ -753,7 +827,7 @@ def test_input_transport_or_invalid_binary_is_typed_failure(self) -> None: result = pipeline.ControlledPipelineV1( build_backend=backend, ).build(_request()) - self.assertIs(type(result), pipeline.BuildRejectedV1) + self.assertIs(type(result), build_transport.BuildRejectedV1) self.assertEqual(result.attempt, 1) self.assertEqual(result.reason, reason) @@ -829,29 +903,29 @@ def test_native_observer_promotion_is_not_representable_in_v1(self) -> None: def test_mutable_exact_native_build_backend_cannot_upgrade_fabricated_build(self) -> None: binary = _static_elf(b"self-mutating-build") - backend = pipeline.NativeDockerBuildBackendV1( + backend = build_transport.NativeDockerBuildBackendV1( Path("/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, platform_name="linux", machine_name="x86_64", ) - def probe(_self: object) -> pipeline.DockerCapabilityReportV1: - return pipeline.DockerSupportedV1( - pipeline.OCI_IMAGE_REFERENCE_V1, - pipeline.OCI_PLATFORM_V1, - _digest("fabricated-daemon"), + def probe(_self: object) -> build_transport.DockerCapabilityReportV1: + return _docker_capability( + docker_path=Path("/bin/true"), + daemon_marker=b"fabricated-daemon", ) def run_build( _self: object, - request: pipeline.DockerBuildRequestV1, - ) -> pipeline.DockerBuildProcessObservationV1: - transfer = pipeline._completed_build_input_transfer_v1( + request: build_transport.DockerBuildRequestV1, + ) -> build_transport.DockerBuildProcessObservationV1: + transfer = build_transport._completed_build_input_transfer_v1( request.input_bundle, request.input_bundle.length, request.input_bundle.sha256, ) - return pipeline._docker_build_exited_v1(0, binary, b"", transfer) + return build_transport._docker_build_exited_v1(0, binary, b"", transfer) backend.probe = MethodType(probe, backend) backend.run_build = MethodType(run_build, backend) @@ -865,26 +939,33 @@ def run_build( class DockerCommandContractTests(unittest.TestCase): - def test_command_is_exact_digest_offline_read_only_and_capability_free(self) -> None: + def test_command_is_exact_digest_offline_read_only_and_capability_bound(self) -> None: with tempfile.TemporaryDirectory() as temporary: root = Path(temporary).resolve() - request = pipeline.DockerBuildRequestV1( + backend = build_transport.NativeDockerBuildBackendV1( + Path("/usr/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + platform_name="linux", + machine_name="x86_64", + host_user=(501, 20), + ) + capability = _probe_native_backend( + backend, + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + ) + request = build_transport.DockerBuildRequestV1( 1, + capability, pipeline._seal_build_input_bundle_v1(_request()), _limits().max_executable_bytes, root / "container.cid", "labcolors-arb-build-v1-test", ) - backend = pipeline.NativeDockerBuildBackendV1( - Path("/usr/bin/docker"), - platform_name="linux", - machine_name="x86_64", - ) command = backend.command_for(request) joined = " ".join(command) - self.assertEqual(command[0], "/usr/bin/docker") + self.assertEqual(command[0], "/usr/bin/true") self.assertIn(pipeline.OCI_IMAGE_REFERENCE_V1, command) self.assertNotIn("gcc:latest", joined) for fragment in ( @@ -906,18 +987,26 @@ def test_command_is_exact_digest_offline_read_only_and_capability_free(self) -> def test_command_exposes_only_a_private_non_executable_standard_tmp(self) -> None: with tempfile.TemporaryDirectory() as temporary: root = Path(temporary).resolve() - request = pipeline.DockerBuildRequestV1( + backend = build_transport.NativeDockerBuildBackendV1( + Path("/usr/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + platform_name="linux", + machine_name="x86_64", + host_user=(501, 20), + ) + capability = _probe_native_backend( + backend, + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + ) + request = build_transport.DockerBuildRequestV1( 1, + capability, pipeline._seal_build_input_bundle_v1(_request()), _limits().max_executable_bytes, root / "container.cid", "labcolors-arb-build-v1-test", ) - command = pipeline.NativeDockerBuildBackendV1( - Path("/usr/bin/docker"), - platform_name="linux", - machine_name="x86_64", - ).command_for(request) + command = backend.command_for(request) tmpfs_indexes = tuple( index for index, item in enumerate(command) if item == "--tmpfs" @@ -938,23 +1027,26 @@ def test_command_exposes_only_a_private_non_executable_standard_tmp(self) -> Non self.assertNotIn("--volume", command) def test_native_probe_fails_closed_without_linux_or_exact_docker(self) -> None: - non_linux = pipeline.NativeDockerBuildBackendV1( + non_linux = build_transport.NativeDockerBuildBackendV1( Path("/usr/bin/docker"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, platform_name="darwin", machine_name="arm64", ).probe() - missing = pipeline.NativeDockerBuildBackendV1( + missing = build_transport.NativeDockerBuildBackendV1( Path("/definitely/missing/docker"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, platform_name="linux", machine_name="x86_64", ).probe() - self.assertEqual(non_linux.reason, pipeline.DockerBlockerReasonV1.HOST_NOT_LINUX_AMD64) - self.assertEqual(missing.reason, pipeline.DockerBlockerReasonV1.DOCKER_UNAVAILABLE) + self.assertEqual(non_linux.reason, build_transport.DockerBlockerReasonV1.HOST_NOT_LINUX_AMD64) + self.assertEqual(missing.reason, build_transport.DockerBlockerReasonV1.DOCKER_UNAVAILABLE) def test_native_command_observer_caps_probe_output_before_allocation(self) -> None: - backend = pipeline.NativeDockerBuildBackendV1( + backend = build_transport.NativeDockerBuildBackendV1( Path("/bin/sh"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, platform_name="linux", machine_name="x86_64", ) @@ -973,19 +1065,22 @@ def test_native_command_observer_caps_probe_output_before_allocation(self) -> No self.assertEqual( result, - pipeline.DockerBuildOutputLimitV1( - pipeline.DockerOutputStreamV1.STDOUT, + build_transport.DockerBuildOutputLimitV1( + build_transport.DockerOutputStreamV1.STDOUT, b"x" * 8, b"", ), ) def test_cleanup_falls_back_to_exact_name_for_absent_or_invalid_cidfile(self) -> None: - backend = pipeline.NativeDockerBuildBackendV1( - Path("/bin/sh"), + backend = build_transport.NativeDockerBuildBackendV1( + Path("/usr/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, platform_name="linux", machine_name="x86_64", + host_user=(501, 20), ) + _probe_native_backend(backend, pipeline.ARB_BUILD_TRANSPORT_POLICY_V1) name = "labcolors-arb-build-v1-cleanup-test" for cid_contents in (None, b"partial-or-foreign"): with self.subTest(cid_contents=cid_contents): @@ -994,8 +1089,8 @@ def test_cleanup_falls_back_to_exact_name_for_absent_or_invalid_cidfile(self) -> if cid_contents is not None: cid_file.write_bytes(cid_contents) observations = ( - pipeline._docker_command_exited_v1(1, b"", b"not found"), - pipeline._docker_command_exited_v1(0, b"", b""), + build_transport._docker_command_exited_v1(1, b"", b"not found"), + build_transport._docker_command_exited_v1(0, b"", b""), ) with mock.patch.object( backend, @@ -1011,8 +1106,9 @@ def test_cleanup_falls_back_to_exact_name_for_absent_or_invalid_cidfile(self) -> self.assertNotIn("partial-or-foreign", " ".join(commands[0])) def test_unverified_container_removal_is_typed_cleanup_failure(self) -> None: - backend = pipeline.NativeDockerBuildBackendV1( + backend = build_transport.NativeDockerBuildBackendV1( Path("/bin/sh"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, platform_name="linux", machine_name="x86_64", ) @@ -1032,10 +1128,10 @@ def test_unverified_container_removal_is_typed_cleanup_failure(self) -> None: container_name="labcolors-arb-build-v1-cleanup-failure", ) - self.assertIs(type(result), pipeline.DockerBuildCleanupFailureV1) + self.assertIs(type(result), build_transport.DockerBuildCleanupFailureV1) self.assertEqual( result.trigger, - pipeline.DockerCleanupTriggerV1.PROCESS_EXIT, + build_transport.DockerCleanupTriggerV1.PROCESS_EXIT, ) diff --git a/proof/region/v1/arb/tests/test_receipt.py b/proof/region/v1/arb/tests/test_receipt.py index 80beade1..95933054 100644 --- a/proof/region/v1/arb/tests/test_receipt.py +++ b/proof/region/v1/arb/tests/test_receipt.py @@ -22,6 +22,8 @@ TESTS = ARB / "tests" sys.path[:0] = [str(PROOF), str(ARB), str(TESTS)] +from build import transport as build_transport # noqa: E402 + import executor # noqa: E402 import pipeline # noqa: E402 import provenance # noqa: E402 @@ -34,6 +36,7 @@ ) from test_pipeline import ( # noqa: E402 _BuildBackend, + _docker_capability, _foreign_comparator, _job, _request, @@ -127,13 +130,13 @@ def _controller( ) return controller, ( mock.patch.object( - pipeline.NativeDockerBuildBackendV1, + build_transport.NativeDockerBuildBackendV1, "probe", autospec=True, side_effect=lambda _self: build_backend.probe(), ), mock.patch.object( - pipeline.NativeDockerBuildBackendV1, + build_transport.NativeDockerBuildBackendV1, "run_build", autospec=True, side_effect=lambda _self, request: build_backend.run_build(request), @@ -219,9 +222,10 @@ def _replace_invocation( class SourceBoundReceiptTests(unittest.TestCase): def test_source_bound_policy_identity_binds_immutable_coordinates(self) -> None: + capability = _docker_capability() self.assertEqual( - receipt.source_bound_policy_identity_v1().hex(), - "a7cf0c142397a1e8ce3f9bb9dd4168120cdf78814745d1d4596d08a8e88a6b1b", + receipt.source_bound_policy_identity_v2(capability).hex(), + "522f089a81e68062f0db4260b00c6e6e0ed2074322247229a99d4714cc5997a5", ) def test_identity_rejection_remains_typed_at_the_receipt_boundary(self) -> None: @@ -275,7 +279,9 @@ def test_only_controller_execution_can_seal_a_receipt(self) -> None: self.assertEqual(result.evidence.identity, result.claim.replay_evidence_identity) self.assertEqual( result.claim.provenance_policy_identity, - receipt.source_bound_policy_identity_v1(), + receipt.source_bound_policy_identity_v2( + result.evidence.build.docker_capability + ), ) self.assertTrue(receipt.replay_evidence_is_well_bound_v1(result.evidence)) self.assertEqual(len(backend.requests), 1) @@ -314,8 +320,11 @@ def test_receipt_uses_only_versioned_public_cross_module_verifiers(self) -> None self.assertNotIn("pipeline._sealed_build_input_bundle_is_well_bound_v1", source) self.assertNotIn("pipeline._build_process_bytes_v1", source) self.assertNotIn("executor._execution_identity_v1", source) - self.assertTrue(hasattr(pipeline, "sealed_build_input_bundle_is_well_bound_v1")) - self.assertTrue(hasattr(pipeline, "build_process_bytes_v1")) + self.assertNotIn("sealed_build_input_bundle_is_well_bound_v1", source) + self.assertIn("pipeline.arb_input_is_bound_v1", source) + self.assertIn("build_transport.build_process_bytes_v1", source) + self.assertTrue(hasattr(pipeline, "arb_input_is_bound_v1")) + self.assertTrue(hasattr(build_transport, "build_process_bytes_v1")) self.assertTrue(hasattr(executor, "invocation_identity_v1")) self.assertTrue(hasattr(executor, "platform_identity_v1")) @@ -323,7 +332,10 @@ def test_reference_does_not_describe_shipped_arb_receipt_as_future(self) -> None documentation = (PROOF / "PROTOCOL.md").read_text(encoding="utf-8") prose = " ".join(documentation.split()) - self.assertIn("## Source-bound Arb replay", documentation) + self.assertIn( + "## Воспроизведение Arb, связанное с источником", + documentation, + ) self.assertIn("SourceBoundEvaluatorReceiptV1", documentation) for stale_claim in ( "заявленные результаты будущих Arb/MPFI processes", @@ -372,8 +384,8 @@ def test_job_first_binds_at_run_not_source_or_build(self) -> None: self.assertEqual(first_source, second_source) self.assertEqual(first_build.input_bundle_identity, second_build.input_bundle_identity) self.assertEqual( - receipt._build_identity_v1(request, first_source, first_build), - receipt._build_identity_v1( + receipt._build_identity_v2(request, first_source, first_build), + receipt._build_identity_v2( different_request, second_source, second_build, @@ -402,7 +414,6 @@ def test_root_and_build_coordinates_are_recomputed(self) -> None: "pipeline_policy_identity", "flint_commit_content_identity", "flint_project_pinned_release_only_identity", - "docker_daemon_observation_sha256", "binary_sha256", "input_bundle_identity", "input_bundle_sha256", @@ -414,6 +425,22 @@ def test_root_and_build_coordinates_are_recomputed(self) -> None: _tamper(dag, "build", build) ) ) + different_capability = _docker_capability( + daemon_marker=b"different-docker-daemon" + ) + self.assertFalse( + receipt.replay_evidence_is_well_bound_v1( + _tamper( + dag, + "build", + _tamper( + dag.build, + "docker_capability", + different_capability, + ), + ) + ) + ) for field_name in ( "flint_commit_content_file_count", "flint_project_pinned_release_only_file_count", @@ -465,41 +492,12 @@ def test_source_process_transfer_and_comparator_mutations_fail(self) -> None: ) first = dag.build.build_processes[0] - for field_name in ("bundle_identity", "expected_sha256", "written_sha256"): - with self.subTest(transfer=field_name): - transfer = _tamper( - first.input_transfer, - field_name, - _digest(field_name), - ) - process = _tamper(first, "input_transfer", transfer) - build = _tamper( - dag.build, - "build_processes", - (process, dag.build.build_processes[1]), - ) - self.assertFalse( - receipt.replay_evidence_is_well_bound_v1( - _tamper(dag, "build", build) - ) - ) - for field_name in ("expected_length", "written_length"): - transfer = _tamper( - first.input_transfer, - field_name, - first.input_transfer.expected_length + 1, - ) - process = _tamper(first, "input_transfer", transfer) - build = _tamper( - dag.build, - "build_processes", - (process, dag.build.build_processes[1]), - ) - self.assertFalse( - receipt.replay_evidence_is_well_bound_v1( - _tamper(dag, "build", build) - ) - ) + self.assertFalse(hasattr(first.input_transfer, "__dict__")) + self.assertFalse(hasattr(first, "__dict__")) + with self.assertRaises(TypeError): + object.__new__(type(first.input_transfer)) + with self.assertRaises(TypeError): + object.__new__(type(first)) preimages = _tamper( dag.build.comparator.preimages, diff --git a/proof/region/v1/arb/tests/test_transport.py b/proof/region/v1/arb/tests/test_transport.py index 63db7075..8011748d 100644 --- a/proof/region/v1/arb/tests/test_transport.py +++ b/proof/region/v1/arb/tests/test_transport.py @@ -3,7 +3,6 @@ from __future__ import annotations -import dataclasses import hashlib import io import inspect @@ -24,31 +23,34 @@ sys.path.insert(0, str(ARB)) sys.path.insert(0, str(TESTS)) +from build import input as build_input # noqa: E402 +from build import transport as build_transport # noqa: E402 import pipeline # noqa: E402 -from test_pipeline import _request # noqa: E402 +from test_pipeline import ( # noqa: E402 + _docker_capability, + _probe_native_backend, + _request, +) BUILD_RECIPE = ARB / "build.sh" NATIVE_GATE = ARB / "tests" / "native_gate.py" +_TEST_CANONICAL_LIMITS = build_input.CanonicalInputLimitsV1(64, 1024, 4096) def _digest(label: str) -> bytes: return hashlib.sha256(label.encode("ascii")).digest() -def _bundle(length: int = 1024 * 1024) -> pipeline.SealedBuildInputBundleV1: +def _bundle(length: int = 1024 * 1024) -> build_input.SealedInputV1: contents = (b"0123456789abcdef" * ((length + 15) // 16))[:length] - return pipeline.SealedBuildInputBundleV1( - _digest("source"), - _digest("build-input"), - contents, - _token=pipeline._BUILD_INPUT_BUNDLE_TOKEN, - ) + return build_input.seal_input_v1(_digest("opaque-binding"), contents) -def _backend() -> pipeline.NativeDockerBuildBackendV1: - return pipeline.NativeDockerBuildBackendV1( +def _backend() -> build_transport.NativeDockerBuildBackendV1: + return build_transport.NativeDockerBuildBackendV1( Path("/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, platform_name="linux", machine_name="x86_64", ) @@ -56,12 +58,12 @@ def _backend() -> pipeline.NativeDockerBuildBackendV1: def _observe( source: str, - bundle: pipeline.SealedBuildInputBundleV1, + bundle: build_input.SealedInputV1, *, stdout_limit: int = 2 * 1024 * 1024, stderr_limit: int = 2 * 1024 * 1024, timeout_ns: int = 5_000_000_000, -) -> pipeline.DockerBuildProcessObservationV1: +) -> build_transport.DockerBuildProcessObservationV1: return _backend()._observe_command( (sys.executable, "-c", source), stdout_limit=stdout_limit, @@ -79,11 +81,11 @@ def test_bundle_is_reproducible_normalized_ustar_with_no_host_authority(self) -> second = pipeline._seal_build_input_bundle_v1(request) self.assertIsNot(first, second) - self.assertIs(first._contents, first._contents) - self.assertEqual(first._contents, second._contents) + self.assertIs(first.contents, first.contents) + self.assertEqual(first.contents, second.contents) self.assertEqual(first.sha256, second.sha256) - self.assertEqual(first.identity, second.identity) - self.assertTrue(pipeline.sealed_build_input_bundle_is_well_bound_v1(first)) + self.assertEqual(first.binding_identity, second.binding_identity) + self.assertTrue(pipeline.arb_input_is_bound_v1(request, first)) source_entries = tuple( entry @@ -117,7 +119,7 @@ def test_bundle_is_reproducible_normalized_ustar_with_no_host_authority(self) -> sorted(expected_directories, key=lambda value: (value.count("/"), value)) ) + tuple(sorted(expected_files)) - with tarfile.open(fileobj=io.BytesIO(first._contents), mode="r:") as archive: + with tarfile.open(fileobj=io.BytesIO(first.contents), mode="r:") as archive: members = tuple(archive) self.assertFalse(archive.pax_headers) self.assertEqual(tuple(member.name for member in members), expected_order) @@ -145,36 +147,134 @@ def test_bundle_is_reproducible_normalized_ustar_with_no_host_authority(self) -> self.assertEqual(stream.read(), body) def test_canonical_encoder_rejects_reorder_collision_and_unencodable_path(self) -> None: + def reject( + values: object, + reason: build_input.InputReasonV1, + field: str, + limits: build_input.CanonicalInputLimitsV1 = _TEST_CANONICAL_LIMITS, + ) -> None: + with self.assertRaises(build_input.InputErrorV1) as caught: + build_input.canonical_ustar_v1(values, limits) + self.assertEqual(caught.exception.reason, reason) + self.assertEqual(caught.exception.field, field) + entries = (("a/b", 0o644, b"x"), ("c", 0o755, b"y")) - encoded = pipeline._canonical_tar_v1(entries) + encoded = build_input.canonical_ustar_v1(entries, _TEST_CANONICAL_LIMITS) self.assertEqual( hashlib.sha256(encoded).hexdigest(), "11bc313cba907e89535876eb8ce46194472367007053ab58b723338676f99427", ) - with self.assertRaises(TypeError): - pipeline._canonical_tar_v1(tuple(reversed(entries))) - with self.assertRaises(TypeError): - pipeline._canonical_tar_v1((("a", 0o644, b"x"), ("a/b", 0o644, b"y"))) - with self.assertRaises(TypeError): - pipeline._canonical_tar_v1((("A", 0o644, b"x"), ("a", 0o644, b"y"))) - with self.assertRaises((TypeError, ValueError)): - pipeline._canonical_tar_v1((("a" * 256, 0o644, b"x"),)) + for hostile, reason, field in ( + ( + tuple(reversed(entries)), + build_input.InputReasonV1.NONCANONICAL_SET, + "entries", + ), + ( + (("a", 0o644, b"x"), ("a/b", 0o644, b"y")), + build_input.InputReasonV1.NONCANONICAL_SET, + "a", + ), + ( + (("A", 0o644, b"x"), ("a", 0o644, b"y")), + build_input.InputReasonV1.NONCANONICAL_SET, + "a", + ), + ( + (("a" * 256, 0o644, b"x"),), + build_input.InputReasonV1.INVALID_PATH, + "a" * 256, + ), + ( + ((1, 0o644, b"x"),), + build_input.InputReasonV1.INVALID_PATH, + "path", + ), + ( + ((["a"], 0o644, b"x"),), + build_input.InputReasonV1.INVALID_PATH, + "path", + ), + ( + (("a" * 101, 0o644, b"x"),), + build_input.InputReasonV1.INVALID_PATH, + "a" * 101, + ), + ( + (("A", 0o644, b"x"), ("a/b", 0o644, b"y")), + build_input.InputReasonV1.NONCANONICAL_SET, + "A", + ), + ( + ((("a" * 120) + "/f", 0o644, b"x"),), + build_input.InputReasonV1.INVALID_PATH, + "a" * 120, + ), + ): + with self.subTest(hostile=repr(hostile)): + reject(hostile, reason, field) + + resource_cases = ( + ( + (("a", 0o644, b"x"), ("b", 0o644, b"y")), + build_input.CanonicalInputLimitsV1(1, 1, 2), + "max_members", + ), + ( + (("a", 0o644, b"xy"),), + build_input.CanonicalInputLimitsV1(1, 1, 2), + "max_file_bytes", + ), + ( + (("a", 0o644, b"x"), ("b", 0o644, b"y")), + build_input.CanonicalInputLimitsV1(2, 1, 1), + "max_payload_bytes", + ), + ( + (("a/b", 0o644, b"x"),), + build_input.CanonicalInputLimitsV1(1, 1, 1), + "max_members", + ), + ( + (("a", 0o644, b"x"),), + build_input.CanonicalInputLimitsV1(1, 1, 1, 10_239), + "max_encoded_bytes", + ), + ) + for values, limits, field in resource_cases: + with self.subTest(resource=field): + reject( + values, + build_input.InputReasonV1.RESOURCE_LIMIT, + field, + limits, + ) + exact_cap = build_input.CanonicalInputLimitsV1(1, 1, 1, 10_240) + self.assertEqual( + len(build_input.canonical_ustar_v1((("a", 0o644, b"x"),), exact_cap)), + exact_cap.max_encoded_bytes, + ) def test_omission_or_content_mutation_changes_bundle_identity(self) -> None: entries = (("a", 0o644, b"x"), ("b", 0o644, b"y")) - original = pipeline._canonical_tar_v1(entries) - omitted = pipeline._canonical_tar_v1(entries[:1]) - mutated = pipeline._canonical_tar_v1( - (("a", 0o644, b"x"), ("b", 0o644, b"z")) + original = build_input.canonical_ustar_v1(entries, _TEST_CANONICAL_LIMITS) + omitted = build_input.canonical_ustar_v1( + entries[:1], + _TEST_CANONICAL_LIMITS, + ) + mutated = build_input.canonical_ustar_v1( + (("a", 0o644, b"x"), ("b", 0o644, b"z")), + _TEST_CANONICAL_LIMITS, ) identities = { - pipeline.SealedBuildInputBundleV1( - _digest("source"), - _digest("build-input"), - body, - _token=pipeline._BUILD_INPUT_BUNDLE_TOKEN, - ).identity + ( + sealed.binding_identity, + sealed.sha256, + ) for body in (original, omitted, mutated) + for sealed in ( + build_input.seal_input_v1(_digest("opaque-binding"), body), + ) } self.assertEqual(len(identities), 3) @@ -184,7 +284,7 @@ def test_replayed_source_coordinates_must_match_the_admitted_capability(self) -> original = admitted.tree_identity object.__setattr__(admitted, "tree_identity", _digest("mutated-tree")) try: - with self.assertRaises(TypeError): + with self.assertRaises(pipeline.PipelineInputErrorV1): pipeline._normalized_source_entries_v1( request.source_lock.sources[0], admitted, @@ -195,17 +295,72 @@ def test_replayed_source_coordinates_must_match_the_admitted_capability(self) -> def test_transport_authorities_cannot_be_directly_forged(self) -> None: bundle = _bundle(1024) with self.assertRaises(TypeError): - pipeline.BuildInputTransferProgressV1( - bundle.identity, + build_transport.BuildInputTransferProgressV1( + bundle.binding_identity, bundle.length, bundle.sha256, bundle.length, bundle.sha256, ) with self.assertRaises(TypeError): - pipeline.BuildInputTransferV1(object()) + build_transport.BuildInputTransferV1(object()) + with self.assertRaises(TypeError): + build_transport.DockerBuildExitedV1(0, b"binary", b"", object()) + with self.assertRaises(TypeError): + build_transport.DockerBuildPolicyV1( + "gcc@sha256:bad@sha256:" + "0" * 64, + *pipeline.ARB_BUILD_TRANSPORT_POLICY_V1[1:], + ) + report = _docker_capability() + self.assertFalse(hasattr(report, "__dict__")) + with self.assertRaises((AttributeError, TypeError)): + object.__setattr__(report, "platform", "foreign") + forged_policy = tuple.__new__(build_transport.DockerBuildPolicyV1, ()) with self.assertRaises(TypeError): - pipeline.DockerBuildExitedV1(0, b"binary", b"", object()) + build_transport.ControlledBuildTransportV1( + policy=forged_policy, + backend=object(), + ) + + class ForgedProbeBackend: + def probe(self) -> object: + return tuple.__new__(build_transport.DockerSupportedV1, ()) + + probed = build_transport.ControlledBuildTransportV1( + policy=pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + backend=ForgedProbeBackend(), + ).probe() + self.assertIs(type(probed), build_transport.DockerUnsupportedV1) + self.assertEqual( + probed.reason, + build_transport.DockerBlockerReasonV1.BACKEND_CONTRACT, + ) + + class ForgedProcessBackend: + def probe(self) -> object: + return report + + def run_build(self, _request: object) -> object: + return tuple.__new__(build_transport.DockerBuildExitedV1, ()) + + controller = build_transport.ControlledBuildTransportV1( + policy=pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + backend=ForgedProcessBackend(), + ) + observed_report = controller.probe() + self.assertIs(type(observed_report), build_transport.DockerSupportedV1) + rejected = controller.build( + observed_report, + bundle, + 1, + input_admission=lambda _value: True, + output_admission=lambda _value: True, + ) + self.assertIs(type(rejected), build_transport.BuildRejectedV1) + self.assertEqual( + rejected.reason, + build_transport.BuildFailureReasonV1.CONTRACT_VIOLATION, + ) class BuildInputObserverTests(unittest.TestCase): @@ -216,7 +371,7 @@ def test_positive_partial_writes_are_normal_and_commit_exact_transfer(self) -> N def partial_write(descriptor: int, contents: object) -> int: return real_write(descriptor, contents[:997]) - with mock.patch.object(pipeline.os, "write", side_effect=partial_write): + with mock.patch.object(build_transport.os, "write", side_effect=partial_write): result = _observe( "import hashlib,sys; d=sys.stdin.buffer.read(); " "sys.stdout.buffer.write(hashlib.sha256(d).digest()); " @@ -224,10 +379,10 @@ def partial_write(descriptor: int, contents: object) -> int: bundle, ) - self.assertIs(type(result), pipeline.DockerBuildExitedV1, result) + self.assertIs(type(result), build_transport.DockerBuildExitedV1, result) self.assertEqual(result.stdout, bundle.sha256) self.assertEqual(result.stderr, b"observed") - self.assertEqual(result.input_transfer.bundle_identity, bundle.identity) + self.assertEqual(result.input_transfer.bundle_identity, bundle.binding_identity) self.assertEqual(result.input_transfer.expected_length, bundle.length) self.assertEqual(result.input_transfer.expected_sha256, bundle.sha256) self.assertEqual(result.input_transfer.written_length, bundle.length) @@ -235,18 +390,18 @@ def partial_write(descriptor: int, contents: object) -> int: def test_zero_write_and_epipe_are_typed_with_exact_partial_progress(self) -> None: bundle = _bundle() - with mock.patch.object(pipeline.os, "write", return_value=0): + with mock.patch.object(build_transport.os, "write", return_value=0): zero = _observe("import sys; sys.stdin.buffer.read()", bundle) - self.assertIs(type(zero), pipeline.DockerBuildInputRejectedV1, zero) + self.assertIs(type(zero), build_transport.DockerBuildInputRejectedV1, zero) self.assertEqual(zero.written_length, 0) self.assertEqual(zero.written_sha256, hashlib.sha256(b"").digest()) closed = _observe("import os,time; os.close(0); time.sleep(1)", bundle) - self.assertIs(type(closed), pipeline.DockerBuildInputRejectedV1, closed) + self.assertIs(type(closed), build_transport.DockerBuildInputRejectedV1, closed) self.assertLess(closed.written_length, bundle.length) self.assertEqual( closed.written_sha256, - hashlib.sha256(bundle._contents[: closed.written_length]).digest(), + hashlib.sha256(bundle.contents[: closed.written_length]).digest(), ) def test_final_stdin_close_failure_is_a_typed_observer_failure(self) -> None: @@ -273,14 +428,14 @@ def spawn(*args: object, **kwargs: object) -> subprocess.Popen[bytes]: process.stdin = CloseFailsOnce(process.stdin) return process - with mock.patch.object(pipeline.subprocess, "Popen", side_effect=spawn): + with mock.patch.object(build_transport.subprocess, "Popen", side_effect=spawn): result = _observe( "import time; time.sleep(1)", _bundle(2 * 1024 * 1024), timeout_ns=100_000_000, ) - self.assertIs(type(result), pipeline.DockerBuildObserverFailureV1, result) + self.assertIs(type(result), build_transport.DockerBuildObserverFailureV1, result) def test_full_duplex_backpressure_does_not_deadlock_or_drop_bytes(self) -> None: bundle = _bundle(512 * 1024) @@ -293,7 +448,7 @@ def test_full_duplex_backpressure_does_not_deadlock_or_drop_bytes(self) -> None: " os.write(2,b'e'*len(d))\n", bundle, ) - self.assertIs(type(result), pipeline.DockerBuildExitedV1, result) + self.assertIs(type(result), build_transport.DockerBuildExitedV1, result) self.assertEqual(len(result.stdout), bundle.length) self.assertEqual(len(result.stderr), bundle.length) self.assertEqual(result.input_transfer.written_sha256, bundle.sha256) @@ -305,8 +460,8 @@ def test_timeout_and_output_limit_preserve_input_progress(self) -> None: bundle, timeout_ns=100_000_000, ) - self.assertIs(type(timed), pipeline.DockerBuildTimedOutV1, timed) - self.assertIs(type(timed.input_progress), pipeline.BuildInputTransferProgressV1) + self.assertIs(type(timed), build_transport.DockerBuildTimedOutV1, timed) + self.assertIs(type(timed.input_progress), build_transport.BuildInputTransferProgressV1) self.assertGreater(timed.input_progress.written_length, 0) self.assertLess(timed.input_progress.written_length, bundle.length) @@ -315,10 +470,10 @@ def test_timeout_and_output_limit_preserve_input_progress(self) -> None: bundle, stdout_limit=8, ) - self.assertIs(type(limited), pipeline.DockerBuildOutputLimitV1, limited) - self.assertEqual(limited.stream, pipeline.DockerOutputStreamV1.STDOUT) + self.assertIs(type(limited), build_transport.DockerBuildOutputLimitV1, limited) + self.assertEqual(limited.stream, build_transport.DockerOutputStreamV1.STDOUT) self.assertEqual(limited.stdout, b"x" * 8) - self.assertIs(type(limited.input_progress), pipeline.BuildInputTransferProgressV1) + self.assertIs(type(limited.input_progress), build_transport.BuildInputTransferProgressV1) def test_cleanup_failure_preserves_input_trigger_and_progress(self) -> None: bundle = _bundle() @@ -337,32 +492,50 @@ def test_cleanup_failure_preserves_input_trigger_and_progress(self) -> None: container_name="labcolors-arb-build-v1-transport-test", input_bundle=bundle, ) - self.assertIs(type(result), pipeline.DockerBuildCleanupFailureV1, result) - self.assertEqual(result.trigger, pipeline.DockerCleanupTriggerV1.INPUT_TRANSFER) + self.assertIs(type(result), build_transport.DockerBuildCleanupFailureV1, result) + self.assertEqual(result.trigger, build_transport.DockerCleanupTriggerV1.INPUT_TRANSFER) self.assertEqual(result.detail, "forced cleanup failure") - self.assertIs(type(result.input_progress), pipeline.BuildInputTransferProgressV1) + self.assertIs(type(result.input_progress), build_transport.BuildInputTransferProgressV1) self.assertLess(result.input_progress.written_length, bundle.length) class SealedBuildTransportContractTests(unittest.TestCase): def test_controller_owns_one_sealed_bundle_for_both_builds(self) -> None: - build_source = inspect.getsource(pipeline.ControlledPipelineV1.build) - self.assertEqual(build_source.count("_seal_build_input_bundle_v1("), 1) - self.assertIn("for attempt in (1, 2)", build_source) + pipeline_source = inspect.getsource(pipeline.ControlledPipelineV1.build) + transport_source = inspect.getsource( + build_transport.ControlledBuildTransportV1.build + ) + self.assertEqual(pipeline_source.count("_seal_build_input_bundle_v1("), 1) + self.assertIn("for attempt in (1, 2)", transport_source) def test_docker_request_has_no_semantic_host_path_authority(self) -> None: - fields = {item.name for item in dataclasses.fields(pipeline.DockerBuildRequestV1)} + fields = set(inspect.signature(build_transport.DockerBuildRequestV1).parameters) self.assertEqual( fields, - {"attempt", "input_bundle", "max_executable_bytes", "cid_file", "container_name"}, + { + "attempt", + "capability", + "input_bundle", + "max_output_bytes", + "cid_file", + "container_name", + }, ) - command = pipeline.NativeDockerBuildBackendV1( - Path("/usr/bin/docker"), + backend = build_transport.NativeDockerBuildBackendV1( + Path("/usr/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, platform_name="linux", machine_name="x86_64", - ).command_for( - pipeline.DockerBuildRequestV1( + host_user=(501, 20), + ) + capability = _probe_native_backend( + backend, + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + ) + command = backend.command_for( + build_transport.DockerBuildRequestV1( 1, + capability, _bundle(1024), 1024, Path("/tmp/container.cid"), diff --git a/proof/region/v1/build/__init__.py b/proof/region/v1/build/__init__.py new file mode 100644 index 00000000..dd2df699 --- /dev/null +++ b/proof/region/v1/build/__init__.py @@ -0,0 +1 @@ +"""Owned shared BUILD package; public contracts live in focused leaf modules.""" diff --git a/proof/region/v1/build/input.py b/proof/region/v1/build/input.py new file mode 100644 index 00000000..3ebe0b32 --- /dev/null +++ b/proof/region/v1/build/input.py @@ -0,0 +1,321 @@ +#!/usr/bin/env python3 +"""Canonical BUILD input bytes without engine or recipe semantics.""" + +from __future__ import annotations + +import hashlib +import io +import tarfile +from dataclasses import dataclass +from enum import StrEnum +from typing import NoReturn + + +_SEALED_INPUT_TOKEN = object() +_USTAR_BLOCK_BYTES = 512 +_USTAR_RECORD_BYTES = 20 * _USTAR_BLOCK_BYTES +_USTAR_EOF_BLOCKS = 2 + + +def _valid_digest(value: object) -> bool: + return type(value) is bytes and len(value) == 32 and value != bytes(32) + + +class InputReasonV1(StrEnum): + WRONG_TYPE = "wrong_type" + INVALID_PATH = "invalid_path" + INVALID_MODE = "invalid_mode" + NONCANONICAL_SET = "noncanonical_set" + RESOURCE_LIMIT = "resource_limit" + + +@dataclass(frozen=True) +class InputErrorV1(ValueError): + reason: InputReasonV1 + field: str + + def __str__(self) -> str: + return f"{self.reason.value}: {self.field}" + + +def _fail(reason: InputReasonV1, field_name: str) -> NoReturn: + raise InputErrorV1(reason, field_name) + + +def _logical_path(value: object) -> str: + if type(value) is not str or not value or value.startswith("/") or "\\" in value: + _fail(InputReasonV1.INVALID_PATH, "path") + try: + encoded = value.encode("ascii") + except UnicodeEncodeError: + _fail(InputReasonV1.INVALID_PATH, "path") + if ( + len(encoded) > 4096 + or any(byte < 0x20 or byte == 0x7F for byte in encoded) + or any(part in ("", ".", "..") for part in value.split("/")) + ): + _fail(InputReasonV1.INVALID_PATH, "path") + return value + + +class CanonicalInputLimitsV1(tuple): + """Caller-owned resource bounds for one in-memory canonical archive.""" + + __slots__ = () + + def __new__( + cls, + max_members: int, + max_file_bytes: int, + max_payload_bytes: int, + max_encoded_bytes: int | None = None, + ) -> CanonicalInputLimitsV1: + base_values = (max_members, max_file_bytes, max_payload_bytes) + if any( + type(value) is not int or value <= 0 or value >= 1 << 64 + for value in base_values + ): + raise TypeError("canonical input limits must be positive u64 values") + if max_encoded_bytes is None: + # USTAR adds one header block per member, at most one partial data + # block per member, two EOF blocks, then pads to one record. This + # is derived from the caller's bounds, not a fixture-specific cap. + maximum_unpadded = ( + max_payload_bytes + + (2 * _USTAR_BLOCK_BYTES - 1) * max_members + + _USTAR_EOF_BLOCKS * _USTAR_BLOCK_BYTES + ) + max_encoded_bytes = _round_up( + maximum_unpadded, + _USTAR_RECORD_BYTES, + ) + values = (*base_values, max_encoded_bytes) + if any(type(value) is not int or value <= 0 or value >= 1 << 64 for value in values): + raise TypeError("canonical input limits must be positive u64 values") + return tuple.__new__(cls, values) + + @property + def max_members(self) -> int: + return self[0] + + @property + def max_file_bytes(self) -> int: + return self[1] + + @property + def max_payload_bytes(self) -> int: + return self[2] + + @property + def max_encoded_bytes(self) -> int: + return self[3] + + +def _limits_are_valid(value: object) -> bool: + if type(value) is not CanonicalInputLimitsV1: + return False + try: + return tuple(CanonicalInputLimitsV1(*tuple(value))) == tuple(value) + except Exception: + return False + + +def _ustar_path_is_encodable(path: str) -> bool: + encoded = path.encode("ascii") + if len(encoded) <= 100: + return True + return any( + 0 < index <= 155 and len(encoded) - index - 1 <= 100 + for index, byte in enumerate(encoded) + if byte == ord("/") + ) + + +def _round_up(value: int, quantum: int) -> int: + return ((value + quantum - 1) // quantum) * quantum + + +def _encoded_ustar_length( + entries: tuple[tuple[str, int, bytes], ...], + directory_count: int, +) -> int: + data_bytes = sum( + _round_up(len(contents), _USTAR_BLOCK_BYTES) + for _path, _mode, contents in entries + ) + raw_bytes = ( + (len(entries) + directory_count + _USTAR_EOF_BLOCKS) + * _USTAR_BLOCK_BYTES + + data_bytes + ) + return _round_up(raw_bytes, _USTAR_RECORD_BYTES) + + +class SealedInputV1(tuple): + """Owned exact bytes carrying only integrity and an opaque caller binding.""" + + __slots__ = () + + def __new__( + cls, + binding_identity: bytes, + contents: bytes, + *, + _token: object, + ) -> SealedInputV1: + if _token is not _SEALED_INPUT_TOKEN: + raise TypeError("SealedInputV1 is created only by seal_input_v1") + if not _valid_digest(binding_identity): + raise TypeError("binding_identity must be one opaque nonzero digest") + if type(contents) is not bytes or not contents: + raise TypeError("sealed input must own nonempty exact bytes") + return tuple.__new__( + cls, + ( + binding_identity, + hashlib.sha256(contents).digest(), + len(contents), + contents, + ), + ) + + @property + def binding_identity(self) -> bytes: + return self[0] + + @property + def sha256(self) -> bytes: + return self[1] + + @property + def length(self) -> int: + return self[2] + + @property + def contents(self) -> bytes: + return self[3] + + +def seal_input_v1(binding_identity: bytes, contents: bytes) -> SealedInputV1: + """Seal exact bytes while treating their semantic binding as opaque.""" + + return SealedInputV1( + binding_identity, + contents, + _token=_SEALED_INPUT_TOKEN, + ) + + +def sealed_input_is_intact_v1(value: object) -> bool: + """Recheck byte integrity without interpreting the caller-owned binding.""" + + if type(value) is not SealedInputV1: + return False + try: + return ( + _valid_digest(value.binding_identity) + and type(value.contents) is bytes + and bool(value.contents) + and value.length == len(value.contents) + and value.sha256 == hashlib.sha256(value.contents).digest() + ) + except Exception: + return False + + +def canonical_ustar_v1( + entries: tuple[tuple[str, int, bytes], ...], + limits: CanonicalInputLimitsV1, +) -> bytes: + """Encode one canonical normalized USTAR file tree.""" + + if ( + type(entries) is not tuple + or not entries + or not _limits_are_valid(limits) + ): + _fail(InputReasonV1.NONCANONICAL_SET, "entries") + if len(entries) > limits.max_members: + _fail(InputReasonV1.RESOURCE_LIMIT, "max_members") + parsed: list[tuple[str, int, bytes]] = [] + total_bytes = 0 + for entry in entries: + if type(entry) is not tuple or len(entry) != 3: + _fail(InputReasonV1.WRONG_TYPE, "entries") + path, mode, contents = entry + path = _logical_path(path) + if not _ustar_path_is_encodable(path): + _fail(InputReasonV1.INVALID_PATH, path) + if type(mode) is not int or mode not in (0o644, 0o755): + _fail(InputReasonV1.INVALID_MODE, path) + if type(contents) is not bytes: + _fail(InputReasonV1.WRONG_TYPE, path) + total_bytes += len(contents) + if len(contents) > limits.max_file_bytes: + _fail(InputReasonV1.RESOURCE_LIMIT, "max_file_bytes") + if total_bytes > limits.max_payload_bytes: + _fail(InputReasonV1.RESOURCE_LIMIT, "max_payload_bytes") + parsed.append((path, mode, contents)) + owned = tuple(parsed) + paths = tuple(path for path, _mode, _contents in owned) + if paths != tuple(sorted(paths)) or len(set(paths)) != len(entries): + _fail(InputReasonV1.NONCANONICAL_SET, "entries") + directories: set[str] = set() + for path, _mode, _contents in owned: + parts = path.split("/")[:-1] + for length in range(1, len(parts) + 1): + directories.add("/".join(parts[:length])) + for path in directories: + if not _ustar_path_is_encodable(path): + _fail(InputReasonV1.INVALID_PATH, path) + namespace: dict[str, tuple[str, str]] = {} + for kind, values in (("directory", tuple(sorted(directories))), ("file", paths)): + for path in values: + folded = path.lower() + prior = namespace.get(folded) + coordinate = (kind, path) + if prior is not None and prior != coordinate: + _fail(InputReasonV1.NONCANONICAL_SET, path) + namespace[folded] = coordinate + if ( + directories.intersection(paths) + or len(directories) + len(owned) > limits.max_members + ): + if directories.intersection(paths): + _fail(InputReasonV1.NONCANONICAL_SET, "file-directory collision") + _fail(InputReasonV1.RESOURCE_LIMIT, "max_members") + encoded_length = _encoded_ustar_length(owned, len(directories)) + if encoded_length > limits.max_encoded_bytes: + _fail(InputReasonV1.RESOURCE_LIMIT, "max_encoded_bytes") + + output = io.BytesIO() + try: + with tarfile.open(fileobj=output, mode="w", format=tarfile.USTAR_FORMAT) as archive: + for path in sorted(directories, key=lambda value: (value.count("/"), value)): + member = tarfile.TarInfo(path) + member.type = tarfile.DIRTYPE + member.mode = 0o755 + member.uid = 0 + member.gid = 0 + member.uname = "" + member.gname = "" + member.mtime = 0 + member.size = 0 + archive.addfile(member) + for path, mode, contents in owned: + member = tarfile.TarInfo(path) + member.type = tarfile.REGTYPE + member.mode = mode + member.uid = 0 + member.gid = 0 + member.uname = "" + member.gname = "" + member.mtime = 0 + member.size = len(contents) + archive.addfile(member, io.BytesIO(contents)) + except (OSError, OverflowError, tarfile.TarError, ValueError): + _fail(InputReasonV1.INVALID_PATH, "USTAR encoding") + encoded = output.getvalue() + if len(encoded) != encoded_length: + _fail(InputReasonV1.NONCANONICAL_SET, "USTAR size mismatch") + return encoded diff --git a/proof/region/v1/build/transport.py b/proof/region/v1/build/transport.py new file mode 100644 index 00000000..40f96817 --- /dev/null +++ b/proof/region/v1/build/transport.py @@ -0,0 +1,3114 @@ +#!/usr/bin/env python3 +"""Engine-neutral, causally observed Docker BUILD transport.""" + +from __future__ import annotations + +import hashlib +import json +import os +import platform +import re +import selectors +import signal +import stat +import subprocess +import tempfile +import time +from enum import StrEnum +from pathlib import Path +from typing import Callable, Protocol, TypeAlias + +from . import input + + +# These ceilings preserve the already shipped Arb V1 observer contract; they +# are not physical constants or evidence that every build fits. Changing one +# requires a new transport version, a streaming/resource design review, and a +# targeted native high-water gate. A lane policy may only tighten them. +BUILD_STDOUT_LIMIT_V1 = 16 * 1024 * 1024 +BUILD_STDERR_LIMIT_V1 = 16 * 1024 * 1024 +BUILD_TIMEOUT_NS_V1 = 2 * 60 * 60 * 1_000_000_000 +DOCKER_PROBE_OUTPUT_LIMIT_V1 = 1024 * 1024 +DOCKER_PROBE_TIMEOUT_NS_V1 = 30 * 1_000_000_000 + +# These are observer scheduling/termination mechanics retained from Arb V1, +# not successful-build evidence coordinates. CPU, RAM and PID containment is +# owned by the declared disposable worker, outside this Docker transport. +_IO_CHUNK_BYTES_V1 = 64 * 1024 +_POLL_SLICE_SECONDS_V1 = 0.1 +_PROCESS_STOP_TIMEOUT_SECONDS_V1 = 30 +_PATH_TYPE = type(Path("/")) + +_BUILD_INPUT_PROGRESS_TOKEN = object() +_BUILD_INPUT_TRANSFER_TOKEN = object() +_DOCKER_COMMAND_EXITED_TOKEN = object() +_DOCKER_BUILD_EXITED_TOKEN = object() +_BUILD_CLEANUP_FAILURE_TOKEN = object() +_BUILD_SESSION_TOKEN = object() +_TWO_BUILD_OBSERVATION_TOKEN = object() + + +def _valid_digest(value: object) -> bool: + return type(value) is bytes and len(value) == 32 and value != bytes(32) + + +def _blob(value: bytes) -> bytes: + return len(value).to_bytes(8, "big") + value + + +def _identity(label: bytes, chunks: tuple[bytes, ...]) -> bytes: + payload = b"".join(_blob(chunk) for chunk in chunks) + return hashlib.sha256(label + len(payload).to_bytes(8, "big") + payload).digest() + + +def _pinned_image_reference(value: object) -> bool: + if type(value) is not str or value.count("@sha256:") != 1: + return False + repository, digest = value.split("@sha256:", 1) + components = repository.split("/") + if any(not component for component in components): + return False + first, *path_components = components + if ":" in first: + domain, separator, port = first.rpartition(":") + if ( + not separator + or not domain + or not port + or any(character not in "0123456789" for character in port) + ): + return False + first = domain + repository_component = re.compile( + r"[a-z0-9]+(?:[._-]+[a-z0-9]+)*\Z" + ) + return ( + bool(repository) + and repository[0].isalnum() + and repository[-1].isalnum() + and repository == repository.lower() + and repository_component.fullmatch(first) is not None + and all( + repository_component.fullmatch(component) is not None + for component in path_components + ) + and len(digest) == 64 + and all(character in "0123456789abcdef" for character in digest) + ) + + +def _encoded_policy_text( + value: object, + maximum: int, + field_name: str, + *, + allow_newlines: bool = False, +) -> str: + if type(value) is not str or not value or "\0" in value: + raise TypeError(f"invalid {field_name}") + try: + encoded = value.encode("utf-8") + except UnicodeEncodeError as error: + raise TypeError(f"invalid {field_name}") from error + if ( + len(encoded) > maximum + or not allow_newlines and ("\n" in value or "\r" in value) + ): + raise TypeError(f"invalid {field_name}") + return value + + +class DockerUserModeV1(StrEnum): + """Declare which unsealed-host user coordinates Docker observes.""" + + HOST_EFFECTIVE_IDS = "host_effective_ids" + + +class DockerBuildPolicyV1(tuple): + """Deeply immutable coordinates for one bounded Docker build transport.""" + + __slots__ = () + + def __new__( + cls, + image_reference: str, + platform: str, + hostname: str, + container_name_prefix: str, + bootstrap: str, + bootstrap_argv0: str, + tmpfs_specs: tuple[str, ...], + user_mode: DockerUserModeV1, + stdout_limit: int, + stderr_limit: int, + build_timeout_ns: int, + probe_output_limit: int, + probe_timeout_ns: int, + ) -> DockerBuildPolicyV1: + strings = tuple( + _encoded_policy_text( + value, + maximum, + field_name, + allow_newlines=field_name == "bootstrap", + ) + for field_name, value, maximum in ( + ("image_reference", image_reference, 512), + ("platform", platform, 64), + ("hostname", hostname, 64), + ("container_name_prefix", container_name_prefix, 64), + ("bootstrap", bootstrap, 64 * 1024), + ("bootstrap_argv0", bootstrap_argv0, 128), + ) + ) + ( + image_reference, + platform, + hostname, + container_name_prefix, + bootstrap, + bootstrap_argv0, + ) = strings + if ( + platform != "linux/amd64" + or not _pinned_image_reference(image_reference) + ): + raise TypeError("policy requires one pinned linux/amd64 image") + if ( + any( + character not in "abcdefghijklmnopqrstuvwxyz0123456789-" + for character in hostname + ) + or any( + character not in "abcdefghijklmnopqrstuvwxyz0123456789-" + for character in container_name_prefix + ) + or not container_name_prefix.endswith("-") + ): + raise TypeError("invalid Docker names") + if type(tmpfs_specs) is not tuple or not tmpfs_specs: + raise TypeError("invalid tmpfs_specs") + owned_tmpfs: list[str] = [] + for spec in tmpfs_specs: + parsed = _encoded_policy_text(spec, 4096, "tmpfs_specs") + if not parsed.startswith("/"): + raise TypeError("invalid tmpfs_specs") + owned_tmpfs.append(parsed) + tmpfs_specs = tuple(owned_tmpfs) + if len(set(tmpfs_specs)) != len(tmpfs_specs): + raise TypeError("invalid tmpfs_specs") + if type(user_mode) is not DockerUserModeV1: + raise TypeError("invalid Docker user mode") + limits = ( + (stdout_limit, BUILD_STDOUT_LIMIT_V1, "stdout_limit"), + (stderr_limit, BUILD_STDERR_LIMIT_V1, "stderr_limit"), + (build_timeout_ns, BUILD_TIMEOUT_NS_V1, "build_timeout_ns"), + (probe_output_limit, DOCKER_PROBE_OUTPUT_LIMIT_V1, "probe_output_limit"), + (probe_timeout_ns, DOCKER_PROBE_TIMEOUT_NS_V1, "probe_timeout_ns"), + ) + if any( + type(value) is not int or value <= 0 or value > maximum + for value, maximum, _name in limits + ): + raise TypeError("invalid Docker policy limit") + return tuple.__new__( + cls, + ( + image_reference, + platform, + hostname, + container_name_prefix, + bootstrap, + bootstrap_argv0, + tmpfs_specs, + user_mode, + stdout_limit, + stderr_limit, + build_timeout_ns, + probe_output_limit, + probe_timeout_ns, + ), + ) + + @property + def image_reference(self) -> str: + return self[0] + + @property + def platform(self) -> str: + return self[1] + + @property + def hostname(self) -> str: + return self[2] + + @property + def container_name_prefix(self) -> str: + return self[3] + + @property + def bootstrap(self) -> str: + return self[4] + + @property + def bootstrap_argv0(self) -> str: + return self[5] + + @property + def tmpfs_specs(self) -> tuple[str, ...]: + return self[6] + + @property + def user_mode(self) -> DockerUserModeV1: + return self[7] + + @property + def stdout_limit(self) -> int: + return self[8] + + @property + def stderr_limit(self) -> int: + return self[9] + + @property + def build_timeout_ns(self) -> int: + return self[10] + + @property + def probe_output_limit(self) -> int: + return self[11] + + @property + def probe_timeout_ns(self) -> int: + return self[12] + + +def docker_policy_is_valid_v1(value: object) -> bool: + if type(value) is not DockerBuildPolicyV1: + return False + try: + return tuple(DockerBuildPolicyV1(*tuple(value))) == tuple(value) + except Exception: + return False + + +def transport_policy_identity_v1(policy: DockerBuildPolicyV1) -> bytes: + """Bind every declared transport-policy coordinate in constructor order.""" + + if not docker_policy_is_valid_v1(policy): + raise TypeError("policy must be canonical DockerBuildPolicyV1") + return _identity( + b"labcolors.proof-region.docker-transport-policy.v1\0", + ( + policy.image_reference.encode("utf-8"), + policy.platform.encode("utf-8"), + policy.hostname.encode("utf-8"), + policy.container_name_prefix.encode("utf-8"), + policy.bootstrap.encode("utf-8"), + policy.bootstrap_argv0.encode("utf-8"), + len(policy.tmpfs_specs).to_bytes(4, "big"), + *(spec.encode("utf-8") for spec in policy.tmpfs_specs), + policy.user_mode.value.encode("ascii"), + policy.stdout_limit.to_bytes(8, "big"), + policy.stderr_limit.to_bytes(8, "big"), + policy.build_timeout_ns.to_bytes(8, "big"), + policy.probe_output_limit.to_bytes(8, "big"), + policy.probe_timeout_ns.to_bytes(8, "big"), + ), + ) + + +class _NativeCommandSlotV1(StrEnum): + CLI_PATH = "cli_path" + IMAGE_REFERENCE = "image_reference" + PLATFORM = "platform" + ORDERED_TMPFS_SPECS = "ordered_tmpfs_specs" + CONTAINER_NAME = "container_name" + HOSTNAME = "hostname" + HOST_USER = "host_user" + CID_FILE = "cid_file" + BOOTSTRAP = "bootstrap" + BOOTSTRAP_ARGV0 = "bootstrap_argv0" + INPUT_LENGTH = "input_length" + INPUT_SHA256 = "input_sha256" + CONTAINER_COORDINATE = "container_coordinate" + CONTAINER_FILTER = "container_filter" + + +class _NativeCommandTokenV1(tuple): + """One tagged literal or named slot in the native command grammar.""" + + __slots__ = () + + def __new__( + cls, + literal: str | None = None, + slot: _NativeCommandSlotV1 | None = None, + ) -> _NativeCommandTokenV1: + if (literal is None) == (slot is None): + raise TypeError("command token must be exactly one literal or slot") + if literal is not None: + if type(literal) is not str or not literal or "\0" in literal: + raise TypeError("invalid native command literal") + return tuple.__new__(cls, (b"literal", literal)) + if type(slot) is not _NativeCommandSlotV1: + raise TypeError("invalid native command slot") + return tuple.__new__(cls, (b"slot", slot)) + + @property + def tag(self) -> bytes: + return self[0] + + @property + def value(self) -> str | _NativeCommandSlotV1: + return self[1] + + +def _literal_v1(value: str) -> _NativeCommandTokenV1: + return _NativeCommandTokenV1(literal=value) + + +def _slot_v1(value: _NativeCommandSlotV1) -> _NativeCommandTokenV1: + return _NativeCommandTokenV1(slot=value) + + +_NATIVE_COMMAND_TEMPLATES_V1: tuple[ + tuple[str, tuple[_NativeCommandTokenV1, ...]], ... +] = ( + ( + "version_probe", + ( + _slot_v1(_NativeCommandSlotV1.CLI_PATH), + _literal_v1("version"), + _literal_v1("--format"), + _literal_v1("{{json .Server}}"), + ), + ), + ( + "image_inspect", + ( + _slot_v1(_NativeCommandSlotV1.CLI_PATH), + _literal_v1("image"), + _literal_v1("inspect"), + _slot_v1(_NativeCommandSlotV1.IMAGE_REFERENCE), + ), + ), + ( + "build", + ( + _slot_v1(_NativeCommandSlotV1.CLI_PATH), + _literal_v1("run"), + _literal_v1("--rm"), + _literal_v1("--interactive"), + _literal_v1("--pull"), + _literal_v1("never"), + _literal_v1("--platform"), + _slot_v1(_NativeCommandSlotV1.PLATFORM), + _literal_v1("--network"), + _literal_v1("none"), + _literal_v1("--read-only"), + _literal_v1("--tmpfs"), + _slot_v1(_NativeCommandSlotV1.ORDERED_TMPFS_SPECS), + _literal_v1("--cap-drop"), + _literal_v1("ALL"), + _literal_v1("--security-opt"), + _literal_v1("no-new-privileges:true"), + _literal_v1("--name"), + _slot_v1(_NativeCommandSlotV1.CONTAINER_NAME), + _literal_v1("--hostname"), + _slot_v1(_NativeCommandSlotV1.HOSTNAME), + _literal_v1("--user"), + _slot_v1(_NativeCommandSlotV1.HOST_USER), + _literal_v1("--workdir"), + _literal_v1("/"), + _literal_v1("--cidfile"), + _slot_v1(_NativeCommandSlotV1.CID_FILE), + _literal_v1("--entrypoint"), + _literal_v1("/usr/bin/env"), + _slot_v1(_NativeCommandSlotV1.IMAGE_REFERENCE), + _literal_v1("-i"), + _literal_v1("PATH=/usr/local/bin:/usr/bin:/bin"), + _literal_v1("LC_ALL=C"), + _literal_v1("LANG=C"), + _literal_v1("TZ=UTC"), + _literal_v1("HOME=/nonexistent"), + _literal_v1("/bin/sh"), + _literal_v1("-c"), + _slot_v1(_NativeCommandSlotV1.BOOTSTRAP), + _slot_v1(_NativeCommandSlotV1.BOOTSTRAP_ARGV0), + _slot_v1(_NativeCommandSlotV1.INPUT_LENGTH), + _slot_v1(_NativeCommandSlotV1.INPUT_SHA256), + ), + ), + ( + "cleanup_rm", + ( + _slot_v1(_NativeCommandSlotV1.CLI_PATH), + _literal_v1("container"), + _literal_v1("rm"), + _literal_v1("--force"), + _slot_v1(_NativeCommandSlotV1.CONTAINER_COORDINATE), + ), + ), + ( + "cleanup_ls", + ( + _slot_v1(_NativeCommandSlotV1.CLI_PATH), + _literal_v1("container"), + _literal_v1("ls"), + _literal_v1("--all"), + _literal_v1("--quiet"), + _literal_v1("--no-trunc"), + _literal_v1("--filter"), + _slot_v1(_NativeCommandSlotV1.CONTAINER_FILTER), + ), + ), +) + + +def native_command_contract_identity_v1() -> bytes: + chunks: list[bytes] = [len(_NATIVE_COMMAND_TEMPLATES_V1).to_bytes(4, "big")] + for name, tokens in _NATIVE_COMMAND_TEMPLATES_V1: + chunks.extend((name.encode("ascii"), len(tokens).to_bytes(4, "big"))) + for token in tokens: + value = token.value + chunks.extend( + ( + token.tag, + ( + value.value.encode("ascii") + if type(value) is _NativeCommandSlotV1 + else value.encode("utf-8") + ), + ) + ) + return _identity( + b"labcolors.proof-region.native-command-contract.v1\0", + tuple(chunks), + ) + + +def _native_command_path_v1(value: object) -> tuple[Path, bytes]: + if type(value) is not _PATH_TYPE or not value.is_absolute(): + raise TypeError("native command path must be an absolute Path") + try: + encoded = os.fsencode(value) + except (TypeError, UnicodeEncodeError) as error: + raise TypeError("native command path is not filesystem-encodable") from error + if not encoded or b"\0" in encoded: + raise TypeError("invalid native command path bytes") + return value, encoded + + +class NativeCommandCoordinateV1(tuple): + """Exact filesystem command coordinate bound to the native argv grammar.""" + + __slots__ = () + + def __new__( + cls, + path_bytes: bytes, + command_contract_identity: bytes, + ) -> NativeCommandCoordinateV1: + if type(path_bytes) is not bytes or not path_bytes or b"\0" in path_bytes: + raise TypeError("invalid native command path bytes") + try: + path = Path(os.fsdecode(path_bytes)) + except (TypeError, UnicodeDecodeError) as error: + raise TypeError("invalid native command path bytes") from error + _owned_path, encoded = _native_command_path_v1(path) + if encoded != path_bytes: + raise TypeError("native command path bytes are not canonical") + if command_contract_identity != native_command_contract_identity_v1(): + raise TypeError("foreign native command contract") + return tuple.__new__(cls, (path_bytes, command_contract_identity)) + + @property + def path_bytes(self) -> bytes: + return self[0] + + @property + def path(self) -> Path: + return Path(os.fsdecode(self.path_bytes)) + + @property + def command_contract_identity(self) -> bytes: + return self[1] + + @property + def identity(self) -> bytes: + return _native_command_coordinate_identity_v1(self) + + +def native_command_coordinate_v1(path: Path) -> NativeCommandCoordinateV1: + _owned, encoded = _native_command_path_v1(path) + return NativeCommandCoordinateV1( + encoded, + native_command_contract_identity_v1(), + ) + + +def _native_command_coordinate_identity_v1( + coordinate: NativeCommandCoordinateV1, +) -> bytes: + if type(coordinate) is not NativeCommandCoordinateV1: + raise TypeError("coordinate must be NativeCommandCoordinateV1") + canonical = NativeCommandCoordinateV1(*tuple(coordinate)) + if tuple(canonical) != tuple(coordinate): + raise TypeError("coordinate is not canonical") + return _identity( + b"labcolors.proof-region.native-command-coordinate.v1\0", + ( + coordinate.command_contract_identity, + coordinate.path_bytes, + ), + ) + + +def _render_native_command_v1( + template_name: str, + command_coordinate: NativeCommandCoordinateV1, + values: dict[_NativeCommandSlotV1, tuple[str, ...]], +) -> tuple[str, ...]: + if type(template_name) is not str or type(values) is not dict: + raise TypeError("invalid native command expansion") + canonical_coordinate = NativeCommandCoordinateV1(*tuple(command_coordinate)) + templates = dict(_NATIVE_COMMAND_TEMPLATES_V1) + try: + tokens = templates[template_name] + except KeyError as error: + raise TypeError("unknown native command template") from error + owned_values = dict(values) + if _NativeCommandSlotV1.CLI_PATH in owned_values: + raise TypeError("native command path is owned by its coordinate") + owned_values[_NativeCommandSlotV1.CLI_PATH] = ( + os.fsdecode(canonical_coordinate.path_bytes), + ) + expected_slots = { + token.value + for token in tokens + if token.tag == b"slot" + } + if set(owned_values) != expected_slots: + raise TypeError("native command slots do not match its template") + command: list[str] = [] + for token in tokens: + if token.tag == b"literal": + command.append(token.value) + continue + slot = token.value + expanded = owned_values[slot] + if ( + type(expanded) is not tuple + or ( + slot is not _NativeCommandSlotV1.ORDERED_TMPFS_SPECS + and len(expanded) != 1 + ) + or any( + type(value) is not str or not value or "\0" in value + for value in expanded + ) + ): + raise TypeError("invalid native command slot expansion") + if slot is _NativeCommandSlotV1.ORDERED_TMPFS_SPECS: + if not expanded or not command: + raise TypeError("invalid ordered tmpfs template") + repeated_literal = command.pop() + for value in expanded: + command.extend((repeated_literal, value)) + continue + command.extend(expanded) + try: + tuple(os.fsencode(value) for value in command) + except (TypeError, UnicodeEncodeError) as error: + raise TypeError("native command contains an unencodable coordinate") from error + return tuple(command) + + +class DockerBlockerReasonV1(StrEnum): + HOST_NOT_LINUX_AMD64 = "host_not_linux_amd64" + DOCKER_UNAVAILABLE = "docker_unavailable" + IMAGE_UNAVAILABLE = "image_unavailable" + IMAGE_IDENTITY_MISMATCH = "image_identity_mismatch" + BACKEND_CONTRACT = "backend_contract" + + +class DockerUnsupportedV1(tuple): + __slots__ = () + + def __new__( + cls, + reason: DockerBlockerReasonV1, + detail: str, + ) -> DockerUnsupportedV1: + if type(reason) is not DockerBlockerReasonV1: + raise TypeError("invalid Docker blocker reason") + if type(detail) is not str or not detail or len(detail) > 4096: + raise TypeError("invalid Docker blocker detail") + return tuple.__new__(cls, (reason, detail)) + + @property + def reason(self) -> DockerBlockerReasonV1: + return self[0] + + @property + def detail(self) -> str: + return self[1] + + +class DockerDaemonObservationV1(tuple): + """Exact stdout bytes observed from the two admitted Docker probes.""" + + __slots__ = () + + def __new__( + cls, + server_stdout: bytes, + image_inspect_stdout: bytes, + ) -> DockerDaemonObservationV1: + for value, field_name in ( + (server_stdout, "server_stdout"), + (image_inspect_stdout, "image_inspect_stdout"), + ): + if ( + type(value) is not bytes + or not value + or len(value) > DOCKER_PROBE_OUTPUT_LIMIT_V1 + ): + raise TypeError(f"invalid Docker daemon {field_name}") + return tuple.__new__(cls, (server_stdout, image_inspect_stdout)) + + @property + def server_stdout(self) -> bytes: + return self[0] + + @property + def image_inspect_stdout(self) -> bytes: + return self[1] + + @property + def identity(self) -> bytes: + return _docker_daemon_observation_identity_v1(self) + + +def _docker_daemon_observation_identity_v1( + observation: DockerDaemonObservationV1, +) -> bytes: + if type(observation) is not DockerDaemonObservationV1: + raise TypeError("observation must be DockerDaemonObservationV1") + canonical = DockerDaemonObservationV1(*tuple(observation)) + if tuple(canonical) != tuple(observation): + raise TypeError("daemon observation is not canonical") + return _identity( + b"labcolors.proof-region.docker-daemon-observation.v1\0", + ( + observation.server_stdout, + observation.image_inspect_stdout, + ), + ) + + +def _host_user_identity_v1(host_user: tuple[int, int]) -> bytes: + owned = _host_user_coordinates(host_user) + return _identity( + b"labcolors.proof-region.host-user.v1\0", + ( + owned[0].to_bytes(4, "big"), + owned[1].to_bytes(4, "big"), + ), + ) + + +class DockerSupportedV1(tuple): + """Canonical capability observed for one exact native Docker coordinate.""" + + __slots__ = () + + def __new__( + cls, + policy: DockerBuildPolicyV1, + daemon_observation: DockerDaemonObservationV1, + command_coordinate: NativeCommandCoordinateV1, + host_user: tuple[int, int], + ) -> DockerSupportedV1: + if not docker_policy_is_valid_v1(policy): + raise TypeError("invalid Docker policy capability") + if type(daemon_observation) is not DockerDaemonObservationV1: + raise TypeError("invalid Docker daemon observation") + canonical_daemon = DockerDaemonObservationV1(*tuple(daemon_observation)) + if ( + tuple(canonical_daemon) != tuple(daemon_observation) + or len(canonical_daemon.server_stdout) > policy.probe_output_limit + or len(canonical_daemon.image_inspect_stdout) + > policy.probe_output_limit + ): + raise TypeError("Docker daemon observation is not canonical") + if type(command_coordinate) is not NativeCommandCoordinateV1: + raise TypeError("invalid native Docker command coordinate") + canonical_command = NativeCommandCoordinateV1(*tuple(command_coordinate)) + if tuple(canonical_command) != tuple(command_coordinate): + raise TypeError("native Docker command coordinate is not canonical") + owned_user = _host_user_coordinates(host_user) + return tuple.__new__( + cls, + (policy, daemon_observation, command_coordinate, owned_user), + ) + + @property + def policy(self) -> DockerBuildPolicyV1: + return self[0] + + @property + def daemon_observation(self) -> DockerDaemonObservationV1: + return self[1] + + @property + def command_coordinate(self) -> NativeCommandCoordinateV1: + return self[2] + + @property + def host_user(self) -> tuple[int, int]: + return self[3] + + @property + def policy_identity(self) -> bytes: + return transport_policy_identity_v1(self.policy) + + @property + def daemon_observation_identity(self) -> bytes: + return _docker_daemon_observation_identity_v1(self.daemon_observation) + + @property + def command_coordinate_identity(self) -> bytes: + return _native_command_coordinate_identity_v1(self.command_coordinate) + + @property + def host_user_identity(self) -> bytes: + return _host_user_identity_v1(self.host_user) + + @property + def identity(self) -> bytes: + return docker_capability_identity_v1(self) + + +def _docker_supported_is_valid_v1(value: object) -> bool: + if type(value) is not DockerSupportedV1: + return False + try: + return tuple(DockerSupportedV1(*tuple(value))) == tuple(value) + except Exception: + return False + + +def docker_capability_identity_v1(capability: DockerSupportedV1) -> bytes: + if not _docker_supported_is_valid_v1(capability): + raise TypeError("capability must be canonical DockerSupportedV1") + return _identity( + b"labcolors.proof-region.docker-capability.v1\0", + ( + capability.policy_identity, + capability.command_coordinate_identity, + capability.daemon_observation_identity, + capability.host_user[0].to_bytes(4, "big"), + capability.host_user[1].to_bytes(4, "big"), + ), + ) + + +DockerCapabilityReportV1: TypeAlias = DockerSupportedV1 | DockerUnsupportedV1 + + +def _docker_unsupported_is_valid_v1(value: object) -> bool: + if type(value) is not DockerUnsupportedV1: + return False + try: + return tuple(DockerUnsupportedV1(*tuple(value))) == tuple(value) + except Exception: + return False + + +def _absolute_path(value: object, field_name: str) -> Path: + if type(value) is not _PATH_TYPE or not value.is_absolute(): + raise TypeError(f"{field_name} must be an absolute Path") + try: + encoded = os.fsencode(value) + except (TypeError, UnicodeEncodeError) as error: + raise TypeError(f"{field_name} is not filesystem-encodable") from error + if ( + not encoded + or b"\0" in encoded + or any(character in str(value) for character in (",", "\n", "\r")) + ): + raise TypeError(f"{field_name} is not Docker-mount-safe") + return value + + +def _host_user_coordinates(value: object) -> tuple[int, int]: + if ( + type(value) is not tuple + or len(value) != 2 + or any(type(item) is not int or item < 0 or item >= 1 << 32 for item in value) + ): + raise TypeError("host_user must be one exact Linux uid/gid pair") + return value + + +def _container_name(value: object, prefix: str) -> str: + if ( + type(value) is not str + or type(prefix) is not str + or not value.startswith(prefix) + or len(value) > 128 + or any(character not in "abcdefghijklmnopqrstuvwxyz0123456789-" for character in value) + ): + raise TypeError("invalid controller-owned Docker container name") + return value + + +class DockerBuildRequestV1(tuple): + __slots__ = () + + def __new__( + cls, + attempt: int, + capability: DockerSupportedV1, + input_bundle: input.SealedInputV1, + max_output_bytes: int, + cid_file: Path, + container_name: str, + ) -> DockerBuildRequestV1: + if type(attempt) is not int or attempt not in (1, 2): + raise TypeError("attempt must be 1 or 2") + if not _docker_supported_is_valid_v1(capability): + raise TypeError("capability must be canonical DockerSupportedV1") + if not input.sealed_input_is_intact_v1(input_bundle): + raise TypeError("input_bundle must preserve exact sealed bytes") + if ( + type(max_output_bytes) is not int + or max_output_bytes <= 0 + or max_output_bytes > capability.policy.stdout_limit + ): + raise TypeError("invalid executable output limit") + _absolute_path(cid_file, "cid_file") + _container_name(container_name, capability.policy.container_name_prefix) + return tuple.__new__( + cls, + ( + attempt, + capability, + input_bundle, + max_output_bytes, + cid_file, + container_name, + ), + ) + + @property + def attempt(self) -> int: + return self[0] + + @property + def capability(self) -> DockerSupportedV1: + return self[1] + + @property + def input_bundle(self) -> input.SealedInputV1: + return self[2] + + @property + def max_output_bytes(self) -> int: + return self[3] + + @property + def cid_file(self) -> Path: + return self[4] + + @property + def container_name(self) -> str: + return self[5] + + +def _docker_build_request_is_valid_v1( + value: object, + capability: DockerSupportedV1, +) -> bool: + if ( + type(value) is not DockerBuildRequestV1 + or not _docker_supported_is_valid_v1(capability) + ): + return False + try: + canonical = DockerBuildRequestV1(*tuple(value)) + return ( + tuple(canonical) == tuple(value) + and canonical.capability == capability + and input.sealed_input_is_intact_v1(canonical.input_bundle) + ) + except Exception: + return False + + +def _bounded_bytes(value: object, maximum: int, field_name: str) -> bytes: + if type(value) is not bytes or len(value) > maximum: + raise TypeError(f"invalid {field_name}") + return value + + +class BuildInputTransferProgressV1(tuple): + __slots__ = () + + def __new__( + cls, + bundle_identity: bytes, + expected_length: int, + expected_sha256: bytes, + written_length: int, + written_sha256: bytes, + *, + _token: object, + ) -> BuildInputTransferProgressV1: + if _token is not _BUILD_INPUT_PROGRESS_TOKEN: + raise TypeError("build input progress is controller-observed") + if not _valid_digest(bundle_identity) or not _valid_digest(expected_sha256): + raise TypeError("invalid build input progress coordinates") + if ( + type(expected_length) is not int + or expected_length <= 0 + or expected_length >= 1 << 64 + or type(written_length) is not int + or written_length < 0 + or written_length >= 1 << 64 + or written_length > expected_length + or type(written_sha256) is not bytes + or len(written_sha256) != 32 + ): + raise TypeError("invalid build input progress") + return tuple.__new__( + cls, + ( + bundle_identity, + expected_length, + expected_sha256, + written_length, + written_sha256, + ), + ) + + @property + def bundle_identity(self) -> bytes: + return self[0] + + @property + def expected_length(self) -> int: + return self[1] + + @property + def expected_sha256(self) -> bytes: + return self[2] + + @property + def written_length(self) -> int: + return self[3] + + @property + def written_sha256(self) -> bytes: + return self[4] + + +def _build_input_progress_v1( + bundle: input.SealedInputV1, + written_length: int, + written_sha256: bytes, +) -> BuildInputTransferProgressV1: + if not input.sealed_input_is_intact_v1(bundle): + raise TypeError("build input bytes are not intact") + if ( + type(written_length) is not int + or written_length < 0 + or written_length > bundle.length + or type(written_sha256) is not bytes + or written_sha256 + != hashlib.sha256(bundle.contents[:written_length]).digest() + ): + raise TypeError("build input progress does not match the sealed bytes") + return BuildInputTransferProgressV1( + bundle.binding_identity, + bundle.length, + bundle.sha256, + written_length, + written_sha256, + _token=_BUILD_INPUT_PROGRESS_TOKEN, + ) + + +def _input_progress_matches_v1( + value: object, + bundle: input.SealedInputV1, +) -> bool: + if ( + type(value) is not BuildInputTransferProgressV1 + or not input.sealed_input_is_intact_v1(bundle) + ): + return False + try: + canonical = _build_input_progress_v1( + bundle, + value.written_length, + value.written_sha256, + ) + return tuple(canonical) == tuple(value) + except Exception: + return False + + +class BuildInputTransferV1(tuple): + __slots__ = () + + def __new__( + cls, + progress: BuildInputTransferProgressV1, + *, + _token: object, + ) -> BuildInputTransferV1: + if _token is not _BUILD_INPUT_TRANSFER_TOKEN: + raise TypeError("build input transfer is controller-observed") + if ( + type(progress) is not BuildInputTransferProgressV1 + or progress.written_length != progress.expected_length + or progress.written_sha256 != progress.expected_sha256 + ): + raise TypeError("completed build input transfer must be exact") + return tuple.__new__(cls, tuple(progress)) + + @property + def bundle_identity(self) -> bytes: + return self[0] + + @property + def expected_length(self) -> int: + return self[1] + + @property + def expected_sha256(self) -> bytes: + return self[2] + + @property + def written_length(self) -> int: + return self[3] + + @property + def written_sha256(self) -> bytes: + return self[4] + + +def _input_transfer_is_structurally_valid_v1(value: object) -> bool: + if type(value) is not BuildInputTransferV1: + return False + try: + return ( + len(value) == 5 + and _valid_digest(value.bundle_identity) + and type(value.expected_length) is int + and 0 < value.expected_length < 1 << 64 + and _valid_digest(value.expected_sha256) + and value.written_length == value.expected_length + and value.written_sha256 == value.expected_sha256 + ) + except Exception: + return False + + +def _completed_build_input_transfer_v1( + bundle: input.SealedInputV1, + written_length: int, + written_sha256: bytes, +) -> BuildInputTransferV1: + progress = _build_input_progress_v1(bundle, written_length, written_sha256) + return BuildInputTransferV1( + progress, + _token=_BUILD_INPUT_TRANSFER_TOKEN, + ) + + +class _DockerCommandExitedV1(tuple): + __slots__ = () + + def __new__( + cls, + returncode: int, + stdout: bytes, + stderr: bytes, + *, + _token: object, + ) -> _DockerCommandExitedV1: + if _token is not _DOCKER_COMMAND_EXITED_TOKEN: + raise TypeError("Docker command exit is controller-observed") + if type(returncode) is not int or not -(1 << 31) <= returncode < 1 << 31: + raise TypeError("invalid Docker returncode") + _bounded_bytes(stdout, BUILD_STDOUT_LIMIT_V1, "stdout") + _bounded_bytes(stderr, BUILD_STDERR_LIMIT_V1, "stderr") + return tuple.__new__(cls, (returncode, stdout, stderr)) + + @property + def returncode(self) -> int: + return self[0] + + @property + def stdout(self) -> bytes: + return self[1] + + @property + def stderr(self) -> bytes: + return self[2] + + +def _docker_command_exited_v1( + returncode: int, + stdout: bytes, + stderr: bytes, +) -> _DockerCommandExitedV1: + return _DockerCommandExitedV1( + returncode, + stdout, + stderr, + _token=_DOCKER_COMMAND_EXITED_TOKEN, + ) + + +class DockerBuildExitedV1(tuple): + __slots__ = () + + def __new__( + cls, + returncode: int, + stdout: bytes, + stderr: bytes, + input_transfer: BuildInputTransferV1, + *, + _token: object, + ) -> DockerBuildExitedV1: + if _token is not _DOCKER_BUILD_EXITED_TOKEN: + raise TypeError("Docker build exit is controller-observed") + if type(returncode) is not int or not -(1 << 31) <= returncode < 1 << 31: + raise TypeError("invalid Docker returncode") + _bounded_bytes(stdout, BUILD_STDOUT_LIMIT_V1, "stdout") + _bounded_bytes(stderr, BUILD_STDERR_LIMIT_V1, "stderr") + if not _input_transfer_is_structurally_valid_v1(input_transfer): + raise TypeError("invalid Docker build input transfer") + return tuple.__new__( + cls, + (returncode, stdout, stderr, input_transfer), + ) + + @property + def returncode(self) -> int: + return self[0] + + @property + def stdout(self) -> bytes: + return self[1] + + @property + def stderr(self) -> bytes: + return self[2] + + @property + def input_transfer(self) -> BuildInputTransferV1: + return self[3] + + +def _docker_build_exited_v1( + returncode: int, + stdout: bytes, + stderr: bytes, + input_transfer: BuildInputTransferV1, +) -> DockerBuildExitedV1: + return DockerBuildExitedV1( + returncode, + stdout, + stderr, + input_transfer, + _token=_DOCKER_BUILD_EXITED_TOKEN, + ) + + +def docker_build_exited_is_valid_v1( + value: object, + input_value: input.SealedInputV1, + max_output_bytes: int, + max_stderr_bytes: int, +) -> bool: + if ( + type(value) is not DockerBuildExitedV1 + or not input.sealed_input_is_intact_v1(input_value) + or type(max_output_bytes) is not int + or max_output_bytes <= 0 + or type(max_stderr_bytes) is not int + or max_stderr_bytes <= 0 + ): + return False + try: + return ( + len(value) == 4 + and type(value.returncode) is int + and -(1 << 31) <= value.returncode < 1 << 31 + and type(value.stdout) is bytes + and len(value.stdout) <= max_output_bytes + and type(value.stderr) is bytes + and len(value.stderr) <= max_stderr_bytes + and _input_transfer_is_structurally_valid_v1(value.input_transfer) + and value.input_transfer.bundle_identity + == input_value.binding_identity + and value.input_transfer.expected_length == input_value.length + and value.input_transfer.expected_sha256 == input_value.sha256 + and value.input_transfer.written_length == input_value.length + and value.input_transfer.written_sha256 == input_value.sha256 + ) + except Exception: + return False + + +class DockerBuildTimedOutV1(tuple): + __slots__ = () + + def __new__( + cls, + stdout: bytes, + stderr: bytes, + input_progress: BuildInputTransferProgressV1 | None = None, + ) -> DockerBuildTimedOutV1: + _bounded_bytes(stdout, BUILD_STDOUT_LIMIT_V1, "stdout") + _bounded_bytes(stderr, BUILD_STDERR_LIMIT_V1, "stderr") + if input_progress is not None and type( + input_progress + ) is not BuildInputTransferProgressV1: + raise TypeError("invalid timed-out build input progress") + return tuple.__new__(cls, (stdout, stderr, input_progress)) + + @property + def stdout(self) -> bytes: + return self[0] + + @property + def stderr(self) -> bytes: + return self[1] + + @property + def input_progress(self) -> BuildInputTransferProgressV1 | None: + return self[2] + + +class DockerOutputStreamV1(StrEnum): + STDOUT = "stdout" + STDERR = "stderr" + + +class DockerBuildOutputLimitV1(tuple): + __slots__ = () + + def __new__( + cls, + stream: DockerOutputStreamV1, + stdout: bytes, + stderr: bytes, + input_progress: BuildInputTransferProgressV1 | None = None, + ) -> DockerBuildOutputLimitV1: + if type(stream) is not DockerOutputStreamV1: + raise TypeError("invalid Docker output stream") + _bounded_bytes(stdout, BUILD_STDOUT_LIMIT_V1, "stdout") + _bounded_bytes(stderr, BUILD_STDERR_LIMIT_V1, "stderr") + if input_progress is not None and type( + input_progress + ) is not BuildInputTransferProgressV1: + raise TypeError("invalid output-limited build input progress") + return tuple.__new__(cls, (stream, stdout, stderr, input_progress)) + + @property + def stream(self) -> DockerOutputStreamV1: + return self[0] + + @property + def stdout(self) -> bytes: + return self[1] + + @property + def stderr(self) -> bytes: + return self[2] + + @property + def input_progress(self) -> BuildInputTransferProgressV1 | None: + return self[3] + + +class DockerBuildObserverFailureV1(tuple): + __slots__ = () + + def __new__( + cls, + detail: str, + stdout: bytes, + stderr: bytes, + input_progress: BuildInputTransferProgressV1 | None = None, + ) -> DockerBuildObserverFailureV1: + if type(detail) is not str or not detail or len(detail) > 4096: + raise TypeError("invalid Docker observer failure") + _bounded_bytes(stdout, BUILD_STDOUT_LIMIT_V1, "stdout") + _bounded_bytes(stderr, BUILD_STDERR_LIMIT_V1, "stderr") + if input_progress is not None and type( + input_progress + ) is not BuildInputTransferProgressV1: + raise TypeError("invalid observer-failure build input progress") + return tuple.__new__(cls, (detail, stdout, stderr, input_progress)) + + @property + def detail(self) -> str: + return self[0] + + @property + def stdout(self) -> bytes: + return self[1] + + @property + def stderr(self) -> bytes: + return self[2] + + @property + def input_progress(self) -> BuildInputTransferProgressV1 | None: + return self[3] + + +class DockerBuildInputRejectedV1(tuple): + __slots__ = () + + def __new__( + cls, + input_progress: BuildInputTransferProgressV1, + stdout: bytes, + stderr: bytes, + ) -> DockerBuildInputRejectedV1: + if type(input_progress) is not BuildInputTransferProgressV1: + raise TypeError("invalid partial build input progress") + _bounded_bytes(stdout, BUILD_STDOUT_LIMIT_V1, "stdout") + _bounded_bytes(stderr, BUILD_STDERR_LIMIT_V1, "stderr") + return tuple.__new__(cls, (input_progress, stdout, stderr)) + + @property + def input_progress(self) -> BuildInputTransferProgressV1: + return self[0] + + @property + def stdout(self) -> bytes: + return self[1] + + @property + def stderr(self) -> bytes: + return self[2] + + @property + def written_length(self) -> int: + return self.input_progress.written_length + + @property + def written_sha256(self) -> bytes: + return self.input_progress.written_sha256 + + +class DockerCleanupTriggerV1(StrEnum): + PROCESS_EXIT = "process_exit" + INPUT_TRANSFER = "input_transfer" + TIMEOUT = "timeout" + OUTPUT_LIMIT = "output_limit" + OBSERVER_FAILURE = "observer_failure" + + +class CleanupResourceV1(StrEnum): + DOCKER_CLI_PROCESS = "docker_cli_process" + DOCKER_CONTAINER = "docker_container" + TEMPORARY_ROOT = "temporary_root" + + +class CleanupFailureRecordV1(tuple): + __slots__ = () + + def __new__( + cls, + resource: CleanupResourceV1, + detail: str, + ) -> CleanupFailureRecordV1: + if type(resource) is not CleanupResourceV1: + raise TypeError("invalid cleanup resource") + if type(detail) is not str or not detail or len(detail) > 4096: + raise TypeError("invalid cleanup failure detail") + return tuple.__new__(cls, (resource, detail)) + + @property + def resource(self) -> CleanupResourceV1: + return self[0] + + @property + def detail(self) -> str: + return self[1] + + +def _cleanup_failure_records_v1( + value: object, + allowed_order: tuple[CleanupResourceV1, ...], +) -> tuple[CleanupFailureRecordV1, ...]: + if type(value) is not tuple or not value: + raise TypeError("cleanup failures must be one nonempty tuple") + order = {resource: index for index, resource in enumerate(allowed_order)} + owned: list[CleanupFailureRecordV1] = [] + indexes: list[int] = [] + for record in value: + if type(record) is not CleanupFailureRecordV1: + raise TypeError("cleanup failure record is not canonical") + canonical = CleanupFailureRecordV1(*tuple(record)) + if tuple(canonical) != tuple(record) or canonical.resource not in order: + raise TypeError("cleanup failure record is not canonical") + owned.append(canonical) + indexes.append(order[canonical.resource]) + if indexes != sorted(set(indexes)): + raise TypeError("cleanup failure records are not in stable resource order") + return tuple(owned) + + +class DockerBuildCleanupFailureV1(tuple): + __slots__ = () + + def __new__( + cls, + trigger: DockerCleanupTriggerV1, + failures: tuple[CleanupFailureRecordV1, ...], + stdout: bytes, + stderr: bytes, + input_progress: BuildInputTransferProgressV1 | None = None, + ) -> DockerBuildCleanupFailureV1: + if type(trigger) is not DockerCleanupTriggerV1: + raise TypeError("invalid Docker cleanup trigger") + owned_failures = _cleanup_failure_records_v1( + failures, + ( + CleanupResourceV1.DOCKER_CLI_PROCESS, + CleanupResourceV1.DOCKER_CONTAINER, + ), + ) + _bounded_bytes(stdout, BUILD_STDOUT_LIMIT_V1, "stdout") + _bounded_bytes(stderr, BUILD_STDERR_LIMIT_V1, "stderr") + if input_progress is not None and type( + input_progress + ) is not BuildInputTransferProgressV1: + raise TypeError("invalid cleanup build input progress") + return tuple.__new__( + cls, + (trigger, owned_failures, stdout, stderr, input_progress), + ) + + @property + def trigger(self) -> DockerCleanupTriggerV1: + return self[0] + + @property + def failures(self) -> tuple[CleanupFailureRecordV1, ...]: + return self[1] + + @property + def detail(self) -> str: + """Render all typed records for diagnostic-only consumers.""" + + return "; ".join(record.detail for record in self.failures) + + @property + def stdout(self) -> bytes: + return self[2] + + @property + def stderr(self) -> bytes: + return self[3] + + @property + def input_progress(self) -> BuildInputTransferProgressV1 | None: + return self[4] + + +DockerBuildProcessObservationV1: TypeAlias = ( + DockerBuildExitedV1 + | DockerBuildTimedOutV1 + | DockerBuildOutputLimitV1 + | DockerBuildObserverFailureV1 + | DockerBuildInputRejectedV1 + | DockerBuildCleanupFailureV1 +) + + +class BuildCleanupFailureV1(tuple): + """Controller-owned cleanup failure with any preceding backend observation.""" + + __slots__ = () + + def __new__( + cls, + failures: tuple[CleanupFailureRecordV1, ...], + current_process: DockerBuildProcessObservationV1 | None, + *, + _token: object, + ) -> BuildCleanupFailureV1: + if _token is not _BUILD_CLEANUP_FAILURE_TOKEN: + raise TypeError("build cleanup failure is controller-observed") + owned_failures = _cleanup_failure_records_v1( + failures, + (CleanupResourceV1.TEMPORARY_ROOT,), + ) + if current_process is not None and type(current_process) not in ( + DockerBuildExitedV1, + DockerBuildTimedOutV1, + DockerBuildOutputLimitV1, + DockerBuildObserverFailureV1, + DockerBuildInputRejectedV1, + DockerBuildCleanupFailureV1, + ): + raise TypeError("cleanup failure lost its current process observation") + return tuple.__new__(cls, (owned_failures, current_process)) + + @property + def failures(self) -> tuple[CleanupFailureRecordV1, ...]: + return self[0] + + @property + def current_process(self) -> DockerBuildProcessObservationV1 | None: + return self[1] + + +_DockerCommandObservationV1: TypeAlias = ( + _DockerCommandExitedV1 | DockerBuildProcessObservationV1 +) + + +def _canonical_progress_v1( + value: object, + input_value: input.SealedInputV1, +) -> BuildInputTransferProgressV1 | None: + if value is None: + return None + if not _input_progress_matches_v1(value, input_value): + raise TypeError("build input progress is not canonical") + return _build_input_progress_v1( + input_value, + value.written_length, + value.written_sha256, + ) + + +def _canonical_process_observation_v1( + value: object, + input_value: input.SealedInputV1, + max_output_bytes: int, + max_stderr_bytes: int, +) -> DockerBuildProcessObservationV1: + """Own a backend observation before classification or retention.""" + + if ( + not input.sealed_input_is_intact_v1(input_value) + or type(max_output_bytes) is not int + or max_output_bytes <= 0 + or type(max_stderr_bytes) is not int + or max_stderr_bytes <= 0 + ): + raise TypeError("invalid process-observation boundary") + try: + if type(value) is DockerBuildExitedV1: + if not docker_build_exited_is_valid_v1( + value, + input_value, + max_output_bytes, + max_stderr_bytes, + ): + raise TypeError("invalid exited build observation") + transfer = _completed_build_input_transfer_v1( + input_value, + value.input_transfer.written_length, + value.input_transfer.written_sha256, + ) + canonical = _docker_build_exited_v1( + value.returncode, + bytes(value.stdout), + bytes(value.stderr), + transfer, + ) + if tuple(canonical) != tuple(value): + raise TypeError("exited build observation is not canonical") + return value + if type(value) is DockerBuildTimedOutV1: + progress = _canonical_progress_v1(value.input_progress, input_value) + if progress is None: + raise TypeError("build timeout did not retain input progress") + canonical = DockerBuildTimedOutV1( + _bounded_bytes(value.stdout, max_output_bytes, "stdout"), + _bounded_bytes(value.stderr, max_stderr_bytes, "stderr"), + progress, + ) + if tuple(canonical) != tuple(value): + raise TypeError("timeout observation is not canonical") + return value + if type(value) is DockerBuildOutputLimitV1: + progress = _canonical_progress_v1(value.input_progress, input_value) + if progress is None: + raise TypeError("build output limit did not retain input progress") + stdout = _bounded_bytes(value.stdout, max_output_bytes, "stdout") + stderr = _bounded_bytes(value.stderr, max_stderr_bytes, "stderr") + if ( + ( + value.stream is DockerOutputStreamV1.STDOUT + and len(stdout) != max_output_bytes + ) + or ( + value.stream is DockerOutputStreamV1.STDERR + and len(stderr) != max_stderr_bytes + ) + ): + raise TypeError("output-limit observation did not reach its cap") + canonical = DockerBuildOutputLimitV1( + value.stream, + stdout, + stderr, + progress, + ) + if tuple(canonical) != tuple(value): + raise TypeError("output-limit observation is not canonical") + return value + if type(value) is DockerBuildObserverFailureV1: + canonical = DockerBuildObserverFailureV1( + value.detail, + _bounded_bytes(value.stdout, max_output_bytes, "stdout"), + _bounded_bytes(value.stderr, max_stderr_bytes, "stderr"), + _canonical_progress_v1(value.input_progress, input_value), + ) + if tuple(canonical) != tuple(value): + raise TypeError("observer failure is not canonical") + return value + if type(value) is DockerBuildInputRejectedV1: + progress = _canonical_progress_v1(value.input_progress, input_value) + if progress is None or progress.written_length >= progress.expected_length: + raise TypeError("input rejection did not retain partial progress") + canonical = DockerBuildInputRejectedV1( + progress, + _bounded_bytes(value.stdout, max_output_bytes, "stdout"), + _bounded_bytes(value.stderr, max_stderr_bytes, "stderr"), + ) + if tuple(canonical) != tuple(value): + raise TypeError("input rejection is not canonical") + return value + if type(value) is DockerBuildCleanupFailureV1: + progress = _canonical_progress_v1(value.input_progress, input_value) + if progress is None: + raise TypeError("build cleanup failure did not retain input progress") + canonical = DockerBuildCleanupFailureV1( + value.trigger, + value.failures, + _bounded_bytes(value.stdout, max_output_bytes, "stdout"), + _bounded_bytes(value.stderr, max_stderr_bytes, "stderr"), + progress, + ) + if tuple(canonical) != tuple(value): + raise TypeError("cleanup failure is not canonical") + return value + except (AttributeError, IndexError, TypeError, ValueError) as error: + raise TypeError("backend process observation is not canonical") from error + raise TypeError("backend returned an unknown process observation") + + +def _build_cleanup_failure_v1( + current_process: DockerBuildProcessObservationV1 | None, + detail: str, +) -> BuildCleanupFailureV1: + return BuildCleanupFailureV1( + ( + CleanupFailureRecordV1( + CleanupResourceV1.TEMPORARY_ROOT, + detail, + ), + ), + current_process, + _token=_BUILD_CLEANUP_FAILURE_TOKEN, + ) + + +def _canonical_build_cleanup_failure_v1( + value: object, + input_value: input.SealedInputV1, + max_output_bytes: int, + max_stderr_bytes: int, +) -> BuildCleanupFailureV1: + if type(value) is not BuildCleanupFailureV1: + raise TypeError("unknown build cleanup observation") + try: + canonical_process = ( + None + if value.current_process is None + else _canonical_process_observation_v1( + value.current_process, + input_value, + max_output_bytes, + max_stderr_bytes, + ) + ) + canonical = BuildCleanupFailureV1( + value.failures, + canonical_process, + _token=_BUILD_CLEANUP_FAILURE_TOKEN, + ) + if tuple(canonical) != tuple(value): + raise TypeError("build cleanup observation is not canonical") + return canonical + except (AttributeError, IndexError, TypeError, ValueError) as error: + raise TypeError("build cleanup observation is not canonical") from error + + +class DockerBuildBackendV1(Protocol): + def probe(self) -> DockerCapabilityReportV1: ... + + def run_build( + self, + request: DockerBuildRequestV1, + ) -> DockerBuildProcessObservationV1: ... + + +def build_process_bytes_v1(process: DockerBuildExitedV1) -> bytes: + if type(process) is not DockerBuildExitedV1: + raise TypeError("only successful typed build observations are encodable") + try: + transfer = process.input_transfer + if ( + process.returncode != 0 + or not -(1 << 31) <= process.returncode < 1 << 31 + or type(process.stdout) is not bytes + or len(process.stdout) > BUILD_STDOUT_LIMIT_V1 + or type(process.stderr) is not bytes + or len(process.stderr) > BUILD_STDERR_LIMIT_V1 + or not _input_transfer_is_structurally_valid_v1(transfer) + ): + raise TypeError("successful build observation is not canonical") + except (AttributeError, IndexError, OverflowError, TypeError) as error: + raise TypeError( + "only successful canonical build observations are encodable" + ) from error + return b"".join( + ( + process.returncode.to_bytes(4, "big", signed=True), + len(process.stdout).to_bytes(8, "big"), + hashlib.sha256(process.stdout).digest(), + len(process.stderr).to_bytes(8, "big"), + hashlib.sha256(process.stderr).digest(), + transfer.bundle_identity, + transfer.expected_length.to_bytes(8, "big"), + transfer.expected_sha256, + transfer.written_length.to_bytes(8, "big"), + transfer.written_sha256, + ) + ) + + +class NativeDockerBuildBackendV1: + """Docker adapter whose probe observes only Linux x64 and its daemon.""" + + def __init__( + self, + docker_path: Path, + policy: DockerBuildPolicyV1, + *, + platform_name: str | None = None, + machine_name: str | None = None, + monotonic_ns: object = time.monotonic_ns, + host_user: tuple[int, int] | None = None, + ) -> None: + _absolute_path(docker_path, "docker_path") + if not docker_policy_is_valid_v1(policy): + raise TypeError("policy must be DockerBuildPolicyV1") + if policy.user_mode is not DockerUserModeV1.HOST_EFFECTIVE_IDS: + raise TypeError("unsupported Docker user policy") + observed_user = ( + (os.geteuid(), os.getegid()) if host_user is None else host_user + ) + observed_platform = ( + platform.system().lower() if platform_name is None else platform_name + ) + observed_machine = ( + platform.machine() if machine_name is None else machine_name + ) + self._command_coordinate = native_command_coordinate_v1(docker_path) + self._policy = DockerBuildPolicyV1(*tuple(policy)) + self._platform_name = _encoded_policy_text( + observed_platform, + 64, + "platform_name", + ) + self._machine_name = _encoded_policy_text( + observed_machine, + 64, + "machine_name", + ) + self._monotonic_ns = monotonic_ns + self._host_user = _host_user_coordinates(observed_user) + self._probed_capability: DockerSupportedV1 | None = None + + @staticmethod + def _environment() -> dict[str, str]: + return { + "HOME": "/nonexistent", + "PATH": "/usr/bin:/bin", + "DOCKER_CONFIG": "/nonexistent", + } + + def probe(self) -> DockerCapabilityReportV1: + self._probed_capability = None + if self._platform_name != "linux" or self._machine_name.lower() not in ( + "x86_64", + "amd64", + ): + return DockerUnsupportedV1( + DockerBlockerReasonV1.HOST_NOT_LINUX_AMD64, + "controlled build requires a Linux amd64 Docker host", + ) + try: + metadata = self._command_coordinate.path.lstat() + except OSError: + return DockerUnsupportedV1( + DockerBlockerReasonV1.DOCKER_UNAVAILABLE, + "exact Docker CLI path is unavailable", + ) + if not stat.S_ISREG(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode): + return DockerUnsupportedV1( + DockerBlockerReasonV1.DOCKER_UNAVAILABLE, + "Docker CLI must be one regular non-symlink path", + ) + commands = ( + _render_native_command_v1( + "version_probe", + self._command_coordinate, + {}, + ), + _render_native_command_v1( + "image_inspect", + self._command_coordinate, + { + _NativeCommandSlotV1.IMAGE_REFERENCE: ( + self._policy.image_reference, + ), + }, + ), + ) + outputs: list[bytes] = [] + for index, command in enumerate(commands): + result = self._observe_command( + command, + stdout_limit=self._policy.probe_output_limit, + stderr_limit=self._policy.probe_output_limit, + timeout_ns=self._policy.probe_timeout_ns, + cid_file=None, + ) + if ( + type(result) is not _DockerCommandExitedV1 + or result.returncode != 0 + or not result.stdout + or result.stderr + ): + return DockerUnsupportedV1( + DockerBlockerReasonV1.DOCKER_UNAVAILABLE + if index == 0 + else DockerBlockerReasonV1.IMAGE_UNAVAILABLE, + "Docker daemon probe failed" + if index == 0 + else "pinned image is not locally inspectable", + ) + outputs.append(result.stdout) + try: + inspected = json.loads(outputs[1]) + if type(inspected) is not list or len(inspected) != 1: + raise ValueError("wrong image inspection cardinality") + image = inspected[0] + if type(image) is not dict: + raise ValueError("wrong image inspection shape") + repo_digests = image.get("RepoDigests") + if ( + image.get("Os") != "linux" + or image.get("Architecture") not in ("amd64", "x86_64") + or type(repo_digests) is not list + or self._policy.image_reference not in repo_digests + ): + raise ValueError("foreign image coordinate") + except (ValueError, TypeError, json.JSONDecodeError): + return DockerUnsupportedV1( + DockerBlockerReasonV1.IMAGE_IDENTITY_MISMATCH, + "local image does not match pinned linux/amd64 manifest", + ) + daemon_observation = DockerDaemonObservationV1( + outputs[0], + outputs[1], + ) + capability = DockerSupportedV1( + self._policy, + daemon_observation, + self._command_coordinate, + self._host_user, + ) + self._probed_capability = capability + return capability + + def command_for(self, request: DockerBuildRequestV1) -> tuple[str, ...]: + if type(request) is not DockerBuildRequestV1: + raise TypeError("request must be DockerBuildRequestV1") + try: + capability = request.capability + except (AttributeError, IndexError) as error: + raise TypeError("request lost its Docker capability") from error + if ( + self._probed_capability is None + or not _docker_build_request_is_valid_v1(request, capability) + or capability is not self._probed_capability + ): + raise TypeError("request capability does not match this backend probe") + policy = capability.policy + return _render_native_command_v1( + "build", + capability.command_coordinate, + { + _NativeCommandSlotV1.PLATFORM: (policy.platform,), + _NativeCommandSlotV1.ORDERED_TMPFS_SPECS: policy.tmpfs_specs, + _NativeCommandSlotV1.CONTAINER_NAME: (request.container_name,), + _NativeCommandSlotV1.HOSTNAME: (policy.hostname,), + _NativeCommandSlotV1.HOST_USER: ( + f"{capability.host_user[0]}:{capability.host_user[1]}", + ), + _NativeCommandSlotV1.CID_FILE: (str(request.cid_file),), + _NativeCommandSlotV1.IMAGE_REFERENCE: (policy.image_reference,), + _NativeCommandSlotV1.BOOTSTRAP: (policy.bootstrap,), + _NativeCommandSlotV1.BOOTSTRAP_ARGV0: (policy.bootstrap_argv0,), + _NativeCommandSlotV1.INPUT_LENGTH: ( + str(request.input_bundle.length), + ), + _NativeCommandSlotV1.INPUT_SHA256: ( + request.input_bundle.sha256.hex(), + ), + }, + ) + + def run_build( + self, + request: DockerBuildRequestV1, + ) -> DockerBuildProcessObservationV1: + command = self.command_for(request) + capability = request.capability + policy = capability.policy + return self._observe_command( + command, + stdout_limit=request.max_output_bytes, + stderr_limit=policy.stderr_limit, + timeout_ns=policy.build_timeout_ns, + cid_file=request.cid_file, + container_name=request.container_name, + input_bundle=request.input_bundle, + ) + + def _observe_command( + self, + command: tuple[str, ...], + *, + stdout_limit: int, + stderr_limit: int, + timeout_ns: int, + cid_file: Path | None, + container_name: str | None = None, + input_bundle: input.SealedInputV1 | None = None, + ) -> _DockerCommandObservationV1: + if ( + type(command) is not tuple + or not command + or any(type(item) is not str or not item or "\0" in item for item in command) + ): + raise TypeError("command must be a nonempty string tuple") + try: + tuple(os.fsencode(item) for item in command) + except (TypeError, UnicodeEncodeError) as error: + raise TypeError("command contains an unencodable coordinate") from error + if ( + type(stdout_limit) is not int + or stdout_limit <= 0 + or stdout_limit > BUILD_STDOUT_LIMIT_V1 + or type(stderr_limit) is not int + or stderr_limit <= 0 + or stderr_limit > BUILD_STDERR_LIMIT_V1 + or type(timeout_ns) is not int + or timeout_ns <= 0 + or timeout_ns > BUILD_TIMEOUT_NS_V1 + ): + raise TypeError("invalid Docker observation limits") + if (cid_file is None) != (container_name is None): + raise TypeError("Docker cleanup requires both CID file and exact name") + if cid_file is not None: + _absolute_path(cid_file, "cid_file") + active_policy = ( + self._probed_capability.policy + if self._probed_capability is not None + else self._policy + ) + _container_name(container_name, active_policy.container_name_prefix) + if input_bundle is not None and type(input_bundle) is not input.SealedInputV1: + raise TypeError("input_bundle must be controller sealed") + if input_bundle is not None and not input.sealed_input_is_intact_v1( + input_bundle + ): + return DockerBuildObserverFailureV1( + "build input bytes are not intact", + b"", + b"", + ) + try: + process = subprocess.Popen( + command, + stdin=subprocess.PIPE if input_bundle is not None else subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd="/", + env=self._environment(), + close_fds=True, + start_new_session=True, + ) + except (OSError, UnicodeEncodeError): + return DockerBuildObserverFailureV1( + "cannot start Docker CLI", + b"", + b"", + ) + stdout = bytearray() + stderr = bytearray() + selector: selectors.BaseSelector | None = None + terminal: DockerOutputStreamV1 | None = None + timed_out = False + observer_failed = False + input_failed = False + written = 0 + input_hasher = hashlib.sha256() + bundle_view: memoryview | None = None + input_progress: BuildInputTransferProgressV1 | None = None + stop_detail: str | None = None + cleanup_detail: str | None = None + input_descriptor: int | None = None + stdout_descriptor: int | None = None + stderr_descriptor: int | None = None + try: + if ( + process.stdout is None + or process.stderr is None + or (input_bundle is not None and process.stdin is None) + ): + observer_failed = True + raise RuntimeError("Docker pipes unavailable") + stdout_descriptor = process.stdout.fileno() + stderr_descriptor = process.stderr.fileno() + if process.stdin is not None: + input_descriptor = process.stdin.fileno() + selector = selectors.DefaultSelector() + bundle_view = ( + memoryview(input_bundle.contents) + if input_bundle is not None + else None + ) + streams = ( + ( + stdout_descriptor, + DockerOutputStreamV1.STDOUT, + stdout, + stdout_limit, + ), + ( + stderr_descriptor, + DockerOutputStreamV1.STDERR, + stderr, + stderr_limit, + ), + ) + for descriptor, stream, target, maximum in streams: + os.set_blocking(descriptor, False) + selector.register( + descriptor, + selectors.EVENT_READ, + ("read", stream, target, maximum), + ) + if process.stdin is not None: + os.set_blocking(input_descriptor, False) + selector.register( + input_descriptor, + selectors.EVENT_WRITE, + ("write",), + ) + start = self._clock() + deadline = start + timeout_ns + while selector.get_map() or process.poll() is None: + now = self._clock() + if now >= deadline: + timed_out = True + break + timeout = min( + (deadline - now) / 1_000_000_000, + _POLL_SLICE_SECONDS_V1, + ) + for key, _events in selector.select(timeout): + if key.data[0] == "read": + _kind, stream, target, maximum = key.data + try: + chunk = os.read( + key.fd, + min( + _IO_CHUNK_BYTES_V1, + maximum + 1 - len(target), + ), + ) + except BlockingIOError: + continue + if not chunk: + selector.unregister(key.fd) + continue + target.extend(chunk) + if len(target) > maximum: + del target[maximum:] + terminal = stream + break + continue + if input_bundle is None or bundle_view is None: + observer_failed = True + break + try: + count = os.write( + key.fd, + bundle_view[written : written + _IO_CHUNK_BYTES_V1], + ) + except BlockingIOError: + continue + except BrokenPipeError: + input_failed = True + break + if count <= 0: + input_failed = True + break + input_hasher.update(bundle_view[written : written + count]) + written += count + if written == input_bundle.length: + selector.unregister(key.fd) + if process.stdin is not None: + process.stdin.close() + if terminal is not None or input_failed or observer_failed: + break + except Exception: + observer_failed = True + finally: + if selector is not None: + try: + selector.close() + except Exception: + observer_failed = True + if process.stdin is not None: + if self._close_owned_stream(process.stdin, input_descriptor): + observer_failed = True + if bundle_view is not None: + try: + bundle_view.release() + except Exception: + observer_failed = True + if input_bundle is not None: + try: + input_progress = _build_input_progress_v1( + input_bundle, + written, + input_hasher.digest(), + ) + except Exception: + observer_failed = True + try: + process_running = process.poll() is None + except Exception: + process_running = True + stop_detail = "Docker CLI process state could not be observed" + if process_running: + if not ( + timed_out + or terminal is not None + or observer_failed + or input_failed + ): + timed_out = True + try: + observed_stop = self._stop_process(process) + except Exception: + observed_stop = "Docker CLI process termination raised" + stop_detail = stop_detail or observed_stop + for stream, descriptor in ( + (process.stdout, stdout_descriptor), + (process.stderr, stderr_descriptor), + ): + if stream is None: + continue + if self._close_owned_stream(stream, descriptor): + observer_failed = True + if cid_file is not None and container_name is not None: + try: + cleanup_detail = self._cleanup_container( + cid_file, + container_name, + ) + except Exception: + cleanup_detail = "Docker container cleanup observer raised" + if stop_detail is not None or cleanup_detail is not None: + trigger = DockerCleanupTriggerV1.PROCESS_EXIT + if observer_failed: + trigger = DockerCleanupTriggerV1.OBSERVER_FAILURE + elif terminal is not None: + trigger = DockerCleanupTriggerV1.OUTPUT_LIMIT + elif timed_out: + trigger = DockerCleanupTriggerV1.TIMEOUT + elif input_failed: + trigger = DockerCleanupTriggerV1.INPUT_TRANSFER + failures: list[CleanupFailureRecordV1] = [] + if stop_detail is not None: + failures.append( + CleanupFailureRecordV1( + CleanupResourceV1.DOCKER_CLI_PROCESS, + stop_detail, + ) + ) + if cleanup_detail is not None: + failures.append( + CleanupFailureRecordV1( + CleanupResourceV1.DOCKER_CONTAINER, + cleanup_detail, + ) + ) + return DockerBuildCleanupFailureV1( + trigger, + tuple(failures), + bytes(stdout), + bytes(stderr), + input_progress, + ) + if input_failed: + if input_progress is None: + return DockerBuildObserverFailureV1( + "build input progress could not be retained", + bytes(stdout), + bytes(stderr), + input_progress, + ) + return DockerBuildInputRejectedV1( + input_progress, + bytes(stdout), + bytes(stderr), + ) + if observer_failed: + return DockerBuildObserverFailureV1( + "Docker output observation failed", + bytes(stdout), + bytes(stderr), + input_progress, + ) + if terminal is not None: + return DockerBuildOutputLimitV1( + terminal, + bytes(stdout), + bytes(stderr), + input_progress, + ) + if timed_out: + return DockerBuildTimedOutV1( + bytes(stdout), + bytes(stderr), + input_progress, + ) + if type(process.returncode) is not int: + return DockerBuildObserverFailureV1( + "Docker returncode unavailable", + bytes(stdout), + bytes(stderr), + input_progress, + ) + if input_bundle is not None: + if ( + input_progress is None + or written != input_bundle.length + or input_hasher.digest() != input_bundle.sha256 + ): + return DockerBuildObserverFailureV1( + "completed build input transfer invariant failed", + bytes(stdout), + bytes(stderr), + input_progress, + ) + input_transfer = _completed_build_input_transfer_v1( + input_bundle, + written, + input_hasher.digest(), + ) + return _docker_build_exited_v1( + process.returncode, + bytes(stdout), + bytes(stderr), + input_transfer, + ) + return _docker_command_exited_v1( + process.returncode, + bytes(stdout), + bytes(stderr), + ) + + @staticmethod + def _close_owned_stream(stream: object, descriptor: int | None) -> bool: + """Close the file object, then its captured owned FD if close raised.""" + + close_failed = False + try: + closed = stream.closed is True + except Exception: + closed = False + close_failed = True + if not closed: + try: + stream.close() + except Exception: + close_failed = True + if close_failed and type(descriptor) is int and descriptor >= 0: + try: + os.close(descriptor) + except Exception: + pass + return close_failed + + def _clock(self) -> int: + value = self._monotonic_ns() + if type(value) is not int or value < 0: + raise RuntimeError("invalid monotonic clock") + return value + + def _stop_process( + self, + process: subprocess.Popen[bytes], + ) -> str | None: + failed = False + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + except OSError: + try: + process.kill() + except ProcessLookupError: + pass + except OSError: + failed = True + try: + process.wait(timeout=_PROCESS_STOP_TIMEOUT_SECONDS_V1) + except subprocess.TimeoutExpired: + failed = True + if process.poll() is None: + failed = True + return "Docker CLI process could not be terminated" if failed else None + + @staticmethod + def _admitted_container_id(cid_file: Path) -> str | None: + try: + descriptor = os.open( + cid_file, + os.O_RDONLY + | getattr(os, "O_CLOEXEC", 0) + | getattr(os, "O_NOFOLLOW", 0), + ) + except OSError: + return None + try: + metadata = os.fstat(descriptor) + if ( + not stat.S_ISREG(metadata.st_mode) + or metadata.st_nlink != 1 + or metadata.st_size not in (64, 65) + ): + return None + raw = os.read(descriptor, 66) + except OSError: + return None + finally: + os.close(descriptor) + if len(raw) == 65 and raw.endswith(b"\n"): + raw = raw[:-1] + if len(raw) != 64 or any( + byte not in b"0123456789abcdef" for byte in raw + ): + return None + return raw.decode("ascii") + + def _observe_cleanup_command( + self, + command: tuple[str, ...], + ) -> _DockerCommandObservationV1: + if self._probed_capability is None: + raise TypeError("Docker cleanup requires an observed capability") + policy = self._probed_capability.policy + return self._observe_command( + command, + stdout_limit=policy.probe_output_limit, + stderr_limit=policy.probe_output_limit, + timeout_ns=policy.probe_timeout_ns, + cid_file=None, + ) + + def _cleanup_container(self, cid_file: Path, container_name: str) -> str | None: + if self._probed_capability is None: + raise TypeError("Docker cleanup requires an observed capability") + capability = self._probed_capability + policy = capability.policy + _absolute_path(cid_file, "cid_file") + _container_name(container_name, policy.container_name_prefix) + container_id = self._admitted_container_id(cid_file) + removal_coordinates = ( + (container_id, container_name) + if container_id is not None + else (container_name,) + ) + try: + for coordinate in removal_coordinates: + self._observe_cleanup_command( + _render_native_command_v1( + "cleanup_rm", + capability.command_coordinate, + { + _NativeCommandSlotV1.CONTAINER_COORDINATE: ( + coordinate, + ), + }, + ) + ) + filters = [f"name=^/{container_name}$"] + if container_id is not None: + filters.append(f"id={container_id}") + for filter_value in filters: + observation = self._observe_cleanup_command( + _render_native_command_v1( + "cleanup_ls", + capability.command_coordinate, + { + _NativeCommandSlotV1.CONTAINER_FILTER: ( + filter_value, + ), + }, + ) + ) + if ( + type(observation) is not _DockerCommandExitedV1 + or observation.returncode != 0 + or observation.stdout + or observation.stderr + ): + return "Docker container absence could not be verified" + except Exception: + return "Docker container cleanup observer raised" + return None + + +class BuildFailureReasonV1(StrEnum): + CONTRACT_VIOLATION = "contract_violation" + PROCESS_FAILED = "process_failed" + CLEANUP_FAILED = "cleanup_failed" + INPUT_TRANSFER_FAILED = "input_transfer_failed" + TIMEOUT = "timeout" + OUTPUT_LIMIT = "output_limit" + OBSERVER_FAILURE = "observer_failure" + INVALID_OUTPUT = "invalid_output" + + +BuildAttemptObservationV1: TypeAlias = ( + DockerBuildProcessObservationV1 | BuildCleanupFailureV1 +) + + +class BuildSessionV1(tuple): + """Owned coordinates shared by every attempt in one two-build session.""" + + __slots__ = () + + def __new__( + cls, + capability: DockerSupportedV1, + input_value: input.SealedInputV1, + max_output_bytes: int, + *, + _token: object, + ) -> BuildSessionV1: + if ( + _token is not _BUILD_SESSION_TOKEN + or not _docker_supported_is_valid_v1(capability) + or not input.sealed_input_is_intact_v1(input_value) + or type(max_output_bytes) is not int + or max_output_bytes <= 0 + or max_output_bytes > capability.policy.stdout_limit + ): + raise TypeError("invalid two-build session coordinates") + return tuple.__new__( + cls, + ( + capability, + input_value, + max_output_bytes, + ), + ) + + @property + def policy(self) -> DockerBuildPolicyV1: + return self.capability.policy + + @property + def capability(self) -> DockerSupportedV1: + return self[0] + + @property + def input_value(self) -> input.SealedInputV1: + value = self[1] + if not input.sealed_input_is_intact_v1(value): + raise RuntimeError("build session lost exact input bytes") + return value + + @property + def max_output_bytes(self) -> int: + return self[2] + + +def _build_session_v1( + capability: DockerSupportedV1, + input_value: input.SealedInputV1, + max_output_bytes: int, +) -> BuildSessionV1: + return BuildSessionV1( + capability, + input_value, + max_output_bytes, + _token=_BUILD_SESSION_TOKEN, + ) + + +def _build_session_is_valid_v1(value: object) -> bool: + if type(value) is not BuildSessionV1: + return False + try: + canonical = _build_session_v1( + value.capability, + value.input_value, + value.max_output_bytes, + ) + return tuple(canonical) == tuple(value) + except Exception: + return False + + +class BuildByteRelationV1(StrEnum): + IDENTICAL = "identical" + DIFFERENT = "different" + + +class TwoBuildObservationV1(tuple): + """Two fresh successful attempts and their observed byte relation.""" + + __slots__ = () + + def __new__( + cls, + session: BuildSessionV1, + processes: tuple[DockerBuildExitedV1, DockerBuildExitedV1], + *, + _token: object, + ) -> TwoBuildObservationV1: + if ( + _token is not _TWO_BUILD_OBSERVATION_TOKEN + or not _build_session_is_valid_v1(session) + or type(processes) is not tuple + or len(processes) != 2 + ): + raise TypeError("invalid two-build observation") + owned: list[DockerBuildExitedV1] = [] + for process in processes: + canonical = _canonical_process_observation_v1( + process, + session.input_value, + session.max_output_bytes, + session.policy.stderr_limit, + ) + if type(canonical) is not DockerBuildExitedV1 or canonical.returncode != 0: + raise TypeError("two-build observation requires successful exits") + owned.append(canonical) + owned_processes = (owned[0], owned[1]) + relation = ( + BuildByteRelationV1.IDENTICAL + if owned_processes[0].stdout == owned_processes[1].stdout + else BuildByteRelationV1.DIFFERENT + ) + return tuple.__new__(cls, (session, relation, owned_processes)) + + @property + def session(self) -> BuildSessionV1: + return self[0] + + @property + def relation(self) -> BuildByteRelationV1: + return self[1] + + @property + def policy(self) -> DockerBuildPolicyV1: + return self.session.policy + + @property + def capability(self) -> DockerSupportedV1: + return self.session.capability + + @property + def input_value(self) -> input.SealedInputV1: + return self.session.input_value + + @property + def max_output_bytes(self) -> int: + return self.session.max_output_bytes + + @property + def processes( + self, + ) -> tuple[DockerBuildExitedV1, DockerBuildExitedV1]: + return self[2] + + @property + def outputs(self) -> tuple[bytes, bytes]: + return self.processes[0].stdout, self.processes[1].stdout + + @property + def first_sha256(self) -> bytes: + return hashlib.sha256(self.outputs[0]).digest() + + @property + def second_sha256(self) -> bytes: + return hashlib.sha256(self.outputs[1]).digest() + + +class BuildRejectedV1(tuple): + """Typed failed attempt retaining the successful causal prefix.""" + + __slots__ = () + + def __new__( + cls, + attempt: int, + reason: BuildFailureReasonV1, + process: BuildAttemptObservationV1 | None = None, + *, + session: BuildSessionV1 | None = None, + completed_processes: tuple[DockerBuildExitedV1, ...] = (), + ) -> BuildRejectedV1: + if ( + type(attempt) is not int + or attempt not in (1, 2) + or type(reason) is not BuildFailureReasonV1 + or type(completed_processes) is not tuple + ): + raise TypeError("invalid build rejection") + if session is None: + if ( + reason is not BuildFailureReasonV1.CONTRACT_VIOLATION + or process is not None + or completed_processes + ): + raise TypeError("context-free rejection must be a contract violation") + return tuple.__new__(cls, (attempt, reason, None, None, ())) + if ( + not _build_session_is_valid_v1(session) + or len(completed_processes) != attempt - 1 + ): + raise TypeError("build rejection lost its causal prefix") + owned_completed: list[DockerBuildExitedV1] = [] + for completed in completed_processes: + canonical = _canonical_process_observation_v1( + completed, + session.input_value, + session.max_output_bytes, + session.policy.stderr_limit, + ) + if type(canonical) is not DockerBuildExitedV1 or canonical.returncode != 0: + raise TypeError("causal prefix contains a failed attempt") + owned_completed.append(canonical) + if process is None: + owned_process: BuildAttemptObservationV1 | None = None + elif type(process) is BuildCleanupFailureV1: + owned_process = _canonical_build_cleanup_failure_v1( + process, + session.input_value, + session.max_output_bytes, + session.policy.stderr_limit, + ) + else: + owned_process = _canonical_process_observation_v1( + process, + session.input_value, + session.max_output_bytes, + session.policy.stderr_limit, + ) + expected_process_types: dict[BuildFailureReasonV1, tuple[type, ...]] = { + BuildFailureReasonV1.CONTRACT_VIOLATION: (), + BuildFailureReasonV1.PROCESS_FAILED: (DockerBuildExitedV1,), + BuildFailureReasonV1.CLEANUP_FAILED: ( + DockerBuildCleanupFailureV1, + BuildCleanupFailureV1, + ), + BuildFailureReasonV1.INPUT_TRANSFER_FAILED: ( + DockerBuildInputRejectedV1, + ), + BuildFailureReasonV1.TIMEOUT: (DockerBuildTimedOutV1,), + BuildFailureReasonV1.OUTPUT_LIMIT: (DockerBuildOutputLimitV1,), + BuildFailureReasonV1.OBSERVER_FAILURE: ( + DockerBuildObserverFailureV1, + ), + BuildFailureReasonV1.INVALID_OUTPUT: (DockerBuildExitedV1,), + } + expected = expected_process_types[reason] + if not expected: + if owned_process is not None: + raise TypeError("contract-violation rejection cannot retain authority") + elif type(owned_process) not in expected: + raise TypeError("build rejection reason and observation disagree") + if ( + ( + reason is BuildFailureReasonV1.PROCESS_FAILED + and owned_process.returncode == 0 + ) + or ( + reason is BuildFailureReasonV1.INVALID_OUTPUT + and owned_process.returncode != 0 + ) + ): + raise TypeError("build rejection exit status disagrees with reason") + return tuple.__new__( + cls, + ( + attempt, + reason, + owned_process, + session, + tuple(owned_completed), + ), + ) + + @property + def attempt(self) -> int: + return self[0] + + @property + def reason(self) -> BuildFailureReasonV1: + return self[1] + + @property + def process(self) -> BuildAttemptObservationV1 | None: + return self[2] + + @property + def session(self) -> BuildSessionV1 | None: + return self[3] + + @property + def completed_processes(self) -> tuple[DockerBuildExitedV1, ...]: + return self[4] + + +def two_build_observation_matches_v1( + value: object, + session: BuildSessionV1, +) -> bool: + if ( + type(value) is not TwoBuildObservationV1 + or not _build_session_is_valid_v1(session) + ): + return False + try: + replayed = TwoBuildObservationV1( + session, + value.processes, + _token=_TWO_BUILD_OBSERVATION_TOKEN, + ) + return tuple(replayed) == tuple(value) + except Exception: + return False + + +BuildTransportResultV1: TypeAlias = ( + TwoBuildObservationV1 | BuildRejectedV1 +) + + +class ControlledBuildTransportV1: + """Own two fresh attempts; callers own semantic input and output admission.""" + + def __init__( + self, + *, + policy: DockerBuildPolicyV1, + backend: DockerBuildBackendV1, + ) -> None: + if not docker_policy_is_valid_v1(policy): + raise TypeError("policy must be DockerBuildPolicyV1") + self._policy = DockerBuildPolicyV1(*tuple(policy)) + self._backend = backend + self._probed_capability: DockerSupportedV1 | None = None + self._consumed = False + + def probe(self) -> DockerCapabilityReportV1: + if self._consumed or self._probed_capability is not None: + return DockerUnsupportedV1( + DockerBlockerReasonV1.BACKEND_CONTRACT, + "build transport capability is one-shot", + ) + try: + report = self._backend.probe() + except Exception: + return DockerUnsupportedV1( + DockerBlockerReasonV1.BACKEND_CONTRACT, + "Docker capability probe raised", + ) + if type(report) is DockerUnsupportedV1: + if _docker_unsupported_is_valid_v1(report): + return DockerUnsupportedV1(*tuple(report)) + return DockerUnsupportedV1( + DockerBlockerReasonV1.BACKEND_CONTRACT, + "Docker capability rejection is not canonical", + ) + if ( + not _docker_supported_is_valid_v1(report) + or report.policy != self._policy + ): + return DockerUnsupportedV1( + DockerBlockerReasonV1.BACKEND_CONTRACT, + "Docker capability report does not match build policy", + ) + self._probed_capability = report + return report + + def build( + self, + capability: DockerSupportedV1, + input_value: input.SealedInputV1, + max_output_bytes: int, + *, + input_admission: Callable[[input.SealedInputV1], bool], + output_admission: Callable[[bytes], bool], + ) -> BuildTransportResultV1: + if ( + self._consumed + or capability is not self._probed_capability + or not _docker_supported_is_valid_v1(capability) + or capability.policy != self._policy + ): + return BuildRejectedV1(1, BuildFailureReasonV1.CONTRACT_VIOLATION) + self._consumed = True + if ( + type(max_output_bytes) is not int + or max_output_bytes <= 0 + or max_output_bytes > capability.policy.stdout_limit + or not callable(input_admission) + or not callable(output_admission) + or not input.sealed_input_is_intact_v1(input_value) + ): + return BuildRejectedV1(1, BuildFailureReasonV1.CONTRACT_VIOLATION) + session = _build_session_v1( + capability, + input_value, + max_output_bytes, + ) + completed: list[DockerBuildExitedV1] = [] + for attempt in (1, 2): + if ( + not input.sealed_input_is_intact_v1(input_value) + or not self._admitted(input_admission, input_value) + ): + return BuildRejectedV1( + attempt, + BuildFailureReasonV1.CONTRACT_VIOLATION, + session=session, + completed_processes=tuple(completed), + ) + built = self._build_once( + attempt, + session, + output_admission, + tuple(completed), + ) + if type(built) is BuildRejectedV1: + return built + completed.append(built) + return TwoBuildObservationV1( + session, + (completed[0], completed[1]), + _token=_TWO_BUILD_OBSERVATION_TOKEN, + ) + + @staticmethod + def _admitted( + admission: Callable[[object], bool], + value: object, + ) -> bool: + try: + return admission(value) is True + except Exception: + return False + + def _build_once( + self, + attempt: int, + session: BuildSessionV1, + output_admission: Callable[[bytes], bool], + completed_processes: tuple[DockerBuildExitedV1, ...], + ) -> DockerBuildExitedV1 | BuildRejectedV1: + if not _build_session_is_valid_v1(session): + return BuildRejectedV1( + attempt, + BuildFailureReasonV1.CONTRACT_VIOLATION, + ) + contract_rejection = BuildRejectedV1( + attempt, + BuildFailureReasonV1.CONTRACT_VIOLATION, + session=session, + completed_processes=completed_processes, + ) + try: + temporary_root = tempfile.TemporaryDirectory( + prefix=f"{session.policy.container_name_prefix}{attempt}-" + ) + except Exception: + return contract_rejection + current_process: DockerBuildProcessObservationV1 | None = None + try: + result, current_process = self._observe_build_attempt_v1( + attempt, + session, + output_admission, + completed_processes, + Path(temporary_root.name).resolve(), + ) + except Exception: + result = contract_rejection + try: + temporary_root.cleanup() + except Exception: + try: + cleanup = _build_cleanup_failure_v1( + current_process, + "temporary build root cleanup failed", + ) + return BuildRejectedV1( + attempt, + BuildFailureReasonV1.CLEANUP_FAILED, + cleanup, + session=session, + completed_processes=completed_processes, + ) + except Exception: + return contract_rejection + return result + + def _observe_build_attempt_v1( + self, + attempt: int, + session: BuildSessionV1, + output_admission: Callable[[bytes], bool], + completed_processes: tuple[DockerBuildExitedV1, ...], + root: Path, + ) -> tuple[ + DockerBuildExitedV1 | BuildRejectedV1, + DockerBuildProcessObservationV1 | None, + ]: + contract_rejection = BuildRejectedV1( + attempt, + BuildFailureReasonV1.CONTRACT_VIOLATION, + session=session, + completed_processes=completed_processes, + ) + request = DockerBuildRequestV1( + attempt, + session.capability, + session.input_value, + session.max_output_bytes, + root / "container.cid", + session.policy.container_name_prefix + + hashlib.sha256( + os.fsencode(root) + bytes((attempt,)) + ).hexdigest(), + ) + try: + observed = self._backend.run_build(request) + except Exception: + return contract_rejection, None + try: + process = _canonical_process_observation_v1( + observed, + session.input_value, + session.max_output_bytes, + session.policy.stderr_limit, + ) + except TypeError: + return contract_rejection, None + reason_by_type: dict[type, BuildFailureReasonV1] = { + DockerBuildCleanupFailureV1: BuildFailureReasonV1.CLEANUP_FAILED, + DockerBuildInputRejectedV1: BuildFailureReasonV1.INPUT_TRANSFER_FAILED, + DockerBuildTimedOutV1: BuildFailureReasonV1.TIMEOUT, + DockerBuildOutputLimitV1: BuildFailureReasonV1.OUTPUT_LIMIT, + DockerBuildObserverFailureV1: BuildFailureReasonV1.OBSERVER_FAILURE, + } + failure_reason = reason_by_type.get(type(process)) + if failure_reason is not None: + return ( + BuildRejectedV1( + attempt, + failure_reason, + process, + session=session, + completed_processes=completed_processes, + ), + process, + ) + if type(process) is not DockerBuildExitedV1: + return contract_rejection, None + if not docker_build_exited_is_valid_v1( + process, + session.input_value, + session.max_output_bytes, + session.policy.stderr_limit, + ): + return contract_rejection, None + if process.returncode != 0: + return ( + BuildRejectedV1( + attempt, + BuildFailureReasonV1.PROCESS_FAILED, + process, + session=session, + completed_processes=completed_processes, + ), + process, + ) + transfer = process.input_transfer + if ( + type(transfer) is not BuildInputTransferV1 + or transfer.bundle_identity != session.input_value.binding_identity + or transfer.expected_length != session.input_value.length + or transfer.expected_sha256 != session.input_value.sha256 + or transfer.written_length != session.input_value.length + or transfer.written_sha256 != session.input_value.sha256 + ): + return contract_rejection, None + if not self._admitted(output_admission, process.stdout): + return ( + BuildRejectedV1( + attempt, + BuildFailureReasonV1.INVALID_OUTPUT, + process, + session=session, + completed_processes=completed_processes, + ), + process, + ) + return process, process diff --git a/proof/region/v1/tests/test_build.py b/proof/region/v1/tests/test_build.py new file mode 100644 index 00000000..c313d13f --- /dev/null +++ b/proof/region/v1/tests/test_build.py @@ -0,0 +1,1189 @@ +#!/usr/bin/env python3 +"""RED contract for an identity-preserving engine-neutral BUILD leaf.""" + +from __future__ import annotations + +import ast +import hashlib +import importlib +import os +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + + +PROOF = Path(__file__).resolve().parents[1] +ARB = PROOF / "arb" +ARB_TESTS = ARB / "tests" +sys.path[:0] = (str(PROOF), str(ARB), str(ARB_TESTS)) + +import pipeline # noqa: E402 +from proof.region.v1.arb.tests import gate as arb_gate # noqa: E402 +from test_pipeline import ( # noqa: E402 + _docker_capability, + _probe_native_backend, + _request, +) +from test_receipt import _execute # noqa: E402 + + +# These inventory goldens describe the complete Arb gate. Identity-version +# tests deliberately change this inventory and must update both values from the +# gate's independent enumeration in the same slice. +ARB_INVENTORY_SHA256_V1 = ( + "383672bd1ac2a2d472fdba33d3ec4c770a897ecd13192ec41e7c12dc1e563219" +) +ARB_ORDER_SHA256_V1 = ( + "c020118a926070e36f14757f0b281ac05dc460b8affa03947f827baa73d5d172" +) + +MOVED_INPUT_SURFACE_V1 = ( + "CanonicalInputLimitsV1", + "SealedInputV1", + "seal_input_v1", + "sealed_input_is_intact_v1", + "canonical_ustar_v1", +) + +MOVED_TRANSPORT_SURFACE_V1 = ( + "DockerBuildPolicyV1", + "DockerUserModeV1", + "DockerBlockerReasonV1", + "DockerUnsupportedV1", + "DockerSupportedV1", + "DockerDaemonObservationV1", + "NativeCommandCoordinateV1", + "DockerBuildRequestV1", + "transport_policy_identity_v1", + "native_command_contract_identity_v1", + "native_command_coordinate_v1", + "docker_capability_identity_v1", + "BuildInputTransferProgressV1", + "BuildInputTransferV1", + "DockerBuildExitedV1", + "DockerBuildTimedOutV1", + "DockerOutputStreamV1", + "DockerBuildOutputLimitV1", + "DockerBuildObserverFailureV1", + "DockerBuildInputRejectedV1", + "DockerCleanupTriggerV1", + "CleanupResourceV1", + "CleanupFailureRecordV1", + "DockerBuildCleanupFailureV1", + "BuildCleanupFailureV1", + "DockerBuildBackendV1", + "NativeDockerBuildBackendV1", + "ControlledBuildTransportV1", + "BuildFailureReasonV1", + "BuildRejectedV1", + "BuildByteRelationV1", + "TwoBuildObservationV1", + "build_process_bytes_v1", +) + +REMOVED_TRANSPORT_SURFACE_V1 = ( + "NonReproducibleBuildV1", + "ReproducibleBuildV1", + "docker_report_matches_policy_v1", +) + +FORBIDDEN_INPUT_IMPORTS_V1 = ( + "arb", + "mpfi", + "pipeline", + "formula", + "comparator", + "receipt", + "region_proof_protocol", + "provenance", +) + +FORBIDDEN_TRANSPORT_IMPORTS_V1 = FORBIDDEN_INPUT_IMPORTS_V1 + ("provenance",) + + +def _imported_modules(source: str) -> tuple[str, ...]: + modules: list[str] = [] + for node in ast.walk(ast.parse(source)): + if isinstance(node, ast.Import): + modules.extend(alias.name for alias in node.names) + elif isinstance(node, ast.ImportFrom): + modules.append(node.module or "") + return tuple(modules) + + +def _digest(label: str) -> bytes: + return hashlib.sha256(label.encode("ascii")).digest() + + +def _sealed_input() -> object: + build_input = importlib.import_module("build.input") + return build_input.seal_input_v1(_digest("generic-build-binding"), b"input") + + +def _docker_capability_fixture(policy: object) -> object: + return _docker_capability(policy) + + +def _completed_process( + transport: object, + input_value: object, + stdout: bytes, + *, + returncode: int = 0, + stderr: bytes = b"", +) -> object: + transfer = transport._completed_build_input_transfer_v1( + input_value, + input_value.length, + input_value.sha256, + ) + return transport._docker_build_exited_v1( + returncode, + stdout, + stderr, + transfer, + ) + + +def _initial_progress(transport: object, input_value: object) -> object: + return transport._build_input_progress_v1( + input_value, + 0, + hashlib.sha256(b"").digest(), + ) + + +def _forged_exact_type(value_type: type[object]) -> object: + if issubclass(value_type, tuple): + return tuple.__new__(value_type, ()) + return object.__new__(value_type) + + +class _ScriptedBuildBackend: + def __init__(self, report: object, observations: tuple[object, ...]) -> None: + self._report = report + self._observations = list(observations) + self.requests: list[object] = [] + + def probe(self) -> object: + return self._report + + def run_build(self, request: object) -> object: + self.requests.append(request) + return self._observations.pop(0) + + +def _controlled_build( + transport: object, + policy: object, + observations: tuple[object, ...], + *, + max_output_bytes: int = 64, + input_value: object | None = None, +) -> tuple[object, _ScriptedBuildBackend, object, object]: + if input_value is None: + input_value = _sealed_input() + capability = _docker_capability_fixture(policy) + backend = _ScriptedBuildBackend(capability, observations) + controller = transport.ControlledBuildTransportV1( + policy=policy, + backend=backend, + ) + owned_capability = controller.probe() + result = controller.build( + owned_capability, + input_value, + max_output_bytes, + input_admission=lambda _value: True, + output_admission=lambda _value: True, + ) + return result, backend, owned_capability, input_value + + +class ExistingArbGateTests(unittest.TestCase): + def test_existing_arb_suite_keeps_exact_count_order_and_inventory(self) -> None: + tests = tuple(arb_gate._iter_tests_v1(arb_gate.full_suite_v1())) + identifiers = tuple(test.id() for test in tests) + ordered_preimage = b"".join( + identifier.encode("utf-8") + b"\n" for identifier in identifiers + ) + + self.assertEqual(len(identifiers), 160) + self.assertEqual(len(set(identifiers)), 160) + self.assertEqual( + arb_gate.test_inventory_sha256_v1(arb_gate.full_suite_v1()), + ARB_INVENTORY_SHA256_V1, + ) + self.assertEqual( + hashlib.sha256(ordered_preimage).hexdigest(), + ARB_ORDER_SHA256_V1, + ) + + +class ArbBuildIdentityCharacterizationTests(unittest.TestCase): + def test_arb_input_and_process_stay_exact_while_capability_identities_move_to_v2(self) -> None: + transport = importlib.import_module("build.transport") + request = _request() + result, _backend = _execute() + observed = result.evidence.build + process_bytes = transport.build_process_bytes_v1( + observed.build_processes[0] + ) + + self.assertEqual(observed.input_bundle_length, 174_080) + self.assertEqual( + observed.input_bundle_sha256.hex(), + "5d6e789a721aeed1a8ff023f0af5389711f85f6fe95294d8290b20301235f4df", + ) + self.assertEqual( + observed.input_bundle_identity.hex(), + "6e88d9105d581ef1898dd1b0ac2ee6362c1bf15e8495990e1518fba35e7a8bd0", + ) + self.assertEqual( + pipeline.pipeline_policy_identity_v2( + request.host_trust, + observed.docker_capability.policy, + ).hex(), + "66af6f844dda8eae548eac026f277845ccde1842c14c39824c5108f027247f39", + ) + self.assertEqual(len(process_bytes), 196) + self.assertEqual( + hashlib.sha256(process_bytes).hexdigest(), + "401aaf23753b09b35482080e6046499e6a8a0a4ea2cea6c658ed377efebac58c", + ) + self.assertEqual( + result.comparator.identity.hex(), + "e4e8e4dd47ddda5585531f67bfe3112f157a032cb728cd1c61766edf26de6c6c", + ) + self.assertEqual( + result.evidence.source_identity.hex(), + "07d85ad695ec17104bdb34f6e9819d25be08afb3aa485918c44a363d7679f7c9", + ) + self.assertEqual( + result.evidence.build_identity.hex(), + "dfe01f51132d938be3f8a8fad32c91d99fc7b22d69c4c9f488c07f2d00806412", + ) + self.assertEqual( + result.evidence.run_identity.hex(), + "0033f6e70d0090ff2839d364cccaf1a3f5bf79eb2c857ce762b236cbbc730542", + ) + self.assertEqual( + result.evidence.identity.hex(), + "80dd866e156d882a749a78b768cfc66838f92f6608baaf9ddfbf3f3a31870324", + ) + self.assertEqual( + result.claim.identity.hex(), + "c0c200282fc3cd800bb0aa53a8e3c2d3fa2edf1a6350185aeff86f410ccef1bb", + ) + + +class SharedBuildExtractionTests(unittest.TestCase): + def test_build_namespace_has_two_focused_shared_leaves(self) -> None: + package = importlib.import_module("build") + build_input = importlib.import_module("build.input") + transport = importlib.import_module("build.transport") + namespace = PROOF / "build" + + self.assertEqual( + Path(package.__file__).resolve(), + (namespace / "__init__.py").resolve(), + ) + self.assertEqual( + tuple(Path(item).resolve() for item in package.__path__), + (namespace.resolve(),), + ) + self.assertFalse((PROOF / "build.py").exists()) + self.assertEqual( + Path(build_input.__file__).resolve(), + (namespace / "input.py").resolve(), + ) + self.assertEqual( + Path(transport.__file__).resolve(), + (namespace / "transport.py").resolve(), + ) + self.assertFalse((ARB / "build").exists()) + self.assertFalse((PROOF / "mpfi/build").exists()) + self.assertIs(transport.input, build_input) + self.assertFalse(hasattr(build_input, "normalized_source_entries_v1")) + for name in MOVED_INPUT_SURFACE_V1: + with self.subTest(name=name): + self.assertTrue(hasattr(build_input, name)) + for name in MOVED_TRANSPORT_SURFACE_V1: + with self.subTest(name=name): + self.assertTrue(hasattr(transport, name)) + for name in REMOVED_TRANSPORT_SURFACE_V1: + with self.subTest(removed=name): + self.assertFalse(hasattr(transport, name)) + + def test_shared_leaves_import_no_engine_or_proof_semantics(self) -> None: + surfaces = ( + ( + importlib.import_module("build.input"), + FORBIDDEN_INPUT_IMPORTS_V1, + ), + ( + importlib.import_module("build.transport"), + FORBIDDEN_TRANSPORT_IMPORTS_V1, + ), + ) + for surface, forbidden_imports in surfaces: + source = Path(surface.__file__).read_text(encoding="utf-8") + for module in _imported_modules(source): + with self.subTest(surface=surface.__name__, module=module): + top_level = module.lstrip(".").split(".", 1)[0].lower() + self.assertNotIn(top_level, forbidden_imports, module) + + def test_observation_contract_uses_current_non_claiming_language(self) -> None: + transport_source = (PROOF / "build" / "transport.py").read_text( + encoding="utf-8" + ) + pipeline_source = (ARB / "pipeline.py").read_text(encoding="utf-8") + self.assertNotIn( + "backend-contract rejection cannot retain authority", + transport_source, + ) + self.assertNotIn("invalid reproducible-build digests", pipeline_source) + + def test_arb_consumers_move_atomically_without_compatibility_reexports(self) -> None: + build_input = importlib.import_module("build.input") + transport = importlib.import_module("build.transport") + arb_pipeline = pipeline + arb_receipt = importlib.import_module("receipt") + request = _request() + bundle = arb_pipeline._seal_build_input_bundle_v1(request) + + self.assertIs(arb_pipeline.build_input, build_input) + self.assertIs(arb_pipeline.build_transport, transport) + self.assertIs(arb_receipt.build_transport, transport) + self.assertFalse(hasattr(arb_receipt, "build_input")) + self.assertIs(type(bundle), build_input.SealedInputV1) + for name in MOVED_INPUT_SURFACE_V1 + MOVED_TRANSPORT_SURFACE_V1: + with self.subTest(consumer=arb_pipeline.__name__, name=name): + self.assertFalse(hasattr(arb_pipeline, name)) + for name in MOVED_TRANSPORT_SURFACE_V1: + with self.subTest(consumer=arb_receipt.__name__, name=name): + self.assertFalse(hasattr(arb_receipt, name)) + for name in REMOVED_TRANSPORT_SURFACE_V1: + with self.subTest(consumer=arb_pipeline.__name__, removed=name): + self.assertFalse(hasattr(arb_pipeline, name)) + with self.subTest(consumer=arb_receipt.__name__, removed=name): + self.assertFalse(hasattr(arb_receipt, name)) + + def test_shared_input_and_policy_are_deeply_immutable_coordinates(self) -> None: + build_input = importlib.import_module("build.input") + transport = importlib.import_module("build.transport") + sealed = build_input.seal_input_v1( + hashlib.sha256(b"binding").digest(), + b"exact bytes", + ) + policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 + + for value in (sealed, policy): + with self.subTest(value=type(value).__name__): + self.assertFalse(hasattr(value, "__dict__")) + with self.assertRaises((AttributeError, TypeError)): + value[0] = value[0] + with self.assertRaises((AttributeError, TypeError)): + object.__setattr__(value, "foreign", object()) + self.assertIs(type(sealed), build_input.SealedInputV1) + self.assertIs(type(policy), transport.DockerBuildPolicyV1) + + def test_forged_source_authorities_fail_in_the_arb_taxonomy(self) -> None: + build_input = importlib.import_module("build.input") + provenance = importlib.import_module("provenance") + request = _request() + lock = request.source_lock.sources[0] + admitted = request.admitted_sources.sources[0] + + for hostile_lock, hostile_admitted in ( + (object.__new__(provenance.SourceReleaseLockV1), admitted), + (lock, object.__new__(provenance.SafeSourceArchiveV1)), + ): + with self.subTest(authority=type(hostile_lock).__name__): + with self.assertRaises(pipeline.PipelineInputErrorV1) as raised: + pipeline._normalized_source_entries_v1( + hostile_lock, + hostile_admitted, + ) + self.assertEqual( + raised.exception.reason, + pipeline.PipelineInputReasonV1.FOREIGN_SOURCE_CAPABILITY, + ) + + +class SharedBuildTransportTargetTests(unittest.TestCase): + def test_public_build_contract_violations_are_typed_before_backend(self) -> None: + build_input = importlib.import_module("build.input") + transport = importlib.import_module("build.transport") + policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 + cases = ( + ("wrong type", None, lambda _value: True), + ( + "forged sealed input", + tuple.__new__(build_input.SealedInputV1, ()), + lambda _value: True, + ), + ("lane admission", _sealed_input(), lambda _value: False), + ) + for name, hostile, admission in cases: + with self.subTest(case=name): + capability = _docker_capability_fixture(policy) + backend = _ScriptedBuildBackend(capability, ()) + controller = transport.ControlledBuildTransportV1( + policy=policy, + backend=backend, + ) + owned_capability = controller.probe() + result = controller.build( + owned_capability, + hostile, + 64, + input_admission=admission, + output_admission=lambda _value: True, + ) + self.assertIs(type(result), transport.BuildRejectedV1) + self.assertEqual( + result.reason, + transport.BuildFailureReasonV1.CONTRACT_VIOLATION, + ) + self.assertEqual(backend.requests, []) + + def test_capability_owns_injected_host_user_and_rejects_surrogate_coordinates(self) -> None: + transport = importlib.import_module("build.transport") + self.assertTrue(hasattr(transport, "DockerUserModeV1")) + user_mode = transport.DockerUserModeV1.HOST_EFFECTIVE_IDS + shipped = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 + coordinates = { + "image_reference": shipped.image_reference, + "platform": shipped.platform, + "hostname": shipped.hostname, + "container_name_prefix": shipped.container_name_prefix, + "bootstrap": shipped.bootstrap, + "bootstrap_argv0": shipped.bootstrap_argv0, + "tmpfs_specs": shipped.tmpfs_specs, + "user_mode": user_mode, + "stdout_limit": shipped.stdout_limit, + "stderr_limit": shipped.stderr_limit, + "build_timeout_ns": shipped.build_timeout_ns, + "probe_output_limit": shipped.probe_output_limit, + "probe_timeout_ns": shipped.probe_timeout_ns, + } + policy = transport.DockerBuildPolicyV1(**coordinates) + input_value = _sealed_input() + backends = tuple( + transport.NativeDockerBuildBackendV1( + Path("/usr/bin/true"), + policy, + host_user=(501, 20), + platform_name="linux", + machine_name="x86_64", + ) + for _ in range(2) + ) + capabilities = tuple( + _probe_native_backend(backend, policy) for backend in backends + ) + requests = tuple( + transport.DockerBuildRequestV1( + 1, + capability, + input_value, + 64, + Path("/tmp/lab-colors-red-user.cid"), + policy.container_name_prefix + "red-user", + ) + for capability in capabilities + ) + with mock.patch.object( + transport.os, + "geteuid", + side_effect=AssertionError("command_for performed ambient uid IO"), + ), mock.patch.object( + transport.os, + "getegid", + side_effect=AssertionError("command_for performed ambient gid IO"), + ): + commands = tuple( + backend.command_for(request) + for backend, request in zip(backends, requests, strict=True) + ) + self.assertEqual(commands[0], commands[1]) + user_index = commands[0].index("--user") + self.assertEqual(commands[0][user_index + 1], "501:20") + + for field_name, value in ( + ("bootstrap", "\ud800"), + ("tmpfs_specs", ("/tmp/\ud800:rw",)), + ): + with self.subTest(field=field_name): + hostile = dict(coordinates) + hostile[field_name] = value + with self.assertRaises(TypeError): + transport.DockerBuildPolicyV1(**hostile) + with self.assertRaises(TypeError): + transport.NativeDockerBuildBackendV1( + Path("/tmp/\ud800"), + policy, + host_user=(501, 20), + platform_name="linux", + machine_name="x86_64", + ) + + def test_native_host_coordinates_are_exact_strings_and_oci_ports_are_ascii(self) -> None: + transport = importlib.import_module("build.transport") + policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 + + class StringSubclass(str): + pass + + for field_name, value in ( + ("platform_name", StringSubclass("linux")), + ("machine_name", StringSubclass("x86_64")), + ("platform_name", 7), + ("machine_name", object()), + ): + with self.subTest(field=field_name, value_type=type(value).__name__): + coordinates = { + "platform_name": "linux", + "machine_name": "x86_64", + } + coordinates[field_name] = value + with self.assertRaises(TypeError): + transport.NativeDockerBuildBackendV1( + Path("/usr/bin/docker"), + policy, + host_user=(501, 20), + **coordinates, + ) + + hostile_policy = { + "image_reference": ( + "registry.example:\u0661/toolchain@sha256:" + "a" * 64 + ), + "platform": policy.platform, + "hostname": policy.hostname, + "container_name_prefix": policy.container_name_prefix, + "bootstrap": policy.bootstrap, + "bootstrap_argv0": policy.bootstrap_argv0, + "tmpfs_specs": policy.tmpfs_specs, + "user_mode": policy.user_mode, + "stdout_limit": policy.stdout_limit, + "stderr_limit": policy.stderr_limit, + "build_timeout_ns": policy.build_timeout_ns, + "probe_output_limit": policy.probe_output_limit, + "probe_timeout_ns": policy.probe_timeout_ns, + } + with self.assertRaises(TypeError): + transport.DockerBuildPolicyV1(**hostile_policy) + + def test_stream_close_failure_fallback_closes_fd_and_retains_evidence(self) -> None: + transport = importlib.import_module("build.transport") + backend = transport.NativeDockerBuildBackendV1( + Path("/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + host_user=(501, 20), + platform_name="linux", + machine_name="x86_64", + ) + real_popen = subprocess.Popen + real_close = os.close + spawned: list[subprocess.Popen[bytes]] = [] + fallback_closed: list[int] = [] + wrapped_streams: list[object] = [] + + class CloseRaises: + def __init__(self, wrapped: object) -> None: + self.wrapped = wrapped + self.descriptor = wrapped.fileno() + self.close_calls = 0 + + @property + def closed(self) -> bool: + return False + + def fileno(self) -> int: + return self.descriptor + + def close(self) -> None: + self.close_calls += 1 + raise OSError("forced close failure") + + def spawn(*args: object, **kwargs: object) -> subprocess.Popen[bytes]: + process = real_popen(*args, **kwargs) + spawned.append(process) + wrapped = CloseRaises(process.stdout) + wrapped_streams.append(wrapped) + process.stdout = wrapped + return process + + def close(descriptor: int) -> None: + fallback_closed.append(descriptor) + real_close(descriptor) + + input_value = _sealed_input() + command = ( + sys.executable, + "-c", + ( + "import sys; sys.stdin.buffer.read(); " + "sys.stdout.buffer.write(b'evidence')" + ), + ) + try: + with mock.patch.object( + transport.subprocess, + "Popen", + side_effect=spawn, + ), mock.patch.object(transport.os, "close", side_effect=close): + result = backend._observe_command( + command, + stdout_limit=64, + stderr_limit=64, + timeout_ns=1_000_000_000, + cid_file=None, + input_bundle=input_value, + ) + finally: + for process in spawned: + if process.poll() is None: + process.kill() + process.wait(timeout=5) + for stream in (process.stdin, process.stderr): + if stream is not None and not stream.closed: + stream.close() + original = wrapped_streams[0].wrapped + try: + original.close() + except OSError: + pass + + self.assertIs(type(result), transport.DockerBuildObserverFailureV1) + self.assertEqual(result.stdout, b"evidence") + self.assertEqual(result.stderr, b"") + self.assertIsNotNone(result.input_progress) + self.assertEqual(result.input_progress.written_length, input_value.length) + self.assertEqual(result.input_progress.written_sha256, input_value.sha256) + self.assertIn(wrapped_streams[0].descriptor, fallback_closed) + self.assertEqual(wrapped_streams[0].close_calls, 1) + + def test_post_popen_failures_always_close_streams_and_cleanup_once(self) -> None: + transport = importlib.import_module("build.transport") + real_popen = subprocess.Popen + + def exercise( + *, + selector_failure: bool, + ) -> tuple[object, bool, int]: + backend = transport.NativeDockerBuildBackendV1( + Path("/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + host_user=(501, 20), + platform_name="linux", + machine_name="x86_64", + ) + spawned: list[subprocess.Popen[bytes]] = [] + cleanup_calls: list[tuple[Path, str]] = [] + + def spawn(*args: object, **kwargs: object) -> subprocess.Popen[bytes]: + process = real_popen(*args, **kwargs) + spawned.append(process) + return process + + def cleanup(cid_file: Path, container_name: str) -> None: + cleanup_calls.append((cid_file, container_name)) + return None + + def stop_raises(process: subprocess.Popen[bytes]) -> None: + process.kill() + process.wait(timeout=5) + raise RuntimeError("forced stop failure") + + selector_patch = ( + mock.patch.object( + transport.selectors, + "DefaultSelector", + side_effect=RuntimeError("forced selector failure"), + ) + if selector_failure + else mock.patch.object( + backend, + "_stop_process", + side_effect=stop_raises, + ) + ) + command = ( + sys.executable, + "-c", + "pass" if selector_failure else "import time; time.sleep(5)", + ) + with tempfile.TemporaryDirectory() as temporary, mock.patch.object( + transport.subprocess, + "Popen", + side_effect=spawn, + ), mock.patch.object( + backend, + "_cleanup_container", + side_effect=cleanup, + ), selector_patch: + try: + result: object = backend._observe_command( + command, + stdout_limit=64, + stderr_limit=64, + timeout_ns=1 if not selector_failure else 1_000_000_000, + cid_file=Path(temporary).resolve() / "container.cid", + container_name=( + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1.container_name_prefix + + ("selector-red" if selector_failure else "stop-red") + ), + ) + except Exception as error: + result = error + self.assertEqual(len(spawned), 1) + process = spawned[0] + streams_closed = bool( + process.stdout is not None + and process.stdout.closed + and process.stderr is not None + and process.stderr.closed + ) + if process.poll() is None: + process.kill() + process.wait(timeout=5) + for stream in (process.stdin, process.stdout, process.stderr): + if stream is not None and not stream.closed: + stream.close() + return result, streams_closed, len(cleanup_calls) + + selector_result, selector_closed, selector_cleanups = exercise( + selector_failure=True + ) + self.assertIs(type(selector_result), transport.DockerBuildObserverFailureV1) + self.assertTrue(selector_closed) + self.assertEqual(selector_cleanups, 1) + + stop_result, stop_closed, stop_cleanups = exercise(selector_failure=False) + self.assertIn( + type(stop_result), + ( + transport.DockerBuildObserverFailureV1, + transport.DockerBuildCleanupFailureV1, + ), + ) + self.assertTrue(stop_closed) + self.assertEqual(stop_cleanups, 1) + + def test_process_and_container_cleanup_failures_are_both_retained_in_order(self) -> None: + transport = importlib.import_module("build.transport") + backend = transport.NativeDockerBuildBackendV1( + Path("/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + host_user=(501, 20), + platform_name="linux", + machine_name="x86_64", + ) + real_popen = subprocess.Popen + + def stop(process: subprocess.Popen[bytes]) -> str: + process.kill() + process.wait(timeout=5) + return "process stop failed" + + with tempfile.TemporaryDirectory() as temporary, mock.patch.object( + transport.subprocess, + "Popen", + side_effect=real_popen, + ), mock.patch.object( + backend, + "_stop_process", + side_effect=stop, + ), mock.patch.object( + backend, + "_cleanup_container", + return_value="container cleanup failed", + ): + result = backend._observe_command( + (sys.executable, "-c", "import time; time.sleep(5)"), + stdout_limit=64, + stderr_limit=64, + timeout_ns=1, + cid_file=Path(temporary).resolve() / "container.cid", + container_name=( + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1.container_name_prefix + + "cleanup-records" + ), + input_bundle=_sealed_input(), + ) + + self.assertIs(type(result), transport.DockerBuildCleanupFailureV1) + self.assertEqual( + tuple(record.resource for record in result.failures), + ( + transport.CleanupResourceV1.DOCKER_CLI_PROCESS, + transport.CleanupResourceV1.DOCKER_CONTAINER, + ), + ) + self.assertEqual( + tuple(record.detail for record in result.failures), + ("process stop failed", "container cleanup failed"), + ) + self.assertTrue( + all( + type(record) is transport.CleanupFailureRecordV1 + for record in result.failures + ) + ) + + def test_impossible_build_failures_without_input_progress_are_contract_violations(self) -> None: + transport = importlib.import_module("build.transport") + policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 + cleanup = transport.CleanupFailureRecordV1( + transport.CleanupResourceV1.DOCKER_CONTAINER, + "container cleanup failed", + ) + impossible = ( + transport.DockerBuildTimedOutV1(b"", b""), + transport.DockerBuildOutputLimitV1( + transport.DockerOutputStreamV1.STDOUT, + b"x" * 64, + b"", + ), + transport.DockerBuildCleanupFailureV1( + transport.DockerCleanupTriggerV1.TIMEOUT, + (cleanup,), + b"", + b"", + ), + ) + for observation in impossible: + with self.subTest(observation=type(observation).__name__): + result, _backend, _report, _input = _controlled_build( + transport, + policy, + (observation,), + ) + self.assertIs(type(result), transport.BuildRejectedV1) + self.assertEqual( + result.reason, + transport.BuildFailureReasonV1.CONTRACT_VIOLATION, + ) + self.assertIsNone(result.process) + + def test_temporary_root_cleanup_failure_retains_current_process_generically(self) -> None: + transport = importlib.import_module("build.transport") + policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 + input_value = _sealed_input() + first = _completed_process(transport, input_value, b"first") + current = _completed_process(transport, input_value, b"second") + real_temporary_directory = tempfile.TemporaryDirectory + allocated: list[object] = [] + allocations = 0 + + class CleanupFailsOnce: + def __init__(self, *args: object, **kwargs: object) -> None: + self._inner = real_temporary_directory(*args, **kwargs) + self.name = self._inner.name + self._failed = False + allocated.append(self) + + def __enter__(self) -> str: + return self.name + + def __exit__(self, *_args: object) -> None: + self.cleanup() + + def cleanup(self) -> None: + if not self._failed: + self._failed = True + raise OSError("forced temporary-root cleanup failure") + self._inner.cleanup() + + def temporary_directory(*args: object, **kwargs: object) -> object: + nonlocal allocations + allocations += 1 + if allocations == 1: + return real_temporary_directory(*args, **kwargs) + return CleanupFailsOnce(*args, **kwargs) + + try: + with mock.patch.object( + transport.tempfile, + "TemporaryDirectory", + side_effect=temporary_directory, + ): + result, _backend, _report, _input = _controlled_build( + transport, + policy, + (first, current), + input_value=input_value, + ) + finally: + for temporary in allocated: + temporary.cleanup() + + self.assertIs(type(result), transport.BuildRejectedV1) + self.assertEqual( + result.reason, + transport.BuildFailureReasonV1.CLEANUP_FAILED, + ) + self.assertEqual(result.attempt, 2) + self.assertEqual(result.completed_processes, (first,)) + self.assertIs(type(result.process), transport.BuildCleanupFailureV1) + self.assertIs(result.process.current_process, current) + self.assertEqual(len(result.process.failures), 1) + self.assertEqual( + result.process.failures[0].resource, + transport.CleanupResourceV1.TEMPORARY_ROOT, + ) + + def test_temporary_root_cleanup_failure_is_typed_without_a_process(self) -> None: + transport = importlib.import_module("build.transport") + policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 + real_temporary_directory = tempfile.TemporaryDirectory + inner = real_temporary_directory() + + class CleanupFailsOnce: + name = inner.name + + def __init__(self) -> None: + self.failed = False + + def cleanup(self) -> None: + if not self.failed: + self.failed = True + raise OSError("forced temporary-root cleanup failure") + inner.cleanup() + + temporary = CleanupFailsOnce() + try: + with mock.patch.object( + transport.tempfile, + "TemporaryDirectory", + return_value=temporary, + ): + result, _backend, _capability, _input = _controlled_build( + transport, + policy, + (), + ) + finally: + temporary.cleanup() + + self.assertIs(type(result), transport.BuildRejectedV1) + self.assertEqual( + result.reason, + transport.BuildFailureReasonV1.CLEANUP_FAILED, + ) + self.assertIs(type(result.process), transport.BuildCleanupFailureV1) + self.assertIsNone(result.process.current_process) + self.assertEqual( + tuple(record.resource for record in result.process.failures), + (transport.CleanupResourceV1.TEMPORARY_ROOT,), + ) + + def test_forged_backend_failures_canonicalize_to_contract_violation(self) -> None: + transport = importlib.import_module("build.transport") + policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 + for failure_type in ( + transport.DockerBuildTimedOutV1, + transport.DockerBuildOutputLimitV1, + transport.DockerBuildObserverFailureV1, + transport.DockerBuildInputRejectedV1, + transport.DockerBuildCleanupFailureV1, + transport.BuildCleanupFailureV1, + ): + with self.subTest(failure=failure_type.__name__): + forged = _forged_exact_type(failure_type) + if hasattr(forged, "__dict__"): + object.__setattr__(forged, "foreign", object()) + result, _backend, _report, _input = _controlled_build( + transport, + policy, + (forged,), + ) + self.assertIs(type(result), transport.BuildRejectedV1) + self.assertEqual( + result.reason, + transport.BuildFailureReasonV1.CONTRACT_VIOLATION, + ) + self.assertIsNone(result.process) + self.assertEqual(result.completed_processes, ()) + + def test_observer_failure_evidence_is_bounded_and_progress_is_exact(self) -> None: + transport = importlib.import_module("build.transport") + policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 + input_value = _sealed_input() + foreign_input = importlib.import_module("build.input").seal_input_v1( + _digest("foreign-build-binding"), + input_value.contents, + ) + cases = ( + transport.DockerBuildObserverFailureV1( + "oversized stdout", + b"x" * 65, + b"", + _initial_progress(transport, input_value), + ), + transport.DockerBuildObserverFailureV1( + "foreign progress", + b"", + b"", + _initial_progress(transport, foreign_input), + ), + ) + for observation in cases: + with self.subTest(detail=observation.detail): + result, _backend, _report, _input = _controlled_build( + transport, + policy, + (observation,), + input_value=input_value, + ) + self.assertIs(type(result), transport.BuildRejectedV1) + self.assertEqual( + result.reason, + transport.BuildFailureReasonV1.CONTRACT_VIOLATION, + ) + self.assertIsNone(result.process) + + def test_top_level_failure_reason_preserves_observer_outcome(self) -> None: + transport = importlib.import_module("build.transport") + policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 + expected_reasons = ( + transport.BuildFailureReasonV1.TIMEOUT, + transport.BuildFailureReasonV1.OUTPUT_LIMIT, + transport.BuildFailureReasonV1.OBSERVER_FAILURE, + transport.BuildFailureReasonV1.PROCESS_FAILED, + ) + input_value = _sealed_input() + progress = _initial_progress(transport, input_value) + observations = ( + transport.DockerBuildTimedOutV1(b"", b"", progress), + transport.DockerBuildOutputLimitV1( + transport.DockerOutputStreamV1.STDOUT, + b"x" * 64, + b"", + progress, + ), + transport.DockerBuildObserverFailureV1( + "observer failed", + b"observer stdout", + b"observer stderr", + progress, + ), + _completed_process( + transport, + input_value, + b"", + returncode=7, + ), + ) + for observation, reason in zip( + observations, + expected_reasons, + strict=True, + ): + with self.subTest(reason=reason): + result, _backend, _report, _input = _controlled_build( + transport, + policy, + (observation,), + input_value=input_value, + ) + self.assertIs(type(result), transport.BuildRejectedV1) + self.assertEqual(result.reason, reason) + + def test_second_attempt_failure_retains_first_completed_process(self) -> None: + transport = importlib.import_module("build.transport") + policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 + input_value = _sealed_input() + first = _completed_process(transport, input_value, b"first") + progress = _initial_progress(transport, input_value) + result, _backend, _report, _input = _controlled_build( + transport, + policy, + (first, transport.DockerBuildTimedOutV1(b"", b"", progress)), + input_value=input_value, + ) + self.assertIs(type(result), transport.BuildRejectedV1) + self.assertEqual(result.attempt, 2) + self.assertTrue(hasattr(result, "completed_processes")) + self.assertIs(type(result.completed_processes), tuple) + self.assertEqual(result.completed_processes, (first,)) + self.assertIs(result.completed_processes[0], first) + self.assertFalse(hasattr(result, "__dict__")) + with self.assertRaises((AttributeError, TypeError)): + object.__setattr__(result, "completed_processes", ()) + + def test_two_build_observation_derives_byte_relation_and_binds_session(self) -> None: + transport = importlib.import_module("build.transport") + self.assertTrue(hasattr(transport, "BuildByteRelationV1")) + self.assertTrue(hasattr(transport, "TwoBuildObservationV1")) + self.assertFalse(hasattr(transport, "ReproducibleBuildV1")) + self.assertFalse(hasattr(transport, "NonReproducibleBuildV1")) + policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 + for outputs, relation in ( + ((b"same", b"same"), transport.BuildByteRelationV1.IDENTICAL), + ((b"first", b"second"), transport.BuildByteRelationV1.DIFFERENT), + ): + with self.subTest(relation=relation): + input_value = _sealed_input() + processes = tuple( + _completed_process(transport, input_value, output) + for output in outputs + ) + result, _backend, capability, _input = _controlled_build( + transport, + policy, + processes, + max_output_bytes=64, + input_value=input_value, + ) + self.assertIs(type(result), transport.TwoBuildObservationV1) + self.assertEqual(result.relation, relation) + self.assertEqual(result.policy, policy) + self.assertEqual(result.capability, capability) + self.assertIs(result.input_value, input_value) + self.assertEqual(result.max_output_bytes, 64) + self.assertEqual(result.processes, processes) + self.assertEqual( + result.relation, + ( + transport.BuildByteRelationV1.IDENTICAL + if processes[0].stdout == processes[1].stdout + else transport.BuildByteRelationV1.DIFFERENT + ), + ) + + def test_build_process_encoding_is_total_and_keeps_exact_golden(self) -> None: + transport = importlib.import_module("build.transport") + result, _backend = _execute() + process = result.evidence.build.build_processes[0] + encoded = transport.build_process_bytes_v1(process) + self.assertEqual(len(encoded), 196) + self.assertEqual( + hashlib.sha256(encoded).hexdigest(), + "401aaf23753b09b35482080e6046499e6a8a0a4ea2cea6c658ed377efebac58c", + ) + + forged = tuple.__new__(transport.DockerBuildExitedV1, ()) + with self.assertRaises(TypeError): + transport.build_process_bytes_v1(forged) + overflow = tuple.__new__( + transport.DockerBuildExitedV1, + ( + 1 << 40, + process.stdout, + process.stderr, + process.input_transfer, + ), + ) + with self.assertRaises(TypeError): + transport.build_process_bytes_v1(overflow) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/proof/region/v1/tests/test_build_identity.py b/proof/region/v1/tests/test_build_identity.py new file mode 100644 index 00000000..4df559a2 --- /dev/null +++ b/proof/region/v1/tests/test_build_identity.py @@ -0,0 +1,809 @@ +#!/usr/bin/env python3 +"""RED contract for orthogonal Docker BUILD capability identities.""" + +from __future__ import annotations + +import hashlib +import inspect +import json +import os +import sys +import unittest +from pathlib import Path +from unittest import mock + + +PROOF = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(PROOF)) + +from build import input as build_input # noqa: E402 +from build import transport # noqa: E402 + + +_POLICY_FIELDS_V1 = ( + "image_reference", + "platform", + "hostname", + "container_name_prefix", + "bootstrap", + "bootstrap_argv0", + "tmpfs_specs", + "user_mode", + "stdout_limit", + "stderr_limit", + "build_timeout_ns", + "probe_output_limit", + "probe_timeout_ns", +) + +def _literal(value: str) -> tuple[str, str]: + return "literal", value + + +def _slot(value: str) -> tuple[str, str]: + return "slot", value + + +_NATIVE_COMMAND_TEMPLATES_V1 = ( + ( + "version_probe", + ( + _slot("cli_path"), + _literal("version"), + _literal("--format"), + _literal("{{json .Server}}"), + ), + ), + ( + "image_inspect", + ( + _slot("cli_path"), + _literal("image"), + _literal("inspect"), + _slot("image_reference"), + ), + ), + ( + "build", + ( + _slot("cli_path"), + _literal("run"), + _literal("--rm"), + _literal("--interactive"), + _literal("--pull"), + _literal("never"), + _literal("--platform"), + _slot("platform"), + _literal("--network"), + _literal("none"), + _literal("--read-only"), + _literal("--tmpfs"), + _slot("ordered_tmpfs_specs"), + _literal("--cap-drop"), + _literal("ALL"), + _literal("--security-opt"), + _literal("no-new-privileges:true"), + _literal("--name"), + _slot("container_name"), + _literal("--hostname"), + _slot("hostname"), + _literal("--user"), + _slot("host_user"), + _literal("--workdir"), + _literal("/"), + _literal("--cidfile"), + _slot("cid_file"), + _literal("--entrypoint"), + _literal("/usr/bin/env"), + _slot("image_reference"), + _literal("-i"), + _literal("PATH=/usr/local/bin:/usr/bin:/bin"), + _literal("LC_ALL=C"), + _literal("LANG=C"), + _literal("TZ=UTC"), + _literal("HOME=/nonexistent"), + _literal("/bin/sh"), + _literal("-c"), + _slot("bootstrap"), + _slot("bootstrap_argv0"), + _slot("input_length"), + _slot("input_sha256"), + ), + ), + ( + "cleanup_rm", + ( + _slot("cli_path"), + _literal("container"), + _literal("rm"), + _literal("--force"), + _slot("container_coordinate"), + ), + ), + ( + "cleanup_ls", + ( + _slot("cli_path"), + _literal("container"), + _literal("ls"), + _literal("--all"), + _literal("--quiet"), + _literal("--no-trunc"), + _literal("--filter"), + _slot("container_filter"), + ), + ), +) + + +def _blob(value: bytes) -> bytes: + return len(value).to_bytes(8, "big") + value + + +def _identity(label: bytes, chunks: tuple[bytes, ...]) -> bytes: + payload = b"".join(_blob(chunk) for chunk in chunks) + return hashlib.sha256( + label + len(payload).to_bytes(8, "big") + payload + ).digest() + + +def _policy_coordinates(policy: object) -> dict[str, object]: + return {name: getattr(policy, name) for name in _POLICY_FIELDS_V1} + + +def _policy_chunks(coordinates: dict[str, object]) -> tuple[bytes, ...]: + tmpfs_specs = coordinates["tmpfs_specs"] + user_mode = coordinates["user_mode"] + assert type(tmpfs_specs) is tuple + return ( + coordinates["image_reference"].encode("utf-8"), + coordinates["platform"].encode("utf-8"), + coordinates["hostname"].encode("utf-8"), + coordinates["container_name_prefix"].encode("utf-8"), + coordinates["bootstrap"].encode("utf-8"), + coordinates["bootstrap_argv0"].encode("utf-8"), + len(tmpfs_specs).to_bytes(4, "big"), + *(item.encode("utf-8") for item in tmpfs_specs), + user_mode.value.encode("ascii"), + coordinates["stdout_limit"].to_bytes(8, "big"), + coordinates["stderr_limit"].to_bytes(8, "big"), + coordinates["build_timeout_ns"].to_bytes(8, "big"), + coordinates["probe_output_limit"].to_bytes(8, "big"), + coordinates["probe_timeout_ns"].to_bytes(8, "big"), + ) + + +def _expected_policy_identity(coordinates: dict[str, object]) -> bytes: + return _identity( + b"labcolors.proof-region.docker-transport-policy.v1\0", + _policy_chunks(coordinates), + ) + + +def _expected_command_contract_identity() -> bytes: + chunks: list[bytes] = [len(_NATIVE_COMMAND_TEMPLATES_V1).to_bytes(4, "big")] + for name, tokens in _NATIVE_COMMAND_TEMPLATES_V1: + chunks.extend((name.encode("ascii"), len(tokens).to_bytes(4, "big"))) + for tag, value in tokens: + chunks.extend((tag.encode("ascii"), value.encode("utf-8"))) + return _identity( + b"labcolors.proof-region.native-command-contract.v1\0", + tuple(chunks), + ) + + +def _expected_command_coordinate(docker_path: Path) -> bytes: + return _identity( + b"labcolors.proof-region.native-command-coordinate.v1\0", + (_expected_command_contract_identity(), os.fsencode(docker_path)), + ) + + +def _expected_daemon_identity( + server_stdout: bytes, + image_inspect_stdout: bytes, +) -> bytes: + return _identity( + b"labcolors.proof-region.docker-daemon-observation.v1\0", + (server_stdout, image_inspect_stdout), + ) + + +def _expected_host_user_identity(host_user: tuple[int, int]) -> bytes: + return _identity( + b"labcolors.proof-region.host-user.v1\0", + ( + host_user[0].to_bytes(4, "big"), + host_user[1].to_bytes(4, "big"), + ), + ) + + +def _expected_capability_identity( + policy_identity: bytes, + daemon_identity: bytes, + command_coordinate: bytes, + host_user: tuple[int, int], +) -> bytes: + return _identity( + b"labcolors.proof-region.docker-capability.v1\0", + ( + policy_identity, + command_coordinate, + daemon_identity, + host_user[0].to_bytes(4, "big"), + host_user[1].to_bytes(4, "big"), + ), + ) + + +def _policy(**changes: object) -> object: + coordinates: dict[str, object] = { + "image_reference": ( + "registry.example/toolchain@sha256:" + "1" * 64 + ), + "platform": "linux/amd64", + "hostname": "lc-build", + "container_name_prefix": "lc-build-", + "bootstrap": "set -eu\ncat", + "bootstrap_argv0": "labcolors-build-v1", + "tmpfs_specs": ( + "/work:rw,nosuid,nodev,noexec,size=1048576", + "/tmp:rw,nosuid,nodev,noexec,size=2097152", + ), + "user_mode": transport.DockerUserModeV1.HOST_EFFECTIVE_IDS, + "stdout_limit": 4096, + "stderr_limit": 2048, + "build_timeout_ns": 5_000_000_000, + "probe_output_limit": 1024, + "probe_timeout_ns": 1_000_000_000, + } + coordinates.update(changes) + return transport.DockerBuildPolicyV1(**coordinates) + + +def _daemon( + *, + server_stdout: bytes = b'{"Version":"identity-test"}\n', + image_inspect_stdout: bytes = b'[{"Id":"sha256:identity-test"}]\n', +) -> object: + return transport.DockerDaemonObservationV1( + server_stdout, + image_inspect_stdout, + ) + + +def _capability( + *, + policy: object | None = None, + daemon: object | None = None, + docker_path: Path = Path("/usr/bin/true"), + host_user: tuple[int, int] = (501, 20), +) -> object: + owned_policy = _policy() if policy is None else policy + owned_daemon = _daemon() if daemon is None else daemon + return transport.DockerSupportedV1( + owned_policy, + owned_daemon, + transport.native_command_coordinate_v1(docker_path), + host_user, + ) + + +def _sealed_input() -> object: + return build_input.seal_input_v1( + hashlib.sha256(b"build-identity-test-binding").digest(), + b"identity-test-input", + ) + + +def _assert_deeply_immutable( + case: unittest.TestCase, + value: object, +) -> None: + case.assertFalse(hasattr(value, "__dict__"), type(value).__name__) + with case.assertRaises((AttributeError, TypeError)): + value[0] = value[0] + with case.assertRaises((AttributeError, TypeError)): + object.__setattr__(value, "foreign", object()) + + +class BuildIdentitySurfaceTests(unittest.TestCase): + def test_identity_surface_is_exact_and_has_no_legacy_report_aliases(self) -> None: + for name in ( + "DockerDaemonObservationV1", + "NativeCommandCoordinateV1", + "transport_policy_identity_v1", + "native_command_contract_identity_v1", + "native_command_coordinate_v1", + "docker_capability_identity_v1", + ): + with self.subTest(required=name): + self.assertTrue(hasattr(transport, name), name) + + self.assertEqual( + tuple(inspect.signature(transport.DockerDaemonObservationV1).parameters), + ("server_stdout", "image_inspect_stdout"), + ) + self.assertEqual( + tuple(inspect.signature(transport.DockerSupportedV1).parameters), + ( + "policy", + "daemon_observation", + "command_coordinate", + "host_user", + ), + ) + self.assertEqual( + tuple(inspect.signature(transport.DockerBuildRequestV1).parameters), + ( + "attempt", + "capability", + "input_bundle", + "max_output_bytes", + "cid_file", + "container_name", + ), + ) + self.assertFalse(hasattr(transport, "docker_report_matches_policy_v1")) + self.assertNotIn( + "docker_report", + Path(transport.__file__).read_text(encoding="utf-8"), + ) + + capability = _capability() + request = transport.DockerBuildRequestV1( + 1, + capability, + _sealed_input(), + 64, + Path("/tmp/lab-colors-identity.cid"), + capability.policy.container_name_prefix + "identity", + ) + for legacy in ( + "image_reference", + "platform", + "daemon_observation_sha256", + ): + with self.subTest(legacy_capability_property=legacy): + self.assertFalse(hasattr(capability, legacy)) + self.assertFalse(hasattr(request, "policy")) + self.assertIs(request.capability, capability) + + with self.assertRaises(TypeError): + transport.DockerSupportedV1( + capability.policy.image_reference, + capability.policy.platform, + capability.daemon_observation.identity, + capability.host_user, + ) + + def test_policy_identity_binds_all_thirteen_coordinates(self) -> None: + policy = _policy() + coordinates = _policy_coordinates(policy) + identity = transport.transport_policy_identity_v1(policy) + + self.assertEqual(identity, _expected_policy_identity(coordinates)) + self.assertIs(type(identity), bytes) + self.assertEqual(len(identity), 32) + + # V1 has one admitted platform and one admitted user mode. Their + # mutation cannot be represented as a valid policy, so the independent + # literal preimage and source guard prove that neither closed-domain + # coordinate silently disappears from the versioned identity. + source = inspect.getsource(transport.transport_policy_identity_v1) + for field_name in _POLICY_FIELDS_V1: + with self.subTest(source_coordinate=field_name): + self.assertIn(f".{field_name}", source) + + raw_mutations: dict[str, object] = { + "image_reference": ( + "registry.example/toolchain@sha256:" + "2" * 64 + ), + "platform": "linux/arm64", + "hostname": "lc-build-alt", + "container_name_prefix": "lc-alt-", + "bootstrap": "set -eu\nprintf changed", + "bootstrap_argv0": "labcolors-build-v1-alt", + "tmpfs_specs": coordinates["tmpfs_specs"] + ("/run:rw,size=4096",), + "user_mode": _AlternateUserMode("explicit_ids"), + "stdout_limit": coordinates["stdout_limit"] + 1, + "stderr_limit": coordinates["stderr_limit"] + 1, + "build_timeout_ns": coordinates["build_timeout_ns"] + 1, + "probe_output_limit": coordinates["probe_output_limit"] + 1, + "probe_timeout_ns": coordinates["probe_timeout_ns"] + 1, + } + for field_name, changed_value in raw_mutations.items(): + with self.subTest(preimage_coordinate=field_name): + changed = dict(coordinates) + changed[field_name] = changed_value + self.assertNotEqual( + _expected_policy_identity(changed), + _expected_policy_identity(coordinates), + ) + + valid_mutations = { + key: value + for key, value in raw_mutations.items() + if key not in ("platform", "user_mode") + } + for field_name, changed_value in valid_mutations.items(): + with self.subTest(runtime_coordinate=field_name): + changed_policy = _policy(**{field_name: changed_value}) + self.assertNotEqual( + transport.transport_policy_identity_v1(changed_policy), + identity, + ) + + +class _AlternateUserMode: + """Test-only value with the encoder surface of the closed production enum.""" + + def __init__(self, value: str) -> None: + self.value = value + + +class BuildCapabilityIdentityTests(unittest.TestCase): + def test_daemon_identity_owns_only_the_two_raw_probe_outputs(self) -> None: + server_stdout = b'{"Version":"26.1.4","Os":"linux"}\n' + image_stdout = b'[{"Os":"linux","Architecture":"amd64"}]\n' + daemon = transport.DockerDaemonObservationV1( + server_stdout, + image_stdout, + ) + + self.assertEqual(daemon.server_stdout, server_stdout) + self.assertEqual(daemon.image_inspect_stdout, image_stdout) + self.assertEqual( + daemon.identity, + _expected_daemon_identity(server_stdout, image_stdout), + ) + self.assertEqual( + transport.DockerDaemonObservationV1( + server_stdout, + image_stdout, + ).identity, + daemon.identity, + ) + self.assertNotEqual( + transport.DockerDaemonObservationV1( + server_stdout + b" ", + image_stdout, + ).identity, + daemon.identity, + ) + self.assertNotEqual( + transport.DockerDaemonObservationV1( + server_stdout, + image_stdout + b" ", + ).identity, + daemon.identity, + ) + _assert_deeply_immutable(self, daemon) + + def test_native_command_coordinate_binds_path_and_literal_template(self) -> None: + first_path = Path("/usr/bin/true") + second_path = Path("/usr/bin/false") + expected_contract = _expected_command_contract_identity() + first = transport.native_command_coordinate_v1(first_path) + second = transport.native_command_coordinate_v1(second_path) + + self.assertEqual( + transport.native_command_contract_identity_v1(), + expected_contract, + ) + self.assertEqual( + transport.native_command_contract_identity_v1(), + transport.native_command_contract_identity_v1(), + ) + self.assertIs(type(first), transport.NativeCommandCoordinateV1) + self.assertEqual(first.path, first_path) + self.assertEqual(first.path_bytes, os.fsencode(first_path)) + self.assertEqual(first.command_contract_identity, expected_contract) + self.assertEqual(first.identity, _expected_command_coordinate(first_path)) + self.assertEqual(second.path, second_path) + self.assertEqual(second.identity, _expected_command_coordinate(second_path)) + self.assertNotEqual(first.identity, second.identity) + _assert_deeply_immutable(self, first) + + def test_capability_identity_keeps_policy_daemon_path_and_user_orthogonal(self) -> None: + policy = _policy() + daemon = _daemon() + command_coordinate = transport.native_command_coordinate_v1( + Path("/usr/bin/true") + ) + host_user = (501, 20) + capability = transport.DockerSupportedV1( + policy, + daemon, + command_coordinate, + host_user, + ) + expected_policy_identity = transport.transport_policy_identity_v1(policy) + expected = _expected_capability_identity( + expected_policy_identity, + daemon.identity, + command_coordinate.identity, + host_user, + ) + + self.assertIs(capability.policy, policy) + self.assertIs(capability.daemon_observation, daemon) + self.assertIs(capability.command_coordinate, command_coordinate) + self.assertEqual(capability.host_user, host_user) + self.assertEqual(capability.policy_identity, expected_policy_identity) + self.assertEqual( + capability.daemon_observation_identity, + daemon.identity, + ) + self.assertEqual( + capability.command_coordinate_identity, + command_coordinate.identity, + ) + self.assertEqual( + capability.host_user_identity, + _expected_host_user_identity(host_user), + ) + self.assertEqual(capability.identity, expected) + self.assertEqual( + transport.docker_capability_identity_v1(capability), + expected, + ) + + variants = ( + _capability(policy=_policy(hostname="lc-build-other"), daemon=daemon), + _capability( + policy=policy, + daemon=_daemon(server_stdout=b'{"Version":"other"}\n'), + ), + _capability(policy=policy, daemon=daemon, docker_path=Path("/bin/sh")), + _capability(policy=policy, daemon=daemon, host_user=(502, 20)), + _capability(policy=policy, daemon=daemon, host_user=(501, 21)), + ) + for variant in variants: + with self.subTest(component=variant): + self.assertNotEqual(variant.identity, capability.identity) + + # Changing outer capability coordinates never contaminates the raw + # daemon-observation identity. + self.assertTrue( + all( + variant.daemon_observation_identity + == variant.daemon_observation.identity + for variant in variants + ) + ) + self.assertEqual( + variants[0].daemon_observation_identity, + daemon.identity, + ) + self.assertEqual( + variants[2].daemon_observation_identity, + daemon.identity, + ) + self.assertEqual( + variants[3].daemon_observation_identity, + daemon.identity, + ) + self.assertEqual( + variants[4].daemon_observation_identity, + daemon.identity, + ) + + for value in (policy, capability): + with self.subTest(immutable=type(value).__name__): + _assert_deeply_immutable(self, value) + + +class NativeCommandAndRequestTests(unittest.TestCase): + def test_native_backend_requires_its_exact_probe_lease(self) -> None: + policy = _policy() + backend = transport.NativeDockerBuildBackendV1( + Path("/usr/bin/true"), + policy, + platform_name="linux", + machine_name="x86_64", + host_user=(501, 20), + ) + unobserved = _capability(policy=policy) + request = transport.DockerBuildRequestV1( + 1, + unobserved, + _sealed_input(), + 64, + Path("/tmp/lab-colors-identity.cid"), + policy.container_name_prefix + "identity", + ) + with self.assertRaises(TypeError): + backend.command_for(request) + + server_stdout = b'{"Version":"identity-test"}\n' + image_stdout = json.dumps( + [ + { + "Os": "linux", + "Architecture": "amd64", + "RepoDigests": [policy.image_reference], + } + ], + separators=(",", ":"), + ).encode("ascii") + observed = ( + transport._docker_command_exited_v1(0, server_stdout, b""), + transport._docker_command_exited_v1(0, image_stdout, b""), + ) + with mock.patch.object(backend, "_observe_command", side_effect=observed): + capability = backend.probe() + self.assertIs(type(capability), transport.DockerSupportedV1) + + equal_but_foreign = transport.DockerSupportedV1(*tuple(capability)) + self.assertEqual(equal_but_foreign, capability) + self.assertIsNot(equal_but_foreign, capability) + cloned_request = transport.DockerBuildRequestV1( + 1, + equal_but_foreign, + _sealed_input(), + 64, + Path("/tmp/lab-colors-identity.cid"), + policy.container_name_prefix + "identity", + ) + with self.assertRaises(TypeError): + backend.command_for(cloned_request) + + def test_command_for_expands_the_versioned_template_to_exact_argv(self) -> None: + policy = _policy() + docker_path = Path("/usr/bin/true") + backend = transport.NativeDockerBuildBackendV1( + docker_path, + policy, + platform_name="linux", + machine_name="x86_64", + host_user=(501, 20), + ) + server_stdout = b'{"Version":"identity-test"}\n' + image_stdout = json.dumps( + [ + { + "Os": "linux", + "Architecture": "amd64", + "RepoDigests": [policy.image_reference], + } + ], + separators=(",", ":"), + ).encode("ascii") + observed = ( + transport._docker_command_exited_v1(0, server_stdout, b""), + transport._docker_command_exited_v1(0, image_stdout, b""), + ) + with mock.patch.object( + backend, + "_observe_command", + side_effect=observed, + ): + capability = backend.probe() + self.assertIs(type(capability), transport.DockerSupportedV1) + + input_bundle = _sealed_input() + cid_file = Path("/tmp/lab-colors-identity.cid") + container_name = policy.container_name_prefix + "identity" + request = transport.DockerBuildRequestV1( + 1, + capability, + input_bundle, + 64, + cid_file, + container_name, + ) + command = backend.command_for(request) + expected = ( + str(docker_path), + "run", + "--rm", + "--interactive", + "--pull", + "never", + "--platform", + policy.platform, + "--network", + "none", + "--read-only", + "--tmpfs", + policy.tmpfs_specs[0], + "--tmpfs", + policy.tmpfs_specs[1], + "--cap-drop", + "ALL", + "--security-opt", + "no-new-privileges:true", + "--name", + container_name, + "--hostname", + policy.hostname, + "--user", + "501:20", + "--workdir", + "/", + "--cidfile", + str(cid_file), + "--entrypoint", + "/usr/bin/env", + policy.image_reference, + "-i", + "PATH=/usr/local/bin:/usr/bin:/bin", + "LC_ALL=C", + "LANG=C", + "TZ=UTC", + "HOME=/nonexistent", + "/bin/sh", + "-c", + policy.bootstrap, + policy.bootstrap_argv0, + str(input_bundle.length), + input_bundle.sha256.hex(), + ) + self.assertEqual(command, expected) + self.assertEqual(command.count("--tmpfs"), len(policy.tmpfs_specs)) + self.assertLess( + command.index(policy.tmpfs_specs[0]), + command.index(policy.tmpfs_specs[1]), + ) + self.assertEqual( + transport.native_command_contract_identity_v1(), + _expected_command_contract_identity(), + ) + + def test_foreign_capability_is_rejected_before_backend_run(self) -> None: + policy = _policy() + owned = _capability(policy=policy) + foreign = _capability(policy=policy, docker_path=Path("/bin/sh")) + + class Backend: + def __init__(self) -> None: + self.requests: list[object] = [] + + def probe(self) -> object: + return owned + + def run_build(self, request: object) -> object: + self.requests.append(request) + raise AssertionError("foreign capability reached backend") + + backend = Backend() + controller = transport.ControlledBuildTransportV1( + policy=policy, + backend=backend, + ) + self.assertIs(controller.probe(), owned) + result = controller.build( + foreign, + _sealed_input(), + 64, + input_admission=lambda _value: True, + output_admission=lambda _value: True, + ) + self.assertIs(type(result), transport.BuildRejectedV1) + self.assertEqual( + result.reason, + transport.BuildFailureReasonV1.CONTRACT_VIOLATION, + ) + self.assertEqual(backend.requests, []) + + def test_request_is_deeply_immutable_and_owns_capability_not_policy(self) -> None: + capability = _capability() + request = transport.DockerBuildRequestV1( + 1, + capability, + _sealed_input(), + 64, + Path("/tmp/lab-colors-identity.cid"), + capability.policy.container_name_prefix + "identity", + ) + + self.assertIs(request.capability, capability) + self.assertFalse(hasattr(request, "policy")) + _assert_deeply_immutable(self, request) + _assert_deeply_immutable(self, request.capability) + + +if __name__ == "__main__": + unittest.main(verbosity=2) From 2e415d3a92c6d9e08766868ee9f5d9eb465662a1 Mon Sep 17 00:00:00 2001 From: Daniel from Labpics <63733699+lemone112@users.noreply.github.com> Date: Sat, 1 Aug 2026 08:56:37 +0300 Subject: [PATCH 02/20] =?UTF-8?q?Proof:=20=D0=B8=D0=B7=D0=BE=D0=BB=D0=B8?= =?UTF-8?q?=D1=80=D0=BE=D0=B2=D0=B0=D1=82=D1=8C=20CID=20cleanup=20=D0=B8?= =?UTF-8?q?=20lifecycle?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- proof/region/v1/PROTOCOL.md | 27 +- proof/region/v1/arb/pipeline.py | 13 +- proof/region/v1/arb/tests/gate.py | 2 +- .../v1/arb/tests/test_build_identity_v2.py | 5 - proof/region/v1/arb/tests/test_pipeline.py | 389 +++-- proof/region/v1/arb/tests/test_receipt.py | 2 +- proof/region/v1/arb/tests/test_transport.py | 90 +- proof/region/v1/build/transport.py | 1488 ++++++++++++----- proof/region/v1/tests/test_build.py | 1130 +++++++++++-- proof/region/v1/tests/test_build_identity.py | 257 ++- 10 files changed, 2605 insertions(+), 798 deletions(-) diff --git a/proof/region/v1/PROTOCOL.md b/proof/region/v1/PROTOCOL.md index 12447821..c8dc4eea 100644 --- a/proof/region/v1/PROTOCOL.md +++ b/proof/region/v1/PROTOCOL.md @@ -249,15 +249,32 @@ fixture-specific cap. 1. transport policy identity связывает все поля точной policy; 2. native command contract identity связывает один типизированный grammar для - probe, build и cleanup; фактический argv строится только этим grammar; + probe, build и cleanup и один immutable child-launch context + (environment, cwd, umask, stdio topology, FD и session behavior); фактический argv и + Popen kwargs строятся только этими значениями; 3. daemon observation identity связывает только два raw probe stdout; 4. Docker capability identity связывает policy, command contract и exact CLI path, daemon observation и наблюдённые host uid/gid. -`BuildSessionV1` и каждый `DockerBuildRequestV1` сохраняют ту же capability, -те же input bytes и output cap. Чужая либо не полученная текущим probe -capability отвергается до process spawn; ambient path/user повторно не -считываются. `TwoBuildObservationV1` хранит обе успешные попытки и только +`BuildSessionV1` и каждый `DockerBuildRequestV1` сохраняют только ту же +capability, те же input bytes и output cap. Request не содержит host path, +CID file или имя контейнера: native adapter сам создаёт свежий приватный CID +path. Native cleanup поддерживается только в fresh one-job VM workflow Arb: +другой субъект с тем же effective UID либо Docker-daemon authority там не +сосуществует. Права `0700` закрывают лишь cross-UID pathname access и не +аутентифицируют same-UID writer. В этой объявленной operational boundary для +cleanup допускается только полный ID, который Docker записал в CID path; перед +`rm --force ` adapter сверяет, что `docker container inspect` вернул тот же +ID. Имя контейнера и fallback-координата в cleanup не участвуют. Вне этой +границы CID path не является доказательством ownership. Чужая либо не +полученная текущим probe capability отвергается до process spawn; ambient +path/user повторно не считываются. Разрешение принадлежит создавшему process: +fork и конкурентное повторное использование отвергаются до блокировки. После +возврата Popen handle `BaseException` до повторного выброса исходного +interruption запускает детерминированные попытки остановить и reap CLI, закрыть +streams и очистить допущенный container. Во время самого Popen construction +handle может ещё отсутствовать: тогда возможна только best-effort попытка CID +cleanup, без ложного заявления о reap CLI. `TwoBuildObservationV1` хранит обе успешные попытки и только классифицирует их байты как identical или different, не называя пару универсальным доказательством воспроизводимости. При отказе сохраняется весь уже завершённый causal prefix. Transport не знает formula, ELF, comparator или diff --git a/proof/region/v1/arb/pipeline.py b/proof/region/v1/arb/pipeline.py index 299b189f..b581a980 100644 --- a/proof/region/v1/arb/pipeline.py +++ b/proof/region/v1/arb/pipeline.py @@ -65,11 +65,13 @@ for path in sorted(_PINNED_BUILD_SOURCE_SHA256_V1) ) -BUILD_STDOUT_LIMIT_V1 = 16 * 1024 * 1024 -BUILD_STDERR_LIMIT_V1 = 16 * 1024 * 1024 -BUILD_TIMEOUT_NS_V1 = 2 * 60 * 60 * 1_000_000_000 -DOCKER_PROBE_OUTPUT_LIMIT_V1 = 1024 * 1024 -DOCKER_PROBE_TIMEOUT_NS_V1 = 30 * 1_000_000_000 +# The generic transport owns universal observer ceilings. This lane binds to +# those coordinates rather than recreating a coincident copy of the policy. +BUILD_STDOUT_LIMIT_V1 = build_transport.BUILD_STDOUT_LIMIT_V1 +BUILD_STDERR_LIMIT_V1 = build_transport.BUILD_STDERR_LIMIT_V1 +BUILD_TIMEOUT_NS_V1 = build_transport.BUILD_TIMEOUT_NS_V1 +DOCKER_PROBE_OUTPUT_LIMIT_V1 = build_transport.DOCKER_PROBE_OUTPUT_LIMIT_V1 +DOCKER_PROBE_TIMEOUT_NS_V1 = build_transport.DOCKER_PROBE_TIMEOUT_NS_V1 MAX_BUILD_SOURCE_FILE_BYTES_V1 = 16 * 1024 * 1024 MAX_BUILD_SOURCE_TOTAL_BYTES_V1 = 32 * 1024 * 1024 @@ -338,7 +340,6 @@ def admit_build_sources_v1( OCI_IMAGE_REFERENCE_V1, OCI_PLATFORM_V1, "labcolors-arb-build-v1", - "labcolors-arb-build-v1-", _BUILD_BOOTSTRAP_V1, "labcolors-arb-build-bootstrap-v1", (_BUILD_TMPFS_SPEC_V1, _BUILD_STATE_TMPFS_SPEC_V1), diff --git a/proof/region/v1/arb/tests/gate.py b/proof/region/v1/arb/tests/gate.py index bdba130e..50a6e012 100644 --- a/proof/region/v1/arb/tests/gate.py +++ b/proof/region/v1/arb/tests/gate.py @@ -15,7 +15,7 @@ REPO = Path(__file__).resolve().parents[5] sys.path.insert(0, str(REPO)) EXPECTED_TEST_INVENTORY_SHA256 = ( - "383672bd1ac2a2d472fdba33d3ec4c770a897ecd13192ec41e7c12dc1e563219" + "4853e06c6e8c1864bc65e0b4c0cd9cdbe0881e0d5907daecb6c8a9fea42f3643" ) _EVALUATOR_REASON = "set LABCOLORS_ARB_EVALUATOR to the controlled C17 binary" EXPECTED_SKIPS = frozenset( diff --git a/proof/region/v1/arb/tests/test_build_identity_v2.py b/proof/region/v1/arb/tests/test_build_identity_v2.py index bc66e090..d7187a76 100644 --- a/proof/region/v1/arb/tests/test_build_identity_v2.py +++ b/proof/region/v1/arb/tests/test_build_identity_v2.py @@ -30,7 +30,6 @@ "image_reference", "platform", "hostname", - "container_name_prefix", "bootstrap", "bootstrap_argv0", "tmpfs_specs", @@ -94,10 +93,6 @@ def _policy_mutants( ), ), ("hostname", _policy_with(policy, hostname="labcolors-build-mutant")), - ( - "container_name_prefix", - _policy_with(policy, container_name_prefix="labcolors-mutant-"), - ), ("bootstrap", _policy_with(policy, bootstrap=policy.bootstrap + "\n:")), ( "bootstrap_argv0", diff --git a/proof/region/v1/arb/tests/test_pipeline.py b/proof/region/v1/arb/tests/test_pipeline.py index 7fd65557..6bf07ea5 100644 --- a/proof/region/v1/arb/tests/test_pipeline.py +++ b/proof/region/v1/arb/tests/test_pipeline.py @@ -662,7 +662,7 @@ def test_pipeline_policy_identity_binds_the_stream_bootstrap(self) -> None: trust = pipeline.HostTrustBoundaryV1.UNSEALED_LINUX_X64_DOCKER_HOST policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 coordinates = list(policy) - coordinates[4] = policy.bootstrap + "\nexit 1" + coordinates[3] = policy.bootstrap + "\nexit 1" changed_policy = build_transport.DockerBuildPolicyV1(*coordinates) original = pipeline.pipeline_policy_identity_v2(trust, policy) changed = pipeline.pipeline_policy_identity_v2(trust, changed_policy) @@ -673,7 +673,7 @@ def test_pipeline_policy_identity_binds_the_private_tmpfs_policy(self) -> None: trust = pipeline.HostTrustBoundaryV1.UNSEALED_LINUX_X64_DOCKER_HOST policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 coordinates = list(policy) - coordinates[6] = ( + coordinates[5] = ( "/tmp:rw,exec,suid,dev,size=536870912,mode=1777", policy.tmpfs_specs[1], ) @@ -940,73 +940,75 @@ def run_build( class DockerCommandContractTests(unittest.TestCase): def test_command_is_exact_digest_offline_read_only_and_capability_bound(self) -> None: - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary).resolve() - backend = build_transport.NativeDockerBuildBackendV1( - Path("/usr/bin/true"), - pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, - platform_name="linux", - machine_name="x86_64", - host_user=(501, 20), - ) - capability = _probe_native_backend( - backend, - pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, - ) - request = build_transport.DockerBuildRequestV1( - 1, - capability, - pipeline._seal_build_input_bundle_v1(_request()), - _limits().max_executable_bytes, - root / "container.cid", - "labcolors-arb-build-v1-test", - ) - - command = backend.command_for(request) - - joined = " ".join(command) - self.assertEqual(command[0], "/usr/bin/true") - self.assertIn(pipeline.OCI_IMAGE_REFERENCE_V1, command) - self.assertNotIn("gcc:latest", joined) - for fragment in ( - "--pull never", - "--platform linux/amd64", - "--network none", - "--read-only", - "--interactive", - "--cap-drop ALL", - "--security-opt no-new-privileges:true", - "--name labcolors-arb-build-v1-test", - "--rm", - ): - with self.subTest(fragment=fragment): - self.assertIn(fragment, joined) - for forbidden in ("--privileged", "--network host", ":latest"): - self.assertNotIn(forbidden, joined) + backend = build_transport.NativeDockerBuildBackendV1( + Path("/usr/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + platform_name="linux", + machine_name="x86_64", + host_user=(501, 20), + ) + capability = _probe_native_backend( + backend, + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + ) + request = build_transport.DockerBuildRequestV1( + 1, + capability, + pipeline._seal_build_input_bundle_v1(_request()), + _limits().max_executable_bytes, + ) + lease = backend._next_run_lease_v1(capability) + try: + command = backend._command_for_v1(request, lease) + + joined = " ".join(command) + self.assertEqual(command[0], "/usr/bin/true") + self.assertIn(pipeline.OCI_IMAGE_REFERENCE_V1, command) + self.assertNotIn("gcc:latest", joined) + for fragment in ( + "--pull never", + "--platform linux/amd64", + "--network none", + "--read-only", + "--interactive", + "--cap-drop ALL", + "--security-opt no-new-privileges:true", + f"--cidfile {lease.cid_file}", + "--rm", + ): + with self.subTest(fragment=fragment): + self.assertIn(fragment, joined) + for forbidden in ( + "--name", + "--privileged", + "--network host", + ":latest", + ): + self.assertNotIn(forbidden, joined) + finally: + backend._release_run_lease_v1(lease) def test_command_exposes_only_a_private_non_executable_standard_tmp(self) -> None: - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary).resolve() - backend = build_transport.NativeDockerBuildBackendV1( - Path("/usr/bin/true"), - pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, - platform_name="linux", - machine_name="x86_64", - host_user=(501, 20), - ) - capability = _probe_native_backend( - backend, - pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, - ) - request = build_transport.DockerBuildRequestV1( - 1, - capability, - pipeline._seal_build_input_bundle_v1(_request()), - _limits().max_executable_bytes, - root / "container.cid", - "labcolors-arb-build-v1-test", - ) - command = backend.command_for(request) + backend = build_transport.NativeDockerBuildBackendV1( + Path("/usr/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + platform_name="linux", + machine_name="x86_64", + host_user=(501, 20), + ) + capability = _probe_native_backend( + backend, + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + ) + request = build_transport.DockerBuildRequestV1( + 1, + capability, + pipeline._seal_build_input_bundle_v1(_request()), + _limits().max_executable_bytes, + ) + lease = backend._next_run_lease_v1(capability) + self.addCleanup(backend._release_run_lease_v1, lease) + command = backend._command_for_v1(request, lease) tmpfs_indexes = tuple( index for index, item in enumerate(command) if item == "--tmpfs" @@ -1060,7 +1062,6 @@ def test_native_command_observer_caps_probe_output_before_allocation(self) -> No stdout_limit=8, stderr_limit=8, timeout_ns=5_000_000_000, - cid_file=None, ) self.assertEqual( @@ -1072,7 +1073,7 @@ def test_native_command_observer_caps_probe_output_before_allocation(self) -> No ), ) - def test_cleanup_falls_back_to_exact_name_for_absent_or_invalid_cidfile(self) -> None: + def test_cleanup_uses_only_a_docker_issued_id_from_its_private_lease(self) -> None: backend = build_transport.NativeDockerBuildBackendV1( Path("/usr/bin/true"), pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, @@ -1080,30 +1081,210 @@ def test_cleanup_falls_back_to_exact_name_for_absent_or_invalid_cidfile(self) -> machine_name="x86_64", host_user=(501, 20), ) - _probe_native_backend(backend, pipeline.ARB_BUILD_TRANSPORT_POLICY_V1) - name = "labcolors-arb-build-v1-cleanup-test" - for cid_contents in (None, b"partial-or-foreign"): - with self.subTest(cid_contents=cid_contents): - with tempfile.TemporaryDirectory() as temporary: - cid_file = Path(temporary) / "container.cid" - if cid_contents is not None: - cid_file.write_bytes(cid_contents) - observations = ( - build_transport._docker_command_exited_v1(1, b"", b"not found"), - build_transport._docker_command_exited_v1(0, b"", b""), - ) + capability = _probe_native_backend( + backend, + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + ) + lease = backend._next_run_lease_v1(capability) + lease.cid_file.write_text("b" * 64 + "\n", encoding="ascii") + backend._mark_run_lease_launched_v1(lease) + foreign_id = "a" * 64 + observations = ( + build_transport._docker_command_exited_v1(0, b"b" * 64 + b"\n", b""), + build_transport._docker_command_exited_v1(0, b"b" * 64 + b"\n", b""), + build_transport._docker_command_exited_v1(0, b"", b""), + ) + try: + with mock.patch.object( + backend, + "_observe_cleanup_command", + side_effect=observations, + ) as observe: + detail = backend._cleanup_container(lease) + + self.assertIsNone(detail) + commands = tuple(call.args[1] for call in observe.call_args_list) + self.assertEqual(len(commands), 3) + self.assertEqual(commands[0][-1], "b" * 64) + self.assertEqual(commands[1][-1], "b" * 64) + self.assertIn("id=" + "b" * 64, commands[2]) + self.assertNotIn( + foreign_id, + " ".join(" ".join(command) for command in commands), + ) + self.assertNotIn("name=", " ".join(" ".join(command) for command in commands)) + finally: + backend._release_run_lease_v1(lease) + + def test_absent_cidfile_never_falls_back_to_a_name_or_docker_io(self) -> None: + backend = build_transport.NativeDockerBuildBackendV1( + Path("/usr/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + platform_name="linux", + machine_name="x86_64", + host_user=(501, 20), + ) + capability = _probe_native_backend( + backend, + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + ) + lease = backend._next_run_lease_v1(capability) + try: + with mock.patch.object(backend, "_observe_cleanup_command") as observe: + self.assertIsNone(backend._cleanup_container(lease)) + observe.assert_not_called() + backend._mark_run_lease_launched_v1(lease) + with mock.patch.object(backend, "_observe_cleanup_command") as observe: + self.assertEqual( + backend._cleanup_container(lease), + "Docker-issued cleanup ID is unavailable", + ) + observe.assert_not_called() + finally: + backend._release_run_lease_v1(lease) + + def test_malformed_or_aliased_cid_never_reaches_destructive_cleanup(self) -> None: + backend = build_transport.NativeDockerBuildBackendV1( + Path("/usr/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + platform_name="linux", + machine_name="x86_64", + host_user=(501, 20), + ) + capability = _probe_native_backend( + backend, + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + ) + for raw, alias in ((b"B" * 64, False), (b"c" * 64, True)): + with self.subTest(alias=alias, raw=raw[:1]): + lease = backend._next_run_lease_v1(capability) + try: + lease.cid_file.write_bytes(raw) + if alias: + os.link(lease.cid_file, lease.cid_file.parent / "cid-alias") + backend._mark_run_lease_launched_v1(lease) with mock.patch.object( backend, "_observe_cleanup_command", - side_effect=observations, ) as observe: - detail = backend._cleanup_container(cid_file, name) + self.assertEqual( + backend._cleanup_container(lease), + "Docker-issued cleanup ID is unavailable", + ) + observe.assert_not_called() + finally: + backend._release_run_lease_v1(lease) + + def test_foreign_name_matching_a_stale_id_never_reaches_rm(self) -> None: + backend = build_transport.NativeDockerBuildBackendV1( + Path("/usr/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + platform_name="linux", + machine_name="x86_64", + host_user=(501, 20), + ) + capability = _probe_native_backend( + backend, + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + ) + lease = backend._next_run_lease_v1(capability) + container_id = "d" * 64 + foreign_id = "e" * 64 + try: + lease.cid_file.write_text(container_id + "\n", encoding="ascii") + backend._mark_run_lease_launched_v1(lease) + with mock.patch.object( + backend, + "_observe_cleanup_command", + return_value=build_transport._docker_command_exited_v1( + 0, + foreign_id.encode("ascii") + b"\n", + b"", + ), + ) as observe: + self.assertEqual( + backend._cleanup_container(lease), + "Docker-issued cleanup ID did not resolve exactly", + ) + commands = tuple(call.args[1] for call in observe.call_args_list) + self.assertEqual(len(commands), 1) + self.assertIn("inspect", commands[0]) + self.assertNotIn("rm", commands[0]) + finally: + backend._release_run_lease_v1(lease) + + def test_cleanup_uses_capability_captured_by_the_run_lease(self) -> None: + backend = build_transport.NativeDockerBuildBackendV1( + Path("/usr/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + platform_name="linux", + machine_name="x86_64", + host_user=(501, 20), + ) + capability = _probe_native_backend( + backend, + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + ) + lease = backend._next_run_lease_v1(capability) + try: + lease.cid_file.write_text("f" * 64 + "\n", encoding="ascii") + backend._mark_run_lease_launched_v1(lease) + backend._probed_capability = None + observations = ( + build_transport._docker_command_exited_v1( + 0, + b"f" * 64 + b"\n", + b"", + ), + build_transport._docker_command_exited_v1( + 0, + b"f" * 64 + b"\n", + b"", + ), + build_transport._docker_command_exited_v1(0, b"", b""), + ) + with mock.patch.object( + backend, + "_observe_cleanup_command", + side_effect=observations, + ) as observe: + self.assertIsNone(backend._cleanup_container(lease)) + self.assertEqual( + tuple(call.args[0] for call in observe.call_args_list), + (capability, capability, capability), + ) + finally: + backend._release_run_lease_v1(lease) - self.assertIsNone(detail) - commands = tuple(call.args[0] for call in observe.call_args_list) - self.assertEqual(commands[0][-1], name) - self.assertIn(f"name=^/{name}$", commands[1]) - self.assertNotIn("partial-or-foreign", " ".join(commands[0])) + def test_cleanup_rejects_another_adapter_lease_before_docker_io(self) -> None: + backend = build_transport.NativeDockerBuildBackendV1( + Path("/usr/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + platform_name="linux", + machine_name="x86_64", + host_user=(501, 20), + ) + _probe_native_backend(backend, pipeline.ARB_BUILD_TRANSPORT_POLICY_V1) + foreign_backend = build_transport.NativeDockerBuildBackendV1( + Path("/usr/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + platform_name="linux", + machine_name="x86_64", + host_user=(501, 20), + ) + foreign_capability = _probe_native_backend( + foreign_backend, + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + ) + foreign_lease = foreign_backend._next_run_lease_v1(foreign_capability) + try: + with mock.patch.object(backend, "_observe_cleanup_command") as observe: + with self.assertRaises(TypeError): + backend._cleanup_container(foreign_lease) + + observe.assert_not_called() + finally: + foreign_backend._release_run_lease_v1(foreign_lease) def test_unverified_container_removal_is_typed_cleanup_failure(self) -> None: backend = build_transport.NativeDockerBuildBackendV1( @@ -1112,21 +1293,23 @@ def test_unverified_container_removal_is_typed_cleanup_failure(self) -> None: platform_name="linux", machine_name="x86_64", ) - with tempfile.TemporaryDirectory() as temporary: - cid_file = Path(temporary).resolve() / "container.cid" - with mock.patch.object( - backend, - "_cleanup_container", - return_value="container absence could not be verified", - ): - result = backend._observe_command( - (sys.executable, "-c", "pass"), - stdout_limit=8, - stderr_limit=8, - timeout_ns=5_000_000_000, - cid_file=cid_file, - container_name="labcolors-arb-build-v1-cleanup-failure", - ) + capability = _docker_capability( + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + docker_path=Path("/bin/sh"), + ) + lease = backend._next_run_lease_v1(capability) + with mock.patch.object( + backend, + "_cleanup_container", + return_value="container absence could not be verified", + ): + result = backend._observe_command( + (sys.executable, "-c", "pass"), + stdout_limit=8, + stderr_limit=8, + timeout_ns=5_000_000_000, + lease=lease, + ) self.assertIs(type(result), build_transport.DockerBuildCleanupFailureV1) self.assertEqual( diff --git a/proof/region/v1/arb/tests/test_receipt.py b/proof/region/v1/arb/tests/test_receipt.py index 95933054..f7e711f7 100644 --- a/proof/region/v1/arb/tests/test_receipt.py +++ b/proof/region/v1/arb/tests/test_receipt.py @@ -225,7 +225,7 @@ def test_source_bound_policy_identity_binds_immutable_coordinates(self) -> None: capability = _docker_capability() self.assertEqual( receipt.source_bound_policy_identity_v2(capability).hex(), - "522f089a81e68062f0db4260b00c6e6e0ed2074322247229a99d4714cc5997a5", + "f223e1a1569ca5cf6251fd012af8a789a75aedd830e3ccb8f13db77d7ac67bd4", ) def test_identity_rejection_remains_typed_at_the_receipt_boundary(self) -> None: diff --git a/proof/region/v1/arb/tests/test_transport.py b/proof/region/v1/arb/tests/test_transport.py index 8011748d..e145df05 100644 --- a/proof/region/v1/arb/tests/test_transport.py +++ b/proof/region/v1/arb/tests/test_transport.py @@ -10,7 +10,6 @@ import subprocess import sys import tarfile -import tempfile import unittest from pathlib import Path from unittest import mock @@ -69,7 +68,6 @@ def _observe( stdout_limit=stdout_limit, stderr_limit=stderr_limit, timeout_ns=timeout_ns, - cid_file=None, input_bundle=bundle, ) @@ -478,20 +476,28 @@ def test_timeout_and_output_limit_preserve_input_progress(self) -> None: def test_cleanup_failure_preserves_input_trigger_and_progress(self) -> None: bundle = _bundle() backend = _backend() - with tempfile.TemporaryDirectory() as temporary, mock.patch.object( - backend, - "_cleanup_container", - return_value="forced cleanup failure", - ): - result = backend._observe_command( - (sys.executable, "-c", "import os,time; os.close(0); time.sleep(1)"), - stdout_limit=1024, - stderr_limit=1024, - timeout_ns=5_000_000_000, - cid_file=Path(temporary).resolve() / "container.cid", - container_name="labcolors-arb-build-v1-transport-test", - input_bundle=bundle, + lease = backend._next_run_lease_v1( + _docker_capability( + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + docker_path=Path("/bin/true"), ) + ) + try: + with mock.patch.object( + backend, + "_cleanup_container", + return_value="forced cleanup failure", + ): + result = backend._observe_command( + (sys.executable, "-c", "import os,time; os.close(0); time.sleep(1)"), + stdout_limit=1024, + stderr_limit=1024, + timeout_ns=5_000_000_000, + lease=lease, + input_bundle=bundle, + ) + finally: + backend._release_run_lease_v1(lease) self.assertIs(type(result), build_transport.DockerBuildCleanupFailureV1, result) self.assertEqual(result.trigger, build_transport.DockerCleanupTriggerV1.INPUT_TRANSFER) self.assertEqual(result.detail, "forced cleanup failure") @@ -508,7 +514,7 @@ def test_controller_owns_one_sealed_bundle_for_both_builds(self) -> None: self.assertEqual(pipeline_source.count("_seal_build_input_bundle_v1("), 1) self.assertIn("for attempt in (1, 2)", transport_source) - def test_docker_request_has_no_semantic_host_path_authority(self) -> None: + def test_docker_request_carries_only_semantic_build_coordinates(self) -> None: fields = set(inspect.signature(build_transport.DockerBuildRequestV1).parameters) self.assertEqual( fields, @@ -517,8 +523,6 @@ def test_docker_request_has_no_semantic_host_path_authority(self) -> None: "capability", "input_bundle", "max_output_bytes", - "cid_file", - "container_name", }, ) backend = build_transport.NativeDockerBuildBackendV1( @@ -532,23 +536,53 @@ def test_docker_request_has_no_semantic_host_path_authority(self) -> None: backend, pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, ) - command = backend.command_for( + request = build_transport.DockerBuildRequestV1( + 1, + capability, + _bundle(1024), + 1024, + ) + self.assertFalse(hasattr(request, "cid_file")) + self.assertFalse(hasattr(request, "container_name")) + with self.assertRaises(TypeError): build_transport.DockerBuildRequestV1( 1, capability, _bundle(1024), 1024, - Path("/tmp/container.cid"), - "labcolors-arb-build-v1-command-test", + Path("/tmp/foreign.cid"), + "labcolors-arb-build-v1-foreign", ) + + def test_native_adapter_mints_private_docker_issued_cleanup_authority(self) -> None: + backend = build_transport.NativeDockerBuildBackendV1( + Path("/usr/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + platform_name="linux", + machine_name="x86_64", + host_user=(501, 20), + ) + capability = _probe_native_backend( + backend, + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + ) + request = build_transport.DockerBuildRequestV1( + 1, + capability, + _bundle(1024), + 1024, ) - self.assertNotIn("--mount", command) - self.assertEqual(command.count("--tmpfs"), 2) - self.assertIn(pipeline._BUILD_TMPFS_SPEC_V1, command) - self.assertIn(pipeline._BUILD_STATE_TMPFS_SPEC_V1, command) - self.assertIn("--interactive", command) - self.assertIn("/usr/bin/env", command) - self.assertIn("-i", command) + lease = backend._next_run_lease_v1(capability) + try: + command = backend._command_for_v1(request, lease) + + self.assertIn("--cidfile", command) + self.assertNotIn("--name", command) + self.assertFalse(hasattr(lease, "container_name")) + self.assertTrue(lease.cid_file.is_absolute()) + self.assertFalse(lease.cid_file.exists()) + finally: + backend._release_run_lease_v1(lease) def test_recipe_is_transport_agnostic_and_bootstrap_owns_binary_stdout(self) -> None: source = BUILD_RECIPE.read_text(encoding="utf-8") diff --git a/proof/region/v1/build/transport.py b/proof/region/v1/build/transport.py index 40f96817..bbeb83a0 100644 --- a/proof/region/v1/build/transport.py +++ b/proof/region/v1/build/transport.py @@ -9,10 +9,12 @@ import platform import re import selectors +import shutil import signal import stat import subprocess import tempfile +import threading import time from enum import StrEnum from pathlib import Path @@ -21,33 +23,39 @@ from . import input -# These ceilings preserve the already shipped Arb V1 observer contract; they -# are not physical constants or evidence that every build fits. Changing one -# requires a new transport version, a streaming/resource design review, and a -# targeted native high-water gate. A lane policy may only tighten them. +# These versioned observer bounds are not physical constants or a claim that +# every client build fits. Changing one requires a transport-version, +# streaming/resource review, and a targeted native high-water gate. A client +# policy may only tighten them. BUILD_STDOUT_LIMIT_V1 = 16 * 1024 * 1024 BUILD_STDERR_LIMIT_V1 = 16 * 1024 * 1024 BUILD_TIMEOUT_NS_V1 = 2 * 60 * 60 * 1_000_000_000 DOCKER_PROBE_OUTPUT_LIMIT_V1 = 1024 * 1024 DOCKER_PROBE_TIMEOUT_NS_V1 = 30 * 1_000_000_000 -# These are observer scheduling/termination mechanics retained from Arb V1, -# not successful-build evidence coordinates. CPU, RAM and PID containment is -# owned by the declared disposable worker, outside this Docker transport. +# These are observer scheduling/termination mechanics, not successful-build +# evidence coordinates. CPU, RAM and PID containment belong to the declared +# disposable worker, outside this Docker transport. _IO_CHUNK_BYTES_V1 = 64 * 1024 _POLL_SLICE_SECONDS_V1 = 0.1 _PROCESS_STOP_TIMEOUT_SECONDS_V1 = 30 _PATH_TYPE = type(Path("/")) +_NATIVE_CID_ROOT_PREFIX_V1 = "labcolors-docker-cid-" _BUILD_INPUT_PROGRESS_TOKEN = object() _BUILD_INPUT_TRANSFER_TOKEN = object() _DOCKER_COMMAND_EXITED_TOKEN = object() _DOCKER_BUILD_EXITED_TOKEN = object() -_BUILD_CLEANUP_FAILURE_TOKEN = object() +_NATIVE_RUN_LEASE_TOKEN = object() +_DOCKER_ISSUED_CONTAINER_ID_TOKEN = object() _BUILD_SESSION_TOKEN = object() _TWO_BUILD_OBSERVATION_TOKEN = object() +class _NativeOwnershipLostV1(RuntimeError): + """A post-fork copy must not act on its creator's native resources.""" + + def _valid_digest(value: object) -> bool: return type(value) is bytes and len(value) == 32 and value != bytes(32) @@ -134,7 +142,6 @@ def __new__( image_reference: str, platform: str, hostname: str, - container_name_prefix: str, bootstrap: str, bootstrap_argv0: str, tmpfs_specs: tuple[str, ...], @@ -156,7 +163,6 @@ def __new__( ("image_reference", image_reference, 512), ("platform", platform, 64), ("hostname", hostname, 64), - ("container_name_prefix", container_name_prefix, 64), ("bootstrap", bootstrap, 64 * 1024), ("bootstrap_argv0", bootstrap_argv0, 128), ) @@ -165,7 +171,6 @@ def __new__( image_reference, platform, hostname, - container_name_prefix, bootstrap, bootstrap_argv0, ) = strings @@ -179,13 +184,8 @@ def __new__( character not in "abcdefghijklmnopqrstuvwxyz0123456789-" for character in hostname ) - or any( - character not in "abcdefghijklmnopqrstuvwxyz0123456789-" - for character in container_name_prefix - ) - or not container_name_prefix.endswith("-") ): - raise TypeError("invalid Docker names") + raise TypeError("invalid Docker hostname") if type(tmpfs_specs) is not tuple or not tmpfs_specs: raise TypeError("invalid tmpfs_specs") owned_tmpfs: list[str] = [] @@ -217,7 +217,6 @@ def __new__( image_reference, platform, hostname, - container_name_prefix, bootstrap, bootstrap_argv0, tmpfs_specs, @@ -242,45 +241,41 @@ def platform(self) -> str: def hostname(self) -> str: return self[2] - @property - def container_name_prefix(self) -> str: - return self[3] - @property def bootstrap(self) -> str: - return self[4] + return self[3] @property def bootstrap_argv0(self) -> str: - return self[5] + return self[4] @property def tmpfs_specs(self) -> tuple[str, ...]: - return self[6] + return self[5] @property def user_mode(self) -> DockerUserModeV1: - return self[7] + return self[6] @property def stdout_limit(self) -> int: - return self[8] + return self[7] @property def stderr_limit(self) -> int: - return self[9] + return self[8] @property def build_timeout_ns(self) -> int: - return self[10] + return self[9] @property def probe_output_limit(self) -> int: - return self[11] + return self[10] @property def probe_timeout_ns(self) -> int: - return self[12] + return self[11] def docker_policy_is_valid_v1(value: object) -> bool: @@ -303,7 +298,6 @@ def transport_policy_identity_v1(policy: DockerBuildPolicyV1) -> bytes: policy.image_reference.encode("utf-8"), policy.platform.encode("utf-8"), policy.hostname.encode("utf-8"), - policy.container_name_prefix.encode("utf-8"), policy.bootstrap.encode("utf-8"), policy.bootstrap_argv0.encode("utf-8"), len(policy.tmpfs_specs).to_bytes(4, "big"), @@ -323,7 +317,6 @@ class _NativeCommandSlotV1(StrEnum): IMAGE_REFERENCE = "image_reference" PLATFORM = "platform" ORDERED_TMPFS_SPECS = "ordered_tmpfs_specs" - CONTAINER_NAME = "container_name" HOSTNAME = "hostname" HOST_USER = "host_user" CID_FILE = "cid_file" @@ -413,8 +406,6 @@ def _slot_v1(value: _NativeCommandSlotV1) -> _NativeCommandTokenV1: _literal_v1("ALL"), _literal_v1("--security-opt"), _literal_v1("no-new-privileges:true"), - _literal_v1("--name"), - _slot_v1(_NativeCommandSlotV1.CONTAINER_NAME), _literal_v1("--hostname"), _slot_v1(_NativeCommandSlotV1.HOSTNAME), _literal_v1("--user"), @@ -450,6 +441,17 @@ def _slot_v1(value: _NativeCommandSlotV1) -> _NativeCommandTokenV1: _slot_v1(_NativeCommandSlotV1.CONTAINER_COORDINATE), ), ), + ( + "cleanup_inspect", + ( + _slot_v1(_NativeCommandSlotV1.CLI_PATH), + _literal_v1("container"), + _literal_v1("inspect"), + _literal_v1("--format"), + _literal_v1("{{.Id}}"), + _slot_v1(_NativeCommandSlotV1.CONTAINER_COORDINATE), + ), + ), ( "cleanup_ls", ( @@ -466,6 +468,224 @@ def _slot_v1(value: _NativeCommandSlotV1) -> _NativeCommandTokenV1: ) +class _NativeStdioModeV1(StrEnum): + PIPE = "pipe" + DEVNULL = "devnull" + + +def _native_stdio_value_v1(mode: _NativeStdioModeV1) -> int: + if type(mode) is not _NativeStdioModeV1: + raise TypeError("invalid native stdio mode") + if mode is _NativeStdioModeV1.PIPE: + return subprocess.PIPE + if mode is _NativeStdioModeV1.DEVNULL: + return subprocess.DEVNULL + raise TypeError("invalid native stdio mode") + + +class _NativeProcessContextV1(tuple): + """One fixed, identity-bound child-launch context for the Docker CLI.""" + + __slots__ = () + + def __new__( + cls, + environment: tuple[tuple[str, str], ...], + cwd: str, + umask: int, + close_fds: bool, + restore_signals: bool, + start_new_session: bool, + stdin_with_input: _NativeStdioModeV1, + stdin_without_input: _NativeStdioModeV1, + stdout: _NativeStdioModeV1, + stderr: _NativeStdioModeV1, + ) -> _NativeProcessContextV1: + if type(environment) is not tuple or not environment: + raise TypeError("invalid native process environment") + owned_environment: list[tuple[str, str]] = [] + for entry in environment: + if type(entry) is not tuple or len(entry) != 2: + raise TypeError("invalid native process environment") + name, value = entry + if ( + type(name) is not str + or not name + or re.fullmatch(r"[A-Z_][A-Z0-9_]*", name) is None + or type(value) is not str + or "\0" in value + or "\n" in value + or "\r" in value + ): + raise TypeError("invalid native process environment") + try: + value.encode("utf-8") + except UnicodeEncodeError as error: + raise TypeError("invalid native process environment") from error + owned_environment.append((name, value)) + canonical_environment = tuple(owned_environment) + if ( + canonical_environment != tuple(sorted(canonical_environment)) + or len({name for name, _value in canonical_environment}) + != len(canonical_environment) + ): + raise TypeError("native process environment must be ordered and unique") + if ( + type(cwd) is not str + or not cwd + or "\0" in cwd + or "\n" in cwd + or "\r" in cwd + or not os.path.isabs(cwd) + ): + raise TypeError("invalid native process cwd") + try: + cwd.encode("utf-8") + except UnicodeEncodeError as error: + raise TypeError("invalid native process cwd") from error + if type(umask) is not int or umask < 0 or umask > 0o777: + raise TypeError("invalid native process umask") + if any( + type(value) is not bool + for value in (close_fds, restore_signals, start_new_session) + ): + raise TypeError("invalid native process launch flags") + if any( + type(value) is not _NativeStdioModeV1 + for value in ( + stdin_with_input, + stdin_without_input, + stdout, + stderr, + ) + ): + raise TypeError("invalid native stdio topology") + return tuple.__new__( + cls, + ( + canonical_environment, + cwd, + umask, + close_fds, + restore_signals, + start_new_session, + stdin_with_input, + stdin_without_input, + stdout, + stderr, + ), + ) + + @property + def environment(self) -> tuple[tuple[str, str], ...]: + return self[0] + + @property + def cwd(self) -> str: + return self[1] + + @property + def umask(self) -> int: + return self[2] + + @property + def close_fds(self) -> bool: + return self[3] + + @property + def restore_signals(self) -> bool: + return self[4] + + @property + def start_new_session(self) -> bool: + return self[5] + + @property + def stdin_with_input(self) -> _NativeStdioModeV1: + return self[6] + + @property + def stdin_without_input(self) -> _NativeStdioModeV1: + return self[7] + + @property + def stdout(self) -> _NativeStdioModeV1: + return self[8] + + @property + def stderr(self) -> _NativeStdioModeV1: + return self[9] + + def identity_chunks_v1(self) -> tuple[bytes, ...]: + return ( + b"native-process-context.v1", + len(self.environment).to_bytes(4, "big"), + *( + chunk + for name, value in self.environment + for chunk in (name.encode("ascii"), value.encode("utf-8")) + ), + self.cwd.encode("utf-8"), + self.umask.to_bytes(4, "big"), + bytes((self.close_fds,)), + bytes((self.restore_signals,)), + bytes((self.start_new_session,)), + b"native-stdio-topology.v1", + b"stdin-with-input", + self.stdin_with_input.value.encode("ascii"), + b"stdin-without-input", + self.stdin_without_input.value.encode("ascii"), + b"stdout", + self.stdout.value.encode("ascii"), + b"stderr", + self.stderr.value.encode("ascii"), + ) + + def popen_kwargs_v1(self, receives_stdin: bool) -> dict[str, object]: + if type(receives_stdin) is not bool: + raise TypeError("receives_stdin must be bool") + return { + "stdin": _native_stdio_value_v1( + self.stdin_with_input + if receives_stdin + else self.stdin_without_input + ), + "stdout": _native_stdio_value_v1(self.stdout), + "stderr": _native_stdio_value_v1(self.stderr), + "cwd": self.cwd, + "env": dict(self.environment), + "close_fds": self.close_fds, + "restore_signals": self.restore_signals, + "start_new_session": self.start_new_session, + "umask": self.umask, + } + + +# The Docker CLI is part of an evidence-producing observation. Its launch +# cannot inherit locale, config, cwd, umask or session state from the host: +# this immutable value both renders Popen kwargs and enters the command +# identity, so a future change cannot silently alter what the observer ran. +_NATIVE_PROCESS_CONTEXT_V1 = _NativeProcessContextV1( + ( + ("DOCKER_CONFIG", "/nonexistent"), + ("HOME", "/nonexistent"), + ("LANG", "C"), + ("LC_ALL", "C"), + ("PATH", "/usr/bin:/bin"), + ("TZ", "UTC"), + ), + "/", + 0o077, + True, + True, + True, + _NativeStdioModeV1.PIPE, + _NativeStdioModeV1.DEVNULL, + _NativeStdioModeV1.PIPE, + _NativeStdioModeV1.PIPE, +) + + def native_command_contract_identity_v1() -> bytes: chunks: list[bytes] = [len(_NATIVE_COMMAND_TEMPLATES_V1).to_bytes(4, "big")] for name, tokens in _NATIVE_COMMAND_TEMPLATES_V1: @@ -482,6 +702,7 @@ def native_command_contract_identity_v1() -> bytes: ), ) ) + chunks.extend(_NATIVE_PROCESS_CONTEXT_V1.identity_chunks_v1()) return _identity( b"labcolors.proof-region.native-command-contract.v1\0", tuple(chunks), @@ -627,6 +848,7 @@ def _render_native_command_v1( class DockerBlockerReasonV1(StrEnum): HOST_NOT_LINUX_AMD64 = "host_not_linux_amd64" + HOST_USER_UNAVAILABLE = "host_user_unavailable" DOCKER_UNAVAILABLE = "docker_unavailable" IMAGE_UNAVAILABLE = "image_unavailable" IMAGE_IDENTITY_MISMATCH = "image_identity_mismatch" @@ -853,19 +1075,9 @@ def _host_user_coordinates(value: object) -> tuple[int, int]: return value -def _container_name(value: object, prefix: str) -> str: - if ( - type(value) is not str - or type(prefix) is not str - or not value.startswith(prefix) - or len(value) > 128 - or any(character not in "abcdefghijklmnopqrstuvwxyz0123456789-" for character in value) - ): - raise TypeError("invalid controller-owned Docker container name") - return value - - class DockerBuildRequestV1(tuple): + """Semantic BUILD coordinates; the adapter owns all host resources.""" + __slots__ = () def __new__( @@ -874,8 +1086,6 @@ def __new__( capability: DockerSupportedV1, input_bundle: input.SealedInputV1, max_output_bytes: int, - cid_file: Path, - container_name: str, ) -> DockerBuildRequestV1: if type(attempt) is not int or attempt not in (1, 2): raise TypeError("attempt must be 1 or 2") @@ -889,8 +1099,6 @@ def __new__( or max_output_bytes > capability.policy.stdout_limit ): raise TypeError("invalid executable output limit") - _absolute_path(cid_file, "cid_file") - _container_name(container_name, capability.policy.container_name_prefix) return tuple.__new__( cls, ( @@ -898,8 +1106,6 @@ def __new__( capability, input_bundle, max_output_bytes, - cid_file, - container_name, ), ) @@ -919,14 +1125,6 @@ def input_bundle(self) -> input.SealedInputV1: def max_output_bytes(self) -> int: return self[3] - @property - def cid_file(self) -> Path: - return self[4] - - @property - def container_name(self) -> str: - return self[5] - def _docker_build_request_is_valid_v1( value: object, @@ -948,6 +1146,89 @@ def _docker_build_request_is_valid_v1( return False +class _NativeRunLeaseV1: + """Adapter-owned authority for one Docker-issued container ID file.""" + + __slots__ = ( + "_owner", + "_creator_pid", + "_capability", + "_root", + "_cid_file", + "_launched", + "_released", + ) + + def __init__( + self, + owner: object, + creator_pid: int, + capability: DockerSupportedV1, + root: Path, + cid_file: Path, + *, + _token: object, + ) -> None: + if ( + _token is not _NATIVE_RUN_LEASE_TOKEN + or type(owner) is not object + or type(creator_pid) is not int + or creator_pid <= 0 + or not _docker_supported_is_valid_v1(capability) + or type(root) is not _PATH_TYPE + or not root.is_absolute() + or type(cid_file) is not _PATH_TYPE + or not cid_file.is_absolute() + ): + raise TypeError("native run lease is adapter-owned") + self._owner = owner + self._creator_pid = creator_pid + self._capability = capability + self._root = root + self._cid_file = cid_file + self._launched = False + self._released = False + + @property + def owner(self) -> object: + return self._owner + + @property + def creator_pid(self) -> int: + return self._creator_pid + + @property + def capability(self) -> DockerSupportedV1: + return self._capability + + @property + def cid_file(self) -> Path: + return self._cid_file + + @property + def launched(self) -> bool: + return self._launched + + +class _DockerIssuedContainerIdV1(str): + """A full ID read from the adapter-private CID file Docker created.""" + + def __new__( + cls, + value: str, + *, + _token: object, + ) -> _DockerIssuedContainerIdV1: + if ( + _token is not _DOCKER_ISSUED_CONTAINER_ID_TOKEN + or type(value) is not str + or len(value) != 64 + or any(character not in "0123456789abcdef" for character in value) + ): + raise TypeError("invalid Docker-issued container ID") + return str.__new__(cls, value) + + def _bounded_bytes(value: object, maximum: int, field_name: str) -> bytes: if type(value) is not bytes or len(value) > maximum: raise TypeError(f"invalid {field_name}") @@ -1424,7 +1705,7 @@ class DockerCleanupTriggerV1(StrEnum): class CleanupResourceV1(StrEnum): DOCKER_CLI_PROCESS = "docker_cli_process" DOCKER_CONTAINER = "docker_container" - TEMPORARY_ROOT = "temporary_root" + DOCKER_CID_ROOT = "docker_cid_root" class CleanupFailureRecordV1(tuple): @@ -1490,6 +1771,7 @@ def __new__( ( CleanupResourceV1.DOCKER_CLI_PROCESS, CleanupResourceV1.DOCKER_CONTAINER, + CleanupResourceV1.DOCKER_CID_ROOT, ), ) _bounded_bytes(stdout, BUILD_STDOUT_LIMIT_V1, "stdout") @@ -1540,44 +1822,6 @@ def input_progress(self) -> BuildInputTransferProgressV1 | None: ) -class BuildCleanupFailureV1(tuple): - """Controller-owned cleanup failure with any preceding backend observation.""" - - __slots__ = () - - def __new__( - cls, - failures: tuple[CleanupFailureRecordV1, ...], - current_process: DockerBuildProcessObservationV1 | None, - *, - _token: object, - ) -> BuildCleanupFailureV1: - if _token is not _BUILD_CLEANUP_FAILURE_TOKEN: - raise TypeError("build cleanup failure is controller-observed") - owned_failures = _cleanup_failure_records_v1( - failures, - (CleanupResourceV1.TEMPORARY_ROOT,), - ) - if current_process is not None and type(current_process) not in ( - DockerBuildExitedV1, - DockerBuildTimedOutV1, - DockerBuildOutputLimitV1, - DockerBuildObserverFailureV1, - DockerBuildInputRejectedV1, - DockerBuildCleanupFailureV1, - ): - raise TypeError("cleanup failure lost its current process observation") - return tuple.__new__(cls, (owned_failures, current_process)) - - @property - def failures(self) -> tuple[CleanupFailureRecordV1, ...]: - return self[0] - - @property - def current_process(self) -> DockerBuildProcessObservationV1 | None: - return self[1] - - _DockerCommandObservationV1: TypeAlias = ( _DockerCommandExitedV1 | DockerBuildProcessObservationV1 ) @@ -1716,53 +1960,6 @@ def _canonical_process_observation_v1( raise TypeError("backend returned an unknown process observation") -def _build_cleanup_failure_v1( - current_process: DockerBuildProcessObservationV1 | None, - detail: str, -) -> BuildCleanupFailureV1: - return BuildCleanupFailureV1( - ( - CleanupFailureRecordV1( - CleanupResourceV1.TEMPORARY_ROOT, - detail, - ), - ), - current_process, - _token=_BUILD_CLEANUP_FAILURE_TOKEN, - ) - - -def _canonical_build_cleanup_failure_v1( - value: object, - input_value: input.SealedInputV1, - max_output_bytes: int, - max_stderr_bytes: int, -) -> BuildCleanupFailureV1: - if type(value) is not BuildCleanupFailureV1: - raise TypeError("unknown build cleanup observation") - try: - canonical_process = ( - None - if value.current_process is None - else _canonical_process_observation_v1( - value.current_process, - input_value, - max_output_bytes, - max_stderr_bytes, - ) - ) - canonical = BuildCleanupFailureV1( - value.failures, - canonical_process, - _token=_BUILD_CLEANUP_FAILURE_TOKEN, - ) - if tuple(canonical) != tuple(value): - raise TypeError("build cleanup observation is not canonical") - return canonical - except (AttributeError, IndexError, TypeError, ValueError) as error: - raise TypeError("build cleanup observation is not canonical") from error - - class DockerBuildBackendV1(Protocol): def probe(self) -> DockerCapabilityReportV1: ... @@ -1826,7 +2023,7 @@ def __init__( if policy.user_mode is not DockerUserModeV1.HOST_EFFECTIVE_IDS: raise TypeError("unsupported Docker user policy") observed_user = ( - (os.geteuid(), os.getegid()) if host_user is None else host_user + None if host_user is None else _host_user_coordinates(host_user) ) observed_platform = ( platform.system().lower() if platform_name is None else platform_name @@ -1847,19 +2044,21 @@ def __init__( "machine_name", ) self._monotonic_ns = monotonic_ns - self._host_user = _host_user_coordinates(observed_user) + self._configured_host_user = observed_user + self._run_lease_owner = object() + self._owner_pid = os.getpid() self._probed_capability: DockerSupportedV1 | None = None - @staticmethod - def _environment() -> dict[str, str]: - return { - "HOME": "/nonexistent", - "PATH": "/usr/bin:/bin", - "DOCKER_CONFIG": "/nonexistent", - } + def _in_owner_process_v1(self) -> bool: + return os.getpid() == self._owner_pid def probe(self) -> DockerCapabilityReportV1: self._probed_capability = None + if not self._in_owner_process_v1(): + return DockerUnsupportedV1( + DockerBlockerReasonV1.BACKEND_CONTRACT, + "native Docker capability belongs to its creator process", + ) if self._platform_name != "linux" or self._machine_name.lower() not in ( "x86_64", "amd64", @@ -1868,6 +2067,15 @@ def probe(self) -> DockerCapabilityReportV1: DockerBlockerReasonV1.HOST_NOT_LINUX_AMD64, "controlled build requires a Linux amd64 Docker host", ) + host_user = self._configured_host_user + if host_user is None: + try: + host_user = _host_user_coordinates((os.geteuid(), os.getegid())) + except (AttributeError, OSError, TypeError): + return DockerUnsupportedV1( + DockerBlockerReasonV1.HOST_USER_UNAVAILABLE, + "host effective uid/gid are unavailable", + ) try: metadata = self._command_coordinate.path.lstat() except OSError: @@ -1903,7 +2111,6 @@ def probe(self) -> DockerCapabilityReportV1: stdout_limit=self._policy.probe_output_limit, stderr_limit=self._policy.probe_output_limit, timeout_ns=self._policy.probe_timeout_ns, - cid_file=None, ) if ( type(result) is not _DockerCommandExitedV1 @@ -1948,12 +2155,15 @@ def probe(self) -> DockerCapabilityReportV1: self._policy, daemon_observation, self._command_coordinate, - self._host_user, + host_user, ) self._probed_capability = capability return capability - def command_for(self, request: DockerBuildRequestV1) -> tuple[str, ...]: + def _bound_request_capability_v1( + self, + request: DockerBuildRequestV1, + ) -> DockerSupportedV1: if type(request) is not DockerBuildRequestV1: raise TypeError("request must be DockerBuildRequestV1") try: @@ -1966,6 +2176,101 @@ def command_for(self, request: DockerBuildRequestV1) -> tuple[str, ...]: or capability is not self._probed_capability ): raise TypeError("request capability does not match this backend probe") + return capability + + def _next_run_lease_v1( + self, + capability: DockerSupportedV1, + ) -> _NativeRunLeaseV1: + if not self._in_owner_process_v1(): + raise RuntimeError("native Docker build belongs to its creator process") + if not _docker_supported_is_valid_v1(capability): + raise TypeError("run lease requires one canonical Docker capability") + root = Path( + tempfile.mkdtemp( + prefix=_NATIVE_CID_ROOT_PREFIX_V1, + dir="/tmp", + ) + ) + try: + metadata = root.lstat() + if ( + not root.is_absolute() + or not stat.S_ISDIR(metadata.st_mode) + or stat.S_ISLNK(metadata.st_mode) + or metadata.st_mode & 0o077 + ): + raise RuntimeError("native Docker CID root is not private") + cid_file = root / "cid" + if cid_file.exists() or cid_file.is_symlink(): + raise RuntimeError("fresh native Docker CID path already exists") + return _NativeRunLeaseV1( + self._run_lease_owner, + self._owner_pid, + capability, + root, + cid_file, + _token=_NATIVE_RUN_LEASE_TOKEN, + ) + except BaseException: + try: + shutil.rmtree(root) + except BaseException: + pass + raise + + def _owns_run_lease_v1(self, lease: object) -> bool: + if type(lease) is not _NativeRunLeaseV1: + return False + try: + return ( + lease.owner is self._run_lease_owner + and lease.creator_pid == self._owner_pid + and _docker_supported_is_valid_v1(lease.capability) + and lease.cid_file.is_absolute() + ) + except (AttributeError, TypeError): + return False + + def _lease_belongs_to_current_process_v1(self, lease: _NativeRunLeaseV1) -> bool: + return self._owns_run_lease_v1(lease) and os.getpid() == lease.creator_pid + + def _mark_run_lease_launched_v1(self, lease: _NativeRunLeaseV1) -> None: + if not self._lease_belongs_to_current_process_v1(lease): + raise RuntimeError("native Docker run lease belongs to another process") + if lease._released: + raise RuntimeError("native Docker run lease was already released") + lease._launched = True + + def _release_run_lease_v1(self, lease: _NativeRunLeaseV1) -> str | None: + if not self._owns_run_lease_v1(lease): + raise TypeError("native run lease does not belong to this adapter") + if os.getpid() != lease.creator_pid: + return None + if lease._released: + return None + try: + shutil.rmtree(lease._root) + except Exception: + return "native Docker CID root cleanup failed" + # This flag certifies completed removal, not merely an attempted one: + # an interrupted caller may safely retry with the same private lease. + lease._released = True + return None + + def _command_for_v1( + self, + request: DockerBuildRequestV1, + lease: _NativeRunLeaseV1, + ) -> tuple[str, ...]: + if not self._owns_run_lease_v1(lease): + raise TypeError("native run lease does not belong to this adapter") + capability = lease.capability + if ( + not _docker_build_request_is_valid_v1(request, capability) + or request.capability is not capability + ): + raise TypeError("request capability does not match this native run lease") policy = capability.policy return _render_native_command_v1( "build", @@ -1973,12 +2278,11 @@ def command_for(self, request: DockerBuildRequestV1) -> tuple[str, ...]: { _NativeCommandSlotV1.PLATFORM: (policy.platform,), _NativeCommandSlotV1.ORDERED_TMPFS_SPECS: policy.tmpfs_specs, - _NativeCommandSlotV1.CONTAINER_NAME: (request.container_name,), _NativeCommandSlotV1.HOSTNAME: (policy.hostname,), _NativeCommandSlotV1.HOST_USER: ( f"{capability.host_user[0]}:{capability.host_user[1]}", ), - _NativeCommandSlotV1.CID_FILE: (str(request.cid_file),), + _NativeCommandSlotV1.CID_FILE: (str(lease.cid_file),), _NativeCommandSlotV1.IMAGE_REFERENCE: (policy.image_reference,), _NativeCommandSlotV1.BOOTSTRAP: (policy.bootstrap,), _NativeCommandSlotV1.BOOTSTRAP_ARGV0: (policy.bootstrap_argv0,), @@ -1995,28 +2299,144 @@ def run_build( self, request: DockerBuildRequestV1, ) -> DockerBuildProcessObservationV1: - command = self.command_for(request) - capability = request.capability - policy = capability.policy - return self._observe_command( - command, - stdout_limit=request.max_output_bytes, - stderr_limit=policy.stderr_limit, - timeout_ns=policy.build_timeout_ns, - cid_file=request.cid_file, - container_name=request.container_name, - input_bundle=request.input_bundle, - ) - - def _observe_command( + if not self._in_owner_process_v1(): + return DockerBuildObserverFailureV1( + "native Docker build belongs to its creator process", + b"", + b"", + ) + lease: _NativeRunLeaseV1 | None = None + observation: DockerBuildProcessObservationV1 | None = None + retained_base_exception: BaseException | None = None + try: + capability = self._bound_request_capability_v1(request) + lease = self._next_run_lease_v1(capability) + command = self._command_for_v1(request, lease) + if not self._lease_belongs_to_current_process_v1(lease): + return DockerBuildObserverFailureV1( + "native Docker build belongs to its creator process", + b"", + b"", + ) + policy = capability.policy + observation = self._observe_command( + command, + stdout_limit=request.max_output_bytes, + stderr_limit=policy.stderr_limit, + timeout_ns=policy.build_timeout_ns, + lease=lease, + input_bundle=request.input_bundle, + ) + except Exception: + observation = DockerBuildObserverFailureV1( + "native Docker build request could not be materialized", + b"", + b"", + ) + except BaseException as error: + retained_base_exception = error + finally: + release_detail: str | None = None + if lease is not None: + try: + release_detail = self._release_run_lease_v1(lease) + except Exception: + release_detail = "native Docker CID root cleanup observer raised" + except BaseException as error: + retained_base_exception = retained_base_exception or error + release_detail = "native Docker CID root cleanup was interrupted" + if retained_base_exception is None and release_detail is not None: + observation = self._with_cid_root_cleanup_failure_v1( + observation, + release_detail, + ) + if retained_base_exception is not None: + raise retained_base_exception.with_traceback( + retained_base_exception.__traceback__ + ) + if observation is None: + return DockerBuildObserverFailureV1( + "native Docker build observation was unavailable", + b"", + b"", + ) + return observation + + @staticmethod + def _with_cid_root_cleanup_failure_v1( + observation: DockerBuildProcessObservationV1 | None, + detail: str, + ) -> DockerBuildProcessObservationV1: + """Retain a completed causal prefix when native CID-root release fails.""" + + if observation is None: + return DockerBuildObserverFailureV1(detail, b"", b"") + if type(observation) is DockerBuildCleanupFailureV1: + return DockerBuildCleanupFailureV1( + observation.trigger, + observation.failures + + ( + CleanupFailureRecordV1( + CleanupResourceV1.DOCKER_CID_ROOT, + detail, + ), + ), + observation.stdout, + observation.stderr, + observation.input_progress, + ) + if type(observation) is DockerBuildExitedV1: + transfer = observation.input_transfer + progress = BuildInputTransferProgressV1( + transfer.bundle_identity, + transfer.expected_length, + transfer.expected_sha256, + transfer.written_length, + transfer.written_sha256, + _token=_BUILD_INPUT_PROGRESS_TOKEN, + ) + trigger = DockerCleanupTriggerV1.PROCESS_EXIT + elif type(observation) is DockerBuildTimedOutV1: + progress = observation.input_progress + trigger = DockerCleanupTriggerV1.TIMEOUT + elif type(observation) is DockerBuildOutputLimitV1: + progress = observation.input_progress + trigger = DockerCleanupTriggerV1.OUTPUT_LIMIT + elif type(observation) is DockerBuildInputRejectedV1: + progress = observation.input_progress + trigger = DockerCleanupTriggerV1.INPUT_TRANSFER + elif type(observation) is DockerBuildObserverFailureV1: + progress = observation.input_progress + trigger = DockerCleanupTriggerV1.OBSERVER_FAILURE + else: + raise TypeError("unknown native Docker build observation") + if progress is None: + return DockerBuildObserverFailureV1( + detail, + observation.stdout, + observation.stderr, + ) + return DockerBuildCleanupFailureV1( + trigger, + ( + CleanupFailureRecordV1( + CleanupResourceV1.DOCKER_CID_ROOT, + detail, + ), + ), + observation.stdout, + observation.stderr, + progress, + ) + + def _observe_command( self, command: tuple[str, ...], *, stdout_limit: int, stderr_limit: int, timeout_ns: int, - cid_file: Path | None, - container_name: str | None = None, + lease: _NativeRunLeaseV1 | None = None, input_bundle: input.SealedInputV1 | None = None, ) -> _DockerCommandObservationV1: if ( @@ -2041,16 +2461,16 @@ def _observe_command( or timeout_ns > BUILD_TIMEOUT_NS_V1 ): raise TypeError("invalid Docker observation limits") - if (cid_file is None) != (container_name is None): - raise TypeError("Docker cleanup requires both CID file and exact name") - if cid_file is not None: - _absolute_path(cid_file, "cid_file") - active_policy = ( - self._probed_capability.policy - if self._probed_capability is not None - else self._policy - ) - _container_name(container_name, active_policy.container_name_prefix) + if lease is not None and not self._owns_run_lease_v1(lease): + raise TypeError("native run lease does not belong to this adapter") + if not self._in_owner_process_v1() or ( + lease is not None and not self._lease_belongs_to_current_process_v1(lease) + ): + return DockerBuildObserverFailureV1( + "native Docker observation belongs to its creator process", + b"", + b"", + ) if input_bundle is not None and type(input_bundle) is not input.SealedInputV1: raise TypeError("input_bundle must be controller sealed") if input_bundle is not None and not input.sealed_input_is_intact_v1( @@ -2061,25 +2481,11 @@ def _observe_command( b"", b"", ) - try: - process = subprocess.Popen( - command, - stdin=subprocess.PIPE if input_bundle is not None else subprocess.DEVNULL, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - cwd="/", - env=self._environment(), - close_fds=True, - start_new_session=True, - ) - except (OSError, UnicodeEncodeError): - return DockerBuildObserverFailureV1( - "cannot start Docker CLI", - b"", - b"", - ) - stdout = bytearray() - stderr = bytearray() + # The one protected region starts before Popen. Once Popen returns + # its handle, every following Python bytecode has that handle under + # the finalizer; state allocation cannot create a post-spawn gap. + stdout: bytearray | bytes = b"" + stderr: bytearray | bytes = b"" selector: selectors.BaseSelector | None = None terminal: DockerOutputStreamV1 | None = None timed_out = False @@ -2094,7 +2500,27 @@ def _observe_command( input_descriptor: int | None = None stdout_descriptor: int | None = None stderr_descriptor: int | None = None + retained_base_exception: BaseException | None = None + ownership_lost = False + process: subprocess.Popen[bytes] | None = None try: + try: + process = subprocess.Popen( + command, + **_NATIVE_PROCESS_CONTEXT_V1.popen_kwargs_v1( + input_bundle is not None + ), + ) + except (OSError, UnicodeEncodeError): + return DockerBuildObserverFailureV1( + "cannot start Docker CLI", + b"", + b"", + ) + stdout = bytearray() + stderr = bytearray() + if lease is not None: + self._mark_run_lease_launched_v1(lease) if ( process.stdout is None or process.stderr is None @@ -2197,65 +2623,131 @@ def _observe_command( process.stdin.close() if terminal is not None or input_failed or observer_failed: break + except _NativeOwnershipLostV1: + ownership_lost = True + observer_failed = True except Exception: observer_failed = True + except BaseException as error: + # Cancellation is not an excuse to leak a child or a container. It + # is re-raised only after every independently-owned resource got a + # best-effort deterministic release attempt. + retained_base_exception = error finally: - if selector is not None: - try: - selector.close() - except Exception: - observer_failed = True - if process.stdin is not None: - if self._close_owned_stream(process.stdin, input_descriptor): - observer_failed = True - if bundle_view is not None: - try: - bundle_view.release() - except Exception: - observer_failed = True - if input_bundle is not None: - try: - input_progress = _build_input_progress_v1( - input_bundle, - written, - input_hasher.digest(), + if process is None: + # Popen can itself be interrupted after the daemon received a + # launch request but before Python returned a handle. There + # is no safe CLI PID to reap then, yet a CID cleanup attempt + # can still release a Docker container without replacing the + # caller's original interruption. + if retained_base_exception is not None and lease is not None: + try: + self._cleanup_container( + lease, + spawn_may_have_started=True, + ) + except BaseException: + pass + else: + if selector is not None: + try: + selector.close() + except Exception: + observer_failed = True + except BaseException as error: + retained_base_exception = retained_base_exception or error + observer_failed = True + if process.stdin is not None: + close_failed, close_interrupt = self._close_owned_stream( + process.stdin, + input_descriptor, ) - except Exception: - observer_failed = True - try: - process_running = process.poll() is None - except Exception: - process_running = True - stop_detail = "Docker CLI process state could not be observed" - if process_running: - if not ( - timed_out - or terminal is not None - or observer_failed - or input_failed + if close_failed: + observer_failed = True + retained_base_exception = retained_base_exception or close_interrupt + if bundle_view is not None: + try: + bundle_view.release() + except Exception: + observer_failed = True + except BaseException as error: + retained_base_exception = retained_base_exception or error + observer_failed = True + if input_bundle is not None: + try: + input_progress = _build_input_progress_v1( + input_bundle, + written, + input_hasher.digest(), + ) + except Exception: + observer_failed = True + except BaseException as error: + retained_base_exception = retained_base_exception or error + observer_failed = True + ownership_lost = ownership_lost or not self._in_owner_process_v1() + if not ownership_lost: + try: + process_running = process.poll() is None + except Exception: + process_running = True + stop_detail = "Docker CLI process state could not be observed" + except BaseException as error: + retained_base_exception = retained_base_exception or error + process_running = True + stop_detail = "Docker CLI process state observation was interrupted" + if process_running: + if not ( + timed_out + or terminal is not None + or observer_failed + or input_failed + ): + timed_out = True + try: + observed_stop = self._stop_process(process) + except Exception: + observed_stop = "Docker CLI process termination raised" + except BaseException as error: + retained_base_exception = retained_base_exception or error + observed_stop = "Docker CLI process termination was interrupted" + fallback_stop = self._force_reap_after_interruption_v1(process) + stop_detail = stop_detail or observed_stop or fallback_stop + for stream, descriptor in ( + (process.stdout, stdout_descriptor), + (process.stderr, stderr_descriptor), ): - timed_out = True - try: - observed_stop = self._stop_process(process) - except Exception: - observed_stop = "Docker CLI process termination raised" - stop_detail = stop_detail or observed_stop - for stream, descriptor in ( - (process.stdout, stdout_descriptor), - (process.stderr, stderr_descriptor), - ): - if stream is None: - continue - if self._close_owned_stream(stream, descriptor): - observer_failed = True - if cid_file is not None and container_name is not None: - try: - cleanup_detail = self._cleanup_container( - cid_file, - container_name, + if stream is None: + continue + close_failed, close_interrupt = self._close_owned_stream( + stream, + descriptor, ) - except Exception: - cleanup_detail = "Docker container cleanup observer raised" + if close_failed: + observer_failed = True + retained_base_exception = retained_base_exception or close_interrupt + if lease is not None and not ownership_lost: + try: + cleanup_detail = self._cleanup_container( + lease, + spawn_may_have_started=True, + ) + except Exception: + cleanup_detail = "Docker container cleanup observer raised" + except BaseException as error: + retained_base_exception = retained_base_exception or error + cleanup_detail = "Docker container cleanup was interrupted" + if retained_base_exception is not None: + raise retained_base_exception.with_traceback( + retained_base_exception.__traceback__ + ) + if ownership_lost: + return DockerBuildObserverFailureV1( + "native Docker observation left its creator process", + bytes(stdout), + bytes(stderr), + input_progress, + ) if stop_detail is not None or cleanup_detail is not None: trigger = DockerCleanupTriggerV1.PROCESS_EXIT if observer_failed: @@ -2358,29 +2850,50 @@ def _observe_command( ) @staticmethod - def _close_owned_stream(stream: object, descriptor: int | None) -> bool: - """Close the file object, then its captured owned FD if close raised.""" + def _close_owned_stream( + stream: object, + descriptor: int | None, + ) -> tuple[bool, BaseException | None]: + """Release a stream even when one release operation is interrupted.""" close_failed = False + retained_base_exception: BaseException | None = None try: closed = stream.closed is True except Exception: closed = False close_failed = True + except BaseException as error: + closed = False + close_failed = True + retained_base_exception = error if not closed: try: stream.close() except Exception: close_failed = True + except BaseException as error: + close_failed = True + retained_base_exception = retained_base_exception or error if close_failed and type(descriptor) is int and descriptor >= 0: try: os.close(descriptor) except Exception: pass - return close_failed + except BaseException as error: + retained_base_exception = retained_base_exception or error + return close_failed, retained_base_exception def _clock(self) -> int: + if not self._in_owner_process_v1(): + raise _NativeOwnershipLostV1( + "native Docker observation belongs to its creator process" + ) value = self._monotonic_ns() + if not self._in_owner_process_v1(): + raise _NativeOwnershipLostV1( + "native Docker observation left its creator process" + ) if type(value) is not int or value < 0: raise RuntimeError("invalid monotonic clock") return value @@ -2410,10 +2923,49 @@ def _stop_process( return "Docker CLI process could not be terminated" if failed else None @staticmethod - def _admitted_container_id(cid_file: Path) -> str | None: + def _force_reap_after_interruption_v1( + process: subprocess.Popen[bytes], + ) -> str | None: + """Use an independent, interruption-safe kill/reap path after stop fails.""" + + failed = False + try: + running = process.poll() is None + except BaseException: + running = True + failed = True + if running: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + except BaseException: + try: + process.kill() + except ProcessLookupError: + pass + except BaseException: + failed = True + try: + process.wait(timeout=_PROCESS_STOP_TIMEOUT_SECONDS_V1) + except BaseException: + failed = True + try: + if process.poll() is None: + failed = True + except BaseException: + failed = True + return "Docker CLI process could not be force-reaped" if failed else None + + @staticmethod + def _docker_issued_container_id_v1( + lease: _NativeRunLeaseV1, + ) -> _DockerIssuedContainerIdV1 | None: + """Admit only the exact ID written into this fresh private CID path.""" + try: descriptor = os.open( - cid_file, + lease.cid_file, os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), @@ -2432,78 +2984,121 @@ def _admitted_container_id(cid_file: Path) -> str | None: except OSError: return None finally: - os.close(descriptor) + try: + os.close(descriptor) + except OSError: + pass if len(raw) == 65 and raw.endswith(b"\n"): raw = raw[:-1] - if len(raw) != 64 or any( - byte not in b"0123456789abcdef" for byte in raw - ): + if len(raw) != 64 or any(byte not in b"0123456789abcdef" for byte in raw): return None - return raw.decode("ascii") + return _DockerIssuedContainerIdV1( + raw.decode("ascii"), + _token=_DOCKER_ISSUED_CONTAINER_ID_TOKEN, + ) def _observe_cleanup_command( self, + capability: DockerSupportedV1, command: tuple[str, ...], ) -> _DockerCommandObservationV1: - if self._probed_capability is None: - raise TypeError("Docker cleanup requires an observed capability") - policy = self._probed_capability.policy + if not self._in_owner_process_v1(): + raise RuntimeError("native Docker cleanup belongs to its creator process") + if not _docker_supported_is_valid_v1(capability): + raise TypeError("Docker cleanup requires one observed capability") + policy = capability.policy return self._observe_command( command, stdout_limit=policy.probe_output_limit, stderr_limit=policy.probe_output_limit, timeout_ns=policy.probe_timeout_ns, - cid_file=None, ) - def _cleanup_container(self, cid_file: Path, container_name: str) -> str | None: - if self._probed_capability is None: - raise TypeError("Docker cleanup requires an observed capability") - capability = self._probed_capability - policy = capability.policy - _absolute_path(cid_file, "cid_file") - _container_name(container_name, policy.container_name_prefix) - container_id = self._admitted_container_id(cid_file) - removal_coordinates = ( - (container_id, container_name) - if container_id is not None - else (container_name,) + def _container_is_absent_v1( + self, + capability: DockerSupportedV1, + container_id: _DockerIssuedContainerIdV1, + ) -> bool: + observation = self._observe_cleanup_command( + capability, + _render_native_command_v1( + "cleanup_ls", + capability.command_coordinate, + { + _NativeCommandSlotV1.CONTAINER_FILTER: ( + f"id={str(container_id)}", + ), + }, + ), ) + return ( + type(observation) is _DockerCommandExitedV1 + and observation.returncode == 0 + and observation.stdout == b"" + and observation.stderr == b"" + ) + + def _cleanup_container( + self, + lease: _NativeRunLeaseV1, + *, + spawn_may_have_started: bool = False, + ) -> str | None: + if not self._owns_run_lease_v1(lease): + raise TypeError("native run lease does not belong to this adapter") + if type(spawn_may_have_started) is not bool: + raise TypeError("native Docker spawn state must be bool") + if not self._lease_belongs_to_current_process_v1(lease): + return "native Docker run lease belongs to another process" + if not lease.launched and not spawn_may_have_started: + return None + capability = lease.capability + container_id = self._docker_issued_container_id_v1(lease) + if container_id is None: + return "Docker-issued cleanup ID is unavailable" try: - for coordinate in removal_coordinates: - self._observe_cleanup_command( - _render_native_command_v1( - "cleanup_rm", - capability.command_coordinate, - { - _NativeCommandSlotV1.CONTAINER_COORDINATE: ( - coordinate, - ), - }, - ) + inspection = self._observe_cleanup_command( + capability, + _render_native_command_v1( + "cleanup_inspect", + capability.command_coordinate, + { + _NativeCommandSlotV1.CONTAINER_COORDINATE: ( + str(container_id), + ), + }, + ), + ) + if ( + type(inspection) is _DockerCommandExitedV1 + and inspection.returncode != 0 + ): + return ( + None + if self._container_is_absent_v1(capability, container_id) + else "Docker container absence could not be verified" ) - filters = [f"name=^/{container_name}$"] - if container_id is not None: - filters.append(f"id={container_id}") - for filter_value in filters: - observation = self._observe_cleanup_command( - _render_native_command_v1( - "cleanup_ls", - capability.command_coordinate, - { - _NativeCommandSlotV1.CONTAINER_FILTER: ( - filter_value, - ), - }, - ) + if ( + type(inspection) is not _DockerCommandExitedV1 + or inspection.returncode != 0 + or inspection.stdout != container_id.encode("ascii") + b"\n" + or inspection.stderr + ): + return "Docker-issued cleanup ID did not resolve exactly" + self._observe_cleanup_command( + capability, + _render_native_command_v1( + "cleanup_rm", + capability.command_coordinate, + { + _NativeCommandSlotV1.CONTAINER_COORDINATE: ( + str(container_id), + ), + }, ) - if ( - type(observation) is not _DockerCommandExitedV1 - or observation.returncode != 0 - or observation.stdout - or observation.stderr - ): - return "Docker container absence could not be verified" + ) + if not self._container_is_absent_v1(capability, container_id): + return "Docker container absence could not be verified" except Exception: return "Docker container cleanup observer raised" return None @@ -2520,9 +3115,7 @@ class BuildFailureReasonV1(StrEnum): INVALID_OUTPUT = "invalid_output" -BuildAttemptObservationV1: TypeAlias = ( - DockerBuildProcessObservationV1 | BuildCleanupFailureV1 -) +BuildAttemptObservationV1: TypeAlias = DockerBuildProcessObservationV1 class BuildSessionV1(tuple): @@ -2736,13 +3329,6 @@ def __new__( owned_completed.append(canonical) if process is None: owned_process: BuildAttemptObservationV1 | None = None - elif type(process) is BuildCleanupFailureV1: - owned_process = _canonical_build_cleanup_failure_v1( - process, - session.input_value, - session.max_output_bytes, - session.policy.stderr_limit, - ) else: owned_process = _canonical_process_observation_v1( process, @@ -2753,10 +3339,7 @@ def __new__( expected_process_types: dict[BuildFailureReasonV1, tuple[type, ...]] = { BuildFailureReasonV1.CONTRACT_VIOLATION: (), BuildFailureReasonV1.PROCESS_FAILED: (DockerBuildExitedV1,), - BuildFailureReasonV1.CLEANUP_FAILED: ( - DockerBuildCleanupFailureV1, - BuildCleanupFailureV1, - ), + BuildFailureReasonV1.CLEANUP_FAILED: (DockerBuildCleanupFailureV1,), BuildFailureReasonV1.INPUT_TRANSFER_FAILED: ( DockerBuildInputRejectedV1, ), @@ -2854,39 +3437,82 @@ def __init__( raise TypeError("policy must be DockerBuildPolicyV1") self._policy = DockerBuildPolicyV1(*tuple(policy)) self._backend = backend + # Fork copies Python object state and may copy a locked mutex. This + # controller's capability is therefore valid only in its creator; + # every public operation checks PID before it can touch that mutex. + self._owner_pid = os.getpid() self._probed_capability: DockerSupportedV1 | None = None self._consumed = False + # Probe result and its one-shot BUILD right are one causal state. A + # lock makes the state transition indivisible across reentrant or + # concurrent callers without holding it during caller/backend IO. + self._lease_lock = threading.Lock() + self._probe_in_flight = False + + def _in_owner_process_v1(self) -> bool: + return os.getpid() == self._owner_pid def probe(self) -> DockerCapabilityReportV1: - if self._consumed or self._probed_capability is not None: + if not self._in_owner_process_v1(): return DockerUnsupportedV1( DockerBlockerReasonV1.BACKEND_CONTRACT, - "build transport capability is one-shot", + "build transport capability belongs to its creator process", ) + with self._lease_lock: + if ( + self._consumed + or self._probed_capability is not None + or self._probe_in_flight + ): + return DockerUnsupportedV1( + DockerBlockerReasonV1.BACKEND_CONTRACT, + "build transport capability is one-shot", + ) + self._probe_in_flight = True + outcome: DockerCapabilityReportV1 | None = None try: report = self._backend.probe() except Exception: - return DockerUnsupportedV1( + outcome: DockerCapabilityReportV1 = DockerUnsupportedV1( DockerBlockerReasonV1.BACKEND_CONTRACT, "Docker capability probe raised", ) - if type(report) is DockerUnsupportedV1: - if _docker_unsupported_is_valid_v1(report): - return DockerUnsupportedV1(*tuple(report)) - return DockerUnsupportedV1( - DockerBlockerReasonV1.BACKEND_CONTRACT, - "Docker capability rejection is not canonical", - ) - if ( - not _docker_supported_is_valid_v1(report) - or report.policy != self._policy - ): - return DockerUnsupportedV1( - DockerBlockerReasonV1.BACKEND_CONTRACT, - "Docker capability report does not match build policy", - ) - self._probed_capability = report - return report + else: + if type(report) is DockerUnsupportedV1: + if _docker_unsupported_is_valid_v1(report): + outcome = DockerUnsupportedV1(*tuple(report)) + else: + outcome = DockerUnsupportedV1( + DockerBlockerReasonV1.BACKEND_CONTRACT, + "Docker capability rejection is not canonical", + ) + elif ( + not _docker_supported_is_valid_v1(report) + or report.policy != self._policy + ): + outcome = DockerUnsupportedV1( + DockerBlockerReasonV1.BACKEND_CONTRACT, + "Docker capability report does not match build policy", + ) + else: + outcome = report + finally: + # BaseException must not permanently leave this controller in the + # transient PROBING state. It still propagates to the caller; the + # cleanup only revokes that incomplete external observation. + if not self._in_owner_process_v1(): + outcome = DockerUnsupportedV1( + DockerBlockerReasonV1.BACKEND_CONTRACT, + "build transport capability belongs to its creator process", + ) + else: + with self._lease_lock: + self._probe_in_flight = False + if type(outcome) is DockerSupportedV1: + self._probed_capability = outcome + if outcome is None: + raise RuntimeError("probe outcome was not produced") + return outcome def build( self, @@ -2897,16 +3523,12 @@ def build( input_admission: Callable[[input.SealedInputV1], bool], output_admission: Callable[[bytes], bool], ) -> BuildTransportResultV1: - if ( - self._consumed - or capability is not self._probed_capability - or not _docker_supported_is_valid_v1(capability) - or capability.policy != self._policy - ): + if not self._in_owner_process_v1(): return BuildRejectedV1(1, BuildFailureReasonV1.CONTRACT_VIOLATION) - self._consumed = True if ( - type(max_output_bytes) is not int + not _docker_supported_is_valid_v1(capability) + or capability.policy != self._policy + or type(max_output_bytes) is not int or max_output_bytes <= 0 or max_output_bytes > capability.policy.stdout_limit or not callable(input_admission) @@ -2914,16 +3536,49 @@ def build( or not input.sealed_input_is_intact_v1(input_value) ): return BuildRejectedV1(1, BuildFailureReasonV1.CONTRACT_VIOLATION) - session = _build_session_v1( - capability, - input_value, - max_output_bytes, - ) + if not self._in_owner_process_v1(): + return BuildRejectedV1(1, BuildFailureReasonV1.CONTRACT_VIOLATION) + with self._lease_lock: + if ( + self._consumed + or capability is not self._probed_capability + or not _docker_supported_is_valid_v1(capability) + or capability.policy != self._policy + ): + return BuildRejectedV1(1, BuildFailureReasonV1.CONTRACT_VIOLATION) + # A rejected declaration has not reached an owned execution attempt, + # so it must not burn the lease. Recheck after claim below because a + # callback is external and may be reentrant or mutate hostile input. + if not self._admitted(input_admission, input_value): + return BuildRejectedV1(1, BuildFailureReasonV1.CONTRACT_VIOLATION) + if not self._in_owner_process_v1(): + return BuildRejectedV1(1, BuildFailureReasonV1.CONTRACT_VIOLATION) + with self._lease_lock: + if ( + self._consumed + or capability is not self._probed_capability + or not _docker_supported_is_valid_v1(capability) + or capability.policy != self._policy + ): + return BuildRejectedV1(1, BuildFailureReasonV1.CONTRACT_VIOLATION) + self._consumed = True + if not input.sealed_input_is_intact_v1(input_value): + return BuildRejectedV1(1, BuildFailureReasonV1.CONTRACT_VIOLATION) + try: + session = _build_session_v1( + capability, + input_value, + max_output_bytes, + ) + except Exception: + return BuildRejectedV1(1, BuildFailureReasonV1.CONTRACT_VIOLATION) completed: list[DockerBuildExitedV1] = [] for attempt in (1, 2): if ( - not input.sealed_input_is_intact_v1(input_value) + not self._in_owner_process_v1() + or not input.sealed_input_is_intact_v1(input_value) or not self._admitted(input_admission, input_value) + or not self._in_owner_process_v1() ): return BuildRejectedV1( attempt, @@ -2937,6 +3592,13 @@ def build( output_admission, tuple(completed), ) + if not self._in_owner_process_v1(): + return BuildRejectedV1( + attempt, + BuildFailureReasonV1.CONTRACT_VIOLATION, + session=session, + completed_processes=tuple(completed), + ) if type(built) is BuildRejectedV1: return built completed.append(built) @@ -2975,40 +3637,14 @@ def _build_once( completed_processes=completed_processes, ) try: - temporary_root = tempfile.TemporaryDirectory( - prefix=f"{session.policy.container_name_prefix}{attempt}-" - ) - except Exception: - return contract_rejection - current_process: DockerBuildProcessObservationV1 | None = None - try: - result, current_process = self._observe_build_attempt_v1( + return self._observe_build_attempt_v1( attempt, session, output_admission, completed_processes, - Path(temporary_root.name).resolve(), ) except Exception: - result = contract_rejection - try: - temporary_root.cleanup() - except Exception: - try: - cleanup = _build_cleanup_failure_v1( - current_process, - "temporary build root cleanup failed", - ) - return BuildRejectedV1( - attempt, - BuildFailureReasonV1.CLEANUP_FAILED, - cleanup, - session=session, - completed_processes=completed_processes, - ) - except Exception: - return contract_rejection - return result + return contract_rejection def _observe_build_attempt_v1( self, @@ -3016,11 +3652,7 @@ def _observe_build_attempt_v1( session: BuildSessionV1, output_admission: Callable[[bytes], bool], completed_processes: tuple[DockerBuildExitedV1, ...], - root: Path, - ) -> tuple[ - DockerBuildExitedV1 | BuildRejectedV1, - DockerBuildProcessObservationV1 | None, - ]: + ) -> DockerBuildExitedV1 | BuildRejectedV1: contract_rejection = BuildRejectedV1( attempt, BuildFailureReasonV1.CONTRACT_VIOLATION, @@ -3032,16 +3664,13 @@ def _observe_build_attempt_v1( session.capability, session.input_value, session.max_output_bytes, - root / "container.cid", - session.policy.container_name_prefix - + hashlib.sha256( - os.fsencode(root) + bytes((attempt,)) - ).hexdigest(), ) try: observed = self._backend.run_build(request) except Exception: - return contract_rejection, None + return contract_rejection + if not self._in_owner_process_v1(): + return contract_rejection try: process = _canonical_process_observation_v1( observed, @@ -3050,7 +3679,7 @@ def _observe_build_attempt_v1( session.policy.stderr_limit, ) except TypeError: - return contract_rejection, None + return contract_rejection reason_by_type: dict[type, BuildFailureReasonV1] = { DockerBuildCleanupFailureV1: BuildFailureReasonV1.CLEANUP_FAILED, DockerBuildInputRejectedV1: BuildFailureReasonV1.INPUT_TRANSFER_FAILED, @@ -3060,35 +3689,29 @@ def _observe_build_attempt_v1( } failure_reason = reason_by_type.get(type(process)) if failure_reason is not None: - return ( - BuildRejectedV1( - attempt, - failure_reason, - process, - session=session, - completed_processes=completed_processes, - ), + return BuildRejectedV1( + attempt, + failure_reason, process, + session=session, + completed_processes=completed_processes, ) if type(process) is not DockerBuildExitedV1: - return contract_rejection, None + return contract_rejection if not docker_build_exited_is_valid_v1( process, session.input_value, session.max_output_bytes, session.policy.stderr_limit, ): - return contract_rejection, None + return contract_rejection if process.returncode != 0: - return ( - BuildRejectedV1( - attempt, - BuildFailureReasonV1.PROCESS_FAILED, - process, - session=session, - completed_processes=completed_processes, - ), + return BuildRejectedV1( + attempt, + BuildFailureReasonV1.PROCESS_FAILED, process, + session=session, + completed_processes=completed_processes, ) transfer = process.input_transfer if ( @@ -3099,16 +3722,15 @@ def _observe_build_attempt_v1( or transfer.written_length != session.input_value.length or transfer.written_sha256 != session.input_value.sha256 ): - return contract_rejection, None + return contract_rejection if not self._admitted(output_admission, process.stdout): - return ( - BuildRejectedV1( - attempt, - BuildFailureReasonV1.INVALID_OUTPUT, - process, - session=session, - completed_processes=completed_processes, - ), + return BuildRejectedV1( + attempt, + BuildFailureReasonV1.INVALID_OUTPUT, process, + session=session, + completed_processes=completed_processes, ) - return process, process + if not self._in_owner_process_v1(): + return contract_rejection + return process diff --git a/proof/region/v1/tests/test_build.py b/proof/region/v1/tests/test_build.py index c313d13f..d590934b 100644 --- a/proof/region/v1/tests/test_build.py +++ b/proof/region/v1/tests/test_build.py @@ -4,12 +4,15 @@ from __future__ import annotations import ast +import dis +import gc import hashlib import importlib import os +import select import subprocess import sys -import tempfile +import threading import unittest from pathlib import Path from unittest import mock @@ -34,10 +37,10 @@ # tests deliberately change this inventory and must update both values from the # gate's independent enumeration in the same slice. ARB_INVENTORY_SHA256_V1 = ( - "383672bd1ac2a2d472fdba33d3ec4c770a897ecd13192ec41e7c12dc1e563219" + "4853e06c6e8c1864bc65e0b4c0cd9cdbe0881e0d5907daecb6c8a9fea42f3643" ) ARB_ORDER_SHA256_V1 = ( - "c020118a926070e36f14757f0b281ac05dc460b8affa03947f827baa73d5d172" + "82b8e00867bc0bed7bd4020f8d9b9531cd195f7c712ddff9a4d73ef7fc0484d5" ) MOVED_INPUT_SURFACE_V1 = ( @@ -73,7 +76,6 @@ "CleanupResourceV1", "CleanupFailureRecordV1", "DockerBuildCleanupFailureV1", - "BuildCleanupFailureV1", "DockerBuildBackendV1", "NativeDockerBuildBackendV1", "ControlledBuildTransportV1", @@ -176,6 +178,61 @@ def run_build(self, request: object) -> object: return self._observations.pop(0) +def _racing_build_transport( + transport: object, + *, + policy: object, + backend: object, +) -> object: + """Force the former unlocked check→consume race without scheduler guesses.""" + + class TrackingLock: + def __init__(self) -> None: + self._lock = threading.Lock() + self._owner: int | None = None + + def __enter__(self) -> TrackingLock: + self._lock.acquire() + self._owner = threading.get_ident() + return self + + def __exit__( + self, + _exception_type: object, + _exception: object, + _traceback: object, + ) -> None: + self._owner = None + self._lock.release() + + def held_by_current_thread(self) -> bool: + return self._owner == threading.get_ident() + + class RacingController(transport.ControlledBuildTransportV1): + def __init__(self) -> None: + self._consume_barrier = threading.Barrier(2) + self._race_armed = False + super().__init__(policy=policy, backend=backend) + self._lease_lock = TrackingLock() + + def arm_consume_race(self) -> None: + self._race_armed = True + + def __getattribute__(self, name: str) -> object: + if ( + name == "_consumed" + and object.__getattribute__(self, "_race_armed") + and not object.__getattribute__( + self, + "_lease_lock", + ).held_by_current_thread() + ): + object.__getattribute__(self, "_consume_barrier").wait(timeout=2) + return super().__getattribute__(name) + + return RacingController() + + def _controlled_build( transport: object, policy: object, @@ -211,8 +268,8 @@ def test_existing_arb_suite_keeps_exact_count_order_and_inventory(self) -> None: identifier.encode("utf-8") + b"\n" for identifier in identifiers ) - self.assertEqual(len(identifiers), 160) - self.assertEqual(len(set(identifiers)), 160) + self.assertEqual(len(identifiers), 166) + self.assertEqual(len(set(identifiers)), 166) self.assertEqual( arb_gate.test_inventory_sha256_v1(arb_gate.full_suite_v1()), ARB_INVENTORY_SHA256_V1, @@ -247,7 +304,7 @@ def test_arb_input_and_process_stay_exact_while_capability_identities_move_to_v2 request.host_trust, observed.docker_capability.policy, ).hex(), - "66af6f844dda8eae548eac026f277845ccde1842c14c39824c5108f027247f39", + "5ff9cac8af5fee7ffb05d18da33721842150dafe43edd6f0e356566c7be12144", ) self.assertEqual(len(process_bytes), 196) self.assertEqual( @@ -256,7 +313,7 @@ def test_arb_input_and_process_stay_exact_while_capability_identities_move_to_v2 ) self.assertEqual( result.comparator.identity.hex(), - "e4e8e4dd47ddda5585531f67bfe3112f157a032cb728cd1c61766edf26de6c6c", + "965004e9a45d4ff724f2ca39043086adf29bf860efc9b47367f67473ba6c52ac", ) self.assertEqual( result.evidence.source_identity.hex(), @@ -264,19 +321,19 @@ def test_arb_input_and_process_stay_exact_while_capability_identities_move_to_v2 ) self.assertEqual( result.evidence.build_identity.hex(), - "dfe01f51132d938be3f8a8fad32c91d99fc7b22d69c4c9f488c07f2d00806412", + "5200b47ecae538174dea9f9c67e487859af70f59dd77eb46ca870d337b866bf9", ) self.assertEqual( result.evidence.run_identity.hex(), - "0033f6e70d0090ff2839d364cccaf1a3f5bf79eb2c857ce762b236cbbc730542", + "3036f9f4e49d0822d48447eaa08a0a2aaf052923e2f6cdb9362585dd044acc8e", ) self.assertEqual( result.evidence.identity.hex(), - "80dd866e156d882a749a78b768cfc66838f92f6608baaf9ddfbf3f3a31870324", + "5a3041c6462401a919940d3a7ad1ed99039c7654d3d6b946901e44dd69c9dc53", ) self.assertEqual( result.claim.identity.hex(), - "c0c200282fc3cd800bb0aa53a8e3c2d3fa2edf1a6350185aeff86f410ccef1bb", + "71d1e5d6580404cd8ff4fef677d7664ba18e4fc99cbacb0a942756d56d59eb25", ) @@ -341,11 +398,16 @@ def test_observation_contract_uses_current_non_claiming_language(self) -> None: encoding="utf-8" ) pipeline_source = (ARB / "pipeline.py").read_text(encoding="utf-8") + protocol_source = (PROOF / "PROTOCOL.md").read_text(encoding="utf-8") self.assertNotIn( "backend-contract rejection cannot retain authority", transport_source, ) self.assertNotIn("invalid reproducible-build digests", pipeline_source) + self.assertIn("fresh one-job VM workflow Arb", protocol_source) + self.assertIn("same-UID writer", protocol_source) + self.assertIn("Popen construction", protocol_source) + self.assertNotIn("cleanup выполняет только по его точному имени", protocol_source) def test_arb_consumers_move_atomically_without_compatibility_reexports(self) -> None: build_input = importlib.import_module("build.input") @@ -372,6 +434,20 @@ def test_arb_consumers_move_atomically_without_compatibility_reexports(self) -> with self.subTest(consumer=arb_receipt.__name__, removed=name): self.assertFalse(hasattr(arb_receipt, name)) + def test_arb_policy_reuses_generic_observer_ceiling_ssot(self) -> None: + transport = importlib.import_module("build.transport") + pipeline_source = (ARB / "pipeline.py").read_text(encoding="utf-8") + for name in ( + "BUILD_STDOUT_LIMIT_V1", + "BUILD_STDERR_LIMIT_V1", + "BUILD_TIMEOUT_NS_V1", + "DOCKER_PROBE_OUTPUT_LIMIT_V1", + "DOCKER_PROBE_TIMEOUT_NS_V1", + ): + with self.subTest(name=name): + self.assertIs(getattr(pipeline, name), getattr(transport, name)) + self.assertIn(f"build_transport.{name}", pipeline_source) + def test_shared_input_and_policy_are_deeply_immutable_coordinates(self) -> None: build_input = importlib.import_module("build.input") transport = importlib.import_module("build.transport") @@ -415,6 +491,248 @@ def test_forged_source_authorities_fail_in_the_arb_taxonomy(self) -> None: class SharedBuildTransportTargetTests(unittest.TestCase): + def test_overlapping_probe_is_rejected_without_a_second_backend_probe(self) -> None: + transport = importlib.import_module("build.transport") + policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 + capability = _docker_capability_fixture(policy) + + class BlockingBackend: + def __init__(self) -> None: + self.entered = threading.Event() + self.release = threading.Event() + self.calls = 0 + + def probe(self) -> object: + self.calls += 1 + self.entered.set() + self.release.wait(timeout=2) + return capability + + def run_build(self, _request: object) -> object: + raise AssertionError("probe-only test reached build") + + backend = BlockingBackend() + controller = transport.ControlledBuildTransportV1( + policy=policy, + backend=backend, + ) + first_results: list[object] = [] + second_results: list[object] = [] + second_done = threading.Event() + first = threading.Thread(target=lambda: first_results.append(controller.probe())) + + def second_probe() -> None: + try: + second_results.append(controller.probe()) + finally: + second_done.set() + + second = threading.Thread(target=second_probe) + first.start() + self.assertTrue(backend.entered.wait(timeout=1)) + second.start() + try: + self.assertTrue(second_done.wait(timeout=1)) + finally: + backend.release.set() + first.join(timeout=2) + second.join(timeout=2) + + self.assertFalse(first.is_alive()) + self.assertFalse(second.is_alive()) + self.assertEqual(backend.calls, 1) + self.assertEqual(first_results, [capability]) + self.assertEqual(len(second_results), 1) + self.assertIs(type(second_results[0]), transport.DockerUnsupportedV1) + self.assertEqual( + second_results[0].reason, + transport.DockerBlockerReasonV1.BACKEND_CONTRACT, + ) + + def test_one_probe_lease_cannot_start_two_concurrent_two_build_sessions(self) -> None: + transport = importlib.import_module("build.transport") + policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 + input_value = _sealed_input() + capability = _docker_capability_fixture(policy) + backend = _ScriptedBuildBackend( + capability, + tuple( + _completed_process(transport, input_value, b"same executable") + for _ in range(4) + ), + ) + controller = _racing_build_transport( + transport, + policy=policy, + backend=backend, + ) + owned_capability = controller.probe() + controller.arm_consume_race() + start = threading.Barrier(3) + results: list[object] = [] + failures: list[BaseException] = [] + + def build() -> None: + try: + start.wait(timeout=2) + results.append( + controller.build( + owned_capability, + input_value, + 64, + input_admission=lambda _value: True, + output_admission=lambda _value: True, + ) + ) + except BaseException as error: + failures.append(error) + + workers = tuple(threading.Thread(target=build) for _ in range(2)) + for worker in workers: + worker.start() + start.wait(timeout=2) + for worker in workers: + worker.join(timeout=3) + self.assertFalse(worker.is_alive()) + + self.assertEqual(failures, []) + self.assertEqual(len(results), 2) + self.assertEqual( + sum(type(result) is transport.TwoBuildObservationV1 for result in results), + 1, + ) + rejections = tuple( + result + for result in results + if type(result) is transport.BuildRejectedV1 + ) + self.assertEqual(len(rejections), 1) + self.assertEqual( + rejections[0].reason, + transport.BuildFailureReasonV1.CONTRACT_VIOLATION, + ) + self.assertEqual(len(backend.requests), 2) + + def test_rejected_preflight_preserves_the_unconsumed_build_lease(self) -> None: + transport = importlib.import_module("build.transport") + policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 + input_value = _sealed_input() + capability = _docker_capability_fixture(policy) + backend = _ScriptedBuildBackend( + capability, + ( + _completed_process(transport, input_value, b"same executable"), + _completed_process(transport, input_value, b"same executable"), + ), + ) + controller = transport.ControlledBuildTransportV1( + policy=policy, + backend=backend, + ) + owned_capability = controller.probe() + + rejected = controller.build( + owned_capability, + input_value, + 64, + input_admission=lambda _value: False, + output_admission=lambda _value: True, + ) + self.assertIs(type(rejected), transport.BuildRejectedV1) + self.assertEqual( + rejected.reason, + transport.BuildFailureReasonV1.CONTRACT_VIOLATION, + ) + self.assertEqual(backend.requests, []) + + admitted = controller.build( + owned_capability, + input_value, + 64, + input_admission=lambda _value: True, + output_admission=lambda _value: True, + ) + self.assertIs(type(admitted), transport.TwoBuildObservationV1) + self.assertEqual(len(backend.requests), 2) + + @unittest.skipUnless(hasattr(os, "fork"), "requires POSIX fork") + def test_forked_child_cannot_wait_on_or_duplicate_a_build_lease(self) -> None: + transport = importlib.import_module("build.transport") + policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 + input_value = _sealed_input() + capability = _docker_capability_fixture(policy) + backend = _ScriptedBuildBackend( + capability, + ( + _completed_process(transport, input_value, b"same executable"), + _completed_process(transport, input_value, b"same executable"), + ), + ) + controller = transport.ControlledBuildTransportV1( + policy=policy, + backend=backend, + ) + owned_capability = controller.probe() + read_fd, write_fd = os.pipe() + controller._lease_lock.acquire() + child_pid: int | None = None + child_reaped = False + try: + child_pid = os.fork() + if child_pid == 0: + os.close(read_fd) + try: + child_result = controller.build( + owned_capability, + input_value, + 64, + input_admission=lambda _value: True, + output_admission=lambda _value: True, + ) + os.write( + write_fd, + ( + f"{type(child_result).__name__}:" + f"{len(backend.requests)}" + ).encode("ascii"), + ) + finally: + os.close(write_fd) + os._exit(0) + os.close(write_fd) + ready, _write_ready, _errors = select.select([read_fd], [], [], 1) + self.assertEqual(ready, [read_fd]) + child_message = os.read(read_fd, 128).decode("ascii") + _waited_pid, status = os.waitpid(child_pid, 0) + child_reaped = True + finally: + controller._lease_lock.release() + if child_pid is not None and not child_reaped: + try: + os.kill(child_pid, 9) + except ProcessLookupError: + pass + try: + os.waitpid(child_pid, 0) + except ChildProcessError: + pass + try: + os.close(read_fd) + except OSError: + pass + + self.assertTrue(os.WIFEXITED(status)) + self.assertEqual(child_message, "BuildRejectedV1:0") + parent_result = controller.build( + owned_capability, + input_value, + 64, + input_admission=lambda _value: True, + output_admission=lambda _value: True, + ) + self.assertIs(type(parent_result), transport.TwoBuildObservationV1) + self.assertEqual(len(backend.requests), 2) + def test_public_build_contract_violations_are_typed_before_backend(self) -> None: build_input = importlib.import_module("build.input") transport = importlib.import_module("build.transport") @@ -460,7 +778,6 @@ def test_capability_owns_injected_host_user_and_rejects_surrogate_coordinates(se "image_reference": shipped.image_reference, "platform": shipped.platform, "hostname": shipped.hostname, - "container_name_prefix": shipped.container_name_prefix, "bootstrap": shipped.bootstrap, "bootstrap_argv0": shipped.bootstrap_argv0, "tmpfs_specs": shipped.tmpfs_specs, @@ -492,11 +809,18 @@ def test_capability_owns_injected_host_user_and_rejects_surrogate_coordinates(se capability, input_value, 64, - Path("/tmp/lab-colors-red-user.cid"), - policy.container_name_prefix + "red-user", ) for capability in capabilities ) + commands: list[tuple[str, ...]] = [] + + def observe( + command: tuple[str, ...], + **_kwargs: object, + ) -> object: + commands.append(command) + return _completed_process(transport, input_value, b"") + with mock.patch.object( transport.os, "geteuid", @@ -506,11 +830,26 @@ def test_capability_owns_injected_host_user_and_rejects_surrogate_coordinates(se "getegid", side_effect=AssertionError("command_for performed ambient gid IO"), ): - commands = tuple( - backend.command_for(request) - for backend, request in zip(backends, requests, strict=True) - ) - self.assertEqual(commands[0], commands[1]) + for backend, request in zip(backends, requests, strict=True): + with mock.patch.object( + backend, + "_observe_command", + side_effect=observe, + ): + self.assertIs( + type(backend.run_build(request)), + transport.DockerBuildExitedV1, + ) + self.assertEqual(len(commands), 2) + + def without_native_cid_path(command: tuple[str, ...]) -> tuple[str, ...]: + index = command.index("--cidfile") + return command[: index + 1] + command[index + 2 :] + + self.assertEqual( + without_native_cid_path(commands[0]), + without_native_cid_path(commands[1]), + ) user_index = commands[0].index("--user") self.assertEqual(commands[0][user_index + 1], "501:20") @@ -532,6 +871,79 @@ def test_capability_owns_injected_host_user_and_rejects_surrogate_coordinates(se machine_name="x86_64", ) + def test_native_backend_defers_host_user_observation_to_supported_probe(self) -> None: + transport = importlib.import_module("build.transport") + policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 + + with mock.patch.object( + transport.os, + "geteuid", + side_effect=AttributeError("not available on this host"), + ), mock.patch.object( + transport.os, + "getegid", + side_effect=AttributeError("not available on this host"), + ): + unsupported_backend = transport.NativeDockerBuildBackendV1( + Path("/usr/bin/true"), + policy, + platform_name="windows", + machine_name="amd64", + ) + unsupported = unsupported_backend.probe() + + self.assertIs(type(unsupported), transport.DockerUnsupportedV1) + self.assertEqual( + unsupported.reason, + transport.DockerBlockerReasonV1.HOST_NOT_LINUX_AMD64, + ) + + supported_backend = transport.NativeDockerBuildBackendV1( + Path("/usr/bin/true"), + policy, + platform_name="linux", + machine_name="x86_64", + ) + with mock.patch.object( + transport.os, + "geteuid", + side_effect=AttributeError("not available on this host"), + ): + unavailable = supported_backend.probe() + + self.assertIs(type(unavailable), transport.DockerUnsupportedV1) + self.assertEqual( + unavailable.reason, + transport.DockerBlockerReasonV1.HOST_USER_UNAVAILABLE, + ) + + def test_native_probe_observes_unconfigured_host_user_each_time(self) -> None: + """Ambient uid/gid belong to a capability observation, never backend cache.""" + + transport = importlib.import_module("build.transport") + policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 + backend = transport.NativeDockerBuildBackendV1( + Path("/usr/bin/true"), + policy, + platform_name="linux", + machine_name="x86_64", + ) + + with mock.patch.object( + transport.os, + "geteuid", + side_effect=(501, 502), + ), mock.patch.object( + transport.os, + "getegid", + side_effect=(20, 21), + ): + first = _probe_native_backend(backend, policy) + second = _probe_native_backend(backend, policy) + + self.assertEqual(first.host_user, (501, 20)) + self.assertEqual(second.host_user, (502, 21)) + def test_native_host_coordinates_are_exact_strings_and_oci_ports_are_ascii(self) -> None: transport = importlib.import_module("build.transport") policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 @@ -565,7 +977,6 @@ class StringSubclass(str): ), "platform": policy.platform, "hostname": policy.hostname, - "container_name_prefix": policy.container_name_prefix, "bootstrap": policy.bootstrap, "bootstrap_argv0": policy.bootstrap_argv0, "tmpfs_specs": policy.tmpfs_specs, @@ -643,7 +1054,6 @@ def close(descriptor: int) -> None: stdout_limit=64, stderr_limit=64, timeout_ns=1_000_000_000, - cid_file=None, input_bundle=input_value, ) finally: @@ -685,15 +1095,15 @@ def exercise( machine_name="x86_64", ) spawned: list[subprocess.Popen[bytes]] = [] - cleanup_calls: list[tuple[Path, str]] = [] + cleanup_calls: list[object] = [] def spawn(*args: object, **kwargs: object) -> subprocess.Popen[bytes]: process = real_popen(*args, **kwargs) spawned.append(process) return process - def cleanup(cid_file: Path, container_name: str) -> None: - cleanup_calls.append((cid_file, container_name)) + def cleanup(lease: object, **_kwargs: object) -> None: + cleanup_calls.append(lease) return None def stop_raises(process: subprocess.Popen[bytes]) -> None: @@ -719,7 +1129,11 @@ def stop_raises(process: subprocess.Popen[bytes]) -> None: "-c", "pass" if selector_failure else "import time; time.sleep(5)", ) - with tempfile.TemporaryDirectory() as temporary, mock.patch.object( + lease = backend._next_run_lease_v1( + _docker_capability_fixture(pipeline.ARB_BUILD_TRANSPORT_POLICY_V1) + ) + self.addCleanup(backend._release_run_lease_v1, lease) + with mock.patch.object( transport.subprocess, "Popen", side_effect=spawn, @@ -734,11 +1148,7 @@ def stop_raises(process: subprocess.Popen[bytes]) -> None: stdout_limit=64, stderr_limit=64, timeout_ns=1 if not selector_failure else 1_000_000_000, - cid_file=Path(temporary).resolve() / "container.cid", - container_name=( - pipeline.ARB_BUILD_TRANSPORT_POLICY_V1.container_name_prefix - + ("selector-red" if selector_failure else "stop-red") - ), + lease=lease, ) except Exception as error: result = error @@ -776,6 +1186,511 @@ def stop_raises(process: subprocess.Popen[bytes]) -> None: self.assertTrue(stop_closed) self.assertEqual(stop_cleanups, 1) + def test_base_exception_during_stop_still_reaps_streams_and_container(self) -> None: + transport = importlib.import_module("build.transport") + backend = transport.NativeDockerBuildBackendV1( + Path("/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + host_user=(501, 20), + platform_name="linux", + machine_name="x86_64", + ) + real_popen = subprocess.Popen + spawned: list[subprocess.Popen[bytes]] = [] + cleanup_calls: list[object] = [] + + def spawn(*args: object, **kwargs: object) -> subprocess.Popen[bytes]: + process = real_popen(*args, **kwargs) + spawned.append(process) + return process + + def cleanup(lease: object, **_kwargs: object) -> None: + cleanup_calls.append(lease) + + lease = backend._next_run_lease_v1( + _docker_capability_fixture(pipeline.ARB_BUILD_TRANSPORT_POLICY_V1) + ) + self.addCleanup(backend._release_run_lease_v1, lease) + + with mock.patch.object( + transport.subprocess, + "Popen", + side_effect=spawn, + ), mock.patch.object( + backend, + "_stop_process", + side_effect=KeyboardInterrupt("interrupt during stop"), + ), mock.patch.object( + backend, + "_cleanup_container", + side_effect=cleanup, + ): + with self.assertRaises(KeyboardInterrupt): + backend._observe_command( + (sys.executable, "-c", "import time; time.sleep(5)"), + stdout_limit=64, + stderr_limit=64, + timeout_ns=1, + lease=lease, + ) + process = spawned[0] + running_before_test_cleanup = process.poll() is None + streams_closed = bool( + process.stdout is not None + and process.stdout.closed + and process.stderr is not None + and process.stderr.closed + ) + if running_before_test_cleanup: + process.kill() + process.wait(timeout=5) + for stream in (process.stdin, process.stdout, process.stderr): + if stream is not None and not stream.closed: + stream.close() + + self.assertFalse(running_before_test_cleanup) + self.assertTrue(streams_closed) + self.assertEqual(len(cleanup_calls), 1) + + def test_interrupt_after_spawn_still_reaps_and_attempts_cid_cleanup(self) -> None: + """A post-spawn interruption cannot bypass the native finalizer.""" + + transport = importlib.import_module("build.transport") + backend = transport.NativeDockerBuildBackendV1( + Path("/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + host_user=(501, 20), + platform_name="linux", + machine_name="x86_64", + ) + lease = backend._next_run_lease_v1( + _docker_capability_fixture(pipeline.ARB_BUILD_TRANSPORT_POLICY_V1) + ) + self.addCleanup(backend._release_run_lease_v1, lease) + real_popen = subprocess.Popen + spawned: list[subprocess.Popen[bytes]] = [] + + def spawn(*args: object, **kwargs: object) -> subprocess.Popen[bytes]: + process = real_popen(*args, **kwargs) + spawned.append(process) + return process + + with mock.patch.object( + transport.subprocess, + "Popen", + side_effect=spawn, + ), mock.patch.object( + backend, + "_mark_run_lease_launched_v1", + side_effect=KeyboardInterrupt("interrupt after spawn"), + ), mock.patch.object( + backend, + "_cleanup_container", + return_value=None, + ) as cleanup: + with self.assertRaisesRegex(KeyboardInterrupt, "after spawn"): + backend._observe_command( + (sys.executable, "-c", "import time; time.sleep(5)"), + stdout_limit=64, + stderr_limit=64, + timeout_ns=1, + lease=lease, + ) + + process = spawned[0] + self.assertIsNotNone(process.poll()) + cleanup.assert_called_once_with(lease, spawn_may_have_started=True) + + def test_interrupt_during_post_spawn_state_initialization_reaps_and_cleans(self) -> None: + """No allocation between Popen and the finalizer may leak a child.""" + + transport = importlib.import_module("build.transport") + backend = transport.NativeDockerBuildBackendV1( + Path("/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + host_user=(501, 20), + platform_name="linux", + machine_name="x86_64", + ) + lease = backend._next_run_lease_v1( + _docker_capability_fixture(pipeline.ARB_BUILD_TRANSPORT_POLICY_V1) + ) + self.addCleanup(backend._release_run_lease_v1, lease) + real_popen = subprocess.Popen + spawned: list[subprocess.Popen[bytes]] = [] + + def spawn(*args: object, **kwargs: object) -> subprocess.Popen[bytes]: + process = real_popen(*args, **kwargs) + spawned.append(process) + return process + + with mock.patch.object( + transport.subprocess, + "Popen", + side_effect=spawn, + ), mock.patch.object( + transport, + "bytearray", + side_effect=KeyboardInterrupt("interrupt during post-spawn allocation"), + create=True, + ), mock.patch.object( + backend, + "_cleanup_container", + return_value=None, + ) as cleanup: + with self.assertRaisesRegex( + KeyboardInterrupt, + "post-spawn allocation", + ): + backend._observe_command( + (sys.executable, "-c", "import time; time.sleep(5)"), + stdout_limit=64, + stderr_limit=64, + timeout_ns=1, + lease=lease, + ) + + process = spawned[0] + try: + self.assertIsNotNone(process.poll()) + cleanup.assert_called_once_with(lease, spawn_may_have_started=True) + finally: + if process.poll() is None: + process.kill() + process.wait(timeout=5) + for stream in (process.stdin, process.stdout, process.stderr): + if stream is not None and not stream.closed: + stream.close() + + def test_post_popen_handler_gap_cannot_bypass_finalizer(self) -> None: + """An interrupt at the first bytecode after Popen still owns its child.""" + + transport = importlib.import_module("build.transport") + backend = transport.NativeDockerBuildBackendV1( + Path("/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + host_user=(501, 20), + platform_name="linux", + machine_name="x86_64", + ) + lease = backend._next_run_lease_v1( + _docker_capability_fixture(pipeline.ARB_BUILD_TRANSPORT_POLICY_V1) + ) + self.addCleanup(backend._release_run_lease_v1, lease) + observe = backend._observe_command + instructions = tuple(dis.Bytecode(observe)) + process_store = next( + index + for index, instruction in enumerate(instructions) + if ( + instruction.opname == "STORE_FAST" + and instruction.argval == "process" + and index > 0 + and instructions[index - 1].opname == "CALL_FUNCTION_EX" + ) + ) + interruption_offset = instructions[process_store + 1].offset + real_popen = subprocess.Popen + spawned: list[subprocess.Popen[bytes]] = [] + injected = False + + def spawn(*args: object, **kwargs: object) -> subprocess.Popen[bytes]: + process = real_popen(*args, **kwargs) + spawned.append(process) + return process + + def tracer(frame: object, event: str, _arg: object) -> object: + nonlocal injected + if getattr(frame, "f_code", None) is observe.__code__: + frame.f_trace_opcodes = True + if ( + not injected + and event == "opcode" + and frame.f_lasti == interruption_offset + ): + injected = True + raise KeyboardInterrupt("interrupt in post-Popen handler gap") + return tracer + + with mock.patch.object( + transport.subprocess, + "Popen", + side_effect=spawn, + ), mock.patch.object( + backend, + "_cleanup_container", + return_value=None, + ) as cleanup: + previous = sys.gettrace() + sys.settrace(tracer) + try: + with self.assertRaisesRegex(KeyboardInterrupt, "handler gap"): + observe( + (sys.executable, "-c", "import time; time.sleep(5)"), + stdout_limit=64, + stderr_limit=64, + timeout_ns=1, + lease=lease, + ) + finally: + sys.settrace(previous) + + self.assertTrue(injected) + process = spawned[0] + try: + self.assertIsNotNone(process.poll()) + cleanup.assert_called_once_with(lease, spawn_may_have_started=True) + finally: + if process.poll() is None: + process.kill() + process.wait(timeout=5) + for stream in (process.stdin, process.stdout, process.stderr): + if stream is not None and not stream.closed: + stream.close() + + def test_popen_construction_interrupt_attempts_cid_cleanup_without_a_handle(self) -> None: + """The pre-handle boundary retains the interruption and tries CID cleanup.""" + + transport = importlib.import_module("build.transport") + backend = transport.NativeDockerBuildBackendV1( + Path("/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + host_user=(501, 20), + platform_name="linux", + machine_name="x86_64", + ) + lease = backend._next_run_lease_v1( + _docker_capability_fixture(pipeline.ARB_BUILD_TRANSPORT_POLICY_V1) + ) + self.addCleanup(backend._release_run_lease_v1, lease) + + with mock.patch.object( + transport.subprocess, + "Popen", + side_effect=KeyboardInterrupt("interrupt during Popen construction"), + ), mock.patch.object( + backend, + "_cleanup_container", + return_value=None, + ) as cleanup: + with self.assertRaisesRegex(KeyboardInterrupt, "Popen construction"): + backend._observe_command( + (sys.executable, "-c", "pass"), + stdout_limit=64, + stderr_limit=64, + timeout_ns=1, + lease=lease, + ) + + cleanup.assert_called_once_with(lease, spawn_may_have_started=True) + + @unittest.skipUnless(hasattr(os, "fork"), "requires POSIX fork") + def test_forked_child_gc_cannot_delete_parent_cid_root(self) -> None: + transport = importlib.import_module("build.transport") + backend = transport.NativeDockerBuildBackendV1( + Path("/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + host_user=(501, 20), + platform_name="linux", + machine_name="x86_64", + ) + lease = backend._next_run_lease_v1( + _docker_capability_fixture(pipeline.ARB_BUILD_TRANSPORT_POLICY_V1) + ) + root = lease.cid_file.parent + try: + child = os.fork() + if child == 0: + del lease + gc.collect() + os._exit(0) + _pid, status = os.waitpid(child, 0) + self.assertEqual(os.waitstatus_to_exitcode(status), 0) + self.assertTrue(root.is_dir()) + finally: + backend._release_run_lease_v1(lease) + + def test_interrupted_cid_root_release_remains_retryable(self) -> None: + """A failed root release must not permanently consume its cleanup lease.""" + + transport = importlib.import_module("build.transport") + backend = transport.NativeDockerBuildBackendV1( + Path("/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + host_user=(501, 20), + platform_name="linux", + machine_name="x86_64", + ) + lease = backend._next_run_lease_v1( + _docker_capability_fixture(pipeline.ARB_BUILD_TRANSPORT_POLICY_V1) + ) + root = lease.cid_file.parent + real_rmtree = transport.shutil.rmtree + try: + with mock.patch.object( + transport.shutil, + "rmtree", + side_effect=KeyboardInterrupt("interrupt during CID-root release"), + ): + with self.assertRaisesRegex(KeyboardInterrupt, "CID-root release"): + backend._release_run_lease_v1(lease) + + self.assertTrue(root.is_dir()) + self.assertFalse(lease._released) + self.assertIsNone(backend._release_run_lease_v1(lease)) + self.assertFalse(root.exists()) + finally: + if root.exists(): + real_rmtree(root) + + def test_stop_interrupt_survives_container_cleanup_failure(self) -> None: + """A later cleanup error cannot replace the caller's interruption.""" + + transport = importlib.import_module("build.transport") + backend = transport.NativeDockerBuildBackendV1( + Path("/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + host_user=(501, 20), + platform_name="linux", + machine_name="x86_64", + ) + real_popen = subprocess.Popen + spawned: list[subprocess.Popen[bytes]] = [] + + def spawn(*args: object, **kwargs: object) -> subprocess.Popen[bytes]: + process = real_popen(*args, **kwargs) + spawned.append(process) + return process + + lease = backend._next_run_lease_v1( + _docker_capability_fixture(pipeline.ARB_BUILD_TRANSPORT_POLICY_V1) + ) + self.addCleanup(backend._release_run_lease_v1, lease) + + with mock.patch.object( + transport.subprocess, + "Popen", + side_effect=spawn, + ), mock.patch.object( + backend, + "_stop_process", + side_effect=KeyboardInterrupt("interrupt during stop"), + ), mock.patch.object( + backend, + "_cleanup_container", + side_effect=OSError("cleanup failed after interruption"), + ) as cleanup: + with self.assertRaisesRegex(KeyboardInterrupt, "interrupt during stop"): + backend._observe_command( + (sys.executable, "-c", "import time; time.sleep(5)"), + stdout_limit=64, + stderr_limit=64, + timeout_ns=1, + lease=lease, + ) + process = spawned[0] + running_before_test_cleanup = process.poll() is None + streams_closed = bool( + process.stdout is not None + and process.stdout.closed + and process.stderr is not None + and process.stderr.closed + ) + if running_before_test_cleanup: + process.kill() + process.wait(timeout=5) + for stream in (process.stdin, process.stdout, process.stderr): + if stream is not None and not stream.closed: + stream.close() + + self.assertFalse(running_before_test_cleanup) + self.assertTrue(streams_closed) + self.assertEqual(cleanup.call_count, 1) + + def test_stream_close_interrupt_still_closes_siblings_and_cleans_container(self) -> None: + transport = importlib.import_module("build.transport") + backend = transport.NativeDockerBuildBackendV1( + Path("/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + host_user=(501, 20), + platform_name="linux", + machine_name="x86_64", + ) + real_popen = subprocess.Popen + spawned: list[subprocess.Popen[bytes]] = [] + wrapped_stdout: list[object] = [] + cleanup_calls: list[object] = [] + + class CloseInterrupts: + def __init__(self, wrapped: object) -> None: + self.wrapped = wrapped + self.descriptor = wrapped.fileno() + + @property + def closed(self) -> bool: + return False + + def fileno(self) -> int: + return self.descriptor + + def close(self) -> None: + raise KeyboardInterrupt("interrupt during stdout close") + + def spawn(*args: object, **kwargs: object) -> subprocess.Popen[bytes]: + process = real_popen(*args, **kwargs) + spawned.append(process) + wrapper = CloseInterrupts(process.stdout) + wrapped_stdout.append(wrapper) + process.stdout = wrapper + return process + + def cleanup(lease: object, **_kwargs: object) -> None: + cleanup_calls.append(lease) + + lease = backend._next_run_lease_v1( + _docker_capability_fixture(pipeline.ARB_BUILD_TRANSPORT_POLICY_V1) + ) + self.addCleanup(backend._release_run_lease_v1, lease) + with mock.patch.object( + transport.subprocess, + "Popen", + side_effect=spawn, + ), mock.patch.object( + backend, + "_cleanup_container", + side_effect=cleanup, + ): + with self.assertRaisesRegex(KeyboardInterrupt, "stdout close"): + backend._observe_command( + (sys.executable, "-c", "pass"), + stdout_limit=64, + stderr_limit=64, + timeout_ns=1_000_000_000, + lease=lease, + ) + process = spawned[0] + stderr_closed = process.stderr is not None and process.stderr.closed + try: + os.fstat(wrapped_stdout[0].descriptor) + except OSError: + stdout_descriptor_closed = True + else: + stdout_descriptor_closed = False + if process.poll() is None: + process.kill() + process.wait(timeout=5) + for stream in (process.stdin, process.stderr): + if stream is not None and not stream.closed: + stream.close() + try: + wrapped_stdout[0].wrapped.close() + except OSError: + pass + + self.assertTrue(stderr_closed) + self.assertTrue(stdout_descriptor_closed) + self.assertEqual(cleanup_calls, [lease]) + def test_process_and_container_cleanup_failures_are_both_retained_in_order(self) -> None: transport = importlib.import_module("build.transport") backend = transport.NativeDockerBuildBackendV1( @@ -792,7 +1707,12 @@ def stop(process: subprocess.Popen[bytes]) -> str: process.wait(timeout=5) return "process stop failed" - with tempfile.TemporaryDirectory() as temporary, mock.patch.object( + lease = backend._next_run_lease_v1( + _docker_capability_fixture(pipeline.ARB_BUILD_TRANSPORT_POLICY_V1) + ) + self.addCleanup(backend._release_run_lease_v1, lease) + + with mock.patch.object( transport.subprocess, "Popen", side_effect=real_popen, @@ -810,11 +1730,7 @@ def stop(process: subprocess.Popen[bytes]) -> str: stdout_limit=64, stderr_limit=64, timeout_ns=1, - cid_file=Path(temporary).resolve() / "container.cid", - container_name=( - pipeline.ARB_BUILD_TRANSPORT_POLICY_V1.container_name_prefix - + "cleanup-records" - ), + lease=lease, input_bundle=_sealed_input(), ) @@ -872,117 +1788,62 @@ def test_impossible_build_failures_without_input_progress_are_contract_violation ) self.assertIsNone(result.process) - def test_temporary_root_cleanup_failure_retains_current_process_generically(self) -> None: + def test_controller_passes_only_semantic_build_request_to_its_backend(self) -> None: transport = importlib.import_module("build.transport") policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 input_value = _sealed_input() - first = _completed_process(transport, input_value, b"first") - current = _completed_process(transport, input_value, b"second") - real_temporary_directory = tempfile.TemporaryDirectory - allocated: list[object] = [] - allocations = 0 - - class CleanupFailsOnce: - def __init__(self, *args: object, **kwargs: object) -> None: - self._inner = real_temporary_directory(*args, **kwargs) - self.name = self._inner.name - self._failed = False - allocated.append(self) - - def __enter__(self) -> str: - return self.name - - def __exit__(self, *_args: object) -> None: - self.cleanup() - - def cleanup(self) -> None: - if not self._failed: - self._failed = True - raise OSError("forced temporary-root cleanup failure") - self._inner.cleanup() - - def temporary_directory(*args: object, **kwargs: object) -> object: - nonlocal allocations - allocations += 1 - if allocations == 1: - return real_temporary_directory(*args, **kwargs) - return CleanupFailsOnce(*args, **kwargs) - - try: - with mock.patch.object( - transport.tempfile, - "TemporaryDirectory", - side_effect=temporary_directory, - ): - result, _backend, _report, _input = _controlled_build( - transport, - policy, - (first, current), - input_value=input_value, - ) - finally: - for temporary in allocated: - temporary.cleanup() - - self.assertIs(type(result), transport.BuildRejectedV1) - self.assertEqual( - result.reason, - transport.BuildFailureReasonV1.CLEANUP_FAILED, + observations = ( + _completed_process(transport, input_value, b"first"), + _completed_process(transport, input_value, b"second"), ) - self.assertEqual(result.attempt, 2) - self.assertEqual(result.completed_processes, (first,)) - self.assertIs(type(result.process), transport.BuildCleanupFailureV1) - self.assertIs(result.process.current_process, current) - self.assertEqual(len(result.process.failures), 1) - self.assertEqual( - result.process.failures[0].resource, - transport.CleanupResourceV1.TEMPORARY_ROOT, + result, backend, _capability, _input = _controlled_build( + transport, + policy, + observations, + input_value=input_value, ) - def test_temporary_root_cleanup_failure_is_typed_without_a_process(self) -> None: + self.assertIs(type(result), transport.TwoBuildObservationV1) + self.assertEqual(len(backend.requests), 2) + for request in backend.requests: + self.assertEqual(len(tuple(request)), 4) + self.assertFalse(hasattr(request, "cid_file")) + self.assertFalse(hasattr(request, "container_name")) + + def test_backend_interrupt_propagates_without_controller_cleanup_authority(self) -> None: transport = importlib.import_module("build.transport") policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 - real_temporary_directory = tempfile.TemporaryDirectory - inner = real_temporary_directory() - - class CleanupFailsOnce: - name = inner.name + input_value = _sealed_input() + capability = _docker_capability_fixture(policy) + class InterruptingBackend: def __init__(self) -> None: - self.failed = False + self.requests: list[object] = [] - def cleanup(self) -> None: - if not self.failed: - self.failed = True - raise OSError("forced temporary-root cleanup failure") - inner.cleanup() + def probe(self) -> object: + return capability - temporary = CleanupFailsOnce() - try: - with mock.patch.object( - transport.tempfile, - "TemporaryDirectory", - return_value=temporary, - ): - result, _backend, _capability, _input = _controlled_build( - transport, - policy, - (), - ) - finally: - temporary.cleanup() + def run_build(self, request: object) -> object: + self.requests.append(request) + raise KeyboardInterrupt("interrupt during build observation") - self.assertIs(type(result), transport.BuildRejectedV1) - self.assertEqual( - result.reason, - transport.BuildFailureReasonV1.CLEANUP_FAILED, - ) - self.assertIs(type(result.process), transport.BuildCleanupFailureV1) - self.assertIsNone(result.process.current_process) - self.assertEqual( - tuple(record.resource for record in result.process.failures), - (transport.CleanupResourceV1.TEMPORARY_ROOT,), + backend = InterruptingBackend() + controller = transport.ControlledBuildTransportV1( + policy=policy, + backend=backend, ) + owned_capability = controller.probe() + with self.assertRaisesRegex(KeyboardInterrupt, "interrupt during build observation"): + controller.build( + owned_capability, + input_value, + 64, + input_admission=lambda _value: True, + output_admission=lambda _value: True, + ) + + self.assertEqual(len(backend.requests), 1) + self.assertEqual(len(tuple(backend.requests[0])), 4) def test_forged_backend_failures_canonicalize_to_contract_violation(self) -> None: transport = importlib.import_module("build.transport") @@ -993,7 +1854,6 @@ def test_forged_backend_failures_canonicalize_to_contract_violation(self) -> Non transport.DockerBuildObserverFailureV1, transport.DockerBuildInputRejectedV1, transport.DockerBuildCleanupFailureV1, - transport.BuildCleanupFailureV1, ): with self.subTest(failure=failure_type.__name__): forged = _forged_exact_type(failure_type) diff --git a/proof/region/v1/tests/test_build_identity.py b/proof/region/v1/tests/test_build_identity.py index 4df559a2..4c7ce861 100644 --- a/proof/region/v1/tests/test_build_identity.py +++ b/proof/region/v1/tests/test_build_identity.py @@ -7,6 +7,7 @@ import inspect import json import os +import subprocess import sys import unittest from pathlib import Path @@ -24,7 +25,6 @@ "image_reference", "platform", "hostname", - "container_name_prefix", "bootstrap", "bootstrap_argv0", "tmpfs_specs", @@ -83,8 +83,6 @@ def _slot(value: str) -> tuple[str, str]: _literal("ALL"), _literal("--security-opt"), _literal("no-new-privileges:true"), - _literal("--name"), - _slot("container_name"), _literal("--hostname"), _slot("hostname"), _literal("--user"), @@ -120,6 +118,17 @@ def _slot(value: str) -> tuple[str, str]: _slot("container_coordinate"), ), ), + ( + "cleanup_inspect", + ( + _slot("cli_path"), + _literal("container"), + _literal("inspect"), + _literal("--format"), + _literal("{{.Id}}"), + _slot("container_coordinate"), + ), + ), ( "cleanup_ls", ( @@ -135,6 +144,24 @@ def _slot(value: str) -> tuple[str, str]: ), ) +_NATIVE_PROCESS_ENVIRONMENT_V1 = ( + ("DOCKER_CONFIG", "/nonexistent"), + ("HOME", "/nonexistent"), + ("LANG", "C"), + ("LC_ALL", "C"), + ("PATH", "/usr/bin:/bin"), + ("TZ", "UTC"), +) +_NATIVE_PROCESS_CWD_V1 = "/" +_NATIVE_PROCESS_UMASK_V1 = 0o077 +_NATIVE_PROCESS_CLOSE_FDS_V1 = True +_NATIVE_PROCESS_RESTORE_SIGNALS_V1 = True +_NATIVE_PROCESS_START_NEW_SESSION_V1 = True +_NATIVE_PROCESS_STDIN_WITH_INPUT_V1 = "pipe" +_NATIVE_PROCESS_STDIN_WITHOUT_INPUT_V1 = "devnull" +_NATIVE_PROCESS_STDOUT_V1 = "pipe" +_NATIVE_PROCESS_STDERR_V1 = "pipe" + def _blob(value: bytes) -> bytes: return len(value).to_bytes(8, "big") + value @@ -159,7 +186,6 @@ def _policy_chunks(coordinates: dict[str, object]) -> tuple[bytes, ...]: coordinates["image_reference"].encode("utf-8"), coordinates["platform"].encode("utf-8"), coordinates["hostname"].encode("utf-8"), - coordinates["container_name_prefix"].encode("utf-8"), coordinates["bootstrap"].encode("utf-8"), coordinates["bootstrap_argv0"].encode("utf-8"), len(tmpfs_specs).to_bytes(4, "big"), @@ -186,6 +212,31 @@ def _expected_command_contract_identity() -> bytes: chunks.extend((name.encode("ascii"), len(tokens).to_bytes(4, "big"))) for tag, value in tokens: chunks.extend((tag.encode("ascii"), value.encode("utf-8"))) + chunks.extend( + ( + b"native-process-context.v1", + len(_NATIVE_PROCESS_ENVIRONMENT_V1).to_bytes(4, "big"), + *( + item + for key, value in _NATIVE_PROCESS_ENVIRONMENT_V1 + for item in (key.encode("ascii"), value.encode("utf-8")) + ), + _NATIVE_PROCESS_CWD_V1.encode("ascii"), + _NATIVE_PROCESS_UMASK_V1.to_bytes(4, "big"), + bytes((_NATIVE_PROCESS_CLOSE_FDS_V1,)), + bytes((_NATIVE_PROCESS_RESTORE_SIGNALS_V1,)), + bytes((_NATIVE_PROCESS_START_NEW_SESSION_V1,)), + b"native-stdio-topology.v1", + b"stdin-with-input", + _NATIVE_PROCESS_STDIN_WITH_INPUT_V1.encode("ascii"), + b"stdin-without-input", + _NATIVE_PROCESS_STDIN_WITHOUT_INPUT_V1.encode("ascii"), + b"stdout", + _NATIVE_PROCESS_STDOUT_V1.encode("ascii"), + b"stderr", + _NATIVE_PROCESS_STDERR_V1.encode("ascii"), + ) + ) return _identity( b"labcolors.proof-region.native-command-contract.v1\0", tuple(chunks), @@ -244,7 +295,6 @@ def _policy(**changes: object) -> object: ), "platform": "linux/amd64", "hostname": "lc-build", - "container_name_prefix": "lc-build-", "bootstrap": "set -eu\ncat", "bootstrap_argv0": "labcolors-build-v1", "tmpfs_specs": ( @@ -341,8 +391,6 @@ def test_identity_surface_is_exact_and_has_no_legacy_report_aliases(self) -> Non "capability", "input_bundle", "max_output_bytes", - "cid_file", - "container_name", ), ) self.assertFalse(hasattr(transport, "docker_report_matches_policy_v1")) @@ -357,8 +405,6 @@ def test_identity_surface_is_exact_and_has_no_legacy_report_aliases(self) -> Non capability, _sealed_input(), 64, - Path("/tmp/lab-colors-identity.cid"), - capability.policy.container_name_prefix + "identity", ) for legacy in ( "image_reference", @@ -368,6 +414,8 @@ def test_identity_surface_is_exact_and_has_no_legacy_report_aliases(self) -> Non with self.subTest(legacy_capability_property=legacy): self.assertFalse(hasattr(capability, legacy)) self.assertFalse(hasattr(request, "policy")) + self.assertFalse(hasattr(request, "cid_file")) + self.assertFalse(hasattr(request, "container_name")) self.assertIs(request.capability, capability) with self.assertRaises(TypeError): @@ -378,7 +426,7 @@ def test_identity_surface_is_exact_and_has_no_legacy_report_aliases(self) -> Non capability.host_user, ) - def test_policy_identity_binds_all_thirteen_coordinates(self) -> None: + def test_policy_identity_binds_all_twelve_coordinates(self) -> None: policy = _policy() coordinates = _policy_coordinates(policy) identity = transport.transport_policy_identity_v1(policy) @@ -402,7 +450,6 @@ def test_policy_identity_binds_all_thirteen_coordinates(self) -> None: ), "platform": "linux/arm64", "hostname": "lc-build-alt", - "container_name_prefix": "lc-alt-", "bootstrap": "set -eu\nprintf changed", "bootstrap_argv0": "labcolors-build-v1-alt", "tmpfs_specs": coordinates["tmpfs_specs"] + ("/run:rw,size=4096",), @@ -596,6 +643,62 @@ def test_capability_identity_keeps_policy_daemon_path_and_user_orthogonal(self) class NativeCommandAndRequestTests(unittest.TestCase): + def test_native_process_context_is_one_identity_bound_launch_renderer(self) -> None: + expected_base = { + "stdout": subprocess.PIPE, + "stderr": subprocess.PIPE, + "cwd": _NATIVE_PROCESS_CWD_V1, + "env": dict(_NATIVE_PROCESS_ENVIRONMENT_V1), + "close_fds": _NATIVE_PROCESS_CLOSE_FDS_V1, + "restore_signals": _NATIVE_PROCESS_RESTORE_SIGNALS_V1, + "start_new_session": _NATIVE_PROCESS_START_NEW_SESSION_V1, + "umask": _NATIVE_PROCESS_UMASK_V1, + } + context = transport._NATIVE_PROCESS_CONTEXT_V1 + for receives_stdin, stdin in ( + (False, subprocess.DEVNULL), + (True, subprocess.PIPE), + ): + with self.subTest(receives_stdin=receives_stdin): + expected = {"stdin": stdin, **expected_base} + first = context.popen_kwargs_v1(receives_stdin) + second = context.popen_kwargs_v1(receives_stdin) + self.assertEqual(first, expected) + self.assertEqual(second, expected) + self.assertIsNot(first, second) + self.assertIsNot(first["env"], second["env"]) + + backend = transport.NativeDockerBuildBackendV1( + Path("/usr/bin/true"), + _policy(), + platform_name="linux", + machine_name="x86_64", + host_user=(501, 20), + ) + with mock.patch.object( + transport.subprocess, + "Popen", + side_effect=OSError("do not launch in identity test"), + ) as spawn: + for receives_stdin, stdin in ( + (False, subprocess.DEVNULL), + (True, subprocess.PIPE), + ): + with self.subTest(receives_stdin=receives_stdin): + result = backend._observe_command( + ("/usr/bin/true",), + stdout_limit=64, + stderr_limit=64, + timeout_ns=1_000_000_000, + input_bundle=_sealed_input() if receives_stdin else None, + ) + self.assertIs( + type(result), + transport.DockerBuildObserverFailureV1, + ) + expected = {"stdin": stdin, **expected_base} + self.assertEqual(spawn.call_args.kwargs, expected) + def test_native_backend_requires_its_exact_probe_lease(self) -> None: policy = _policy() backend = transport.NativeDockerBuildBackendV1( @@ -611,11 +714,9 @@ def test_native_backend_requires_its_exact_probe_lease(self) -> None: unobserved, _sealed_input(), 64, - Path("/tmp/lab-colors-identity.cid"), - policy.container_name_prefix + "identity", ) with self.assertRaises(TypeError): - backend.command_for(request) + backend._bound_request_capability_v1(request) server_stdout = b'{"Version":"identity-test"}\n' image_stdout = json.dumps( @@ -644,13 +745,11 @@ def test_native_backend_requires_its_exact_probe_lease(self) -> None: equal_but_foreign, _sealed_input(), 64, - Path("/tmp/lab-colors-identity.cid"), - policy.container_name_prefix + "identity", ) with self.assertRaises(TypeError): - backend.command_for(cloned_request) + backend._bound_request_capability_v1(cloned_request) - def test_command_for_expands_the_versioned_template_to_exact_argv(self) -> None: + def test_native_adapter_expands_the_versioned_template_to_exact_argv(self) -> None: policy = _policy() docker_path = Path("/usr/bin/true") backend = transport.NativeDockerBuildBackendV1( @@ -684,73 +783,71 @@ def test_command_for_expands_the_versioned_template_to_exact_argv(self) -> None: self.assertIs(type(capability), transport.DockerSupportedV1) input_bundle = _sealed_input() - cid_file = Path("/tmp/lab-colors-identity.cid") - container_name = policy.container_name_prefix + "identity" request = transport.DockerBuildRequestV1( 1, capability, input_bundle, 64, - cid_file, - container_name, - ) - command = backend.command_for(request) - expected = ( - str(docker_path), - "run", - "--rm", - "--interactive", - "--pull", - "never", - "--platform", - policy.platform, - "--network", - "none", - "--read-only", - "--tmpfs", - policy.tmpfs_specs[0], - "--tmpfs", - policy.tmpfs_specs[1], - "--cap-drop", - "ALL", - "--security-opt", - "no-new-privileges:true", - "--name", - container_name, - "--hostname", - policy.hostname, - "--user", - "501:20", - "--workdir", - "/", - "--cidfile", - str(cid_file), - "--entrypoint", - "/usr/bin/env", - policy.image_reference, - "-i", - "PATH=/usr/local/bin:/usr/bin:/bin", - "LC_ALL=C", - "LANG=C", - "TZ=UTC", - "HOME=/nonexistent", - "/bin/sh", - "-c", - policy.bootstrap, - policy.bootstrap_argv0, - str(input_bundle.length), - input_bundle.sha256.hex(), - ) - self.assertEqual(command, expected) - self.assertEqual(command.count("--tmpfs"), len(policy.tmpfs_specs)) - self.assertLess( - command.index(policy.tmpfs_specs[0]), - command.index(policy.tmpfs_specs[1]), - ) - self.assertEqual( - transport.native_command_contract_identity_v1(), - _expected_command_contract_identity(), ) + lease = backend._next_run_lease_v1(capability) + try: + command = backend._command_for_v1(request, lease) + expected = ( + str(docker_path), + "run", + "--rm", + "--interactive", + "--pull", + "never", + "--platform", + policy.platform, + "--network", + "none", + "--read-only", + "--tmpfs", + policy.tmpfs_specs[0], + "--tmpfs", + policy.tmpfs_specs[1], + "--cap-drop", + "ALL", + "--security-opt", + "no-new-privileges:true", + "--hostname", + policy.hostname, + "--user", + "501:20", + "--workdir", + "/", + "--cidfile", + str(lease.cid_file), + "--entrypoint", + "/usr/bin/env", + policy.image_reference, + "-i", + "PATH=/usr/local/bin:/usr/bin:/bin", + "LC_ALL=C", + "LANG=C", + "TZ=UTC", + "HOME=/nonexistent", + "/bin/sh", + "-c", + policy.bootstrap, + policy.bootstrap_argv0, + str(input_bundle.length), + input_bundle.sha256.hex(), + ) + self.assertEqual(command, expected) + self.assertEqual(command.count("--tmpfs"), len(policy.tmpfs_specs)) + self.assertLess( + command.index(policy.tmpfs_specs[0]), + command.index(policy.tmpfs_specs[1]), + ) + self.assertEqual( + transport.native_command_contract_identity_v1(), + _expected_command_contract_identity(), + ) + finally: + backend._release_run_lease_v1(lease) def test_foreign_capability_is_rejected_before_backend_run(self) -> None: policy = _policy() @@ -795,8 +892,6 @@ def test_request_is_deeply_immutable_and_owns_capability_not_policy(self) -> Non capability, _sealed_input(), 64, - Path("/tmp/lab-colors-identity.cid"), - capability.policy.container_name_prefix + "identity", ) self.assertIs(request.capability, capability) From f34fa37913b8cd7f812ac6ac9babcc4487c41b1a Mon Sep 17 00:00:00 2001 From: Daniel from Labpics <63733699+lemone112@users.noreply.github.com> Date: Sat, 1 Aug 2026 09:22:20 +0300 Subject: [PATCH 03/20] =?UTF-8?q?Proof:=20=D1=83=D0=BA=D1=80=D0=B5=D0=BF?= =?UTF-8?q?=D0=B8=D1=82=D1=8C=20transport=20boundary=20=D0=B8=20receipt=20?= =?UTF-8?q?trust?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- proof/region/v1/PROTOCOL.md | 6 +- proof/region/v1/arb/receipt.py | 13 +++- proof/region/v1/arb/tests/gate.py | 2 +- .../v1/arb/tests/test_build_identity_v2.py | 6 +- proof/region/v1/arb/tests/test_receipt.py | 75 +++++++++++++++++-- proof/region/v1/arb/tests/test_transport.py | 74 ++++++++++++++++-- proof/region/v1/build/input.py | 8 +- proof/region/v1/build/transport.py | 64 ++++++++-------- proof/region/v1/tests/test_build.py | 34 +++++++-- proof/region/v1/tests/test_build_identity.py | 22 ++++-- 10 files changed, 236 insertions(+), 68 deletions(-) diff --git a/proof/region/v1/PROTOCOL.md b/proof/region/v1/PROTOCOL.md index c8dc4eea..70b455c4 100644 --- a/proof/region/v1/PROTOCOL.md +++ b/proof/region/v1/PROTOCOL.md @@ -276,8 +276,10 @@ streams и очистить допущенный container. Во время са handle может ещё отсутствовать: тогда возможна только best-effort попытка CID cleanup, без ложного заявления о reap CLI. `TwoBuildObservationV1` хранит обе успешные попытки и только классифицирует их байты как identical или different, не называя пару -универсальным доказательством воспроизводимости. При отказе сохраняется весь -уже завершённый causal prefix. Transport не знает formula, ELF, comparator или +универсальным доказательством воспроизводимости. При отказе после создания +валидной session сохраняется весь уже завершённый causal prefix; нарушение +контракта, выявленное до неё, может не иметь ни session, ни process prefix. +Transport не знает formula, ELF, comparator или source provenance: lane отдельно перепроверяет semantic input binding перед каждым process и передаёт output admission. Arb объявляет собственную exact policy; MPFI обязан объявить другую, а не заимствовать Arb semantics. diff --git a/proof/region/v1/arb/receipt.py b/proof/region/v1/arb/receipt.py index d5191f05..eb04237d 100644 --- a/proof/region/v1/arb/receipt.py +++ b/proof/region/v1/arb/receipt.py @@ -50,6 +50,7 @@ def _identity(label: bytes, chunks: tuple[bytes, ...]) -> bytes: def source_bound_policy_identity_v2( capability: build_transport.DockerSupportedV1, + host_trust: pipeline.HostTrustBoundaryV1, ) -> bytes: """Identity of the exact observation rules and observed BUILD capability.""" @@ -59,7 +60,7 @@ def source_bound_policy_identity_v2( _SOURCE_BOUND_POLICY_ID_LABEL_V2, ( pipeline.pipeline_policy_identity_v2( - pipeline.HostTrustBoundaryV1.UNSEALED_LINUX_X64_DOCKER_HOST, + host_trust, capability.policy, ), capability_identity, @@ -506,7 +507,10 @@ def __init__( raise TypeError("SourceBoundEvaluatorReceiptV1 is controller-sealed") if ( claim.provenance_policy_identity - != source_bound_policy_identity_v2(evidence.build.docker_capability) + != source_bound_policy_identity_v2( + evidence.build.docker_capability, + evidence.request.host_trust, + ) or claim.run_claim_identity != evidence.run_claim.identity or claim.replay_evidence_identity != evidence.identity ): @@ -800,7 +804,10 @@ def execute(self, request: pipeline.PipelineRequestV1) -> SourceBoundResultV1: _token=_EVIDENCE_TOKEN, ) claim = protocol.EvaluatorProvenanceClaimV1( - source_bound_policy_identity_v2(built.docker_capability), + source_bound_policy_identity_v2( + built.docker_capability, + replay_request.host_trust, + ), run_claim.identity, evidence.identity, ) diff --git a/proof/region/v1/arb/tests/gate.py b/proof/region/v1/arb/tests/gate.py index 50a6e012..b518606b 100644 --- a/proof/region/v1/arb/tests/gate.py +++ b/proof/region/v1/arb/tests/gate.py @@ -15,7 +15,7 @@ REPO = Path(__file__).resolve().parents[5] sys.path.insert(0, str(REPO)) EXPECTED_TEST_INVENTORY_SHA256 = ( - "4853e06c6e8c1864bc65e0b4c0cd9cdbe0881e0d5907daecb6c8a9fea42f3643" + "e93060f8fa2ff5035bcc394f92dccc5f7f8baf7f9e019fc13cda933295393dce" ) _EVALUATOR_REASON = "set LABCOLORS_ARB_EVALUATOR to the controlled C17 binary" EXPECTED_SKIPS = frozenset( diff --git a/proof/region/v1/arb/tests/test_build_identity_v2.py b/proof/region/v1/arb/tests/test_build_identity_v2.py index d7187a76..1341eaee 100644 --- a/proof/region/v1/arb/tests/test_build_identity_v2.py +++ b/proof/region/v1/arb/tests/test_build_identity_v2.py @@ -175,7 +175,8 @@ def _observed_build_coordinates( result, ) source_bound_policy = receipt.source_bound_policy_identity_v2( - result.docker_capability + result.docker_capability, + request.host_trust, ) process_encodings = tuple( build_transport.build_process_bytes_v1(process) @@ -367,8 +368,7 @@ def test_path_uid_daemon_and_hostname_flow_to_downstream_build_identity_only(sel bundle_bytes, source_bound_policy, ) = variant - if name != "hostname": - self.assertNotEqual(capability_identity, baseline_capability) + self.assertNotEqual(capability_identity, baseline_capability) self.assertNotEqual(comparator_build, baseline_comparator_build) self.assertNotEqual(receipt_build, baseline_receipt_build) self.assertNotEqual( diff --git a/proof/region/v1/arb/tests/test_receipt.py b/proof/region/v1/arb/tests/test_receipt.py index f7e711f7..368014b3 100644 --- a/proof/region/v1/arb/tests/test_receipt.py +++ b/proof/region/v1/arb/tests/test_receipt.py @@ -223,11 +223,28 @@ def _replace_invocation( class SourceBoundReceiptTests(unittest.TestCase): def test_source_bound_policy_identity_binds_immutable_coordinates(self) -> None: capability = _docker_capability() + request = _request() + # This golden belongs to the exact observed capability fixture; changing + # its daemon, CLI path or host user must deliberately rederive it. self.assertEqual( - receipt.source_bound_policy_identity_v2(capability).hex(), + receipt.source_bound_policy_identity_v2( + capability, + request.host_trust, + ).hex(), "f223e1a1569ca5cf6251fd012af8a789a75aedd830e3ccb8f13db77d7ac67bd4", ) + def test_source_bound_policy_identity_consumes_explicit_trust_coordinate(self) -> None: + capability = _docker_capability() + trust = object() + with mock.patch.object( + receipt.pipeline, + "pipeline_policy_identity_v2", + return_value=_digest("pipeline-policy"), + ) as policy_identity: + receipt.source_bound_policy_identity_v2(capability, trust) + policy_identity.assert_called_once_with(trust, capability.policy) + def test_identity_rejection_remains_typed_at_the_receipt_boundary(self) -> None: invocation_rejection = executor.ExecutionIdentityRejectedV1( executor.ExecutionIdentityReasonV1.REQUEST_NOT_ADMITTED, @@ -280,7 +297,8 @@ def test_only_controller_execution_can_seal_a_receipt(self) -> None: self.assertEqual( result.claim.provenance_policy_identity, receipt.source_bound_policy_identity_v2( - result.evidence.build.docker_capability + result.evidence.build.docker_capability, + result.evidence.request.host_trust, ), ) self.assertTrue(receipt.replay_evidence_is_well_bound_v1(result.evidence)) @@ -494,10 +512,55 @@ def test_source_process_transfer_and_comparator_mutations_fail(self) -> None: first = dag.build.build_processes[0] self.assertFalse(hasattr(first.input_transfer, "__dict__")) self.assertFalse(hasattr(first, "__dict__")) - with self.assertRaises(TypeError): - object.__new__(type(first.input_transfer)) - with self.assertRaises(TypeError): - object.__new__(type(first)) + forged_transfer = tuple.__new__( + type(first.input_transfer), + ( + first.input_transfer.bundle_identity, + first.input_transfer.expected_length + 1, + first.input_transfer.expected_sha256, + first.input_transfer.written_length, + first.input_transfer.written_sha256, + ), + ) + forged_process = tuple.__new__( + type(first), + ( + first.returncode, + first.stdout, + first.stderr, + forged_transfer, + ), + ) + forged_build = _tamper( + dag.build, + "build_processes", + (forged_process, dag.build.build_processes[1]), + ) + self.assertFalse( + receipt.replay_evidence_is_well_bound_v1( + _tamper(dag, "build", forged_build) + ) + ) + + forged_process = tuple.__new__( + type(first), + ( + first.returncode + 1, + first.stdout, + first.stderr, + first.input_transfer, + ), + ) + forged_build = _tamper( + dag.build, + "build_processes", + (forged_process, dag.build.build_processes[1]), + ) + self.assertFalse( + receipt.replay_evidence_is_well_bound_v1( + _tamper(dag, "build", forged_build) + ) + ) preimages = _tamper( dag.build.comparator.preimages, diff --git a/proof/region/v1/arb/tests/test_transport.py b/proof/region/v1/arb/tests/test_transport.py index e145df05..b6c941fa 100644 --- a/proof/region/v1/arb/tests/test_transport.py +++ b/proof/region/v1/arb/tests/test_transport.py @@ -6,10 +6,12 @@ import hashlib import io import inspect +import json import os import subprocess import sys import tarfile +import tempfile import unittest from pathlib import Path from unittest import mock @@ -203,11 +205,6 @@ def reject( build_input.InputReasonV1.NONCANONICAL_SET, "A", ), - ( - ((("a" * 120) + "/f", 0o644, b"x"),), - build_input.InputReasonV1.INVALID_PATH, - "a" * 120, - ), ): with self.subTest(hostile=repr(hostile)): reject(hostile, reason, field) @@ -253,6 +250,26 @@ def reject( exact_cap.max_encoded_bytes, ) + def test_long_implicit_directory_uses_the_ustar_trailing_separator(self) -> None: + directory = "d" * 155 + entries = ((f"{directory}/f", 0o644, b"x"),) + limits = build_input.CanonicalInputLimitsV1(2, 1, 1) + + encoded = build_input.canonical_ustar_v1(entries, limits) + + with tarfile.open(fileobj=io.BytesIO(encoded), mode="r:") as archive: + self.assertEqual( + tuple(member.name for member in archive), + (directory, f"{directory}/f"), + ) + with self.assertRaises(build_input.InputErrorV1) as caught: + build_input.canonical_ustar_v1( + ((f"{'d' * 156}/f", 0o644, b"x"),), + limits, + ) + self.assertEqual(caught.exception.reason, build_input.InputReasonV1.INVALID_PATH) + self.assertEqual(caught.exception.field, f"{'d' * 156}/f") + def test_omission_or_content_mutation_changes_bundle_identity(self) -> None: entries = (("a", 0o644, b"x"), ("b", 0o644, b"y")) original = build_input.canonical_ustar_v1(entries, _TEST_CANONICAL_LIMITS) @@ -506,6 +523,53 @@ def test_cleanup_failure_preserves_input_trigger_and_progress(self) -> None: class SealedBuildTransportContractTests(unittest.TestCase): + def test_successful_probe_keeps_machine_readable_stdout_despite_cli_warning(self) -> None: + policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 + with tempfile.TemporaryDirectory() as temporary: + docker_path = Path(temporary) / "docker" + docker_path.write_bytes(b"fixture") + docker_path.chmod(0o755) + backend = build_transport.NativeDockerBuildBackendV1( + docker_path, + policy, + platform_name="linux", + machine_name="x86_64", + host_user=(501, 20), + ) + image = json.dumps( + [ + { + "Os": "linux", + "Architecture": "amd64", + "RepoDigests": [policy.image_reference], + } + ], + separators=(",", ":"), + ).encode("ascii") + with mock.patch.object( + backend, + "_observe_command", + side_effect=( + build_transport._docker_command_exited_v1( + 0, + b'{"Version":"fixture"}', + b"warning: CLI hint\n", + ), + build_transport._docker_command_exited_v1( + 0, + image, + b"warning: local metadata\n", + ), + ), + ): + capability = backend.probe() + + self.assertIs(type(capability), build_transport.DockerSupportedV1) + self.assertEqual( + capability.daemon_observation.server_stdout, + b'{"Version":"fixture"}', + ) + def test_controller_owns_one_sealed_bundle_for_both_builds(self) -> None: pipeline_source = inspect.getsource(pipeline.ControlledPipelineV1.build) transport_source = inspect.getsource( diff --git a/proof/region/v1/build/input.py b/proof/region/v1/build/input.py index 3ebe0b32..eb4e685a 100644 --- a/proof/region/v1/build/input.py +++ b/proof/region/v1/build/input.py @@ -120,8 +120,12 @@ def _limits_are_valid(value: object) -> bool: return False -def _ustar_path_is_encodable(path: str) -> bool: +def _ustar_path_is_encodable(path: str, *, directory: bool = False) -> bool: encoded = path.encode("ascii") + if directory: + # tarfile writes DIRTYPE without a trailing separator as one with it; + # the USTAR prefix split must validate the exact emitted header name. + encoded += b"/" if len(encoded) <= 100: return True return any( @@ -266,7 +270,7 @@ def canonical_ustar_v1( for length in range(1, len(parts) + 1): directories.add("/".join(parts[:length])) for path in directories: - if not _ustar_path_is_encodable(path): + if not _ustar_path_is_encodable(path, directory=True): _fail(InputReasonV1.INVALID_PATH, path) namespace: dict[str, tuple[str, str]] = {} for kind, values in (("directory", tuple(sorted(directories))), ("file", paths)): diff --git a/proof/region/v1/build/transport.py b/proof/region/v1/build/transport.py index bbeb83a0..c6693239 100644 --- a/proof/region/v1/build/transport.py +++ b/proof/region/v1/build/transport.py @@ -20,7 +20,7 @@ from pathlib import Path from typing import Callable, Protocol, TypeAlias -from . import input +from . import input as _build_input # These versioned observer bounds are not physical constants or a claim that @@ -1084,14 +1084,14 @@ def __new__( cls, attempt: int, capability: DockerSupportedV1, - input_bundle: input.SealedInputV1, + input_bundle: _build_input.SealedInputV1, max_output_bytes: int, ) -> DockerBuildRequestV1: if type(attempt) is not int or attempt not in (1, 2): raise TypeError("attempt must be 1 or 2") if not _docker_supported_is_valid_v1(capability): raise TypeError("capability must be canonical DockerSupportedV1") - if not input.sealed_input_is_intact_v1(input_bundle): + if not _build_input.sealed_input_is_intact_v1(input_bundle): raise TypeError("input_bundle must preserve exact sealed bytes") if ( type(max_output_bytes) is not int @@ -1118,7 +1118,7 @@ def capability(self) -> DockerSupportedV1: return self[1] @property - def input_bundle(self) -> input.SealedInputV1: + def input_bundle(self) -> _build_input.SealedInputV1: return self[2] @property @@ -1140,7 +1140,7 @@ def _docker_build_request_is_valid_v1( return ( tuple(canonical) == tuple(value) and canonical.capability == capability - and input.sealed_input_is_intact_v1(canonical.input_bundle) + and _build_input.sealed_input_is_intact_v1(canonical.input_bundle) ) except Exception: return False @@ -1297,11 +1297,11 @@ def written_sha256(self) -> bytes: def _build_input_progress_v1( - bundle: input.SealedInputV1, + bundle: _build_input.SealedInputV1, written_length: int, written_sha256: bytes, ) -> BuildInputTransferProgressV1: - if not input.sealed_input_is_intact_v1(bundle): + if not _build_input.sealed_input_is_intact_v1(bundle): raise TypeError("build input bytes are not intact") if ( type(written_length) is not int @@ -1324,11 +1324,11 @@ def _build_input_progress_v1( def _input_progress_matches_v1( value: object, - bundle: input.SealedInputV1, + bundle: _build_input.SealedInputV1, ) -> bool: if ( type(value) is not BuildInputTransferProgressV1 - or not input.sealed_input_is_intact_v1(bundle) + or not _build_input.sealed_input_is_intact_v1(bundle) ): return False try: @@ -1400,7 +1400,7 @@ def _input_transfer_is_structurally_valid_v1(value: object) -> bool: def _completed_build_input_transfer_v1( - bundle: input.SealedInputV1, + bundle: _build_input.SealedInputV1, written_length: int, written_sha256: bytes, ) -> BuildInputTransferV1: @@ -1515,13 +1515,13 @@ def _docker_build_exited_v1( def docker_build_exited_is_valid_v1( value: object, - input_value: input.SealedInputV1, + input_value: _build_input.SealedInputV1, max_output_bytes: int, max_stderr_bytes: int, ) -> bool: if ( type(value) is not DockerBuildExitedV1 - or not input.sealed_input_is_intact_v1(input_value) + or not _build_input.sealed_input_is_intact_v1(input_value) or type(max_output_bytes) is not int or max_output_bytes <= 0 or type(max_stderr_bytes) is not int @@ -1829,7 +1829,7 @@ def input_progress(self) -> BuildInputTransferProgressV1 | None: def _canonical_progress_v1( value: object, - input_value: input.SealedInputV1, + input_value: _build_input.SealedInputV1, ) -> BuildInputTransferProgressV1 | None: if value is None: return None @@ -1844,14 +1844,14 @@ def _canonical_progress_v1( def _canonical_process_observation_v1( value: object, - input_value: input.SealedInputV1, + input_value: _build_input.SealedInputV1, max_output_bytes: int, max_stderr_bytes: int, ) -> DockerBuildProcessObservationV1: """Own a backend observation before classification or retention.""" if ( - not input.sealed_input_is_intact_v1(input_value) + not _build_input.sealed_input_is_intact_v1(input_value) or type(max_output_bytes) is not int or max_output_bytes <= 0 or type(max_stderr_bytes) is not int @@ -2112,11 +2112,12 @@ def probe(self) -> DockerCapabilityReportV1: stderr_limit=self._policy.probe_output_limit, timeout_ns=self._policy.probe_timeout_ns, ) + # The versioned capability observes machine-readable stdout; + # successful Docker CLI warnings are diagnostic, not absence proof. if ( type(result) is not _DockerCommandExitedV1 or result.returncode != 0 or not result.stdout - or result.stderr ): return DockerUnsupportedV1( DockerBlockerReasonV1.DOCKER_UNAVAILABLE @@ -2437,7 +2438,7 @@ def _observe_command( stderr_limit: int, timeout_ns: int, lease: _NativeRunLeaseV1 | None = None, - input_bundle: input.SealedInputV1 | None = None, + input_bundle: _build_input.SealedInputV1 | None = None, ) -> _DockerCommandObservationV1: if ( type(command) is not tuple @@ -2471,9 +2472,9 @@ def _observe_command( b"", b"", ) - if input_bundle is not None and type(input_bundle) is not input.SealedInputV1: + if input_bundle is not None and type(input_bundle) is not _build_input.SealedInputV1: raise TypeError("input_bundle must be controller sealed") - if input_bundle is not None and not input.sealed_input_is_intact_v1( + if input_bundle is not None and not _build_input.sealed_input_is_intact_v1( input_bundle ): return DockerBuildObserverFailureV1( @@ -3126,7 +3127,7 @@ class BuildSessionV1(tuple): def __new__( cls, capability: DockerSupportedV1, - input_value: input.SealedInputV1, + input_value: _build_input.SealedInputV1, max_output_bytes: int, *, _token: object, @@ -3134,7 +3135,7 @@ def __new__( if ( _token is not _BUILD_SESSION_TOKEN or not _docker_supported_is_valid_v1(capability) - or not input.sealed_input_is_intact_v1(input_value) + or not _build_input.sealed_input_is_intact_v1(input_value) or type(max_output_bytes) is not int or max_output_bytes <= 0 or max_output_bytes > capability.policy.stdout_limit @@ -3158,11 +3159,8 @@ def capability(self) -> DockerSupportedV1: return self[0] @property - def input_value(self) -> input.SealedInputV1: - value = self[1] - if not input.sealed_input_is_intact_v1(value): - raise RuntimeError("build session lost exact input bytes") - return value + def input_value(self) -> _build_input.SealedInputV1: + return self[1] @property def max_output_bytes(self) -> int: @@ -3171,7 +3169,7 @@ def max_output_bytes(self) -> int: def _build_session_v1( capability: DockerSupportedV1, - input_value: input.SealedInputV1, + input_value: _build_input.SealedInputV1, max_output_bytes: int, ) -> BuildSessionV1: return BuildSessionV1( @@ -3256,7 +3254,7 @@ def capability(self) -> DockerSupportedV1: return self.session.capability @property - def input_value(self) -> input.SealedInputV1: + def input_value(self) -> _build_input.SealedInputV1: return self.session.input_value @property @@ -3517,10 +3515,10 @@ def probe(self) -> DockerCapabilityReportV1: def build( self, capability: DockerSupportedV1, - input_value: input.SealedInputV1, + input_value: _build_input.SealedInputV1, max_output_bytes: int, *, - input_admission: Callable[[input.SealedInputV1], bool], + input_admission: Callable[[_build_input.SealedInputV1], bool], output_admission: Callable[[bytes], bool], ) -> BuildTransportResultV1: if not self._in_owner_process_v1(): @@ -3533,7 +3531,7 @@ def build( or max_output_bytes > capability.policy.stdout_limit or not callable(input_admission) or not callable(output_admission) - or not input.sealed_input_is_intact_v1(input_value) + or not _build_input.sealed_input_is_intact_v1(input_value) ): return BuildRejectedV1(1, BuildFailureReasonV1.CONTRACT_VIOLATION) if not self._in_owner_process_v1(): @@ -3562,7 +3560,7 @@ def build( ): return BuildRejectedV1(1, BuildFailureReasonV1.CONTRACT_VIOLATION) self._consumed = True - if not input.sealed_input_is_intact_v1(input_value): + if not _build_input.sealed_input_is_intact_v1(input_value): return BuildRejectedV1(1, BuildFailureReasonV1.CONTRACT_VIOLATION) try: session = _build_session_v1( @@ -3576,7 +3574,7 @@ def build( for attempt in (1, 2): if ( not self._in_owner_process_v1() - or not input.sealed_input_is_intact_v1(input_value) + or not _build_input.sealed_input_is_intact_v1(input_value) or not self._admitted(input_admission, input_value) or not self._in_owner_process_v1() ): diff --git a/proof/region/v1/tests/test_build.py b/proof/region/v1/tests/test_build.py index d590934b..92d24669 100644 --- a/proof/region/v1/tests/test_build.py +++ b/proof/region/v1/tests/test_build.py @@ -21,7 +21,8 @@ PROOF = Path(__file__).resolve().parents[1] ARB = PROOF / "arb" ARB_TESTS = ARB / "tests" -sys.path[:0] = (str(PROOF), str(ARB), str(ARB_TESTS)) +REPO = PROOF.parents[2] +sys.path[:0] = (str(REPO), str(PROOF), str(ARB), str(ARB_TESTS)) import pipeline # noqa: E402 from proof.region.v1.arb.tests import gate as arb_gate # noqa: E402 @@ -37,10 +38,10 @@ # tests deliberately change this inventory and must update both values from the # gate's independent enumeration in the same slice. ARB_INVENTORY_SHA256_V1 = ( - "4853e06c6e8c1864bc65e0b4c0cd9cdbe0881e0d5907daecb6c8a9fea42f3643" + "e93060f8fa2ff5035bcc394f92dccc5f7f8baf7f9e019fc13cda933295393dce" ) ARB_ORDER_SHA256_V1 = ( - "82b8e00867bc0bed7bd4020f8d9b9531cd195f7c712ddff9a4d73ef7fc0484d5" + "78712585ffac242f31c3a385ab98c047a3501df1037b5830d5428ec9f39bf9d6" ) MOVED_INPUT_SURFACE_V1 = ( @@ -103,7 +104,9 @@ "provenance", ) -FORBIDDEN_TRANSPORT_IMPORTS_V1 = FORBIDDEN_INPUT_IMPORTS_V1 + ("provenance",) +# Both shared leaves must remain unaware of engine semantics; separate names +# keep the two contracts legible without making their import policy diverge. +FORBIDDEN_TRANSPORT_IMPORTS_V1 = FORBIDDEN_INPUT_IMPORTS_V1 def _imported_modules(source: str) -> tuple[str, ...]: @@ -268,8 +271,8 @@ def test_existing_arb_suite_keeps_exact_count_order_and_inventory(self) -> None: identifier.encode("utf-8") + b"\n" for identifier in identifiers ) - self.assertEqual(len(identifiers), 166) - self.assertEqual(len(set(identifiers)), 166) + self.assertEqual(len(identifiers), 169) + self.assertEqual(len(set(identifiers)), 169) self.assertEqual( arb_gate.test_inventory_sha256_v1(arb_gate.full_suite_v1()), ARB_INVENTORY_SHA256_V1, @@ -363,7 +366,8 @@ def test_build_namespace_has_two_focused_shared_leaves(self) -> None: ) self.assertFalse((ARB / "build").exists()) self.assertFalse((PROOF / "mpfi/build").exists()) - self.assertIs(transport.input, build_input) + self.assertFalse(hasattr(transport, "input")) + self.assertFalse(hasattr(transport, "build_input")) self.assertFalse(hasattr(build_input, "normalized_source_entries_v1")) for name in MOVED_INPUT_SURFACE_V1: with self.subTest(name=name): @@ -491,6 +495,22 @@ def test_forged_source_authorities_fail_in_the_arb_taxonomy(self) -> None: class SharedBuildTransportTargetTests(unittest.TestCase): + def test_session_property_is_pure_while_boundary_validator_rejects_forgery(self) -> None: + transport = importlib.import_module("build.transport") + build_input = importlib.import_module("build.input") + forged_input = tuple.__new__(build_input.SealedInputV1, ()) + session = tuple.__new__( + transport.BuildSessionV1, + ( + _docker_capability_fixture(pipeline.ARB_BUILD_TRANSPORT_POLICY_V1), + forged_input, + 64, + ), + ) + + self.assertIs(session.input_value, forged_input) + self.assertFalse(transport._build_session_is_valid_v1(session)) + def test_overlapping_probe_is_rejected_without_a_second_backend_probe(self) -> None: transport = importlib.import_module("build.transport") policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 diff --git a/proof/region/v1/tests/test_build_identity.py b/proof/region/v1/tests/test_build_identity.py index 4c7ce861..19032312 100644 --- a/proof/region/v1/tests/test_build_identity.py +++ b/proof/region/v1/tests/test_build_identity.py @@ -163,6 +163,8 @@ def _slot(value: str) -> tuple[str, str]: _NATIVE_PROCESS_STDERR_V1 = "pipe" +# This literal oracle intentionally does not call production encoders: changing +# a production preimage silently must turn a test failure, not rewrite its proof. def _blob(value: bytes) -> bytes: return len(value).to_bytes(8, "big") + value @@ -181,7 +183,8 @@ def _policy_coordinates(policy: object) -> dict[str, object]: def _policy_chunks(coordinates: dict[str, object]) -> tuple[bytes, ...]: tmpfs_specs = coordinates["tmpfs_specs"] user_mode = coordinates["user_mode"] - assert type(tmpfs_specs) is tuple + if type(tmpfs_specs) is not tuple: + raise TypeError("tmpfs_specs must be an exact tuple") return ( coordinates["image_reference"].encode("utf-8"), coordinates["platform"].encode("utf-8"), @@ -358,6 +361,13 @@ def _assert_deeply_immutable( object.__setattr__(value, "foreign", object()) +class _AlternateUserMode: + """Test-only value with the encoder surface of the closed production enum.""" + + def __init__(self, value: str) -> None: + self.value = value + + class BuildIdentitySurfaceTests(unittest.TestCase): def test_identity_surface_is_exact_and_has_no_legacy_report_aliases(self) -> None: for name in ( @@ -482,12 +492,12 @@ def test_policy_identity_binds_all_twelve_coordinates(self) -> None: identity, ) + def test_literal_oracle_rejects_non_tuple_without_asserts(self) -> None: + coordinates = _policy_coordinates(_policy()) + coordinates["tmpfs_specs"] = object() -class _AlternateUserMode: - """Test-only value with the encoder surface of the closed production enum.""" - - def __init__(self, value: str) -> None: - self.value = value + with self.assertRaises(TypeError): + _policy_chunks(coordinates) class BuildCapabilityIdentityTests(unittest.TestCase): From 660022368045aa853a884dd057b33e214756a60b Mon Sep 17 00:00:00 2001 From: Daniel from Labpics <63733699+lemone112@users.noreply.github.com> Date: Sat, 1 Aug 2026 09:57:09 +0300 Subject: [PATCH 04/20] Proof: preserve stream ownership during cleanup --- proof/region/v1/arb/tests/test_pipeline.py | 1 + proof/region/v1/arb/tests/test_transport.py | 4 +- proof/region/v1/build/transport.py | 32 ++- proof/region/v1/tests/test_build.py | 228 +++++++++++++++++--- 4 files changed, 223 insertions(+), 42 deletions(-) diff --git a/proof/region/v1/arb/tests/test_pipeline.py b/proof/region/v1/arb/tests/test_pipeline.py index 6bf07ea5..afe828d1 100644 --- a/proof/region/v1/arb/tests/test_pipeline.py +++ b/proof/region/v1/arb/tests/test_pipeline.py @@ -1298,6 +1298,7 @@ def test_unverified_container_removal_is_typed_cleanup_failure(self) -> None: docker_path=Path("/bin/sh"), ) lease = backend._next_run_lease_v1(capability) + self.addCleanup(backend._release_run_lease_v1, lease) with mock.patch.object( backend, "_cleanup_container", diff --git a/proof/region/v1/arb/tests/test_transport.py b/proof/region/v1/arb/tests/test_transport.py index b6c941fa..be598901 100644 --- a/proof/region/v1/arb/tests/test_transport.py +++ b/proof/region/v1/arb/tests/test_transport.py @@ -329,7 +329,7 @@ def test_transport_authorities_cannot_be_directly_forged(self) -> None: report = _docker_capability() self.assertFalse(hasattr(report, "__dict__")) with self.assertRaises((AttributeError, TypeError)): - object.__setattr__(report, "platform", "foreign") + object.__setattr__(report, "host_user", (0, 0)) forged_policy = tuple.__new__(build_transport.DockerBuildPolicyV1, ()) with self.assertRaises(TypeError): build_transport.ControlledBuildTransportV1( @@ -608,6 +608,8 @@ def test_docker_request_carries_only_semantic_build_coordinates(self) -> None: ) self.assertFalse(hasattr(request, "cid_file")) self.assertFalse(hasattr(request, "container_name")) + # The request has no positional slot for adapter-owned host cleanup + # authority; extra values cannot smuggle a CID path or container name. with self.assertRaises(TypeError): build_transport.DockerBuildRequestV1( 1, diff --git a/proof/region/v1/build/transport.py b/proof/region/v1/build/transport.py index c6693239..7fbcee1c 100644 --- a/proof/region/v1/build/transport.py +++ b/proof/region/v1/build/transport.py @@ -2661,7 +2661,6 @@ def _observe_command( if process.stdin is not None: close_failed, close_interrupt = self._close_owned_stream( process.stdin, - input_descriptor, ) if close_failed: observer_failed = True @@ -2714,15 +2713,11 @@ def _observe_command( observed_stop = "Docker CLI process termination was interrupted" fallback_stop = self._force_reap_after_interruption_v1(process) stop_detail = stop_detail or observed_stop or fallback_stop - for stream, descriptor in ( - (process.stdout, stdout_descriptor), - (process.stderr, stderr_descriptor), - ): + for stream in (process.stdout, process.stderr): if stream is None: continue close_failed, close_interrupt = self._close_owned_stream( stream, - descriptor, ) if close_failed: observer_failed = True @@ -2853,11 +2848,11 @@ def _observe_command( @staticmethod def _close_owned_stream( stream: object, - descriptor: int | None, ) -> tuple[bool, BaseException | None]: - """Release a stream even when one release operation is interrupted.""" + """Release through the stream owner without aliasing its descriptor.""" close_failed = False + close_raised = False retained_base_exception: BaseException | None = None try: closed = stream.closed is True @@ -2873,16 +2868,31 @@ def _close_owned_stream( stream.close() except Exception: close_failed = True + close_raised = True except BaseException as error: close_failed = True + close_raised = True retained_base_exception = retained_base_exception or error - if close_failed and type(descriptor) is int and descriptor >= 0: + if close_raised: try: - os.close(descriptor) + still_open = stream.closed is False except Exception: - pass + still_open = False except BaseException as error: retained_base_exception = retained_base_exception or error + still_open = False + if still_open: + # close() may fail before it releases the resource, but a + # saved FD can already name another resource. Retrying the + # same owner is the only bounded release attempt that keeps + # ownership unambiguous. + try: + stream.close() + except Exception: + close_failed = True + except BaseException as error: + close_failed = True + retained_base_exception = retained_base_exception or error return close_failed, retained_base_exception def _clock(self) -> int: diff --git a/proof/region/v1/tests/test_build.py b/proof/region/v1/tests/test_build.py index 92d24669..2932cf88 100644 --- a/proof/region/v1/tests/test_build.py +++ b/proof/region/v1/tests/test_build.py @@ -34,15 +34,16 @@ from test_receipt import _execute # noqa: E402 -# These inventory goldens describe the complete Arb gate. Identity-version -# tests deliberately change this inventory and must update both values from the -# gate's independent enumeration in the same slice. +# These literals are an independent outer oracle for the Arb gate: importing +# its expected hash here would let a coordinated gate edit hide inventory drift. +# A deliberate test-set change updates both values from fresh enumeration. ARB_INVENTORY_SHA256_V1 = ( "e93060f8fa2ff5035bcc394f92dccc5f7f8baf7f9e019fc13cda933295393dce" ) ARB_ORDER_SHA256_V1 = ( "78712585ffac242f31c3a385ab98c047a3501df1037b5830d5428ec9f39bf9d6" ) +ARB_TEST_COUNT_V1 = 169 MOVED_INPUT_SURFACE_V1 = ( "CanonicalInputLimitsV1", @@ -271,8 +272,8 @@ def test_existing_arb_suite_keeps_exact_count_order_and_inventory(self) -> None: identifier.encode("utf-8") + b"\n" for identifier in identifiers ) - self.assertEqual(len(identifiers), 169) - self.assertEqual(len(set(identifiers)), 169) + self.assertEqual(len(identifiers), ARB_TEST_COUNT_V1) + self.assertEqual(len(set(identifiers)), ARB_TEST_COUNT_V1) self.assertEqual( arb_gate.test_inventory_sha256_v1(arb_gate.full_suite_v1()), ARB_INVENTORY_SHA256_V1, @@ -495,6 +496,88 @@ def test_forged_source_authorities_fail_in_the_arb_taxonomy(self) -> None: class SharedBuildTransportTargetTests(unittest.TestCase): + def test_stream_close_fallback_requires_current_stream_ownership(self) -> None: + transport = importlib.import_module("build.transport") + real_close = os.close + + descriptor = os.open(os.devnull, os.O_RDONLY) + + class ClosesThenRaises: + closed = False + replacement: int | None = None + close_calls = 0 + + def close(self) -> None: + self.close_calls += 1 + real_close(descriptor) + self.closed = True + self.replacement = os.open(os.devnull, os.O_RDONLY) + raise OSError("stream close released its descriptor") + + stream = ClosesThenRaises() + try: + with mock.patch.object( + transport.os, + "close", + side_effect=AssertionError("helper closed a numeric descriptor"), + ) as direct_close: + failed, interruption = ( + transport.NativeDockerBuildBackendV1._close_owned_stream( + stream, + ) + ) + + self.assertTrue(failed) + self.assertIsNone(interruption) + self.assertEqual(stream.replacement, descriptor) + self.assertEqual(stream.close_calls, 1) + direct_close.assert_not_called() + os.fstat(descriptor) + finally: + if stream.replacement is not None: + try: + real_close(stream.replacement) + except OSError: + pass + + descriptor = os.open(os.devnull, os.O_RDONLY) + + class RaisesBeforeClose: + closed = False + close_calls = 0 + + def close(self) -> None: + self.close_calls += 1 + if self.close_calls == 1: + raise OSError("stream close kept its descriptor") + real_close(descriptor) + self.closed = True + + stream = RaisesBeforeClose() + try: + with mock.patch.object( + transport.os, + "close", + side_effect=AssertionError("helper closed a numeric descriptor"), + ) as direct_close: + failed, interruption = ( + transport.NativeDockerBuildBackendV1._close_owned_stream( + stream, + ) + ) + + self.assertTrue(failed) + self.assertIsNone(interruption) + self.assertEqual(stream.close_calls, 2) + direct_close.assert_not_called() + with self.assertRaises(OSError): + os.fstat(descriptor) + finally: + try: + real_close(descriptor) + except OSError: + pass + def test_session_property_is_pure_while_boundary_validator_rejects_forgery(self) -> None: transport = importlib.import_module("build.transport") build_input = importlib.import_module("build.input") @@ -1010,7 +1093,7 @@ class StringSubclass(str): with self.assertRaises(TypeError): transport.DockerBuildPolicyV1(**hostile_policy) - def test_stream_close_failure_fallback_closes_fd_and_retains_evidence(self) -> None: + def test_stream_close_failure_retries_owner_and_retains_evidence(self) -> None: transport = importlib.import_module("build.transport") backend = transport.NativeDockerBuildBackendV1( Path("/bin/true"), @@ -1020,9 +1103,7 @@ def test_stream_close_failure_fallback_closes_fd_and_retains_evidence(self) -> N machine_name="x86_64", ) real_popen = subprocess.Popen - real_close = os.close spawned: list[subprocess.Popen[bytes]] = [] - fallback_closed: list[int] = [] wrapped_streams: list[object] = [] class CloseRaises: @@ -1033,14 +1114,16 @@ def __init__(self, wrapped: object) -> None: @property def closed(self) -> bool: - return False + return self.wrapped.closed def fileno(self) -> int: return self.descriptor def close(self) -> None: self.close_calls += 1 - raise OSError("forced close failure") + if self.close_calls == 1: + raise OSError("forced close failure") + self.wrapped.close() def spawn(*args: object, **kwargs: object) -> subprocess.Popen[bytes]: process = real_popen(*args, **kwargs) @@ -1050,10 +1133,6 @@ def spawn(*args: object, **kwargs: object) -> subprocess.Popen[bytes]: process.stdout = wrapped return process - def close(descriptor: int) -> None: - fallback_closed.append(descriptor) - real_close(descriptor) - input_value = _sealed_input() command = ( sys.executable, @@ -1068,7 +1147,7 @@ def close(descriptor: int) -> None: transport.subprocess, "Popen", side_effect=spawn, - ), mock.patch.object(transport.os, "close", side_effect=close): + ): result = backend._observe_command( command, stdout_limit=64, @@ -1085,10 +1164,8 @@ def close(descriptor: int) -> None: if stream is not None and not stream.closed: stream.close() original = wrapped_streams[0].wrapped - try: + if not original.closed: original.close() - except OSError: - pass self.assertIs(type(result), transport.DockerBuildObserverFailureV1) self.assertEqual(result.stdout, b"evidence") @@ -1096,8 +1173,8 @@ def close(descriptor: int) -> None: self.assertIsNotNone(result.input_progress) self.assertEqual(result.input_progress.written_length, input_value.length) self.assertEqual(result.input_progress.written_sha256, input_value.sha256) - self.assertIn(wrapped_streams[0].descriptor, fallback_closed) - self.assertEqual(wrapped_streams[0].close_calls, 1) + self.assertTrue(wrapped_streams[0].wrapped.closed) + self.assertEqual(wrapped_streams[0].close_calls, 2) def test_post_popen_failures_always_close_streams_and_cleanup_once(self) -> None: transport = importlib.import_module("build.transport") @@ -1400,15 +1477,23 @@ def test_post_popen_handler_gap_cannot_bypass_finalizer(self) -> None: observe = backend._observe_command instructions = tuple(dis.Bytecode(observe)) process_store = next( - index - for index, instruction in enumerate(instructions) - if ( - instruction.opname == "STORE_FAST" - and instruction.argval == "process" - and index > 0 - and instructions[index - 1].opname == "CALL_FUNCTION_EX" - ) + ( + index + for index, instruction in enumerate(instructions) + if ( + instruction.opname == "STORE_FAST" + and instruction.argval == "process" + and index > 0 + and instructions[index - 1].opname == "CALL_FUNCTION_EX" + ) + ), + None, ) + if process_store is None: + self.fail( + "CPython bytecode no longer exposes CALL_FUNCTION_EX before " + f"STORE_FAST process (Python {sys.version})" + ) interruption_offset = instructions[process_store + 1].offset real_popen = subprocess.Popen spawned: list[subprocess.Popen[bytes]] = [] @@ -1645,16 +1730,20 @@ class CloseInterrupts: def __init__(self, wrapped: object) -> None: self.wrapped = wrapped self.descriptor = wrapped.fileno() + self.close_calls = 0 @property def closed(self) -> bool: - return False + return self.wrapped.closed def fileno(self) -> int: return self.descriptor def close(self) -> None: - raise KeyboardInterrupt("interrupt during stdout close") + self.close_calls += 1 + if self.close_calls == 1: + raise KeyboardInterrupt("interrupt during stdout close") + self.wrapped.close() def spawn(*args: object, **kwargs: object) -> subprocess.Popen[bytes]: process = real_popen(*args, **kwargs) @@ -1709,6 +1798,85 @@ def cleanup(lease: object, **_kwargs: object) -> None: self.assertTrue(stderr_closed) self.assertTrue(stdout_descriptor_closed) + self.assertEqual(wrapped_stdout[0].close_calls, 2) + self.assertEqual(cleanup_calls, [lease]) + + def test_persistent_stream_close_interrupt_keeps_release_failure_honest(self) -> None: + transport = importlib.import_module("build.transport") + backend = transport.NativeDockerBuildBackendV1( + Path("/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + host_user=(501, 20), + platform_name="linux", + machine_name="x86_64", + ) + real_popen = subprocess.Popen + spawned: list[subprocess.Popen[bytes]] = [] + wrapped_stdout: list[object] = [] + cleanup_calls: list[object] = [] + + class CloseAlwaysInterrupts: + def __init__(self, wrapped: object) -> None: + self.wrapped = wrapped + self.descriptor = wrapped.fileno() + self.close_calls = 0 + + @property + def closed(self) -> bool: + return False + + def fileno(self) -> int: + return self.descriptor + + def close(self) -> None: + self.close_calls += 1 + raise KeyboardInterrupt("persistent stdout close interrupt") + + def spawn(*args: object, **kwargs: object) -> subprocess.Popen[bytes]: + process = real_popen(*args, **kwargs) + spawned.append(process) + wrapper = CloseAlwaysInterrupts(process.stdout) + wrapped_stdout.append(wrapper) + process.stdout = wrapper + return process + + def cleanup(lease: object, **_kwargs: object) -> None: + cleanup_calls.append(lease) + + lease = backend._next_run_lease_v1( + _docker_capability_fixture(pipeline.ARB_BUILD_TRANSPORT_POLICY_V1) + ) + self.addCleanup(backend._release_run_lease_v1, lease) + with mock.patch.object( + transport.subprocess, + "Popen", + side_effect=spawn, + ), mock.patch.object( + backend, + "_cleanup_container", + side_effect=cleanup, + ): + with self.assertRaisesRegex(KeyboardInterrupt, "persistent stdout"): + backend._observe_command( + (sys.executable, "-c", "pass"), + stdout_limit=64, + stderr_limit=64, + timeout_ns=1_000_000_000, + lease=lease, + ) + process = spawned[0] + stderr_closed = process.stderr is not None and process.stderr.closed + os.fstat(wrapped_stdout[0].descriptor) + if process.poll() is None: + process.kill() + process.wait(timeout=5) + for stream in (process.stdin, process.stderr): + if stream is not None and not stream.closed: + stream.close() + wrapped_stdout[0].wrapped.close() + + self.assertTrue(stderr_closed) + self.assertEqual(wrapped_stdout[0].close_calls, 2) self.assertEqual(cleanup_calls, [lease]) def test_process_and_container_cleanup_failures_are_both_retained_in_order(self) -> None: From 1b3ffbf94d5eb537711b33d572f0a95c3b29cd6b Mon Sep 17 00:00:00 2001 From: Daniel from Labpics <63733699+lemone112@users.noreply.github.com> Date: Sat, 1 Aug 2026 10:02:14 +0300 Subject: [PATCH 05/20] Test: always reap hostile observer fixture --- proof/region/v1/tests/test_build.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/proof/region/v1/tests/test_build.py b/proof/region/v1/tests/test_build.py index 2932cf88..4e3c7488 100644 --- a/proof/region/v1/tests/test_build.py +++ b/proof/region/v1/tests/test_build.py @@ -1832,12 +1832,29 @@ def close(self) -> None: self.close_calls += 1 raise KeyboardInterrupt("persistent stdout close interrupt") + def cleanup_spawned( + process: subprocess.Popen[bytes], + wrapper: CloseAlwaysInterrupts, + ) -> None: + if process.poll() is None: + try: + process.kill() + except ProcessLookupError: + pass + process.wait(timeout=5) + for stream in (process.stdin, process.stderr): + if stream is not None and not stream.closed: + stream.close() + if not wrapper.wrapped.closed: + wrapper.wrapped.close() + def spawn(*args: object, **kwargs: object) -> subprocess.Popen[bytes]: process = real_popen(*args, **kwargs) spawned.append(process) wrapper = CloseAlwaysInterrupts(process.stdout) wrapped_stdout.append(wrapper) process.stdout = wrapper + self.addCleanup(cleanup_spawned, process, wrapper) return process def cleanup(lease: object, **_kwargs: object) -> None: @@ -1867,13 +1884,6 @@ def cleanup(lease: object, **_kwargs: object) -> None: process = spawned[0] stderr_closed = process.stderr is not None and process.stderr.closed os.fstat(wrapped_stdout[0].descriptor) - if process.poll() is None: - process.kill() - process.wait(timeout=5) - for stream in (process.stdin, process.stderr): - if stream is not None and not stream.closed: - stream.close() - wrapped_stdout[0].wrapped.close() self.assertTrue(stderr_closed) self.assertEqual(wrapped_stdout[0].close_calls, 2) From 08e02434c23bdc3946e356cfe4c121b035b642e9 Mon Sep 17 00:00:00 2001 From: Daniel from Labpics <63733699+lemone112@users.noreply.github.com> Date: Sat, 1 Aug 2026 10:04:38 +0300 Subject: [PATCH 06/20] Test: register observer cleanup before fixture setup --- proof/region/v1/tests/test_build.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/proof/region/v1/tests/test_build.py b/proof/region/v1/tests/test_build.py index 4e3c7488..5348d0e7 100644 --- a/proof/region/v1/tests/test_build.py +++ b/proof/region/v1/tests/test_build.py @@ -1834,7 +1834,7 @@ def close(self) -> None: def cleanup_spawned( process: subprocess.Popen[bytes], - wrapper: CloseAlwaysInterrupts, + original_stdout: object, ) -> None: if process.poll() is None: try: @@ -1842,19 +1842,19 @@ def cleanup_spawned( except ProcessLookupError: pass process.wait(timeout=5) - for stream in (process.stdin, process.stderr): + for stream in (process.stdin, process.stderr, original_stdout): if stream is not None and not stream.closed: stream.close() - if not wrapper.wrapped.closed: - wrapper.wrapped.close() def spawn(*args: object, **kwargs: object) -> subprocess.Popen[bytes]: process = real_popen(*args, **kwargs) + # Fixture setup may fail after Popen; ownership starts with its + # original stream, before the hostile wrapper exists. + self.addCleanup(cleanup_spawned, process, process.stdout) spawned.append(process) wrapper = CloseAlwaysInterrupts(process.stdout) wrapped_stdout.append(wrapper) process.stdout = wrapper - self.addCleanup(cleanup_spawned, process, wrapper) return process def cleanup(lease: object, **_kwargs: object) -> None: From c010af9b117ea56efe08d84e8c662d12ce5615fb Mon Sep 17 00:00:00 2001 From: Daniel from Labpics <63733699+lemone112@users.noreply.github.com> Date: Sat, 1 Aug 2026 10:59:44 +0300 Subject: [PATCH 07/20] =?UTF-8?q?Build:=20=D1=82=D0=B8=D0=BF=D0=B8=D0=B7?= =?UTF-8?q?=D0=B8=D1=80=D0=BE=D0=B2=D0=B0=D1=82=D1=8C=20=D0=BD=D0=B5=D0=B2?= =?UTF-8?q?=D0=B0=D0=BB=D0=B8=D0=B4=D0=BD=D1=8B=D0=B9=20public=20input?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- proof/region/v1/arb/tests/gate.py | 2 +- proof/region/v1/arb/tests/test_transport.py | 94 +++++++++++++++++++++ proof/region/v1/build/input.py | 40 ++++++--- proof/region/v1/tests/test_build.py | 6 +- 4 files changed, 128 insertions(+), 14 deletions(-) diff --git a/proof/region/v1/arb/tests/gate.py b/proof/region/v1/arb/tests/gate.py index b518606b..06e58f28 100644 --- a/proof/region/v1/arb/tests/gate.py +++ b/proof/region/v1/arb/tests/gate.py @@ -15,7 +15,7 @@ REPO = Path(__file__).resolve().parents[5] sys.path.insert(0, str(REPO)) EXPECTED_TEST_INVENTORY_SHA256 = ( - "e93060f8fa2ff5035bcc394f92dccc5f7f8baf7f9e019fc13cda933295393dce" + "bec51ba4a7bcbb0bd332c611fce4ac9d0ddb028eec569401f8ae0123d3bfe611" ) _EVALUATOR_REASON = "set LABCOLORS_ARB_EVALUATOR to the controlled C17 binary" EXPECTED_SKIPS = frozenset( diff --git a/proof/region/v1/arb/tests/test_transport.py b/proof/region/v1/arb/tests/test_transport.py index be598901..d7eddda6 100644 --- a/proof/region/v1/arb/tests/test_transport.py +++ b/proof/region/v1/arb/tests/test_transport.py @@ -3,6 +3,7 @@ from __future__ import annotations +from collections.abc import Callable import hashlib import io import inspect @@ -75,6 +76,99 @@ def _observe( class CanonicalBuildBundleTests(unittest.TestCase): + def test_public_input_constructors_use_typed_value_errors(self) -> None: + def reject( + constructor: Callable[[], object], + reason: build_input.InputReasonV1, + field: str, + ) -> None: + with self.assertRaises(build_input.InputErrorV1) as caught: + constructor() + self.assertEqual(caught.exception.reason, reason) + self.assertEqual(caught.exception.field, field) + + def limits( + max_members: object = 1, + max_file_bytes: object = 1, + max_payload_bytes: object = 1, + max_encoded_bytes: object = None, + ) -> object: + return build_input.CanonicalInputLimitsV1( + max_members, + max_file_bytes, + max_payload_bytes, + max_encoded_bytes, + ) + + def seal( + binding_identity: object = _digest("binding"), + contents: object = b"x", + ) -> object: + return build_input.seal_input_v1(binding_identity, contents) + + limit_fields = ( + "max_members", + "max_file_bytes", + "max_payload_bytes", + "max_encoded_bytes", + ) + self.assertIs( + type(build_input.CanonicalInputLimitsV1(1, 1, 1, None)), + build_input.CanonicalInputLimitsV1, + ) + for field, value in ( + ("max_members", True), + ("max_file_bytes", 1.0), + ("max_payload_bytes", object()), + ("max_encoded_bytes", b"1"), + ): + with self.subTest(kind="wrong_type", field=field): + reject( + lambda field=field, value=value: limits(**{field: value}), + build_input.InputReasonV1.WRONG_TYPE, + field, + ) + for field in limit_fields: + for value in (0, 1 << 64): + with self.subTest(kind="invalid_limit", field=field, value=value): + reject( + lambda field=field, value=value: limits(**{field: value}), + build_input.InputReasonV1.INVALID_VALUE, + field, + ) + reject( + lambda: limits(max_payload_bytes=(1 << 64) - 1), + build_input.InputReasonV1.INVALID_VALUE, + "max_encoded_bytes", + ) + + for field, constructor in ( + ("binding_identity", lambda: seal(bytearray(_digest("binding")))), + ("contents", lambda: seal(contents=bytearray(b"x"))), + ): + with self.subTest(kind="wrong_type", field=field): + reject(constructor, build_input.InputReasonV1.WRONG_TYPE, field) + for field, constructor in ( + ("binding_identity", lambda: seal(bytes(32))), + ("binding_identity", lambda: seal(b"x" * 31)), + ("contents", lambda: seal(contents=b"")), + ): + with self.subTest(kind="invalid_value", field=field): + reject(constructor, build_input.InputReasonV1.INVALID_VALUE, field) + + for constructor in ( + lambda: build_input.CanonicalInputLimitsV1(1, 1), + lambda: build_input.seal_input_v1(_digest("binding")), + lambda: build_input.SealedInputV1( + _digest("binding"), + b"x", + _token=object(), + ), + ): + with self.subTest(kind="private_or_call_shape"): + with self.assertRaises(TypeError): + constructor() + def test_bundle_is_reproducible_normalized_ustar_with_no_host_authority(self) -> None: request = _request() first = pipeline._seal_build_input_bundle_v1(request) diff --git a/proof/region/v1/build/input.py b/proof/region/v1/build/input.py index eb4e685a..f09cf886 100644 --- a/proof/region/v1/build/input.py +++ b/proof/region/v1/build/input.py @@ -23,6 +23,7 @@ def _valid_digest(value: object) -> bool: class InputReasonV1(StrEnum): WRONG_TYPE = "wrong_type" + INVALID_VALUE = "invalid_value" INVALID_PATH = "invalid_path" INVALID_MODE = "invalid_mode" NONCANONICAL_SET = "noncanonical_set" @@ -42,6 +43,14 @@ def _fail(reason: InputReasonV1, field_name: str) -> NoReturn: raise InputErrorV1(reason, field_name) +def _positive_u64(value: object, field_name: str) -> int: + if type(value) is not int: + _fail(InputReasonV1.WRONG_TYPE, field_name) + if value <= 0 or value >= 1 << 64: + _fail(InputReasonV1.INVALID_VALUE, field_name) + return value + + def _logical_path(value: object) -> str: if type(value) is not str or not value or value.startswith("/") or "\\" in value: _fail(InputReasonV1.INVALID_PATH, "path") @@ -70,12 +79,9 @@ def __new__( max_payload_bytes: int, max_encoded_bytes: int | None = None, ) -> CanonicalInputLimitsV1: - base_values = (max_members, max_file_bytes, max_payload_bytes) - if any( - type(value) is not int or value <= 0 or value >= 1 << 64 - for value in base_values - ): - raise TypeError("canonical input limits must be positive u64 values") + max_members = _positive_u64(max_members, "max_members") + max_file_bytes = _positive_u64(max_file_bytes, "max_file_bytes") + max_payload_bytes = _positive_u64(max_payload_bytes, "max_payload_bytes") if max_encoded_bytes is None: # USTAR adds one header block per member, at most one partial data # block per member, two EOF blocks, then pads to one record. This @@ -89,10 +95,16 @@ def __new__( maximum_unpadded, _USTAR_RECORD_BYTES, ) - values = (*base_values, max_encoded_bytes) - if any(type(value) is not int or value <= 0 or value >= 1 << 64 for value in values): - raise TypeError("canonical input limits must be positive u64 values") - return tuple.__new__(cls, values) + max_encoded_bytes = _positive_u64(max_encoded_bytes, "max_encoded_bytes") + return tuple.__new__( + cls, + ( + max_members, + max_file_bytes, + max_payload_bytes, + max_encoded_bytes, + ), + ) @property def max_members(self) -> int: @@ -203,6 +215,14 @@ def contents(self) -> bytes: def seal_input_v1(binding_identity: bytes, contents: bytes) -> SealedInputV1: """Seal exact bytes while treating their semantic binding as opaque.""" + if type(binding_identity) is not bytes: + _fail(InputReasonV1.WRONG_TYPE, "binding_identity") + if not _valid_digest(binding_identity): + _fail(InputReasonV1.INVALID_VALUE, "binding_identity") + if type(contents) is not bytes: + _fail(InputReasonV1.WRONG_TYPE, "contents") + if not contents: + _fail(InputReasonV1.INVALID_VALUE, "contents") return SealedInputV1( binding_identity, contents, diff --git a/proof/region/v1/tests/test_build.py b/proof/region/v1/tests/test_build.py index 5348d0e7..b3c6e6bd 100644 --- a/proof/region/v1/tests/test_build.py +++ b/proof/region/v1/tests/test_build.py @@ -38,12 +38,12 @@ # its expected hash here would let a coordinated gate edit hide inventory drift. # A deliberate test-set change updates both values from fresh enumeration. ARB_INVENTORY_SHA256_V1 = ( - "e93060f8fa2ff5035bcc394f92dccc5f7f8baf7f9e019fc13cda933295393dce" + "bec51ba4a7bcbb0bd332c611fce4ac9d0ddb028eec569401f8ae0123d3bfe611" ) ARB_ORDER_SHA256_V1 = ( - "78712585ffac242f31c3a385ab98c047a3501df1037b5830d5428ec9f39bf9d6" + "bf04abd1245e57afc751e9803ad981df05af5509e0d32fe0e39dc76dc2421790" ) -ARB_TEST_COUNT_V1 = 169 +ARB_TEST_COUNT_V1 = 170 MOVED_INPUT_SURFACE_V1 = ( "CanonicalInputLimitsV1", From 9e10df299703fe2b8421746698a08d31c7bf75f8 Mon Sep 17 00:00:00 2001 From: Daniel from Labpics <63733699+lemone112@users.noreply.github.com> Date: Sat, 1 Aug 2026 12:49:12 +0300 Subject: [PATCH 08/20] =?UTF-8?q?Proof:=20=D1=81=D0=BE=D1=85=D1=80=D0=B0?= =?UTF-8?q?=D0=BD=D0=B8=D1=82=D1=8C=20Arb=20binding=20=D0=B8=20total=20pro?= =?UTF-8?q?be?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- proof/region/v1/arb/tests/gate.py | 2 +- .../v1/arb/tests/test_build_identity_v2.py | 60 +++++++++++++++++++ proof/region/v1/arb/tests/test_transport.py | 54 +++++++++++++++++ proof/region/v1/build/transport.py | 11 ++-- proof/region/v1/tests/test_build.py | 6 +- 5 files changed, 125 insertions(+), 8 deletions(-) diff --git a/proof/region/v1/arb/tests/gate.py b/proof/region/v1/arb/tests/gate.py index 06e58f28..f04c3327 100644 --- a/proof/region/v1/arb/tests/gate.py +++ b/proof/region/v1/arb/tests/gate.py @@ -15,7 +15,7 @@ REPO = Path(__file__).resolve().parents[5] sys.path.insert(0, str(REPO)) EXPECTED_TEST_INVENTORY_SHA256 = ( - "bec51ba4a7bcbb0bd332c611fce4ac9d0ddb028eec569401f8ae0123d3bfe611" + "8a6c293af9283193dffe24ba45b864e343ad5c7d64fed2519fdd1d87c8b28d34" ) _EVALUATOR_REASON = "set LABCOLORS_ARB_EVALUATOR to the controlled C17 binary" EXPECTED_SKIPS = frozenset( diff --git a/proof/region/v1/arb/tests/test_build_identity_v2.py b/proof/region/v1/arb/tests/test_build_identity_v2.py index 1341eaee..2569fbdd 100644 --- a/proof/region/v1/arb/tests/test_build_identity_v2.py +++ b/proof/region/v1/arb/tests/test_build_identity_v2.py @@ -112,6 +112,31 @@ def _policy_mutants( return tuple(mutants) +def _arb_input_binding_oracle_v1( + source_identity: bytes, + build_input_identity: bytes, + contents: bytes, + bootstrap: str, +) -> bytes: + """Independent frozen formula for the unchanged Arb input binding.""" + + chunks = ( + source_identity, + build_input_identity, + len(contents).to_bytes(8, "big"), + hashlib.sha256(contents).digest(), + hashlib.sha256(bootstrap.encode("utf-8")).digest(), + ) + payload = b"".join( + len(chunk).to_bytes(8, "big") + chunk for chunk in chunks + ) + return hashlib.sha256( + b"labcolors.proof-region.arb-build-input-bundle.v1\0" + + len(payload).to_bytes(8, "big") + + payload + ).digest() + + def _capability( docker_path: Path, policy: build_transport.DockerBuildPolicyV1, @@ -247,6 +272,41 @@ def test_pipeline_policy_consumes_both_owned_transport_identities(self) -> None: with self.assertRaises(TypeError): pipeline.pipeline_policy_identity_v2(trust, surrogate) + def test_generic_sealing_preserves_the_frozen_arb_binding_formula(self) -> None: + """Moving byte storage cannot silently change an unchanged protocol ID.""" + + request = _request() + baseline_policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 + baseline = pipeline._seal_build_input_bundle_v1(request) + expected = _arb_input_binding_oracle_v1( + request.admitted_sources.identity, + request.build_sources.build_input_identity, + baseline.contents, + baseline_policy.bootstrap, + ) + self.assertEqual(baseline.binding_identity, expected) + + changed_policy = _policy_with( + baseline_policy, + bootstrap=baseline_policy.bootstrap + "\n:", + ) + with mock.patch.object( + pipeline, + "ARB_BUILD_TRANSPORT_POLICY_V1", + changed_policy, + ): + changed = pipeline._seal_build_input_bundle_v1(request) + self.assertTrue(pipeline.arb_input_is_bound_v1(request, changed)) + expected_changed = _arb_input_binding_oracle_v1( + request.admitted_sources.identity, + request.build_sources.build_input_identity, + changed.contents, + changed_policy.bootstrap, + ) + self.assertEqual(changed.contents, baseline.contents) + self.assertEqual(changed.binding_identity, expected_changed) + self.assertNotEqual(changed.binding_identity, baseline.binding_identity) + def test_every_admitted_policy_mutation_changes_transport_and_pipeline_identity(self) -> None: trust = pipeline.HostTrustBoundaryV1.UNSEALED_LINUX_X64_DOCKER_HOST policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 diff --git a/proof/region/v1/arb/tests/test_transport.py b/proof/region/v1/arb/tests/test_transport.py index d7eddda6..fa44ea5b 100644 --- a/proof/region/v1/arb/tests/test_transport.py +++ b/proof/region/v1/arb/tests/test_transport.py @@ -3,6 +3,7 @@ from __future__ import annotations +import ast from collections.abc import Callable import hashlib import io @@ -13,6 +14,7 @@ import sys import tarfile import tempfile +import textwrap import unittest from pathlib import Path from unittest import mock @@ -76,6 +78,58 @@ def _observe( class CanonicalBuildBundleTests(unittest.TestCase): + def test_public_probe_starts_with_a_typed_terminal_outcome(self) -> None: + """A backend cannot leave the public probe in a non-report state.""" + + source = inspect.getsource(build_transport.ControlledBuildTransportV1.probe) + tree = ast.parse(textwrap.dedent(source)) + function = tree.body[0] + if not isinstance(function, ast.FunctionDef): + self.fail("public probe source must remain one function definition") + typed_initializers = [ + node + for node in function.body + if isinstance(node, ast.AnnAssign) + and isinstance(node.target, ast.Name) + and node.target.id == "outcome" + and isinstance(node.annotation, ast.Name) + and node.annotation.id == "DockerCapabilityReportV1" + and isinstance(node.value, ast.Call) + and isinstance(node.value.func, ast.Name) + and node.value.func.id == "DockerUnsupportedV1" + ] + self.assertEqual(len(typed_initializers), 1) + self.assertNotIn('raise RuntimeError("probe outcome was not produced")', source) + + def test_probe_releases_its_transient_lease_after_backend_failure(self) -> None: + report = _docker_capability(pipeline.ARB_BUILD_TRANSPORT_POLICY_V1) + + class FlakyProbeBackend: + def __init__(self) -> None: + self.calls = 0 + + def probe(self) -> object: + self.calls += 1 + if self.calls == 1: + raise ValueError("forced backend failure") + return report + + backend = FlakyProbeBackend() + controller = build_transport.ControlledBuildTransportV1( + policy=pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + backend=backend, + ) + first = controller.probe() + second = controller.probe() + + self.assertIs(type(first), build_transport.DockerUnsupportedV1) + self.assertEqual( + first.reason, + build_transport.DockerBlockerReasonV1.BACKEND_CONTRACT, + ) + self.assertIs(second, report) + self.assertEqual(backend.calls, 2) + def test_public_input_constructors_use_typed_value_errors(self) -> None: def reject( constructor: Callable[[], object], diff --git a/proof/region/v1/build/transport.py b/proof/region/v1/build/transport.py index 7fbcee1c..153ddb71 100644 --- a/proof/region/v1/build/transport.py +++ b/proof/region/v1/build/transport.py @@ -3477,11 +3477,16 @@ def probe(self) -> DockerCapabilityReportV1: "build transport capability is one-shot", ) self._probe_in_flight = True - outcome: DockerCapabilityReportV1 | None = None + # Start at a typed rejection before foreign IO so every ordinary exit + # has a public report. A BaseException still propagates unchanged. + outcome: DockerCapabilityReportV1 = DockerUnsupportedV1( + DockerBlockerReasonV1.BACKEND_CONTRACT, + "Docker capability probe produced no canonical report", + ) try: report = self._backend.probe() except Exception: - outcome: DockerCapabilityReportV1 = DockerUnsupportedV1( + outcome = DockerUnsupportedV1( DockerBlockerReasonV1.BACKEND_CONTRACT, "Docker capability probe raised", ) @@ -3518,8 +3523,6 @@ def probe(self) -> DockerCapabilityReportV1: self._probe_in_flight = False if type(outcome) is DockerSupportedV1: self._probed_capability = outcome - if outcome is None: - raise RuntimeError("probe outcome was not produced") return outcome def build( diff --git a/proof/region/v1/tests/test_build.py b/proof/region/v1/tests/test_build.py index b3c6e6bd..ea1876bb 100644 --- a/proof/region/v1/tests/test_build.py +++ b/proof/region/v1/tests/test_build.py @@ -38,12 +38,12 @@ # its expected hash here would let a coordinated gate edit hide inventory drift. # A deliberate test-set change updates both values from fresh enumeration. ARB_INVENTORY_SHA256_V1 = ( - "bec51ba4a7bcbb0bd332c611fce4ac9d0ddb028eec569401f8ae0123d3bfe611" + "8a6c293af9283193dffe24ba45b864e343ad5c7d64fed2519fdd1d87c8b28d34" ) ARB_ORDER_SHA256_V1 = ( - "bf04abd1245e57afc751e9803ad981df05af5509e0d32fe0e39dc76dc2421790" + "b3c75d673b4878623c9383daffe043ed4000b208632b9e33a7e944ed13568098" ) -ARB_TEST_COUNT_V1 = 170 +ARB_TEST_COUNT_V1 = 173 MOVED_INPUT_SURFACE_V1 = ( "CanonicalInputLimitsV1", From f4654134707bb4a26a245714b4267fd2ea11734e Mon Sep 17 00:00:00 2001 From: Daniel from Labpics <63733699+lemone112@users.noreply.github.com> Date: Sat, 1 Aug 2026 12:51:11 +0300 Subject: [PATCH 09/20] Proof: totalize native cleanup failure --- proof/region/v1/arb/tests/gate.py | 2 +- proof/region/v1/arb/tests/test_transport.py | 42 +++++++++++++++++++++ proof/region/v1/build/transport.py | 10 +++-- proof/region/v1/tests/test_build.py | 6 +-- 4 files changed, 53 insertions(+), 7 deletions(-) diff --git a/proof/region/v1/arb/tests/gate.py b/proof/region/v1/arb/tests/gate.py index f04c3327..5a82beb1 100644 --- a/proof/region/v1/arb/tests/gate.py +++ b/proof/region/v1/arb/tests/gate.py @@ -15,7 +15,7 @@ REPO = Path(__file__).resolve().parents[5] sys.path.insert(0, str(REPO)) EXPECTED_TEST_INVENTORY_SHA256 = ( - "8a6c293af9283193dffe24ba45b864e343ad5c7d64fed2519fdd1d87c8b28d34" + "3c03f5b9ad9f19c24fa78380b6b9a78ce559a175cf603b2f065e85cba285030b" ) _EVALUATOR_REASON = "set LABCOLORS_ARB_EVALUATOR to the controlled C17 binary" EXPECTED_SKIPS = frozenset( diff --git a/proof/region/v1/arb/tests/test_transport.py b/proof/region/v1/arb/tests/test_transport.py index fa44ea5b..ddba43d2 100644 --- a/proof/region/v1/arb/tests/test_transport.py +++ b/proof/region/v1/arb/tests/test_transport.py @@ -798,6 +798,48 @@ def test_native_adapter_mints_private_docker_issued_cleanup_authority(self) -> N finally: backend._release_run_lease_v1(lease) + def test_native_adapter_keeps_cleanup_failure_typed_for_unknown_observation(self) -> None: + backend = build_transport.NativeDockerBuildBackendV1( + Path("/usr/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + platform_name="linux", + machine_name="x86_64", + host_user=(501, 20), + ) + capability = _probe_native_backend( + backend, + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + ) + request = build_transport.DockerBuildRequestV1( + 1, + capability, + _bundle(1024), + 1024, + ) + release = backend._release_run_lease_v1 + + def report_cleanup_failure(lease: object) -> str: + self.assertIsNone(release(lease)) + return "forced CID-root cleanup failure" + + with mock.patch.object(backend, "_observe_command", return_value=object()): + with mock.patch.object( + backend, + "_release_run_lease_v1", + side_effect=report_cleanup_failure, + ): + result = backend.run_build(request) + + self.assertIs(type(result), build_transport.DockerBuildObserverFailureV1) + self.assertEqual( + result.detail, + "native Docker build observation is not canonical; " + "forced CID-root cleanup failure", + ) + self.assertEqual(result.stdout, b"") + self.assertEqual(result.stderr, b"") + self.assertIsNone(result.input_progress) + def test_recipe_is_transport_agnostic_and_bootstrap_owns_binary_stdout(self) -> None: source = BUILD_RECIPE.read_text(encoding="utf-8") self.assertIn("readonly inputs=/build/snapshot/inputs", source) diff --git a/proof/region/v1/build/transport.py b/proof/region/v1/build/transport.py index 153ddb71..cf2fa660 100644 --- a/proof/region/v1/build/transport.py +++ b/proof/region/v1/build/transport.py @@ -2365,10 +2365,10 @@ def run_build( @staticmethod def _with_cid_root_cleanup_failure_v1( - observation: DockerBuildProcessObservationV1 | None, + observation: object, detail: str, ) -> DockerBuildProcessObservationV1: - """Retain a completed causal prefix when native CID-root release fails.""" + """Retain a canonical prefix, otherwise report a typed observer failure.""" if observation is None: return DockerBuildObserverFailureV1(detail, b"", b"") @@ -2410,7 +2410,11 @@ def _with_cid_root_cleanup_failure_v1( progress = observation.input_progress trigger = DockerCleanupTriggerV1.OBSERVER_FAILURE else: - raise TypeError("unknown native Docker build observation") + return DockerBuildObserverFailureV1( + "native Docker build observation is not canonical; " + detail, + b"", + b"", + ) if progress is None: return DockerBuildObserverFailureV1( detail, diff --git a/proof/region/v1/tests/test_build.py b/proof/region/v1/tests/test_build.py index ea1876bb..cad7dde6 100644 --- a/proof/region/v1/tests/test_build.py +++ b/proof/region/v1/tests/test_build.py @@ -38,12 +38,12 @@ # its expected hash here would let a coordinated gate edit hide inventory drift. # A deliberate test-set change updates both values from fresh enumeration. ARB_INVENTORY_SHA256_V1 = ( - "8a6c293af9283193dffe24ba45b864e343ad5c7d64fed2519fdd1d87c8b28d34" + "3c03f5b9ad9f19c24fa78380b6b9a78ce559a175cf603b2f065e85cba285030b" ) ARB_ORDER_SHA256_V1 = ( - "b3c75d673b4878623c9383daffe043ed4000b208632b9e33a7e944ed13568098" + "e58edaad85f3f89ef29124e038a6aebe5e58f5caa1a58ae81297ce4f3c392e67" ) -ARB_TEST_COUNT_V1 = 173 +ARB_TEST_COUNT_V1 = 174 MOVED_INPUT_SURFACE_V1 = ( "CanonicalInputLimitsV1", From ba538b0bb92ff39f8e29e244040b10aa105f3652 Mon Sep 17 00:00:00 2001 From: Daniel from Labpics <63733699+lemone112@users.noreply.github.com> Date: Sat, 1 Aug 2026 12:54:18 +0300 Subject: [PATCH 10/20] =?UTF-8?q?Test:=20=D1=83=D1=82=D0=BE=D1=87=D0=BD?= =?UTF-8?q?=D0=B8=D1=82=D1=8C=20fixed-arity=20boundary?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- proof/region/v1/arb/tests/test_transport.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/proof/region/v1/arb/tests/test_transport.py b/proof/region/v1/arb/tests/test_transport.py index ddba43d2..db417fc9 100644 --- a/proof/region/v1/arb/tests/test_transport.py +++ b/proof/region/v1/arb/tests/test_transport.py @@ -756,8 +756,8 @@ def test_docker_request_carries_only_semantic_build_coordinates(self) -> None: ) self.assertFalse(hasattr(request, "cid_file")) self.assertFalse(hasattr(request, "container_name")) - # The request has no positional slot for adapter-owned host cleanup - # authority; extra values cannot smuggle a CID path or container name. + # The field assertion above proves no cleanup coordinate is modeled. + # This call separately guards the fixed-arity boundary against extras. with self.assertRaises(TypeError): build_transport.DockerBuildRequestV1( 1, From f645d75b01fc21916a9dd8d35f0e15faa8845e69 Mon Sep 17 00:00:00 2001 From: Daniel from Labpics <63733699+lemone112@users.noreply.github.com> Date: Sat, 1 Aug 2026 13:24:04 +0300 Subject: [PATCH 11/20] Proof: totalize native build observations --- proof/region/v1/arb/tests/gate.py | 2 +- proof/region/v1/arb/tests/test_transport.py | 466 +++++++++++++++++++- proof/region/v1/build/transport.py | 148 ++++++- proof/region/v1/tests/test_build.py | 6 +- 4 files changed, 588 insertions(+), 34 deletions(-) diff --git a/proof/region/v1/arb/tests/gate.py b/proof/region/v1/arb/tests/gate.py index 5a82beb1..20f49288 100644 --- a/proof/region/v1/arb/tests/gate.py +++ b/proof/region/v1/arb/tests/gate.py @@ -15,7 +15,7 @@ REPO = Path(__file__).resolve().parents[5] sys.path.insert(0, str(REPO)) EXPECTED_TEST_INVENTORY_SHA256 = ( - "3c03f5b9ad9f19c24fa78380b6b9a78ce559a175cf603b2f065e85cba285030b" + "bd62145334bcd64d9c1d95551b2856ced34f97e6fee3f60447fe0170ae116930" ) _EVALUATOR_REASON = "set LABCOLORS_ARB_EVALUATOR to the controlled C17 binary" EXPECTED_SKIPS = frozenset( diff --git a/proof/region/v1/arb/tests/test_transport.py b/proof/region/v1/arb/tests/test_transport.py index db417fc9..cb63cd88 100644 --- a/proof/region/v1/arb/tests/test_transport.py +++ b/proof/region/v1/arb/tests/test_transport.py @@ -671,6 +671,51 @@ def test_cleanup_failure_preserves_input_trigger_and_progress(self) -> None: class SealedBuildTransportContractTests(unittest.TestCase): + def test_diagnostic_details_have_one_strict_admission_law(self) -> None: + constructors = ( + ( + "unsupported", + lambda detail: build_transport.DockerUnsupportedV1( + build_transport.DockerBlockerReasonV1.BACKEND_CONTRACT, + detail, + ), + ), + ( + "observer-failure", + lambda detail: build_transport.DockerBuildObserverFailureV1( + detail, + b"", + b"", + ), + ), + ( + "cleanup-record", + lambda detail: build_transport.CleanupFailureRecordV1( + build_transport.CleanupResourceV1.DOCKER_CID_ROOT, + detail, + ), + ), + ) + + class DetailSubclass(str): + pass + + for constructor_name, constructor in constructors: + with self.subTest(constructor=constructor_name, detail="valid"): + self.assertIsNotNone(constructor("valid diagnostic detail")) + for name, invalid_detail in ( + ("empty", ""), + ("subclass", DetailSubclass("detail")), + ( + "too-long", + "x" * (build_transport._DIAGNOSTIC_DETAIL_TEXT_LIMIT_V1 + 1), + ), + ("wrong-type", object()), + ): + with self.subTest(constructor=constructor_name, detail=name): + with self.assertRaises(TypeError): + constructor(invalid_detail) + def test_successful_probe_keeps_machine_readable_stdout_despite_cli_warning(self) -> None: policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 with tempfile.TemporaryDirectory() as temporary: @@ -798,7 +843,343 @@ def test_native_adapter_mints_private_docker_issued_cleanup_authority(self) -> N finally: backend._release_run_lease_v1(lease) - def test_native_adapter_keeps_cleanup_failure_typed_for_unknown_observation(self) -> None: + def test_native_adapter_rejects_malformed_nominal_observations_typed(self) -> None: + raw_observations = ( + ("unknown", object()), + *( + (kind.__name__, tuple.__new__(kind, ())) + for kind in ( + build_transport.DockerBuildExitedV1, + build_transport.DockerBuildTimedOutV1, + build_transport.DockerBuildOutputLimitV1, + build_transport.DockerBuildObserverFailureV1, + build_transport.DockerBuildInputRejectedV1, + build_transport.DockerBuildCleanupFailureV1, + ) + ), + ( + "DockerBuildObserverFailureV1/invalid-progress", + tuple.__new__( + build_transport.DockerBuildObserverFailureV1, + ("forged", b"untrusted stdout", b"untrusted stderr", object()), + ), + ), + ) + + def observe(raw: object, *, cleanup_fails: bool) -> object: + backend = build_transport.NativeDockerBuildBackendV1( + Path("/usr/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + platform_name="linux", + machine_name="x86_64", + host_user=(501, 20), + ) + capability = _probe_native_backend( + backend, + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + ) + request = build_transport.DockerBuildRequestV1( + 1, + capability, + _bundle(1024), + 1024, + ) + if not cleanup_fails: + with mock.patch.object( + backend, + "_observe_command", + return_value=raw, + ): + return backend.run_build(request) + + release = backend._release_run_lease_v1 + + def report_cleanup_failure(lease: object) -> str: + self.assertIsNone(release(lease)) + return "forced CID-root cleanup failure" + + with mock.patch.object( + backend, + "_observe_command", + return_value=raw, + ), mock.patch.object( + backend, + "_release_run_lease_v1", + side_effect=report_cleanup_failure, + ): + return backend.run_build(request) + + for name, raw in raw_observations: + with self.subTest(observation=name, cleanup_fails=False): + without_cleanup_failure = observe(raw, cleanup_fails=False) + self.assertIs( + type(without_cleanup_failure), + build_transport.DockerBuildObserverFailureV1, + ) + self.assertEqual( + without_cleanup_failure.detail, + "native Docker build observation is not canonical", + ) + self.assertEqual(without_cleanup_failure.stdout, b"") + self.assertEqual(without_cleanup_failure.stderr, b"") + self.assertIsNone(without_cleanup_failure.input_progress) + with self.subTest(observation=name, cleanup_fails=True): + with_cleanup_failure = observe(raw, cleanup_fails=True) + self.assertIs( + type(with_cleanup_failure), + build_transport.DockerBuildObserverFailureV1, + ) + self.assertEqual( + with_cleanup_failure.detail, + "native Docker build observation is not canonical; " + "forced CID-root cleanup failure", + ) + self.assertEqual(with_cleanup_failure.stdout, b"") + self.assertEqual(with_cleanup_failure.stderr, b"") + self.assertIsNone(with_cleanup_failure.input_progress) + + def test_native_adapter_rejects_a_preexisting_cid_root_claim(self) -> None: + backend = build_transport.NativeDockerBuildBackendV1( + Path("/usr/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + platform_name="linux", + machine_name="x86_64", + host_user=(501, 20), + ) + capability = _probe_native_backend( + backend, + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + ) + bundle = _bundle(1024) + request = build_transport.DockerBuildRequestV1( + 1, + capability, + bundle, + 1024, + ) + raw = build_transport.DockerBuildCleanupFailureV1( + build_transport.DockerCleanupTriggerV1.PROCESS_EXIT, + ( + build_transport.CleanupFailureRecordV1( + build_transport.CleanupResourceV1.DOCKER_CID_ROOT, + "forged prior CID-root cleanup failure", + ), + ), + b"retained stdout", + b"retained stderr", + build_transport._build_input_progress_v1( + bundle, + bundle.length, + bundle.sha256, + ), + ) + with mock.patch.object( + backend, + "_observe_command", + return_value=raw, + ): + without_cleanup_failure = backend.run_build(request) + + self.assertIs( + type(without_cleanup_failure), + build_transport.DockerBuildObserverFailureV1, + ) + self.assertEqual( + without_cleanup_failure.detail, + "native Docker build observation already contains a CID-root " + "cleanup failure", + ) + self.assertEqual(without_cleanup_failure.stdout, b"retained stdout") + self.assertEqual(without_cleanup_failure.stderr, b"retained stderr") + self.assertIsNone(without_cleanup_failure.input_progress) + + release = backend._release_run_lease_v1 + + def report_cleanup_failure(lease: object) -> str: + self.assertIsNone(release(lease)) + return "forced CID-root cleanup failure" + + with mock.patch.object( + backend, + "_observe_command", + return_value=raw, + ), mock.patch.object( + backend, + "_release_run_lease_v1", + side_effect=report_cleanup_failure, + ): + with_cleanup_failure = backend.run_build(request) + + self.assertIs( + type(with_cleanup_failure), + build_transport.DockerBuildObserverFailureV1, + ) + self.assertEqual( + with_cleanup_failure.detail, + "native Docker build observation already contains a CID-root " + "cleanup failure; forced CID-root cleanup failure", + ) + self.assertEqual(with_cleanup_failure.stdout, b"retained stdout") + self.assertEqual(with_cleanup_failure.stderr, b"retained stderr") + self.assertIsNone(with_cleanup_failure.input_progress) + + def test_native_adapter_appends_cid_root_to_canonical_cleanup_prefix(self) -> None: + backend = build_transport.NativeDockerBuildBackendV1( + Path("/usr/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + platform_name="linux", + machine_name="x86_64", + host_user=(501, 20), + ) + capability = _probe_native_backend( + backend, + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + ) + bundle = _bundle(1024) + request = build_transport.DockerBuildRequestV1( + 1, + capability, + bundle, + 1024, + ) + progress = build_transport._build_input_progress_v1( + bundle, + bundle.length, + bundle.sha256, + ) + raw = build_transport.DockerBuildCleanupFailureV1( + build_transport.DockerCleanupTriggerV1.PROCESS_EXIT, + ( + build_transport.CleanupFailureRecordV1( + build_transport.CleanupResourceV1.DOCKER_CLI_PROCESS, + "prior CLI cleanup failure", + ), + build_transport.CleanupFailureRecordV1( + build_transport.CleanupResourceV1.DOCKER_CONTAINER, + "prior container cleanup failure", + ), + ), + b"retained stdout", + b"retained stderr", + progress, + ) + release = backend._release_run_lease_v1 + + def report_cleanup_failure(lease: object) -> str: + self.assertIsNone(release(lease)) + return "forced CID-root cleanup failure" + + with mock.patch.object( + backend, + "_observe_command", + return_value=raw, + ), mock.patch.object( + backend, + "_release_run_lease_v1", + side_effect=report_cleanup_failure, + ): + result = backend.run_build(request) + + self.assertIs(type(result), build_transport.DockerBuildCleanupFailureV1) + self.assertEqual(result.trigger, build_transport.DockerCleanupTriggerV1.PROCESS_EXIT) + self.assertEqual( + tuple(record.resource for record in result.failures), + ( + build_transport.CleanupResourceV1.DOCKER_CLI_PROCESS, + build_transport.CleanupResourceV1.DOCKER_CONTAINER, + build_transport.CleanupResourceV1.DOCKER_CID_ROOT, + ), + ) + self.assertEqual( + tuple(record.detail for record in result.failures), + ( + "prior CLI cleanup failure", + "prior container cleanup failure", + "forced CID-root cleanup failure", + ), + ) + self.assertEqual(result.stdout, b"retained stdout") + self.assertEqual(result.stderr, b"retained stderr") + self.assertIs(result.input_progress, progress) + + def test_native_adapter_reowns_invalid_cid_cleanup_details(self) -> None: + backend = build_transport.NativeDockerBuildBackendV1( + Path("/usr/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + platform_name="linux", + machine_name="x86_64", + host_user=(501, 20), + ) + capability = _probe_native_backend( + backend, + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + ) + bundle = _bundle(1024) + request = build_transport.DockerBuildRequestV1( + 1, + capability, + bundle, + 1024, + ) + transfer = build_transport._completed_build_input_transfer_v1( + bundle, + bundle.length, + bundle.sha256, + ) + raw = build_transport._docker_build_exited_v1( + 0, + b"built stdout", + b"built stderr", + transfer, + ) + release = backend._release_run_lease_v1 + + for name, invalid_detail in ( + ("empty", ""), + ("wrong-type", object()), + ( + "too-long", + "x" * (build_transport._DIAGNOSTIC_DETAIL_TEXT_LIMIT_V1 + 1), + ), + ): + with self.subTest(detail=name): + + def release_with_invalid_detail( + lease: object, + *, + detail: object = invalid_detail, + ) -> object: + self.assertIsNone(release(lease)) + return detail + + with mock.patch.object( + backend, + "_observe_command", + return_value=raw, + ), mock.patch.object( + backend, + "_release_run_lease_v1", + side_effect=release_with_invalid_detail, + ): + result = backend.run_build(request) + + self.assertIs(type(result), build_transport.DockerBuildCleanupFailureV1) + self.assertEqual( + result.detail, + "native Docker CID root cleanup detail is not canonical", + ) + self.assertEqual(result.stdout, b"built stdout") + self.assertEqual(result.stderr, b"built stderr") + self.assertEqual( + result.input_progress, + build_transport._build_input_progress_v1( + bundle, + bundle.length, + bundle.sha256, + ), + ) + + def test_native_adapter_keeps_dual_failure_typed_at_detail_limit(self) -> None: backend = build_transport.NativeDockerBuildBackendV1( Path("/usr/bin/true"), pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, @@ -816,29 +1197,88 @@ def test_native_adapter_keeps_cleanup_failure_typed_for_unknown_observation(self _bundle(1024), 1024, ) + raw = build_transport.DockerBuildObserverFailureV1( + "x" * build_transport._DIAGNOSTIC_DETAIL_TEXT_LIMIT_V1, + b"retained stdout", + b"retained stderr", + ) release = backend._release_run_lease_v1 def report_cleanup_failure(lease: object) -> str: self.assertIsNone(release(lease)) return "forced CID-root cleanup failure" - with mock.patch.object(backend, "_observe_command", return_value=object()): - with mock.patch.object( - backend, - "_release_run_lease_v1", - side_effect=report_cleanup_failure, - ): - result = backend.run_build(request) + with mock.patch.object( + backend, + "_observe_command", + return_value=raw, + ), mock.patch.object( + backend, + "_release_run_lease_v1", + side_effect=report_cleanup_failure, + ): + result = backend.run_build(request) self.assertIs(type(result), build_transport.DockerBuildObserverFailureV1) self.assertEqual( result.detail, - "native Docker build observation is not canonical; " - "forced CID-root cleanup failure", + "native Docker build observation and CID root cleanup both failed", + ) + self.assertEqual(result.stdout, b"retained stdout") + self.assertEqual(result.stderr, b"retained stderr") + + def test_native_adapter_releases_before_propagating_first_canonicalization_interruption( + self, + ) -> None: + backend = build_transport.NativeDockerBuildBackendV1( + Path("/usr/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + platform_name="linux", + machine_name="x86_64", + host_user=(501, 20), + ) + capability = _probe_native_backend( + backend, + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + ) + request = build_transport.DockerBuildRequestV1( + 1, + capability, + _bundle(1024), + 1024, ) - self.assertEqual(result.stdout, b"") - self.assertEqual(result.stderr, b"") - self.assertIsNone(result.input_progress) + release = backend._release_run_lease_v1 + released: list[None] = [] + + class FalseyInterrupt(BaseException): + def __bool__(self) -> bool: + return False + + original = FalseyInterrupt("first interruption") + + def release_then_record(lease: object) -> None: + self.assertIsNone(release(lease)) + released.append(None) + raise KeyboardInterrupt("later cleanup interruption") + + with mock.patch.object( + backend, + "_observe_command", + return_value=object(), + ), mock.patch.object( + build_transport, + "_canonical_process_observation_v1", + side_effect=original, + ), mock.patch.object( + backend, + "_release_run_lease_v1", + side_effect=release_then_record, + ): + with self.assertRaises(FalseyInterrupt) as raised: + backend.run_build(request) + + self.assertIs(raised.exception, original) + self.assertEqual(released, [None]) def test_recipe_is_transport_agnostic_and_bootstrap_owns_binary_stdout(self) -> None: source = BUILD_RECIPE.read_text(encoding="utf-8") diff --git a/proof/region/v1/build/transport.py b/proof/region/v1/build/transport.py index cf2fa660..4bb22f1c 100644 --- a/proof/region/v1/build/transport.py +++ b/proof/region/v1/build/transport.py @@ -39,6 +39,12 @@ _IO_CHUNK_BYTES_V1 = 64 * 1024 _POLL_SLICE_SECONDS_V1 = 0.1 _PROCESS_STOP_TIMEOUT_SECONDS_V1 = 30 +# Details enter receipts as bounded diagnostic evidence; this avoids allowing +# an adapter error string to become an unbounded transport payload. +_DIAGNOSTIC_DETAIL_TEXT_LIMIT_V1 = 4096 +_INVALID_CID_ROOT_CLEANUP_DETAIL_V1 = ( + "native Docker CID root cleanup detail is not canonical" +) _PATH_TYPE = type(Path("/")) _NATIVE_CID_ROOT_PREFIX_V1 = "labcolors-docker-cid-" @@ -69,6 +75,32 @@ def _identity(label: bytes, chunks: tuple[bytes, ...]) -> bytes: return hashlib.sha256(label + len(payload).to_bytes(8, "big") + payload).digest() +def _retain_first_base_exception_v1( + retained: BaseException | None, + current: BaseException, +) -> BaseException: + return retained if retained is not None else current + + +def _canonical_diagnostic_detail_v1(value: object) -> str | None: + if ( + type(value) is str + and value + and len(value) <= _DIAGNOSTIC_DETAIL_TEXT_LIMIT_V1 + ): + return value + return None + + +def _canonical_cid_root_cleanup_detail_v1(value: object) -> str | None: + if value is None: + return None + detail = _canonical_diagnostic_detail_v1(value) + if detail is not None: + return detail + return _INVALID_CID_ROOT_CLEANUP_DETAIL_V1 + + def _pinned_image_reference(value: object) -> bool: if type(value) is not str or value.count("@sha256:") != 1: return False @@ -865,7 +897,7 @@ def __new__( ) -> DockerUnsupportedV1: if type(reason) is not DockerBlockerReasonV1: raise TypeError("invalid Docker blocker reason") - if type(detail) is not str or not detail or len(detail) > 4096: + if _canonical_diagnostic_detail_v1(detail) is None: raise TypeError("invalid Docker blocker detail") return tuple.__new__(cls, (reason, detail)) @@ -1631,7 +1663,7 @@ def __new__( stderr: bytes, input_progress: BuildInputTransferProgressV1 | None = None, ) -> DockerBuildObserverFailureV1: - if type(detail) is not str or not detail or len(detail) > 4096: + if _canonical_diagnostic_detail_v1(detail) is None: raise TypeError("invalid Docker observer failure") _bounded_bytes(stdout, BUILD_STDOUT_LIMIT_V1, "stdout") _bounded_bytes(stderr, BUILD_STDERR_LIMIT_V1, "stderr") @@ -1718,7 +1750,7 @@ def __new__( ) -> CleanupFailureRecordV1: if type(resource) is not CleanupResourceV1: raise TypeError("invalid cleanup resource") - if type(detail) is not str or not detail or len(detail) > 4096: + if _canonical_diagnostic_detail_v1(detail) is None: raise TypeError("invalid cleanup failure detail") return tuple.__new__(cls, (resource, detail)) @@ -2320,7 +2352,7 @@ def run_build( b"", ) policy = capability.policy - observation = self._observe_command( + raw_observation = self._observe_command( command, stdout_limit=request.max_output_bytes, stderr_limit=policy.stderr_limit, @@ -2328,6 +2360,35 @@ def run_build( lease=lease, input_bundle=request.input_bundle, ) + try: + canonical_observation = _canonical_process_observation_v1( + raw_observation, + request.input_bundle, + request.max_output_bytes, + policy.stderr_limit, + ) + except Exception: + observation = DockerBuildObserverFailureV1( + "native Docker build observation is not canonical", + b"", + b"", + ) + else: + if ( + type(canonical_observation) is DockerBuildCleanupFailureV1 + and any( + record.resource is CleanupResourceV1.DOCKER_CID_ROOT + for record in canonical_observation.failures + ) + ): + observation = DockerBuildObserverFailureV1( + "native Docker build observation already contains a " + "CID-root cleanup failure", + canonical_observation.stdout, + canonical_observation.stderr, + ) + else: + observation = canonical_observation except Exception: observation = DockerBuildObserverFailureV1( "native Docker build request could not be materialized", @@ -2340,11 +2401,16 @@ def run_build( release_detail: str | None = None if lease is not None: try: - release_detail = self._release_run_lease_v1(lease) + release_detail = _canonical_cid_root_cleanup_detail_v1( + self._release_run_lease_v1(lease) + ) except Exception: release_detail = "native Docker CID root cleanup observer raised" except BaseException as error: - retained_base_exception = retained_base_exception or error + retained_base_exception = _retain_first_base_exception_v1( + retained_base_exception, + error, + ) release_detail = "native Docker CID root cleanup was interrupted" if retained_base_exception is None and release_detail is not None: observation = self._with_cid_root_cleanup_failure_v1( @@ -2416,6 +2482,19 @@ def _with_cid_root_cleanup_failure_v1( b"", ) if progress is None: + if type(observation) is DockerBuildObserverFailureV1: + try: + return DockerBuildObserverFailureV1( + observation.detail + "; " + detail, + observation.stdout, + observation.stderr, + ) + except Exception: + return DockerBuildObserverFailureV1( + "native Docker build observation and CID root cleanup both failed", + observation.stdout, + observation.stderr, + ) return DockerBuildObserverFailureV1( detail, observation.stdout, @@ -2660,7 +2739,10 @@ def _observe_command( except Exception: observer_failed = True except BaseException as error: - retained_base_exception = retained_base_exception or error + retained_base_exception = _retain_first_base_exception_v1( + retained_base_exception, + error, + ) observer_failed = True if process.stdin is not None: close_failed, close_interrupt = self._close_owned_stream( @@ -2668,14 +2750,21 @@ def _observe_command( ) if close_failed: observer_failed = True - retained_base_exception = retained_base_exception or close_interrupt + if close_interrupt is not None: + retained_base_exception = _retain_first_base_exception_v1( + retained_base_exception, + close_interrupt, + ) if bundle_view is not None: try: bundle_view.release() except Exception: observer_failed = True except BaseException as error: - retained_base_exception = retained_base_exception or error + retained_base_exception = _retain_first_base_exception_v1( + retained_base_exception, + error, + ) observer_failed = True if input_bundle is not None: try: @@ -2687,7 +2776,10 @@ def _observe_command( except Exception: observer_failed = True except BaseException as error: - retained_base_exception = retained_base_exception or error + retained_base_exception = _retain_first_base_exception_v1( + retained_base_exception, + error, + ) observer_failed = True ownership_lost = ownership_lost or not self._in_owner_process_v1() if not ownership_lost: @@ -2697,7 +2789,10 @@ def _observe_command( process_running = True stop_detail = "Docker CLI process state could not be observed" except BaseException as error: - retained_base_exception = retained_base_exception or error + retained_base_exception = _retain_first_base_exception_v1( + retained_base_exception, + error, + ) process_running = True stop_detail = "Docker CLI process state observation was interrupted" if process_running: @@ -2713,7 +2808,10 @@ def _observe_command( except Exception: observed_stop = "Docker CLI process termination raised" except BaseException as error: - retained_base_exception = retained_base_exception or error + retained_base_exception = _retain_first_base_exception_v1( + retained_base_exception, + error, + ) observed_stop = "Docker CLI process termination was interrupted" fallback_stop = self._force_reap_after_interruption_v1(process) stop_detail = stop_detail or observed_stop or fallback_stop @@ -2725,7 +2823,11 @@ def _observe_command( ) if close_failed: observer_failed = True - retained_base_exception = retained_base_exception or close_interrupt + if close_interrupt is not None: + retained_base_exception = _retain_first_base_exception_v1( + retained_base_exception, + close_interrupt, + ) if lease is not None and not ownership_lost: try: cleanup_detail = self._cleanup_container( @@ -2735,7 +2837,10 @@ def _observe_command( except Exception: cleanup_detail = "Docker container cleanup observer raised" except BaseException as error: - retained_base_exception = retained_base_exception or error + retained_base_exception = _retain_first_base_exception_v1( + retained_base_exception, + error, + ) cleanup_detail = "Docker container cleanup was interrupted" if retained_base_exception is not None: raise retained_base_exception.with_traceback( @@ -2876,14 +2981,20 @@ def _close_owned_stream( except BaseException as error: close_failed = True close_raised = True - retained_base_exception = retained_base_exception or error + retained_base_exception = _retain_first_base_exception_v1( + retained_base_exception, + error, + ) if close_raised: try: still_open = stream.closed is False except Exception: still_open = False except BaseException as error: - retained_base_exception = retained_base_exception or error + retained_base_exception = _retain_first_base_exception_v1( + retained_base_exception, + error, + ) still_open = False if still_open: # close() may fail before it releases the resource, but a @@ -2896,7 +3007,10 @@ def _close_owned_stream( close_failed = True except BaseException as error: close_failed = True - retained_base_exception = retained_base_exception or error + retained_base_exception = _retain_first_base_exception_v1( + retained_base_exception, + error, + ) return close_failed, retained_base_exception def _clock(self) -> int: diff --git a/proof/region/v1/tests/test_build.py b/proof/region/v1/tests/test_build.py index cad7dde6..4fa43959 100644 --- a/proof/region/v1/tests/test_build.py +++ b/proof/region/v1/tests/test_build.py @@ -38,12 +38,12 @@ # its expected hash here would let a coordinated gate edit hide inventory drift. # A deliberate test-set change updates both values from fresh enumeration. ARB_INVENTORY_SHA256_V1 = ( - "3c03f5b9ad9f19c24fa78380b6b9a78ce559a175cf603b2f065e85cba285030b" + "bd62145334bcd64d9c1d95551b2856ced34f97e6fee3f60447fe0170ae116930" ) ARB_ORDER_SHA256_V1 = ( - "e58edaad85f3f89ef29124e038a6aebe5e58f5caa1a58ae81297ce4f3c392e67" + "48e9b88583a250ec4122c064cc8e9c479cd8f1622a5cd0f88bbf46af62c5cb7b" ) -ARB_TEST_COUNT_V1 = 174 +ARB_TEST_COUNT_V1 = 180 MOVED_INPUT_SURFACE_V1 = ( "CanonicalInputLimitsV1", From 284ee162c00be98c3b24ca69aa5f99c161d794af Mon Sep 17 00:00:00 2001 From: Daniel from Labpics <63733699+lemone112@users.noreply.github.com> Date: Sat, 1 Aug 2026 15:54:43 +0300 Subject: [PATCH 12/20] Proof: retain native cleanup evidence --- proof/region/v1/arb/tests/gate.py | 2 +- proof/region/v1/arb/tests/test_transport.py | 247 ++++++++------------ proof/region/v1/build/transport.py | 16 +- proof/region/v1/tests/test_build.py | 6 +- 4 files changed, 118 insertions(+), 153 deletions(-) diff --git a/proof/region/v1/arb/tests/gate.py b/proof/region/v1/arb/tests/gate.py index 20f49288..8026abcf 100644 --- a/proof/region/v1/arb/tests/gate.py +++ b/proof/region/v1/arb/tests/gate.py @@ -15,7 +15,7 @@ REPO = Path(__file__).resolve().parents[5] sys.path.insert(0, str(REPO)) EXPECTED_TEST_INVENTORY_SHA256 = ( - "bd62145334bcd64d9c1d95551b2856ced34f97e6fee3f60447fe0170ae116930" + "cbacd035c919cb7a18a3f05d41319ae5bf6c93bbcca9612312357e9be23aedd5" ) _EVALUATOR_REASON = "set LABCOLORS_ARB_EVALUATOR to the controlled C17 binary" EXPECTED_SKIPS = frozenset( diff --git a/proof/region/v1/arb/tests/test_transport.py b/proof/region/v1/arb/tests/test_transport.py index cb63cd88..5ce29920 100644 --- a/proof/region/v1/arb/tests/test_transport.py +++ b/proof/region/v1/arb/tests/test_transport.py @@ -60,6 +60,49 @@ def _backend() -> build_transport.NativeDockerBuildBackendV1: ) +def _native_backend_with_request( + bundle: build_input.SealedInputV1 | None = None, +) -> tuple[ + build_transport.NativeDockerBuildBackendV1, + build_transport.DockerBuildRequestV1, +]: + """One fixture owns the native request shape used by cleanup tests.""" + + backend = build_transport.NativeDockerBuildBackendV1( + Path("/usr/bin/true"), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + platform_name="linux", + machine_name="x86_64", + host_user=(501, 20), + ) + capability = _probe_native_backend( + backend, + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + ) + return backend, build_transport.DockerBuildRequestV1( + 1, + capability, + _bundle(1024) if bundle is None else bundle, + 1024, + ) + + +def _report_released_cid_root( + backend: build_transport.NativeDockerBuildBackendV1, + detail: object = "forced CID-root cleanup failure", +) -> Callable[[object], object]: + """Keep failure fixtures honest: a reported cleanup failure follows release.""" + + release = backend._release_run_lease_v1 + + def release_then_report(lease: object) -> object: + if release(lease) is not None: + raise AssertionError("native CID-root fixture lease did not release") + return detail + + return release_then_report + + def _observe( source: str, bundle: build_input.SealedInputV1, @@ -814,23 +857,8 @@ def test_docker_request_carries_only_semantic_build_coordinates(self) -> None: ) def test_native_adapter_mints_private_docker_issued_cleanup_authority(self) -> None: - backend = build_transport.NativeDockerBuildBackendV1( - Path("/usr/bin/true"), - pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, - platform_name="linux", - machine_name="x86_64", - host_user=(501, 20), - ) - capability = _probe_native_backend( - backend, - pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, - ) - request = build_transport.DockerBuildRequestV1( - 1, - capability, - _bundle(1024), - 1024, - ) + backend, request = _native_backend_with_request() + capability = request.capability lease = backend._next_run_lease_v1(capability) try: command = backend._command_for_v1(request, lease) @@ -867,23 +895,7 @@ def test_native_adapter_rejects_malformed_nominal_observations_typed(self) -> No ) def observe(raw: object, *, cleanup_fails: bool) -> object: - backend = build_transport.NativeDockerBuildBackendV1( - Path("/usr/bin/true"), - pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, - platform_name="linux", - machine_name="x86_64", - host_user=(501, 20), - ) - capability = _probe_native_backend( - backend, - pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, - ) - request = build_transport.DockerBuildRequestV1( - 1, - capability, - _bundle(1024), - 1024, - ) + backend, request = _native_backend_with_request() if not cleanup_fails: with mock.patch.object( backend, @@ -892,12 +904,6 @@ def observe(raw: object, *, cleanup_fails: bool) -> object: ): return backend.run_build(request) - release = backend._release_run_lease_v1 - - def report_cleanup_failure(lease: object) -> str: - self.assertIsNone(release(lease)) - return "forced CID-root cleanup failure" - with mock.patch.object( backend, "_observe_command", @@ -905,7 +911,7 @@ def report_cleanup_failure(lease: object) -> str: ), mock.patch.object( backend, "_release_run_lease_v1", - side_effect=report_cleanup_failure, + side_effect=_report_released_cid_root(backend), ): return backend.run_build(request) @@ -939,24 +945,8 @@ def report_cleanup_failure(lease: object) -> str: self.assertIsNone(with_cleanup_failure.input_progress) def test_native_adapter_rejects_a_preexisting_cid_root_claim(self) -> None: - backend = build_transport.NativeDockerBuildBackendV1( - Path("/usr/bin/true"), - pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, - platform_name="linux", - machine_name="x86_64", - host_user=(501, 20), - ) - capability = _probe_native_backend( - backend, - pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, - ) bundle = _bundle(1024) - request = build_transport.DockerBuildRequestV1( - 1, - capability, - bundle, - 1024, - ) + backend, request = _native_backend_with_request(bundle) raw = build_transport.DockerBuildCleanupFailureV1( build_transport.DockerCleanupTriggerV1.PROCESS_EXIT, ( @@ -991,13 +981,7 @@ def test_native_adapter_rejects_a_preexisting_cid_root_claim(self) -> None: ) self.assertEqual(without_cleanup_failure.stdout, b"retained stdout") self.assertEqual(without_cleanup_failure.stderr, b"retained stderr") - self.assertIsNone(without_cleanup_failure.input_progress) - - release = backend._release_run_lease_v1 - - def report_cleanup_failure(lease: object) -> str: - self.assertIsNone(release(lease)) - return "forced CID-root cleanup failure" + self.assertIs(without_cleanup_failure.input_progress, raw.input_progress) with mock.patch.object( backend, @@ -1006,42 +990,62 @@ def report_cleanup_failure(lease: object) -> str: ), mock.patch.object( backend, "_release_run_lease_v1", - side_effect=report_cleanup_failure, + side_effect=_report_released_cid_root(backend), ): with_cleanup_failure = backend.run_build(request) self.assertIs( type(with_cleanup_failure), - build_transport.DockerBuildObserverFailureV1, + build_transport.DockerBuildCleanupFailureV1, ) self.assertEqual( - with_cleanup_failure.detail, - "native Docker build observation already contains a CID-root " - "cleanup failure; forced CID-root cleanup failure", + with_cleanup_failure.trigger, + build_transport.DockerCleanupTriggerV1.OBSERVER_FAILURE, + ) + self.assertEqual( + with_cleanup_failure.failures, + ( + build_transport.CleanupFailureRecordV1( + build_transport.CleanupResourceV1.DOCKER_CID_ROOT, + "forced CID-root cleanup failure", + ), + ), ) self.assertEqual(with_cleanup_failure.stdout, b"retained stdout") self.assertEqual(with_cleanup_failure.stderr, b"retained stderr") - self.assertIsNone(with_cleanup_failure.input_progress) + self.assertIs(with_cleanup_failure.input_progress, raw.input_progress) + + def test_native_adapter_bounds_unknown_observation_cleanup_detail(self) -> None: + prefix = "native Docker build observation is not canonical; " + for name, detail, expected_detail in ( + ( + "retained", + "forced CID-root cleanup failure", + prefix + "forced CID-root cleanup failure", + ), + ( + "bounded", + "x" * build_transport._DIAGNOSTIC_DETAIL_TEXT_LIMIT_V1, + "native Docker build observation and CID root cleanup both failed", + ), + ): + with self.subTest(detail=name): + result = ( + build_transport.NativeDockerBuildBackendV1._with_cid_root_cleanup_failure_v1( + object(), + detail, + ) + ) + + self.assertIs(type(result), build_transport.DockerBuildObserverFailureV1) + self.assertEqual(result.detail, expected_detail) + self.assertEqual(result.stdout, b"") + self.assertEqual(result.stderr, b"") + self.assertIsNone(result.input_progress) def test_native_adapter_appends_cid_root_to_canonical_cleanup_prefix(self) -> None: - backend = build_transport.NativeDockerBuildBackendV1( - Path("/usr/bin/true"), - pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, - platform_name="linux", - machine_name="x86_64", - host_user=(501, 20), - ) - capability = _probe_native_backend( - backend, - pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, - ) bundle = _bundle(1024) - request = build_transport.DockerBuildRequestV1( - 1, - capability, - bundle, - 1024, - ) + backend, request = _native_backend_with_request(bundle) progress = build_transport._build_input_progress_v1( bundle, bundle.length, @@ -1063,12 +1067,6 @@ def test_native_adapter_appends_cid_root_to_canonical_cleanup_prefix(self) -> No b"retained stderr", progress, ) - release = backend._release_run_lease_v1 - - def report_cleanup_failure(lease: object) -> str: - self.assertIsNone(release(lease)) - return "forced CID-root cleanup failure" - with mock.patch.object( backend, "_observe_command", @@ -1076,7 +1074,7 @@ def report_cleanup_failure(lease: object) -> str: ), mock.patch.object( backend, "_release_run_lease_v1", - side_effect=report_cleanup_failure, + side_effect=_report_released_cid_root(backend), ): result = backend.run_build(request) @@ -1103,24 +1101,8 @@ def report_cleanup_failure(lease: object) -> str: self.assertIs(result.input_progress, progress) def test_native_adapter_reowns_invalid_cid_cleanup_details(self) -> None: - backend = build_transport.NativeDockerBuildBackendV1( - Path("/usr/bin/true"), - pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, - platform_name="linux", - machine_name="x86_64", - host_user=(501, 20), - ) - capability = _probe_native_backend( - backend, - pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, - ) bundle = _bundle(1024) - request = build_transport.DockerBuildRequestV1( - 1, - capability, - bundle, - 1024, - ) + backend, request = _native_backend_with_request(bundle) transfer = build_transport._completed_build_input_transfer_v1( bundle, bundle.length, @@ -1132,8 +1114,6 @@ def test_native_adapter_reowns_invalid_cid_cleanup_details(self) -> None: b"built stderr", transfer, ) - release = backend._release_run_lease_v1 - for name, invalid_detail in ( ("empty", ""), ("wrong-type", object()), @@ -1144,14 +1124,6 @@ def test_native_adapter_reowns_invalid_cid_cleanup_details(self) -> None: ): with self.subTest(detail=name): - def release_with_invalid_detail( - lease: object, - *, - detail: object = invalid_detail, - ) -> object: - self.assertIsNone(release(lease)) - return detail - with mock.patch.object( backend, "_observe_command", @@ -1159,7 +1131,10 @@ def release_with_invalid_detail( ), mock.patch.object( backend, "_release_run_lease_v1", - side_effect=release_with_invalid_detail, + side_effect=_report_released_cid_root( + backend, + invalid_detail, + ), ): result = backend.run_build(request) @@ -1180,34 +1155,12 @@ def release_with_invalid_detail( ) def test_native_adapter_keeps_dual_failure_typed_at_detail_limit(self) -> None: - backend = build_transport.NativeDockerBuildBackendV1( - Path("/usr/bin/true"), - pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, - platform_name="linux", - machine_name="x86_64", - host_user=(501, 20), - ) - capability = _probe_native_backend( - backend, - pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, - ) - request = build_transport.DockerBuildRequestV1( - 1, - capability, - _bundle(1024), - 1024, - ) + backend, request = _native_backend_with_request() raw = build_transport.DockerBuildObserverFailureV1( "x" * build_transport._DIAGNOSTIC_DETAIL_TEXT_LIMIT_V1, b"retained stdout", b"retained stderr", ) - release = backend._release_run_lease_v1 - - def report_cleanup_failure(lease: object) -> str: - self.assertIsNone(release(lease)) - return "forced CID-root cleanup failure" - with mock.patch.object( backend, "_observe_command", @@ -1215,7 +1168,7 @@ def report_cleanup_failure(lease: object) -> str: ), mock.patch.object( backend, "_release_run_lease_v1", - side_effect=report_cleanup_failure, + side_effect=_report_released_cid_root(backend), ): result = backend.run_build(request) diff --git a/proof/region/v1/build/transport.py b/proof/region/v1/build/transport.py index 4bb22f1c..53bc552a 100644 --- a/proof/region/v1/build/transport.py +++ b/proof/region/v1/build/transport.py @@ -45,6 +45,9 @@ _INVALID_CID_ROOT_CLEANUP_DETAIL_V1 = ( "native Docker CID root cleanup detail is not canonical" ) +_OBSERVER_AND_CID_ROOT_CLEANUP_FAILURE_V1 = ( + "native Docker build observation and CID root cleanup both failed" +) _PATH_TYPE = type(Path("/")) _NATIVE_CID_ROOT_PREFIX_V1 = "labcolors-docker-cid-" @@ -2386,6 +2389,7 @@ def run_build( "CID-root cleanup failure", canonical_observation.stdout, canonical_observation.stderr, + canonical_observation.input_progress, ) else: observation = canonical_observation @@ -2476,8 +2480,16 @@ def _with_cid_root_cleanup_failure_v1( progress = observation.input_progress trigger = DockerCleanupTriggerV1.OBSERVER_FAILURE else: + fallback_detail = ( + "native Docker build observation is not canonical; " + detail + ) + # Preserve a canonical cleanup diagnostic when it fits. A complete + # one can fill the budget, so overflow has the declared dual-failure + # meaning rather than turning the typed fallback into an exception. + if _canonical_diagnostic_detail_v1(fallback_detail) is None: + fallback_detail = _OBSERVER_AND_CID_ROOT_CLEANUP_FAILURE_V1 return DockerBuildObserverFailureV1( - "native Docker build observation is not canonical; " + detail, + fallback_detail, b"", b"", ) @@ -2491,7 +2503,7 @@ def _with_cid_root_cleanup_failure_v1( ) except Exception: return DockerBuildObserverFailureV1( - "native Docker build observation and CID root cleanup both failed", + _OBSERVER_AND_CID_ROOT_CLEANUP_FAILURE_V1, observation.stdout, observation.stderr, ) diff --git a/proof/region/v1/tests/test_build.py b/proof/region/v1/tests/test_build.py index 4fa43959..2132332d 100644 --- a/proof/region/v1/tests/test_build.py +++ b/proof/region/v1/tests/test_build.py @@ -38,12 +38,12 @@ # its expected hash here would let a coordinated gate edit hide inventory drift. # A deliberate test-set change updates both values from fresh enumeration. ARB_INVENTORY_SHA256_V1 = ( - "bd62145334bcd64d9c1d95551b2856ced34f97e6fee3f60447fe0170ae116930" + "cbacd035c919cb7a18a3f05d41319ae5bf6c93bbcca9612312357e9be23aedd5" ) ARB_ORDER_SHA256_V1 = ( - "48e9b88583a250ec4122c064cc8e9c479cd8f1622a5cd0f88bbf46af62c5cb7b" + "506d3d1f82102affc23e846b9500fbe95334134552800a141147a0595b476aea" ) -ARB_TEST_COUNT_V1 = 180 +ARB_TEST_COUNT_V1 = 181 MOVED_INPUT_SURFACE_V1 = ( "CanonicalInputLimitsV1", From 33dfdede8a2fae1fb5d7a3a80c225b301641b18d Mon Sep 17 00:00:00 2001 From: Daniel from Labpics <63733699+lemone112@users.noreply.github.com> Date: Sat, 1 Aug 2026 17:00:10 +0300 Subject: [PATCH 13/20] =?UTF-8?q?Proof:=20=D0=B7=D0=B0=D1=84=D0=B8=D0=BA?= =?UTF-8?q?=D1=81=D0=B8=D1=80=D0=BE=D0=B2=D0=B0=D1=82=D1=8C=20BUILD=20bind?= =?UTF-8?q?ing=20=D0=BD=D0=B0=20capability?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- proof/region/v1/arb/pipeline.py | 33 +++-- proof/region/v1/arb/receipt.py | 8 +- proof/region/v1/arb/tests/gate.py | 2 +- .../v1/arb/tests/test_build_identity_v2.py | 123 +++++++++++++++++- proof/region/v1/arb/tests/test_pipeline.py | 10 +- proof/region/v1/arb/tests/test_transport.py | 7 +- proof/region/v1/tests/test_build.py | 11 +- 7 files changed, 167 insertions(+), 27 deletions(-) diff --git a/proof/region/v1/arb/pipeline.py b/proof/region/v1/arb/pipeline.py index b581a980..bc6b0920 100644 --- a/proof/region/v1/arb/pipeline.py +++ b/proof/region/v1/arb/pipeline.py @@ -356,12 +356,14 @@ def _arb_input_binding_identity_v1( source_identity: bytes, build_input_identity: bytes, contents: bytes, + exact_policy: build_transport.DockerBuildPolicyV1, ) -> bytes: if ( not _valid_digest(source_identity) or not _valid_digest(build_input_identity) or type(contents) is not bytes or not contents + or not build_transport.docker_policy_is_valid_v1(exact_policy) ): raise TypeError("invalid Arb build input binding coordinates") digest = hashlib.sha256(contents).digest() @@ -372,8 +374,12 @@ def _arb_input_binding_identity_v1( build_input_identity, len(contents).to_bytes(8, "big"), digest, + # This inner identity fixes only the stream-to-tree program. V1 + # never reads shell $0; argv0 instead remains in the outer + # transport identity. A bootstrap that consumes $0 needs a new + # binding schema rather than silently widening this preimage. hashlib.sha256( - ARB_BUILD_TRANSPORT_POLICY_V1.bootstrap.encode("utf-8") + exact_policy.bootstrap.encode("utf-8") ).digest(), ), ) @@ -381,6 +387,7 @@ def _arb_input_binding_identity_v1( def arb_input_is_bound_v1( request: object, + exact_policy: object, value: object, ) -> bool: """Recompute Arb semantics independently of generic byte integrity.""" @@ -396,6 +403,7 @@ def arb_input_is_bound_v1( request.admitted_sources.identity, request.build_sources.build_input_identity, value.contents, + exact_policy, ) except Exception: return False @@ -403,9 +411,12 @@ def arb_input_is_bound_v1( def _seal_build_input_bundle_v1( request: "PipelineRequestV1", + exact_policy: build_transport.DockerBuildPolicyV1, ) -> build_input.SealedInputV1: if type(request) is not PipelineRequestV1: raise TypeError("request must be PipelineRequestV1") + if not build_transport.docker_policy_is_valid_v1(exact_policy): + raise TypeError("exact_policy must be canonical DockerBuildPolicyV1") source_entries = tuple( entry for lock, admitted in zip( @@ -450,6 +461,7 @@ def _seal_build_input_bundle_v1( request.admitted_sources.identity, request.build_sources.build_input_identity, contents, + exact_policy, ), contents, ) @@ -1036,8 +1048,6 @@ def _derive_arb_comparator_for_build_v1( or rebuild_sha256s != (binary_sha256, binary_sha256) ): raise TypeError("comparator derivation requires two equal successful builds") - if docker_capability.policy != ARB_BUILD_TRANSPORT_POLICY_V1: - raise TypeError("Docker capability does not bind the Arb transport policy") docker_capability_identity = build_transport.docker_capability_identity_v1( docker_capability ) @@ -1383,10 +1393,7 @@ def __init__( canonical_capability = build_transport.DockerSupportedV1( *tuple(docker_capability) ) - if ( - tuple(canonical_capability) != tuple(docker_capability) - or canonical_capability.policy != ARB_BUILD_TRANSPORT_POLICY_V1 - ): + if tuple(canonical_capability) != tuple(docker_capability): raise TypeError("diagnostic build does not bind the exact Arb capability") if ( type(rebuild_sha256s) is not tuple @@ -1409,6 +1416,7 @@ def __init__( structural_source_identity, build_input_identity, input_bundle.contents, + canonical_capability.policy, ) ): raise TypeError("diagnostic build lost its sealed input bundle") @@ -1564,7 +1572,10 @@ def build(self, request: PipelineRequestV1) -> BuildResultV1: ) docker_capability = probe_result try: - input_bundle = _seal_build_input_bundle_v1(request) + input_bundle = _seal_build_input_bundle_v1( + request, + docker_capability.policy, + ) except ( OSError, TypeError, @@ -1582,7 +1593,11 @@ def build(self, request: PipelineRequestV1) -> BuildResultV1: docker_capability, input_bundle, request.execution_limits.max_executable_bytes, - input_admission=lambda value: arb_input_is_bound_v1(request, value), + input_admission=lambda value: arb_input_is_bound_v1( + request, + docker_capability.policy, + value, + ), output_admission=self._admit_arb_output_v1, ) if type(built) is build_transport.BuildRejectedV1: diff --git a/proof/region/v1/arb/receipt.py b/proof/region/v1/arb/receipt.py index eb04237d..61ee8f92 100644 --- a/proof/region/v1/arb/receipt.py +++ b/proof/region/v1/arb/receipt.py @@ -214,10 +214,12 @@ def _build_identity_v2( request.host_trust, build.docker_capability.policy, ) - or build.docker_capability.policy - != pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 or build.host_trust is not request.host_trust - or not pipeline.arb_input_is_bound_v1(request, bundle) + or not pipeline.arb_input_is_bound_v1( + request, + build.docker_capability.policy, + bundle, + ) or build.input_bundle_identity != bundle.binding_identity or build.input_bundle_sha256 != bundle.sha256 or build.input_bundle_length != bundle.length diff --git a/proof/region/v1/arb/tests/gate.py b/proof/region/v1/arb/tests/gate.py index 8026abcf..160a2ba7 100644 --- a/proof/region/v1/arb/tests/gate.py +++ b/proof/region/v1/arb/tests/gate.py @@ -15,7 +15,7 @@ REPO = Path(__file__).resolve().parents[5] sys.path.insert(0, str(REPO)) EXPECTED_TEST_INVENTORY_SHA256 = ( - "cbacd035c919cb7a18a3f05d41319ae5bf6c93bbcca9612312357e9be23aedd5" + "75462b6e595a3642705ce5135ca6a38b5c634be61b0ef33ea81f73abe17b564b" ) _EVALUATOR_REASON = "set LABCOLORS_ARB_EVALUATOR to the controlled C17 binary" EXPECTED_SKIPS = frozenset( diff --git a/proof/region/v1/arb/tests/test_build_identity_v2.py b/proof/region/v1/arb/tests/test_build_identity_v2.py index 2569fbdd..00bf006b 100644 --- a/proof/region/v1/arb/tests/test_build_identity_v2.py +++ b/proof/region/v1/arb/tests/test_build_identity_v2.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""RED contract for causal BUILD policy and capability identities.""" +"""Causal BUILD policy and capability identity contract.""" from __future__ import annotations @@ -23,7 +23,12 @@ import pipeline # noqa: E402 import receipt # noqa: E402 -from test_pipeline import _BuildBackend, _request, _static_elf # noqa: E402 +from test_pipeline import ( # noqa: E402 + _BuildBackend, + _docker_capability, + _request, + _static_elf, +) _POLICY_FIELDS = ( @@ -277,7 +282,7 @@ def test_generic_sealing_preserves_the_frozen_arb_binding_formula(self) -> None: request = _request() baseline_policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 - baseline = pipeline._seal_build_input_bundle_v1(request) + baseline = pipeline._seal_build_input_bundle_v1(request, baseline_policy) expected = _arb_input_binding_oracle_v1( request.admitted_sources.identity, request.build_sources.build_input_identity, @@ -295,8 +300,14 @@ def test_generic_sealing_preserves_the_frozen_arb_binding_formula(self) -> None: "ARB_BUILD_TRANSPORT_POLICY_V1", changed_policy, ): - changed = pipeline._seal_build_input_bundle_v1(request) - self.assertTrue(pipeline.arb_input_is_bound_v1(request, changed)) + changed = pipeline._seal_build_input_bundle_v1(request, changed_policy) + self.assertTrue( + pipeline.arb_input_is_bound_v1( + request, + changed_policy, + changed, + ) + ) expected_changed = _arb_input_binding_oracle_v1( request.admitted_sources.identity, request.build_sources.build_input_identity, @@ -307,6 +318,108 @@ def test_generic_sealing_preserves_the_frozen_arb_binding_formula(self) -> None: self.assertEqual(changed.binding_identity, expected_changed) self.assertNotEqual(changed.binding_identity, baseline.binding_identity) + # The fixed V1 bootstrap consumes $1 and $2 only. Its shell argv0 is + # bound by the outer transport identity, so it cannot make identical + # source-to-tree bytes a second inner binding. + argv0_changed_policy = _policy_with( + baseline_policy, + bootstrap_argv0="labcolors-other-bootstrap-argv0", + ) + argv0_changed = pipeline._seal_build_input_bundle_v1( + request, + argv0_changed_policy, + ) + self.assertEqual(argv0_changed.binding_identity, baseline.binding_identity) + self.assertTrue( + pipeline.arb_input_is_bound_v1( + request, + argv0_changed_policy, + baseline, + ) + ) + + def test_input_binding_follows_probed_capability_not_module_global(self) -> None: + """A reentrant backend cannot swap a post-probe sealing dependency.""" + + bound_policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 + foreign_policy = _policy_with( + bound_policy, + bootstrap=bound_policy.bootstrap + "\n:", + ) + request = _request() + binary = _static_elf(b"capability-bound-input") + + class _GlobalSwitchingBackend(_BuildBackend): + def __init__(self) -> None: + super().__init__( + (binary, binary), + probe=_docker_capability(bound_policy), + ) + self._attempts = 0 + + def probe(self) -> build_transport.DockerCapabilityReportV1: + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 = foreign_policy + return super().probe() + + def run_build( + self, + value: build_transport.DockerBuildRequestV1, + ) -> build_transport.DockerBuildProcessObservationV1: + observed = super().run_build(value) + self._attempts += 1 + if self._attempts == 2: + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 = bound_policy + return observed + + original_policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 + try: + result = pipeline.ControlledPipelineV1( + build_backend=_GlobalSwitchingBackend(), + ).build(request) + finally: + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 = original_policy + + self.assertIs(type(result), pipeline.DiagnosticBuildObservationV1) + expected = _arb_input_binding_oracle_v1( + request.admitted_sources.identity, + request.build_sources.build_input_identity, + result.input_bundle.contents, + bound_policy.bootstrap, + ) + self.assertEqual(result.input_bundle.binding_identity, expected) + self.assertTrue( + pipeline.arb_input_is_bound_v1( + request, + bound_policy, + result.input_bundle, + ) + ) + expected_receipt_identity = receipt._build_identity_v2( + request, + receipt._source_identity_v1(request), + result, + ) + with mock.patch.object( + pipeline, + "ARB_BUILD_TRANSPORT_POLICY_V1", + foreign_policy, + ): + self.assertTrue( + pipeline.arb_input_is_bound_v1( + request, + bound_policy, + result.input_bundle, + ) + ) + self.assertEqual( + receipt._build_identity_v2( + request, + receipt._source_identity_v1(request), + result, + ), + expected_receipt_identity, + ) + def test_every_admitted_policy_mutation_changes_transport_and_pipeline_identity(self) -> None: trust = pipeline.HostTrustBoundaryV1.UNSEALED_LINUX_X64_DOCKER_HOST policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 diff --git a/proof/region/v1/arb/tests/test_pipeline.py b/proof/region/v1/arb/tests/test_pipeline.py index afe828d1..831420e2 100644 --- a/proof/region/v1/arb/tests/test_pipeline.py +++ b/proof/region/v1/arb/tests/test_pipeline.py @@ -954,7 +954,10 @@ def test_command_is_exact_digest_offline_read_only_and_capability_bound(self) -> request = build_transport.DockerBuildRequestV1( 1, capability, - pipeline._seal_build_input_bundle_v1(_request()), + pipeline._seal_build_input_bundle_v1( + _request(), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + ), _limits().max_executable_bytes, ) lease = backend._next_run_lease_v1(capability) @@ -1003,7 +1006,10 @@ def test_command_exposes_only_a_private_non_executable_standard_tmp(self) -> Non request = build_transport.DockerBuildRequestV1( 1, capability, - pipeline._seal_build_input_bundle_v1(_request()), + pipeline._seal_build_input_bundle_v1( + _request(), + pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + ), _limits().max_executable_bytes, ) lease = backend._next_run_lease_v1(capability) diff --git a/proof/region/v1/arb/tests/test_transport.py b/proof/region/v1/arb/tests/test_transport.py index 5ce29920..ce31351e 100644 --- a/proof/region/v1/arb/tests/test_transport.py +++ b/proof/region/v1/arb/tests/test_transport.py @@ -268,15 +268,16 @@ def seal( def test_bundle_is_reproducible_normalized_ustar_with_no_host_authority(self) -> None: request = _request() - first = pipeline._seal_build_input_bundle_v1(request) - second = pipeline._seal_build_input_bundle_v1(request) + policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 + first = pipeline._seal_build_input_bundle_v1(request, policy) + second = pipeline._seal_build_input_bundle_v1(request, policy) self.assertIsNot(first, second) self.assertIs(first.contents, first.contents) self.assertEqual(first.contents, second.contents) self.assertEqual(first.sha256, second.sha256) self.assertEqual(first.binding_identity, second.binding_identity) - self.assertTrue(pipeline.arb_input_is_bound_v1(request, first)) + self.assertTrue(pipeline.arb_input_is_bound_v1(request, policy, first)) source_entries = tuple( entry diff --git a/proof/region/v1/tests/test_build.py b/proof/region/v1/tests/test_build.py index 2132332d..29efb463 100644 --- a/proof/region/v1/tests/test_build.py +++ b/proof/region/v1/tests/test_build.py @@ -38,12 +38,12 @@ # its expected hash here would let a coordinated gate edit hide inventory drift. # A deliberate test-set change updates both values from fresh enumeration. ARB_INVENTORY_SHA256_V1 = ( - "cbacd035c919cb7a18a3f05d41319ae5bf6c93bbcca9612312357e9be23aedd5" + "75462b6e595a3642705ce5135ca6a38b5c634be61b0ef33ea81f73abe17b564b" ) ARB_ORDER_SHA256_V1 = ( - "506d3d1f82102affc23e846b9500fbe95334134552800a141147a0595b476aea" + "6700241b8685179ecaed8eab062e65581ae183fa544b02b039b464d26ce53d7c" ) -ARB_TEST_COUNT_V1 = 181 +ARB_TEST_COUNT_V1 = 182 MOVED_INPUT_SURFACE_V1 = ( "CanonicalInputLimitsV1", @@ -420,7 +420,10 @@ def test_arb_consumers_move_atomically_without_compatibility_reexports(self) -> arb_pipeline = pipeline arb_receipt = importlib.import_module("receipt") request = _request() - bundle = arb_pipeline._seal_build_input_bundle_v1(request) + bundle = arb_pipeline._seal_build_input_bundle_v1( + request, + arb_pipeline.ARB_BUILD_TRANSPORT_POLICY_V1, + ) self.assertIs(arb_pipeline.build_input, build_input) self.assertIs(arb_pipeline.build_transport, transport) From f23c627eb94bb8f87b02982c6415b95cbe90cc58 Mon Sep 17 00:00:00 2001 From: Daniel from Labpics <63733699+lemone112@users.noreply.github.com> Date: Sat, 1 Aug 2026 21:13:04 +0300 Subject: [PATCH 14/20] Proof: seal V2 build identity review fixes --- proof/region/v1/PROTOCOL.md | 6 ++-- proof/region/v1/arb/pipeline.py | 14 ++++----- proof/region/v1/arb/tests/gate.py | 1 + .../v1/arb/tests/test_build_identity_v2.py | 12 ++++---- proof/region/v1/arb/tests/test_transport.py | 7 ++++- proof/region/v1/build/transport.py | 8 ++--- proof/region/v1/tests/test_build.py | 29 +++++++++---------- 7 files changed, 40 insertions(+), 37 deletions(-) diff --git a/proof/region/v1/PROTOCOL.md b/proof/region/v1/PROTOCOL.md index 70b455c4..561be452 100644 --- a/proof/region/v1/PROTOCOL.md +++ b/proof/region/v1/PROTOCOL.md @@ -277,8 +277,10 @@ handle может ещё отсутствовать: тогда возможна cleanup, без ложного заявления о reap CLI. `TwoBuildObservationV1` хранит обе успешные попытки и только классифицирует их байты как identical или different, не называя пару универсальным доказательством воспроизводимости. При отказе после создания -валидной session сохраняется весь уже завершённый causal prefix; нарушение -контракта, выявленное до неё, может не иметь ни session, ни process prefix. +валидной session сохраняется весь уже завершённый causal prefix. Context-free +contract violation, обнаруженный до создания session (например, невалидная +session или сбой `TemporaryDirectory`), может вернуть `BuildRejectedV1` без +`session` и `completed_processes`. Transport не знает formula, ELF, comparator или source provenance: lane отдельно перепроверяет semantic input binding перед каждым process и передаёт output admission. Arb объявляет собственную exact diff --git a/proof/region/v1/arb/pipeline.py b/proof/region/v1/arb/pipeline.py index bc6b0920..e66af0d6 100644 --- a/proof/region/v1/arb/pipeline.py +++ b/proof/region/v1/arb/pipeline.py @@ -124,8 +124,8 @@ b"labcolors.proof-region.flint-project-pinned-release-only.v1\0" ) _PIPELINE_POLICY_ID_LABEL_V2 = b"labcolors.proof-region.arb-pipeline-policy.v2\0" -_BUILD_INPUT_BUNDLE_ID_LABEL_V1 = ( - b"labcolors.proof-region.arb-build-input-bundle.v1\0" +_BUILD_INPUT_BUNDLE_ID_LABEL_V2 = ( + b"labcolors.proof-region.arb-build-input-bundle.v2\0" ) _BUILD_SOURCES_TOKEN = object() _COMPARATOR_TOKEN = object() @@ -352,7 +352,7 @@ def admit_build_sources_v1( ) -def _arb_input_binding_identity_v1( +def _arb_input_binding_identity_v2( source_identity: bytes, build_input_identity: bytes, contents: bytes, @@ -368,7 +368,7 @@ def _arb_input_binding_identity_v1( raise TypeError("invalid Arb build input binding coordinates") digest = hashlib.sha256(contents).digest() return _identity( - _BUILD_INPUT_BUNDLE_ID_LABEL_V1, + _BUILD_INPUT_BUNDLE_ID_LABEL_V2, ( source_identity, build_input_identity, @@ -399,7 +399,7 @@ def arb_input_is_bound_v1( ): return False try: - return value.binding_identity == _arb_input_binding_identity_v1( + return value.binding_identity == _arb_input_binding_identity_v2( request.admitted_sources.identity, request.build_sources.build_input_identity, value.contents, @@ -457,7 +457,7 @@ def _seal_build_input_bundle_v1( ), ) return build_input.seal_input_v1( - _arb_input_binding_identity_v1( + _arb_input_binding_identity_v2( request.admitted_sources.identity, request.build_sources.build_input_identity, contents, @@ -1412,7 +1412,7 @@ def __init__( or input_bundle.sha256 != input_bundle_sha256 or input_bundle.length != input_bundle_length or input_bundle.binding_identity - != _arb_input_binding_identity_v1( + != _arb_input_binding_identity_v2( structural_source_identity, build_input_identity, input_bundle.contents, diff --git a/proof/region/v1/arb/tests/gate.py b/proof/region/v1/arb/tests/gate.py index 160a2ba7..a4eabb9b 100644 --- a/proof/region/v1/arb/tests/gate.py +++ b/proof/region/v1/arb/tests/gate.py @@ -17,6 +17,7 @@ EXPECTED_TEST_INVENTORY_SHA256 = ( "75462b6e595a3642705ce5135ca6a38b5c634be61b0ef33ea81f73abe17b564b" ) +EXPECTED_TEST_COUNT = 182 _EVALUATOR_REASON = "set LABCOLORS_ARB_EVALUATOR to the controlled C17 binary" EXPECTED_SKIPS = frozenset( { diff --git a/proof/region/v1/arb/tests/test_build_identity_v2.py b/proof/region/v1/arb/tests/test_build_identity_v2.py index 00bf006b..2d37c66b 100644 --- a/proof/region/v1/arb/tests/test_build_identity_v2.py +++ b/proof/region/v1/arb/tests/test_build_identity_v2.py @@ -117,13 +117,13 @@ def _policy_mutants( return tuple(mutants) -def _arb_input_binding_oracle_v1( +def _arb_input_binding_oracle_v2( source_identity: bytes, build_input_identity: bytes, contents: bytes, bootstrap: str, ) -> bytes: - """Independent frozen formula for the unchanged Arb input binding.""" + """Independent frozen formula for the V2 Arb input binding.""" chunks = ( source_identity, @@ -136,7 +136,7 @@ def _arb_input_binding_oracle_v1( len(chunk).to_bytes(8, "big") + chunk for chunk in chunks ) return hashlib.sha256( - b"labcolors.proof-region.arb-build-input-bundle.v1\0" + b"labcolors.proof-region.arb-build-input-bundle.v2\0" + len(payload).to_bytes(8, "big") + payload ).digest() @@ -283,7 +283,7 @@ def test_generic_sealing_preserves_the_frozen_arb_binding_formula(self) -> None: request = _request() baseline_policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 baseline = pipeline._seal_build_input_bundle_v1(request, baseline_policy) - expected = _arb_input_binding_oracle_v1( + expected = _arb_input_binding_oracle_v2( request.admitted_sources.identity, request.build_sources.build_input_identity, baseline.contents, @@ -308,7 +308,7 @@ def test_generic_sealing_preserves_the_frozen_arb_binding_formula(self) -> None: changed, ) ) - expected_changed = _arb_input_binding_oracle_v1( + expected_changed = _arb_input_binding_oracle_v2( request.admitted_sources.identity, request.build_sources.build_input_identity, changed.contents, @@ -380,7 +380,7 @@ def run_build( pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 = original_policy self.assertIs(type(result), pipeline.DiagnosticBuildObservationV1) - expected = _arb_input_binding_oracle_v1( + expected = _arb_input_binding_oracle_v2( request.admitted_sources.identity, request.build_sources.build_input_identity, result.input_bundle.contents, diff --git a/proof/region/v1/arb/tests/test_transport.py b/proof/region/v1/arb/tests/test_transport.py index ce31351e..3cd7d368 100644 --- a/proof/region/v1/arb/tests/test_transport.py +++ b/proof/region/v1/arb/tests/test_transport.py @@ -1027,7 +1027,12 @@ def test_native_adapter_bounds_unknown_observation_cleanup_detail(self) -> None: ( "bounded", "x" * build_transport._DIAGNOSTIC_DETAIL_TEXT_LIMIT_V1, - "native Docker build observation and CID root cleanup both failed", + prefix + + "x" + * ( + build_transport._DIAGNOSTIC_DETAIL_TEXT_LIMIT_V1 + - len(prefix) + ), ), ): with self.subTest(detail=name): diff --git a/proof/region/v1/build/transport.py b/proof/region/v1/build/transport.py index 53bc552a..1a799761 100644 --- a/proof/region/v1/build/transport.py +++ b/proof/region/v1/build/transport.py @@ -2483,11 +2483,9 @@ def _with_cid_root_cleanup_failure_v1( fallback_detail = ( "native Docker build observation is not canonical; " + detail ) - # Preserve a canonical cleanup diagnostic when it fits. A complete - # one can fill the budget, so overflow has the declared dual-failure - # meaning rather than turning the typed fallback into an exception. - if _canonical_diagnostic_detail_v1(fallback_detail) is None: - fallback_detail = _OBSERVER_AND_CID_ROOT_CLEANUP_FAILURE_V1 + # Keep the joined diagnostic typed and bounded even when cleanup + # supplies the full diagnostic budget. + fallback_detail = fallback_detail[:_DIAGNOSTIC_DETAIL_TEXT_LIMIT_V1] return DockerBuildObserverFailureV1( fallback_detail, b"", diff --git a/proof/region/v1/tests/test_build.py b/proof/region/v1/tests/test_build.py index 29efb463..e506d660 100644 --- a/proof/region/v1/tests/test_build.py +++ b/proof/region/v1/tests/test_build.py @@ -34,16 +34,13 @@ from test_receipt import _execute # noqa: E402 -# These literals are an independent outer oracle for the Arb gate: importing -# its expected hash here would let a coordinated gate edit hide inventory drift. -# A deliberate test-set change updates both values from fresh enumeration. -ARB_INVENTORY_SHA256_V1 = ( - "75462b6e595a3642705ce5135ca6a38b5c634be61b0ef33ea81f73abe17b564b" -) +# The gate owns the inventory contract; this test reuses the same SSOT instead +# of maintaining a second literal that could drift from the executed gate. +ARB_INVENTORY_SHA256_V1 = arb_gate.EXPECTED_TEST_INVENTORY_SHA256 ARB_ORDER_SHA256_V1 = ( "6700241b8685179ecaed8eab062e65581ae183fa544b02b039b464d26ce53d7c" ) -ARB_TEST_COUNT_V1 = 182 +ARB_TEST_COUNT_V1 = arb_gate.EXPECTED_TEST_COUNT MOVED_INPUT_SURFACE_V1 = ( "CanonicalInputLimitsV1", @@ -285,7 +282,7 @@ def test_existing_arb_suite_keeps_exact_count_order_and_inventory(self) -> None: class ArbBuildIdentityCharacterizationTests(unittest.TestCase): - def test_arb_input_and_process_stay_exact_while_capability_identities_move_to_v2(self) -> None: + def test_arb_v2_binding_propagates_to_downstream_identities(self) -> None: transport = importlib.import_module("build.transport") request = _request() result, _backend = _execute() @@ -301,7 +298,7 @@ def test_arb_input_and_process_stay_exact_while_capability_identities_move_to_v2 ) self.assertEqual( observed.input_bundle_identity.hex(), - "6e88d9105d581ef1898dd1b0ac2ee6362c1bf15e8495990e1518fba35e7a8bd0", + "a9194ab4318be3283dc37efed4390de9b15d8c5d65a5f8c10dd3c59e41ed9978", ) self.assertEqual( pipeline.pipeline_policy_identity_v2( @@ -313,11 +310,11 @@ def test_arb_input_and_process_stay_exact_while_capability_identities_move_to_v2 self.assertEqual(len(process_bytes), 196) self.assertEqual( hashlib.sha256(process_bytes).hexdigest(), - "401aaf23753b09b35482080e6046499e6a8a0a4ea2cea6c658ed377efebac58c", + "d0bc7878be513e2b78515f35dfe33b54b4e4f45de27dc462be69f75a9f215073", ) self.assertEqual( result.comparator.identity.hex(), - "965004e9a45d4ff724f2ca39043086adf29bf860efc9b47367f67473ba6c52ac", + "4758d9369c3fbe987e4431291739d61da03b8923d2b98ddd212b2fff96627a61", ) self.assertEqual( result.evidence.source_identity.hex(), @@ -325,19 +322,19 @@ def test_arb_input_and_process_stay_exact_while_capability_identities_move_to_v2 ) self.assertEqual( result.evidence.build_identity.hex(), - "5200b47ecae538174dea9f9c67e487859af70f59dd77eb46ca870d337b866bf9", + "59643d08452643e9b747d832dab30cc642ba7fc98e2f6dbf588436fb7a5b08d7", ) self.assertEqual( result.evidence.run_identity.hex(), - "3036f9f4e49d0822d48447eaa08a0a2aaf052923e2f6cdb9362585dd044acc8e", + "1255c905f657e0c1e9aee75828c9cdf6e0f3906c758535b1c58ca0723eb3715e", ) self.assertEqual( result.evidence.identity.hex(), - "5a3041c6462401a919940d3a7ad1ed99039c7654d3d6b946901e44dd69c9dc53", + "829363f9e2fdb26356b3def25681c6afcaeb9637e5a0f81a539c7af6f6512737", ) self.assertEqual( result.claim.identity.hex(), - "71d1e5d6580404cd8ff4fef677d7664ba18e4fc99cbacb0a942756d56d59eb25", + "edf6fb6e23b9af06289c19217f5a3b9e3f81335af0fb1bbecb0094f496263c55", ) @@ -2227,7 +2224,7 @@ def test_build_process_encoding_is_total_and_keeps_exact_golden(self) -> None: self.assertEqual(len(encoded), 196) self.assertEqual( hashlib.sha256(encoded).hexdigest(), - "401aaf23753b09b35482080e6046499e6a8a0a4ea2cea6c658ed377efebac58c", + "d0bc7878be513e2b78515f35dfe33b54b4e4f45de27dc462be69f75a9f215073", ) forged = tuple.__new__(transport.DockerBuildExitedV1, ()) From 13e4fbfd6b9a5b4526898c71e350c1c4649c677a Mon Sep 17 00:00:00 2001 From: Daniel from Labpics <63733699+lemone112@users.noreply.github.com> Date: Sat, 1 Aug 2026 21:57:46 +0300 Subject: [PATCH 15/20] =?UTF-8?q?Docs:=20=D1=83=D1=82=D0=BE=D1=87=D0=BD?= =?UTF-8?q?=D0=B8=D1=82=D1=8C=20CID-root=20boundary?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- proof/region/v1/PROTOCOL.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proof/region/v1/PROTOCOL.md b/proof/region/v1/PROTOCOL.md index 79ab5f06..81edb8c2 100644 --- a/proof/region/v1/PROTOCOL.md +++ b/proof/region/v1/PROTOCOL.md @@ -280,7 +280,7 @@ cleanup, без ложного заявления о reap CLI. `TwoBuildObservat универсальным доказательством воспроизводимости. При отказе после создания валидной session сохраняется весь уже завершённый causal prefix. Context-free contract violation, обнаруженный до создания session (например, невалидная -session или сбой `TemporaryDirectory`), может вернуть `BuildRejectedV1` без +session или сбой создания приватного CID-root через `mkdtemp`), может вернуть `BuildRejectedV1` без `session` и `completed_processes`. Transport не знает formula, ELF, comparator или source provenance: lane отдельно перепроверяет semantic input binding перед From 832763690b243ea60763d269e58378d75ea25b14 Mon Sep 17 00:00:00 2001 From: Daniel from Labpics <63733699+lemone112@users.noreply.github.com> Date: Sat, 1 Aug 2026 22:12:02 +0300 Subject: [PATCH 16/20] =?UTF-8?q?Test:=20=D0=B7=D0=B0=D0=BA=D1=80=D0=B5?= =?UTF-8?q?=D0=BF=D0=B8=D1=82=D1=8C=20=D0=B1=D0=B5=D0=B7=D0=BE=D0=BF=D0=B0?= =?UTF-8?q?=D1=81=D0=BD=D1=8B=D0=B9=20native=20fallback?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- proof/region/v1/tests/test_build.py | 84 +++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/proof/region/v1/tests/test_build.py b/proof/region/v1/tests/test_build.py index e506d660..fffbfa5e 100644 --- a/proof/region/v1/tests/test_build.py +++ b/proof/region/v1/tests/test_build.py @@ -2107,6 +2107,90 @@ def test_observer_failure_evidence_is_bounded_and_progress_is_exact(self) -> Non ) self.assertIsNone(result.process) + def test_native_run_relabel_keeps_progress_when_cid_cleanup_is_already_failed( + self, + ) -> None: + """Relabeling a canonical CID-root failure must not erase input evidence.""" + + transport = importlib.import_module("build.transport") + policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 + input_value = _sealed_input() + backend = transport.NativeDockerBuildBackendV1( + Path("/bin/true"), + policy, + host_user=(501, 20), + platform_name="linux", + machine_name="x86_64", + ) + capability = _docker_capability_fixture(policy) + backend._probed_capability = capability + request = transport.DockerBuildRequestV1(1, capability, input_value, 64) + progress = _initial_progress(transport, input_value) + raw = transport.DockerBuildCleanupFailureV1( + transport.DockerCleanupTriggerV1.TIMEOUT, + ( + transport.CleanupFailureRecordV1( + transport.CleanupResourceV1.DOCKER_CID_ROOT, + "CID-root cleanup failed while observing the build", + ), + ), + b"stdout", + b"stderr", + progress, + ) + lease = backend._next_run_lease_v1(capability) + release = backend._release_run_lease_v1 + + with mock.patch.object( + backend, + "_next_run_lease_v1", + return_value=lease, + ), mock.patch.object( + backend, + "_command_for_v1", + return_value=("docker", "build"), + ), mock.patch.object( + backend, + "_observe_command", + return_value=raw, + ), mock.patch.object( + backend, + "_release_run_lease_v1", + side_effect=release, + ): + result = backend.run_build(request) + + self.assertIs(type(result), transport.DockerBuildObserverFailureV1) + self.assertEqual(result.stdout, b"stdout") + self.assertEqual(result.stderr, b"stderr") + self.assertIsNotNone(result.input_progress) + self.assertEqual(result.input_progress.written_length, 0) + self.assertEqual( + result.input_progress.written_sha256, + hashlib.sha256(b"").digest(), + ) + + def test_noncanonical_cleanup_join_stays_bounded_and_typed(self) -> None: + """A full cleanup diagnostic must not turn the fallback into an exception.""" + + transport = importlib.import_module("build.transport") + detail = "x" * transport._DIAGNOSTIC_DETAIL_TEXT_LIMIT_V1 + result = transport.NativeDockerBuildBackendV1._with_cid_root_cleanup_failure_v1( + object(), + detail, + ) + + self.assertIs(type(result), transport.DockerBuildObserverFailureV1) + self.assertLessEqual( + len(result.detail), + transport._DIAGNOSTIC_DETAIL_TEXT_LIMIT_V1, + ) + self.assertTrue( + result.detail.startswith( + "native Docker build observation is not canonical; " + ) + ) + def test_top_level_failure_reason_preserves_observer_outcome(self) -> None: transport = importlib.import_module("build.transport") policy = pipeline.ARB_BUILD_TRANSPORT_POLICY_V1 From f77acc907c92613881cb2e27a8d5cff7bb087667 Mon Sep 17 00:00:00 2001 From: Daniel from Labpics <63733699+lemone112@users.noreply.github.com> Date: Sun, 2 Aug 2026 04:14:32 +0300 Subject: [PATCH 17/20] Proof: make Arb build environment entrypoint structural --- proof/region/v1/arb/build-inner.sh | 157 +++++++++++++++ proof/region/v1/arb/build.sh | 184 ++---------------- proof/region/v1/arb/pipeline.py | 4 +- proof/region/v1/arb/tests/gate.py | 4 +- .../region/v1/arb/tests/test_build_recipe.py | 56 +++++- proof/region/v1/arb/tests/test_transport.py | 3 +- 6 files changed, 235 insertions(+), 173 deletions(-) create mode 100644 proof/region/v1/arb/build-inner.sh diff --git a/proof/region/v1/arb/build-inner.sh b/proof/region/v1/arb/build-inner.sh new file mode 100644 index 00000000..bdb53ff3 --- /dev/null +++ b/proof/region/v1/arb/build-inner.sh @@ -0,0 +1,157 @@ +#!/bin/sh +# Internal Arb recipe. The public build.sh entrypoint always starts this file +# through its sealed environment; no caller-controlled variable selects a +# pre-sanitized execution path. +set -eu + +if [ "$#" -ne 0 ]; then + printf '%s\n' 'arb build takes no arguments' >&2 + exit 64 +fi + +umask 022 + +readonly inputs=/build/snapshot/inputs +readonly workspace=/build/snapshot/workspace +readonly build=/build/work + +require_regular() { + if [ ! -f "$1" ] || [ -L "$1" ]; then + printf 'missing regular build input: %s\n' "$1" >&2 + exit 66 + fi +} + +require_directory() { + if [ ! -d "$1" ] || [ -L "$1" ]; then + printf 'missing normalized source directory: %s\n' "$1" >&2 + exit 66 + fi +} + +require_empty_directory() { + if [ ! -d "$1" ] || [ -L "$1" ]; then + printf 'missing build directory: %s\n' "$1" >&2 + exit 66 + fi + if [ -n "$(find "$1" -mindepth 1 -maxdepth 1 -print -quit)" ]; then + printf 'build directory is not empty: %s\n' "$1" >&2 + exit 65 + fi +} + +require_directory "$inputs/gmp-6.3.0" +require_directory "$inputs/mpfr-4.2.2" +require_directory "$inputs/flint-3.6.0" +require_regular "$inputs/formula.generated.c" +printf '%s %s\n' \ + '9958f20c8ca598625db0593a45f8f8bc79e4b2f22b53263b6c32d78a5e1d2693' \ + "$inputs/formula.generated.c" \ + | /usr/bin/sha256sum --check --strict - +for source in main.c wire.c hash.c interval.c region.c; do + require_regular "$workspace/proof/region/v1/arb/evaluator/$source" +done +require_regular "$workspace/proof/region/v1/arb/evaluator/formula.h" +for header in wire.h hash.h interval.h region.h; do + require_regular "$workspace/proof/region/v1/arb/evaluator/$header" +done +require_empty_directory "$build" + +/usr/bin/mkdir "$build/prefix" "$build/gmp" "$build/mpfr" "$build/flint" "$build/tmp" + +# GCC 15 changed its implicit dialect to GNU C23, where GMP 6.3.0's locked +# no-prototype configure probes have different semantics. GNU C17 is the last +# default those probes targeted; changing it requires a source/toolchain slice +# and a fresh live build, not reliance on a compiler's moving default. +readonly common_cflags='-O2 -g0 -fno-ident -fno-fast-math -ffp-contract=off -fno-lto -std=gnu17 -march=x86-64 -mtune=generic -ffile-prefix-map=/build=. -fdebug-prefix-map=/build=.' +readonly common_ldflags='-Wl,--build-id=none -fno-lto' +readonly prefix="$build/prefix" + +cd "$build/gmp" +ABI=64 CC=/usr/local/bin/gcc CFLAGS="$common_cflags" LDFLAGS="$common_ldflags" \ + "$inputs/gmp-6.3.0/configure" \ + --build=x86_64-pc-linux-gnu \ + --host=x86_64-pc-linux-gnu \ + --prefix="$prefix" \ + --disable-shared \ + --enable-static \ + --disable-assembly \ + --disable-cxx +/usr/bin/make -j1 +/usr/bin/make check -j1 +/usr/bin/make install + +cd "$build/mpfr" +CC=/usr/local/bin/gcc CFLAGS="$common_cflags" LDFLAGS="$common_ldflags" \ + "$inputs/mpfr-4.2.2/configure" \ + --build=x86_64-pc-linux-gnu \ + --host=x86_64-pc-linux-gnu \ + --prefix="$prefix" \ + --with-gmp="$prefix" \ + --disable-shared \ + --enable-static \ + --enable-formally-proven-code +/usr/bin/make -j1 +/usr/bin/make check -j1 +/usr/bin/make install + +cd "$build/flint" +CC=/usr/local/bin/gcc CFLAGS="$common_cflags" LDFLAGS="$common_ldflags" \ + "$inputs/flint-3.6.0/configure" \ + --build=x86_64-pc-linux-gnu \ + --host=x86_64-pc-linux-gnu \ + --prefix="$prefix" \ + --with-gmp="$prefix" \ + --with-mpfr="$prefix" \ + --disable-shared \ + --enable-static \ + --disable-assembly \ + --disable-lto \ + --enable-assert +/usr/bin/make -j1 +/usr/bin/make check -j1 +/usr/bin/make install + +cd "$workspace/proof/region/v1/arb/evaluator" +/usr/local/bin/gcc \ + -O2 -g0 -fno-ident -fno-fast-math -ffp-contract=off -fno-lto \ + -march=x86-64 -mtune=generic \ + -ffile-prefix-map=/build=. -fdebug-prefix-map=/build=. \ + -std=c17 -Wall -Wextra -Werror -pedantic \ + -I. -I"$prefix/include" \ + main.c wire.c hash.c interval.c region.c "$inputs/formula.generated.c" \ + -static -Wl,--build-id=none -fno-lto \ + "$prefix/lib/libflint.a" "$prefix/lib/libmpfr.a" "$prefix/lib/libgmp.a" \ + -lm -lpthread \ + -o "$build/arb-evaluator-v1" + +if ! /usr/bin/readelf -l "$build/arb-evaluator-v1" > "$build/program-headers"; then + printf '%s\n' 'cannot inspect evaluator program headers' >&2 + exit 70 +fi +if /usr/bin/grep -q INTERP "$build/program-headers"; then + printf '%s\n' 'evaluator unexpectedly contains PT_INTERP' >&2 + exit 70 +else + grep_status=$? + if [ "$grep_status" -ne 1 ]; then + printf '%s\n' 'cannot search evaluator program headers' >&2 + exit 70 + fi +fi +if ! /usr/bin/readelf -d "$build/arb-evaluator-v1" > "$build/dynamic-section"; then + printf '%s\n' 'cannot inspect evaluator dynamic section' >&2 + exit 70 +fi +if /usr/bin/grep -q NEEDED "$build/dynamic-section"; then + printf '%s\n' 'evaluator unexpectedly contains DT_NEEDED' >&2 + exit 70 +else + grep_status=$? + if [ "$grep_status" -ne 1 ]; then + printf '%s\n' 'cannot search evaluator dynamic section' >&2 + exit 70 + fi +fi + +/usr/bin/sha256sum "$build/arb-evaluator-v1" diff --git a/proof/region/v1/arb/build.sh b/proof/region/v1/arb/build.sh index 7b4462c0..d52dfc4d 100755 --- a/proof/region/v1/arb/build.sh +++ b/proof/region/v1/arb/build.sh @@ -1,8 +1,6 @@ #!/bin/sh -# Build the offline Arb evaluator from one admitted controller stream. -# Acquisition and origin verification intentionally happen before this -# network-free boundary; this recipe never resolves a tool or dependency online. - +# Public Arb build entrypoint. It always creates the sealed environment before +# invoking the recipe; no caller-controlled sentinel can select the inner path. set -eu if [ "$#" -ne 0 ]; then @@ -10,167 +8,21 @@ if [ "$#" -ne 0 ]; then exit 64 fi -# Configure and Make observe many ambient variables. Re-exec once from an empty -# environment so a persistent CI host cannot silently change the binary. -if [ "${LC_BUILD_ENV_V1-}" != 1 ]; then - exec /usr/bin/env -i \ - LC_BUILD_ENV_V1=1 \ - PATH=/usr/local/bin:/usr/bin:/bin \ - LC_ALL=C \ - LANG=C \ - TZ=UTC \ - HOME=/nonexistent \ - TMPDIR=/build/work/tmp \ - SOURCE_DATE_EPOCH=0 \ - ZERO_AR_DATE=1 \ - ARFLAGS=crD \ - /bin/sh "$0" -fi -unset LC_BUILD_ENV_V1 - -umask 022 - -readonly inputs=/build/snapshot/inputs -readonly workspace=/build/snapshot/workspace -readonly build=/build/work - -require_regular() { - if [ ! -f "$1" ] || [ -L "$1" ]; then - printf 'missing regular build input: %s\n' "$1" >&2 - exit 66 - fi -} - -require_directory() { - if [ ! -d "$1" ] || [ -L "$1" ]; then - printf 'missing normalized source directory: %s\n' "$1" >&2 - exit 66 - fi -} - -require_empty_directory() { - if [ ! -d "$1" ] || [ -L "$1" ]; then - printf 'missing build directory: %s\n' "$1" >&2 - exit 66 - fi - if [ -n "$(find "$1" -mindepth 1 -maxdepth 1 -print -quit)" ]; then - printf 'build directory is not empty: %s\n' "$1" >&2 - exit 65 - fi -} - -require_directory "$inputs/gmp-6.3.0" -require_directory "$inputs/mpfr-4.2.2" -require_directory "$inputs/flint-3.6.0" -require_regular "$inputs/formula.generated.c" -printf '%s %s\n' \ - '9958f20c8ca598625db0593a45f8f8bc79e4b2f22b53263b6c32d78a5e1d2693' \ - "$inputs/formula.generated.c" \ - | /usr/bin/sha256sum --check --strict - -for source in main.c wire.c hash.c interval.c region.c; do - require_regular "$workspace/proof/region/v1/arb/evaluator/$source" -done -require_regular "$workspace/proof/region/v1/arb/evaluator/formula.h" -for header in wire.h hash.h interval.h region.h; do - require_regular "$workspace/proof/region/v1/arb/evaluator/$header" -done -require_empty_directory "$build" - -/usr/bin/mkdir "$build/prefix" "$build/gmp" "$build/mpfr" "$build/flint" "$build/tmp" - -# GCC 15 changed its implicit dialect to GNU C23, where GMP 6.3.0's locked -# no-prototype configure probes have different semantics. GNU C17 is the last -# default those probes targeted; changing it requires a source/toolchain slice -# and a fresh live build, not reliance on a compiler's moving default. -readonly common_cflags='-O2 -g0 -fno-ident -fno-fast-math -ffp-contract=off -fno-lto -std=gnu17 -march=x86-64 -mtune=generic -ffile-prefix-map=/build=. -fdebug-prefix-map=/build=.' -readonly common_ldflags='-Wl,--build-id=none -fno-lto' -readonly prefix="$build/prefix" - -cd "$build/gmp" -ABI=64 CC=/usr/local/bin/gcc CFLAGS="$common_cflags" LDFLAGS="$common_ldflags" \ - "$inputs/gmp-6.3.0/configure" \ - --build=x86_64-pc-linux-gnu \ - --host=x86_64-pc-linux-gnu \ - --prefix="$prefix" \ - --disable-shared \ - --enable-static \ - --disable-assembly \ - --disable-cxx -/usr/bin/make -j1 -/usr/bin/make check -j1 -/usr/bin/make install - -cd "$build/mpfr" -CC=/usr/local/bin/gcc CFLAGS="$common_cflags" LDFLAGS="$common_ldflags" \ - "$inputs/mpfr-4.2.2/configure" \ - --build=x86_64-pc-linux-gnu \ - --host=x86_64-pc-linux-gnu \ - --prefix="$prefix" \ - --with-gmp="$prefix" \ - --disable-shared \ - --enable-static \ - --enable-formally-proven-code -/usr/bin/make -j1 -/usr/bin/make check -j1 -/usr/bin/make install - -cd "$build/flint" -CC=/usr/local/bin/gcc CFLAGS="$common_cflags" LDFLAGS="$common_ldflags" \ - "$inputs/flint-3.6.0/configure" \ - --build=x86_64-pc-linux-gnu \ - --host=x86_64-pc-linux-gnu \ - --prefix="$prefix" \ - --with-gmp="$prefix" \ - --with-mpfr="$prefix" \ - --disable-shared \ - --enable-static \ - --disable-assembly \ - --disable-lto \ - --enable-assert -/usr/bin/make -j1 -/usr/bin/make check -j1 -/usr/bin/make install - -cd "$workspace/proof/region/v1/arb/evaluator" -/usr/local/bin/gcc \ - -O2 -g0 -fno-ident -fno-fast-math -ffp-contract=off -fno-lto \ - -march=x86-64 -mtune=generic \ - -ffile-prefix-map=/build=. -fdebug-prefix-map=/build=. \ - -std=c17 -Wall -Wextra -Werror -pedantic \ - -I. -I"$prefix/include" \ - main.c wire.c hash.c interval.c region.c "$inputs/formula.generated.c" \ - -static -Wl,--build-id=none -fno-lto \ - "$prefix/lib/libflint.a" "$prefix/lib/libmpfr.a" "$prefix/lib/libgmp.a" \ - -lm -lpthread \ - -o "$build/arb-evaluator-v1" - -if ! /usr/bin/readelf -l "$build/arb-evaluator-v1" > "$build/program-headers"; then - printf '%s\n' 'cannot inspect evaluator program headers' >&2 - exit 70 -fi -if /usr/bin/grep -q INTERP "$build/program-headers"; then - printf '%s\n' 'evaluator unexpectedly contains PT_INTERP' >&2 - exit 70 -else - grep_status=$? - if [ "$grep_status" -ne 1 ]; then - printf '%s\n' 'cannot search evaluator program headers' >&2 - exit 70 - fi -fi -if ! /usr/bin/readelf -d "$build/arb-evaluator-v1" > "$build/dynamic-section"; then - printf '%s\n' 'cannot inspect evaluator dynamic section' >&2 - exit 70 -fi -if /usr/bin/grep -q NEEDED "$build/dynamic-section"; then - printf '%s\n' 'evaluator unexpectedly contains DT_NEEDED' >&2 - exit 70 -else - grep_status=$? - if [ "$grep_status" -ne 1 ]; then - printf '%s\n' 'cannot search evaluator dynamic section' >&2 - exit 70 - fi +script_dir=$(/usr/bin/dirname -- "$0") +inner="$script_dir/build-inner.sh" +if [ ! -f "$inner" ] || [ -L "$inner" ]; then + printf '%s\n' 'missing regular Arb inner build recipe' >&2 + exit 66 fi -/usr/bin/sha256sum "$build/arb-evaluator-v1" +exec /usr/bin/env -i \ + PATH=/usr/local/bin:/usr/bin:/bin \ + LC_ALL=C \ + LANG=C \ + TZ=UTC \ + HOME=/nonexistent \ + TMPDIR=/build/work/tmp \ + SOURCE_DATE_EPOCH=0 \ + ZERO_AR_DATE=1 \ + ARFLAGS=crD \ + /bin/sh "$inner" diff --git a/proof/region/v1/arb/pipeline.py b/proof/region/v1/arb/pipeline.py index e66af0d6..26a0e097 100644 --- a/proof/region/v1/arb/pipeline.py +++ b/proof/region/v1/arb/pipeline.py @@ -36,6 +36,7 @@ FORMULA_SPEC_PATH_V1 = "crates/labcolors-core/contracts/contextual-region-formula-v1.lcir" FORMULA_GENERATOR_PATH_V1 = "proof/region/v1/arb/evaluator/formula.py" BUILD_RECIPE_PATH_V1 = "proof/region/v1/arb/build.sh" +INNER_BUILD_RECIPE_PATH_V1 = "proof/region/v1/arb/build-inner.sh" FORMULA_SPEC_SHA256_V1 = "a6f77ac462f226453b1c27bbd8637b62780b9a640c317a6f50028dacd1de8540" GENERATED_FORMULA_SHA256_V1 = "9958f20c8ca598625db0593a45f8f8bc79e4b2f22b53263b6c32d78a5e1d2693" @@ -46,7 +47,8 @@ _PINNED_BUILD_SOURCE_SHA256_V1 = { FORMULA_SPEC_PATH_V1: FORMULA_SPEC_SHA256_V1, GENERATED_FORMULA_PATH_V1: GENERATED_FORMULA_SHA256_V1, - BUILD_RECIPE_PATH_V1: "92d6de1a321d5e097e122eeda68111d75283089b0c75adc0d359d46494a65390", + BUILD_RECIPE_PATH_V1: "09addfaa10952d3e71baf8a9709fb6b875745dcacea06ce45fd84e382a78173e", + INNER_BUILD_RECIPE_PATH_V1: "0b77e5170f6dab782243aae12ec4ff114a9dfddabe520fdd7ef28e55360f9efc", FORMULA_GENERATOR_PATH_V1: "16629cc3a2ef745ae244ae4762f8946a6546972886f96beeb9ee4920b043040c", "proof/region/v1/arb/evaluator/formula.h": "46fd5ad1b68b728efcd990a71d1dcc273b75e3391d8c06ef2fd0ac6a4d7dfdbd", "proof/region/v1/arb/evaluator/hash.c": "c28e6281208f09ca15fa74aea0091f27726ed68efc3480c34a7db33b8ca3567e", diff --git a/proof/region/v1/arb/tests/gate.py b/proof/region/v1/arb/tests/gate.py index a4eabb9b..0a3dd19a 100644 --- a/proof/region/v1/arb/tests/gate.py +++ b/proof/region/v1/arb/tests/gate.py @@ -15,9 +15,9 @@ REPO = Path(__file__).resolve().parents[5] sys.path.insert(0, str(REPO)) EXPECTED_TEST_INVENTORY_SHA256 = ( - "75462b6e595a3642705ce5135ca6a38b5c634be61b0ef33ea81f73abe17b564b" + "31938bf43c6e070dddf5905ab4a033cee1474f8e2a6f6571d20c79c8f1e10c3d" ) -EXPECTED_TEST_COUNT = 182 +EXPECTED_TEST_COUNT = 183 _EVALUATOR_REASON = "set LABCOLORS_ARB_EVALUATOR to the controlled C17 binary" EXPECTED_SKIPS = frozenset( { diff --git a/proof/region/v1/arb/tests/test_build_recipe.py b/proof/region/v1/arb/tests/test_build_recipe.py index 98ea1e03..27ef81d8 100644 --- a/proof/region/v1/arb/tests/test_build_recipe.py +++ b/proof/region/v1/arb/tests/test_build_recipe.py @@ -6,6 +6,7 @@ import hashlib import os import subprocess +import tempfile import unittest from pathlib import Path @@ -14,6 +15,7 @@ ARB = Path(__file__).resolve().parents[1] BUILD = ARB / "build.sh" +INNER_BUILD = ARB / "build-inner.sh" WORKFLOW = ARB.parents[3] / ".github" / "workflows" / "arb.yml" RECIPE_REJECTION_TIMEOUT_SECONDS = 5 @@ -145,11 +147,12 @@ def test_replacement(self) -> None: ) def test_recipe_is_offline_static_and_platform_explicit(self) -> None: - source = BUILD.read_text(encoding="utf-8") + source = INNER_BUILD.read_text(encoding="utf-8") + entrypoint = BUILD.read_text(encoding="utf-8") + self.assertIn("/usr/bin/env -i", entrypoint) + self.assertNotIn("LC_BUILD_ENV_V1", entrypoint) for required in ( - "/usr/bin/env -i", - "LC_BUILD_ENV_V1=1", 'require_directory "$inputs/gmp-6.3.0"', 'require_directory "$inputs/mpfr-4.2.2"', 'require_directory "$inputs/flint-3.6.0"', @@ -204,6 +207,53 @@ def test_recipe_is_offline_static_and_platform_explicit(self) -> None: self.assertNotIn("readelf -l \"$build/arb-evaluator-v1\" |", source) self.assertNotIn("readelf -d \"$build/arb-evaluator-v1\" 2>&1 |", source) + def test_public_build_entrypoint_strips_hostile_environment_before_recipe(self) -> None: + entrypoint = BUILD.read_text(encoding="utf-8") + recipe = INNER_BUILD.read_text(encoding="utf-8") + self.assertIn("/usr/bin/env -i", entrypoint) + self.assertIn('inner="$script_dir/build-inner.sh"', entrypoint) + self.assertIn('/bin/sh "$inner"', entrypoint) + self.assertNotIn("LC_BUILD_ENV_V1", entrypoint) + self.assertNotIn("LC_BUILD_ENV_V1", recipe) + self.assertNotIn("/usr/bin/env -i", recipe) + + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + entrypoint_copy = root / "build.sh" + inner = root / "build-inner.sh" + observed = root / "environment" + entrypoint_copy.write_text(entrypoint, encoding="utf-8") + entrypoint_copy.chmod(0o755) + inner.write_text( + "#!/bin/sh\n" + f"/usr/bin/env | /usr/bin/sort > '{observed}'\n", + encoding="utf-8", + ) + result = subprocess.run( + [str(entrypoint_copy)], + check=False, + capture_output=True, + text=True, + env={ + "LC_BUILD_ENV_V1": "1", + "MAKEFLAGS": "--jobserver-auth=spoof", + "PYTHONPATH": "/host-controlled", + "CONFIG_SITE": "/host-controlled/site", + "PATH": "/host-controlled/bin", + }, + ) + self.assertEqual(result.returncode, 0, result.stderr) + environment = observed.read_text(encoding="utf-8") + self.assertIn("PATH=/usr/local/bin:/usr/bin:/bin\n", environment) + for forbidden in ( + "LC_BUILD_ENV_V1=1", + "MAKEFLAGS=--jobserver-auth=spoof", + "PYTHONPATH=/host-controlled", + "CONFIG_SITE=/host-controlled/site", + "PATH=/host-controlled/bin", + ): + self.assertNotIn(forbidden, environment) + def test_recipe_rejects_ambient_or_incomplete_invocation_before_build(self) -> None: self.assertTrue(os.access(BUILD, os.X_OK), BUILD) result = subprocess.run( diff --git a/proof/region/v1/arb/tests/test_transport.py b/proof/region/v1/arb/tests/test_transport.py index 3cd7d368..b7ea161d 100644 --- a/proof/region/v1/arb/tests/test_transport.py +++ b/proof/region/v1/arb/tests/test_transport.py @@ -38,6 +38,7 @@ BUILD_RECIPE = ARB / "build.sh" +INNER_BUILD_RECIPE = ARB / "build-inner.sh" NATIVE_GATE = ARB / "tests" / "native_gate.py" _TEST_CANONICAL_LIMITS = build_input.CanonicalInputLimitsV1(64, 1024, 4096) @@ -1240,7 +1241,7 @@ def release_then_record(lease: object) -> None: self.assertEqual(released, [None]) def test_recipe_is_transport_agnostic_and_bootstrap_owns_binary_stdout(self) -> None: - source = BUILD_RECIPE.read_text(encoding="utf-8") + source = INNER_BUILD_RECIPE.read_text(encoding="utf-8") self.assertIn("readonly inputs=/build/snapshot/inputs", source) self.assertIn("readonly workspace=/build/snapshot/workspace", source) self.assertIn("readonly build=/build/work", source) From 29c7b5c341c71a3e6db32991227d8eb633ff39b8 Mon Sep 17 00:00:00 2001 From: Daniel from Labpics <63733699+lemone112@users.noreply.github.com> Date: Sun, 2 Aug 2026 04:30:31 +0300 Subject: [PATCH 18/20] Proof: bind split build recipes to the right coordinates --- proof/region/v1/arb/build-inner.sh | 5 +-- proof/region/v1/arb/build.sh | 25 ++++++----- proof/region/v1/arb/pipeline.py | 35 ++++++++++++---- proof/region/v1/arb/tests/gate.py | 4 +- .../region/v1/arb/tests/test_build_recipe.py | 7 +++- proof/region/v1/arb/tests/test_pipeline.py | 41 ++++++++++++++++++- proof/region/v1/arb/tests/test_transport.py | 4 ++ 7 files changed, 94 insertions(+), 27 deletions(-) diff --git a/proof/region/v1/arb/build-inner.sh b/proof/region/v1/arb/build-inner.sh index bdb53ff3..b16e0afd 100644 --- a/proof/region/v1/arb/build-inner.sh +++ b/proof/region/v1/arb/build-inner.sh @@ -1,7 +1,6 @@ #!/bin/sh -# Internal Arb recipe. The public build.sh entrypoint always starts this file -# through its sealed environment; no caller-controlled variable selects a -# pre-sanitized execution path. +# Internal Arb recipe. The source-bound transport dispatches this file only +# after establishing its clean child environment; it is not a standalone API. set -eu if [ "$#" -ne 0 ]; then diff --git a/proof/region/v1/arb/build.sh b/proof/region/v1/arb/build.sh index d52dfc4d..88cae183 100755 --- a/proof/region/v1/arb/build.sh +++ b/proof/region/v1/arb/build.sh @@ -1,6 +1,8 @@ #!/bin/sh -# Public Arb build entrypoint. It always creates the sealed environment before -# invoking the recipe; no caller-controlled sentinel can select the inner path. +# Source-owned Arb dispatcher. The trusted Docker transport invokes this file +# from the fixed bundle path with a clean environment. Keep the outer shell +# limited to builtins: path resolution must happen only in the clean child. +# shellcheck disable=SC2016 set -eu if [ "$#" -ne 0 ]; then @@ -8,13 +10,6 @@ if [ "$#" -ne 0 ]; then exit 64 fi -script_dir=$(/usr/bin/dirname -- "$0") -inner="$script_dir/build-inner.sh" -if [ ! -f "$inner" ] || [ -L "$inner" ]; then - printf '%s\n' 'missing regular Arb inner build recipe' >&2 - exit 66 -fi - exec /usr/bin/env -i \ PATH=/usr/local/bin:/usr/bin:/bin \ LC_ALL=C \ @@ -25,4 +20,14 @@ exec /usr/bin/env -i \ SOURCE_DATE_EPOCH=0 \ ZERO_AR_DATE=1 \ ARFLAGS=crD \ - /bin/sh "$inner" + /bin/sh -c ' + set -eu + script_path=$(/usr/bin/readlink -f -- "$1") + script_dir=$(/usr/bin/dirname -- "$script_path") + inner="$script_dir/build-inner.sh" + if [ ! -f "$inner" ] || [ -L "$inner" ]; then + printf "%s\\n" "missing regular Arb inner build recipe" >&2 + exit 66 + fi + exec /bin/sh "$inner" + ' /bin/sh "$0" diff --git a/proof/region/v1/arb/pipeline.py b/proof/region/v1/arb/pipeline.py index 26a0e097..37d51e35 100644 --- a/proof/region/v1/arb/pipeline.py +++ b/proof/region/v1/arb/pipeline.py @@ -37,6 +37,11 @@ FORMULA_GENERATOR_PATH_V1 = "proof/region/v1/arb/evaluator/formula.py" BUILD_RECIPE_PATH_V1 = "proof/region/v1/arb/build.sh" INNER_BUILD_RECIPE_PATH_V1 = "proof/region/v1/arb/build-inner.sh" +# One ordered recipe coordinate is shared by BUILD and test-observation +# preimages. Keeping the dispatcher and the actual make/check body together +# prevents a recipe-only mutation from masquerading as evaluator source. +BUILD_RECIPE_PATHS_V1 = (BUILD_RECIPE_PATH_V1, INNER_BUILD_RECIPE_PATH_V1) +BUILD_RECIPE_PATH_SET_V1 = frozenset(BUILD_RECIPE_PATHS_V1) FORMULA_SPEC_SHA256_V1 = "a6f77ac462f226453b1c27bbd8637b62780b9a640c317a6f50028dacd1de8540" GENERATED_FORMULA_SHA256_V1 = "9958f20c8ca598625db0593a45f8f8bc79e4b2f22b53263b6c32d78a5e1d2693" @@ -47,8 +52,8 @@ _PINNED_BUILD_SOURCE_SHA256_V1 = { FORMULA_SPEC_PATH_V1: FORMULA_SPEC_SHA256_V1, GENERATED_FORMULA_PATH_V1: GENERATED_FORMULA_SHA256_V1, - BUILD_RECIPE_PATH_V1: "09addfaa10952d3e71baf8a9709fb6b875745dcacea06ce45fd84e382a78173e", - INNER_BUILD_RECIPE_PATH_V1: "0b77e5170f6dab782243aae12ec4ff114a9dfddabe520fdd7ef28e55360f9efc", + BUILD_RECIPE_PATH_V1: "067db346c2d13b393104964fdd8298d3204b77e13bae213d9b1f3a0bf820f94f", + INNER_BUILD_RECIPE_PATH_V1: "b70ee1a6b0259745d5f9326c14f12f83ab026b5f5bf57a01f794edd1c5aa1d00", FORMULA_GENERATOR_PATH_V1: "16629cc3a2ef745ae244ae4762f8946a6546972886f96beeb9ee4920b043040c", "proof/region/v1/arb/evaluator/formula.h": "46fd5ad1b68b728efcd990a71d1dcc273b75e3391d8c06ef2fd0ac6a4d7dfdbd", "proof/region/v1/arb/evaluator/hash.c": "c28e6281208f09ca15fa74aea0091f27726ed68efc3480c34a7db33b8ca3567e", @@ -793,7 +798,14 @@ def comparator_build_preimage_v2( return _comparator_preimage_v2( b"labcolors.proof-region.arb-comparator.build-identity.v2\0", ( - build_sources.contents(BUILD_RECIPE_PATH_V1), + _encoded_build_file_set_v1( + b"labcolors.proof-region.arb-comparator.build-recipes.v1\0", + tuple( + item + for item in build_sources.files + if item.path in BUILD_RECIPE_PATH_SET_V1 + ), + ), build_sources.build_input_identity, build_sources.formula_support_identity, docker_capability_identity, @@ -1149,14 +1161,16 @@ def _derive_arb_comparator_for_build_v1( wrapper_files = tuple( item for item in request.build_sources.files if item.path in wrapper_paths ) + recipe_files = tuple( + item + for item in request.build_sources.files + if item.path in BUILD_RECIPE_PATH_SET_V1 + ) evaluator_files = tuple( item for item in request.build_sources.files - if item.path not in ( - FORMULA_SPEC_PATH_V1, - FORMULA_GENERATOR_PATH_V1, - BUILD_RECIPE_PATH_V1, - ) + if item.path not in (FORMULA_SPEC_PATH_V1, FORMULA_GENERATOR_PATH_V1) + and item.path not in BUILD_RECIPE_PATH_SET_V1 and item.path not in wrapper_paths ) wrapper_source = _encoded_build_file_set_v1( @@ -1184,7 +1198,10 @@ def _derive_arb_comparator_for_build_v1( b"labcolors.proof-region.arb-comparator.test-observation.v1\0", ( b"kind:aggregate-outer-process-observation-no-per-test-records", - request.build_sources.contents(BUILD_RECIPE_PATH_V1), + _encoded_build_file_set_v1( + b"labcolors.proof-region.arb-comparator.build-recipes.v1\0", + recipe_files, + ), len(build_processes).to_bytes(4, "big"), *process_bytes, ), diff --git a/proof/region/v1/arb/tests/gate.py b/proof/region/v1/arb/tests/gate.py index 0a3dd19a..d1291ca6 100644 --- a/proof/region/v1/arb/tests/gate.py +++ b/proof/region/v1/arb/tests/gate.py @@ -15,9 +15,9 @@ REPO = Path(__file__).resolve().parents[5] sys.path.insert(0, str(REPO)) EXPECTED_TEST_INVENTORY_SHA256 = ( - "31938bf43c6e070dddf5905ab4a033cee1474f8e2a6f6571d20c79c8f1e10c3d" + "b886ec2cb791fc54c168801fa91cab95f4a91aaaa5f85871831b3fbfbed3b70c" ) -EXPECTED_TEST_COUNT = 183 +EXPECTED_TEST_COUNT = 184 _EVALUATOR_REASON = "set LABCOLORS_ARB_EVALUATOR to the controlled C17 binary" EXPECTED_SKIPS = frozenset( { diff --git a/proof/region/v1/arb/tests/test_build_recipe.py b/proof/region/v1/arb/tests/test_build_recipe.py index 27ef81d8..3a861446 100644 --- a/proof/region/v1/arb/tests/test_build_recipe.py +++ b/proof/region/v1/arb/tests/test_build_recipe.py @@ -207,12 +207,17 @@ def test_recipe_is_offline_static_and_platform_explicit(self) -> None: self.assertNotIn("readelf -l \"$build/arb-evaluator-v1\" |", source) self.assertNotIn("readelf -d \"$build/arb-evaluator-v1\" 2>&1 |", source) - def test_public_build_entrypoint_strips_hostile_environment_before_recipe(self) -> None: + def test_source_owned_dispatch_cleans_child_environment(self) -> None: entrypoint = BUILD.read_text(encoding="utf-8") recipe = INNER_BUILD.read_text(encoding="utf-8") self.assertIn("/usr/bin/env -i", entrypoint) + self.assertIn("/usr/bin/readlink -f", entrypoint) self.assertIn('inner="$script_dir/build-inner.sh"', entrypoint) self.assertIn('/bin/sh "$inner"', entrypoint) + self.assertLess( + entrypoint.index("exec /usr/bin/env -i"), + entrypoint.index("script_path="), + ) self.assertNotIn("LC_BUILD_ENV_V1", entrypoint) self.assertNotIn("LC_BUILD_ENV_V1", recipe) self.assertNotIn("/usr/bin/env -i", recipe) diff --git a/proof/region/v1/arb/tests/test_pipeline.py b/proof/region/v1/arb/tests/test_pipeline.py index 831420e2..b76e1b4f 100644 --- a/proof/region/v1/arb/tests/test_pipeline.py +++ b/proof/region/v1/arb/tests/test_pipeline.py @@ -581,8 +581,7 @@ def test_wrapper_and_evaluator_file_sets_are_exact_and_disjoint(self) -> None: excluded = wrapper_paths | { pipeline.FORMULA_SPEC_PATH_V1, pipeline.FORMULA_GENERATOR_PATH_V1, - pipeline.BUILD_RECIPE_PATH_V1, - } + } | pipeline.BUILD_RECIPE_PATH_SET_V1 wrapper_files = tuple(item for item in files if item.path in wrapper_paths) evaluator_files = tuple(item for item in files if item.path not in excluded) @@ -602,6 +601,44 @@ def test_wrapper_and_evaluator_file_sets_are_exact_and_disjoint(self) -> None: ), ) + def test_inner_recipe_mutation_is_not_evaluator_source(self) -> None: + original = self._result() + files = _build_sources().files + inner_index = next( + index + for index, item in enumerate(files) + if item.path == pipeline.INNER_BUILD_RECIPE_PATH_V1 + ) + mutated_files = files[:inner_index] + ( + replace(files[inner_index], contents=files[inner_index].contents + b"\n"), + ) + files[inner_index + 1 :] + mutated_sources = pipeline.AdmittedBuildSourcesV1( + mutated_files, + pipeline._build_sources_identity(mutated_files), + _token=pipeline._BUILD_SOURCES_TOKEN, + ) + mutated_request = replace(_request(), build_sources=mutated_sources) + mutated = pipeline._derive_arb_comparator_for_build_v1( + mutated_request, + original.docker_capability, + original._binary, + original.rebuild_sha256s, + original.build_processes, + ) + + self.assertEqual( + mutated.preimages.evaluator_source, + original.comparator.preimages.evaluator_source, + ) + self.assertNotEqual( + mutated.preimages.build_identity, + original.comparator.preimages.build_identity, + ) + self.assertNotEqual( + mutated.preimages.test_observation, + original.comparator.preimages.test_observation, + ) + def test_build_stdout_cannot_supply_a_foreign_manifest_or_coordinate(self) -> None: foreign = _foreign_comparator() report = b"manifest=" + foreign.identity.hex().encode("ascii") diff --git a/proof/region/v1/arb/tests/test_transport.py b/proof/region/v1/arb/tests/test_transport.py index b7ea161d..1bd572a1 100644 --- a/proof/region/v1/arb/tests/test_transport.py +++ b/proof/region/v1/arb/tests/test_transport.py @@ -1248,6 +1248,10 @@ def test_recipe_is_transport_agnostic_and_bootstrap_owns_binary_stdout(self) -> self.assertNotIn("/out", source) self.assertNotIn(">&3", source) self.assertIn("exec 3>&1", pipeline._BUILD_BOOTSTRAP_V1) + self.assertIn( + '/bin/sh "$snapshot/workspace/proof/region/v1/arb/build.sh"', + pipeline._BUILD_BOOTSTRAP_V1, + ) self.assertIn("/build/work/arb-evaluator-v1 >&3", pipeline._BUILD_BOOTSTRAP_V1) def test_native_gate_executes_the_one_shot_receipt_controller(self) -> None: From 1268d4d6db5643f2124d00ed3aff692d306800f1 Mon Sep 17 00:00:00 2001 From: Daniel from Labpics <63733699+lemone112@users.noreply.github.com> Date: Sun, 2 Aug 2026 04:33:13 +0300 Subject: [PATCH 19/20] Proof: independently replay every comparator coordinate --- proof/region/v1/arb/receipt.py | 15 +++++++++ proof/region/v1/arb/tests/gate.py | 4 +-- proof/region/v1/arb/tests/test_receipt.py | 38 +++++++++++++++++++++++ 3 files changed, 55 insertions(+), 2 deletions(-) diff --git a/proof/region/v1/arb/receipt.py b/proof/region/v1/arb/receipt.py index 61ee8f92..2d44a47f 100644 --- a/proof/region/v1/arb/receipt.py +++ b/proof/region/v1/arb/receipt.py @@ -131,6 +131,17 @@ def _comparator_replays_v1( build.rebuild_sha256s, len(build.binary), ) + # Re-derive every named comparator coordinate from the retained + # request and BUILD/RUN observation. Re-hashing a supplied comparator + # only proves internal consistency; it cannot prove that its wrapper, + # evaluator, or test-observation coordinates describe this DAG. + expected_comparator = pipeline._derive_arb_comparator_for_build_v1( + request, + build.docker_capability, + build.binary, + build.rebuild_sha256s, + build.build_processes, + ) if ( type(comparator) is not pipeline.DiagnosticArbComparatorV1 or comparator.structural_source_identity @@ -140,6 +151,10 @@ def _comparator_replays_v1( or comparator.pipeline_policy_identity != build.pipeline_policy_identity or comparator.pipeline_policy_identity != expected_pipeline_policy or comparator.preimages.build_identity != expected_build_preimage + or comparator.preimages != expected_comparator.preimages + or comparator.manifest.manifest != expected_comparator.manifest.manifest + or comparator.manifest.identity != expected_comparator.manifest.identity + or comparator.identity != expected_comparator.identity or comparator.binary_sha256 != build.binary_sha256 or comparator.rebuild_sha256s != build.rebuild_sha256s ): diff --git a/proof/region/v1/arb/tests/gate.py b/proof/region/v1/arb/tests/gate.py index d1291ca6..4cff9a81 100644 --- a/proof/region/v1/arb/tests/gate.py +++ b/proof/region/v1/arb/tests/gate.py @@ -15,9 +15,9 @@ REPO = Path(__file__).resolve().parents[5] sys.path.insert(0, str(REPO)) EXPECTED_TEST_INVENTORY_SHA256 = ( - "b886ec2cb791fc54c168801fa91cab95f4a91aaaa5f85871831b3fbfbed3b70c" + "43c97cac58e82df841be8da440c6c2351c4ab5cde578dd95dac921162f608b11" ) -EXPECTED_TEST_COUNT = 184 +EXPECTED_TEST_COUNT = 185 _EVALUATOR_REASON = "set LABCOLORS_ARB_EVALUATOR to the controlled C17 binary" EXPECTED_SKIPS = frozenset( { diff --git a/proof/region/v1/arb/tests/test_receipt.py b/proof/region/v1/arb/tests/test_receipt.py index 368014b3..895376ed 100644 --- a/proof/region/v1/arb/tests/test_receipt.py +++ b/proof/region/v1/arb/tests/test_receipt.py @@ -28,6 +28,7 @@ import pipeline # noqa: E402 import provenance # noqa: E402 import receipt # noqa: E402 +import region_proof_protocol as protocol # noqa: E402 from region_proof_protocol import ( # noqa: E402 BoundaryUnprovenWitnessV1, DecisionTranscriptV1, @@ -591,6 +592,43 @@ def test_source_process_transfer_and_comparator_mutations_fail(self) -> None: receipt.replay_evidence_is_well_bound_v1(_tamper(dag, "build", build)) ) + def test_each_self_consistent_comparator_coordinate_is_rederived(self) -> None: + result, _backend = _execute() + dag = result.evidence + original = dag.build.comparator + for field in original.preimages.__dataclass_fields__: + with self.subTest(field=field): + preimages = _tamper( + original.preimages, + field, + getattr(original.preimages, field) + b"\x00mutation", + ) + names = tuple(original.preimages.__dataclass_fields__) + coordinates = tuple( + hashlib.sha256(getattr(preimages, name)).digest() + for name in names + ) + manifest = protocol.ComparatorManifestV2( + original.manifest.manifest.kind, + *coordinates, + ) + by_digest = { + coordinate: getattr(preimages, name) + for name, coordinate in zip(names, coordinates, strict=True) + } + resolved = protocol.ContentResolvedComparatorManifestV2.admit( + manifest, + by_digest.get, + ) + comparator = _tamper(original, "preimages", preimages) + comparator = _tamper(comparator, "manifest", resolved) + forged_build = _tamper(dag.build, "comparator", comparator) + self.assertFalse( + receipt.replay_evidence_is_well_bound_v1( + _tamper(dag, "build", forged_build) + ) + ) + def test_source_replay_rejects_a_self_consistent_forged_manifest(self) -> None: request = _request() lock = request.source_lock.sources[2] From 8998ada1b4128514579e3e50afbb9d6a68ac91d4 Mon Sep 17 00:00:00 2001 From: Daniel from Labpics <63733699+lemone112@users.noreply.github.com> Date: Sun, 2 Aug 2026 04:36:11 +0300 Subject: [PATCH 20/20] Proof: expose the comparator replay boundary --- proof/region/v1/arb/pipeline.py | 4 ++-- proof/region/v1/arb/receipt.py | 2 +- proof/region/v1/arb/tests/test_build_identity_v2.py | 2 +- proof/region/v1/arb/tests/test_pipeline.py | 2 +- proof/region/v1/arb/tests/test_receipt.py | 2 ++ 5 files changed, 7 insertions(+), 5 deletions(-) diff --git a/proof/region/v1/arb/pipeline.py b/proof/region/v1/arb/pipeline.py index 37d51e35..d937ec05 100644 --- a/proof/region/v1/arb/pipeline.py +++ b/proof/region/v1/arb/pipeline.py @@ -1032,7 +1032,7 @@ def __post_init__(self) -> None: ) -def _derive_arb_comparator_for_build_v1( +def derive_arb_comparator_for_build_v1( request: PipelineRequestV1, docker_capability: build_transport.DockerSupportedV1, binary: bytes, @@ -1647,7 +1647,7 @@ def build(self, request: PipelineRequestV1) -> BuildResultV1: hashlib.sha256(item).digest() for item in built.outputs ) build_processes = built.processes - comparator = _derive_arb_comparator_for_build_v1( + comparator = derive_arb_comparator_for_build_v1( request, docker_capability, binary, diff --git a/proof/region/v1/arb/receipt.py b/proof/region/v1/arb/receipt.py index 2d44a47f..689a37df 100644 --- a/proof/region/v1/arb/receipt.py +++ b/proof/region/v1/arb/receipt.py @@ -135,7 +135,7 @@ def _comparator_replays_v1( # request and BUILD/RUN observation. Re-hashing a supplied comparator # only proves internal consistency; it cannot prove that its wrapper, # evaluator, or test-observation coordinates describe this DAG. - expected_comparator = pipeline._derive_arb_comparator_for_build_v1( + expected_comparator = pipeline.derive_arb_comparator_for_build_v1( request, build.docker_capability, build.binary, diff --git a/proof/region/v1/arb/tests/test_build_identity_v2.py b/proof/region/v1/arb/tests/test_build_identity_v2.py index 2d37c66b..267510da 100644 --- a/proof/region/v1/arb/tests/test_build_identity_v2.py +++ b/proof/region/v1/arb/tests/test_build_identity_v2.py @@ -467,7 +467,7 @@ def test_diagnostic_build_owns_one_capability_and_replayers_consume_its_identity self.assertNotIn(mirror, field_names) comparator_calls = _called_names( - pipeline._derive_arb_comparator_for_build_v1 + pipeline.derive_arb_comparator_for_build_v1 ) comparator_replay_calls = _called_names(receipt._comparator_replays_v1) receipt_build_calls = _called_names(receipt._build_identity_v2) diff --git a/proof/region/v1/arb/tests/test_pipeline.py b/proof/region/v1/arb/tests/test_pipeline.py index b76e1b4f..a228d956 100644 --- a/proof/region/v1/arb/tests/test_pipeline.py +++ b/proof/region/v1/arb/tests/test_pipeline.py @@ -618,7 +618,7 @@ def test_inner_recipe_mutation_is_not_evaluator_source(self) -> None: _token=pipeline._BUILD_SOURCES_TOKEN, ) mutated_request = replace(_request(), build_sources=mutated_sources) - mutated = pipeline._derive_arb_comparator_for_build_v1( + mutated = pipeline.derive_arb_comparator_for_build_v1( mutated_request, original.docker_capability, original._binary, diff --git a/proof/region/v1/arb/tests/test_receipt.py b/proof/region/v1/arb/tests/test_receipt.py index 895376ed..2716e69e 100644 --- a/proof/region/v1/arb/tests/test_receipt.py +++ b/proof/region/v1/arb/tests/test_receipt.py @@ -341,8 +341,10 @@ def test_receipt_uses_only_versioned_public_cross_module_verifiers(self) -> None self.assertNotIn("executor._execution_identity_v1", source) self.assertNotIn("sealed_build_input_bundle_is_well_bound_v1", source) self.assertIn("pipeline.arb_input_is_bound_v1", source) + self.assertIn("pipeline.derive_arb_comparator_for_build_v1", source) self.assertIn("build_transport.build_process_bytes_v1", source) self.assertTrue(hasattr(pipeline, "arb_input_is_bound_v1")) + self.assertTrue(hasattr(pipeline, "derive_arb_comparator_for_build_v1")) self.assertTrue(hasattr(build_transport, "build_process_bytes_v1")) self.assertTrue(hasattr(executor, "invocation_identity_v1")) self.assertTrue(hasattr(executor, "platform_identity_v1"))