diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index 2090a50..31d1282 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -215,3 +215,13 @@ jobs: if __name__ == '__main__': main() + - name: Retain dependency review job summary + if: ${{ always() }} + continue-on-error: true + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: dependency-review-job-summary-${{ github.event.pull_request.head.sha }}-${{ github.run_id }}-${{ github.run_attempt }} + # Retain only the runner's masked copies used for the actual job summary. + path: ${{ runner.temp }}/_runner_file_commands/step_summary_*-scrubbed + retention-days: 7 + if-no-files-found: warn diff --git a/.github/workflows/publish-linux.yml b/.github/workflows/publish-linux.yml index 37456a2..5192f1d 100644 --- a/.github/workflows/publish-linux.yml +++ b/.github/workflows/publish-linux.yml @@ -24,6 +24,8 @@ jobs: publish: runs-on: ubuntu-26.04 timeout-minutes: 360 + env: + CANDIDATE_ONLY: ${{ inputs.candidate_only }} steps: - name: Check out image sources uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -55,6 +57,9 @@ jobs: { echo "image=${image}" + prefix= + if [[ "${CANDIDATE_ONLY}" == true ]]; then prefix=candidate-; fi + echo "publish_tag=${image}:${prefix}sha-${GITHUB_SHA}" echo 'tags<> "${GITHUB_STEP_SUMMARY}" + set -euo pipefail + images=(linux-build classic-build classic-portable-build) + platforms=(ubuntu-26.04 ubuntu-26.04 debian-12) + digests=("${BROAD_DIGEST}" "${CLASSIC_DIGEST}" "${PORTABLE_DIGEST}") + for digest in "${digests[@]}"; do [[ "${digest}" =~ ^sha256:[0-9a-f]{64}$ ]]; done + for index in 0 1 2; do + image="ghcr.io/${GITHUB_REPOSITORY_OWNER,,}/${images[index]}" + tags=(--tag "${image}:latest" --tag "${image}:${platforms[index]}") + if [[ "${GITHUB_REF_TYPE}" == tag ]]; then + [[ "${GITHUB_REF_NAME}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] + tags+=(--tag "${image}:${GITHUB_REF_NAME#v}") + fi + if ! docker buildx imagetools create "${tags[@]}" "${image}@${digests[index]}"; then + echo "Alias promotion incomplete. Rerun the failed job in this workflow run to converge all aliases." >&2 + exit 1 + fi + printf '%s@%s\n' "${image}" "${digests[index]}" >> "${GITHUB_STEP_SUMMARY}" + done + + - name: Retain failed identity evidence + if: ${{ failure() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: linux-published-identity-failure-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/linux-published-identity/${{ github.run_id }}-${{ github.run_attempt }} + if-no-files-found: error + retention-days: 14 diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 54fd342..be82ac3 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -39,6 +39,8 @@ jobs: case "${path}" in .dockerignore | llvm-toolchain.json | llvm-bootstrap-packages.lock | \ tools/install_llvm_toolchain.py | tools/validate_llvm_toolchain.py | \ + tools/verify_published_identity.py | tools/tests/test_verify_published_identity.py | \ + tools/collect-released-sdk-metadata.py | tools/tests/test_collect_released_sdk_metadata.py | \ tools/tests/test_install_llvm_toolchain.py | tools/tests/test_workflow_contracts.py | .github/actionlint.yaml | .github/workflows/* | \ portable/* | classic-packages.lock | classic-curl-toolchain.json | \ classic-shader-toolchain.json | classic-shader-toolchain.spdx.json | \ @@ -51,6 +53,8 @@ jobs: case "${path}" in .dockerignore | llvm-toolchain.json | llvm-bootstrap-packages.lock | \ tools/install_llvm_toolchain.py | tools/validate_llvm_toolchain.py | \ + tools/verify_published_identity.py | tools/tests/test_verify_published_identity.py | \ + tools/collect-released-sdk-metadata.py | tools/tests/test_collect_released_sdk_metadata.py | \ tools/tests/test_install_llvm_toolchain.py | tools/tests/test_workflow_contracts.py | .github/actionlint.yaml | \ .github/workflows/* | linux/* | \ audio-toolchain.json | audio-toolchain.spdx.json | \ @@ -72,6 +76,8 @@ jobs: case "${path}" in .dockerignore | llvm-toolchain.json | llvm-bootstrap-packages.lock | \ tools/install_llvm_toolchain.py | tools/validate_llvm_toolchain.py | \ + tools/verify_published_identity.py | tools/tests/test_verify_published_identity.py | \ + tools/collect-released-sdk-metadata.py | tools/tests/test_collect_released_sdk_metadata.py | \ tools/tests/test_install_llvm_toolchain.py | tools/tests/test_workflow_contracts.py | .github/workflows/publish-linux.yml | \ .github/workflows/validate.yml | linux/* | \ audio-toolchain.json | audio-toolchain.spdx.json | \ diff --git a/AGENTS.md b/AGENTS.md index 19612b4..758075a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -24,7 +24,17 @@ source commits or shader inputs. Schema 2 compiler evidence requires every selected LLVM utility, including separate Clang C/C++ drivers and archiver/ranlib, with canonical paths, versions and hashes; rehash before - consumer builds and publication. Portable curl is an LLVM rebuild of + consumer builds and publication. The separate schema-1 `installed.json.build_tools` + receipt binds fixed `/usr/bin/cmake`, `/usr/bin/ninja` and `/usr/bin/pkgconf` + paths, canonical executables, hashes, sizes and full version stdout. Verify + all three protected identities before probing any tool and fence + configure/build execution; + the mandatory `pkg_config` role also fences inventory module-version and + consumer/smoke/curl package queries through the canonical executable. + reject symlinks in these fixed Debian executable and metadata paths. + Missing or incomplete receipts require a rebuilt producer. The producer OS, libraries and + CMake modules and earlier library-production builds remain trusted; this + receipt does not attest historical execution. Portable curl is an LLVM rebuild of security-patched Debian `7.88.1-10+deb12u15` with c-ares 1.34.6, retaining existing features including LDAP/LDAPS, RTMP, SCP and SFTP. Its separate `portable/curl-toolchain.json`, `portable/build-curl.py`, @@ -75,6 +85,20 @@ `candidate-sha-`. Windows publishes `candidate-sha-` and `classic-check-candidate-sha-` with cross-build and native Windows tests intact. Neither path may move rolling, platform, or version aliases. +- The Linux publisher captures each loaded image's immutable Docker ID and raw + inspection before runtime tests. Curl/TLS, Classic client/server, portable + public-CA and offline consumer checks must execute those captured IDs. After + all runtime gates, publish only SHA candidates, then verify the actual pushed + OCI index, Linux/amd64 manifest and config against the captured RootFS and + complete execution config with `tools/verify_published_identity.py`. Only + exact source/revision publication labels, creation/history and BuildKit inline + cache metadata may differ; unknown raw config and execution fields fail closed. + Keep bounded metadata-only authenticated acquisition confined to the + same-repository hosted publisher. Retain raw inspection, OCI objects, + descriptor relationships and hashed reports as mandatory artifacts before + promoting any release aliases. Rerun the failed job in the same workflow run + to recover partial promotion; preserve historical released-SDK collector + targets and final-cohort gates. - `llvm-toolchain.json`, `llvm-bootstrap-packages.lock`, and the shared LLVM installer/validator own LLVM 23.1.3 Clang/LLD/compiler-rt/libunwind source, licensing, installed metadata, and target contracts. Every affected input @@ -110,6 +134,10 @@ exists; fix actionable introduced risk or record explicit maintainer-accepted existing risk. License metadata does not certify compatibility; existing repository license policies apply. + For programmatic access, read every runner-scrubbed report in the + `dependency-review-job-summary---` artifact after + verifying its digest and latest current-head run/attempt identity; an absent + or unreadable actual report remains unknown, not clean. - Commits and pull-request titles use Conventional Commits. Preserve unrelated work and finish with `git diff --check`. - Update this `AGENTS.md` in the same change when major rework alters image diff --git a/README.md b/README.md index 5fe13b4..82a24eb 100644 --- a/README.md +++ b/README.md @@ -462,8 +462,30 @@ shader-generation record under versions and hashes for every selected LLVM utility, including independent `compiler` (`clang`) and `cxx_compiler` (`clang++`) records plus `llvm-ar` and `llvm-ranlib`; incomplete earlier evidence is rejected. Consumers rehash these -identities before building and publication. Shared application libraries live under `/usr/local`; -Debian packages provide the baseline transitive libraries. Compiler flags use +identities before building and publication. + +`installed.json.build_tools` separately seals CMake, Ninja and pkgconf with schema 1, +fixed `/usr/bin/cmake`, `/usr/bin/ninja` and `/usr/bin/pkgconf` paths, +canonical executable paths, SHA-256 hashes, byte sizes, +executable status and complete version stdout. Its first lines agree with the +existing installed tool version strings. The consumer verifies all three protected +root-owned executables through open descriptors before any version probe and +fences their resolution, bytes and file identity around configure and build. +The mandatory `pkg_config` role also supplies the installed tool version string, +inventory module-version queries, and consumer/smoke/curl package probes through +the same admission and execution fences. Module versions and feature selections +retain their existing contract. +Every directory component must be protected, and these fixed Debian executable +and metadata paths must not contain symlinks. +SDKs without all three producer identities require a rebuild. This check trusts the +producer OS, dynamic libraries and CMake modules; it does not attest the full +transitive execution environment. +It binds the installed executables used by the later consumer; earlier library +builds remain part of trusted SDK production and are not historical execution +attestations. + +Shared application libraries live under `/usr/local`; Debian packages provide +the baseline transitive libraries. Compiler flags use `-march=x86-64 -mtune=generic`. Verification checks actual ELF dependency providers, versioned symbols, loader relocations and CPU notes, plus device-free image/font decoding, audio decoding and OpenSSL provider loading. The diff --git a/linux/Dockerfile b/linux/Dockerfile index f3831e8..64cddfa 100644 --- a/linux/Dockerfile +++ b/linux/Dockerfile @@ -418,12 +418,31 @@ RUN install -d -o "${USERNAME}" -g "${USERNAME}" "/home/${USERNAME}/.codex" \ > "/etc/sudoers.d/${USERNAME}" \ && chmod 0440 "/etc/sudoers.d/${USERNAME}" -# gh-stack v0.1.0 is a non-immutable release tag, so retain the supported pin -# while independently enforcing its attested linux-amd64 asset digest. The +# Fetch the pinned asset directly: gh extension install still discovers the +# latest release through the API even with --pin, which can hit anonymous limits. +# gh-stack v0.1.0 is a non-immutable release tag, so independently enforce its +# attested linux-amd64 asset digest before installation. The # source commit is the verified release-workflow signer/source digest for # github/gh-stack/.github/workflows/release.yml at refs/tags/v0.1.0. -RUN HOME="/home/${USERNAME}" runuser -u "${USERNAME}" -- \ - gh extension install github/gh-stack --pin "v${GH_STACK_VERSION}" \ +RUN curl --fail --location --silent --show-error \ + "https://github.com/github/gh-stack/releases/download/v${GH_STACK_VERSION}/linux-amd64" \ + --output /tmp/gh-stack-linux-amd64 \ + && echo "${GH_STACK_SHA256_AMD64} /tmp/gh-stack-linux-amd64" | sha256sum -c - \ + && install -d -m 0755 -o "${USERNAME}" -g "${USERNAME}" \ + "/home/${USERNAME}/.local" "/home/${USERNAME}/.local/share" \ + "/home/${USERNAME}/.local/share/gh" \ + "/home/${USERNAME}/.local/share/gh/extensions" \ + "/home/${USERNAME}/.local/share/gh/extensions/gh-stack" \ + && install -m 0755 -o "${USERNAME}" -g "${USERNAME}" /tmp/gh-stack-linux-amd64 \ + "/home/${USERNAME}/.local/share/gh/extensions/gh-stack/gh-stack" \ + && printf '%s\n' 'owner: github' 'name: gh-stack' 'host: github.com' \ + "tag: v${GH_STACK_VERSION}" 'ispinned: true' \ + "path: /home/${USERNAME}/.local/share/gh/extensions/gh-stack/gh-stack" \ + > "/home/${USERNAME}/.local/share/gh/extensions/gh-stack/manifest.yml" \ + && chown "${USERNAME}:${USERNAME}" \ + "/home/${USERNAME}/.local/share/gh/extensions/gh-stack/manifest.yml" \ + && chmod 0644 "/home/${USERNAME}/.local/share/gh/extensions/gh-stack/manifest.yml" \ + && rm /tmp/gh-stack-linux-amd64 \ && echo "${GH_STACK_SHA256_AMD64} /home/${USERNAME}/.local/share/gh/extensions/gh-stack/gh-stack" \ | sha256sum -c - \ && runuser -u "${USERNAME}" -- grep -Fx 'owner: github' \ diff --git a/portable/build-curl.py b/portable/build-curl.py index 5d5cd00..034e155 100644 --- a/portable/build-curl.py +++ b/portable/build-curl.py @@ -21,6 +21,17 @@ def digest(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() +def record_curl_production(root: Path) -> dict: + """The cold producer has no installed.json yet; generate real tool identities.""" + from build_tools import record as record_build_tools + from curl_evidence import record + + build_tools = record_build_tools() + installed = {"build_tools": build_tools, + "tools": {role: item["version"].splitlines()[0] for role, item in build_tools["tools"].items()}} + return record(root / "curl-toolchain.json", installed=installed) + + def checksum_table(text: str, field: str) -> dict[str, str]: lines = text.splitlines() start = lines.index(field + ":") + 1 @@ -171,8 +182,7 @@ def main(root: Path, jobs: int) -> None: "cares_manifest_sha256": digest(root / manifest["cares_manifest"])} (root / "curl-source-receipt.json").write_text(json.dumps(receipt, indent=2) + "\n") subprocess.run(["ldconfig"], check=True) - from curl_evidence import record - record(root / "curl-toolchain.json") + record_curl_production(root) if __name__ == "__main__": diff --git a/portable/build_tools.py b/portable/build_tools.py new file mode 100644 index 0000000..88d1f20 --- /dev/null +++ b/portable/build_tools.py @@ -0,0 +1,248 @@ +#!/usr/bin/env python3 +"""Bind portable CMake/Ninja/pkgconf execution to the producer's installed bytes. + +The producer OS, dynamic libraries and CMake modules remain trusted. These +receipts protect the selected executables against an unprivileged consumer; +they do not attest the complete transitive execution environment. +""" +from __future__ import annotations + +import argparse +from contextlib import contextmanager +import hashlib +import json +import os +from pathlib import Path +import posixpath +import re +import stat +import subprocess + +PATHS = {"cmake": "/usr/bin/cmake", "ninja": "/usr/bin/ninja", "pkg_config": "/usr/bin/pkgconf"} +FIELDS = {"path", "resolved_path", "sha256", "size", "executable", "version"} +MAX_BINARY_SIZE = 512 * 1024 * 1024 + + +def validate(installed: dict) -> dict: + if type(installed) is not dict: + raise ValueError("invalid producer installed metadata object") + receipt = installed.get("build_tools") + if receipt is None: + raise ValueError("missing producer build_tools receipt; rebuild the portable SDK with CMake/Ninja/pkgconf identities") + if (type(receipt) is not dict or set(receipt) != {"schema_version", "tools"} + or type(receipt["schema_version"]) is not int or receipt["schema_version"] != 1 + or type(receipt["tools"]) is not dict or set(receipt["tools"]) != set(PATHS)): + raise ValueError("invalid producer build_tools schema or roles; rebuild the portable SDK with all three identities") + legacy = installed.get("tools") + if type(legacy) is not dict: + raise ValueError("missing installed tool version strings") + for role, path in PATHS.items(): + item = receipt["tools"][role] + if type(item) is not dict or set(item) != FIELDS or item["path"] != path: + raise ValueError("invalid producer build tool fields or fixed path: " + role) + resolved = item["resolved_path"] + if (type(resolved) is not str or not resolved.startswith("/") + or resolved.startswith("//") or "\x00" in resolved + or posixpath.normpath(resolved) != resolved or resolved != path): + raise ValueError("invalid canonical build tool path: " + role) + if (type(item["sha256"]) is not str or not re.fullmatch(r"[0-9a-f]{64}", item["sha256"]) + or type(item["size"]) is not int or not 0 < item["size"] <= MAX_BINARY_SIZE + or item["executable"] is not True): + raise ValueError("invalid producer build tool identity: " + role) + version = item["version"] + if (type(version) is not str or not version or len(version) > 65536 + or any((ord(char) < 32 and char != "\n") or ord(char) == 127 for char in version)): + raise ValueError("invalid complete build tool version: " + role) + first = version.splitlines()[0] + pattern = r"cmake version [0-9]+\.[0-9]+\.[0-9]+" if role == "cmake" else r"[0-9]+\.[0-9]+(?:\.[0-9]+)?" + if not re.fullmatch(pattern, first) or type(legacy.get(role)) is not str or legacy[role] != first: + raise ValueError("build tool version disagrees with installed.tools: " + role) + return receipt + + +def _protected_path(path: Path) -> Path: + """Check every component; fixed Debian tool/metadata paths cannot use links.""" + resolved = Path("/") + parts = list(path.parts[1:]) if path.is_absolute() else [*Path.cwd().parts[1:], *path.parts] + + def check(candidate): + info = candidate.lstat() + if stat.S_ISLNK(info.st_mode): + raise ValueError("symlink in installed build tool path: " + str(candidate)) + if info.st_uid != 0 or info.st_mode & 0o022: + raise ValueError("unprotected installed build tool path: " + str(candidate)) + return info + + check(resolved) + while parts: + component = parts.pop(0) + if component in ("", "."): + continue + if component == "..": + resolved = resolved.parent + check(resolved) + continue + candidate = resolved / component + info = check(candidate) + if parts and not stat.S_ISDIR(info.st_mode): + raise ValueError("invalid build tool directory: " + str(candidate)) + resolved = candidate + return resolved + + +def _identity(fd: int) -> tuple[os.stat_result, str]: + info = os.fstat(fd) + if (not stat.S_ISREG(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o6022 + or not info.st_mode & 0o111 or not 0 < info.st_size <= MAX_BINARY_SIZE): + raise ValueError("build tool must be a protected executable regular file") + digest = hashlib.sha256() + os.lseek(fd, 0, os.SEEK_SET) + while chunk := os.read(fd, 1024 * 1024): + digest.update(chunk) + return info, digest.hexdigest() + + +def _coordinates(info: os.stat_result) -> tuple: + return (info.st_dev, info.st_ino, info.st_mode, info.st_uid, info.st_gid, + info.st_size, info.st_mtime_ns, info.st_ctime_ns) + + +@contextmanager +def protected_tools(expected: dict | None = None): + """Keep verified descriptors alive and fence fixed paths around execution.""" + handles = {} + identities = {} + try: + # All roles must pass byte admission BEFORE either --version executes. + for role, value in PATHS.items(): + path = Path(value) + resolved = _protected_path(path) + fd = os.open(resolved, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC | os.O_NONBLOCK) + handles[role] = fd + info, digest = _identity(fd) + if _coordinates(info) != _coordinates(resolved.stat()) or _protected_path(path) != resolved: + raise ValueError("build tool changed during admission: " + role) + item = {"path": value, "resolved_path": str(resolved), "sha256": digest, + "size": info.st_size, "executable": True} + if expected is not None and any(item[key] != expected[role][key] for key in item): + raise ValueError("changed installed producer build tool identity: " + role) + identities[role] = (item, info) + + def fence(): + for role, (item, before) in identities.items(): + resolved = _protected_path(Path(item["path"])) + info, digest = _identity(handles[role]) + if (str(resolved) != item["resolved_path"] or _coordinates(info) != _coordinates(before) + or _coordinates(resolved.stat()) != _coordinates(before) or digest != item["sha256"]): + raise ValueError("installed build tool changed across execution: " + role) + + fence() + try: + yield {role: item for role, (item, _) in identities.items()}, fence + finally: + fence() + finally: + for fd in handles.values(): + os.close(fd) + + +def _versions(items: dict, fence, expected: dict | None = None) -> None: + for role, item in items.items(): + fence() + # Preserve complete stdout, including its final newline, in the receipt. + try: + version = subprocess.check_output([item["path"], "--version"]).decode("utf-8") + finally: + fence() + if expected is not None and version != expected[role]["version"]: + raise ValueError("changed installed producer build tool version: " + role) + item["version"] = version + + +def record() -> dict: + with protected_tools() as (items, fence): + _versions(items, fence) + receipt = {"schema_version": 1, "tools": items} + validate({"build_tools": receipt, "tools": {role: item["version"].splitlines()[0] + for role, item in items.items()}}) + return receipt + + +@contextmanager +def admitted_tools(installed: dict): + """Admit all three roles before any version or selected tool execution.""" + receipt = validate(installed) + with protected_tools(receipt["tools"]) as (items, fence): + _versions(items, fence, receipt["tools"]) + yield items, fence + + +def capture(installed: dict, role: str, arguments: list[str]) -> str: + """Query the sealed executable, preserving stdout and failure-time fencing.""" + if role not in PATHS: + raise ValueError("unsupported portable build tool role") + with admitted_tools(installed) as (items, fence): + fence() + try: + return subprocess.check_output([items[role]["path"], *arguments]).decode("utf-8") + finally: + fence() + + +def verify(installed: dict, *, role: str | None = None, arguments: list[str] | None = None) -> None: + with admitted_tools(installed) as (_, fence): + if role is not None: + if role not in PATHS: + raise ValueError("unsupported portable build tool role") + fence() + subprocess.run([PATHS[role], *(arguments or [])], check=True) + fence() + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--verify", required=True, type=Path, help="producer installed.json") + parser.add_argument("--run", choices=PATHS) + parser.add_argument("arguments", nargs=argparse.REMAINDER) + args = parser.parse_args() + arguments = args.arguments + if arguments[:1] == ["--"]: + arguments = arguments[1:] + if arguments and args.run is None: + parser.error("arguments require --run") + try: + verify(load_installed(args.verify), role=args.run, arguments=arguments) + except (ValueError, OSError, subprocess.CalledProcessError) as exc: + parser.exit(1, f"portable build tool verification failed: {exc}\n") + + +def load_installed(path: Path) -> dict: + """Metadata cannot be supplied by an unprivileged consumer either.""" + resolved = _protected_path(path) + fd = os.open(resolved, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC | os.O_NONBLOCK) + try: + before = os.fstat(fd) + if (not stat.S_ISREG(before.st_mode) or before.st_uid != 0 or before.st_mode & 0o022 + or not 0 < before.st_size <= 1024 * 1024): + raise ValueError("installed metadata must be a protected regular file") + with os.fdopen(os.dup(fd), "rb") as source: + payload = source.read(1024 * 1024 + 1) + if (_protected_path(path) != resolved or _coordinates(os.fstat(fd)) != _coordinates(before) + or _coordinates(resolved.stat()) != _coordinates(before) or len(payload) != before.st_size): + raise ValueError("installed metadata changed during admission") + + def unique(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError("duplicate installed metadata field: " + key) + result[key] = value + return result + + return json.loads(payload, object_pairs_hook=unique) + finally: + os.close(fd) + + +if __name__ == "__main__": + main() diff --git a/portable/consumer.sh b/portable/consumer.sh index e3de735..4b6ca20 100755 --- a/portable/consumer.sh +++ b/portable/consumer.sh @@ -18,14 +18,17 @@ while read -r expected_hash input; do case "${relative}" in client/* | LICENSE.md | ATTRIBUTIONS.md | PROVENANCE.md) ;; *) exit 1 ;; esac printf '%s %s\n' "${expected_hash}" "${source}/${relative}" | sha256sum -c - done < "${root}/shader-inputs.txt" +python3 "${root}/build_tools.py" --verify "${root}/installed.json" "${root}/smoke.sh" python3 "${root}/compiler_evidence.py" --verify "${root}/compiler-evidence.json" +python3 "${root}/build_tools.py" --verify "${root}/installed.json" work=$(mktemp -d) trap 'rm -rf -- "${work}"' EXIT # Resolve pinned providers through the baseline loader cache; omit transient # CMake build-tree RPATH padding, whose empty entries mean the current directory. -/usr/bin/cmake -S "${source}" -B "${work}/build" -G Ninja \ - -DCMAKE_MAKE_PROGRAM=/usr/bin/ninja -DPKG_CONFIG_EXECUTABLE=/usr/bin/pkg-config \ +python3 "${root}/build_tools.py" --verify "${root}/installed.json" --run cmake -- \ + -S "${source}" -B "${work}/build" -G Ninja \ + -DCMAKE_MAKE_PROGRAM=/usr/bin/ninja -DPKG_CONFIG_EXECUTABLE=/usr/bin/pkgconf \ -DCMAKE_AR=/opt/llvm-23.1.3/bin/llvm-ar -DCMAKE_RANLIB=/opt/llvm-23.1.3/bin/llvm-ranlib \ -DCMAKE_BUILD_TYPE=Release -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ \ @@ -37,7 +40,8 @@ trap 'rm -rf -- "${work}"' EXIT '-DCMAKE_C_FLAGS=-O2 -march=x86-64 -mtune=generic' \ '-DCMAKE_CXX_FLAGS=-O2 -march=x86-64 -mtune=generic' python3 "${root}/compiler_evidence.py" --verify "${root}/compiler-evidence.json" -/usr/bin/cmake --build "${work}/build" --parallel 2 +python3 "${root}/build_tools.py" --verify "${root}/installed.json" --run cmake -- \ + --build "${work}/build" --parallel 2 ctest --test-dir "${work}/build" --output-on-failure --timeout 120 python3 - "${work}/build/compile_commands.json" <<'CHECK' import json, shlex, sys @@ -56,4 +60,5 @@ cp "${work}/build/compile_commands.json" "${reports}/compile_commands.json" cp "${work}/build/Testing/Temporary/LastTest.log" "${reports}/consumer-tests.log" printf '%s\n' "${expected}" > "${reports}/classic-commit.txt" python3 "${root}/compiler_evidence.py" --verify "${root}/compiler-evidence.json" +python3 "${root}/build_tools.py" --verify "${root}/installed.json" cp "${root}/installed.json" "${root}/contract.json" "${root}/compiler-evidence.json" "${reports}/" diff --git a/portable/contract.json b/portable/contract.json index 5d3348b..3e79dfe 100644 --- a/portable/contract.json +++ b/portable/contract.json @@ -60,7 +60,7 @@ "shader_manifest": "classic-shader-toolchain.json", "consumer": { "repository": "atrinik/classic", - "commit": "bbaf7785b6f7c5905b3f5c52f079d6198da0d540", + "commit": "bd5317a03cdc82f029105c5dc26428b09fcd54b3", "shader_directory": "/opt/atrinik-portable/shaders", "cmake_argument": "ATRINIK_GPU_SHADER_DIRECTORY", "input_verification": "exact-source-commit-and-shader-input-sha256", diff --git a/portable/curl_evidence.py b/portable/curl_evidence.py index d91ffbc..5b0a1cb 100644 --- a/portable/curl_evidence.py +++ b/portable/curl_evidence.py @@ -13,6 +13,8 @@ import subprocess import tempfile +from build_tools import capture, load_installed + def digest(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() @@ -56,8 +58,10 @@ def tls_results(positive: dict, negative: dict) -> None: raise ValueError("portable public-CA qualification failed") -def record(manifest_path: Path, public_ca: Path | None = None) -> dict: +def record(manifest_path: Path, public_ca: Path | None = None, *, installed: dict | None = None) -> dict: root = manifest_path.parent + if installed is None: + installed = load_installed(root / "installed.json") manifest = json.loads(manifest_path.read_text()) shared_path = root / manifest["cares_manifest"] shared = json.loads(shared_path.read_text()) @@ -66,8 +70,8 @@ def record(manifest_path: Path, public_ca: Path | None = None) -> dict: retained = root / "sources/curl" configuration((retained / "curl_config.h").read_text()) for package, expected in (("libcurl", manifest["version"]), ("libcares", shared["cares"]["version"])): - prefix = subprocess.check_output(["pkg-config", "--variable=prefix", package], text=True).strip() - version = subprocess.check_output(["pkg-config", "--modversion", package], text=True).strip() + prefix = capture(installed, "pkg_config", ["--variable=prefix", package]).strip() + version = capture(installed, "pkg_config", ["--modversion", package]).strip() if prefix != manifest["prefix"] or version != expected: raise ValueError("wrong active portable curl pkg-config: " + package) if public_ca is not None: @@ -75,7 +79,7 @@ def record(manifest_path: Path, public_ca: Path | None = None) -> dict: tls = None with tempfile.TemporaryDirectory(prefix="portable-curl-probe-") as temporary: binary = Path(temporary) / "probe" - package_flags = shlex.split(subprocess.check_output(["pkg-config", "--cflags", "--libs", "libcurl", "libcares"], text=True)) + package_flags = shlex.split(capture(installed, "pkg_config", ["--cflags", "--libs", "libcurl", "libcares"])) command = ["/opt/llvm-23.1.3/bin/clang", "-O2", "-march=x86-64", "-mtune=generic", "-Wall", "-Wextra", "-Werror", str(root / "curl-runtime-probe.c"), "-o", str(binary), *package_flags, "-ldl"] subprocess.run(command, check=True) diff --git a/portable/inventory.py b/portable/inventory.py index f929a92..3ac001f 100644 --- a/portable/inventory.py +++ b/portable/inventory.py @@ -7,6 +7,7 @@ from compiler_evidence import record from curl_evidence import record as record_curl +from build_tools import capture, record as record_build_tools root = Path(__file__).resolve().parent contract = json.loads((root / "contract.json").read_text()) @@ -25,17 +26,22 @@ def output(*args): evidence = record() evidence_path = root / "compiler-evidence.json" evidence_path.write_text(json.dumps(evidence, indent=2) + "\n") +build_tools = record_build_tools() +producer_installed = {"build_tools": build_tools, + "tools": {role: item["version"].splitlines()[0] for role, item in build_tools["tools"].items()}} actual = {"schema_version": 2, "contract_sha256": hashlib.sha256((root / "contract.json").read_bytes()).hexdigest(), "architecture": output("dpkg", "--print-architecture"), "glibc": output("getconf", "GNU_LIBC_VERSION"), "tools": {name: output(*args).splitlines()[0] for name, args in { "cc": ["clang", "--version"], "linker": ["ld.lld", "--version"], - "readelf": ["llvm-readelf", "--version"], "cmake": ["cmake", "--version"], - "ninja": ["ninja", "--version"], "python": ["python3", "--version"], - "git-lfs": ["git", "lfs", "version"], "openssl": ["openssl", "version"]}.items()}, + "readelf": ["llvm-readelf", "--version"], "python": ["python3", "--version"], + "git-lfs": ["git", "lfs", "version"], "openssl": ["openssl", "version"]}.items() + } | producer_installed["tools"], + "build_tools": build_tools, "compiler_target": evidence["target"], "compiler_evidence_sha256": hashlib.sha256(evidence_path.read_bytes()).hexdigest(), - "pkg_config": {name: output("pkg-config", "--modversion", name) for name in contract["pkg_config"]}, - "curl": record_curl(root / contract["curl_manifest"])} + "pkg_config": {name: capture(producer_installed, "pkg_config", ["--modversion", name]).strip() + for name in contract["pkg_config"]}, + "curl": record_curl(root / contract["curl_manifest"], installed=producer_installed)} assert actual["architecture"] == "amd64" assert actual["glibc"] == "glibc 2.36" for name, version in contract["pkg_config"].items(): diff --git a/portable/smoke.sh b/portable/smoke.sh index b498e93..cd55a1a 100755 --- a/portable/smoke.sh +++ b/portable/smoke.sh @@ -2,6 +2,7 @@ set -euo pipefail root=/opt/atrinik-portable test "$(id -u)" != 0 +python3 "${root}/build_tools.py" --verify "${root}/installed.json" python3 - <<'READABLE' import os from pathlib import Path @@ -33,7 +34,9 @@ cd "${work}" python3 "${root}/abi.py" --output abi.json "${root}/audio/sdl3-mixer-probe" "${root}/audio/opus-probe.opus" # pkg-config emits flags for the pinned build inputs; no user-provided shell code. -read -r -a media_flags <<< "$(pkg-config --cflags --libs sdl3-image sdl3-ttf)" +media_output=$(python3 "${root}/build_tools.py" --verify "${root}/installed.json" --run pkg_config -- \ + --cflags --libs sdl3-image sdl3-ttf) +read -r -a media_flags <<< "${media_output}" clang -O2 -march=x86-64 -mtune=generic "${root}/media-smoke.c" \ -o media-smoke "${media_flags[@]}" ./media-smoke /usr/share/fonts/truetype/dejavu/DejaVuSans.ttf @@ -42,3 +45,4 @@ grep -q 'OpenSSL Default Provider' providers.txt grep -q 'OpenSSL Legacy Provider' providers.txt python3 "${root}/abi.py" --output media-abi.json ./media-smoke (cd "${root}/shaders" && sha256sum -c SHA256SUMS) +python3 "${root}/build_tools.py" --verify "${root}/installed.json" diff --git a/portable/test_build_tools.py b/portable/test_build_tools.py new file mode 100644 index 0000000..b62e270 --- /dev/null +++ b/portable/test_build_tools.py @@ -0,0 +1,463 @@ +"""Producer receipts reject replaced code before probing or configuring it.""" +import copy +import hashlib +import importlib.util +import json +import os +from pathlib import Path +import stat +import tempfile +import unittest +from unittest import mock + +spec = importlib.util.spec_from_file_location("build_tools", Path(__file__).with_name("build_tools.py")) +tools = importlib.util.module_from_spec(spec) +spec.loader.exec_module(tools) + + +def installed_fixture(): + versions = {"cmake": "cmake version 3.25.1\n\nCMake suite maintained and supported by Kitware (kitware.com/cmake).\n", + "ninja": "1.11.1\n", "pkg_config": "1.8.1\n"} + return {"tools": {role: version.splitlines()[0] for role, version in versions.items()}, + "build_tools": {"schema_version": 1, "tools": { + role: {"path": path, "resolved_path": path, + "sha256": hashlib.sha256(role.encode()).hexdigest(), "size": len(role), + "executable": True, "version": versions[role]} + for role, path in tools.PATHS.items()}}} + + +class SchemaTests(unittest.TestCase): + def test_complete_receipt(self): + tools.validate(installed_fixture()) + + def test_missing_receipt_is_actionable(self): + with self.assertRaisesRegex(ValueError, "rebuild the portable SDK"): + tools.validate({"tools": {"cmake": "cmake version 3.25.1", "ninja": "1.11.1"}}) + + def test_strict_fields_roles_and_types(self): + for value in (None, [], True): + with self.subTest(value=value), self.assertRaises(ValueError): + tools.validate(value) + mutations = [ + lambda v: v["build_tools"].update(extra=True), + lambda v: v["build_tools"].update(schema_version=True), + lambda v: v["build_tools"].update(schema_version=2), + lambda v: v["build_tools"].update(tools=[]), + lambda v: v["build_tools"]["tools"].pop("ninja"), + lambda v: v["build_tools"]["tools"].pop("pkg_config"), + lambda v: v["build_tools"]["tools"].update(ctest={}), + lambda v: v.update(tools=[]), + ] + for mutate in mutations: + value = installed_fixture() + mutate(value) + with self.subTest(mutate=mutate), self.assertRaises(ValueError): + tools.validate(value) + changes = {"path": ["cmake", "/usr/local/bin/cmake", "/usr/bin/../bin/cmake"], + "resolved_path": ["relative", "/usr/bin/../bin/cmake", "//usr/bin/cmake", "/", None], + "sha256": ["x" * 64, "A" * 64, None], "size": [True, "5", 0, -1, tools.MAX_BINARY_SIZE + 1], + "executable": [False, 1, "true"], "version": ["", None, "wrong\n", "cmake version 3.25.1\r\n", + "cmake version 3.25.1\n\tcontrol\n", "cmake version 3.25.1\n\x7f"]} + for field, values in changes.items(): + for replacement in values: + value = installed_fixture() + value["build_tools"]["tools"]["cmake"][field] = replacement + with self.subTest(field=field, replacement=replacement), self.assertRaises(ValueError): + tools.validate(value) + for role in tools.PATHS: + value = installed_fixture() + value["build_tools"]["tools"][role]["extra"] = True + with self.subTest(role=role), self.assertRaises(ValueError): + tools.validate(value) + del value["build_tools"]["tools"][role]["extra"] + value["tools"][role] += " changed" + with self.subTest(role=role), self.assertRaises(ValueError): + tools.validate(value) + + def test_every_required_entry_field(self): + for field in tools.FIELDS: + value = installed_fixture() + del value["build_tools"]["tools"]["ninja"][field] + with self.subTest(field=field), self.assertRaises(ValueError): + tools.validate(value) + + +class ExecutionTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory(prefix="build-tools-test-") + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.log = self.root / "execution.log" + self.paths = {} + for role, version in (("cmake", "cmake version 3.25.1\n\ncomplete banner\n"), ("ninja", "1.11.1\n"), + ("pkg_config", "1.8.1\n")): + path = self.root / role + path.write_text("#!/usr/bin/python3\nimport sys\nfrom pathlib import Path\n" + f"with Path({str(self.log)!r}).open('a') as log: log.write({role!r} + ' ' + repr(sys.argv[1:]) + '\\n')\n" + f"if sys.argv[1:] == ['--version']: sys.stdout.write({version!r})\n" + "elif sys.argv[1:2] == ['--modversion']: sys.stdout.write('1.2.3\\n')\n" + "elif sys.argv[1:2] == ['--variable=prefix']: sys.stdout.write('/usr/local\\n')\n") + path.chmod(0o755) + self.paths[role] = str(path) + # A test-owned tree intentionally has a consumer UID and writable /tmp + # ancestors. Bypass only OS ownership policy, retaining actual fd hashes, + # resolution, stats, subprocesses and all admission/fencing behavior. + self.addCleanup(mock.patch.stopall) + mock.patch.object(tools, "PATHS", self.paths).start() + mock.patch.object(tools, "_protected_path", side_effect=lambda p: p.resolve(strict=True)).start() + mock.patch.object(tools, "_identity", side_effect=self.fixture_identity).start() + self.receipt = tools.record() + self.installed = {"build_tools": self.receipt, + "tools": {role: item["version"].splitlines()[0] + for role, item in self.receipt["tools"].items()}} + self.log.unlink() + + @staticmethod + def fixture_identity(fd): + info = os.fstat(fd) + os.lseek(fd, 0, os.SEEK_SET) + digest = hashlib.sha256() + while chunk := os.read(fd, 1024 * 1024): + digest.update(chunk) + return info, digest.hexdigest() + + def test_record_actual_bytes_and_full_stdout_and_run_fixed_tool(self): + for role, path in self.paths.items(): + item = self.receipt["tools"][role] + self.assertEqual(item["sha256"], hashlib.sha256(Path(path).read_bytes()).hexdigest()) + self.assertEqual(item["size"], Path(path).stat().st_size) + self.assertEqual(item["resolved_path"], str(Path(path).resolve())) + self.assertTrue(item["version"].endswith("\n")) + tools.verify(self.installed, role="cmake", arguments=["-S", "fixture", "-G", "Ninja"]) + self.assertEqual(self.log.read_text().splitlines(), ["cmake ['--version']", "ninja ['--version']", + "pkg_config ['--version']", + "cmake ['-S', 'fixture', '-G', 'Ninja']"]) + + def test_changed_tool_or_size_rejected_before_any_probe(self): + for role in self.paths: + for field, change in (("sha256", "0" * 64), ("size", 1234), ("resolved_path", "/different")): + value = copy.deepcopy(self.installed) + value["build_tools"]["tools"][role][field] = change + message = "invalid canonical" if field == "resolved_path" else "changed installed" + with self.subTest(role=role, field=field), self.assertRaisesRegex(ValueError, message): + tools.verify(value, role="cmake", arguments=["--configure"]) + self.assertFalse(self.log.exists()) + + def test_replaced_bytes_fail_before_version_or_configure(self): + path = Path(self.paths["ninja"]) + path.write_bytes(path.read_bytes() + b"# changed bytes\n") + with self.assertRaisesRegex(ValueError, "changed installed"): + tools.verify(self.installed, role="cmake", arguments=["--configure"]) + self.assertFalse(self.log.exists()) + + def test_missing_pkgconf_role_rejects_old_receipt_before_any_execution(self): + value = copy.deepcopy(self.installed) + del value["build_tools"]["tools"]["pkg_config"] + with self.assertRaisesRegex(ValueError, "schema or roles"): + tools.capture(value, "pkg_config", ["--modversion", "fixture"]) + self.assertFalse(self.log.exists()) + + def test_replaced_pkgconf_bytes_fail_before_any_version_or_module_query(self): + path = Path(self.paths["pkg_config"]) + path.write_bytes(path.read_bytes() + b"# changed pkgconf bytes\n") + with self.assertRaisesRegex(ValueError, "changed installed"): + tools.capture(self.installed, "pkg_config", ["--modversion", "fixture"]) + self.assertFalse(self.log.exists()) + + def test_curl_consumer_changed_pkgconf_is_rejected_before_any_probe(self): + import curl_evidence + + manifest = self.root / "curl-toolchain.json" + manifest.write_text(json.dumps({"cares_manifest": "classic-curl-toolchain.json", + "prefix": "/usr/local", "version": "7.88.1"})) + (self.root / "classic-curl-toolchain.json").write_text(json.dumps({"cares": {"version": "1.34.6"}})) + retained = self.root / "sources/curl" + retained.mkdir(parents=True) + (retained / "curl_config.h").write_text("#define USE_ARES 1\n") + path = Path(self.paths["pkg_config"]) + path.write_bytes(path.read_bytes() + b"# changed before curl probe\n") + with mock.patch.object(curl_evidence, "load_installed", return_value=self.installed) as loader, \ + mock.patch.object(curl_evidence, "capture", side_effect=tools.capture), \ + self.assertRaisesRegex(ValueError, "changed installed"): + curl_evidence.record(manifest) + loader.assert_called_once_with(self.root / "installed.json") + self.assertFalse(self.log.exists()) + + def test_cold_curl_producer_passes_generated_receipt_and_admits_package_queries(self): + import build_tools + import curl_evidence + + builder_spec = importlib.util.spec_from_file_location("cold_build_curl", Path(__file__).with_name("build-curl.py")) + builder = importlib.util.module_from_spec(builder_spec) + builder_spec.loader.exec_module(builder) + manifest = self.root / "curl-toolchain.json" + manifest.write_text(json.dumps({"cares_manifest": "classic-curl-toolchain.json", + "prefix": "/usr/local", "version": "7.88.1"})) + (self.root / "classic-curl-toolchain.json").write_text(json.dumps({"cares": {"version": "1.34.6"}})) + retained = self.root / "sources/curl" + retained.mkdir(parents=True) + (retained / "curl_config.h").write_text("#define USE_ARES 1\n") + self.assertFalse((self.root / "installed.json").exists()) + with mock.patch.object(build_tools, "record", side_effect=tools.record), \ + mock.patch.object(curl_evidence, "load_installed", side_effect=AssertionError("cold producer must pass a generated receipt")) as loader, \ + mock.patch.object(curl_evidence, "capture", side_effect=tools.capture), \ + mock.patch.object(curl_evidence, "record", wraps=curl_evidence.record) as record, \ + self.assertRaisesRegex(ValueError, "wrong active portable curl pkg-config"): + # The real query emits fixture module version 1.2.3, intentionally + # stopping at existing module validation before an application probe. + builder.record_curl_production(self.root) + loader.assert_not_called() + generated = record.call_args.kwargs["installed"] + self.assertEqual(set(generated["build_tools"]["tools"]), {"cmake", "ninja", "pkg_config"}) + for role, path in self.paths.items(): + self.assertEqual(generated["build_tools"]["tools"][role]["sha256"], hashlib.sha256(Path(path).read_bytes()).hexdigest()) + logs = self.log.read_text().splitlines() + self.assertEqual(logs[:3], ["cmake ['--version']", "ninja ['--version']", "pkg_config ['--version']"]) + self.assertIn("pkg_config ['--variable=prefix', 'libcurl']", logs) + self.assertIn("pkg_config ['--modversion', 'libcurl']", logs) + # Exercise the extracted cold evidence step, and retain the real main + # callsite so future refactors cannot bypass this generated argument. + self.assertIn(' record_curl_production(root)\n', Path(__file__).with_name("build-curl.py").read_text()) + + def test_module_query_uses_admitted_pkgconf_with_complete_stdout(self): + self.assertEqual(tools.capture(self.installed, "pkg_config", ["--modversion", "fixture"]), "1.2.3\n") + self.assertEqual(self.log.read_text().splitlines(), ["cmake ['--version']", "ninja ['--version']", + "pkg_config ['--version']", + "pkg_config ['--modversion', 'fixture']"]) + + def test_module_query_mutation_is_fenced_including_failure(self): + original = tools.subprocess.check_output + for failed in (False, True): + path = Path(self.paths["pkg_config"]) + before = path.read_bytes() + + def mutate_query(*args, **kwargs): + if args[0][1:] == ["--modversion", "fixture"]: + path.write_bytes(before + b"# changed during module query\n") + if failed: + raise tools.subprocess.CalledProcessError(1, args[0]) + return b"1.2.3\n" + return original(*args, **kwargs) + + with self.subTest(failed=failed), mock.patch.object(tools.subprocess, "check_output", side_effect=mutate_query), \ + self.assertRaisesRegex(ValueError, "changed across execution"): + tools.capture(self.installed, "pkg_config", ["--modversion", "fixture"]) + path.write_bytes(before) + + def test_full_stdout_mismatch_fails_before_configure(self): + value = copy.deepcopy(self.installed) + value["build_tools"]["tools"]["cmake"]["version"] = "cmake version 3.25.1\n\nchanged later line\n" + with self.assertRaisesRegex(ValueError, "changed installed producer build tool version"): + tools.verify(value, role="cmake", arguments=["--configure"]) + self.assertEqual(self.log.read_text().splitlines(), ["cmake ['--version']"]) + + def test_inode_replacement_during_probe_is_fenced(self): + original = tools.subprocess.check_output + + def replace_after_probe(*args, **kwargs): + version = original(*args, **kwargs) + path = Path(self.paths["cmake"]) + replacement = path.with_suffix(".replacement") + replacement.write_bytes(path.read_bytes()) + replacement.chmod(0o755) + replacement.replace(path) + return version + + with mock.patch.object(tools.subprocess, "check_output", side_effect=replace_after_probe), \ + self.assertRaisesRegex(ValueError, "changed across execution"): + tools.verify(self.installed, role="cmake", arguments=["--configure"]) + self.assertEqual(self.log.read_text().splitlines(), ["cmake ['--version']"]) + + def test_mutation_during_configure_is_fenced(self): + original = tools.subprocess.run + + def mutate_after_configure(*args, **kwargs): + result = original(*args, **kwargs) + if args[0][1:] == ["--configure"]: + path = Path(self.paths["ninja"]) + path.write_bytes(path.read_bytes() + b"# changed during configure\n") + return result + + with mock.patch.object(tools.subprocess, "run", side_effect=mutate_after_configure), \ + self.assertRaisesRegex(ValueError, "changed across execution"): + tools.verify(self.installed, role="cmake", arguments=["--configure"]) + + def test_failed_configure_still_fences_mutation(self): + original = tools.subprocess.run + + def fail_configure(*args, **kwargs): + if args[0][1:] == ["--configure"]: + path = Path(self.paths["ninja"]) + path.write_bytes(path.read_bytes() + b"# mutated failed configure\n") + raise tools.subprocess.CalledProcessError(1, args[0]) + return original(*args, **kwargs) + + with mock.patch.object(tools.subprocess, "run", side_effect=fail_configure), \ + self.assertRaisesRegex(ValueError, "changed across execution"): + tools.verify(self.installed, role="cmake", arguments=["--configure"]) + + +class ProtectionTests(unittest.TestCase): + def test_unprivileged_or_writable_files_fail_before_hashing(self): + with tempfile.TemporaryFile() as file: + original = os.fstat(file.fileno()) + for mode, uid in ((stat.S_IFREG | 0o755, 1000), (stat.S_IFREG | 0o775, 0), + (stat.S_IFREG | 0o644, 0), (stat.S_IFIFO | 0o755, 0), + (stat.S_IFREG | 0o4755, 0), (stat.S_IFREG | 0o2755, 0)): + values = list(original) + values[0], values[4], values[6] = mode, uid, 1 + with self.subTest(mode=mode, uid=uid), mock.patch.object(tools.os, "fstat", return_value=os.stat_result(values)), \ + mock.patch.object(tools.os, "read") as read, self.assertRaises(ValueError): + tools._identity(file.fileno()) + read.assert_not_called() + + def test_protected_regular_executable_positive(self): + with tempfile.TemporaryDirectory() as temp: + path = Path(temp) / "regular" + path.write_bytes(b"test-owned executable bytes") + path.chmod(0o555) + original_lstat, original_fstat = Path.lstat, os.fstat + + def protected_info(info): + values = list(info) + values[0], values[4] = info.st_mode & ~0o022, 0 + return os.stat_result(values) + + # Root ownership is modeled in a test-owned tree. Directory/file + # traversal, opening, regular mode, size and bytes remain real. + with mock.patch.object(tools.Path, "lstat", side_effect=lambda p: protected_info(original_lstat(p)), autospec=True), \ + mock.patch.object(tools.os, "fstat", side_effect=lambda fd: protected_info(original_fstat(fd))): + resolved = tools._protected_path(path) + self.assertEqual(resolved, path.resolve(strict=True)) + fd = os.open(resolved, os.O_RDONLY | os.O_NOFOLLOW) + try: + info, digest = tools._identity(fd) + self.assertEqual(info.st_size, len(resolved.read_bytes())) + self.assertEqual(digest, hashlib.sha256(resolved.read_bytes()).hexdigest()) + finally: + os.close(fd) + + def test_intermediate_symlink_expansion_cannot_skip_writable_directory(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + trusted, untrusted = root / "trusted", root / "untrusted" + trusted.mkdir() + untrusted.mkdir() + untrusted.chmod(0o777) + real = trusted / "real" + real.write_text("fixture") + real.chmod(0o755) + (trusted / "cmake").symlink_to(untrusted / "hop") + (untrusted / "hop").symlink_to(real) + original = Path.lstat + seen = [] + + def trusted_ancestor_info(path): + info = original(path) + seen.append(path) + # Model root-owned protected fixture ancestors only. Keep the + # actual writable intermediate directory's ownership/mode. + if path == untrusted: + return info + values = list(info) + values[0], values[4] = info.st_mode & ~0o022, 0 + return os.stat_result(values) + + with mock.patch.object(tools.Path, "lstat", side_effect=trusted_ancestor_info, autospec=True): + self.assertEqual(tools._protected_path(real), real) + seen.clear() + with self.assertRaisesRegex(ValueError, "symlink in installed build tool path"): + tools._protected_path(trusted / "cmake") + self.assertIn(trusted / "cmake", seen) + self.assertNotIn(real, seen) + + def test_parent_symlink_cannot_redirect_a_regular_tool_or_metadata(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + (root / "real").mkdir() + (root / "real/tool").write_bytes(b"fixture") + (root / "parent").symlink_to(root / "real", target_is_directory=True) + original_lstat = Path.lstat + + def protected_info(path): + info = original_lstat(path) + values = list(info) + values[0], values[4] = info.st_mode & ~0o022, 0 + return os.stat_result(values) + + with mock.patch.object(tools.Path, "lstat", side_effect=protected_info, autospec=True), \ + self.assertRaisesRegex(ValueError, "symlink in installed build tool path"): + tools._protected_path(root / "parent/tool") + + def test_consumer_owned_tree_is_not_protected(self): + with tempfile.TemporaryDirectory() as temp: + path = Path(temp) / "executable" + path.write_text("fixture") + path.chmod(0o755) + with self.assertRaisesRegex(ValueError, "unprotected"): + tools._protected_path(path) + + def test_consumer_uses_receipt_runner_for_configure_and_build(self): + source = Path(__file__).with_name("consumer.sh").read_text() + self.assertEqual(source.count('--run cmake --'), 2) + self.assertNotIn('/usr/bin/cmake ', source) + self.assertIn('-DCMAKE_MAKE_PROGRAM=/usr/bin/ninja', source) + self.assertIn('-DPKG_CONFIG_EXECUTABLE=/usr/bin/pkgconf', source) + self.assertLess(source.index('build_tools.py'), source.index('"${root}/smoke.sh"')) + inventory = Path(__file__).with_name("inventory.py").read_text() + self.assertIn('"build_tools": build_tools', inventory) + self.assertNotIn('["cmake", "--version"]', inventory) + self.assertNotIn('["ninja", "--version"]', inventory) + self.assertIn('capture(producer_installed, "pkg_config", ["--modversion", name])', inventory) + self.assertNotIn('output("pkg-config"', inventory) + smoke = Path(__file__).with_name("smoke.sh").read_text() + self.assertIn('--run pkg_config --', smoke) + self.assertNotIn('$(pkg-config ', smoke) + curl = Path(__file__).with_name("curl_evidence.py").read_text() + self.assertNotIn('["pkg-config"', curl) + self.assertEqual(curl.count('capture(installed, "pkg_config"'), 3) + + +class MetadataTests(unittest.TestCase): + def test_protected_loader_rejects_consumer_metadata(self): + with tempfile.TemporaryDirectory() as temp: + path = Path(temp) / "installed.json" + path.write_text(json.dumps(installed_fixture())) + with self.assertRaisesRegex(ValueError, "unprotected"): + tools.load_installed(path) + + def test_metadata_duplicates_and_invalid_types_are_rejected(self): + with tempfile.TemporaryDirectory() as temp: + path = Path(temp) / "installed.json" + original_fstat, original_stat = os.fstat, Path.stat + + def root_info(info): + values = list(info) + values[4] = 0 + return os.stat_result(values) + + with mock.patch.object(tools, "_protected_path", side_effect=lambda p: p), \ + mock.patch.object(tools.os, "fstat", side_effect=lambda fd: root_info(original_fstat(fd))), \ + mock.patch.object(tools.Path, "stat", side_effect=lambda p: root_info(original_stat(p)), autospec=True): + path.write_text(json.dumps(installed_fixture())) + self.assertEqual(tools.load_installed(path), installed_fixture()) + for payload in ('{"build_tools": {}, "build_tools": {}}', + '{"build_tools": {"tools": {"cmake": {"path": "a", "path": "b"}}}}'): + path.write_text(payload) + with self.subTest(payload=payload), self.assertRaisesRegex(ValueError, "duplicate"): + tools.load_installed(path) + path.write_text("[]") + with self.assertRaisesRegex(ValueError, "metadata object"): + tools.verify(tools.load_installed(path)) + + def test_cli_preserves_configure_argument_boundary(self): + with mock.patch.object(tools, "load_installed", return_value=installed_fixture()), \ + mock.patch.object(tools, "verify") as verify, \ + mock.patch("sys.argv", ["build_tools.py", "--verify", "/installed.json", "--run", "cmake", + "--", "-S", "/source with spaces", "-G", "Ninja"]): + tools.main() + self.assertEqual(verify.call_args.kwargs, {"role": "cmake", "arguments": ["-S", "/source with spaces", "-G", "Ninja"]}) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/RELEASED_SDK_INVENTORY.md b/tools/RELEASED_SDK_INVENTORY.md new file mode 100644 index 0000000..8c4e5c4 --- /dev/null +++ b/tools/RELEASED_SDK_INVENTORY.md @@ -0,0 +1,102 @@ +# Released portable SDK inventory foundation + +`released_sdk_inventory.py` is an internal acquisition helper for the existing +collector's authenticated, task-owned `StoppedImage` and archive protocol. It +has no CLI, credentials, registry operations, release pins or image execution. +It is not yet imported or invoked by `collect-released-sdk-metadata.py`; the +three fixed 1.18.0 targets and their producer mapping remain authoritative. + +The helper streams `/opt/atrinik-portable` without extraction. It retains only +the fixed `audio-toolchain.json`, `llvm-toolchain.json`, `curl-toolchain.json` +and `classic-curl-toolchain.json` payloads, each nonempty +and at most 1 MiB, as private validation inputs on `ProducerInventory`. Before +parsing a manifest, it binds the retained bytes to the streamed record's +SHA-256 and size. These payloads are excluded from serialized inventory maps and +their digests; copying the records to a plain dictionary loses the payloads and +fails closed during source-archive validation. All other file payloads are +discarded after hashing. It requires a root directory entry, parents before children, unique +canonical relative paths, regular files and safe directory/file modes. It +rejects links, special/sparse files, path/size PAX overrides, malformed headers, +truncated members, missing end markers and nonzero trailing data. Timestamp-only +PAX records are accepted. Bounds are 20,000 members, 1,024-byte paths, 1 GiB per +file, 8 GiB aggregate payload and a shared 900-second deadline; single-file +reads use the same deadline and aggregate accounting. The existing collector's +operation alarm and socket timeout remain required outer bounds. + +Each producer record is `relative_path -> {sha256, size, executable}`, including +zero-byte files. Canonical sorted compact JSON uses the wrapper's +`linux_portable.inventory_digest` format. Common licenses use exactly the +wrapper's fourteen fixed names. Debian notices are selected only by matching +`runtime-sources.json.source_packages` to `debian-sources.json` package/source +rows. Their records retain the requested absolute path as key and add +`resolved_path`. Source archives named by the authenticated runtime and contract +documents must match the complete producer inventory, including custom provider +archives and the audio and LLVM archives declared by their referenced manifests. +Every declared archive record requires an integer size greater than zero and at +most 1 GiB, plus the declared SHA-256; even the matching digest of an empty file +cannot satisfy an archive declaration. +Debian notices, common licenses and build-tool executables must be nonempty; +other producer-tree files may legitimately have zero bytes. + +Archive validation requires the schema-2 `portable-final`, `linux/amd64` +contract, including the fixed audio, LLVM and curl manifest references. The +curl manifest must reference the fixed shared Classic curl manifest. Historical +contracts with absent references cannot qualify through this helper. The +contract's source cohort is exactly `openssl`, `sdl3`, `sdl3-image` and +`sdl3-ttf`; the audio cohort is `SDL3_mixer` with `libogg`, `libopus` and +`libopusfile`. Missing, duplicate or renamed components fail validation. +`runtime-sources.json.custom_providers` must contain exactly `libcurl` and +`libcares`, with the fields emitted by `portable/legal.py`. Their complete +archive maps must match the authenticated curl manifests: the four Debian +curl source/signature files under `sources/curl/`, and the digest-named c-ares +archive under `sources/`. Removing an individual archive from a provider's +declaration cannot bypass its byte check. These checks validate declared source +closure; they do not parse Debian `.dsc` dependency semantics or independently +establish completeness of the producer's Debian source-package archive map. + +Docker HEAD's `LinkTarget` is a fully resolved result, so it cannot establish a +safe link traversal by itself. The notice resolver reads each raw symbolic link +as a single tar member, resolves at most sixteen hops itself beneath +`/usr/share/doc`, rejects even temporary escape and cycles, and crosschecks the +result against HEAD. Parent (`..`) components are accepted only at the start +of a relative link target, before any potentially symbolic component. It +rechecks raw links, canonical regular files and every +observed parent/root before returning. This relies on Moby's +[`containerArchivePath`](https://github.com/moby/moby/blob/v28.5.1/daemon/archive_unix.go) +and [`containerFSView.Stat`](https://github.com/moby/moby/blob/v28.5.1/daemon/containerfs_linux.go) +semantics; final hosted qualification must exercise the actual daemon. The +producer-tree path accepts no links at all. Docker tar generation may omit entries +when traversal fails: complete tar framing proves only the delivered stream. +Before adopting new wrapper pins, final hosted qualification must independently +compare this producer map with the wrapper's `verified_tree_inventory` on that +exact image (or an authenticated complete OCI-overlay reconstruction). Source +archive hash checks alone do not establish that no unrelated file was omitted. + +The build-tool reader requires exactly `cmake`, `ninja` and `pkg_config` at +`/usr/bin/cmake`, `/usr/bin/ninja` and `/usr/bin/pkgconf`, all canonical regular +executables. `pkg_config` uses the numeric version grammar shared with Ninja; +the `pkg-config` alias cannot substitute for the fixed `pkgconf` path. It validates +`installed.json.build_tools = {schema_version: 1, tools: {…}}`, the exact role and +field sets, full producer version strings and legacy first-line compatibility, +then compares streamed bytes, size and executable status. It never executes a +tool to obtain a version. LLVM compiler evidence retains its separate roles. + +Final wiring requires actual merged Classic consumer/shader commit M, final +producer P and released image D. A subsequent reviewed change must add fixed +index, linux/amd64 manifest and config digests plus per-target producer revision; +validate M/P and shader metadata against those fixed constants; bind each saved +metadata file to the producer inventory; emit all three complete maps and their +canonical digests; and invoke the helper inside the collector's existing +cleanup-protected acquisition. Do not add caller-supplied pins, paths or trust +extensions. Original broad/classic/preliminary identities must remain unchanged. +Wrong final M/D fixtures and actual hosted unstarted-image collection remain +required at that integration point. Cleanup failure must still invalidate the +receipt. Even successful acquisition retains `installed-identity-only`, +`image_code_executed: false` and `final_portable_cohort: false`; it does not prove +runtime, export, gameplay or audible behavior. + +Offline validation: + +```sh +python3 -m unittest discover -s tools/tests -p 'test*released*sdk*.py' -v +``` diff --git a/tools/collect-released-sdk-metadata.py b/tools/collect-released-sdk-metadata.py index dca8abd..1b96d8f 100644 --- a/tools/collect-released-sdk-metadata.py +++ b/tools/collect-released-sdk-metadata.py @@ -611,7 +611,7 @@ def pax_times(raw): raw = raw[length:] -def single_file_archive(response, path, expected, *, retain, executable=False): +def single_file_archive(response, path, expected, *, retain, executable=False, allow_empty=False, root_owned=False): """Read one regular file; reject all links/extensions/path changes, never extract.""" require(response.chunked or response.length is not None, 'unframed-archive') require(response.getheader('Content-Encoding') in (None, 'identity') and @@ -620,7 +620,9 @@ def single_file_archive(response, path, expected, *, retain, executable=False): stat_matches(expected, path) regular_mode(expected['mode'], executable) limit = METADATA_LIMIT if retain else BINARY_LIMIT - require(0 < expected['size'] <= limit, 'file-size-limit') + # Only producer-tree inventory reads opt in to legitimate empty source files. + # Curated metadata, notices and canonical tools must always contain bytes. + require((0 if allow_empty else 1) <= expected['size'] <= limit, 'file-size-limit') header = exact_read(response, 512) info = tarfile.TarInfo.frombuf(header, 'utf-8', 'strict') if info.type == tarfile.XHDTYPE: @@ -634,6 +636,7 @@ def single_file_archive(response, path, expected, *, retain, executable=False): regular_mode(info.mode, executable) require(info.mode == expected['mode'] and info.uid >= 0 and info.gid >= 0, 'archive-metadata-mismatch') + require(not root_owned or info.uid == 0, 'archive-owner-mismatch') sha = hashlib.sha256() saved = bytearray() if retain else None remaining = info.size diff --git a/tools/released_sdk_inventory.py b/tools/released_sdk_inventory.py new file mode 100644 index 0000000..afc0697 --- /dev/null +++ b/tools/released_sdk_inventory.py @@ -0,0 +1,656 @@ +# Copyright (c) 2026 Atrinik contributors +# SPDX-License-Identifier: MIT +"""Internal fixed-boundary inventory reader; no CLI or acquisition authority. + +The caller must supply the collector's authenticated, created-state StoppedImage +and its archive protocol. This foundation is deliberately not wired to a release +target: final producer/consumer/index/platform/config pins are still required. +""" +from abc import ABC, abstractmethod +import hashlib +import json +from pathlib import PurePosixPath +import re +import tarfile +import time +import urllib.parse + +ROOT = '/opt/atrinik-portable' +DOC = '/usr/share/doc' +COMMON = '/usr/share/common-licenses' +BUILD_TOOLS = {'cmake': '/usr/bin/cmake', 'ninja': '/usr/bin/ninja', + 'pkg_config': '/usr/bin/pkgconf'} +COMMON_LICENSE_NAMES = ('Apache-2.0', 'Artistic', 'BSD', 'CC0-1.0', 'GFDL-1.2', + 'GFDL-1.3', 'GPL-1', 'GPL-2', 'GPL-3', 'LGPL-2', + 'LGPL-2.1', 'LGPL-3', 'MPL-1.1', 'MPL-2.0') +MAX_MEMBERS = 20000 +MAX_PATH = 1024 +MAX_FILE = 1024 ** 3 +MAX_BUILD_TOOL = 512 * 1024 * 1024 +MAX_TOTAL = 8 * 1024 ** 3 +MAX_SECONDS = 900 +MAX_HOPS = 16 +SOURCE_MANIFESTS = ('audio-toolchain.json', 'llvm-toolchain.json', + 'curl-toolchain.json', 'classic-curl-toolchain.json') +MAX_MANIFEST = 1024 * 1024 +GO_DIRECTORY = 1 << 31 +GO_SYMLINK = 1 << 27 + + +class InventoryError(Exception): + """Only fixed failure codes; never hostile archive or daemon contents.""" + + +class ProviderReader(ABC): + """Authenticated byte reader supplied by the source-inventory caller. + + Implementations own admission to their stopped image or protected filesystem, + and fence actual file/ancestor identities around every read. This interface + is not authentication: callers must use the same source for the producer + inventory, metadata and provider bytes, never an advisory metadata adapter. + """ + @abstractmethod + def read_provider(self, path): + """Read a fixed-namespace library; return sha256, size and executable.""" + + +class ProducerInventory(dict): + """File identities plus bounded source manifests from the same archive. + + Payloads are private validation inputs, excluded from the serialized inventory + and its digest. Copying only the identities cannot satisfy manifest closure. + """ + def __init__(self): + super().__init__() + self.source_manifests = {} + + +def require(ok, code): + if not ok: + raise InventoryError(code) + + +def inventory_digest(records): + return hashlib.sha256(json.dumps(records, sort_keys=True, + separators=(',', ':')).encode()).hexdigest() + + +def relative_path(value): + require(isinstance(value, str) and 0 < len(value.encode('utf-8')) <= MAX_PATH + and all(32 < ord(c) < 127 for c in value) and '\\' not in value + and all(part not in ('', '.', '..') for part in value.split('/')), + 'inventory-path-invalid') + return value + + +def under_doc(value): + require(isinstance(value, str) and value.startswith(DOC + '/'), 'notice-boundary') + relative_path(value[len(DOC) + 1:]) + return value + + +class Budget: + def __init__(self): + self.deadline = time.monotonic() + MAX_SECONDS + self.bytes = 0 + self.members = 0 + self.payload = 0 + + def check(self): + require(time.monotonic() < self.deadline, 'inventory-deadline') + + def read(self, stream, size): + self.check() + raw = stream.read(size) + self.check() + self.bytes += len(raw) + require(self.bytes <= MAX_TOTAL + MAX_MEMBERS * (1024 + 8192) + 10240, + 'inventory-stream-limit') + return raw + + def exact(self, stream, size): + result = bytearray() + while len(result) < size: + raw = self.read(stream, min(65536, size - len(result))) + require(bool(raw), 'inventory-truncated') + result.extend(raw) + return bytes(result) + + def member(self, size): + self.members += 1 + self.payload += size + require(self.members <= MAX_MEMBERS and 0 <= size <= MAX_FILE + and self.payload <= MAX_TOTAL, 'inventory-size-limit') + + +def framing(response): + require(response.chunked or response.length is not None, 'inventory-unframed') + require(response.getheader('Content-Encoding') in (None, 'identity') and + response.getheader('Content-Type', '').split(';')[0] == 'application/x-tar', + 'inventory-encoding') + + +def header(raw): + try: + info = tarfile.TarInfo.frombuf(raw, 'utf-8', 'strict') + # TarInfo strips every directory terminator, so reject repeated raw + # slashes before its normalization can hide a noncanonical spelling. + require(not (info.isdir() and raw[:100].split(b'\0', 1)[0].endswith(b'//')), + 'inventory-path-invalid') + # Moby archives GET / with a raw '/' directory entry. TarInfo strips + # its slash to empty; preserve only this exact canonical root spelling. + if info.isdir() and info.name == '' and raw[:100].split(b'\0', 1)[0] == b'/': + info.name = '/' + return info + except (tarfile.TarError, ValueError, UnicodeError): + raise InventoryError('inventory-header-invalid') from None + + +def next_member(response, budget, protocol, *, symlink=False): + raw = budget.exact(response, 512) + if not any(raw): + require(not any(budget.exact(response, 512)), 'inventory-end-markers') + trailing = 0 + while True: + raw = budget.read(response, 512) + if not raw: + break + trailing += len(raw) + require(trailing <= 10240 and not any(raw), 'inventory-tail') + require(response.length in (None, 0), 'inventory-truncated') + require(trailing % 512 == 0, 'inventory-tail-alignment') + return None + info = header(raw) + if info.type == tarfile.XHDTYPE: + require(0 < info.size <= 8192 and not info.linkname, 'inventory-pax-invalid') + protocol.pax_times(budget.exact(response, info.size)) + require(not any(budget.exact(response, (-info.size) % 512)), 'inventory-padding') + info = header(budget.exact(response, 512)) + require(not info.pax_headers and not info.sparse and + (not info.linkname or (symlink and info.type == tarfile.SYMTYPE)), + 'inventory-extension-or-link') + require(info.uid >= 0 and info.gid >= 0, 'inventory-owner-invalid') + budget.member(info.size) + return info + + +def tree_archive(response, protocol, expected, budget=None): + """Hash ROOT, retaining only the bounded, fixed source-manifest payloads.""" + budget = budget or Budget() + framing(response) + protocol.stat_matches(expected, ROOT, directory=True) + files, entries, directories = ProducerInventory(), set(), set() + root_name = PurePosixPath(ROOT).name + while True: + info = next_member(response, budget, protocol) + if info is None: + break + name = info.name + # header() already rejects repeated raw directory terminators. + # Absolute, dot, double slash and traversal names also fail here. + relative_path(name) + require(name == root_name or name.startswith(root_name + '/'), 'inventory-root-mismatch') + require(name not in entries, 'inventory-duplicate') + entries.add(name) + if name == root_name: + require(len(entries) == 1 and info.isdir() and info.mode == expected['mode'] & 0o777, + 'inventory-root-mismatch') + else: + require(str(PurePosixPath(name).parent) in directories, 'inventory-parent-missing') + if info.type == tarfile.DIRTYPE: + require(info.size == 0 and 0 <= info.mode <= 0o777 and + info.mode & 0o022 == 0 and info.mode & 0o555 == 0o555, + 'inventory-directory-mode') + directories.add(name) + continue + require(info.type in (tarfile.REGTYPE, tarfile.AREGTYPE) and name != root_name, + 'inventory-member-type') + protocol.regular_mode(info.mode) + path = name[len(root_name) + 1:] + payload = bytearray() if path in SOURCE_MANIFESTS else None + if payload is not None: + require(0 < info.size <= MAX_MANIFEST, 'inventory-manifest-size') + sha = hashlib.sha256() + remaining = info.size + while remaining: + raw = budget.exact(response, min(65536, remaining)) + sha.update(raw) + if payload is not None: + payload.extend(raw) + remaining -= len(raw) + require(not any(budget.exact(response, (-info.size) % 512)), 'inventory-padding') + files[path] = {'sha256': sha.hexdigest(), 'size': info.size, + 'executable': bool(info.mode & 0o111)} + if payload is not None: + files.source_manifests[path] = bytes(payload) + require(root_name in directories and bool(files), 'inventory-empty') + return files + + +def notice_requests(runtime, debian): + """Select only runtime source-package notices from authenticated documents.""" + sources = runtime.get('source_packages') + require(isinstance(sources, dict) and 0 < len(sources) <= MAX_MEMBERS + and isinstance(debian, list) and len(debian) <= MAX_MEMBERS, 'notice-documents-invalid') + require(all(isinstance(name, str) and re.fullmatch(r'[a-z0-9][a-z0-9+.-]*', name) + and isinstance(version, str) and 0 < len(version) <= 256 + for name, version in sources.items()), 'notice-source-invalid') + requests, found = set(), set() + for row in debian: + require(isinstance(row, dict), 'notice-row-invalid') + source = row.get('source') + require(isinstance(source, str), 'notice-row-invalid') + if source not in sources: + continue + package = row.get('package') + require(isinstance(package, str) and re.fullmatch(r'[a-z0-9][a-z0-9+.-]*(?::[a-z0-9]+)?', package) + and row.get('source_version') == sources[source], 'notice-package-invalid') + directory = DOC + '/' + package.split(':')[0] + require(row.get('notice_directory') == directory, 'notice-directory-invalid') + requests.add(directory + '/copyright') + found.add(source) + require(found == set(sources), 'notice-source-missing') + return sorted(requests) + + +def source_manifest(records, reference, expected): + """Parse only fixed manifests authenticated by the producer tree read.""" + require(reference == expected, 'inventory-manifest-path') + payloads = getattr(records, 'source_manifests', {}) + require(isinstance(payloads, dict), 'inventory-manifest-payloads') + raw = payloads.get(expected) + require(isinstance(raw, bytes) and 0 < len(raw) <= MAX_MANIFEST, + 'inventory-manifest-missing-or-size') + identity = records.get(expected) + require(isinstance(identity, dict) and type(identity.get('size')) is int + and identity['size'] == len(raw) + and identity.get('sha256') == hashlib.sha256(raw).hexdigest(), + 'inventory-manifest-bytes-mismatch') + + def unique(pairs): + value = {} + for key, item in pairs: + require(key not in value, 'inventory-manifest-duplicate') + value[key] = item + return value + + def invalid_constant(_): + raise InventoryError('inventory-manifest-json') + + try: + value = json.loads(raw.decode('utf-8'), object_pairs_hook=unique, + parse_constant=invalid_constant) + except (ValueError, UnicodeError, RecursionError): + raise InventoryError('inventory-manifest-json') from None + require(isinstance(value, dict) and type(value.get('schema_version')) is int + and value['schema_version'] == 1, 'inventory-manifest-schema') + return value + + +def crosscheck_archives(records, runtime, contract, *, reader=None): + """Validate source closure and bind provider bytes using the same image reader.""" + require(isinstance(records, dict) and isinstance(runtime, dict) + and isinstance(contract, dict), 'inventory-source-documents') + require(type(contract.get('schema_version')) is int and contract['schema_version'] == 2 + and contract.get('target') == 'portable-final' and contract.get('platform') == 'linux/amd64', + 'inventory-contract-schema') + archives, sources = runtime.get('archives'), contract.get('sources') + require(isinstance(archives, dict) and 0 < len(archives) <= MAX_MEMBERS + and isinstance(sources, list) and 0 < len(sources) <= MAX_MEMBERS, + 'inventory-source-documents') + required = {} + def add(path, sha): + relative_path(path) + require(isinstance(sha, str) and re.fullmatch(r'[0-9a-f]{64}', sha), 'inventory-source-digest') + require(path not in required or required[path] == sha, 'inventory-source-conflict') + required[path] = sha + for name, sha in archives.items(): + require(isinstance(name, str) and '/' not in name, 'inventory-source-name') + add('sources/debian/' + name, sha) + source_names = set() + for source in sources: + require(isinstance(source, dict), 'inventory-source-row') + name = source.get('name') + require(isinstance(name, str) and name in {'openssl', 'sdl3', 'sdl3-image', 'sdl3-ttf'} + and name not in source_names, 'inventory-source-cohort') + source_names.add(name) + sha = source.get('sha256') + require(isinstance(sha, str), 'inventory-source-digest') + add('sources/' + sha + '.tar.gz', sha) + require(source_names == {'openssl', 'sdl3', 'sdl3-image', 'sdl3-ttf'}, 'inventory-source-cohort') + audio = source_manifest(records, contract.get('audio_manifest'), 'audio-toolchain.json') + dependencies = audio.get('dependencies') + require(isinstance(dependencies, list) and len(dependencies) == 3, + 'inventory-audio-sources') + mixer = audio.get('sdl_mixer') + require(isinstance(mixer, dict) and mixer.get('name') == 'SDL3_mixer', 'inventory-audio-source') + names = set() + for entry in [mixer, *dependencies]: + require(isinstance(entry, dict), 'inventory-audio-source') + name, source = entry.get('name'), entry.get('source') + require(isinstance(name, str) and re.fullmatch(r'[A-Za-z0-9_.+-]{1,128}', name) + and name not in names and isinstance(source, dict), 'inventory-audio-source') + names.add(name) + sha = source.get('sha256') + require(isinstance(sha, str), 'inventory-source-digest') + add('sources/' + sha + '.tar.gz', sha) + require(names == {'SDL3_mixer', 'libogg', 'libopus', 'libopusfile'}, 'inventory-audio-sources') + compiler = contract.get('compiler') + require(isinstance(compiler, dict), 'inventory-compiler-schema') + llvm = source_manifest(records, compiler.get('toolchain_manifest'), 'llvm-toolchain.json') + version, source, metadata = llvm.get('version'), llvm.get('source'), llvm.get('metadata') + require(isinstance(version, str) and re.fullmatch(r'[0-9]{1,4}\.[0-9]{1,4}\.[0-9]{1,4}', version) + and isinstance(source, dict) and isinstance(metadata, dict), 'inventory-llvm-source') + # The installer writes this exact filename, and portable/Dockerfile + # copies its source directory into sources/llvm without flattening it. + archive = 'llvm-project-' + version + '.src.tar.xz' + require(metadata.get('source_archive') == 'share/atrinik/sources/' + archive, + 'inventory-llvm-archive-path') + add('sources/llvm/' + archive, source.get('sha256')) + + # legal.py emits both providers. Bind their complete archive declarations + # to the producer manifests, so dropping a provider or one archive cannot + # shrink the corresponding-source requirements. + curl = source_manifest(records, contract.get('curl_manifest'), 'curl-toolchain.json') + shared = source_manifest(records, curl.get('cares_manifest'), 'classic-curl-toolchain.json') + version, source_version, archives = curl.get('version'), curl.get('source_version'), curl.get('archives') + require(curl.get('source_package') == 'curl' and isinstance(version, str) + and re.fullmatch(r'[0-9]{1,4}\.[0-9]{1,4}\.[0-9]{1,4}', version) + and isinstance(source_version, str) and len(source_version) <= 128 + and re.fullmatch(re.escape(version) + r'-[0-9A-Za-z.+~]+', source_version), + 'inventory-curl-source') + require(isinstance(archives, dict) and set(archives) == { + 'curl_' + source_version + '.debian.tar.xz', 'curl_' + source_version + '.dsc', + 'curl_' + version + '.orig.tar.gz', 'curl_' + version + '.orig.tar.gz.asc'}, + 'inventory-curl-archives') + cares = shared.get('cares') + require(isinstance(cares, dict) and isinstance(cares.get('version'), str) + and re.fullmatch(r'[0-9]{1,4}\.[0-9]{1,4}\.[0-9]{1,4}', cares['version']) + and isinstance(cares.get('sha256'), str), 'inventory-cares-source') + expected_sources = { + 'libcurl': {'sources/curl/' + name: sha for name, sha in archives.items()}, + 'libcares': {'sources/' + cares['sha256'] + '.tar.gz': cares['sha256']}} + providers = runtime.get('custom_providers') + require(isinstance(providers, dict) and set(providers) == set(expected_sources), + 'inventory-custom-providers') + for name, sources in expected_sources.items(): + provider = providers[name] + require(isinstance(provider, dict) and set(provider) == { + 'path', 'sha256', 'version', 'source_archives', 'build_receipt'}, 'inventory-custom-provider') + require(isinstance(provider['path'], str) and len(provider['path']) <= MAX_PATH and re.fullmatch( + r'/usr/local/lib/' + name + r'\.so(?:\.[0-9]+)+', provider['path']) + and isinstance(provider['sha256'], str) and re.fullmatch(r'[0-9a-f]{64}', provider['sha256']) + and provider['version'] == (version if name == 'libcurl' else cares['version']) + and provider['build_receipt'] == 'curl-source-receipt.json' + and isinstance(provider['source_archives'], dict) and provider['source_archives'] == sources, + 'inventory-custom-provider') + for path, sha in sources.items(): + add(path, sha) + for path, sha in required.items(): + record = records.get(path) + require(isinstance(record, dict) and type(record.get('size')) is int + and 0 < record['size'] <= MAX_FILE and record.get('sha256') == sha, + 'inventory-source-bytes-mismatch') + require(isinstance(reader, ProviderReader), 'inventory-provider-reader-required') + for provider in providers.values(): + observed = reader.read_provider(provider['path']) + require(isinstance(observed, dict) and set(observed) == {'sha256', 'size', 'executable'} + and isinstance(observed['sha256'], str) + and re.fullmatch(r'[0-9a-f]{64}', observed['sha256']) + and type(observed['size']) is int and 0 < observed['size'] <= MAX_FILE + and type(observed['executable']) is bool, 'inventory-provider-identity') + require(observed['sha256'] == provider['sha256'], 'inventory-provider-bytes-mismatch') + + +class BudgetResponse: + """Keep single-file protocol reads under the shared inventory deadline.""" + def __init__(self, response, budget): + self.response, self.budget = response, budget + + def __getattr__(self, name): + return getattr(self.response, name) + + def read(self, size): + return self.budget.read(self.response, size) + + +class InstalledInventory(ProviderReader): + """Use only with an authenticated StoppedImage; does not grant release trust.""" + def __init__(self, image, protocol): + self.image, self.protocol = image, protocol + self.budget = Budget() + + def _fence(self): + self.budget.check() + self.image.fence() + self.budget.check() + + def _directory_owner(self, path, expected): + """Read only the directory header, never its recursive archive contents. + + Docker HEAD omits UID/GID. The first member of its authenticated GET + supplies ownership; close immediately to bound even the root archive. + """ + conn, response = self._get(path) + try: + require(self.protocol.path_stat(response) == expected, 'inventory-path-drift') + framing(response) + info = next_member(response, self.budget, self.protocol) + names = {'/'} if path == '/' else {PurePosixPath(path).name} + require(info is not None and info.name in names and info.isdir() + and info.size == 0 and info.mode == expected['mode'] & 0o777, + 'inventory-directory-entry') + require(info.uid == 0, 'inventory-directory-owner') + finally: + conn.close() + + def _parents(self, path, *, protected=False): + result = {} + for parent in reversed(PurePosixPath(path).parents): + if str(parent) == '/' and not protected: + continue + self.budget.check() + value = self.image.stat(str(parent)) + if str(parent) == '/': + require(value.get('name') == '/', 'inventory-directory-entry') + self.protocol.stat_matches({**value, 'name': ''}, '/', directory=True) + else: + self.protocol.stat_matches(value, str(parent), directory=True) + if protected: + self._directory_owner(str(parent), value) + result[str(parent)] = value + return result + + def _recheck(self, observations): + for path, before in observations.items(): + self.budget.check() + require(self.image.stat(path) == before, 'inventory-path-drift') + self._fence() + + def _get(self, path): + return self.image.docker.request('GET', '/containers/' + self.image.identifier + '/archive?' + + urllib.parse.urlencode({'path': path}), + {'Accept-Encoding': 'identity'}) + + def producer(self): + self._fence() + observations = self._parents(ROOT) + expected = self.image.stat(ROOT) + self.protocol.stat_matches(expected, ROOT, directory=True) + observations[ROOT] = expected + conn, response = self._get(ROOT) + try: + require(self.protocol.path_stat(response) == expected, 'inventory-path-drift') + result = tree_archive(response, self.protocol, expected, self.budget) + finally: + conn.close() + self._recheck(observations) + return result + + def read_provider(self, path): + return self._regular(path, provider=True) + + def common_licenses(self): + return {name: self._regular(COMMON + '/' + name) for name in COMMON_LICENSE_NAMES} + + def _regular(self, path, *, executable=False, provider=False): + # Provider paths have already passed source-manifest validation; enforce + # the fixed library namespace again at the actual archive request. + provider_path = isinstance(path, str) and len(path) <= MAX_PATH and re.fullmatch( + r'/usr/local/lib/lib(?:curl|cares)\.so(?:\.[0-9]+)+', path) is not None + require(not provider or provider_path, 'inventory-file-not-allowed') + require((provider and provider_path) or path in {COMMON + '/' + name for name in COMMON_LICENSE_NAMES} + or path in BUILD_TOOLS.values() or path.startswith(DOC + '/'), + 'inventory-file-not-allowed') + if path.startswith(DOC + '/'): + under_doc(path) + self._fence() + protected = path in BUILD_TOOLS.values() or provider + expected = self.image.stat(path) + self.protocol.stat_matches(expected, path) + self.protocol.regular_mode(expected['mode'], executable=executable) + require(0 < expected['size'] <= self.protocol.BINARY_LIMIT, 'file-size-limit') + observations = self._parents(path, protected=protected) + observations[path] = expected + self.budget.member(expected['size']) + conn, response = self._get(path) + try: + require(self.protocol.path_stat(response) == expected, 'inventory-path-drift') + identity, _ = self.protocol.single_file_archive( + BudgetResponse(response, self.budget), path, expected, retain=False, executable=executable, root_owned=protected) + self.budget.check() + finally: + conn.close() + self._recheck(observations) + return {'sha256': identity['sha256'], 'size': identity['size'], + 'executable': bool(expected['mode'] & 0o111)} + + + def _alias(self, path, expected): + """Read the raw one-hop target; Docker HEAD's LinkTarget is fully resolved. + + Moby v28.5.1 daemon/archive_unix.go archives the named resource without + dereferencing it. Never use the daemon-resolved target as traversal input. + """ + conn, response = self._get(path) + try: + require(self.protocol.path_stat(response) == expected, 'notice-alias-drift') + framing(response) + info = next_member(response, self.budget, self.protocol, symlink=True) + require(info is not None and info.type == tarfile.SYMTYPE and info.size == 0 + and info.name == PurePosixPath(path).name + and info.mode == expected['mode'] & 0o777, + 'notice-alias-entry') + raw = info.linkname + require(isinstance(raw, str) and 0 < len(raw.encode('utf-8')) <= MAX_PATH + and len(raw.encode('utf-8')) == expected['size'] + and all(32 < ord(c) < 127 for c in raw) and '\\' not in raw, + 'notice-alias-target') + require(next_member(response, self.budget, self.protocol) is None, + 'notice-alias-extra-member') + return raw + finally: + conn.close() + + def _notice(self, requested): + under_doc(requested) + self._fence() + observations = self._parents(DOC + '/copyright') + links, visited = {}, set() + + def resolve(path): + under_doc(path) + parts = path[len(DOC) + 1:].split('/') + current = DOC + for index, part in enumerate(parts): + current += '/' + part + self.budget.check() + value = self.image.stat(current) + require(current not in observations or observations[current] == value, + 'notice-alias-drift') + observations[current] = value + mode = value.get('mode') + require(type(mode) is int, 'notice-mode-invalid') + if mode & ~0o777 == GO_SYMLINK: + require(value.get('name') == part and type(value.get('size')) is int + and 0 < value['size'] <= MAX_PATH, 'notice-alias-stat') + under_doc(value.get('linkTarget')) + require(current not in visited and len(visited) < MAX_HOPS, 'notice-alias-cycle-or-limit') + visited.add(current) + raw = self._alias(current, value) + links[current] = raw + # Interpret raw components ourselves. Even temporary escape + # followed by re-entry fails; HEAD's final target is no proof. + if raw.startswith('/'): + require(raw.startswith(DOC + '/'), 'notice-boundary') + resolved_parts = [] + target_parts = raw[len(DOC) + 1:].split('/') + else: + resolved_parts = current[len(DOC) + 1:].split('/')[:-1] + target_parts = raw.split('/') + descended = False + for component in target_parts: + require(component != '', 'notice-alias-target') + if component == '..': + # Only leading parent hops are safe to normalize: + # later components may themselves name symlinks. + require(not descended and bool(resolved_parts), 'notice-boundary') + resolved_parts.pop() + elif component != '.': + descended = True + resolved_parts.append(component) + require(bool(resolved_parts), 'notice-boundary') + canonical = resolve(DOC + '/' + '/'.join(resolved_parts)) + require(canonical == value['linkTarget'], 'notice-alias-drift') + if index + 1 < len(parts): + canonical = resolve(canonical + '/' + '/'.join(parts[index + 1:])) + return canonical + if index + 1 < len(parts) or mode & GO_DIRECTORY: + self.protocol.stat_matches(value, current, directory=True) + else: + self.protocol.stat_matches(value, current) + return current + + canonical = resolve(requested) + result = self._regular(canonical) + for path, raw in links.items(): + require(self._alias(path, observations[path]) == raw, 'notice-alias-drift') + self._recheck(observations) + return {**result, 'resolved_path': canonical} + + def debian_notices(self, runtime, debian): + return {requested: self._notice(requested) + for requested in notice_requests(runtime, debian)} + + + def build_tools(self, installed): + require(isinstance(installed, dict), 'build-tool-schema') + receipt = installed.get('build_tools') + require(isinstance(receipt, dict) and set(receipt) == {'schema_version', 'tools'} + and type(receipt['schema_version']) is int and receipt['schema_version'] == 1 + and isinstance(receipt['tools'], dict) and set(receipt['tools']) == set(BUILD_TOOLS), + 'build-tool-schema') + result = {} + for role, path in BUILD_TOOLS.items(): + entry = receipt['tools'][role] + require(isinstance(entry, dict) and set(entry) == { + 'path', 'resolved_path', 'sha256', 'size', 'executable', 'version'}, 'build-tool-fields') + require(entry['path'] == path and entry['resolved_path'] == path, + 'build-tool-path') + require(isinstance(entry['sha256'], str) and re.fullmatch(r'[0-9a-f]{64}', entry['sha256']) + and type(entry['size']) is int and 0 < entry['size'] <= MAX_BUILD_TOOL + and entry['executable'] is True and isinstance(entry['version'], str) + and 0 < len(entry['version']) <= 65536 + and not any((ord(char) < 32 and char != '\n') or ord(char) == 127 + for char in entry['version']) + and re.fullmatch(r'cmake version [0-9]+\.[0-9]+\.[0-9]+' if role == 'cmake' + else r'[0-9]+\.[0-9]+(?:\.[0-9]+)?', + entry['version'].splitlines()[0]) is not None + and isinstance(installed.get('tools'), dict) + and installed['tools'].get(role) == entry['version'].splitlines()[0], + 'build-tool-identity') + observed = self._regular(path, executable=True) + require(all(observed[key] == entry[key] for key in ('sha256', 'size', 'executable')), + 'build-tool-bytes-mismatch') + result[role] = dict(entry) + return result diff --git a/tools/tests/test_collect_released_sdk_metadata.py b/tools/tests/test_collect_released_sdk_metadata.py index a84c1be..9054b23 100644 --- a/tools/tests/test_collect_released_sdk_metadata.py +++ b/tools/tests/test_collect_released_sdk_metadata.py @@ -469,6 +469,33 @@ def test_archive_rejects_links_types_paths_and_extra_members(self): M.single_file_archive(Response(archive(path, raw, **option)), path, file_stat(path, raw), retain=True) + def test_archive_empty_producer_file_requires_explicit_opt_in(self): + path, raw = '/opt/atrinik-portable/sources/empty.txt', b'' + for retain in (True, False): + with self.subTest(retain=retain): + with self.assertRaisesRegex(M.Stop, 'file-size-limit'): + M.single_file_archive(Response(archive(path, raw)), path, + file_stat(path, raw), retain=retain) + identity, actual = M.single_file_archive(Response(archive(path, raw)), path, + file_stat(path, raw), retain=retain, allow_empty=True) + self.assertEqual(identity['size'], 0) + self.assertEqual(identity['sha256'], M.hashlib.sha256(raw).hexdigest()) + self.assertEqual(actual, raw if retain else None) + + def test_empty_opt_in_preserves_archive_permissions_limits_and_link_rejection(self): + path, raw = '/opt/atrinik-portable/sources/empty.txt', b'' + for option in ({'mode': 0o666}, {'kind': tarfile.SYMTYPE}, + {'kind': tarfile.LNKTYPE}, {'extra': True}): + with self.subTest(option=option), self.assertRaises(M.Stop): + M.single_file_archive(Response(archive(path, raw, **option)), path, + file_stat(path, raw, option.get('mode', 0o644)), + retain=False, allow_empty=True) + for retain, limit in ((True, M.METADATA_LIMIT), (False, M.BINARY_LIMIT)): + for size in (-1, limit + 1): + with self.subTest(retain=retain, size=size), self.assertRaises(M.Stop): + M.single_file_archive(Response(archive(path, raw)), path, + dict(file_stat(path, raw), size=size), retain=retain, allow_empty=True) + def test_archive_pax_timestamps_only(self): path, raw = M.LLVM_MANIFEST, b'normal metadata' result, actual = M.single_file_archive(Response(archive(path, raw, pax={'mtime': '123.456'})), @@ -755,6 +782,21 @@ def test_portable_collection_checks_all_tool_roles_against_streamed_bytes(self): self.assertFalse(result['image_code_executed']) self.assertFalse(result['final_portable_cohort']) + def test_collection_rejects_empty_metadata_notices_and_canonical_tools(self): + for target in M.TARGETS: + for path in (M.LLVM_MANIFEST, M.LLVM_NOTICES[0], + M.PREFIX + '/bin/clang-23', M.PREFIX + '/lib/libunwind.a'): + data, binary = self.portable_fixture() + if target != 'portable-preliminary': + data = {name: data.get(name, b'public metadata fixture') + for name in M.COMMON_FILES + M.TARGET_FILES[target]} + data['/usr/local/share/atrinik/llvm-toolchain.json'] = data[M.LLVM_MANIFEST] + (data if path in data else binary)[path] = b'' + with self.subTest(target=target, path=path): + result, _ = self.collect_portable_fixture(data, binary, target) + self.assertEqual(result['status'], 'stopped') + self.assertEqual(result['reason'], 'file-size-limit') + def test_portable_contract_and_compiler_receipt_hashes_are_required(self): base = '/opt/atrinik-portable/' for name in ('contract.json', 'compiler-evidence.json'): diff --git a/tools/tests/test_released_sdk_inventory.py b/tools/tests/test_released_sdk_inventory.py new file mode 100644 index 0000000..7b2487a --- /dev/null +++ b/tools/tests/test_released_sdk_inventory.py @@ -0,0 +1,1002 @@ +# Copyright (c) 2026 Atrinik contributors +# SPDX-License-Identifier: MIT +"""Offline bounded inventory trust tests. No Docker, network or credentials.""" +import base64 +import copy +import hashlib +import importlib.util +import io +import json +from pathlib import Path +import tarfile +import unittest +from unittest.mock import Mock, patch +import urllib.parse + +ROOT = Path(__file__).resolve().parents[2] + + +def load(name, path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +M = load('released_sdk_inventory', ROOT / 'tools/released_sdk_inventory.py') +P = load('inventory_collector', ROOT / 'tools/collect-released-sdk-metadata.py') + + +class Response(io.BytesIO): + chunked = True + length = None + + def __init__(self, raw, expected=None): + super().__init__(raw) + self.headers = {'Content-Type': 'application/x-tar'} + if expected is not None: + self.headers['X-Docker-Container-Path-Stat'] = base64.b64encode(json.dumps(expected).encode()).decode() + + def getheader(self, name, default=None): + return self.headers.get(name, default) + + +def directory(path): + return {'name': Path(path).name if path != '/' else '/', 'mode': M.GO_DIRECTORY | 0o755, + 'size': 4096, 'mtime': 'fixed', 'linkTarget': ''} + + +def regular(path, raw): + return {'name': Path(path).name, 'mode': 0o644, 'size': len(raw), + 'mtime': 'fixed', 'linkTarget': ''} + + +def tar(entries): + raw = io.BytesIO() + with tarfile.open(fileobj=raw, mode='w', format=tarfile.PAX_FORMAT) as out: + for name, data, kind, extra in entries: + member = tarfile.TarInfo(name) + member.type = kind + member.mode = 0o755 if kind == tarfile.DIRTYPE else 0o644 + member.size = len(data) + for key, value in extra.items(): + setattr(member, key, value) + out.addfile(member, io.BytesIO(data)) + return raw.getvalue() + + +DIR = ('atrinik-portable', b'', tarfile.DIRTYPE, {}) +FILE = ('atrinik-portable/a', b'abc', tarfile.REGTYPE, {}) +EMPTY = ('atrinik-portable/empty', b'', tarfile.REGTYPE, {}) + + +class InventoryTests(unittest.TestCase): + def setUp(self): + for item in (patch.object(P, 'https_get', side_effect=AssertionError('network forbidden')), + patch.object(P.LocalConnection, 'connect', side_effect=AssertionError('Docker forbidden'))): + item.start() + self.addCleanup(item.stop) + + def parse(self, entries=None, raw=None): + return M.tree_archive(Response(raw if raw is not None else tar(entries)), P, directory(M.ROOT)) + + def denied(self, entries=None, raw=None, code=None): + with self.assertRaises((M.InventoryError, P.Stop)) as caught: + self.parse(entries=entries, raw=raw) + if code: + self.assertEqual(str(caught.exception), code) + + def test_complete_inventory_and_zero_byte_file(self): + result = self.parse([DIR, FILE, EMPTY]) + self.assertEqual(result, { + 'a': {'sha256': hashlib.sha256(b'abc').hexdigest(), 'size': 3, 'executable': False}, + 'empty': {'sha256': hashlib.sha256(b'').hexdigest(), 'size': 0, 'executable': False}}) + self.assertEqual(M.inventory_digest(result), + hashlib.sha256(json.dumps(result, sort_keys=True, separators=(',', ':')).encode()).hexdigest()) + + def test_executable_bit_and_pax_times(self): + result = self.parse([DIR, ('atrinik-portable/run', b'x', tarfile.REGTYPE, + {'mode': 0o755, 'pax_headers': {'mtime': '123.25'}})]) + self.assertTrue(result['run']['executable']) + + def test_all_links_special_sparse_and_extensions_fail(self): + for kind in (tarfile.SYMTYPE, tarfile.LNKTYPE, tarfile.FIFOTYPE, + tarfile.CHRTYPE, tarfile.BLKTYPE, tarfile.GNUTYPE_SPARSE, + tarfile.XGLTYPE, tarfile.GNUTYPE_LONGNAME): + with self.subTest(kind=kind): + self.denied([DIR, ('atrinik-portable/bad', b'', kind, {})]) + for key in ('path', 'linkpath', 'size', 'GNU.sparse.map', 'SCHILY.xattr.secret'): + with self.subTest(pax=key): + self.denied([DIR, ('atrinik-portable/a', b'x', tarfile.REGTYPE, + {'pax_headers': {key: '0'}})], code='unsafe-pax-header') + + def test_paths_duplicates_and_collisions_fail(self): + for name in ('/atrinik-portable/a', 'atrinik-portable/../a', 'atrinik-portable/./a', + 'atrinik-portable//a', 'other/a', 'atrinik-portable/a\\b', + 'atrinik-portable/a\nb', 'atrinik-portable/d/file'): + with self.subTest(name=name): + self.denied([DIR, (name, b'x', tarfile.REGTYPE, {})]) + self.denied([DIR, FILE, FILE], code='inventory-duplicate') + self.denied([DIR, FILE, ('atrinik-portable/a', b'', tarfile.DIRTYPE, {})], + code='inventory-duplicate') + self.denied([FILE], code='inventory-parent-missing') + self.denied([DIR], code='inventory-empty') + for name in ('atrinik-portable//', 'atrinik-portable///'): + self.denied([(name, b'', tarfile.DIRTYPE, {}), FILE], code='inventory-path-invalid') + + def test_truncation_extra_members_and_framing(self): + raw = tar([DIR, FILE]) + for size in (0, 511, 512, 1024, 1535, 1536, 2048): + with self.subTest(size=size): + self.denied(raw=raw[:size]) + self.denied(raw=raw[:1536] + bytes(512) + b'x' + bytes(511), code='inventory-end-markers') + self.denied(raw=raw + b'x', code='inventory-tail') + self.denied(raw=raw + bytes(1), code='inventory-tail-alignment') + self.denied(raw=raw + tar([FILE]), code='inventory-tail') + response = Response(raw) + response.chunked = False + with self.assertRaisesRegex(M.InventoryError, 'inventory-unframed'): + M.tree_archive(response, P, directory(M.ROOT)) + response = Response(raw) + response.length = 123 + with self.assertRaisesRegex(M.InventoryError, 'inventory-truncated'): + M.tree_archive(response, P, directory(M.ROOT)) + + def test_limits_and_deadline(self): + for name, value in (('MAX_MEMBERS', 1), ('MAX_FILE', 2), ('MAX_TOTAL', 2), ('MAX_PATH', 2)): + with self.subTest(limit=name), patch.object(M, name, value): + self.denied([DIR, FILE]) + with patch.object(M.time, 'monotonic', side_effect=[0, 1000]): + self.denied([DIR, FILE], code='inventory-deadline') + response = Response(tar([DIR, FILE])) + budget = M.Budget() + with patch.object(M.time, 'monotonic', side_effect=[budget.deadline - 1, budget.deadline]): + with self.assertRaisesRegex(M.InventoryError, 'inventory-deadline'): + budget.read(response, 512) + + def test_single_file_zero_and_extra_member(self): + raw = tar([('empty', b'', tarfile.REGTYPE, {})]) + with self.assertRaisesRegex(P.Stop, 'file-size-limit'): + P.single_file_archive(Response(raw), '/empty', regular('/empty', b''), retain=False) + with self.assertRaises(P.Stop): + P.single_file_archive(Response(tar([('a', b'x', tarfile.REGTYPE, {}), + ('extra', b'x', tarfile.REGTYPE, {})])), + '/a', regular('/a', b'x'), retain=False) + + def fake_image(self): + image = Mock() + image.identifier = 'a' * 64 + image.stat.side_effect = lambda path: directory(path) + return image + + def installed_image(self, payloads=None, metadata=None, missing=()): + """Exercise real archive readers; HEAD intentionally has no UID/GID.""" + payloads = payloads if payloads is not None else { + '/usr/local/lib/' + name + '.so.1.0.0': name.encode() + for name in ('libcurl', 'libcares')} + metadata = metadata or {} + image = self.fake_image() + connections = [] + def stat(path): + if path in missing: + raise P.Stop('docker-http-status') + if path in payloads: + return {**regular(path, payloads[path]), 'mode': metadata.get(path, {}).get('mode', 0o755)} + return directory(path) + def request(method, route, headers): + self.assertEqual(method, 'GET') + path = urllib.parse.parse_qs(urllib.parse.urlsplit(route).query)['path'][0] + expected = stat(path) + conn = Mock() + connections.append(conn) + raw = payloads.get(path, b'') + kind = tarfile.REGTYPE if path in payloads else tarfile.DIRTYPE + extra = {'mode': expected['mode'] & 0o777, **metadata.get(path, {})} + name = Path(path).name if path != '/' else '/' + return conn, Response(tar([(name, raw, kind, extra)]), expected) + image.stat.side_effect = stat + image.docker.request.side_effect = request + image.connections = connections + return image + + def crosscheck(self, records, runtime, contract, *, image=None): + reader = M.InstalledInventory(image if image is not None else self.installed_image(), P) + return M.crosscheck_archives(records, runtime, contract, reader=reader) + + def test_created_image_fences_and_connection_close(self): + image = self.fake_image() + conn = Mock() + image.docker.request.return_value = (conn, Response(tar([DIR, FILE]), directory(M.ROOT))) + result = M.InstalledInventory(image, P).producer() + self.assertIn('a', result) + self.assertEqual(image.fence.call_count, 2) + self.assertEqual(image.stat.call_count, 4) + conn.close.assert_called_once() + request = image.docker.request.call_args + self.assertEqual(urllib.parse.parse_qs(urllib.parse.urlsplit(request.args[1]).query), {'path': [M.ROOT]}) + + def test_root_stat_drift_and_archive_failure_close(self): + for drift in ('root', 'parent', 'get', 'tar'): + image = self.fake_image() + conn = Mock() + states = {'/opt': 0, M.ROOT: 0} + def stat(path): + states[path] += 1 + value = directory(path) + if states[path] == 2 and ((drift == 'root' and path == M.ROOT) or (drift == 'parent' and path == '/opt')): + value['mtime'] = 'changed' + return value + image.stat.side_effect = stat + expected = directory(M.ROOT) + if drift == 'get': + expected['mtime'] = 'changed' + image.docker.request.return_value = (conn, Response(b'bad' if drift == 'tar' else tar([DIR, FILE]), expected)) + with self.subTest(drift=drift), self.assertRaises(M.InventoryError): + M.InstalledInventory(image, P).producer() + conn.close.assert_called_once() + + def documents(self): + return {'source_packages': {'foo': '1.0'}, 'archives': {'foo.tar.xz': 'a' * 64}}, [ + {'source': 'foo', 'source_version': '1.0', 'package': 'foo:amd64', 'notice_directory': M.DOC + '/foo'}, + {'source': 'unrelated', 'notice_directory': '/secret'}] + + def test_notice_request_selection_and_no_descriptor_path_extension(self): + runtime, debian = self.documents() + self.assertEqual(M.notice_requests(runtime, debian), [M.DOC + '/foo/copyright']) + for key, bad in (('notice_directory', '/secret'), ('package', '../foo'), ('source_version', '2.0')): + rows = copy.deepcopy(debian) + rows[0][key] = bad + with self.subTest(key=key), self.assertRaises(M.InventoryError): + M.notice_requests(runtime, rows) + with self.assertRaisesRegex(M.InventoryError, 'notice-source-missing'): + M.notice_requests(runtime, []) + + def test_archive_crosschecks(self): + payloads, runtime, contract = self.manifest_fixture() + records = self.producer_fixture(payloads) + self.crosscheck(records, runtime, contract) + for path in (path for path in records if path.startswith('sources/')): + bad = copy.deepcopy(records) + bad[path]['sha256'] = 'c' * 64 + with self.assertRaisesRegex(M.InventoryError, 'inventory-source-bytes-mismatch'): + self.crosscheck(bad, runtime, contract) + runtime['archives'] = {'../secret': 'a' * 64} + with self.assertRaises(M.InventoryError): + self.crosscheck(records, runtime, contract) + + def manifest_fixture(self): + # Use the producer's actual schemas and path declarations, with small + # stand-ins for the source archives so tests never download or execute. + contract = json.loads((ROOT / 'portable/contract.json').read_text()) + audio = json.loads((ROOT / 'audio-toolchain.json').read_text()) + llvm = json.loads((ROOT / 'llvm-toolchain.json').read_text()) + curl = json.loads((ROOT / 'portable/curl-toolchain.json').read_text()) + shared = json.loads((ROOT / 'classic-curl-toolchain.json').read_text()) + payloads = {'sources/debian/foo.tar.xz': b'debian source'} + runtime = {'archives': {'foo.tar.xz': hashlib.sha256(b'debian source').hexdigest()}} + for entry in contract['sources']: + raw = ('source ' + entry['name']).encode() + entry['sha256'] = hashlib.sha256(raw).hexdigest() + payloads['sources/' + entry['sha256'] + '.tar.gz'] = raw + for entry in [audio['sdl_mixer'], *audio['dependencies']]: + raw = ('source ' + entry['name']).encode() + entry['source']['sha256'] = hashlib.sha256(raw).hexdigest() + payloads['sources/' + entry['source']['sha256'] + '.tar.gz'] = raw + llvm['source']['sha256'] = hashlib.sha256(b'llvm source').hexdigest() + payloads['sources/llvm/' + Path(llvm['metadata']['source_archive']).name] = b'llvm source' + payloads['audio-toolchain.json'] = json.dumps(audio).encode() + payloads['llvm-toolchain.json'] = json.dumps(llvm).encode() + provider_sources = {'libcurl': {}} + for name in curl['archives']: + raw = ('source ' + name).encode() + sha = hashlib.sha256(raw).hexdigest() + curl['archives'][name] = sha + payloads['sources/curl/' + name] = raw + provider_sources['libcurl']['sources/curl/' + name] = sha + sha = hashlib.sha256(b'cares source').hexdigest() + shared['cares']['sha256'] = sha + payloads['sources/' + sha + '.tar.gz'] = b'cares source' + provider_sources['libcares'] = {'sources/' + sha + '.tar.gz': sha} + runtime['custom_providers'] = { + name: {'path': '/usr/local/lib/' + name + '.so.1.0.0', + 'sha256': hashlib.sha256(name.encode()).hexdigest(), 'version': version, + 'source_archives': provider_sources[name], 'build_receipt': 'curl-source-receipt.json'} + for name, version in (('libcurl', curl['version']), ('libcares', shared['cares']['version']))} + payloads['curl-toolchain.json'] = json.dumps(curl).encode() + payloads['classic-curl-toolchain.json'] = json.dumps(shared).encode() + return payloads, runtime, contract + + def producer_fixture(self, payloads): + entries = [DIR] + directories = set() + for name, raw in sorted(payloads.items()): + for parent in reversed(Path(name).parents): + if str(parent) != '.' and str(parent) not in directories: + entries.append(('atrinik-portable/' + str(parent), b'', tarfile.DIRTYPE, {})) + directories.add(str(parent)) + entries.append(('atrinik-portable/' + name, raw, tarfile.REGTYPE, {})) + return self.parse(entries) + + def test_authenticated_manifests_close_audio_and_llvm_sources(self): + payloads, runtime, contract = self.manifest_fixture() + records = self.producer_fixture(payloads) + self.crosscheck(records, runtime, contract) + self.assertEqual(set(records.source_manifests), set(M.SOURCE_MANIFESTS)) + self.assertEqual(M.inventory_digest(records), M.inventory_digest(dict(records))) + # A hash-only inventory cannot manufacture authenticated manifest data. + with self.assertRaisesRegex(M.InventoryError, 'inventory-manifest-missing-or-size'): + self.crosscheck(dict(records), runtime, contract) + + def test_contract_schema_and_manifest_references_are_mandatory(self): + payloads, runtime, contract = self.manifest_fixture() + for key in ('audio_manifest', 'curl_manifest', 'compiler'): + bad = copy.deepcopy(contract) + del bad[key] + # Removing both the declaration and all its archives cannot reduce + # the qualification requirements to the remaining subset. + partial = {name: raw for name, raw in payloads.items() if not name.startswith('sources/')} + with self.subTest(missing=key), self.assertRaises(M.InventoryError): + self.crosscheck(self.producer_fixture(partial), runtime, bad) + bad = copy.deepcopy(contract) + del bad['compiler']['toolchain_manifest'] + with self.assertRaisesRegex(M.InventoryError, 'inventory-manifest-path'): + self.crosscheck(self.producer_fixture(payloads), runtime, bad) + for key, values in (('schema_version', [None, True, 1, 2.0, '2']), + ('target', [None, 'classic-final']), ('platform', [None, 'linux/arm64']), + ('compiler', [None, [], 'llvm-toolchain.json'])): + for value in values: + bad = copy.deepcopy(contract) + bad[key] = value + with self.subTest(key=key, value=value), self.assertRaises(M.InventoryError): + self.crosscheck(self.producer_fixture(payloads), runtime, bad) + + def test_contract_source_cohort_cannot_be_shortened_or_duplicated(self): + payloads, runtime, contract = self.manifest_fixture() + for index in range(len(contract['sources'])): + bad = copy.deepcopy(contract) + removed = bad['sources'].pop(index) + partial = dict(payloads) + del partial['sources/' + removed['sha256'] + '.tar.gz'] + with self.subTest(removed=removed['name']), self.assertRaisesRegex( + M.InventoryError, 'inventory-source-cohort'): + self.crosscheck(self.producer_fixture(partial), runtime, bad) + for rows in ([*contract['sources'], contract['sources'][0]], + [{**row, 'name': 'other'} for row in contract['sources']], [], None, {}): + bad = {**contract, 'sources': rows} + with self.subTest(rows=rows), self.assertRaises(M.InventoryError): + self.crosscheck(self.producer_fixture(payloads), runtime, bad) + + def test_audio_cohort_cannot_be_shortened_or_renamed(self): + payloads, runtime, contract = self.manifest_fixture() + original = json.loads(payloads['audio-toolchain.json']) + for index in range(len(original['dependencies'])): + audio = copy.deepcopy(original) + removed = audio['dependencies'].pop(index) + partial = dict(payloads) + del partial['sources/' + removed['source']['sha256'] + '.tar.gz'] + partial['audio-toolchain.json'] = json.dumps(audio).encode() + with self.subTest(removed=removed['name']), self.assertRaises(M.InventoryError): + self.crosscheck(self.producer_fixture(partial), runtime, contract) + for role in ('sdl_mixer', 'dependency'): + audio = copy.deepcopy(original) + entry = audio['sdl_mixer'] if role == 'sdl_mixer' else audio['dependencies'][0] + entry['name'] = 'other' + partial = {**payloads, 'audio-toolchain.json': json.dumps(audio).encode()} + with self.subTest(role=role), self.assertRaises(M.InventoryError): + self.crosscheck(self.producer_fixture(partial), runtime, contract) + + def test_both_custom_providers_and_all_fields_are_mandatory(self): + payloads, runtime, contract = self.manifest_fixture() + variants = [{}] + for value in (None, [], {}, {'other': runtime['custom_providers']['libcurl']}): + variants.append({**runtime, 'custom_providers': value}) + variants.append({key: value for key, value in runtime.items() if key != 'custom_providers'}) + for name in runtime['custom_providers']: + bad = copy.deepcopy(runtime) + del bad['custom_providers'][name] + variants.append(bad) + for field in runtime['custom_providers'][name]: + bad = copy.deepcopy(runtime) + del bad['custom_providers'][name][field] + variants.append(bad) + for field, value in (('path', '/etc/passwd'), ('path', []), ('sha256', True), + ('version', None), ('build_receipt', '../curl-source-receipt.json'), + ('source_archives', None), ('source_archives', []), ('source_archives', {})): + bad = copy.deepcopy(runtime) + bad['custom_providers'][name][field] = value + variants.append(bad) + bad = copy.deepcopy(runtime) + bad['custom_providers']['other'] = copy.deepcopy(bad['custom_providers']['libcurl']) + variants.append(bad) + for bad in variants: + with self.subTest(runtime=bad), self.assertRaises(M.InventoryError): + self.crosscheck(self.producer_fixture(payloads), bad, contract) + for name, provider in runtime['custom_providers'].items(): + bad = copy.deepcopy(runtime) + del bad['custom_providers'][name] + partial = {path: raw for path, raw in payloads.items() if path not in provider['source_archives']} + with self.subTest(missing_provider_and_archives=name), self.assertRaisesRegex( + M.InventoryError, 'inventory-custom-providers'): + self.crosscheck(self.producer_fixture(partial), bad, contract) + + def test_custom_provider_archives_cannot_be_omitted_replaced_or_substituted(self): + payloads, runtime, contract = self.manifest_fixture() + for name, provider in runtime['custom_providers'].items(): + for path in provider['source_archives']: + for change in ('missing', 'replaced', 'omitted-declaration'): + partial, bad = dict(payloads), copy.deepcopy(runtime) + if change == 'replaced': + partial[path] = b'replaced source' + else: + del partial[path] + if change == 'omitted-declaration': + del bad['custom_providers'][name]['source_archives'][path] + with self.subTest(path=path, change=change), self.assertRaises(M.InventoryError): + self.crosscheck(self.producer_fixture(partial), bad, contract) + bad = copy.deepcopy(runtime) + sha = contract['sources'][0]['sha256'] + bad['custom_providers'][name]['source_archives'] = {'sources/' + sha + '.tar.gz': sha} + with self.subTest(substituted=name), self.assertRaisesRegex(M.InventoryError, 'inventory-custom-provider'): + self.crosscheck(self.producer_fixture(payloads), bad, contract) + + def test_curl_manifests_require_complete_archive_set_and_reference_chain(self): + payloads, runtime, contract = self.manifest_fixture() + original = json.loads(payloads['curl-toolchain.json']) + for name in original['archives']: + curl, bad = copy.deepcopy(original), copy.deepcopy(runtime) + del curl['archives'][name] + del bad['custom_providers']['libcurl']['source_archives']['sources/curl/' + name] + partial = {**payloads, 'curl-toolchain.json': json.dumps(curl).encode()} + del partial['sources/curl/' + name] + with self.subTest(archive=name), self.assertRaisesRegex(M.InventoryError, 'inventory-curl-archives'): + self.crosscheck(self.producer_fixture(partial), bad, contract) + for key, value in (('cares_manifest', None), ('cares_manifest', '../classic-curl-toolchain.json'), + ('cares_manifest', '/etc/passwd'), ('source_package', 'other'), + ('source_version', None), ('source_version', '../7.88.1'), + ('version', True), ('archives', [])): + curl = {**original, key: value} + partial = {**payloads, 'curl-toolchain.json': json.dumps(curl).encode()} + with self.subTest(key=key, value=value), self.assertRaises(M.InventoryError): + self.crosscheck(self.producer_fixture(partial), runtime, contract) + shared = json.loads(payloads['classic-curl-toolchain.json']) + for value in (None, [], {}, {'version': True, 'sha256': 'a' * 64}, + {'version': '1.34.6', 'sha256': None}, {'version': '1.34.6', 'sha256': '../escape'}): + bad = {**shared, 'cares': value} + partial = {**payloads, 'classic-curl-toolchain.json': json.dumps(bad).encode()} + with self.subTest(cares=value), self.assertRaises(M.InventoryError): + self.crosscheck(self.producer_fixture(partial), runtime, contract) + + def test_source_document_types_fail_with_fixed_errors(self): + payloads, runtime, contract = self.manifest_fixture() + records = self.producer_fixture(payloads) + for index in range(3): + for value in (None, [], True, 'document'): + args = [records, runtime, contract] + args[index] = value + with self.subTest(index=index, value=value), self.assertRaisesRegex( + M.InventoryError, 'inventory-source-documents'): + self.crosscheck(*args) + + def test_all_declared_archive_records_require_typed_nonempty_bounded_bytes(self): + payloads, runtime, contract = self.manifest_fixture() + records = self.producer_fixture(payloads) + for path in (path for path in records if path.startswith('sources/')): + original = records[path] + malformed = [None, [], 'record', True, {}, {'sha256': original['sha256']}] + malformed += [{**original, 'size': size} for size in (None, False, True, 0, -1, '1', 1.0, M.MAX_FILE + 1)] + malformed += [{**original, 'sha256': sha} for sha in (None, [], 'A' * 64, '0' * 64)] + for record in malformed: + bad = copy.deepcopy(records) + bad[path] = record + with self.subTest(path=path, record=record), self.assertRaisesRegex( + M.InventoryError, 'inventory-source-bytes-mismatch'): + self.crosscheck(bad, runtime, contract) + + def test_matching_empty_archive_digest_is_rejected_for_every_source_family(self): + empty = hashlib.sha256(b'').hexdigest() + for family in ('debian', 'contract', 'audio', 'llvm', 'libcurl', 'libcares'): + payloads, runtime, contract = self.manifest_fixture() + if family == 'debian': + path = 'sources/debian/foo.tar.xz' + runtime['archives']['foo.tar.xz'] = empty + elif family == 'contract': + contract['sources'][0]['sha256'] = empty + path = 'sources/' + empty + '.tar.gz' + elif family in ('audio', 'llvm'): + name = 'audio-toolchain.json' if family == 'audio' else 'llvm-toolchain.json' + manifest = json.loads(payloads[name]) + source = manifest['sdl_mixer']['source'] if family == 'audio' else manifest['source'] + source['sha256'] = empty + payloads[name] = json.dumps(manifest).encode() + path = ('sources/' + empty + '.tar.gz' if family == 'audio' + else 'sources/llvm/' + Path(manifest['metadata']['source_archive']).name) + elif family == 'libcurl': + curl = json.loads(payloads['curl-toolchain.json']) + name = next(iter(curl['archives'])) + curl['archives'][name] = empty + payloads['curl-toolchain.json'] = json.dumps(curl).encode() + path = 'sources/curl/' + name + runtime['custom_providers'][family]['source_archives'][path] = empty + else: + shared = json.loads(payloads['classic-curl-toolchain.json']) + shared['cares']['sha256'] = empty + payloads['classic-curl-toolchain.json'] = json.dumps(shared).encode() + path = 'sources/' + empty + '.tar.gz' + runtime['custom_providers'][family]['source_archives'] = {path: empty} + payloads[path] = b'' + with self.subTest(family=family), self.assertRaisesRegex(M.InventoryError, 'inventory-source-bytes-mismatch'): + self.crosscheck(self.producer_fixture(payloads), runtime, contract) + + def test_each_declared_audio_and_llvm_source_missing_or_replaced_fails(self): + payloads, runtime, contract = self.manifest_fixture() + audio = json.loads(payloads['audio-toolchain.json']) + paths = ['sources/' + entry['source']['sha256'] + '.tar.gz' + for entry in [audio['sdl_mixer'], *audio['dependencies']]] + paths += [path for path in payloads if path.startswith('sources/llvm/')] + for path in paths: + for missing in (True, False): + bad = dict(payloads) + if missing: + del bad[path] + else: + bad[path] = b'replaced archive' + # Even equal whole-tree inventories cannot bless missing bytes. + records = self.producer_fixture(bad) + self.assertEqual(M.inventory_digest(records), M.inventory_digest(self.producer_fixture(bad))) + with self.subTest(path=path, missing=missing), self.assertRaisesRegex( + M.InventoryError, 'inventory-source-bytes-mismatch'): + self.crosscheck(records, runtime, contract) + + def test_manifest_missing_replaced_identity_and_limits_fail(self): + payloads, runtime, contract = self.manifest_fixture() + for name in M.SOURCE_MANIFESTS: + bad = dict(payloads) + del bad[name] + with self.subTest(name=name), self.assertRaisesRegex( + M.InventoryError, 'inventory-manifest-missing-or-size'): + self.crosscheck(self.producer_fixture(bad), runtime, contract) + records = self.producer_fixture(payloads) + records.source_manifests[name] += b' ' + with self.assertRaisesRegex(M.InventoryError, 'inventory-manifest-bytes-mismatch'): + self.crosscheck(records, runtime, contract) + records = self.producer_fixture(payloads) + records[name]['size'] = float(records[name]['size']) + with self.assertRaisesRegex(M.InventoryError, 'inventory-manifest-bytes-mismatch'): + self.crosscheck(records, runtime, contract) + for raw in (b'', b' ' * (M.MAX_MANIFEST + 1)): + with self.subTest(name=name, size=len(raw)), self.assertRaisesRegex( + M.InventoryError, 'inventory-manifest-size'): + self.producer_fixture({name: raw}) + + def test_manifest_json_duplicates_types_and_depth_fail(self): + payloads, runtime, contract = self.manifest_fixture() + for name in M.SOURCE_MANIFESTS: + for raw in (b'{', b'\xff', b'[]', b'null', b'{"schema_version":true}', + b'{"schema_version":2}', b'{"schema_version":1,"schema_version":1}', + b'{"schema_version":1,"extra":{"sha256":1,"sha256":2}}', + b'{"schema_version":1,"extra":NaN}', b'[' * 2000 + b']' * 2000): + bad = dict(payloads) + bad[name] = raw + with self.subTest(name=name, raw=raw[:80]), self.assertRaises(M.InventoryError): + self.crosscheck(self.producer_fixture(bad), runtime, contract) + + def test_manifest_references_cannot_escape_fixed_boundary(self): + payloads, runtime, contract = self.manifest_fixture() + for reference in ('/etc/passwd', '../audio-toolchain.json', 'sub/audio-toolchain.json', + 'sources/../audio-toolchain.json', 'audio-toolchain.json/extra', + 'audio-toolchain.json\\extra', None, [], {}): + for role in ('audio', 'llvm'): + bad = copy.deepcopy(contract) + if role == 'audio': + bad['audio_manifest'] = reference + else: + bad['compiler']['toolchain_manifest'] = reference + with self.subTest(role=role, reference=reference), self.assertRaisesRegex( + M.InventoryError, 'inventory-manifest-path'): + self.crosscheck(self.producer_fixture(payloads), runtime, bad) + + def test_audio_manifest_invalid_sources_and_duplicate_names_fail(self): + payloads, runtime, contract = self.manifest_fixture() + original = json.loads(payloads['audio-toolchain.json']) + variants = [] + for key, values in (('dependencies', [None, {}, [], [None], [original['dependencies'][0]] * 128]), + ('sdl_mixer', [None, [], {}, {'name': '../escape', 'source': {}}, + {'name': 'mixer', 'source': None}])): + for value in values: + bad = copy.deepcopy(original) + bad[key] = value + variants.append(bad) + bad = copy.deepcopy(original) + bad['dependencies'].append(copy.deepcopy(bad['sdl_mixer'])) + variants.append(bad) + for sha in (None, 1, [], 'A' * 64, '../escape', 'a' * 63): + bad = copy.deepcopy(original) + bad['dependencies'][0]['source']['sha256'] = sha + variants.append(bad) + for value in variants: + bad = dict(payloads) + bad['audio-toolchain.json'] = json.dumps(value).encode() + with self.subTest(value=value), self.assertRaises(M.InventoryError): + self.crosscheck(self.producer_fixture(bad), runtime, contract) + + def test_llvm_manifest_invalid_sources_and_archive_paths_fail(self): + payloads, runtime, contract = self.manifest_fixture() + original = json.loads(payloads['llvm-toolchain.json']) + for key, values in (('version', [None, True, '../escape', '23.1.3/../escape']), + ('source', [None, [], {}, {'sha256': 'A' * 64}]), + ('metadata', [None, [], {}, *[{'source_archive': path} for path in ( + '/etc/passwd', 'share/atrinik/sources/../escape.tar.xz', + 'share/atrinik/sources/nested/llvm-project-23.1.3.src.tar.xz', + 'share/atrinik/sources/llvm-project-23.1.3.src.tar.xz/extra', + 'share/atrinik/sources/llvm-project-23.1.3.src.tar.xz\\extra', + 'share/atrinik/sources/llvm-project-22.1.0.src.tar.xz')]])): + for value in values: + llvm = copy.deepcopy(original) + llvm[key] = value + bad = dict(payloads) + bad['llvm-toolchain.json'] = json.dumps(llvm).encode() + with self.subTest(key=key, value=value), self.assertRaises(M.InventoryError): + self.crosscheck(self.producer_fixture(bad), runtime, contract) + + + def notice_image(self, links=None, drift=False): + links = links or {} + image = self.fake_image() + payload = b'notice' + link_reads = {} + def stat(path): + if path in links: + raw, target = links[path] + return {'name': Path(path).name, 'mode': M.GO_SYMLINK | 0o777, + 'size': len(raw), 'mtime': 'fixed', 'linkTarget': target} + if path.endswith('/copyright') or path.startswith(M.COMMON + '/'): + return regular(path, payload) + return directory(path) + def request(method, route, headers): + self.assertEqual(method, 'GET') + path = urllib.parse.parse_qs(urllib.parse.urlsplit(route).query)['path'][0] + if path in links: + raw, _ = links[path] + link_reads[path] = link_reads.get(path, 0) + 1 + if drift and link_reads[path] > 1: + raw = raw.replace('bar', 'baz') + data = tar([(Path(path).name, b'', tarfile.SYMTYPE, {'linkname': raw, 'mode': 0o777})]) + else: + data = tar([(Path(path).name, payload, tarfile.REGTYPE, {})]) + return Mock(), Response(data, stat(path)) + image.stat.side_effect = stat + image.docker.request.side_effect = request + return image + + def test_notices_resolve_directory_and_file_links_beneath_doc(self): + runtime, debian = self.documents() + for links, canonical in (({}, M.DOC + '/foo/copyright'), + ({M.DOC + '/foo': ('bar', M.DOC + '/bar')}, M.DOC + '/bar/copyright'), + ({M.DOC + '/foo/copyright': ('../bar/copyright', M.DOC + '/bar/copyright')}, + M.DOC + '/bar/copyright')): + with self.subTest(links=links): + result = M.InstalledInventory(self.notice_image(links), P).debian_notices(runtime, debian) + self.assertEqual(set(result), {M.DOC + '/foo/copyright'}) + self.assertEqual(result[M.DOC + '/foo/copyright']['resolved_path'], canonical) + self.assertEqual(result[M.DOC + '/foo/copyright']['sha256'], hashlib.sha256(b'notice').hexdigest()) + + def test_notice_escape_cycle_limit_wrong_resolved_target_and_drift(self): + runtime, debian = self.documents() + cases = [ + {M.DOC + '/foo': ('/secret', '/secret')}, + {M.DOC + '/foo': ('../../../usr/share/doc/bar', M.DOC + '/bar')}, + {M.DOC + '/foo': ('bar', M.DOC + '/bar'), M.DOC + '/bar': ('foo', M.DOC + '/foo')}, + {M.DOC + '/foo': ('bar', M.DOC + '/wrong')}, + {M.DOC + '/foo': ('bar/../baz', M.DOC + '/baz'), + M.DOC + '/bar': ('/outside/dir', '/outside/dir')}, + ] + for links in cases: + with self.subTest(links=links), self.assertRaises(M.InventoryError): + M.InstalledInventory(self.notice_image(links), P).debian_notices(runtime, debian) + links = {M.DOC + '/foo': ('bar', M.DOC + '/bar')} + with patch.object(M, 'MAX_HOPS', 0), self.assertRaises(M.InventoryError): + M.InstalledInventory(self.notice_image(links), P).debian_notices(runtime, debian) + with self.assertRaisesRegex(M.InventoryError, 'notice-alias-drift'): + M.InstalledInventory(self.notice_image(links, drift=True), P).debian_notices(runtime, debian) + + def test_common_license_exact_fixed_names_and_format(self): + image = self.notice_image() + result = M.InstalledInventory(image, P).common_licenses() + self.assertEqual(set(result), set(M.COMMON_LICENSE_NAMES)) + self.assertEqual(len(result), 14) + self.assertEqual(set(result['MIT'] if 'MIT' in result else result['BSD']), + {'sha256', 'size', 'executable'}) + paths = [urllib.parse.parse_qs(urllib.parse.urlsplit(call.args[1]).query)['path'][0] + for call in image.docker.request.call_args_list] + self.assertEqual(set(paths), {M.COMMON + '/' + name for name in M.COMMON_LICENSE_NAMES}) + + def test_empty_inventory_notices_and_build_tools_are_rejected(self): + runtime, debian = self.documents() + for path in (M.DOC + '/foo/copyright', M.COMMON + '/BSD', *M.BUILD_TOOLS.values()): + image = self.fake_image() + executable = path in M.BUILD_TOOLS.values() + expected = regular(path, b'') + if executable: + expected['mode'] = 0o755 + image.stat.side_effect = lambda value: expected if value == path else directory(value) + conn = Mock() + image.docker.request.return_value = ( + conn, Response(tar([(Path(path).name, b'', tarfile.REGTYPE, + {'mode': expected['mode']})]), expected)) + reader = M.InstalledInventory(image, P) + with self.subTest(path=path), self.assertRaisesRegex(M.InventoryError, 'file-size-limit'): + if path.startswith(M.DOC + '/'): + reader.debian_notices(runtime, debian) + else: + reader._regular(path, executable=executable) + image.docker.request.assert_not_called() + + def build_receipt(self): + entries = {role: {'path': path, 'resolved_path': path, 'sha256': hashlib.sha256(b'tool').hexdigest(), + 'size': 4, 'executable': True, + 'version': {'cmake': 'cmake version 3.25.1', 'ninja': '1.11.1', + 'pkg_config': '1.8.1'}[role] + '\nsecond line\n'} + for role, path in M.BUILD_TOOLS.items()} + return {'build_tools': {'schema_version': 1, 'tools': entries}, + 'tools': {role: entry['version'].splitlines()[0] for role, entry in entries.items()}} + + def test_provider_identity_requires_authenticated_reader_and_actual_bytes(self): + payloads, runtime, contract = self.manifest_fixture() + records = self.producer_fixture(payloads) + with self.assertRaisesRegex(M.InventoryError, 'inventory-provider-reader-required'): + M.crosscheck_archives(records, runtime, contract) + image = self.installed_image() + self.crosscheck(records, runtime, contract, image=image) + requests = [urllib.parse.parse_qs(urllib.parse.urlsplit(call.args[1]).query)['path'][0] + for call in image.docker.request.call_args_list] + self.assertEqual(set(requests), {'/', '/usr', '/usr/local', '/usr/local/lib', + *[item['path'] for item in runtime['custom_providers'].values()]}) + for conn in image.connections: + conn.close.assert_called_once() + for name, item in runtime['custom_providers'].items(): + with self.subTest(provider=name): + bad = copy.deepcopy(runtime) + bad['custom_providers'][name]['sha256'] = 'a' * 64 + with self.assertRaisesRegex(M.InventoryError, 'inventory-provider-bytes-mismatch'): + self.crosscheck(records, bad, contract) + absent = copy.deepcopy(runtime) + absent['custom_providers'][name]['path'] += '.123' + with self.assertRaisesRegex(P.Stop, 'docker-http-status'): + self.crosscheck(records, absent, contract, + image=self.installed_image(missing=(item['path'] + '.123',))) + data = {entry['path']: key.encode() for key, entry in runtime['custom_providers'].items()} + data[item['path']] = b'replaced installed binary' + with self.assertRaisesRegex(M.InventoryError, 'inventory-provider-bytes-mismatch'): + self.crosscheck(records, runtime, contract, image=self.installed_image(data)) + + def test_provider_reader_boundary_rejects_missing_reader_and_invalid_identities(self): + payloads, runtime, contract = self.manifest_fixture() + records = self.producer_fixture(payloads) + for reader in (None, {}, {'sha256': 'a' * 64}, object()): + with self.subTest(reader=reader), self.assertRaisesRegex( + M.InventoryError, 'inventory-provider-reader-required'): + M.crosscheck_archives(records, runtime, contract, reader=reader) + class InvalidReader(M.ProviderReader): + def __init__(self, value): + self.value = value + def read_provider(self, path): + return self.value + valid = {'sha256': hashlib.sha256(b'libcurl').hexdigest(), 'size': 7, 'executable': True} + for value in (None, [], {}, {**valid, 'extra': True}, {**valid, 'size': True}, + {**valid, 'size': 0}, {**valid, 'size': M.MAX_FILE + 1}, + {**valid, 'sha256': 'A' * 64}, {**valid, 'executable': 1}): + with self.subTest(value=value), self.assertRaisesRegex(M.InventoryError, 'inventory-provider-identity'): + M.crosscheck_archives(records, runtime, contract, reader=InvalidReader(value)) + + def test_provider_reader_keeps_fixed_namespace_and_no_link_defenses(self): + for path in ('/etc/passwd', '/usr/local/lib/libcurl.so', '/usr/local/lib/libcurl.so.1/extra', + '/usr/local/lib/libcares.so.1/../elsewhere', '/usr/local/lib/libcurl.so.' + '1' * M.MAX_PATH): + image = self.installed_image() + with self.subTest(path=path), self.assertRaisesRegex(M.InventoryError, 'inventory-file-not-allowed'): + M.InstalledInventory(image, P).read_provider(path) + image.docker.request.assert_not_called() + payloads, runtime, contract = self.manifest_fixture() + records = self.producer_fixture(payloads) + for provider in runtime['custom_providers'].values(): + image = self.installed_image(metadata={provider['path']: {'type': tarfile.SYMTYPE, + 'linkname': 'other.so'}}) + with self.subTest(path=provider['path']), self.assertRaisesRegex(P.Stop, 'unexpected-archive-entry'): + self.crosscheck(records, runtime, contract, image=image) + + def test_provider_files_and_ancestors_are_root_owned(self): + payloads, runtime, contract = self.manifest_fixture() + records = self.producer_fixture(payloads) + paths = ('/', '/usr', '/usr/local', '/usr/local/lib', + *[entry['path'] for entry in runtime['custom_providers'].values()]) + for path in paths: + image = self.installed_image(metadata={path: {'uid': 1000, 'gid': 1000}}) + with self.subTest(path=path), self.assertRaisesRegex( + (M.InventoryError, P.Stop), 'inventory-directory-owner|archive-owner-mismatch'): + self.crosscheck(records, runtime, contract, image=image) + for conn in image.connections: + conn.close.assert_called_once() + + def test_build_tools_real_reader_protects_file_and_all_ancestors(self): + data = {path: b'tool' for path in M.BUILD_TOOLS.values()} + installed = self.build_receipt() + image = self.installed_image(data) + self.assertEqual(M.InstalledInventory(image, P).build_tools(installed), installed['build_tools']['tools']) + for conn in image.connections: + conn.close.assert_called_once() + for path in ('/', '/usr', '/usr/bin', *M.BUILD_TOOLS.values()): + image = self.installed_image(data, {path: {'uid': 1000, 'gid': 1000}}) + with self.subTest(path=path), self.assertRaisesRegex( + (M.InventoryError, P.Stop), 'inventory-directory-owner|archive-owner-mismatch'): + M.InstalledInventory(image, P).build_tools(installed) + for conn in image.connections: + conn.close.assert_called_once() + # The portable consumer requires UID 0; GID 0 is not part of that contract. + image = self.installed_image(data, {path: {'gid': 1000} for path in ('/', '/usr', '/usr/bin', *data)}) + self.assertEqual(M.InstalledInventory(image, P).build_tools(installed), installed['build_tools']['tools']) + for role, path in M.BUILD_TOOLS.items(): + changed = {**data, path: b'changed tool'} + with self.subTest(role=role), self.assertRaisesRegex(M.InventoryError, 'build-tool-bytes-mismatch'): + M.InstalledInventory(self.installed_image(changed), P).build_tools(installed) + + def test_protected_directory_header_rejects_ownership_overrides_and_path_drift(self): + data = {path: b'tool' for path in M.BUILD_TOOLS.values()} + for path in ('/', '/usr', '/usr/bin'): + for extra in ({'mode': 0o777}, {'type': tarfile.SYMTYPE, 'linkname': 'elsewhere'}, + {'name': '../escape'}, {'pax_headers': {'uid': '0'}, 'uid': 1000}, + {'pax_headers': {'gid': '0'}}, {'uid': -1}): + image = self.installed_image(data, {path: extra}) + with self.subTest(path=path, extra=extra), self.assertRaises((M.InventoryError, P.Stop)): + M.InstalledInventory(image, P).build_tools(self.build_receipt()) + for conn in image.connections: + conn.close.assert_called_once() + image = self.installed_image(data) + request = image.docker.request.side_effect + def drift(method, route, headers): + conn, response = request(method, route, headers) + if urllib.parse.parse_qs(urllib.parse.urlsplit(route).query)['path'] == ['/usr']: + response.headers['X-Docker-Container-Path-Stat'] = base64.b64encode( + json.dumps({**directory('/usr'), 'mtime': 'changed'}).encode()).decode() + return conn, response + image.docker.request.side_effect = drift + with self.assertRaisesRegex(M.InventoryError, 'inventory-path-drift'): + M.InstalledInventory(image, P).build_tools(self.build_receipt()) + + def test_root_directory_requires_exact_moby_archive_name(self): + data = {path: b'tool' for path in M.BUILD_TOOLS.values()} + for name in ('.', './', '//', '///', 'usr'): + image = self.installed_image(data, {'/': {'name': name}}) + with self.subTest(name=name), self.assertRaises(M.InventoryError): + M.InstalledInventory(image, P).build_tools(self.build_receipt()) + raw = bytearray(tar([('/', b'', tarfile.DIRTYPE, {})])) + raw[:100] = bytes(100) + raw[148:156] = b' ' + raw[148:156] = ('%06o\0 ' % sum(raw[:512])).encode() + image = self.fake_image() + image.docker.request.return_value = Mock(), Response(bytes(raw), directory('/')) + with self.assertRaisesRegex(M.InventoryError, 'inventory-directory-entry'): + M.InstalledInventory(image, P)._directory_owner('/', directory('/')) + image = self.installed_image(data, {'/': {'name': '/'}}) + self.assertEqual(M.InstalledInventory(image, P).build_tools(self.build_receipt()), + self.build_receipt()['build_tools']['tools']) + + def test_directory_ownership_read_is_bounded_to_header(self): + image = self.fake_image() + # Simulate a huge recursive directory response. Do not request body bytes. + response = Response(tar([('usr', b'', tarfile.DIRTYPE, {})]), directory('/usr')) + original = response.read + def read(size): + self.assertEqual(response.tell(), 0) + self.assertEqual(size, 512) + return original(size) + response.read = read + conn = Mock() + image.docker.request.return_value = conn, response + M.InstalledInventory(image, P)._directory_owner('/usr', directory('/usr')) + conn.close.assert_called_once() + + def test_build_tools_accept_full_stdout_with_or_without_final_newline(self): + for cmake, ninja in (('cmake version 3.25.1', '1.11'), + ('cmake version 3.25.1\n\nCMake suite maintained and supported by Kitware.', + '1.11.1\n'), + ('cmake version 3.25.1\nfull stdout\n', '1.11.1')): + installed = self.build_receipt() + for role, version in (('cmake', cmake), ('ninja', ninja), + ('pkg_config', ninja.replace('1.11', '1.8'))): + installed['build_tools']['tools'][role]['version'] = version + installed['tools'][role] = version.splitlines()[0] + reader = M.InstalledInventory(self.fake_image(), P) + identity = {'sha256': hashlib.sha256(b'tool').hexdigest(), 'size': 4, 'executable': True} + with self.subTest(cmake=cmake, ninja=ninja), patch.object(reader, '_regular', return_value=identity): + self.assertEqual(reader.build_tools(installed), installed['build_tools']['tools']) + + def test_build_tools_enforce_shared_512_mib_boundary(self): + for size in (512 * 1024 * 1024, 512 * 1024 * 1024 + 1): + installed = self.build_receipt() + for entry in installed['build_tools']['tools'].values(): + entry['size'] = size + identity = {'sha256': hashlib.sha256(b'tool').hexdigest(), 'size': size, 'executable': True} + reader = M.InstalledInventory(self.fake_image(), P) + with self.subTest(size=size), patch.object(reader, '_regular', return_value=identity) as read: + if size == 512 * 1024 * 1024: + self.assertEqual(reader.build_tools(installed), installed['build_tools']['tools']) + else: + with self.assertRaisesRegex(M.InventoryError, 'build-tool-identity'): + reader.build_tools(installed) + read.assert_not_called() + + def test_build_tools_reject_role_grammar_controls_and_del(self): + cases = {'cmake': ('3.25.1', 'cmake version 3.25', 'cmake version 3.25.1-rc1'), + 'ninja': ('ninja 1.11.1', 'cmake version 3.25.1', '1.11.1-rc1'), + 'pkg_config': ('pkgconf 1.8.1', 'cmake version 3.25.1', '1.8.1-rc1')} + for role in M.BUILD_TOOLS: + valid = 'cmake version 3.25.1' if role == 'cmake' else '1.11.1' + cases[role] += tuple(valid + '\nextra' + chr(code) + for code in (*range(10), *range(11, 32), 127)) + for version in cases[role]: + installed = self.build_receipt() + installed['build_tools']['tools'][role]['version'] = version + installed['tools'][role] = version.splitlines()[0] + reader = M.InstalledInventory(self.fake_image(), P) + identity = {'sha256': hashlib.sha256(b'tool').hexdigest(), 'size': 4, 'executable': True} + with self.subTest(role=role, version=version), patch.object(reader, '_regular', return_value=identity) as read: + with self.assertRaisesRegex(M.InventoryError, 'build-tool-identity'): + reader.build_tools(installed) + self.assertNotIn(M.BUILD_TOOLS[role], [call.args[0] for call in read.call_args_list]) + + def test_build_tools_require_nonroot_executable_before_get(self): + for mode in (0o744, 0o644): + image = self.fake_image() + image.stat.side_effect = lambda path: ( + {**regular(path, b'tool'), 'mode': mode} + if path in M.BUILD_TOOLS.values() else directory(path)) + with self.subTest(mode=mode), self.assertRaisesRegex(P.Stop, 'tool-not-executable'): + M.InstalledInventory(image, P).build_tools(self.build_receipt()) + image.docker.request.assert_not_called() + + def test_pkg_config_is_mandatory_and_alias_cannot_replace_fixed_pkgconf(self): + self.assertEqual(M.BUILD_TOOLS, {'cmake': '/usr/bin/cmake', 'ninja': '/usr/bin/ninja', + 'pkg_config': '/usr/bin/pkgconf'}) + installed = self.build_receipt() + reader = M.InstalledInventory(self.fake_image(), P) + for role in M.BUILD_TOOLS: + bad = copy.deepcopy(installed) + del bad['build_tools']['tools'][role] + with self.subTest(missing=role), patch.object(reader, '_regular') as read: + with self.assertRaisesRegex(M.InventoryError, 'build-tool-schema'): + reader.build_tools(bad) + read.assert_not_called() + identity = {'sha256': hashlib.sha256(b'tool').hexdigest(), 'size': 4, 'executable': True} + for key in ('path', 'resolved_path'): + bad = copy.deepcopy(installed) + bad['build_tools']['tools']['pkg_config'][key] = '/usr/bin/pkg-config' + with self.subTest(key=key), patch.object(reader, '_regular', return_value=identity) as read: + with self.assertRaisesRegex(M.InventoryError, 'build-tool-path'): + reader.build_tools(bad) + self.assertNotIn('/usr/bin/pkgconf', [call.args[0] for call in read.call_args_list]) + bad = copy.deepcopy(installed) + del bad['tools']['pkg_config'] + with patch.object(reader, '_regular', return_value=identity), self.assertRaisesRegex(M.InventoryError, 'build-tool-identity'): + reader.build_tools(bad) + + def test_build_tools_exact_schema_paths_and_streamed_bytes(self): + installed = self.build_receipt() + reader = M.InstalledInventory(self.fake_image(), P) + identity = {'sha256': hashlib.sha256(b'tool').hexdigest(), 'size': 4, 'executable': True} + with patch.object(reader, '_regular', return_value=identity) as read: + self.assertEqual(reader.build_tools(installed), installed['build_tools']['tools']) + self.assertEqual([call.args[0] for call in read.call_args_list], list(M.BUILD_TOOLS.values())) + for role in M.BUILD_TOOLS: + for key, value in (('path', '/secret'), ('resolved_path', '/secret'), ('sha256', 'a' * 64), + ('size', True), ('executable', 1), ('version', 'wrong\n')): + bad = copy.deepcopy(installed) + bad['build_tools']['tools'][role][key] = value + with self.subTest(role=role, key=key), patch.object(reader, '_regular', return_value=identity), self.assertRaises(M.InventoryError): + reader.build_tools(bad) + bad = copy.deepcopy(installed) + bad['build_tools']['tools']['extra'] = {} + with self.assertRaisesRegex(M.InventoryError, 'build-tool-schema'): + reader.build_tools(bad) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/tests/test_verify_published_identity.py b/tools/tests/test_verify_published_identity.py new file mode 100644 index 0000000..b942f66 --- /dev/null +++ b/tools/tests/test_verify_published_identity.py @@ -0,0 +1,627 @@ +# Copyright (c) 2026 Atrinik contributors +# SPDX-License-Identifier: MIT +"""Offline tests for the Linux publisher's loaded-to-published identity fence.""" +import copy +import importlib.util +import io +import json +import os +from pathlib import Path +import tempfile +import unittest +from unittest.mock import Mock, patch + + +ROOT = Path(__file__).resolve().parents[2] +SPEC = importlib.util.spec_from_file_location( + "published_identity", ROOT / "tools/verify_published_identity.py") +M = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(M) +REVISION = "1" * 40 +INDEX_DIGEST = "sha256:" + "9" * 64 +BEARER = "FAKE_SCOPED_PULL_BEARER" +DOCKER_149_DEFAULTS = { + "Hostname": "", "Domainname": "", "Image": "", + "AttachStdin": False, "AttachStdout": False, "AttachStderr": False, + "Tty": False, "OpenStdin": False, "StdinOnce": False, +} + + +def encoded(value): + return json.dumps(value, separators=(",", ":")).encode() + + +def descriptor(raw, media_type): + return {"digest": M.R.digest(raw), "size": len(raw), "mediaType": media_type} + + +def runtime(labels=None): + return { + "Hostname": "", + "Domainname": "", + "User": "1000:1000", + "AttachStdin": False, + "AttachStdout": False, + "AttachStderr": False, + "Tty": False, + "OpenStdin": False, + "StdinOnce": False, + "Env": ["PATH=/usr/bin", "LANG=C.UTF-8"], + "Cmd": ["/bin/true"], + "Entrypoint": None, + "Image": "", + "Labels": labels, + "Volumes": None, + "WorkingDir": "/work", + "NetworkDisabled": False, + "MacAddress": "", + "OnBuild": None, + "StopSignal": "SIGTERM", + "StopTimeout": 10, + "Shell": ["/bin/sh", "-c"], + "ArgsEscaped": False, + "ExposedPorts": {"443/tcp": {}}, + "Healthcheck": {"Test": ["CMD", "/bin/true"], "Interval": 1, + "Timeout": 2, "StartPeriod": 3, "StartInterval": 4, + "Retries": 5}, + } + + +def fixture(): + layers = ["sha256:" + "a" * 64, "sha256:" + "b" * 64] + local = { + "Id": "sha256:" + "c" * 64, + "Os": "linux", + "Architecture": "amd64", + "Variant": "", + "Author": "Atrinik", + "RootFS": {"Type": "layers", "Layers": list(layers)}, + "Config": runtime({"fixture": "same"}), + } + published = { + "architecture": "amd64", + "os": "linux", + "author": "Atrinik", + "created": "2026-10-12T00:00:00Z", + "history": [], + "moby.buildkit.cache.v0": None, + "rootfs": {"type": "layers", "diff_ids": list(layers)}, + "config": runtime({ + "fixture": "same", + "org.opencontainers.image.source": M.R.SOURCE, + "org.opencontainers.image.revision": REVISION, + }), + } + return local, published + + +def public_ubuntu_transport_fixture(api): + """Project actual public Ubuntu inspect/OCI shapes, with synthetic labels. + + Docker 29.1.3 was queried at API 1.49, 1.50 and 1.51 for the cached + ubuntu@sha256:678c6550cc43645e08669028bc177f50be4e7c5b8cca677067b1914d4afc7a03. + The public index/amd64 manifest/config chain was separately hash-verified; + its OCI config SHA256 is 86a1a31fdd84f2dc79bd6d92272100d4369b085790753bc521069eab663074e9. + Config below retains the captured non-label fields and values. Only labels + use synthetic test data; no fixture claims an Atrinik-published Ubuntu image. + """ + local, published = fixture() + config = { + "Env": ["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"], + "Cmd": ["/bin/bash"], + "Labels": {"fixture": "same"}, + } + local["Config"] = copy.deepcopy(config) + local["Config"].update({"Entrypoint": None, "OnBuild": None, "Volumes": None, + "User": "", "WorkingDir": ""}) + if api == "1.49": + local["Config"].update(DOCKER_149_DEFAULTS) + elif api not in {"1.50", "1.51"}: + raise ValueError("fixture API not captured") + published["config"] = config + published["config"]["Labels"].update({ + "org.opencontainers.image.source": M.R.SOURCE, + "org.opencontainers.image.revision": REVISION, + }) + return local, published + + +def context_env(**changes): + ref = changes.pop("GITHUB_REF", "refs/tags/v1.2.3") + env = { + "GITHUB_ACTIONS": "true", + "RUNNER_ENVIRONMENT": "github-hosted", + "GITHUB_REPOSITORY": M.R.REPOSITORY, + "GITHUB_EVENT_NAME": "push", + "GITHUB_REF": ref, + "GITHUB_WORKFLOW_REF": M.R.REPOSITORY + "/" + M.WORKFLOW + "@" + ref, + "GITHUB_SHA": REVISION, + "GITHUB_ACTOR": "trusted-actor", + "GITHUB_RUN_ID": "123", + "GITHUB_RUN_ATTEMPT": "2", + } + env.update(changes) + return env + + +def registry_fixture(config=None): + if config is None: + _, config = fixture() + raw_config = encoded(config) + layers = [ + {"digest": "sha256:" + "d" * 64, "size": 101, + "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip"}, + {"digest": "sha256:" + "e" * 64, "size": 102, + "mediaType": "application/vnd.oci.image.layer.v1.tar+zstd"}, + ] + manifest = {"schemaVersion": 2, "mediaType": M.R.MANIFEST, + "config": descriptor(raw_config, M.R.CONFIG), "layers": layers} + raw_manifest = encoded(manifest) + selected = dict(descriptor(raw_manifest, M.R.MANIFEST), + platform={"os": "linux", "architecture": "amd64"}) + attestation = dict( + descriptor(encoded({"attestation": True}), M.R.MANIFEST), + platform={"os": "unknown", "architecture": "unknown"}, + annotations={"vnd.docker.reference.type": "attestation-manifest", + "vnd.docker.reference.digest": selected["digest"]}) + index = {"schemaVersion": 2, "mediaType": M.R.INDEX, + "manifests": [selected, attestation]} + raw_index = encoded(index) + objects = {INDEX_DIGEST: raw_index, selected["digest"]: raw_manifest, + manifest["config"]["digest"]: raw_config} + return objects, config, manifest, index + + +class Response(io.BytesIO): + def __init__(self, raw): + super().__init__(raw) + self.status = 200 + self.length = len(raw) + + +class OfflineTests(unittest.TestCase): + def setUp(self): + blockers = ( + patch.object(M.R, "https_get", side_effect=AssertionError("network forbidden")), + patch.object(M.R.LocalConnection, "connect", + side_effect=AssertionError("Docker socket forbidden")), + patch.object(M.R.subprocess, "run", + side_effect=AssertionError("subprocess forbidden")), + ) + for blocker in blockers: + blocker.start() + self.addCleanup(blocker.stop) + + def assert_stop(self, code, call, *args): + with self.assertRaisesRegex(M.Stop, code): + call(*args) + + def test_exact_runtime_config_rootfs_and_publication_labels_match(self): + local, published = fixture() + result = M.compare(local, published, REVISION) + self.assertEqual(result["loaded_image_id"], local["Id"]) + self.assertTrue(result["rootfs_equal"]) + self.assertTrue(result["execution_config_equal"]) + self.assertEqual(result["docker_absent_defaults"], M.DEFAULTS) + + def test_public_ubuntu_inspect_and_oci_config_shapes_match_all_supported_apis(self): + for api in ("1.49", "1.50", "1.51"): + with self.subTest(api=api): + local, published = public_ubuntu_transport_fixture(api) + before = copy.deepcopy((local, published)) + result = M.compare(local, published, REVISION) + self.assertEqual((local, published), before) + self.assertTrue(result["rootfs_equal"]) + self.assertTrue(result["execution_config_equal"]) + for key, value in DOCKER_149_DEFAULTS.items(): + self.assertIs(type(result["docker_absent_defaults"][key]), type(value)) + self.assertEqual(result["docker_absent_defaults"][key], value) + + def test_each_docker149_nondefault_differs_from_omitted_or_zero(self): + for key, zero in DOCKER_149_DEFAULTS.items(): + for side in ("loaded", "published"): + for peer in ("omitted", "zero"): + with self.subTest(key=key, side=side, peer=peer): + local, published = public_ubuntu_transport_fixture("1.49") + changed, unchanged = ((local["Config"], published["config"]) + if side == "loaded" else (published["config"], local["Config"])) + changed[key] = True if type(zero) is bool else "changed" + if peer == "omitted": + unchanged.pop(key, None) + else: + unchanged[key] = zero + self.assert_stop("published-runtime-config-mismatch", M.compare, + local, published, REVISION) + + def test_each_docker149_default_rejects_wrong_types_before_normalizing(self): + for key, zero in DOCKER_149_DEFAULTS.items(): + code = "invalid-runtime-boolean" if type(zero) is bool else "invalid-runtime-string" + wrong_values = (0, 1, "false", None, [], {}) if type(zero) is bool else (0, False, None, [], {}) + for side in ("loaded", "published"): + for peer in ("omitted", "zero"): + for wrong in wrong_values: + with self.subTest(key=key, side=side, peer=peer, wrong=wrong): + local, published = public_ubuntu_transport_fixture("1.49") + changed, unchanged = ((local["Config"], published["config"]) + if side == "loaded" else (published["config"], local["Config"])) + changed[key] = wrong + if peer == "omitted": + unchanged.pop(key, None) + else: + unchanged[key] = zero + self.assert_stop(code, M.compare, local, published, REVISION) + + def test_captured_config_shape_still_rejects_unknown_zero_fields(self): + for side in ("loaded", "published", "both"): + local, published = public_ubuntu_transport_fixture("1.49") + if side in ("loaded", "both"): + local["Config"]["UnknownDockerDefault"] = False + if side in ("published", "both"): + published["config"]["UnknownDockerDefault"] = False + with self.subTest(side=side): + self.assert_stop("unknown-runtime-config-field", M.compare, + local, published, REVISION) + + def test_docker_absent_defaults_are_exact_but_null_is_not_empty_array(self): + local, published = fixture() + for key in M.DEFAULTS: + for direction in ("loaded-materialized", "published-materialized"): + left, right = copy.deepcopy(local), copy.deepcopy(published) + if key == "Labels": + if direction == "loaded-materialized": + left["Config"][key] = None + else: + left["Config"].pop(key, None) + right["config"]["Labels"] = { + "org.opencontainers.image.source": M.R.SOURCE, + "org.opencontainers.image.revision": REVISION, + } + elif direction == "loaded-materialized": + left["Config"][key] = copy.deepcopy(M.DEFAULTS[key]) + right["config"].pop(key, None) + else: + left["Config"].pop(key, None) + right["config"][key] = copy.deepcopy(M.DEFAULTS[key]) + with self.subTest(key=key, direction=direction): + M.compare(left, right, REVISION) + for key in ("Cmd", "Entrypoint", "Env", "OnBuild"): + with self.subTest(null_vs_array=key): + left, right = copy.deepcopy(local), copy.deepcopy(published) + left["Config"][key] = None + right["config"][key] = [] + self.assert_stop("published-runtime-config-mismatch", M.compare, + left, right, REVISION) + + def test_environment_order_scalar_types_and_rootfs_order_are_bound(self): + local, published = fixture() + published["config"]["Env"].reverse() + self.assert_stop("published-runtime-config-mismatch", M.compare, + local, published, REVISION) + local, published = fixture() + local["Config"]["StopTimeout"] = True + published["config"]["StopTimeout"] = True + self.assert_stop("invalid-runtime-timeout", M.compare, local, published, REVISION) + local, published = fixture() + published["rootfs"]["diff_ids"].reverse() + self.assert_stop("published-rootfs-mismatch", M.compare, local, published, REVISION) + + def test_unknown_runtime_and_top_level_config_fields_are_rejected(self): + for location, code in (("runtime", "unknown-runtime-config-field"), + ("top", "unknown-image-config-field")): + local, published = fixture() + if location == "runtime": + local["Config"]["Unknown"] = "same" + published["config"]["Unknown"] = "same" + else: + published["unknown"] = "value" + with self.subTest(location=location): + self.assert_stop(code, M.compare, local, published, REVISION) + + def test_runtime_field_types_maps_and_healthcheck_are_strict(self): + cases = [ + ("AttachStdin", 0, "invalid-runtime-boolean"), + ("User", None, "invalid-runtime-string"), + ("Env", [True], "invalid-runtime-array"), + ("Labels", {"x": 1}, "invalid-runtime-map-value"), + ("Volumes", {"/data": None}, "invalid-runtime-map-value"), + ("ExposedPorts", {"80/tcp": []}, "invalid-runtime-map-value"), + ("StopTimeout", -1, "invalid-runtime-timeout"), + ("Healthcheck", {"Unknown": 1}, "invalid-healthcheck"), + ("Healthcheck", {"Test": [1]}, "invalid-healthcheck-value"), + ("Healthcheck", {"Retries": True}, "invalid-healthcheck-value"), + ] + for key, value, code in cases: + local, published = fixture() + local["Config"][key] = value + published["config"][key] = copy.deepcopy(value) + with self.subTest(key=key, value=value): + self.assert_stop(code, M.compare, local, published, REVISION) + + def test_loaded_and_published_platform_fields_are_exact(self): + for key, value in (("OsVersion", "6.8"), ("OsFeatures", ["feature"])): + local, published = fixture() + local[key] = value + with self.subTest(loaded=key): + self.assert_stop("loaded-platform-mismatch", M.compare, + local, published, REVISION) + for key, value in (("os.version", "6.8"), ("os.features", []), + ("variant", "v8")): + local, published = fixture() + published[key] = value + with self.subTest(published=key): + self.assert_stop("published-platform-mismatch", M.compare, + local, published, REVISION) + + def test_author_revision_and_source_are_exact(self): + for mutation, code in ( + (lambda local, published: published.update(author="other"), + "image-author-mismatch"), + (lambda local, published: published["config"]["Labels"].update( + {"org.opencontainers.image.revision": "2" * 40}), + "producer-label-mismatch"), + (lambda local, published: published["config"]["Labels"].update( + {"org.opencontainers.image.source": "https://example.invalid"}), + "producer-label-mismatch")): + local, published = fixture() + mutation(local, published) + self.assert_stop(code, M.compare, local, published, REVISION) + + def test_only_declared_top_level_build_metadata_is_exempt_from_comparison(self): + local, published = fixture() + published.update(created={"opaque": "builder value"}, history=[{"created_by": "x"}], + **{"moby.buildkit.cache.v0": {"arbitrary": [1, True, None]}}) + self.assertTrue(M.compare(local, published, REVISION)["execution_config_equal"]) + + def test_decode_rejects_empty_oversized_duplicate_and_nonfinite_json(self): + self.assert_stop("invalid-json-size", M.decode, b"") + self.assert_stop("invalid-json-size", M.decode, b" " * (M.R.LIMIT + 1)) + self.assert_stop("duplicate-json-key", M.decode, b'{"x":1,"x":2}') + for raw in (b'{"x":NaN}', b'{"x":Infinity}', b'{"x":-Infinity}'): + with self.subTest(raw=raw): + self.assert_stop("invalid-json-value", M.decode, raw) + + def test_registry_binds_complete_descriptor_chain_and_platform(self): + objects, config, manifest, index = registry_fixture() + calls = [] + def get(package, kind, digest, bearer): + calls.append((package, kind, digest, bearer)) + return objects[digest] + with patch.object(M.R, "registry_object", side_effect=get): + blobs, relation, actual = M.registry("atrinik/linux-build", INDEX_DIGEST, BEARER) + self.assertEqual(actual, config) + self.assertEqual(set(blobs), {"index.json", "manifest.json", "config.json"}) + self.assertEqual(relation["platform"], "linux/amd64") + self.assertEqual(relation["manifest_descriptor"], index["manifests"][0]) + self.assertEqual(relation["config_descriptor"], manifest["config"]) + self.assertEqual([item[1] for item in calls], ["manifests", "manifests", "blobs"]) + + def test_registry_rejects_descriptor_size_digest_platform_and_type_confusion(self): + mutations = [ + ("manifest-size", "manifest-size-mismatch", + lambda o, c, m, i: i["manifests"][0].update(size=1)), + ("config-size", "config-size-mismatch", + lambda o, c, m, i: m["config"].update(size=1)), + ("manifest-digest", "invalid-oci-descriptor", + lambda o, c, m, i: i["manifests"][0].update(digest="bad")), + ("descriptor-type", "invalid-oci-descriptor", + lambda o, c, m, i: i["manifests"][0].update(size=True)), + ("platform-extra", "unexpected-platform-metadata", + lambda o, c, m, i: i["manifests"][0]["platform"].update(variant="v8")), + ("schema-bool", "invalid-oci-index", + lambda o, c, m, i: i.update(schemaVersion=True)), + ("manifest-media", "invalid-image-manifest", + lambda o, c, m, i: m.update(mediaType=M.R.INDEX)), + ("config-media", "invalid-oci-descriptor", + lambda o, c, m, i: m["config"].update(mediaType=M.R.MANIFEST)), + ("layer-media", "invalid-oci-descriptor", + lambda o, c, m, i: m["layers"][0].update(mediaType=M.R.CONFIG)), + ] + for name, code, mutate in mutations: + objects, config, manifest, index = registry_fixture() + mutate(objects, config, manifest, index) + raw_config = encoded(config) + if name not in ("config-size", "config-media"): + manifest["config"] = descriptor(raw_config, M.R.CONFIG) + raw_manifest = encoded(manifest) + if name not in ("manifest-size", "manifest-digest", "descriptor-type"): + index["manifests"][0].update(descriptor(raw_manifest, M.R.MANIFEST)) + index["manifests"][1]["annotations"]["vnd.docker.reference.digest"] = \ + index["manifests"][0]["digest"] + raw_index = encoded(index) + objects = {INDEX_DIGEST: raw_index, + index["manifests"][0]["digest"]: raw_manifest, + manifest["config"]["digest"]: raw_config} + with self.subTest(name=name), patch.object( + M.R, "registry_object", side_effect=lambda p, k, d, b, data=objects: data[d]), \ + self.assertRaisesRegex(M.Stop, code): + M.registry("atrinik/linux-build", INDEX_DIGEST, BEARER) + + def test_registry_rejects_ambiguous_platform_attestation_and_layer_count(self): + for name, code in (("ambiguous", "ambiguous-linux-amd64-manifest"), + ("attestation-subject", "attestation-subject-mismatch"), + ("layer-count", "layer-count-mismatch")): + objects, config, manifest, index = registry_fixture() + if name == "ambiguous": + index["manifests"].append(copy.deepcopy(index["manifests"][0])) + elif name == "attestation-subject": + index["manifests"][1]["annotations"]["vnd.docker.reference.digest"] = "sha256:" + "f" * 64 + else: + config["rootfs"]["diff_ids"].pop() + raw_config = encoded(config) + manifest["config"] = descriptor(raw_config, M.R.CONFIG) + raw_manifest = encoded(manifest) + for item in index["manifests"]: + if item["platform"]["os"] == "linux": + item.update(descriptor(raw_manifest, M.R.MANIFEST)) + if name != "attestation-subject": + index["manifests"][1]["annotations"]["vnd.docker.reference.digest"] = \ + index["manifests"][0]["digest"] + raw_index = encoded(index) + data = {INDEX_DIGEST: raw_index, M.R.digest(raw_manifest): raw_manifest, + M.R.digest(raw_config): raw_config} + with self.subTest(name=name), patch.object( + M.R, "registry_object", side_effect=lambda p, k, d, b, data=data: data[d]), \ + self.assertRaisesRegex(M.Stop, code): + M.registry("atrinik/linux-build", INDEX_DIGEST, BEARER) + + def test_context_binds_host_repository_event_ref_workflow_and_source(self): + ctx = M.context(context_env()) + self.assertEqual(ctx["head"], REVISION) + self.assertFalse(ctx["candidate_only"]) + dispatch = context_env( + GITHUB_EVENT_NAME="workflow_dispatch", GITHUB_REF="refs/heads/feature/safe", + GITHUB_WORKFLOW_REF=M.R.REPOSITORY + "/" + M.WORKFLOW + "@refs/heads/feature/safe", + CANDIDATE_ONLY="true") + self.assertTrue(M.context(dispatch)["candidate_only"]) + bad = ( + {"RUNNER_ENVIRONMENT": "self-hosted"}, + {"GITHUB_REPOSITORY": "other/repo"}, + {"GITHUB_EVENT_NAME": "pull_request"}, + {"GITHUB_REF": "refs/heads/main"}, + {"GITHUB_WORKFLOW_REF": "other/workflow@refs/tags/v1.2.3"}, + {"GITHUB_SHA": "not-a-revision"}, + {"GITHUB_ACTOR": "bad actor"}, + {"OPENSSL_CONF": "/tmp/hostile"}, + ) + for changes in bad: + with self.subTest(changes=changes), self.assertRaises(M.Stop): + M.context(context_env(**changes)) + + def test_evidence_files_reject_names_links_modes_and_oversize(self): + with tempfile.TemporaryDirectory() as temporary: + folder = Path(temporary) + os.chmod(folder, 0o700) + for name in ("", ".", "..", "sub/file"): + with self.subTest(save=name), self.assertRaises(M.Stop): + M.save(folder, name, b"x") + with self.subTest(read=name), self.assertRaises(M.Stop): + M.read(folder, name) + path = folder / "evidence" + path.write_bytes(b"x") + os.chmod(path, 0o644) + self.assert_stop("unsafe-evidence-file", M.read, folder, "evidence") + os.chmod(path, 0o600) + hardlink = folder / "hardlink" + os.link(path, hardlink) + self.assert_stop("unsafe-evidence-file", M.read, folder, "evidence") + hardlink.unlink() + path.unlink() + path.symlink_to("missing") + with self.assertRaises(OSError): + M.read(folder, "evidence") + self.assert_stop("invalid-evidence-file", M.save, folder, "large", b"x" * (M.R.LIMIT + 1)) + + def test_directory_requires_absolute_owned_private_nonsymlink_path(self): + ctx = M.context(context_env()) + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + os.chmod(root, 0o700) + with patch.dict(os.environ, {"RUNNER_TEMP": str(root)}): + folder = M.directory(ctx, "broad", True) + self.assertEqual(folder.stat().st_mode & 0o777, 0o700) + os.chmod(root, 0o777) + with patch.dict(os.environ, {"RUNNER_TEMP": str(root)}): + self.assert_stop("unsafe-runner-temp", M.directory, ctx, "broad", False) + with patch.dict(os.environ, {"RUNNER_TEMP": "relative"}): + self.assert_stop("unsafe-runner-temp", M.directory, ctx, "broad", False) + + def capture_fixture(self, folder, output): + local, _ = fixture() + raw = encoded(local) + response = Response(raw) + connection = Mock() + docker = Mock() + docker.negotiate.return_value = {"docker_api": "1.51", "docker_version": "28.5.1"} + docker.request.return_value = (connection, response) + with patch.object(M.R, "runner_docker", return_value=docker), \ + patch.dict(os.environ, {"GITHUB_OUTPUT": str(output)}): + M.capture(M.context(context_env()), "broad", folder) + connection.close.assert_called_once_with() + requested = docker.request.call_args.args[1] + self.assertIn("platform=%7B%22os%22%3A%22linux%22%2C%22architecture%22%3A%22amd64%22%7D", requested) + return local + + def test_capture_writes_bound_evidence_and_only_validated_digest_output(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + folder, output = root / "evidence", root / "output" + folder.mkdir(mode=0o700) + output.touch(mode=0o600) + local = self.capture_fixture(folder, output) + capture = M.decode((folder / "capture.json").read_bytes()) + self.assertEqual(capture["image_id"], local["Id"]) + self.assertEqual(capture["target"], "broad") + self.assertEqual(output.read_text(), "image_id=" + local["Id"] + "\n") + + def test_capture_rejects_output_symlink_hardlink_writable_mode_and_relative_path(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + target = root / "target" + target.touch(mode=0o600) + hardlink = root / "hardlink" + os.link(target, hardlink) + symlink = root / "symlink" + symlink.symlink_to(target) + writable = root / "writable" + writable.touch(mode=0o666) + os.chmod(writable, 0o666) + for output in (symlink, hardlink, writable, Path("relative-output")): + folder = root / ("case-" + str(len(list(root.iterdir())))) + folder.mkdir(mode=0o700) + with self.subTest(output=output), self.assertRaises((M.Stop, OSError)): + self.capture_fixture(folder, output) + + def test_verify_binds_capture_context_image_and_registry_relation(self): + ctx = M.context(context_env()) + local, published = fixture() + raw = encoded(local) + with tempfile.TemporaryDirectory() as temporary: + folder = Path(temporary) + os.chmod(folder, 0o700) + inspection = M.save(folder, "loaded-inspect.json", raw) + M.save(folder, "capture.json", M.encoded({ + "context": ctx, "target": "broad", "docker_api": "1.51", + "docker_version": "28.5.1", "image_id": local["Id"], + "inspection": inspection, + })) + relation = {"package": M.TARGETS["broad"][0], "index_digest": INDEX_DIGEST, + "manifest_digest": "sha256:" + "d" * 64, + "config_digest": "sha256:" + "e" * 64, "platform": "linux/amd64", + "manifest_descriptor": {}, "config_descriptor": {}} + blobs = {"index.json": b"index", "manifest.json": b"manifest", "config.json": b"config"} + with patch.object(M.R, "scoped_bearer", return_value=BEARER) as bearer, \ + patch.object(M, "registry", return_value=(blobs, relation, published)) as registry, \ + patch("builtins.print"): + M.verify(ctx, "broad", folder, INDEX_DIGEST, "ORIGINAL_TOKEN") + bearer.assert_called_once_with(M.TARGETS["broad"][0], "ORIGINAL_TOKEN", ctx["actor"]) + registry.assert_called_once_with(M.TARGETS["broad"][0], INDEX_DIGEST, BEARER) + report = M.decode((folder / "report.json").read_bytes()) + self.assertEqual(report["status"], "verified-published-identity") + self.assertEqual(report["loaded_image_id"], local["Id"]) + self.assertEqual(report["context"], ctx) + + def test_verify_rejects_capture_context_and_image_drift_before_registry(self): + ctx = M.context(context_env()) + local, _ = fixture() + raw = encoded(local) + for change, code in (("context", "capture-context-mismatch"), + ("image", "capture-image-mismatch")): + with tempfile.TemporaryDirectory() as temporary: + folder = Path(temporary) + os.chmod(folder, 0o700) + inspection = M.save(folder, "loaded-inspect.json", raw) + captured = {"context": ctx, "target": "broad", "image_id": local["Id"], + "inspection": inspection} + if change == "context": + captured["context"] = dict(ctx, run_attempt="3") + else: + captured["image_id"] = "sha256:" + "f" * 64 + M.save(folder, "capture.json", M.encoded(captured)) + with self.subTest(change=change), patch.object( + M.R, "scoped_bearer") as bearer, self.assertRaisesRegex(M.Stop, code): + M.verify(ctx, "broad", folder, INDEX_DIGEST, "ORIGINAL_TOKEN") + bearer.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/tests/test_workflow_contracts.py b/tools/tests/test_workflow_contracts.py index 50d4823..ff5fce6 100644 --- a/tools/tests/test_workflow_contracts.py +++ b/tools/tests/test_workflow_contracts.py @@ -21,13 +21,18 @@ def step_script(name, step): start = text.index(f' - name: {step}\n') block = text[start:].split('\n - name: ', 1)[0] script = block.split(' run: |\n', 1)[1] - return '\n'.join(line[10:] for line in script.splitlines()) + '\n' + lines = [] + for line in script.splitlines(): + if line.strip() and not line.startswith(' '): + break + lines.append(line[10:]) + return '\n'.join(lines) + '\n' class WorkflowContracts(unittest.TestCase): def test_portable_public_ca_command_and_failure_are_mandatory(self): for name, image in (('validate.yml', 'atrinik-classic-portable:validation'), - ('publish-linux.yml', 'atrinik-classic-portable:release-validation')): + ('publish-linux.yml', 'sha256:' + 'a' * 64)): for docker_exit in (0, 17): with self.subTest(workflow=name, docker_exit=docker_exit), tempfile.TemporaryDirectory() as directory: root = Path(directory) @@ -40,7 +45,7 @@ def test_portable_public_ca_command_and_failure_are_mandatory(self): docker.chmod(0o755) env = dict(os.environ, PATH=directory + ':' + os.environ['PATH'], GITHUB_WORKSPACE=directory, DOCKER_CALL=str(root / 'docker-call.json'), - DOCKER_EXIT=str(docker_exit), HTTPS_PROXY='http://fixture-user:fixture-password@invalid') + DOCKER_EXIT=str(docker_exit), IMAGE_ID=image, HTTPS_PROXY='http://fixture-user:fixture-password@invalid') result = subprocess.run(['bash', '-euo', 'pipefail', '-c', step_script(name, 'Verify portable candidate public CA')], cwd=directory, env=env, capture_output=True, text=True) @@ -78,7 +83,7 @@ def test_portable_public_ca_gate_precedes_consumer_and_publication(self): self.assertIn('set -euo pipefail', block) if name == 'publish-linux.yml': self.assertLess(gate, text.index(' - name: Publish immutable portable candidate')) - self.assertLess(gate, text.index(' - name: Promote validated portable aliases')) + self.assertLess(gate, text.index(' - name: Promote validated Linux aliases')) def run_step(self, name, step, **values): with tempfile.TemporaryDirectory() as directory: @@ -133,7 +138,7 @@ def test_windows_candidate_and_release_tag_names(self): def test_candidate_mode_keeps_qualification_and_fences_promotion(self): linux = workflow('publish-linux.yml') self.assertEqual(linux.count('if: ${{ !inputs.candidate_only }}'), 1) - self.assertIn('Promote validated portable aliases\n if: ${{ !inputs.candidate_only }}', linux) + self.assertIn('Promote validated Linux aliases\n if: ${{ !inputs.candidate_only }}', linux) for target in ('target: classic-validation', 'target: validation', 'target: portable-validation'): self.assertIn(target, linux) @@ -145,6 +150,109 @@ def test_candidate_mode_keeps_qualification_and_fences_promotion(self): self.assertNotIn('candidate_only', native) self.assertIn('run-classic-native-tests.ps1', native) + def test_linux_publish_uses_only_sha_tags_then_fences_every_alias(self): + text = workflow('publish-linux.yml') + for step, output in (('Build and publish Linux image', 'metadata.outputs.publish_tag'), + ('Build and publish Classic image', 'classic-metadata.outputs.publish_tag'), + ('Publish immutable portable candidate', 'portable-metadata.outputs.tag')): + block = text.split(' - name: ' + step + '\n')[1].split('\n - name: ')[0] + self.assertIn('tags: ${{ steps.' + output + ' }}', block) + self.assertNotIn('outputs.tags', block) + fence = text.index(' - name: Verify all published image identities') + upload = text.index(' - name: Retain published identity evidence') + promote = text.index(' - name: Promote validated Linux aliases') + self.assertLess(text.index(' - name: Publish immutable portable candidate'), fence) + self.assertLess(fence, upload) + self.assertLess(upload, promote) + for block in (text[fence:upload], text[upload:promote]): + self.assertNotIn('continue-on-error', block) + self.assertNotIn(' if:', block) + self.assertIn('if-no-files-found: error', text[upload:promote]) + for candidate in ('false', 'true'): + for step, image in (('Select image tags', 'linux-build'), + ('Select Classic image tags', 'classic-build')): + result = self.run_step('publish-linux.yml', step, IMAGE_NAME=image, + CANDIDATE_ONLY=candidate) + prefix = 'candidate-' if candidate == 'true' else '' + self.assertIn(f'publish_tag=ghcr.io/atrinik/{image}:{prefix}sha-' + 'a' * 40, result) + + def test_linux_runtime_checks_use_captured_ids(self): + text = workflow('publish-linux.yml') + checks = { + 'classic': ['Qualify Classic cancellation and public TLS before publication', + 'Run Classic client and server checks before publication'], + 'broad': ['Qualify broad Linux cancellation and public TLS before publication'], + 'portable': ['Verify portable candidate public CA', + 'Validate real portable consumer before publication'], + } + for target, names in checks.items(): + captured = text.index(' - name: Capture loaded ' + target + ' image identity') + for name in names: + start = text.index(' - name: ' + name + '\n') + self.assertLess(captured, start) + block = text[start:].split('\n - name: ')[0] + self.assertIn('IMAGE_ID: ${{ steps.' + target + '-identity.outputs.image_id }}', block) + self.assertIn('"${IMAGE_ID}"', block) + self.assertNotIn(':qualification', block) + self.assertNotIn(':release-validation', block) + + def test_published_fence_stops_on_first_failed_identity(self): + for failed in ('none', 'broad', 'classic', 'portable'): + with self.subTest(failed=failed), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + command = root / 'python3' + command.write_text('#!/usr/bin/env python3\nimport os, sys\n' + 'from pathlib import Path\n' + 'target = sys.argv[sys.argv.index("--target") + 1]\n' + 'with Path(os.environ["CALLS"]).open("a") as out: out.write(target + "\\n")\n' + 'sys.exit(19 if target == os.environ["FAIL_TARGET"] else 0)\n') + # Avoid recursively selecting the fake python interpreter. + command.write_text(command.read_text().replace('#!/usr/bin/env python3', '#!/usr/bin/python3')) + command.chmod(0o755) + env = dict(os.environ, PATH=directory + ':' + os.environ['PATH'], + CALLS=str(root / 'calls'), FAIL_TARGET=failed, + BROAD_DIGEST='sha256:' + 'a' * 64, CLASSIC_DIGEST='sha256:' + 'b' * 64, + PORTABLE_DIGEST='sha256:' + 'c' * 64) + result = subprocess.run(['bash', '-euo', 'pipefail', '-c', + step_script('publish-linux.yml', 'Verify all published image identities')], + env=env, capture_output=True, text=True) + self.assertEqual(result.returncode, 0 if failed == 'none' else 19) + targets = ['broad', 'classic', 'portable'] + expected = targets if failed == 'none' else targets[:targets.index(failed) + 1] + self.assertEqual((root / 'calls').read_text().splitlines(), expected) + + def test_linux_alias_promotion_uses_verified_digests_and_stops_on_failure(self): + for fail_call in (0, 2): + with self.subTest(fail_call=fail_call), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + docker = root / 'docker' + docker.write_text('#!/usr/bin/python3\nimport json, os, sys\n' + 'from pathlib import Path\n' + 'path = Path(os.environ["CALLS"])\n' + 'calls = json.loads(path.read_text()) if path.exists() else []\n' + 'calls.append(sys.argv[1:]); path.write_text(json.dumps(calls))\n' + 'sys.exit(17 if len(calls) == int(os.environ["FAIL_CALL"]) else 0)\n') + docker.chmod(0o755) + digests = ['sha256:' + char * 64 for char in 'abc'] + env = dict(os.environ, PATH=directory + ':' + os.environ['PATH'], + CALLS=str(root / 'calls'), FAIL_CALL=str(fail_call), + BROAD_DIGEST=digests[0], CLASSIC_DIGEST=digests[1], PORTABLE_DIGEST=digests[2], + GITHUB_REPOSITORY_OWNER='Atrinik', GITHUB_REF_TYPE='tag', GITHUB_REF_NAME='v9.8.7', + GITHUB_STEP_SUMMARY=str(root / 'summary')) + result = subprocess.run(['bash', '-euo', 'pipefail', '-c', + step_script('publish-linux.yml', 'Promote validated Linux aliases')], + env=env, capture_output=True, text=True) + self.assertEqual(result.returncode, 0 if fail_call == 0 else 1, result.stderr) + calls = json.loads((root / 'calls').read_text()) + self.assertEqual(len(calls), 3 if fail_call == 0 else 2) + for index, call in enumerate(calls): + image = 'ghcr.io/atrinik/' + ['linux-build', 'classic-build', 'classic-portable-build'][index] + platform = ['ubuntu-26.04', 'ubuntu-26.04', 'debian-12'][index] + self.assertEqual(call, ['buildx', 'imagetools', 'create', '--tag', image + ':latest', + '--tag', image + ':' + platform, '--tag', image + ':9.8.7', image + '@' + digests[index]]) + if fail_call: + self.assertIn('Rerun the failed job in this workflow run', result.stderr) + def test_shared_llvm_inputs_select_every_image(self): script = step_script('validate.yml', 'Detect image changes') inputs = ('llvm-toolchain.json', 'llvm-bootstrap-packages.lock', @@ -165,6 +273,45 @@ def test_shared_llvm_inputs_select_every_image(self): self.assertEqual(set(output.read_text().splitlines()), {'portable=true', 'classic=true', 'linux=true', 'windows=true'}) + def test_publication_identity_inputs_select_linux_images(self): + script = step_script('validate.yml', 'Detect image changes') + cases = ( + ('tools/verify_published_identity.py', True, False), + ('tools/tests/test_verify_published_identity.py', True, True), + ('tools/collect-released-sdk-metadata.py', True, False), + ('tools/tests/test_collect_released_sdk_metadata.py', True, True), + ('windows/Dockerfile', False, True), + ('README.md', False, False), + ('tools/unrelated-helper.py', False, False), + ) + for changed, linux_selected, windows_selected in cases: + with self.subTest(changed=changed), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + # Real isolated Git history exercises the workflow's exact diff command. + def git(*args): + return subprocess.run(['git', *args], cwd=root, check=True, + capture_output=True, text=True).stdout.strip() + git('init', '-q') + git('-c', 'user.name=Fixture', '-c', 'user.email=fixture@example.invalid', + '-c', 'commit.gpgsign=false', 'commit', '-qm', 'base', '--allow-empty') + base = git('rev-parse', 'HEAD') + source = root / changed + source.parent.mkdir(parents=True, exist_ok=True) + source.write_text('isolated selector input\n') + git('add', '--', changed) + git('-c', 'user.name=Fixture', '-c', 'user.email=fixture@example.invalid', + '-c', 'commit.gpgsign=false', 'commit', '-qm', 'change') + output = root / 'output' + env = dict(os.environ, GITHUB_OUTPUT=str(output), + BASE_SHA=base, HEAD_SHA=git('rev-parse', 'HEAD')) + subprocess.run(['bash', '-euo', 'pipefail', '-c', script], cwd=root, + env=env, check=True, capture_output=True, text=True) + selected = 'true' if linux_selected else 'false' + windows = 'true' if windows_selected else 'false' + self.assertEqual(set(output.read_text().splitlines()), + {f'portable={selected}', f'classic={selected}', + f'linux={selected}', f'windows={windows}'}) + def test_pr_workflow_remains_private_package_free_and_strict(self): text = workflow('validate.yml') self.assertNotIn('packages:', text) diff --git a/tools/verify_published_identity.py b/tools/verify_published_identity.py new file mode 100644 index 0000000..2e6df18 --- /dev/null +++ b/tools/verify_published_identity.py @@ -0,0 +1,367 @@ +#!/usr/bin/python3 -I +# Copyright (c) 2026 Atrinik contributors +# SPDX-License-Identifier: MIT +"""Bind Linux publisher runtime-tested image IDs to actual pushed OCI metadata. + +No registry layers are downloaded and no image code is executed here. Only the +same-repository hosted publisher may capture the local daemon or acquire metadata. +The historical released-SDK collector's target/revision/admission stays unchanged. +""" +import argparse +import importlib.util +import json +import os +from pathlib import Path +import re +import resource +import signal +import stat +import sys +import urllib.parse + +_SPEC = importlib.util.spec_from_file_location( + 'registry_transport', Path(__file__).with_name('collect-released-sdk-metadata.py')) +R = importlib.util.module_from_spec(_SPEC) +_SPEC.loader.exec_module(R) +require = R.require +Stop = R.Stop +TARGETS = { + 'broad': ('atrinik/linux-build', 'atrinik-linux-curl:qualification'), + 'classic': ('atrinik/classic-build', 'atrinik-classic-curl:qualification'), + 'portable': ('atrinik/classic-portable-build', 'atrinik-classic-portable:release-validation'), +} +WORKFLOW = '.github/workflows/publish-linux.yml' +# Docker API 1.49 also materializes nine zero-valued container-only fields; +# API 1.50 removes them. All supported APIs through 1.51 materialize the other +# absence defaults below. Never equate empty arrays/objects with null, nor +# normalize non-default values or unknown keys. +DEFAULTS = {'Cmd': None, 'Entrypoint': None, 'Env': None, 'Labels': None, + 'OnBuild': None, 'Volumes': None, 'User': '', 'WorkingDir': '', + 'Hostname': '', 'Domainname': '', 'Image': '', + 'AttachStdin': False, 'AttachStdout': False, 'AttachStderr': False, + 'Tty': False, 'OpenStdin': False, 'StdinOnce': False} +CONFIG_KEYS = {'architecture', 'os', 'variant', 'os.version', 'os.features', + 'config', 'rootfs', 'author', 'created', 'history', 'moby.buildkit.cache.v0'} +STRING_FIELDS = {'Hostname', 'Domainname', 'User', 'Image', 'WorkingDir', 'MacAddress', 'StopSignal'} +BOOL_FIELDS = {'AttachStdin', 'AttachStdout', 'AttachStderr', 'Tty', 'OpenStdin', 'StdinOnce', + 'ArgsEscaped', 'NetworkDisabled'} +ARRAY_FIELDS = {'Env', 'Cmd', 'Entrypoint', 'OnBuild', 'Shell'} +MAP_FIELDS = {'Labels', 'Volumes', 'ExposedPorts'} +RUNTIME_FIELDS = STRING_FIELDS | BOOL_FIELDS | ARRAY_FIELDS | MAP_FIELDS | {'Healthcheck', 'StopTimeout'} +PUBLICATION_LABELS = {'org.opencontainers.image.source', 'org.opencontainers.image.revision'} + + +def decode(raw): + require(isinstance(raw, bytes) and 0 < len(raw) <= R.LIMIT, 'invalid-json-size') + value = R.decode(raw) + # Reject non-finite numbers accepted by Python's default parser; JSON from + # Docker/OCI cannot contain these. This also bounds hostile deep documents. + try: + json.dumps(value, allow_nan=False) + except (ValueError, RecursionError): + raise Stop('invalid-json-value') from None + return value + + +def context(env): + require(env.get('GITHUB_ACTIONS') == 'true' and + env.get('RUNNER_ENVIRONMENT') == 'github-hosted' and sys.platform == 'linux', + 'hosted-runner-required') + require(env.get('GITHUB_REPOSITORY') == R.REPOSITORY, 'repository-not-allowed') + ref, event = env.get('GITHUB_REF', ''), env.get('GITHUB_EVENT_NAME') + candidate = env.get('CANDIDATE_ONLY') == 'true' + release = re.fullmatch(r'refs/tags/v[0-9]+\.[0-9]+\.[0-9]+', ref) is not None + require((event == 'push' and release and not candidate) or + (event == 'workflow_dispatch' and + (release or (ref == 'refs/heads/main') or + (candidate and re.fullmatch(r'refs/heads/[A-Za-z0-9._/-]+', ref)))), + 'untrusted-workflow-event') + require(env.get('GITHUB_WORKFLOW_REF') == R.REPOSITORY + '/' + WORKFLOW + '@' + ref, + 'workflow-identity-mismatch') + head, actor = env.get('GITHUB_SHA', ''), env.get('GITHUB_ACTOR', '') + require(re.fullmatch(r'[0-9a-f]{40}', head) is not None, 'invalid-workflow-head') + require(re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9-]{0,38}(?:\[bot\])?', actor) is not None, + 'invalid-workflow-actor') + for key in ('GITHUB_RUN_ID', 'GITHUB_RUN_ATTEMPT'): + require(re.fullmatch(r'[1-9][0-9]{0,19}', env.get(key, '')) is not None, 'invalid-workflow-run') + require(not any(env.get(key) for key in ('OPENSSL_CONF', 'OPENSSL_MODULES', 'SSLKEYLOGFILE')), + 'tls-override-not-allowed') + return {'repository': R.REPOSITORY, 'workflow': WORKFLOW, 'ref': ref, 'event': event, + 'head': head, 'actor': actor, 'candidate_only': candidate, + 'run_id': env['GITHUB_RUN_ID'], 'run_attempt': env['GITHUB_RUN_ATTEMPT']} + + +def loaded_identity(local): + require(isinstance(local, dict) and R.valid_digest(local.get('Id')), 'invalid-loaded-image') + require(local.get('Os') == 'linux' and local.get('Architecture') == 'amd64' and + local.get('Variant', '') == '' and not local.get('OsVersion') and + not local.get('OsFeatures'), 'loaded-platform-mismatch') + rootfs = local.get('RootFS') + require(isinstance(rootfs, dict) and set(rootfs) == {'Type', 'Layers'} and + rootfs['Type'] == 'layers' and isinstance(rootfs['Layers'], list) and rootfs['Layers'] and + all(R.valid_digest(item) for item in rootfs['Layers']), 'invalid-loaded-rootfs') + validate_runtime(local.get('Config')) + return local['Id'] + + +def validate_runtime(config): + require(isinstance(config, dict) and set(config) <= RUNTIME_FIELDS, 'unknown-runtime-config-field') + for key, value in config.items(): + if key in STRING_FIELDS: + require(isinstance(value, str), 'invalid-runtime-string') + elif key in BOOL_FIELDS: + require(type(value) is bool, 'invalid-runtime-boolean') + elif key in ARRAY_FIELDS: + require(value is None or (isinstance(value, list) and + all(isinstance(item, str) for item in value)), 'invalid-runtime-array') + elif key in MAP_FIELDS: + require(value is None or isinstance(value, dict), 'invalid-runtime-map') + for item in (value or {}).values(): + require(isinstance(item, str) if key == 'Labels' else item == {}, 'invalid-runtime-map-value') + elif key == 'StopTimeout': + require(type(value) is int and value >= 0, 'invalid-runtime-timeout') + elif key == 'Healthcheck': + require(value is None or (isinstance(value, dict) and set(value) <= { + 'Test', 'Interval', 'Timeout', 'StartPeriod', 'StartInterval', 'Retries'}), + 'invalid-healthcheck') + for field, item in (value or {}).items(): + require((isinstance(item, list) and all(isinstance(part, str) for part in item)) + if field == 'Test' else (type(item) is int and item >= 0), 'invalid-healthcheck-value') + + +def same_json(left, right): + # Python compares True == 1; use JSON equality to retain scalar types. + return json.dumps(left, sort_keys=True, allow_nan=False) == json.dumps(right, sort_keys=True, allow_nan=False) + + +def compare(local, config, revision): + loaded_identity(local) + require(isinstance(config, dict) and set(config) <= CONFIG_KEYS, 'unknown-image-config-field') + require(config.get('os') == 'linux' and config.get('architecture') == 'amd64' and + config.get('variant', '') == '' and 'os.version' not in config and + 'os.features' not in config, 'published-platform-mismatch') + require(config.get('author', '') == local.get('Author', ''), 'image-author-mismatch') + rootfs = config.get('rootfs') + require(isinstance(rootfs, dict) and set(rootfs) == {'type', 'diff_ids'} and + rootfs['type'] == 'layers' and rootfs['diff_ids'] == local['RootFS']['Layers'], + 'published-rootfs-mismatch') + expected, actual = dict(local['Config']), config.get('config') + validate_runtime(actual) + actual = dict(actual) + labels = actual.get('Labels') + require(isinstance(labels, dict) and + labels.get('org.opencontainers.image.source') == R.SOURCE and + labels.get('org.opencontainers.image.revision') == revision, 'producer-label-mismatch') + for cfg in (expected, actual): + labels = cfg.get('Labels') + require(labels is None or (isinstance(labels, dict) and + all(isinstance(k, str) and isinstance(v, str) for k, v in labels.items())), + 'invalid-runtime-labels') + # Only these exact, validated publication labels may differ. + cfg['Labels'] = {k: v for k, v in (labels or {}).items() if k not in PUBLICATION_LABELS} + for key, default in DEFAULTS.items(): + cfg.setdefault(key, default) + require(same_json(expected, actual), 'published-runtime-config-mismatch') + return {'loaded_image_id': local['Id'], 'rootfs_equal': True, 'execution_config_equal': True, + 'publication_exceptions': ['config.Labels.org.opencontainers.image.source', + 'config.Labels.org.opencontainers.image.revision', 'created', 'history', + 'moby.buildkit.cache.v0'], 'docker_absent_defaults': DEFAULTS} + + +def registry(package, index_digest, bearer): + # Reuse only bounded HTTPS/auth/digest primitives, never the historical + # collector's revision gate, pulls, containers, files, or acceptance claims. + index_raw = R.registry_object(package, 'manifests', index_digest, bearer) + index = decode(index_raw) + require(isinstance(index, dict) and type(index.get('schemaVersion')) is int and + index['schemaVersion'] == 2 and index.get('mediaType') == R.INDEX and + isinstance(index.get('manifests'), list), 'invalid-oci-index') + candidates = [] + for item in index['manifests']: + R.descriptor(item, {R.MANIFEST}) + platform, annotations = item.get('platform'), item.get('annotations', {}) + require(isinstance(platform, dict) and isinstance(annotations, dict), 'invalid-index-platform') + if platform.get('os') == 'linux' and platform.get('architecture') == 'amd64': + require(platform == {'os': 'linux', 'architecture': 'amd64'} and + annotations.get('vnd.docker.reference.type') != 'attestation-manifest', + 'unexpected-platform-metadata') + candidates.append(item) + else: + require(platform == {'os': 'unknown', 'architecture': 'unknown'} and + annotations.get('vnd.docker.reference.type') == 'attestation-manifest', + 'unexpected-index-platform') + require(len(candidates) == 1, 'ambiguous-linux-amd64-manifest') + selected = candidates[0] + for item in index['manifests']: + if item is not selected: + require(item['annotations'].get('vnd.docker.reference.digest') == selected['digest'], + 'attestation-subject-mismatch') + manifest_raw = R.registry_object(package, 'manifests', selected['digest'], bearer) + require(len(manifest_raw) == selected['size'], 'manifest-size-mismatch') + manifest = decode(manifest_raw) + require(isinstance(manifest, dict) and type(manifest.get('schemaVersion')) is int and + manifest['schemaVersion'] == 2 and manifest.get('mediaType') == R.MANIFEST and + not manifest.get('artifactType') and not manifest.get('subject'), 'invalid-image-manifest') + desc = R.descriptor(manifest.get('config'), {R.CONFIG}) + require(desc['size'] <= R.LIMIT and isinstance(manifest.get('layers'), list) and + manifest['layers'], 'invalid-image-layers') + for layer in manifest['layers']: + R.descriptor(layer, R.LAYERS) + config_raw = R.registry_object(package, 'blobs', desc['digest'], bearer) + require(len(config_raw) == desc['size'], 'config-size-mismatch') + config = decode(config_raw) + require(isinstance(config, dict) and isinstance(config.get('rootfs'), dict) and + isinstance(config['rootfs'].get('diff_ids'), list) and + len(config['rootfs']['diff_ids']) == len(manifest['layers']), 'layer-count-mismatch') + return {'index.json': index_raw, 'manifest.json': manifest_raw, 'config.json': config_raw}, { + 'package': package, 'index_digest': index_digest, 'manifest_digest': selected['digest'], + 'config_digest': desc['digest'], 'platform': 'linux/amd64', + 'manifest_descriptor': selected, 'config_descriptor': desc}, config + + +def safe_directory(path): + info = path.lstat() + require(stat.S_ISDIR(info.st_mode) and info.st_uid == os.getuid() and + info.st_mode & 0o077 == 0, 'unsafe-evidence-directory') + + +def directory(ctx, target, create): + runner_temp = Path(os.environ['RUNNER_TEMP']) + require(runner_temp.is_absolute() and runner_temp.resolve() == runner_temp, 'unsafe-runner-temp') + info = runner_temp.stat() + require(stat.S_ISDIR(info.st_mode) and info.st_uid == os.getuid() and + not info.st_mode & 0o022, 'unsafe-runner-temp') + root = runner_temp / 'linux-published-identity' + if create: + root.mkdir(mode=0o700, exist_ok=True) + safe_directory(root) + run = root / (ctx['run_id'] + '-' + ctx['run_attempt']) + if create: + run.mkdir(mode=0o700, exist_ok=True) + safe_directory(run) + folder = run / target + if create: + folder.mkdir(mode=0o700) + safe_directory(folder) + return folder + + +def save(folder, name, raw): + require('/' not in name and name not in ('', '.', '..') and len(raw) <= R.LIMIT, + 'invalid-evidence-file') + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW + fd = os.open(folder / name, flags, 0o600) + with os.fdopen(fd, 'wb') as output: + output.write(raw) + return {'file': name, 'digest': R.digest(raw), 'size': len(raw)} + + +def read(folder, name): + require('/' not in name and name not in ('', '.', '..'), 'invalid-evidence-file') + fd = os.open(folder / name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + with os.fdopen(fd, 'rb') as source: + info = os.fstat(source.fileno()) + require(stat.S_ISREG(info.st_mode) and info.st_uid == os.getuid() and info.st_nlink == 1 and + info.st_mode & 0o077 == 0 and 0 < info.st_size <= R.LIMIT, 'unsafe-evidence-file') + raw = source.read(R.LIMIT + 1) + require(len(raw) == info.st_size, 'evidence-file-changed') + return raw + + +def encoded(value): + return (json.dumps(value, sort_keys=True, indent=2, allow_nan=False) + '\n').encode() + + +def capture(ctx, target, folder): + docker = R.runner_docker() + daemon = docker.negotiate() + query = urllib.parse.urlencode({'platform': '{"os":"linux","architecture":"amd64"}'}) + path = '/images/' + urllib.parse.quote(TARGETS[target][1], safe='') + '/json?' + query + conn, response = docker.request('GET', path) + try: + raw = R.bounded_read(response) + finally: + conn.close() + local = decode(raw) + image_id = loaded_identity(local) + artifact = save(folder, 'loaded-inspect.json', raw) + save(folder, 'capture.json', encoded({'context': ctx, 'target': target, **daemon, + 'image_id': image_id, 'inspection': artifact})) + # GitHub owns this output path; it may be readable but never writable by + # other users. Only append the validated digest to the existing regular file. + output_path = Path(os.environ['GITHUB_OUTPUT']) + require(output_path.is_absolute() and output_path.resolve() == output_path, + 'unsafe-github-output') + fd = os.open(output_path, os.O_WRONLY | os.O_APPEND | os.O_NOFOLLOW | os.O_NONBLOCK) + with os.fdopen(fd, 'w', encoding='utf-8') as output: + info = os.fstat(output.fileno()) + require(stat.S_ISREG(info.st_mode) and info.st_uid == os.getuid() and + info.st_nlink == 1 and not info.st_mode & 0o022, 'unsafe-github-output') + output.write('image_id=' + image_id + '\n') + + +def verify(ctx, target, folder, index_digest, original): + require(R.valid_digest(index_digest), 'invalid-published-digest') + require(0 < len(original) < 16384 and all(32 < ord(c) < 127 for c in original), + 'workflow-token-unavailable') + raw = read(folder, 'loaded-inspect.json') + captured = decode(read(folder, 'capture.json')) + require(isinstance(captured, dict) and captured.get('context') == ctx and + captured.get('target') == target and captured.get('inspection') == { + 'file': 'loaded-inspect.json', 'digest': R.digest(raw), 'size': len(raw)}, + 'capture-context-mismatch') + local = decode(raw) + require(captured.get('image_id') == loaded_identity(local), 'capture-image-mismatch') + package = TARGETS[target][0] + bearer = R.scoped_bearer(package, original, ctx['actor']) + blobs, relation, config = registry(package, index_digest, bearer) + artifacts = [save(folder, name, content) for name, content in blobs.items()] + result = compare(local, config, ctx['head']) + report = {'schema_version': 1, 'context': ctx, 'target': target, + 'status': 'verified-published-identity', **relation, **result, + 'artifacts': [captured['inspection'], *artifacts], + 'verifier_digest': R.digest(Path(__file__).read_bytes())} + raw_report = encoded(report) + save(folder, 'report.json', raw_report) + save(folder, 'report.sha256', (R.digest(raw_report).removeprefix('sha256:') + ' report.json\n').encode()) + print(json.dumps({'target': target, 'status': report['status'], 'report_digest': R.digest(raw_report)})) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('operation', choices=('capture', 'verify')) + parser.add_argument('--target', required=True, choices=tuple(TARGETS)) + parser.add_argument('--digest') + args = parser.parse_args() + original = os.environ.pop('GHCR_TOKEN', '') + for key in ('GH_TOKEN', 'GITHUB_TOKEN', 'GH_CONFIG_DIR', 'XDG_CONFIG_HOME', 'GH_DEBUG'): + os.environ.pop(key, None) + def stop(_signum, _frame): + raise Stop('operation-interrupted') + try: + require(sys.flags.isolated == 1, 'isolated-python-required') + os.umask(0o077) + resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) + signal.signal(signal.SIGALRM, stop) + signal.signal(signal.SIGTERM, stop) + signal.signal(signal.SIGINT, stop) + signal.alarm(300) + ctx = context(os.environ) + R.verify_checkout(ctx) + folder = directory(ctx, args.target, args.operation == 'capture') + if args.operation == 'capture': + require(args.digest is None, 'unexpected-capture-digest') + capture(ctx, args.target, folder) + else: + verify(ctx, args.target, folder, args.digest, original) + return 0 + except BaseException: + # No raw daemon/registry/auth/error bodies or signed redirect URLs in logs. + print('{"status":"stopped","reason":"published-identity-fence-failed"}') + return 1 + finally: + signal.alarm(0) + + +if __name__ == '__main__': + sys.exit(main())