From 8a8245ace10c9165713726d07f775d4e5c3d82f1 Mon Sep 17 00:00:00 2001 From: Zoey Rose Date: Sun, 11 Oct 2026 20:06:40 -0500 Subject: [PATCH 01/23] fix(portable): bind CMake and Ninja to installed producer bytes --- AGENTS.md | 8 +- README.md | 19 ++- portable/build_tools.py | 211 ++++++++++++++++++++++++++++ portable/consumer.sh | 8 +- portable/inventory.py | 9 +- portable/test_build_tools.py | 259 +++++++++++++++++++++++++++++++++++ 6 files changed, 506 insertions(+), 8 deletions(-) create mode 100644 portable/build_tools.py create mode 100644 portable/test_build_tools.py diff --git a/AGENTS.md b/AGENTS.md index 6490e6f..e8d029e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -24,7 +24,13 @@ source commits or shader inputs. Schema 2 compiler evidence requires every selected LLVM utility, including separate Clang C/C++ drivers and archiver/ranlib, with canonical paths, versions and hashes; rehash before - consumer builds and publication. Portable curl is an LLVM rebuild of + consumer builds and publication. The separate schema-1 `installed.json.build_tools` + receipt binds fixed `/usr/bin/cmake` and `/usr/bin/ninja` paths, canonical + executables, hashes, sizes and full version stdout. Verify both protected + identities before probing either tool and fence configure/build execution; + missing receipts require a rebuilt producer. The producer OS, libraries and + CMake modules and earlier library-production builds remain trusted; this + receipt does not attest historical execution. Portable curl is an LLVM rebuild of security-patched Debian `7.88.1-10+deb12u15` with c-ares 1.34.6, retaining existing features including LDAP/LDAPS, RTMP, SCP and SFTP. Its separate `portable/curl-toolchain.json`, `portable/build-curl.py`, diff --git a/README.md b/README.md index 5fe13b4..d6965ff 100644 --- a/README.md +++ b/README.md @@ -462,8 +462,23 @@ shader-generation record under versions and hashes for every selected LLVM utility, including independent `compiler` (`clang`) and `cxx_compiler` (`clang++`) records plus `llvm-ar` and `llvm-ranlib`; incomplete earlier evidence is rejected. Consumers rehash these -identities before building and publication. Shared application libraries live under `/usr/local`; -Debian packages provide the baseline transitive libraries. Compiler flags use +identities before building and publication. + +`installed.json.build_tools` separately seals CMake and Ninja with schema 1, +fixed `/usr/bin` paths, canonical executable paths, SHA-256 hashes, byte sizes, +executable status and complete version stdout. Its first lines agree with the +existing installed tool version strings. The consumer verifies both protected +root-owned executables through open descriptors before any version probe and +fences their resolution, bytes and file identity around configure and build. +SDKs without this producer receipt require a rebuild. This check trusts the +producer OS, dynamic libraries and CMake modules; it does not attest the full +transitive execution environment. +It binds the installed executables used by the later consumer; earlier library +builds remain part of trusted SDK production and are not historical execution +attestations. + +Shared application libraries live under `/usr/local`; Debian packages provide +the baseline transitive libraries. Compiler flags use `-march=x86-64 -mtune=generic`. Verification checks actual ELF dependency providers, versioned symbols, loader relocations and CPU notes, plus device-free image/font decoding, audio decoding and OpenSSL provider loading. The diff --git a/portable/build_tools.py b/portable/build_tools.py new file mode 100644 index 0000000..82aa24a --- /dev/null +++ b/portable/build_tools.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python3 +"""Bind portable CMake/Ninja execution to the producer's installed bytes. + +The producer OS, dynamic libraries and CMake modules remain trusted. These +receipts protect the selected executables against an unprivileged consumer; +they do not attest the complete transitive execution environment. +""" +from __future__ import annotations + +import argparse +from contextlib import contextmanager +import hashlib +import json +import os +from pathlib import Path +import posixpath +import re +import stat +import subprocess + +PATHS = {"cmake": "/usr/bin/cmake", "ninja": "/usr/bin/ninja"} +FIELDS = {"path", "resolved_path", "sha256", "size", "executable", "version"} + + +def validate(installed: dict) -> dict: + if type(installed) is not dict: + raise ValueError("invalid producer installed metadata object") + receipt = installed.get("build_tools") + if receipt is None: + raise ValueError("missing producer build_tools receipt; rebuild the portable SDK with CMake/Ninja identities") + if (type(receipt) is not dict or set(receipt) != {"schema_version", "tools"} + or type(receipt["schema_version"]) is not int or receipt["schema_version"] != 1 + or type(receipt["tools"]) is not dict or set(receipt["tools"]) != set(PATHS)): + raise ValueError("invalid producer build_tools schema or roles") + legacy = installed.get("tools") + if type(legacy) is not dict: + raise ValueError("missing installed tool version strings") + for role, path in PATHS.items(): + item = receipt["tools"][role] + if type(item) is not dict or set(item) != FIELDS or item["path"] != path: + raise ValueError("invalid producer build tool fields or fixed path: " + role) + resolved = item["resolved_path"] + if (type(resolved) is not str or not resolved.startswith("/") + or resolved.startswith("//") or "\x00" in resolved + or posixpath.normpath(resolved) != resolved or resolved == "/"): + raise ValueError("invalid canonical build tool path: " + role) + if (type(item["sha256"]) is not str or not re.fullmatch(r"[0-9a-f]{64}", item["sha256"]) + or type(item["size"]) is not int or item["size"] <= 0 + or item["executable"] is not True): + raise ValueError("invalid producer build tool identity: " + role) + version = item["version"] + if (type(version) is not str or not version or len(version) > 65536 + or "\x00" in version or "\r" in version): + raise ValueError("invalid complete build tool version: " + role) + first = version.splitlines()[0] + pattern = r"cmake version [0-9]+\.[0-9]+\.[0-9]+" if role == "cmake" else r"[0-9]+\.[0-9]+(?:\.[0-9]+)?" + if not re.fullmatch(pattern, first) or type(legacy.get(role)) is not str or legacy[role] != first: + raise ValueError("build tool version disagrees with installed.tools: " + role) + return receipt + + +def _protected_path(path: Path) -> Path: + """Check lexical links and their canonical tree before opening any code.""" + resolved = path.resolve(strict=True) + for candidate in dict.fromkeys([*reversed(path.parents), path, + *reversed(resolved.parents), resolved]): + info = candidate.lstat() + if info.st_uid != 0 or (not stat.S_ISLNK(info.st_mode) and info.st_mode & 0o022): + raise ValueError("unprotected installed build tool path: " + str(candidate)) + if candidate not in (path, resolved) and not (stat.S_ISDIR(info.st_mode) or stat.S_ISLNK(info.st_mode)): + raise ValueError("invalid build tool directory: " + str(candidate)) + return resolved + + +def _identity(fd: int) -> tuple[os.stat_result, str]: + info = os.fstat(fd) + if (not stat.S_ISREG(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022 + or not info.st_mode & 0o111 or info.st_size <= 0): + raise ValueError("build tool must be a protected executable regular file") + digest = hashlib.sha256() + os.lseek(fd, 0, os.SEEK_SET) + while chunk := os.read(fd, 1024 * 1024): + digest.update(chunk) + return info, digest.hexdigest() + + +def _coordinates(info: os.stat_result) -> tuple: + return (info.st_dev, info.st_ino, info.st_mode, info.st_uid, info.st_gid, + info.st_size, info.st_mtime_ns, info.st_ctime_ns) + + +@contextmanager +def protected_tools(expected: dict | None = None): + """Keep verified descriptors alive and fence fixed paths around execution.""" + handles = {} + identities = {} + try: + # All roles must pass byte admission BEFORE either --version executes. + for role, value in PATHS.items(): + path = Path(value) + resolved = _protected_path(path) + fd = os.open(resolved, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC | os.O_NONBLOCK) + handles[role] = fd + info, digest = _identity(fd) + if _coordinates(info) != _coordinates(resolved.stat()) or _protected_path(path) != resolved: + raise ValueError("build tool changed during admission: " + role) + item = {"path": value, "resolved_path": str(resolved), "sha256": digest, + "size": info.st_size, "executable": True} + if expected is not None and any(item[key] != expected[role][key] for key in item): + raise ValueError("changed installed producer build tool identity: " + role) + identities[role] = (item, info) + + def fence(): + for role, (item, before) in identities.items(): + resolved = _protected_path(Path(item["path"])) + info, digest = _identity(handles[role]) + if (str(resolved) != item["resolved_path"] or _coordinates(info) != _coordinates(before) + or _coordinates(resolved.stat()) != _coordinates(before) or digest != item["sha256"]): + raise ValueError("installed build tool changed across execution: " + role) + + fence() + try: + yield {role: item for role, (item, _) in identities.items()}, fence + finally: + fence() + finally: + for fd in handles.values(): + os.close(fd) + + +def _versions(items: dict, fence, expected: dict | None = None) -> None: + for role, item in items.items(): + fence() + # Preserve complete stdout, including its final newline, in the receipt. + try: + version = subprocess.check_output([item["path"], "--version"]).decode("utf-8") + finally: + fence() + if expected is not None and version != expected[role]["version"]: + raise ValueError("changed installed producer build tool version: " + role) + item["version"] = version + + +def record() -> dict: + with protected_tools() as (items, fence): + _versions(items, fence) + receipt = {"schema_version": 1, "tools": items} + validate({"build_tools": receipt, "tools": {role: item["version"].splitlines()[0] + for role, item in items.items()}}) + return receipt + + +def verify(installed: dict, *, role: str | None = None, arguments: list[str] | None = None) -> None: + receipt = validate(installed) + with protected_tools(receipt["tools"]) as (items, fence): + _versions(items, fence, receipt["tools"]) + if role is not None: + if role not in PATHS: + raise ValueError("unsupported portable build tool role") + fence() + subprocess.run([PATHS[role], *(arguments or [])], check=True) + fence() + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--verify", required=True, type=Path, help="producer installed.json") + parser.add_argument("--run", choices=PATHS) + parser.add_argument("arguments", nargs=argparse.REMAINDER) + args = parser.parse_args() + arguments = args.arguments + if arguments[:1] == ["--"]: + arguments = arguments[1:] + if arguments and args.run is None: + parser.error("arguments require --run") + try: + verify(load_installed(args.verify), role=args.run, arguments=arguments) + except (ValueError, OSError, subprocess.CalledProcessError) as exc: + parser.exit(1, f"portable build tool verification failed: {exc}\n") + + +def load_installed(path: Path) -> dict: + """Metadata cannot be supplied by an unprivileged consumer either.""" + resolved = _protected_path(path) + fd = os.open(resolved, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC | os.O_NONBLOCK) + try: + before = os.fstat(fd) + if (not stat.S_ISREG(before.st_mode) or before.st_uid != 0 or before.st_mode & 0o022 + or not 0 < before.st_size <= 1024 * 1024): + raise ValueError("installed metadata must be a protected regular file") + with os.fdopen(os.dup(fd), "rb") as source: + payload = source.read(1024 * 1024 + 1) + if (_protected_path(path) != resolved or _coordinates(os.fstat(fd)) != _coordinates(before) + or _coordinates(resolved.stat()) != _coordinates(before) or len(payload) != before.st_size): + raise ValueError("installed metadata changed during admission") + + def unique(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError("duplicate installed metadata field: " + key) + result[key] = value + return result + + return json.loads(payload, object_pairs_hook=unique) + finally: + os.close(fd) + + +if __name__ == "__main__": + main() diff --git a/portable/consumer.sh b/portable/consumer.sh index e3de735..b283621 100755 --- a/portable/consumer.sh +++ b/portable/consumer.sh @@ -20,11 +20,13 @@ while read -r expected_hash input; do done < "${root}/shader-inputs.txt" "${root}/smoke.sh" python3 "${root}/compiler_evidence.py" --verify "${root}/compiler-evidence.json" +python3 "${root}/build_tools.py" --verify "${root}/installed.json" work=$(mktemp -d) trap 'rm -rf -- "${work}"' EXIT # Resolve pinned providers through the baseline loader cache; omit transient # CMake build-tree RPATH padding, whose empty entries mean the current directory. -/usr/bin/cmake -S "${source}" -B "${work}/build" -G Ninja \ +python3 "${root}/build_tools.py" --verify "${root}/installed.json" --run cmake -- \ + -S "${source}" -B "${work}/build" -G Ninja \ -DCMAKE_MAKE_PROGRAM=/usr/bin/ninja -DPKG_CONFIG_EXECUTABLE=/usr/bin/pkg-config \ -DCMAKE_AR=/opt/llvm-23.1.3/bin/llvm-ar -DCMAKE_RANLIB=/opt/llvm-23.1.3/bin/llvm-ranlib \ -DCMAKE_BUILD_TYPE=Release -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ @@ -37,7 +39,8 @@ trap 'rm -rf -- "${work}"' EXIT '-DCMAKE_C_FLAGS=-O2 -march=x86-64 -mtune=generic' \ '-DCMAKE_CXX_FLAGS=-O2 -march=x86-64 -mtune=generic' python3 "${root}/compiler_evidence.py" --verify "${root}/compiler-evidence.json" -/usr/bin/cmake --build "${work}/build" --parallel 2 +python3 "${root}/build_tools.py" --verify "${root}/installed.json" --run cmake -- \ + --build "${work}/build" --parallel 2 ctest --test-dir "${work}/build" --output-on-failure --timeout 120 python3 - "${work}/build/compile_commands.json" <<'CHECK' import json, shlex, sys @@ -56,4 +59,5 @@ cp "${work}/build/compile_commands.json" "${reports}/compile_commands.json" cp "${work}/build/Testing/Temporary/LastTest.log" "${reports}/consumer-tests.log" printf '%s\n' "${expected}" > "${reports}/classic-commit.txt" python3 "${root}/compiler_evidence.py" --verify "${root}/compiler-evidence.json" +python3 "${root}/build_tools.py" --verify "${root}/installed.json" cp "${root}/installed.json" "${root}/contract.json" "${root}/compiler-evidence.json" "${reports}/" diff --git a/portable/inventory.py b/portable/inventory.py index f929a92..6709686 100644 --- a/portable/inventory.py +++ b/portable/inventory.py @@ -7,6 +7,7 @@ from compiler_evidence import record from curl_evidence import record as record_curl +from build_tools import record as record_build_tools root = Path(__file__).resolve().parent contract = json.loads((root / "contract.json").read_text()) @@ -25,13 +26,15 @@ def output(*args): evidence = record() evidence_path = root / "compiler-evidence.json" evidence_path.write_text(json.dumps(evidence, indent=2) + "\n") +build_tools = record_build_tools() actual = {"schema_version": 2, "contract_sha256": hashlib.sha256((root / "contract.json").read_bytes()).hexdigest(), "architecture": output("dpkg", "--print-architecture"), "glibc": output("getconf", "GNU_LIBC_VERSION"), "tools": {name: output(*args).splitlines()[0] for name, args in { "cc": ["clang", "--version"], "linker": ["ld.lld", "--version"], - "readelf": ["llvm-readelf", "--version"], "cmake": ["cmake", "--version"], - "ninja": ["ninja", "--version"], "python": ["python3", "--version"], - "git-lfs": ["git", "lfs", "version"], "openssl": ["openssl", "version"]}.items()}, + "readelf": ["llvm-readelf", "--version"], "python": ["python3", "--version"], + "git-lfs": ["git", "lfs", "version"], "openssl": ["openssl", "version"]}.items() + } | {role: item["version"].splitlines()[0] for role, item in build_tools["tools"].items()}, + "build_tools": build_tools, "compiler_target": evidence["target"], "compiler_evidence_sha256": hashlib.sha256(evidence_path.read_bytes()).hexdigest(), "pkg_config": {name: output("pkg-config", "--modversion", name) for name in contract["pkg_config"]}, diff --git a/portable/test_build_tools.py b/portable/test_build_tools.py new file mode 100644 index 0000000..7e9f50a --- /dev/null +++ b/portable/test_build_tools.py @@ -0,0 +1,259 @@ +"""Producer receipts reject replaced code before probing or configuring it.""" +import copy +import hashlib +import importlib.util +import json +import os +from pathlib import Path +import stat +import tempfile +import unittest +from unittest import mock + +spec = importlib.util.spec_from_file_location("build_tools", Path(__file__).with_name("build_tools.py")) +tools = importlib.util.module_from_spec(spec) +spec.loader.exec_module(tools) + + +def installed_fixture(): + versions = {"cmake": "cmake version 3.25.1\n\nCMake suite maintained and supported by Kitware (kitware.com/cmake).\n", + "ninja": "1.11.1\n"} + return {"tools": {role: version.splitlines()[0] for role, version in versions.items()}, + "build_tools": {"schema_version": 1, "tools": { + role: {"path": path, "resolved_path": path, + "sha256": hashlib.sha256(role.encode()).hexdigest(), "size": len(role), + "executable": True, "version": versions[role]} + for role, path in tools.PATHS.items()}}} + + +class SchemaTests(unittest.TestCase): + def test_complete_receipt(self): + tools.validate(installed_fixture()) + + def test_missing_receipt_is_actionable(self): + with self.assertRaisesRegex(ValueError, "rebuild the portable SDK"): + tools.validate({"tools": {"cmake": "cmake version 3.25.1", "ninja": "1.11.1"}}) + + def test_strict_fields_roles_and_types(self): + for value in (None, [], True): + with self.subTest(value=value), self.assertRaises(ValueError): + tools.validate(value) + mutations = [ + lambda v: v["build_tools"].update(extra=True), + lambda v: v["build_tools"].update(schema_version=True), + lambda v: v["build_tools"].update(schema_version=2), + lambda v: v["build_tools"].update(tools=[]), + lambda v: v["build_tools"]["tools"].pop("ninja"), + lambda v: v["build_tools"]["tools"].update(ctest={}), + lambda v: v.update(tools=[]), + ] + for mutate in mutations: + value = installed_fixture() + mutate(value) + with self.subTest(mutate=mutate), self.assertRaises(ValueError): + tools.validate(value) + changes = {"path": ["cmake", "/usr/local/bin/cmake", "/usr/bin/../bin/cmake"], + "resolved_path": ["relative", "/usr/bin/../bin/cmake", "//usr/bin/cmake", "/", None], + "sha256": ["x" * 64, "A" * 64, None], "size": [True, "5", 0, -1], + "executable": [False, 1, "true"], "version": ["", None, "wrong\n", "cmake version 3.25.1\r\n"]} + for field, values in changes.items(): + for replacement in values: + value = installed_fixture() + value["build_tools"]["tools"]["cmake"][field] = replacement + with self.subTest(field=field, replacement=replacement), self.assertRaises(ValueError): + tools.validate(value) + for role in tools.PATHS: + value = installed_fixture() + value["build_tools"]["tools"][role]["extra"] = True + with self.subTest(role=role), self.assertRaises(ValueError): + tools.validate(value) + del value["build_tools"]["tools"][role]["extra"] + value["tools"][role] += " changed" + with self.subTest(role=role), self.assertRaises(ValueError): + tools.validate(value) + + def test_every_required_entry_field(self): + for field in tools.FIELDS: + value = installed_fixture() + del value["build_tools"]["tools"]["ninja"][field] + with self.subTest(field=field), self.assertRaises(ValueError): + tools.validate(value) + + +class ExecutionTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory(prefix="build-tools-test-") + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.log = self.root / "execution.log" + self.paths = {} + for role, version in (("cmake", "cmake version 3.25.1\n\ncomplete banner\n"), ("ninja", "1.11.1\n")): + path = self.root / role + path.write_text("#!/usr/bin/python3\nimport sys\nfrom pathlib import Path\n" + f"with Path({str(self.log)!r}).open('a') as log: log.write({role!r} + ' ' + repr(sys.argv[1:]) + '\\n')\n" + f"if sys.argv[1:] == ['--version']: sys.stdout.write({version!r})\n") + path.chmod(0o755) + self.paths[role] = str(path) + # A test-owned tree intentionally has a consumer UID and writable /tmp + # ancestors. Bypass only OS ownership policy, retaining actual fd hashes, + # resolution, stats, subprocesses and all admission/fencing behavior. + self.addCleanup(mock.patch.stopall) + mock.patch.object(tools, "PATHS", self.paths).start() + mock.patch.object(tools, "_protected_path", side_effect=lambda p: p.resolve(strict=True)).start() + mock.patch.object(tools, "_identity", side_effect=self.fixture_identity).start() + self.receipt = tools.record() + self.installed = {"build_tools": self.receipt, + "tools": {role: item["version"].splitlines()[0] + for role, item in self.receipt["tools"].items()}} + self.log.unlink() + + @staticmethod + def fixture_identity(fd): + info = os.fstat(fd) + os.lseek(fd, 0, os.SEEK_SET) + digest = hashlib.sha256() + while chunk := os.read(fd, 1024 * 1024): + digest.update(chunk) + return info, digest.hexdigest() + + def test_record_actual_bytes_and_full_stdout_and_run_fixed_tool(self): + for role, path in self.paths.items(): + item = self.receipt["tools"][role] + self.assertEqual(item["sha256"], hashlib.sha256(Path(path).read_bytes()).hexdigest()) + self.assertEqual(item["size"], Path(path).stat().st_size) + self.assertEqual(item["resolved_path"], str(Path(path).resolve())) + self.assertTrue(item["version"].endswith("\n")) + tools.verify(self.installed, role="cmake", arguments=["-S", "fixture", "-G", "Ninja"]) + self.assertEqual(self.log.read_text().splitlines(), ["cmake ['--version']", "ninja ['--version']", + "cmake ['-S', 'fixture', '-G', 'Ninja']"]) + + def test_changed_tool_or_size_rejected_before_any_probe(self): + for role in self.paths: + for field, change in (("sha256", "0" * 64), ("size", 1234), ("resolved_path", "/different")): + value = copy.deepcopy(self.installed) + value["build_tools"]["tools"][role][field] = change + with self.subTest(role=role, field=field), self.assertRaisesRegex(ValueError, "changed installed"): + tools.verify(value, role="cmake", arguments=["--configure"]) + self.assertFalse(self.log.exists()) + + def test_replaced_bytes_fail_before_version_or_configure(self): + path = Path(self.paths["ninja"]) + path.write_bytes(path.read_bytes() + b"# changed bytes\n") + with self.assertRaisesRegex(ValueError, "changed installed"): + tools.verify(self.installed, role="cmake", arguments=["--configure"]) + self.assertFalse(self.log.exists()) + + def test_full_stdout_mismatch_fails_before_configure(self): + value = copy.deepcopy(self.installed) + value["build_tools"]["tools"]["cmake"]["version"] = "cmake version 3.25.1\n\nchanged later line\n" + with self.assertRaisesRegex(ValueError, "changed installed producer build tool version"): + tools.verify(value, role="cmake", arguments=["--configure"]) + self.assertEqual(self.log.read_text().splitlines(), ["cmake ['--version']"]) + + def test_inode_replacement_during_probe_is_fenced(self): + original = tools.subprocess.check_output + + def replace_after_probe(*args, **kwargs): + version = original(*args, **kwargs) + path = Path(self.paths["cmake"]) + replacement = path.with_suffix(".replacement") + replacement.write_bytes(path.read_bytes()) + replacement.chmod(0o755) + replacement.replace(path) + return version + + with mock.patch.object(tools.subprocess, "check_output", side_effect=replace_after_probe), \ + self.assertRaisesRegex(ValueError, "changed across execution"): + tools.verify(self.installed, role="cmake", arguments=["--configure"]) + self.assertEqual(self.log.read_text().splitlines(), ["cmake ['--version']"]) + + def test_mutation_during_configure_is_fenced(self): + original = tools.subprocess.run + + def mutate_after_configure(*args, **kwargs): + result = original(*args, **kwargs) + if args[0][1:] == ["--configure"]: + path = Path(self.paths["ninja"]) + path.write_bytes(path.read_bytes() + b"# changed during configure\n") + return result + + with mock.patch.object(tools.subprocess, "run", side_effect=mutate_after_configure), \ + self.assertRaisesRegex(ValueError, "changed across execution"): + tools.verify(self.installed, role="cmake", arguments=["--configure"]) + + +class ProtectionTests(unittest.TestCase): + def test_unprivileged_or_writable_files_fail_before_hashing(self): + with tempfile.TemporaryFile() as file: + original = os.fstat(file.fileno()) + for mode, uid in ((stat.S_IFREG | 0o755, 1000), (stat.S_IFREG | 0o775, 0), + (stat.S_IFREG | 0o644, 0), (stat.S_IFIFO | 0o755, 0)): + values = list(original) + values[0], values[4], values[6] = mode, uid, 1 + with self.subTest(mode=mode, uid=uid), mock.patch.object(tools.os, "fstat", return_value=os.stat_result(values)), \ + mock.patch.object(tools.os, "read") as read, self.assertRaises(ValueError): + tools._identity(file.fileno()) + read.assert_not_called() + + def test_consumer_owned_tree_is_not_protected(self): + with tempfile.TemporaryDirectory() as temp: + path = Path(temp) / "executable" + path.write_text("fixture") + path.chmod(0o755) + with self.assertRaisesRegex(ValueError, "unprotected"): + tools._protected_path(path) + + def test_consumer_uses_receipt_runner_for_configure_and_build(self): + source = Path(__file__).with_name("consumer.sh").read_text() + self.assertEqual(source.count('--run cmake --'), 2) + self.assertNotIn('/usr/bin/cmake ', source) + self.assertIn('-DCMAKE_MAKE_PROGRAM=/usr/bin/ninja', source) + inventory = Path(__file__).with_name("inventory.py").read_text() + self.assertIn('"build_tools": build_tools', inventory) + self.assertNotIn('["cmake", "--version"]', inventory) + self.assertNotIn('["ninja", "--version"]', inventory) + + +class MetadataTests(unittest.TestCase): + def test_protected_loader_rejects_consumer_metadata(self): + with tempfile.TemporaryDirectory() as temp: + path = Path(temp) / "installed.json" + path.write_text(json.dumps(installed_fixture())) + with self.assertRaisesRegex(ValueError, "unprotected"): + tools.load_installed(path) + + def test_metadata_duplicates_and_invalid_types_are_rejected(self): + with tempfile.TemporaryDirectory() as temp: + path = Path(temp) / "installed.json" + original_fstat, original_stat = os.fstat, Path.stat + + def root_info(info): + values = list(info) + values[4] = 0 + return os.stat_result(values) + + with mock.patch.object(tools, "_protected_path", side_effect=lambda p: p), \ + mock.patch.object(tools.os, "fstat", side_effect=lambda fd: root_info(original_fstat(fd))), \ + mock.patch.object(tools.Path, "stat", side_effect=lambda p: root_info(original_stat(p)), autospec=True): + path.write_text(json.dumps(installed_fixture())) + self.assertEqual(tools.load_installed(path), installed_fixture()) + for payload in ('{"build_tools": {}, "build_tools": {}}', + '{"build_tools": {"tools": {"cmake": {"path": "a", "path": "b"}}}}'): + path.write_text(payload) + with self.subTest(payload=payload), self.assertRaisesRegex(ValueError, "duplicate"): + tools.load_installed(path) + path.write_text("[]") + with self.assertRaisesRegex(ValueError, "metadata object"): + tools.verify(tools.load_installed(path)) + + def test_cli_preserves_configure_argument_boundary(self): + with mock.patch.object(tools, "load_installed", return_value=installed_fixture()), \ + mock.patch.object(tools, "verify") as verify, \ + mock.patch("sys.argv", ["build_tools.py", "--verify", "/installed.json", "--run", "cmake", + "--", "-S", "/source with spaces", "-G", "Ninja"]): + tools.main() + self.assertEqual(verify.call_args.kwargs, {"role": "cmake", "arguments": ["-S", "/source with spaces", "-G", "Ninja"]}) + + +if __name__ == "__main__": + unittest.main() From 96eafd21a020d5bd0c30910122af6fd852bc5cb8 Mon Sep 17 00:00:00 2001 From: Zoey Rose Date: Sun, 11 Oct 2026 20:16:02 -0500 Subject: [PATCH 02/23] fix(portable): reject symlink hops in protected build tool paths --- AGENTS.md | 1 + README.md | 2 + portable/build_tools.py | 40 ++++++++++---- portable/test_build_tools.py | 103 +++++++++++++++++++++++++++++++++-- 4 files changed, 131 insertions(+), 15 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index e8d029e..485e4ed 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -28,6 +28,7 @@ receipt binds fixed `/usr/bin/cmake` and `/usr/bin/ninja` paths, canonical executables, hashes, sizes and full version stdout. Verify both protected identities before probing either tool and fence configure/build execution; + reject symlinks in these fixed Debian executable and metadata paths. missing receipts require a rebuilt producer. The producer OS, libraries and CMake modules and earlier library-production builds remain trusted; this receipt does not attest historical execution. Portable curl is an LLVM rebuild of diff --git a/README.md b/README.md index d6965ff..d4b6e8b 100644 --- a/README.md +++ b/README.md @@ -470,6 +470,8 @@ executable status and complete version stdout. Its first lines agree with the existing installed tool version strings. The consumer verifies both protected root-owned executables through open descriptors before any version probe and fences their resolution, bytes and file identity around configure and build. +Every directory component must be protected, and these fixed Debian executable +and metadata paths must not contain symlinks. SDKs without this producer receipt require a rebuild. This check trusts the producer OS, dynamic libraries and CMake modules; it does not attest the full transitive execution environment. diff --git a/portable/build_tools.py b/portable/build_tools.py index 82aa24a..2194ef9 100644 --- a/portable/build_tools.py +++ b/portable/build_tools.py @@ -20,6 +20,7 @@ PATHS = {"cmake": "/usr/bin/cmake", "ninja": "/usr/bin/ninja"} FIELDS = {"path", "resolved_path", "sha256", "size", "executable", "version"} +MAX_BINARY_SIZE = 512 * 1024 * 1024 def validate(installed: dict) -> dict: @@ -42,15 +43,15 @@ def validate(installed: dict) -> dict: resolved = item["resolved_path"] if (type(resolved) is not str or not resolved.startswith("/") or resolved.startswith("//") or "\x00" in resolved - or posixpath.normpath(resolved) != resolved or resolved == "/"): + or posixpath.normpath(resolved) != resolved or resolved != path): raise ValueError("invalid canonical build tool path: " + role) if (type(item["sha256"]) is not str or not re.fullmatch(r"[0-9a-f]{64}", item["sha256"]) - or type(item["size"]) is not int or item["size"] <= 0 + or type(item["size"]) is not int or not 0 < item["size"] <= MAX_BINARY_SIZE or item["executable"] is not True): raise ValueError("invalid producer build tool identity: " + role) version = item["version"] if (type(version) is not str or not version or len(version) > 65536 - or "\x00" in version or "\r" in version): + or any((ord(char) < 32 and char != "\n") or ord(char) == 127 for char in version)): raise ValueError("invalid complete build tool version: " + role) first = version.splitlines()[0] pattern = r"cmake version [0-9]+\.[0-9]+\.[0-9]+" if role == "cmake" else r"[0-9]+\.[0-9]+(?:\.[0-9]+)?" @@ -60,22 +61,39 @@ def validate(installed: dict) -> dict: def _protected_path(path: Path) -> Path: - """Check lexical links and their canonical tree before opening any code.""" - resolved = path.resolve(strict=True) - for candidate in dict.fromkeys([*reversed(path.parents), path, - *reversed(resolved.parents), resolved]): + """Check every component; fixed Debian tool/metadata paths cannot use links.""" + resolved = Path("/") + parts = list(path.parts[1:]) if path.is_absolute() else [*Path.cwd().parts[1:], *path.parts] + + def check(candidate): info = candidate.lstat() - if info.st_uid != 0 or (not stat.S_ISLNK(info.st_mode) and info.st_mode & 0o022): + if stat.S_ISLNK(info.st_mode): + raise ValueError("symlink in installed build tool path: " + str(candidate)) + if info.st_uid != 0 or info.st_mode & 0o022: raise ValueError("unprotected installed build tool path: " + str(candidate)) - if candidate not in (path, resolved) and not (stat.S_ISDIR(info.st_mode) or stat.S_ISLNK(info.st_mode)): + return info + + check(resolved) + while parts: + component = parts.pop(0) + if component in ("", "."): + continue + if component == "..": + resolved = resolved.parent + check(resolved) + continue + candidate = resolved / component + info = check(candidate) + if parts and not stat.S_ISDIR(info.st_mode): raise ValueError("invalid build tool directory: " + str(candidate)) + resolved = candidate return resolved def _identity(fd: int) -> tuple[os.stat_result, str]: info = os.fstat(fd) - if (not stat.S_ISREG(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022 - or not info.st_mode & 0o111 or info.st_size <= 0): + if (not stat.S_ISREG(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o6022 + or not info.st_mode & 0o111 or not 0 < info.st_size <= MAX_BINARY_SIZE): raise ValueError("build tool must be a protected executable regular file") digest = hashlib.sha256() os.lseek(fd, 0, os.SEEK_SET) diff --git a/portable/test_build_tools.py b/portable/test_build_tools.py index 7e9f50a..3a03dc4 100644 --- a/portable/test_build_tools.py +++ b/portable/test_build_tools.py @@ -54,8 +54,9 @@ def test_strict_fields_roles_and_types(self): tools.validate(value) changes = {"path": ["cmake", "/usr/local/bin/cmake", "/usr/bin/../bin/cmake"], "resolved_path": ["relative", "/usr/bin/../bin/cmake", "//usr/bin/cmake", "/", None], - "sha256": ["x" * 64, "A" * 64, None], "size": [True, "5", 0, -1], - "executable": [False, 1, "true"], "version": ["", None, "wrong\n", "cmake version 3.25.1\r\n"]} + "sha256": ["x" * 64, "A" * 64, None], "size": [True, "5", 0, -1, tools.MAX_BINARY_SIZE + 1], + "executable": [False, 1, "true"], "version": ["", None, "wrong\n", "cmake version 3.25.1\r\n", + "cmake version 3.25.1\n\tcontrol\n", "cmake version 3.25.1\n\x7f"]} for field, values in changes.items(): for replacement in values: value = installed_fixture() @@ -132,7 +133,8 @@ def test_changed_tool_or_size_rejected_before_any_probe(self): for field, change in (("sha256", "0" * 64), ("size", 1234), ("resolved_path", "/different")): value = copy.deepcopy(self.installed) value["build_tools"]["tools"][role][field] = change - with self.subTest(role=role, field=field), self.assertRaisesRegex(ValueError, "changed installed"): + message = "invalid canonical" if field == "resolved_path" else "changed installed" + with self.subTest(role=role, field=field), self.assertRaisesRegex(ValueError, message): tools.verify(value, role="cmake", arguments=["--configure"]) self.assertFalse(self.log.exists()) @@ -181,13 +183,28 @@ def mutate_after_configure(*args, **kwargs): self.assertRaisesRegex(ValueError, "changed across execution"): tools.verify(self.installed, role="cmake", arguments=["--configure"]) + def test_failed_configure_still_fences_mutation(self): + original = tools.subprocess.run + + def fail_configure(*args, **kwargs): + if args[0][1:] == ["--configure"]: + path = Path(self.paths["ninja"]) + path.write_bytes(path.read_bytes() + b"# mutated failed configure\n") + raise tools.subprocess.CalledProcessError(1, args[0]) + return original(*args, **kwargs) + + with mock.patch.object(tools.subprocess, "run", side_effect=fail_configure), \ + self.assertRaisesRegex(ValueError, "changed across execution"): + tools.verify(self.installed, role="cmake", arguments=["--configure"]) + class ProtectionTests(unittest.TestCase): def test_unprivileged_or_writable_files_fail_before_hashing(self): with tempfile.TemporaryFile() as file: original = os.fstat(file.fileno()) for mode, uid in ((stat.S_IFREG | 0o755, 1000), (stat.S_IFREG | 0o775, 0), - (stat.S_IFREG | 0o644, 0), (stat.S_IFIFO | 0o755, 0)): + (stat.S_IFREG | 0o644, 0), (stat.S_IFIFO | 0o755, 0), + (stat.S_IFREG | 0o4755, 0), (stat.S_IFREG | 0o2755, 0)): values = list(original) values[0], values[4], values[6] = mode, uid, 1 with self.subTest(mode=mode, uid=uid), mock.patch.object(tools.os, "fstat", return_value=os.stat_result(values)), \ @@ -195,6 +212,84 @@ def test_unprivileged_or_writable_files_fail_before_hashing(self): tools._identity(file.fileno()) read.assert_not_called() + def test_protected_regular_executable_positive(self): + with tempfile.TemporaryDirectory() as temp: + path = Path(temp) / "regular" + path.write_bytes(b"test-owned executable bytes") + path.chmod(0o555) + original_lstat, original_fstat = Path.lstat, os.fstat + + def protected_info(info): + values = list(info) + values[0], values[4] = info.st_mode & ~0o022, 0 + return os.stat_result(values) + + # Root ownership is modeled in a test-owned tree. Directory/file + # traversal, opening, regular mode, size and bytes remain real. + with mock.patch.object(tools.Path, "lstat", side_effect=lambda p: protected_info(original_lstat(p)), autospec=True), \ + mock.patch.object(tools.os, "fstat", side_effect=lambda fd: protected_info(original_fstat(fd))): + resolved = tools._protected_path(path) + self.assertEqual(resolved, path.resolve(strict=True)) + fd = os.open(resolved, os.O_RDONLY | os.O_NOFOLLOW) + try: + info, digest = tools._identity(fd) + self.assertEqual(info.st_size, len(resolved.read_bytes())) + self.assertEqual(digest, hashlib.sha256(resolved.read_bytes()).hexdigest()) + finally: + os.close(fd) + + def test_intermediate_symlink_expansion_cannot_skip_writable_directory(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + trusted, untrusted = root / "trusted", root / "untrusted" + trusted.mkdir() + untrusted.mkdir() + untrusted.chmod(0o777) + real = trusted / "real" + real.write_text("fixture") + real.chmod(0o755) + (trusted / "cmake").symlink_to(untrusted / "hop") + (untrusted / "hop").symlink_to(real) + original = Path.lstat + seen = [] + + def trusted_ancestor_info(path): + info = original(path) + seen.append(path) + # Model root-owned protected fixture ancestors only. Keep the + # actual writable intermediate directory's ownership/mode. + if path == untrusted: + return info + values = list(info) + values[0], values[4] = info.st_mode & ~0o022, 0 + return os.stat_result(values) + + with mock.patch.object(tools.Path, "lstat", side_effect=trusted_ancestor_info, autospec=True): + self.assertEqual(tools._protected_path(real), real) + seen.clear() + with self.assertRaisesRegex(ValueError, "symlink in installed build tool path"): + tools._protected_path(trusted / "cmake") + self.assertIn(trusted / "cmake", seen) + self.assertNotIn(real, seen) + + def test_parent_symlink_cannot_redirect_a_regular_tool_or_metadata(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + (root / "real").mkdir() + (root / "real/tool").write_bytes(b"fixture") + (root / "parent").symlink_to(root / "real", target_is_directory=True) + original_lstat = Path.lstat + + def protected_info(path): + info = original_lstat(path) + values = list(info) + values[0], values[4] = info.st_mode & ~0o022, 0 + return os.stat_result(values) + + with mock.patch.object(tools.Path, "lstat", side_effect=protected_info, autospec=True), \ + self.assertRaisesRegex(ValueError, "symlink in installed build tool path"): + tools._protected_path(root / "parent/tool") + def test_consumer_owned_tree_is_not_protected(self): with tempfile.TemporaryDirectory() as temp: path = Path(temp) / "executable" From 4079934a0dfab0adf167c1abea8c9f4b6df4bafa Mon Sep 17 00:00:00 2001 From: Zoey Rose Date: Sun, 11 Oct 2026 20:10:41 -0500 Subject: [PATCH 03/23] build(portable): add bounded installed inventory foundation --- tools/RELEASED_SDK_INVENTORY.md | 72 ++++ tools/collect-released-sdk-metadata.py | 2 +- tools/released_sdk_inventory.py | 456 +++++++++++++++++++++ tools/tests/test_released_sdk_inventory.py | 345 ++++++++++++++++ 4 files changed, 874 insertions(+), 1 deletion(-) create mode 100644 tools/RELEASED_SDK_INVENTORY.md create mode 100644 tools/released_sdk_inventory.py create mode 100644 tools/tests/test_released_sdk_inventory.py diff --git a/tools/RELEASED_SDK_INVENTORY.md b/tools/RELEASED_SDK_INVENTORY.md new file mode 100644 index 0000000..50109bb --- /dev/null +++ b/tools/RELEASED_SDK_INVENTORY.md @@ -0,0 +1,72 @@ +# Released portable SDK inventory foundation + +`released_sdk_inventory.py` is an internal acquisition helper for the existing +collector's authenticated, task-owned `StoppedImage` and archive protocol. It +has no CLI, credentials, registry operations, release pins or image execution. +It is not yet imported or invoked by `collect-released-sdk-metadata.py`; the +three fixed 1.18.0 targets and their producer mapping remain authoritative. + +The helper streams `/opt/atrinik-portable` without extraction or retaining file +payloads. It requires a root directory entry, parents before children, unique +canonical relative paths, regular files and safe directory/file modes. It +rejects links, special/sparse files, path/size PAX overrides, malformed headers, +truncated members, missing end markers and nonzero trailing data. Timestamp-only +PAX records are accepted. Bounds are 20,000 members, 1,024-byte paths, 1 GiB per +file, 8 GiB aggregate payload and a shared 900-second deadline; single-file +reads use the same deadline and aggregate accounting. The existing collector's +operation alarm and socket timeout remain required outer bounds. + +Each producer record is `relative_path -> {sha256, size, executable}`, including +zero-byte files. Canonical sorted compact JSON uses the wrapper's +`linux_portable.inventory_digest` format. Common licenses use exactly the +wrapper's fourteen fixed names. Debian notices are selected only by matching +`runtime-sources.json.source_packages` to `debian-sources.json` package/source +rows. Their records retain the requested absolute path as key and add +`resolved_path`. Source archives named by the authenticated runtime and contract +documents must match the complete producer inventory, including custom provider +archives. + +Docker HEAD's `LinkTarget` is a fully resolved result, so it cannot establish a +safe link traversal by itself. The notice resolver reads each raw symbolic link +as a single tar member, resolves at most sixteen hops itself beneath +`/usr/share/doc`, rejects even temporary escape and cycles, and crosschecks the +result against HEAD. Parent (`..`) components are accepted only at the start +of a relative link target, before any potentially symbolic component. It +rechecks raw links, canonical regular files and every +observed parent/root before returning. This relies on Moby's +[`containerArchivePath`](https://github.com/moby/moby/blob/v28.5.1/daemon/archive_unix.go) +and [`containerFSView.Stat`](https://github.com/moby/moby/blob/v28.5.1/daemon/containerfs_linux.go) +semantics; final hosted qualification must exercise the actual daemon. The +producer-tree path accepts no links at all. Docker tar generation may omit entries +when traversal fails: complete tar framing proves only the delivered stream. +Before adopting new wrapper pins, final hosted qualification must independently +compare this producer map with the wrapper's `verified_tree_inventory` on that +exact image (or an authenticated complete OCI-overlay reconstruction). Source +archive hash checks alone do not establish that no unrelated file was omitted. + +The build-tool reader accepts exactly `cmake` and `ninja` at `/usr/bin/cmake` and +`/usr/bin/ninja`, both canonical regular executables. It validates +`installed.json.build_tools = {schema_version: 1, tools: {…}}`, the exact role and +field sets, full producer version strings and legacy first-line compatibility, +then compares streamed bytes, size and executable status. It never executes a +tool to obtain a version. LLVM compiler evidence retains its separate roles. + +Final wiring requires actual merged Classic consumer/shader commit M, final +producer P and released image D. A subsequent reviewed change must add fixed +index, linux/amd64 manifest and config digests plus per-target producer revision; +validate M/P and shader metadata against those fixed constants; bind each saved +metadata file to the producer inventory; emit all three complete maps and their +canonical digests; and invoke the helper inside the collector's existing +cleanup-protected acquisition. Do not add caller-supplied pins, paths or trust +extensions. Original broad/classic/preliminary identities must remain unchanged. +Wrong final M/D fixtures and actual hosted unstarted-image collection remain +required at that integration point. Cleanup failure must still invalidate the +receipt. Even successful acquisition retains `installed-identity-only`, +`image_code_executed: false` and `final_portable_cohort: false`; it does not prove +runtime, export, gameplay or audible behavior. + +Offline validation: + +```sh +python3 -m unittest discover -s tools/tests -p 'test*released*sdk*.py' -v +``` diff --git a/tools/collect-released-sdk-metadata.py b/tools/collect-released-sdk-metadata.py index dca8abd..06f95df 100644 --- a/tools/collect-released-sdk-metadata.py +++ b/tools/collect-released-sdk-metadata.py @@ -620,7 +620,7 @@ def single_file_archive(response, path, expected, *, retain, executable=False): stat_matches(expected, path) regular_mode(expected['mode'], executable) limit = METADATA_LIMIT if retain else BINARY_LIMIT - require(0 < expected['size'] <= limit, 'file-size-limit') + require(0 <= expected['size'] <= limit, 'file-size-limit') header = exact_read(response, 512) info = tarfile.TarInfo.frombuf(header, 'utf-8', 'strict') if info.type == tarfile.XHDTYPE: diff --git a/tools/released_sdk_inventory.py b/tools/released_sdk_inventory.py new file mode 100644 index 0000000..92fc811 --- /dev/null +++ b/tools/released_sdk_inventory.py @@ -0,0 +1,456 @@ +# Copyright (c) 2026 Atrinik contributors +# SPDX-License-Identifier: MIT +"""Internal fixed-boundary inventory reader; no CLI or acquisition authority. + +The caller must supply the collector's authenticated, created-state StoppedImage +and its archive protocol. This foundation is deliberately not wired to a release +target: final producer/consumer/index/platform/config pins are still required. +""" +import hashlib +import json +from pathlib import PurePosixPath +import re +import tarfile +import time +import urllib.parse + +ROOT = '/opt/atrinik-portable' +DOC = '/usr/share/doc' +COMMON = '/usr/share/common-licenses' +BUILD_TOOLS = {'cmake': '/usr/bin/cmake', 'ninja': '/usr/bin/ninja'} +COMMON_LICENSE_NAMES = ('Apache-2.0', 'Artistic', 'BSD', 'CC0-1.0', 'GFDL-1.2', + 'GFDL-1.3', 'GPL-1', 'GPL-2', 'GPL-3', 'LGPL-2', + 'LGPL-2.1', 'LGPL-3', 'MPL-1.1', 'MPL-2.0') +MAX_MEMBERS = 20000 +MAX_PATH = 1024 +MAX_FILE = 1024 ** 3 +MAX_TOTAL = 8 * 1024 ** 3 +MAX_SECONDS = 900 +MAX_HOPS = 16 +GO_DIRECTORY = 1 << 31 +GO_SYMLINK = 1 << 27 + + +class InventoryError(Exception): + """Only fixed failure codes; never hostile archive or daemon contents.""" + + +def require(ok, code): + if not ok: + raise InventoryError(code) + + +def inventory_digest(records): + return hashlib.sha256(json.dumps(records, sort_keys=True, + separators=(',', ':')).encode()).hexdigest() + + +def relative_path(value): + require(isinstance(value, str) and 0 < len(value.encode('utf-8')) <= MAX_PATH + and all(32 < ord(c) < 127 for c in value) and '\\' not in value + and all(part not in ('', '.', '..') for part in value.split('/')), + 'inventory-path-invalid') + return value + + +def under_doc(value): + require(isinstance(value, str) and value.startswith(DOC + '/'), 'notice-boundary') + relative_path(value[len(DOC) + 1:]) + return value + + +class Budget: + def __init__(self): + self.deadline = time.monotonic() + MAX_SECONDS + self.bytes = 0 + self.members = 0 + self.payload = 0 + + def check(self): + require(time.monotonic() < self.deadline, 'inventory-deadline') + + def read(self, stream, size): + self.check() + raw = stream.read(size) + self.check() + self.bytes += len(raw) + require(self.bytes <= MAX_TOTAL + MAX_MEMBERS * (1024 + 8192) + 10240, + 'inventory-stream-limit') + return raw + + def exact(self, stream, size): + result = bytearray() + while len(result) < size: + raw = self.read(stream, min(65536, size - len(result))) + require(bool(raw), 'inventory-truncated') + result.extend(raw) + return bytes(result) + + def member(self, size): + self.members += 1 + self.payload += size + require(self.members <= MAX_MEMBERS and 0 <= size <= MAX_FILE + and self.payload <= MAX_TOTAL, 'inventory-size-limit') + + +def framing(response): + require(response.chunked or response.length is not None, 'inventory-unframed') + require(response.getheader('Content-Encoding') in (None, 'identity') and + response.getheader('Content-Type', '').split(';')[0] == 'application/x-tar', + 'inventory-encoding') + + +def header(raw): + try: + info = tarfile.TarInfo.frombuf(raw, 'utf-8', 'strict') + # TarInfo strips every directory terminator, so reject repeated raw + # slashes before its normalization can hide a noncanonical spelling. + require(not (info.isdir() and raw[:100].split(b'\0', 1)[0].endswith(b'//')), + 'inventory-path-invalid') + return info + except (tarfile.TarError, ValueError, UnicodeError): + raise InventoryError('inventory-header-invalid') from None + + +def next_member(response, budget, protocol, *, symlink=False): + raw = budget.exact(response, 512) + if not any(raw): + require(not any(budget.exact(response, 512)), 'inventory-end-markers') + trailing = 0 + while True: + raw = budget.read(response, 512) + if not raw: + break + trailing += len(raw) + require(trailing <= 10240 and not any(raw), 'inventory-tail') + require(response.length in (None, 0), 'inventory-truncated') + require(trailing % 512 == 0, 'inventory-tail-alignment') + return None + info = header(raw) + if info.type == tarfile.XHDTYPE: + require(0 < info.size <= 8192 and not info.linkname, 'inventory-pax-invalid') + protocol.pax_times(budget.exact(response, info.size)) + require(not any(budget.exact(response, (-info.size) % 512)), 'inventory-padding') + info = header(budget.exact(response, 512)) + require(not info.pax_headers and not info.sparse and + (not info.linkname or (symlink and info.type == tarfile.SYMTYPE)), + 'inventory-extension-or-link') + require(info.uid >= 0 and info.gid >= 0, 'inventory-owner-invalid') + budget.member(info.size) + return info + + +def tree_archive(response, protocol, expected, budget=None): + """Hash a complete Docker archive of ROOT, retaining no file payloads.""" + budget = budget or Budget() + framing(response) + protocol.stat_matches(expected, ROOT, directory=True) + files, entries, directories = {}, set(), set() + root_name = PurePosixPath(ROOT).name + while True: + info = next_member(response, budget, protocol) + if info is None: + break + name = info.name + # header() already rejects repeated raw directory terminators. + # Absolute, dot, double slash and traversal names also fail here. + relative_path(name) + require(name == root_name or name.startswith(root_name + '/'), 'inventory-root-mismatch') + require(name not in entries, 'inventory-duplicate') + entries.add(name) + if name == root_name: + require(len(entries) == 1 and info.isdir() and info.mode == expected['mode'] & 0o777, + 'inventory-root-mismatch') + else: + require(str(PurePosixPath(name).parent) in directories, 'inventory-parent-missing') + if info.type == tarfile.DIRTYPE: + require(info.size == 0 and 0 <= info.mode <= 0o777 and + info.mode & 0o022 == 0 and info.mode & 0o555 == 0o555, + 'inventory-directory-mode') + directories.add(name) + continue + require(info.type in (tarfile.REGTYPE, tarfile.AREGTYPE) and name != root_name, + 'inventory-member-type') + protocol.regular_mode(info.mode) + sha = hashlib.sha256() + remaining = info.size + while remaining: + raw = budget.exact(response, min(65536, remaining)) + sha.update(raw) + remaining -= len(raw) + require(not any(budget.exact(response, (-info.size) % 512)), 'inventory-padding') + files[name[len(root_name) + 1:]] = {'sha256': sha.hexdigest(), 'size': info.size, + 'executable': bool(info.mode & 0o111)} + require(root_name in directories and bool(files), 'inventory-empty') + return files + + +def notice_requests(runtime, debian): + """Select only runtime source-package notices from authenticated documents.""" + sources = runtime.get('source_packages') + require(isinstance(sources, dict) and 0 < len(sources) <= MAX_MEMBERS + and isinstance(debian, list) and len(debian) <= MAX_MEMBERS, 'notice-documents-invalid') + require(all(isinstance(name, str) and re.fullmatch(r'[a-z0-9][a-z0-9+.-]*', name) + and isinstance(version, str) and 0 < len(version) <= 256 + for name, version in sources.items()), 'notice-source-invalid') + requests, found = set(), set() + for row in debian: + require(isinstance(row, dict), 'notice-row-invalid') + source = row.get('source') + require(isinstance(source, str), 'notice-row-invalid') + if source not in sources: + continue + package = row.get('package') + require(isinstance(package, str) and re.fullmatch(r'[a-z0-9][a-z0-9+.-]*(?::[a-z0-9]+)?', package) + and row.get('source_version') == sources[source], 'notice-package-invalid') + directory = DOC + '/' + package.split(':')[0] + require(row.get('notice_directory') == directory, 'notice-directory-invalid') + requests.add(directory + '/copyright') + found.add(source) + require(found == set(sources), 'notice-source-missing') + return sorted(requests) + + +def crosscheck_archives(records, runtime, contract): + archives, sources = runtime.get('archives'), contract.get('sources') + require(isinstance(archives, dict) and 0 < len(archives) <= MAX_MEMBERS + and isinstance(sources, list) and 0 < len(sources) <= MAX_MEMBERS, + 'inventory-source-documents') + required = {} + def add(path, sha): + relative_path(path) + require(isinstance(sha, str) and re.fullmatch(r'[0-9a-f]{64}', sha), 'inventory-source-digest') + require(path not in required or required[path] == sha, 'inventory-source-conflict') + required[path] = sha + for name, sha in archives.items(): + require(isinstance(name, str) and '/' not in name, 'inventory-source-name') + add('sources/debian/' + name, sha) + for source in sources: + require(isinstance(source, dict), 'inventory-source-row') + sha = source.get('sha256') + require(isinstance(sha, str), 'inventory-source-digest') + add('sources/' + sha + '.tar.gz', sha) + providers = runtime.get('custom_providers', {}) + require(isinstance(providers, dict) and len(providers) <= MAX_MEMBERS, 'inventory-custom-providers') + for provider in providers.values(): + require(isinstance(provider, dict) and isinstance(provider.get('source_archives'), dict) + and 0 < len(provider['source_archives']) <= MAX_MEMBERS, 'inventory-custom-provider') + for path, sha in provider['source_archives'].items(): + require(isinstance(path, str) and path.startswith('sources/'), 'inventory-source-name') + add(path, sha) + for path, sha in required.items(): + require(path in records and records[path]['sha256'] == sha, 'inventory-source-bytes-mismatch') + + +class BudgetResponse: + """Keep single-file protocol reads under the shared inventory deadline.""" + def __init__(self, response, budget): + self.response, self.budget = response, budget + + def __getattr__(self, name): + return getattr(self.response, name) + + def read(self, size): + return self.budget.read(self.response, size) + + +class InstalledInventory: + """Use only with an authenticated StoppedImage; does not grant release trust.""" + def __init__(self, image, protocol): + self.image, self.protocol = image, protocol + self.budget = Budget() + + def _fence(self): + self.budget.check() + self.image.fence() + self.budget.check() + + def _parents(self, path): + result = {} + for parent in reversed(PurePosixPath(path).parents): + if str(parent) == '/': + continue + self.budget.check() + value = self.image.stat(str(parent)) + self.protocol.stat_matches(value, str(parent), directory=True) + result[str(parent)] = value + return result + + def _recheck(self, observations): + for path, before in observations.items(): + self.budget.check() + require(self.image.stat(path) == before, 'inventory-path-drift') + self._fence() + + def _get(self, path): + return self.image.docker.request('GET', '/containers/' + self.image.identifier + '/archive?' + + urllib.parse.urlencode({'path': path}), + {'Accept-Encoding': 'identity'}) + + def producer(self): + self._fence() + observations = self._parents(ROOT) + expected = self.image.stat(ROOT) + self.protocol.stat_matches(expected, ROOT, directory=True) + observations[ROOT] = expected + conn, response = self._get(ROOT) + try: + require(self.protocol.path_stat(response) == expected, 'inventory-path-drift') + result = tree_archive(response, self.protocol, expected, self.budget) + finally: + conn.close() + self._recheck(observations) + return result + + def common_licenses(self): + return {name: self._regular(COMMON + '/' + name) for name in COMMON_LICENSE_NAMES} + + def _regular(self, path, *, executable=False): + # Internal only: callers are the fixed licenses and verified doc resolver. + require(path in {COMMON + '/' + name for name in COMMON_LICENSE_NAMES} + or path in BUILD_TOOLS.values() or path.startswith(DOC + '/'), + 'inventory-file-not-allowed') + if path.startswith(DOC + '/'): + under_doc(path) + self._fence() + observations = self._parents(path) + expected = self.image.stat(path) + self.protocol.stat_matches(expected, path) + self.protocol.regular_mode(expected['mode'], executable=executable) + observations[path] = expected + self.budget.member(expected['size']) + conn, response = self._get(path) + try: + require(self.protocol.path_stat(response) == expected, 'inventory-path-drift') + identity, _ = self.protocol.single_file_archive( + BudgetResponse(response, self.budget), path, expected, retain=False, executable=executable) + self.budget.check() + finally: + conn.close() + self._recheck(observations) + return {'sha256': identity['sha256'], 'size': identity['size'], + 'executable': bool(expected['mode'] & 0o111)} + + + def _alias(self, path, expected): + """Read the raw one-hop target; Docker HEAD's LinkTarget is fully resolved. + + Moby v28.5.1 daemon/archive_unix.go archives the named resource without + dereferencing it. Never use the daemon-resolved target as traversal input. + """ + conn, response = self._get(path) + try: + require(self.protocol.path_stat(response) == expected, 'notice-alias-drift') + framing(response) + info = next_member(response, self.budget, self.protocol, symlink=True) + require(info is not None and info.type == tarfile.SYMTYPE and info.size == 0 + and info.name == PurePosixPath(path).name + and info.mode == expected['mode'] & 0o777, + 'notice-alias-entry') + raw = info.linkname + require(isinstance(raw, str) and 0 < len(raw.encode('utf-8')) <= MAX_PATH + and len(raw.encode('utf-8')) == expected['size'] + and all(32 < ord(c) < 127 for c in raw) and '\\' not in raw, + 'notice-alias-target') + require(next_member(response, self.budget, self.protocol) is None, + 'notice-alias-extra-member') + return raw + finally: + conn.close() + + def _notice(self, requested): + under_doc(requested) + self._fence() + observations = self._parents(DOC + '/copyright') + links, visited = {}, set() + + def resolve(path): + under_doc(path) + parts = path[len(DOC) + 1:].split('/') + current = DOC + for index, part in enumerate(parts): + current += '/' + part + self.budget.check() + value = self.image.stat(current) + require(current not in observations or observations[current] == value, + 'notice-alias-drift') + observations[current] = value + mode = value.get('mode') + require(type(mode) is int, 'notice-mode-invalid') + if mode & ~0o777 == GO_SYMLINK: + require(value.get('name') == part and type(value.get('size')) is int + and 0 < value['size'] <= MAX_PATH, 'notice-alias-stat') + under_doc(value.get('linkTarget')) + require(current not in visited and len(visited) < MAX_HOPS, 'notice-alias-cycle-or-limit') + visited.add(current) + raw = self._alias(current, value) + links[current] = raw + # Interpret raw components ourselves. Even temporary escape + # followed by re-entry fails; HEAD's final target is no proof. + if raw.startswith('/'): + require(raw.startswith(DOC + '/'), 'notice-boundary') + resolved_parts = [] + target_parts = raw[len(DOC) + 1:].split('/') + else: + resolved_parts = current[len(DOC) + 1:].split('/')[:-1] + target_parts = raw.split('/') + descended = False + for component in target_parts: + require(component != '', 'notice-alias-target') + if component == '..': + # Only leading parent hops are safe to normalize: + # later components may themselves name symlinks. + require(not descended and bool(resolved_parts), 'notice-boundary') + resolved_parts.pop() + elif component != '.': + descended = True + resolved_parts.append(component) + require(bool(resolved_parts), 'notice-boundary') + canonical = resolve(DOC + '/' + '/'.join(resolved_parts)) + require(canonical == value['linkTarget'], 'notice-alias-drift') + if index + 1 < len(parts): + canonical = resolve(canonical + '/' + '/'.join(parts[index + 1:])) + return canonical + if index + 1 < len(parts) or mode & GO_DIRECTORY: + self.protocol.stat_matches(value, current, directory=True) + else: + self.protocol.stat_matches(value, current) + return current + + canonical = resolve(requested) + result = self._regular(canonical) + for path, raw in links.items(): + require(self._alias(path, observations[path]) == raw, 'notice-alias-drift') + self._recheck(observations) + return {**result, 'resolved_path': canonical} + + def debian_notices(self, runtime, debian): + return {requested: self._notice(requested) + for requested in notice_requests(runtime, debian)} + + + def build_tools(self, installed): + receipt = installed.get('build_tools') + require(isinstance(receipt, dict) and set(receipt) == {'schema_version', 'tools'} + and type(receipt['schema_version']) is int and receipt['schema_version'] == 1 + and isinstance(receipt['tools'], dict) and set(receipt['tools']) == set(BUILD_TOOLS), + 'build-tool-schema') + result = {} + for role, path in BUILD_TOOLS.items(): + entry = receipt['tools'][role] + require(isinstance(entry, dict) and set(entry) == { + 'path', 'resolved_path', 'sha256', 'size', 'executable', 'version'}, 'build-tool-fields') + require(entry['path'] == path and entry['resolved_path'] == path, + 'build-tool-path') + require(isinstance(entry['sha256'], str) and re.fullmatch(r'[0-9a-f]{64}', entry['sha256']) + and type(entry['size']) is int and 0 < entry['size'] <= MAX_FILE + and entry['executable'] is True and isinstance(entry['version'], str) + and 0 < len(entry['version']) <= 65536 and entry['version'].endswith('\n') + and isinstance(installed.get('tools'), dict) + and installed['tools'].get(role) == entry['version'].splitlines()[0], + 'build-tool-identity') + observed = self._regular(path, executable=True) + require(all(observed[key] == entry[key] for key in ('sha256', 'size', 'executable')), + 'build-tool-bytes-mismatch') + result[role] = dict(entry) + return result diff --git a/tools/tests/test_released_sdk_inventory.py b/tools/tests/test_released_sdk_inventory.py new file mode 100644 index 0000000..2d7d7b0 --- /dev/null +++ b/tools/tests/test_released_sdk_inventory.py @@ -0,0 +1,345 @@ +# Copyright (c) 2026 Atrinik contributors +# SPDX-License-Identifier: MIT +"""Offline bounded inventory trust tests. No Docker, network or credentials.""" +import base64 +import copy +import hashlib +import importlib.util +import io +import json +from pathlib import Path +import tarfile +import unittest +from unittest.mock import Mock, patch +import urllib.parse + +ROOT = Path(__file__).resolve().parents[2] + + +def load(name, path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +M = load('released_sdk_inventory', ROOT / 'tools/released_sdk_inventory.py') +P = load('inventory_collector', ROOT / 'tools/collect-released-sdk-metadata.py') + + +class Response(io.BytesIO): + chunked = True + length = None + + def __init__(self, raw, expected=None): + super().__init__(raw) + self.headers = {'Content-Type': 'application/x-tar'} + if expected is not None: + self.headers['X-Docker-Container-Path-Stat'] = base64.b64encode(json.dumps(expected).encode()).decode() + + def getheader(self, name, default=None): + return self.headers.get(name, default) + + +def directory(path): + return {'name': Path(path).name, 'mode': M.GO_DIRECTORY | 0o755, + 'size': 4096, 'mtime': 'fixed', 'linkTarget': ''} + + +def regular(path, raw): + return {'name': Path(path).name, 'mode': 0o644, 'size': len(raw), + 'mtime': 'fixed', 'linkTarget': ''} + + +def tar(entries): + raw = io.BytesIO() + with tarfile.open(fileobj=raw, mode='w', format=tarfile.PAX_FORMAT) as out: + for name, data, kind, extra in entries: + member = tarfile.TarInfo(name) + member.type = kind + member.mode = 0o755 if kind == tarfile.DIRTYPE else 0o644 + member.size = len(data) + for key, value in extra.items(): + setattr(member, key, value) + out.addfile(member, io.BytesIO(data)) + return raw.getvalue() + + +DIR = ('atrinik-portable', b'', tarfile.DIRTYPE, {}) +FILE = ('atrinik-portable/a', b'abc', tarfile.REGTYPE, {}) +EMPTY = ('atrinik-portable/empty', b'', tarfile.REGTYPE, {}) + + +class InventoryTests(unittest.TestCase): + def setUp(self): + for item in (patch.object(P, 'https_get', side_effect=AssertionError('network forbidden')), + patch.object(P.LocalConnection, 'connect', side_effect=AssertionError('Docker forbidden'))): + item.start() + self.addCleanup(item.stop) + + def parse(self, entries=None, raw=None): + return M.tree_archive(Response(raw if raw is not None else tar(entries)), P, directory(M.ROOT)) + + def denied(self, entries=None, raw=None, code=None): + with self.assertRaises((M.InventoryError, P.Stop)) as caught: + self.parse(entries=entries, raw=raw) + if code: + self.assertEqual(str(caught.exception), code) + + def test_complete_inventory_and_zero_byte_file(self): + result = self.parse([DIR, FILE, EMPTY]) + self.assertEqual(result, { + 'a': {'sha256': hashlib.sha256(b'abc').hexdigest(), 'size': 3, 'executable': False}, + 'empty': {'sha256': hashlib.sha256(b'').hexdigest(), 'size': 0, 'executable': False}}) + self.assertEqual(M.inventory_digest(result), + hashlib.sha256(json.dumps(result, sort_keys=True, separators=(',', ':')).encode()).hexdigest()) + + def test_executable_bit_and_pax_times(self): + result = self.parse([DIR, ('atrinik-portable/run', b'x', tarfile.REGTYPE, + {'mode': 0o755, 'pax_headers': {'mtime': '123.25'}})]) + self.assertTrue(result['run']['executable']) + + def test_all_links_special_sparse_and_extensions_fail(self): + for kind in (tarfile.SYMTYPE, tarfile.LNKTYPE, tarfile.FIFOTYPE, + tarfile.CHRTYPE, tarfile.BLKTYPE, tarfile.GNUTYPE_SPARSE, + tarfile.XGLTYPE, tarfile.GNUTYPE_LONGNAME): + with self.subTest(kind=kind): + self.denied([DIR, ('atrinik-portable/bad', b'', kind, {})]) + for key in ('path', 'linkpath', 'size', 'GNU.sparse.map', 'SCHILY.xattr.secret'): + with self.subTest(pax=key): + self.denied([DIR, ('atrinik-portable/a', b'x', tarfile.REGTYPE, + {'pax_headers': {key: '0'}})], code='unsafe-pax-header') + + def test_paths_duplicates_and_collisions_fail(self): + for name in ('/atrinik-portable/a', 'atrinik-portable/../a', 'atrinik-portable/./a', + 'atrinik-portable//a', 'other/a', 'atrinik-portable/a\\b', + 'atrinik-portable/a\nb', 'atrinik-portable/d/file'): + with self.subTest(name=name): + self.denied([DIR, (name, b'x', tarfile.REGTYPE, {})]) + self.denied([DIR, FILE, FILE], code='inventory-duplicate') + self.denied([DIR, FILE, ('atrinik-portable/a', b'', tarfile.DIRTYPE, {})], + code='inventory-duplicate') + self.denied([FILE], code='inventory-parent-missing') + self.denied([DIR], code='inventory-empty') + for name in ('atrinik-portable//', 'atrinik-portable///'): + self.denied([(name, b'', tarfile.DIRTYPE, {}), FILE], code='inventory-path-invalid') + + def test_truncation_extra_members_and_framing(self): + raw = tar([DIR, FILE]) + for size in (0, 511, 512, 1024, 1535, 1536, 2048): + with self.subTest(size=size): + self.denied(raw=raw[:size]) + self.denied(raw=raw[:1536] + bytes(512) + b'x' + bytes(511), code='inventory-end-markers') + self.denied(raw=raw + b'x', code='inventory-tail') + self.denied(raw=raw + bytes(1), code='inventory-tail-alignment') + self.denied(raw=raw + tar([FILE]), code='inventory-tail') + response = Response(raw) + response.chunked = False + with self.assertRaisesRegex(M.InventoryError, 'inventory-unframed'): + M.tree_archive(response, P, directory(M.ROOT)) + response = Response(raw) + response.length = 123 + with self.assertRaisesRegex(M.InventoryError, 'inventory-truncated'): + M.tree_archive(response, P, directory(M.ROOT)) + + def test_limits_and_deadline(self): + for name, value in (('MAX_MEMBERS', 1), ('MAX_FILE', 2), ('MAX_TOTAL', 2), ('MAX_PATH', 2)): + with self.subTest(limit=name), patch.object(M, name, value): + self.denied([DIR, FILE]) + with patch.object(M.time, 'monotonic', side_effect=[0, 1000]): + self.denied([DIR, FILE], code='inventory-deadline') + response = Response(tar([DIR, FILE])) + budget = M.Budget() + with patch.object(M.time, 'monotonic', side_effect=[budget.deadline - 1, budget.deadline]): + with self.assertRaisesRegex(M.InventoryError, 'inventory-deadline'): + budget.read(response, 512) + + def test_single_file_zero_and_extra_member(self): + raw = tar([('empty', b'', tarfile.REGTYPE, {})]) + record, data = P.single_file_archive(Response(raw), '/empty', regular('/empty', b''), retain=False) + self.assertEqual(record['size'], 0) + self.assertIsNone(data) + with self.assertRaises(P.Stop): + P.single_file_archive(Response(tar([('a', b'x', tarfile.REGTYPE, {}), + ('extra', b'x', tarfile.REGTYPE, {})])), + '/a', regular('/a', b'x'), retain=False) + + def fake_image(self): + image = Mock() + image.identifier = 'a' * 64 + image.stat.side_effect = lambda path: directory(path) + return image + + def test_created_image_fences_and_connection_close(self): + image = self.fake_image() + conn = Mock() + image.docker.request.return_value = (conn, Response(tar([DIR, FILE]), directory(M.ROOT))) + result = M.InstalledInventory(image, P).producer() + self.assertIn('a', result) + self.assertEqual(image.fence.call_count, 2) + self.assertEqual(image.stat.call_count, 4) + conn.close.assert_called_once() + request = image.docker.request.call_args + self.assertEqual(urllib.parse.parse_qs(urllib.parse.urlsplit(request.args[1]).query), {'path': [M.ROOT]}) + + def test_root_stat_drift_and_archive_failure_close(self): + for drift in ('root', 'parent', 'get', 'tar'): + image = self.fake_image() + conn = Mock() + states = {'/opt': 0, M.ROOT: 0} + def stat(path): + states[path] += 1 + value = directory(path) + if states[path] == 2 and ((drift == 'root' and path == M.ROOT) or (drift == 'parent' and path == '/opt')): + value['mtime'] = 'changed' + return value + image.stat.side_effect = stat + expected = directory(M.ROOT) + if drift == 'get': + expected['mtime'] = 'changed' + image.docker.request.return_value = (conn, Response(b'bad' if drift == 'tar' else tar([DIR, FILE]), expected)) + with self.subTest(drift=drift), self.assertRaises(M.InventoryError): + M.InstalledInventory(image, P).producer() + conn.close.assert_called_once() + + def documents(self): + return {'source_packages': {'foo': '1.0'}, 'archives': {'foo.tar.xz': 'a' * 64}}, [ + {'source': 'foo', 'source_version': '1.0', 'package': 'foo:amd64', 'notice_directory': M.DOC + '/foo'}, + {'source': 'unrelated', 'notice_directory': '/secret'}] + + def test_notice_request_selection_and_no_descriptor_path_extension(self): + runtime, debian = self.documents() + self.assertEqual(M.notice_requests(runtime, debian), [M.DOC + '/foo/copyright']) + for key, bad in (('notice_directory', '/secret'), ('package', '../foo'), ('source_version', '2.0')): + rows = copy.deepcopy(debian) + rows[0][key] = bad + with self.subTest(key=key), self.assertRaises(M.InventoryError): + M.notice_requests(runtime, rows) + with self.assertRaisesRegex(M.InventoryError, 'notice-source-missing'): + M.notice_requests(runtime, []) + + def test_archive_crosschecks(self): + runtime, _ = self.documents() + contract = {'sources': [{'sha256': 'b' * 64}]} + records = {'sources/debian/foo.tar.xz': {'sha256': 'a' * 64}, + 'sources/' + 'b' * 64 + '.tar.gz': {'sha256': 'b' * 64}} + M.crosscheck_archives(records, runtime, contract) + for path in records: + bad = copy.deepcopy(records) + bad[path]['sha256'] = 'c' * 64 + with self.assertRaisesRegex(M.InventoryError, 'inventory-source-bytes-mismatch'): + M.crosscheck_archives(bad, runtime, contract) + runtime['archives'] = {'../secret': 'a' * 64} + with self.assertRaises(M.InventoryError): + M.crosscheck_archives(records, runtime, contract) + + + def notice_image(self, links=None, drift=False): + links = links or {} + image = self.fake_image() + payload = b'notice' + link_reads = {} + def stat(path): + if path in links: + raw, target = links[path] + return {'name': Path(path).name, 'mode': M.GO_SYMLINK | 0o777, + 'size': len(raw), 'mtime': 'fixed', 'linkTarget': target} + if path.endswith('/copyright') or path.startswith(M.COMMON + '/'): + return regular(path, payload) + return directory(path) + def request(method, route, headers): + self.assertEqual(method, 'GET') + path = urllib.parse.parse_qs(urllib.parse.urlsplit(route).query)['path'][0] + if path in links: + raw, _ = links[path] + link_reads[path] = link_reads.get(path, 0) + 1 + if drift and link_reads[path] > 1: + raw = raw.replace('bar', 'baz') + data = tar([(Path(path).name, b'', tarfile.SYMTYPE, {'linkname': raw, 'mode': 0o777})]) + else: + data = tar([(Path(path).name, payload, tarfile.REGTYPE, {})]) + return Mock(), Response(data, stat(path)) + image.stat.side_effect = stat + image.docker.request.side_effect = request + return image + + def test_notices_resolve_directory_and_file_links_beneath_doc(self): + runtime, debian = self.documents() + for links, canonical in (({}, M.DOC + '/foo/copyright'), + ({M.DOC + '/foo': ('bar', M.DOC + '/bar')}, M.DOC + '/bar/copyright'), + ({M.DOC + '/foo/copyright': ('../bar/copyright', M.DOC + '/bar/copyright')}, + M.DOC + '/bar/copyright')): + with self.subTest(links=links): + result = M.InstalledInventory(self.notice_image(links), P).debian_notices(runtime, debian) + self.assertEqual(set(result), {M.DOC + '/foo/copyright'}) + self.assertEqual(result[M.DOC + '/foo/copyright']['resolved_path'], canonical) + self.assertEqual(result[M.DOC + '/foo/copyright']['sha256'], hashlib.sha256(b'notice').hexdigest()) + + def test_notice_escape_cycle_limit_wrong_resolved_target_and_drift(self): + runtime, debian = self.documents() + cases = [ + {M.DOC + '/foo': ('/secret', '/secret')}, + {M.DOC + '/foo': ('../../../usr/share/doc/bar', M.DOC + '/bar')}, + {M.DOC + '/foo': ('bar', M.DOC + '/bar'), M.DOC + '/bar': ('foo', M.DOC + '/foo')}, + {M.DOC + '/foo': ('bar', M.DOC + '/wrong')}, + {M.DOC + '/foo': ('bar/../baz', M.DOC + '/baz'), + M.DOC + '/bar': ('/outside/dir', '/outside/dir')}, + ] + for links in cases: + with self.subTest(links=links), self.assertRaises(M.InventoryError): + M.InstalledInventory(self.notice_image(links), P).debian_notices(runtime, debian) + links = {M.DOC + '/foo': ('bar', M.DOC + '/bar')} + with patch.object(M, 'MAX_HOPS', 0), self.assertRaises(M.InventoryError): + M.InstalledInventory(self.notice_image(links), P).debian_notices(runtime, debian) + with self.assertRaisesRegex(M.InventoryError, 'notice-alias-drift'): + M.InstalledInventory(self.notice_image(links, drift=True), P).debian_notices(runtime, debian) + + def test_common_license_exact_fixed_names_and_format(self): + image = self.notice_image() + result = M.InstalledInventory(image, P).common_licenses() + self.assertEqual(set(result), set(M.COMMON_LICENSE_NAMES)) + self.assertEqual(len(result), 14) + self.assertEqual(set(result['MIT'] if 'MIT' in result else result['BSD']), + {'sha256', 'size', 'executable'}) + paths = [urllib.parse.parse_qs(urllib.parse.urlsplit(call.args[1]).query)['path'][0] + for call in image.docker.request.call_args_list] + self.assertEqual(set(paths), {M.COMMON + '/' + name for name in M.COMMON_LICENSE_NAMES}) + + def build_receipt(self): + entries = {role: {'path': path, 'resolved_path': path, 'sha256': hashlib.sha256(b'tool').hexdigest(), + 'size': 4, 'executable': True, 'version': role + ' fixture\nsecond line\n'} + for role, path in M.BUILD_TOOLS.items()} + return {'build_tools': {'schema_version': 1, 'tools': entries}, + 'tools': {role: entry['version'].splitlines()[0] for role, entry in entries.items()}} + + def test_build_tools_require_nonroot_executable_before_get(self): + for mode in (0o744, 0o644): + image = self.fake_image() + image.stat.side_effect = lambda path: ( + {**regular(path, b'tool'), 'mode': mode} + if path in M.BUILD_TOOLS.values() else directory(path)) + with self.subTest(mode=mode), self.assertRaisesRegex(P.Stop, 'tool-not-executable'): + M.InstalledInventory(image, P).build_tools(self.build_receipt()) + image.docker.request.assert_not_called() + + def test_build_tools_exact_schema_paths_and_streamed_bytes(self): + installed = self.build_receipt() + reader = M.InstalledInventory(self.fake_image(), P) + identity = {'sha256': hashlib.sha256(b'tool').hexdigest(), 'size': 4, 'executable': True} + with patch.object(reader, '_regular', return_value=identity) as read: + self.assertEqual(reader.build_tools(installed), installed['build_tools']['tools']) + self.assertEqual([call.args[0] for call in read.call_args_list], list(M.BUILD_TOOLS.values())) + for key, value in (('path', '/secret'), ('resolved_path', '/secret'), ('sha256', 'a' * 64), + ('size', True), ('executable', 1), ('version', 'wrong\n')): + bad = copy.deepcopy(installed) + bad['build_tools']['tools']['cmake'][key] = value + with self.subTest(key=key), patch.object(reader, '_regular', return_value=identity), self.assertRaises(M.InventoryError): + reader.build_tools(bad) + bad = copy.deepcopy(installed) + bad['build_tools']['tools']['extra'] = {} + with self.assertRaisesRegex(M.InventoryError, 'build-tool-schema'): + reader.build_tools(bad) + + +if __name__ == '__main__': + unittest.main() From 1ede7ccd732ec457e00ff817c78847fa3be3afe2 Mon Sep 17 00:00:00 2001 From: Zoey Rose Date: Sun, 11 Oct 2026 20:30:36 -0500 Subject: [PATCH 04/23] fix(portable): bind pkgconf queries to producer executable receipt --- AGENTS.md | 11 +++-- README.md | 11 +++-- portable/build_tools.py | 29 +++++++++++--- portable/consumer.sh | 3 +- portable/curl_evidence.py | 12 ++++-- portable/inventory.py | 11 +++-- portable/smoke.sh | 6 ++- portable/test_build_tools.py | 78 ++++++++++++++++++++++++++++++++++-- 8 files changed, 136 insertions(+), 25 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 485e4ed..e6768a6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -25,11 +25,14 @@ selected LLVM utility, including separate Clang C/C++ drivers and archiver/ranlib, with canonical paths, versions and hashes; rehash before consumer builds and publication. The separate schema-1 `installed.json.build_tools` - receipt binds fixed `/usr/bin/cmake` and `/usr/bin/ninja` paths, canonical - executables, hashes, sizes and full version stdout. Verify both protected - identities before probing either tool and fence configure/build execution; + receipt binds fixed `/usr/bin/cmake`, `/usr/bin/ninja` and `/usr/bin/pkgconf` + paths, canonical executables, hashes, sizes and full version stdout. Verify + all three protected identities before probing any tool and fence + configure/build execution; + the mandatory `pkg_config` role also fences inventory module-version and + consumer/smoke/curl package queries through the canonical executable. reject symlinks in these fixed Debian executable and metadata paths. - missing receipts require a rebuilt producer. The producer OS, libraries and + Missing or incomplete receipts require a rebuilt producer. The producer OS, libraries and CMake modules and earlier library-production builds remain trusted; this receipt does not attest historical execution. Portable curl is an LLVM rebuild of security-patched Debian `7.88.1-10+deb12u15` with c-ares 1.34.6, retaining diff --git a/README.md b/README.md index d4b6e8b..54fd85d 100644 --- a/README.md +++ b/README.md @@ -464,15 +464,20 @@ versions and hashes for every selected LLVM utility, including independent `llvm-ranlib`; incomplete earlier evidence is rejected. Consumers rehash these identities before building and publication. -`installed.json.build_tools` separately seals CMake and Ninja with schema 1, -fixed `/usr/bin` paths, canonical executable paths, SHA-256 hashes, byte sizes, +`installed.json.build_tools` separately seals CMake, Ninja and pkgconf with schema 1, +fixed `/usr/bin/cmake`, `/usr/bin/ninja` and `/usr/bin/pkgconf` paths, +canonical executable paths, SHA-256 hashes, byte sizes, executable status and complete version stdout. Its first lines agree with the existing installed tool version strings. The consumer verifies both protected root-owned executables through open descriptors before any version probe and fences their resolution, bytes and file identity around configure and build. +The mandatory `pkg_config` role also supplies the installed tool version string, +inventory module-version queries, and consumer/smoke/curl package probes through +the same admission and execution fences. Module versions and feature selections +retain their existing contract. Every directory component must be protected, and these fixed Debian executable and metadata paths must not contain symlinks. -SDKs without this producer receipt require a rebuild. This check trusts the +SDKs without all three producer identities require a rebuild. This check trusts the producer OS, dynamic libraries and CMake modules; it does not attest the full transitive execution environment. It binds the installed executables used by the later consumer; earlier library diff --git a/portable/build_tools.py b/portable/build_tools.py index 2194ef9..88d1f20 100644 --- a/portable/build_tools.py +++ b/portable/build_tools.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Bind portable CMake/Ninja execution to the producer's installed bytes. +"""Bind portable CMake/Ninja/pkgconf execution to the producer's installed bytes. The producer OS, dynamic libraries and CMake modules remain trusted. These receipts protect the selected executables against an unprivileged consumer; @@ -18,7 +18,7 @@ import stat import subprocess -PATHS = {"cmake": "/usr/bin/cmake", "ninja": "/usr/bin/ninja"} +PATHS = {"cmake": "/usr/bin/cmake", "ninja": "/usr/bin/ninja", "pkg_config": "/usr/bin/pkgconf"} FIELDS = {"path", "resolved_path", "sha256", "size", "executable", "version"} MAX_BINARY_SIZE = 512 * 1024 * 1024 @@ -28,11 +28,11 @@ def validate(installed: dict) -> dict: raise ValueError("invalid producer installed metadata object") receipt = installed.get("build_tools") if receipt is None: - raise ValueError("missing producer build_tools receipt; rebuild the portable SDK with CMake/Ninja identities") + raise ValueError("missing producer build_tools receipt; rebuild the portable SDK with CMake/Ninja/pkgconf identities") if (type(receipt) is not dict or set(receipt) != {"schema_version", "tools"} or type(receipt["schema_version"]) is not int or receipt["schema_version"] != 1 or type(receipt["tools"]) is not dict or set(receipt["tools"]) != set(PATHS)): - raise ValueError("invalid producer build_tools schema or roles") + raise ValueError("invalid producer build_tools schema or roles; rebuild the portable SDK with all three identities") legacy = installed.get("tools") if type(legacy) is not dict: raise ValueError("missing installed tool version strings") @@ -168,10 +168,29 @@ def record() -> dict: return receipt -def verify(installed: dict, *, role: str | None = None, arguments: list[str] | None = None) -> None: +@contextmanager +def admitted_tools(installed: dict): + """Admit all three roles before any version or selected tool execution.""" receipt = validate(installed) with protected_tools(receipt["tools"]) as (items, fence): _versions(items, fence, receipt["tools"]) + yield items, fence + + +def capture(installed: dict, role: str, arguments: list[str]) -> str: + """Query the sealed executable, preserving stdout and failure-time fencing.""" + if role not in PATHS: + raise ValueError("unsupported portable build tool role") + with admitted_tools(installed) as (items, fence): + fence() + try: + return subprocess.check_output([items[role]["path"], *arguments]).decode("utf-8") + finally: + fence() + + +def verify(installed: dict, *, role: str | None = None, arguments: list[str] | None = None) -> None: + with admitted_tools(installed) as (_, fence): if role is not None: if role not in PATHS: raise ValueError("unsupported portable build tool role") diff --git a/portable/consumer.sh b/portable/consumer.sh index b283621..4b6ca20 100755 --- a/portable/consumer.sh +++ b/portable/consumer.sh @@ -18,6 +18,7 @@ while read -r expected_hash input; do case "${relative}" in client/* | LICENSE.md | ATTRIBUTIONS.md | PROVENANCE.md) ;; *) exit 1 ;; esac printf '%s %s\n' "${expected_hash}" "${source}/${relative}" | sha256sum -c - done < "${root}/shader-inputs.txt" +python3 "${root}/build_tools.py" --verify "${root}/installed.json" "${root}/smoke.sh" python3 "${root}/compiler_evidence.py" --verify "${root}/compiler-evidence.json" python3 "${root}/build_tools.py" --verify "${root}/installed.json" @@ -27,7 +28,7 @@ trap 'rm -rf -- "${work}"' EXIT # CMake build-tree RPATH padding, whose empty entries mean the current directory. python3 "${root}/build_tools.py" --verify "${root}/installed.json" --run cmake -- \ -S "${source}" -B "${work}/build" -G Ninja \ - -DCMAKE_MAKE_PROGRAM=/usr/bin/ninja -DPKG_CONFIG_EXECUTABLE=/usr/bin/pkg-config \ + -DCMAKE_MAKE_PROGRAM=/usr/bin/ninja -DPKG_CONFIG_EXECUTABLE=/usr/bin/pkgconf \ -DCMAKE_AR=/opt/llvm-23.1.3/bin/llvm-ar -DCMAKE_RANLIB=/opt/llvm-23.1.3/bin/llvm-ranlib \ -DCMAKE_BUILD_TYPE=Release -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ \ diff --git a/portable/curl_evidence.py b/portable/curl_evidence.py index d91ffbc..5b0a1cb 100644 --- a/portable/curl_evidence.py +++ b/portable/curl_evidence.py @@ -13,6 +13,8 @@ import subprocess import tempfile +from build_tools import capture, load_installed + def digest(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() @@ -56,8 +58,10 @@ def tls_results(positive: dict, negative: dict) -> None: raise ValueError("portable public-CA qualification failed") -def record(manifest_path: Path, public_ca: Path | None = None) -> dict: +def record(manifest_path: Path, public_ca: Path | None = None, *, installed: dict | None = None) -> dict: root = manifest_path.parent + if installed is None: + installed = load_installed(root / "installed.json") manifest = json.loads(manifest_path.read_text()) shared_path = root / manifest["cares_manifest"] shared = json.loads(shared_path.read_text()) @@ -66,8 +70,8 @@ def record(manifest_path: Path, public_ca: Path | None = None) -> dict: retained = root / "sources/curl" configuration((retained / "curl_config.h").read_text()) for package, expected in (("libcurl", manifest["version"]), ("libcares", shared["cares"]["version"])): - prefix = subprocess.check_output(["pkg-config", "--variable=prefix", package], text=True).strip() - version = subprocess.check_output(["pkg-config", "--modversion", package], text=True).strip() + prefix = capture(installed, "pkg_config", ["--variable=prefix", package]).strip() + version = capture(installed, "pkg_config", ["--modversion", package]).strip() if prefix != manifest["prefix"] or version != expected: raise ValueError("wrong active portable curl pkg-config: " + package) if public_ca is not None: @@ -75,7 +79,7 @@ def record(manifest_path: Path, public_ca: Path | None = None) -> dict: tls = None with tempfile.TemporaryDirectory(prefix="portable-curl-probe-") as temporary: binary = Path(temporary) / "probe" - package_flags = shlex.split(subprocess.check_output(["pkg-config", "--cflags", "--libs", "libcurl", "libcares"], text=True)) + package_flags = shlex.split(capture(installed, "pkg_config", ["--cflags", "--libs", "libcurl", "libcares"])) command = ["/opt/llvm-23.1.3/bin/clang", "-O2", "-march=x86-64", "-mtune=generic", "-Wall", "-Wextra", "-Werror", str(root / "curl-runtime-probe.c"), "-o", str(binary), *package_flags, "-ldl"] subprocess.run(command, check=True) diff --git a/portable/inventory.py b/portable/inventory.py index 6709686..3ac001f 100644 --- a/portable/inventory.py +++ b/portable/inventory.py @@ -7,7 +7,7 @@ from compiler_evidence import record from curl_evidence import record as record_curl -from build_tools import record as record_build_tools +from build_tools import capture, record as record_build_tools root = Path(__file__).resolve().parent contract = json.loads((root / "contract.json").read_text()) @@ -27,18 +27,21 @@ def output(*args): evidence_path = root / "compiler-evidence.json" evidence_path.write_text(json.dumps(evidence, indent=2) + "\n") build_tools = record_build_tools() +producer_installed = {"build_tools": build_tools, + "tools": {role: item["version"].splitlines()[0] for role, item in build_tools["tools"].items()}} actual = {"schema_version": 2, "contract_sha256": hashlib.sha256((root / "contract.json").read_bytes()).hexdigest(), "architecture": output("dpkg", "--print-architecture"), "glibc": output("getconf", "GNU_LIBC_VERSION"), "tools": {name: output(*args).splitlines()[0] for name, args in { "cc": ["clang", "--version"], "linker": ["ld.lld", "--version"], "readelf": ["llvm-readelf", "--version"], "python": ["python3", "--version"], "git-lfs": ["git", "lfs", "version"], "openssl": ["openssl", "version"]}.items() - } | {role: item["version"].splitlines()[0] for role, item in build_tools["tools"].items()}, + } | producer_installed["tools"], "build_tools": build_tools, "compiler_target": evidence["target"], "compiler_evidence_sha256": hashlib.sha256(evidence_path.read_bytes()).hexdigest(), - "pkg_config": {name: output("pkg-config", "--modversion", name) for name in contract["pkg_config"]}, - "curl": record_curl(root / contract["curl_manifest"])} + "pkg_config": {name: capture(producer_installed, "pkg_config", ["--modversion", name]).strip() + for name in contract["pkg_config"]}, + "curl": record_curl(root / contract["curl_manifest"], installed=producer_installed)} assert actual["architecture"] == "amd64" assert actual["glibc"] == "glibc 2.36" for name, version in contract["pkg_config"].items(): diff --git a/portable/smoke.sh b/portable/smoke.sh index b498e93..cd55a1a 100755 --- a/portable/smoke.sh +++ b/portable/smoke.sh @@ -2,6 +2,7 @@ set -euo pipefail root=/opt/atrinik-portable test "$(id -u)" != 0 +python3 "${root}/build_tools.py" --verify "${root}/installed.json" python3 - <<'READABLE' import os from pathlib import Path @@ -33,7 +34,9 @@ cd "${work}" python3 "${root}/abi.py" --output abi.json "${root}/audio/sdl3-mixer-probe" "${root}/audio/opus-probe.opus" # pkg-config emits flags for the pinned build inputs; no user-provided shell code. -read -r -a media_flags <<< "$(pkg-config --cflags --libs sdl3-image sdl3-ttf)" +media_output=$(python3 "${root}/build_tools.py" --verify "${root}/installed.json" --run pkg_config -- \ + --cflags --libs sdl3-image sdl3-ttf) +read -r -a media_flags <<< "${media_output}" clang -O2 -march=x86-64 -mtune=generic "${root}/media-smoke.c" \ -o media-smoke "${media_flags[@]}" ./media-smoke /usr/share/fonts/truetype/dejavu/DejaVuSans.ttf @@ -42,3 +45,4 @@ grep -q 'OpenSSL Default Provider' providers.txt grep -q 'OpenSSL Legacy Provider' providers.txt python3 "${root}/abi.py" --output media-abi.json ./media-smoke (cd "${root}/shaders" && sha256sum -c SHA256SUMS) +python3 "${root}/build_tools.py" --verify "${root}/installed.json" diff --git a/portable/test_build_tools.py b/portable/test_build_tools.py index 3a03dc4..adbf1e6 100644 --- a/portable/test_build_tools.py +++ b/portable/test_build_tools.py @@ -17,7 +17,7 @@ def installed_fixture(): versions = {"cmake": "cmake version 3.25.1\n\nCMake suite maintained and supported by Kitware (kitware.com/cmake).\n", - "ninja": "1.11.1\n"} + "ninja": "1.11.1\n", "pkg_config": "1.8.1\n"} return {"tools": {role: version.splitlines()[0] for role, version in versions.items()}, "build_tools": {"schema_version": 1, "tools": { role: {"path": path, "resolved_path": path, @@ -44,6 +44,7 @@ def test_strict_fields_roles_and_types(self): lambda v: v["build_tools"].update(schema_version=2), lambda v: v["build_tools"].update(tools=[]), lambda v: v["build_tools"]["tools"].pop("ninja"), + lambda v: v["build_tools"]["tools"].pop("pkg_config"), lambda v: v["build_tools"]["tools"].update(ctest={}), lambda v: v.update(tools=[]), ] @@ -88,11 +89,13 @@ def setUp(self): self.root = Path(self.temp.name) self.log = self.root / "execution.log" self.paths = {} - for role, version in (("cmake", "cmake version 3.25.1\n\ncomplete banner\n"), ("ninja", "1.11.1\n")): + for role, version in (("cmake", "cmake version 3.25.1\n\ncomplete banner\n"), ("ninja", "1.11.1\n"), + ("pkg_config", "1.8.1\n")): path = self.root / role path.write_text("#!/usr/bin/python3\nimport sys\nfrom pathlib import Path\n" f"with Path({str(self.log)!r}).open('a') as log: log.write({role!r} + ' ' + repr(sys.argv[1:]) + '\\n')\n" - f"if sys.argv[1:] == ['--version']: sys.stdout.write({version!r})\n") + f"if sys.argv[1:] == ['--version']: sys.stdout.write({version!r})\n" + "elif sys.argv[1:2] == ['--modversion']: sys.stdout.write('1.2.3\\n')\n") path.chmod(0o755) self.paths[role] = str(path) # A test-owned tree intentionally has a consumer UID and writable /tmp @@ -126,6 +129,7 @@ def test_record_actual_bytes_and_full_stdout_and_run_fixed_tool(self): self.assertTrue(item["version"].endswith("\n")) tools.verify(self.installed, role="cmake", arguments=["-S", "fixture", "-G", "Ninja"]) self.assertEqual(self.log.read_text().splitlines(), ["cmake ['--version']", "ninja ['--version']", + "pkg_config ['--version']", "cmake ['-S', 'fixture', '-G', 'Ninja']"]) def test_changed_tool_or_size_rejected_before_any_probe(self): @@ -145,6 +149,64 @@ def test_replaced_bytes_fail_before_version_or_configure(self): tools.verify(self.installed, role="cmake", arguments=["--configure"]) self.assertFalse(self.log.exists()) + def test_missing_pkgconf_role_rejects_old_receipt_before_any_execution(self): + value = copy.deepcopy(self.installed) + del value["build_tools"]["tools"]["pkg_config"] + with self.assertRaisesRegex(ValueError, "schema or roles"): + tools.capture(value, "pkg_config", ["--modversion", "fixture"]) + self.assertFalse(self.log.exists()) + + def test_replaced_pkgconf_bytes_fail_before_any_version_or_module_query(self): + path = Path(self.paths["pkg_config"]) + path.write_bytes(path.read_bytes() + b"# changed pkgconf bytes\n") + with self.assertRaisesRegex(ValueError, "changed installed"): + tools.capture(self.installed, "pkg_config", ["--modversion", "fixture"]) + self.assertFalse(self.log.exists()) + + def test_curl_consumer_changed_pkgconf_is_rejected_before_any_probe(self): + import curl_evidence + + manifest = self.root / "curl-toolchain.json" + manifest.write_text(json.dumps({"cares_manifest": "classic-curl-toolchain.json", + "prefix": "/usr/local", "version": "7.88.1"})) + (self.root / "classic-curl-toolchain.json").write_text(json.dumps({"cares": {"version": "1.34.6"}})) + retained = self.root / "sources/curl" + retained.mkdir(parents=True) + (retained / "curl_config.h").write_text("#define USE_ARES 1\n") + path = Path(self.paths["pkg_config"]) + path.write_bytes(path.read_bytes() + b"# changed before curl probe\n") + with mock.patch.object(curl_evidence, "load_installed", return_value=self.installed) as loader, \ + mock.patch.object(curl_evidence, "capture", side_effect=tools.capture), \ + self.assertRaisesRegex(ValueError, "changed installed"): + curl_evidence.record(manifest) + loader.assert_called_once_with(self.root / "installed.json") + self.assertFalse(self.log.exists()) + + def test_module_query_uses_admitted_pkgconf_with_complete_stdout(self): + self.assertEqual(tools.capture(self.installed, "pkg_config", ["--modversion", "fixture"]), "1.2.3\n") + self.assertEqual(self.log.read_text().splitlines(), ["cmake ['--version']", "ninja ['--version']", + "pkg_config ['--version']", + "pkg_config ['--modversion', 'fixture']"]) + + def test_module_query_mutation_is_fenced_including_failure(self): + original = tools.subprocess.check_output + for failed in (False, True): + path = Path(self.paths["pkg_config"]) + before = path.read_bytes() + + def mutate_query(*args, **kwargs): + if args[0][1:] == ["--modversion", "fixture"]: + path.write_bytes(before + b"# changed during module query\n") + if failed: + raise tools.subprocess.CalledProcessError(1, args[0]) + return b"1.2.3\n" + return original(*args, **kwargs) + + with self.subTest(failed=failed), mock.patch.object(tools.subprocess, "check_output", side_effect=mutate_query), \ + self.assertRaisesRegex(ValueError, "changed across execution"): + tools.capture(self.installed, "pkg_config", ["--modversion", "fixture"]) + path.write_bytes(before) + def test_full_stdout_mismatch_fails_before_configure(self): value = copy.deepcopy(self.installed) value["build_tools"]["tools"]["cmake"]["version"] = "cmake version 3.25.1\n\nchanged later line\n" @@ -303,10 +365,20 @@ def test_consumer_uses_receipt_runner_for_configure_and_build(self): self.assertEqual(source.count('--run cmake --'), 2) self.assertNotIn('/usr/bin/cmake ', source) self.assertIn('-DCMAKE_MAKE_PROGRAM=/usr/bin/ninja', source) + self.assertIn('-DPKG_CONFIG_EXECUTABLE=/usr/bin/pkgconf', source) + self.assertLess(source.index('build_tools.py'), source.index('"${root}/smoke.sh"')) inventory = Path(__file__).with_name("inventory.py").read_text() self.assertIn('"build_tools": build_tools', inventory) self.assertNotIn('["cmake", "--version"]', inventory) self.assertNotIn('["ninja", "--version"]', inventory) + self.assertIn('capture(producer_installed, "pkg_config", ["--modversion", name])', inventory) + self.assertNotIn('output("pkg-config"', inventory) + smoke = Path(__file__).with_name("smoke.sh").read_text() + self.assertIn('--run pkg_config --', smoke) + self.assertNotIn('$(pkg-config ', smoke) + curl = Path(__file__).with_name("curl_evidence.py").read_text() + self.assertNotIn('["pkg-config"', curl) + self.assertEqual(curl.count('capture(installed, "pkg_config"'), 3) class MetadataTests(unittest.TestCase): From 84bdeb71cae69ec37baa91aece7e5e1b9d1901f8 Mon Sep 17 00:00:00 2001 From: Zoey Rose Date: Sun, 11 Oct 2026 20:34:17 -0500 Subject: [PATCH 05/23] fix(portable): generate curl producer receipt before inventory --- portable/build-curl.py | 14 +++++++++++-- portable/test_build_tools.py | 39 +++++++++++++++++++++++++++++++++++- 2 files changed, 50 insertions(+), 3 deletions(-) diff --git a/portable/build-curl.py b/portable/build-curl.py index 5d5cd00..034e155 100644 --- a/portable/build-curl.py +++ b/portable/build-curl.py @@ -21,6 +21,17 @@ def digest(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() +def record_curl_production(root: Path) -> dict: + """The cold producer has no installed.json yet; generate real tool identities.""" + from build_tools import record as record_build_tools + from curl_evidence import record + + build_tools = record_build_tools() + installed = {"build_tools": build_tools, + "tools": {role: item["version"].splitlines()[0] for role, item in build_tools["tools"].items()}} + return record(root / "curl-toolchain.json", installed=installed) + + def checksum_table(text: str, field: str) -> dict[str, str]: lines = text.splitlines() start = lines.index(field + ":") + 1 @@ -171,8 +182,7 @@ def main(root: Path, jobs: int) -> None: "cares_manifest_sha256": digest(root / manifest["cares_manifest"])} (root / "curl-source-receipt.json").write_text(json.dumps(receipt, indent=2) + "\n") subprocess.run(["ldconfig"], check=True) - from curl_evidence import record - record(root / "curl-toolchain.json") + record_curl_production(root) if __name__ == "__main__": diff --git a/portable/test_build_tools.py b/portable/test_build_tools.py index adbf1e6..b62e270 100644 --- a/portable/test_build_tools.py +++ b/portable/test_build_tools.py @@ -95,7 +95,8 @@ def setUp(self): path.write_text("#!/usr/bin/python3\nimport sys\nfrom pathlib import Path\n" f"with Path({str(self.log)!r}).open('a') as log: log.write({role!r} + ' ' + repr(sys.argv[1:]) + '\\n')\n" f"if sys.argv[1:] == ['--version']: sys.stdout.write({version!r})\n" - "elif sys.argv[1:2] == ['--modversion']: sys.stdout.write('1.2.3\\n')\n") + "elif sys.argv[1:2] == ['--modversion']: sys.stdout.write('1.2.3\\n')\n" + "elif sys.argv[1:2] == ['--variable=prefix']: sys.stdout.write('/usr/local\\n')\n") path.chmod(0o755) self.paths[role] = str(path) # A test-owned tree intentionally has a consumer UID and writable /tmp @@ -182,6 +183,42 @@ def test_curl_consumer_changed_pkgconf_is_rejected_before_any_probe(self): loader.assert_called_once_with(self.root / "installed.json") self.assertFalse(self.log.exists()) + def test_cold_curl_producer_passes_generated_receipt_and_admits_package_queries(self): + import build_tools + import curl_evidence + + builder_spec = importlib.util.spec_from_file_location("cold_build_curl", Path(__file__).with_name("build-curl.py")) + builder = importlib.util.module_from_spec(builder_spec) + builder_spec.loader.exec_module(builder) + manifest = self.root / "curl-toolchain.json" + manifest.write_text(json.dumps({"cares_manifest": "classic-curl-toolchain.json", + "prefix": "/usr/local", "version": "7.88.1"})) + (self.root / "classic-curl-toolchain.json").write_text(json.dumps({"cares": {"version": "1.34.6"}})) + retained = self.root / "sources/curl" + retained.mkdir(parents=True) + (retained / "curl_config.h").write_text("#define USE_ARES 1\n") + self.assertFalse((self.root / "installed.json").exists()) + with mock.patch.object(build_tools, "record", side_effect=tools.record), \ + mock.patch.object(curl_evidence, "load_installed", side_effect=AssertionError("cold producer must pass a generated receipt")) as loader, \ + mock.patch.object(curl_evidence, "capture", side_effect=tools.capture), \ + mock.patch.object(curl_evidence, "record", wraps=curl_evidence.record) as record, \ + self.assertRaisesRegex(ValueError, "wrong active portable curl pkg-config"): + # The real query emits fixture module version 1.2.3, intentionally + # stopping at existing module validation before an application probe. + builder.record_curl_production(self.root) + loader.assert_not_called() + generated = record.call_args.kwargs["installed"] + self.assertEqual(set(generated["build_tools"]["tools"]), {"cmake", "ninja", "pkg_config"}) + for role, path in self.paths.items(): + self.assertEqual(generated["build_tools"]["tools"][role]["sha256"], hashlib.sha256(Path(path).read_bytes()).hexdigest()) + logs = self.log.read_text().splitlines() + self.assertEqual(logs[:3], ["cmake ['--version']", "ninja ['--version']", "pkg_config ['--version']"]) + self.assertIn("pkg_config ['--variable=prefix', 'libcurl']", logs) + self.assertIn("pkg_config ['--modversion', 'libcurl']", logs) + # Exercise the extracted cold evidence step, and retain the real main + # callsite so future refactors cannot bypass this generated argument. + self.assertIn(' record_curl_production(root)\n', Path(__file__).with_name("build-curl.py").read_text()) + def test_module_query_uses_admitted_pkgconf_with_complete_stdout(self): self.assertEqual(tools.capture(self.installed, "pkg_config", ["--modversion", "fixture"]), "1.2.3\n") self.assertEqual(self.log.read_text().splitlines(), ["cmake ['--version']", "ninja ['--version']", From 35296594f9b0b790a306029de1563446b048adb1 Mon Sep 17 00:00:00 2001 From: Zoey Rose Date: Sun, 11 Oct 2026 20:22:03 -0500 Subject: [PATCH 06/23] fix(portable): align installed build tool receipt validation --- tools/released_sdk_inventory.py | 11 ++++- tools/tests/test_released_sdk_inventory.py | 50 +++++++++++++++++++++- 2 files changed, 58 insertions(+), 3 deletions(-) diff --git a/tools/released_sdk_inventory.py b/tools/released_sdk_inventory.py index 92fc811..f0bb14b 100644 --- a/tools/released_sdk_inventory.py +++ b/tools/released_sdk_inventory.py @@ -24,6 +24,7 @@ MAX_MEMBERS = 20000 MAX_PATH = 1024 MAX_FILE = 1024 ** 3 +MAX_BUILD_TOOL = 512 * 1024 * 1024 MAX_TOTAL = 8 * 1024 ** 3 MAX_SECONDS = 900 MAX_HOPS = 16 @@ -430,6 +431,7 @@ def debian_notices(self, runtime, debian): def build_tools(self, installed): + require(isinstance(installed, dict), 'build-tool-schema') receipt = installed.get('build_tools') require(isinstance(receipt, dict) and set(receipt) == {'schema_version', 'tools'} and type(receipt['schema_version']) is int and receipt['schema_version'] == 1 @@ -443,9 +445,14 @@ def build_tools(self, installed): require(entry['path'] == path and entry['resolved_path'] == path, 'build-tool-path') require(isinstance(entry['sha256'], str) and re.fullmatch(r'[0-9a-f]{64}', entry['sha256']) - and type(entry['size']) is int and 0 < entry['size'] <= MAX_FILE + and type(entry['size']) is int and 0 < entry['size'] <= MAX_BUILD_TOOL and entry['executable'] is True and isinstance(entry['version'], str) - and 0 < len(entry['version']) <= 65536 and entry['version'].endswith('\n') + and 0 < len(entry['version']) <= 65536 + and not any((ord(char) < 32 and char != '\n') or ord(char) == 127 + for char in entry['version']) + and re.fullmatch(r'cmake version [0-9]+\.[0-9]+\.[0-9]+' if role == 'cmake' + else r'[0-9]+\.[0-9]+(?:\.[0-9]+)?', + entry['version'].splitlines()[0]) is not None and isinstance(installed.get('tools'), dict) and installed['tools'].get(role) == entry['version'].splitlines()[0], 'build-tool-identity') diff --git a/tools/tests/test_released_sdk_inventory.py b/tools/tests/test_released_sdk_inventory.py index 2d7d7b0..ba55090 100644 --- a/tools/tests/test_released_sdk_inventory.py +++ b/tools/tests/test_released_sdk_inventory.py @@ -307,11 +307,59 @@ def test_common_license_exact_fixed_names_and_format(self): def build_receipt(self): entries = {role: {'path': path, 'resolved_path': path, 'sha256': hashlib.sha256(b'tool').hexdigest(), - 'size': 4, 'executable': True, 'version': role + ' fixture\nsecond line\n'} + 'size': 4, 'executable': True, + 'version': ('cmake version 3.25.1' if role == 'cmake' else '1.11.1') + '\nsecond line\n'} for role, path in M.BUILD_TOOLS.items()} return {'build_tools': {'schema_version': 1, 'tools': entries}, 'tools': {role: entry['version'].splitlines()[0] for role, entry in entries.items()}} + def test_build_tools_accept_full_stdout_with_or_without_final_newline(self): + for cmake, ninja in (('cmake version 3.25.1', '1.11'), + ('cmake version 3.25.1\n\nCMake suite maintained and supported by Kitware.', + '1.11.1\n'), + ('cmake version 3.25.1\nfull stdout\n', '1.11.1')): + installed = self.build_receipt() + for role, version in (('cmake', cmake), ('ninja', ninja)): + installed['build_tools']['tools'][role]['version'] = version + installed['tools'][role] = version.splitlines()[0] + reader = M.InstalledInventory(self.fake_image(), P) + identity = {'sha256': hashlib.sha256(b'tool').hexdigest(), 'size': 4, 'executable': True} + with self.subTest(cmake=cmake, ninja=ninja), patch.object(reader, '_regular', return_value=identity): + self.assertEqual(reader.build_tools(installed), installed['build_tools']['tools']) + + def test_build_tools_enforce_shared_512_mib_boundary(self): + for size in (512 * 1024 * 1024, 512 * 1024 * 1024 + 1): + installed = self.build_receipt() + for entry in installed['build_tools']['tools'].values(): + entry['size'] = size + identity = {'sha256': hashlib.sha256(b'tool').hexdigest(), 'size': size, 'executable': True} + reader = M.InstalledInventory(self.fake_image(), P) + with self.subTest(size=size), patch.object(reader, '_regular', return_value=identity) as read: + if size == 512 * 1024 * 1024: + self.assertEqual(reader.build_tools(installed), installed['build_tools']['tools']) + else: + with self.assertRaisesRegex(M.InventoryError, 'build-tool-identity'): + reader.build_tools(installed) + read.assert_not_called() + + def test_build_tools_reject_role_grammar_controls_and_del(self): + cases = {'cmake': ('3.25.1', 'cmake version 3.25', 'cmake version 3.25.1-rc1'), + 'ninja': ('ninja 1.11.1', 'cmake version 3.25.1', '1.11.1-rc1')} + for role in M.BUILD_TOOLS: + valid = 'cmake version 3.25.1' if role == 'cmake' else '1.11.1' + cases[role] += tuple(valid + '\nextra' + chr(code) + for code in (*range(10), *range(11, 32), 127)) + for version in cases[role]: + installed = self.build_receipt() + installed['build_tools']['tools'][role]['version'] = version + installed['tools'][role] = version.splitlines()[0] + reader = M.InstalledInventory(self.fake_image(), P) + identity = {'sha256': hashlib.sha256(b'tool').hexdigest(), 'size': 4, 'executable': True} + with self.subTest(role=role, version=version), patch.object(reader, '_regular', return_value=identity) as read: + with self.assertRaisesRegex(M.InventoryError, 'build-tool-identity'): + reader.build_tools(installed) + self.assertNotIn(M.BUILD_TOOLS[role], [call.args[0] for call in read.call_args_list]) + def test_build_tools_require_nonroot_executable_before_get(self): for mode in (0o744, 0o644): image = self.fake_image() From a935976a3f5ad3b8d975e27ac743de2a482d5b17 Mon Sep 17 00:00:00 2001 From: Zoey Rose Date: Sun, 11 Oct 2026 20:26:33 -0500 Subject: [PATCH 07/23] fix(portable): require installed pkgconf identity --- tools/RELEASED_SDK_INVENTORY.md | 6 ++- tools/released_sdk_inventory.py | 3 +- tools/tests/test_released_sdk_inventory.py | 47 +++++++++++++++++----- 3 files changed, 44 insertions(+), 12 deletions(-) diff --git a/tools/RELEASED_SDK_INVENTORY.md b/tools/RELEASED_SDK_INVENTORY.md index 50109bb..ac5680a 100644 --- a/tools/RELEASED_SDK_INVENTORY.md +++ b/tools/RELEASED_SDK_INVENTORY.md @@ -44,8 +44,10 @@ compare this producer map with the wrapper's `verified_tree_inventory` on that exact image (or an authenticated complete OCI-overlay reconstruction). Source archive hash checks alone do not establish that no unrelated file was omitted. -The build-tool reader accepts exactly `cmake` and `ninja` at `/usr/bin/cmake` and -`/usr/bin/ninja`, both canonical regular executables. It validates +The build-tool reader requires exactly `cmake`, `ninja` and `pkg_config` at +`/usr/bin/cmake`, `/usr/bin/ninja` and `/usr/bin/pkgconf`, all canonical regular +executables. `pkg_config` uses the numeric version grammar shared with Ninja; +the `pkg-config` alias cannot substitute for the fixed `pkgconf` path. It validates `installed.json.build_tools = {schema_version: 1, tools: {…}}`, the exact role and field sets, full producer version strings and legacy first-line compatibility, then compares streamed bytes, size and executable status. It never executes a diff --git a/tools/released_sdk_inventory.py b/tools/released_sdk_inventory.py index f0bb14b..9e846be 100644 --- a/tools/released_sdk_inventory.py +++ b/tools/released_sdk_inventory.py @@ -17,7 +17,8 @@ ROOT = '/opt/atrinik-portable' DOC = '/usr/share/doc' COMMON = '/usr/share/common-licenses' -BUILD_TOOLS = {'cmake': '/usr/bin/cmake', 'ninja': '/usr/bin/ninja'} +BUILD_TOOLS = {'cmake': '/usr/bin/cmake', 'ninja': '/usr/bin/ninja', + 'pkg_config': '/usr/bin/pkgconf'} COMMON_LICENSE_NAMES = ('Apache-2.0', 'Artistic', 'BSD', 'CC0-1.0', 'GFDL-1.2', 'GFDL-1.3', 'GPL-1', 'GPL-2', 'GPL-3', 'LGPL-2', 'LGPL-2.1', 'LGPL-3', 'MPL-1.1', 'MPL-2.0') diff --git a/tools/tests/test_released_sdk_inventory.py b/tools/tests/test_released_sdk_inventory.py index ba55090..faa5f9a 100644 --- a/tools/tests/test_released_sdk_inventory.py +++ b/tools/tests/test_released_sdk_inventory.py @@ -308,7 +308,8 @@ def test_common_license_exact_fixed_names_and_format(self): def build_receipt(self): entries = {role: {'path': path, 'resolved_path': path, 'sha256': hashlib.sha256(b'tool').hexdigest(), 'size': 4, 'executable': True, - 'version': ('cmake version 3.25.1' if role == 'cmake' else '1.11.1') + '\nsecond line\n'} + 'version': {'cmake': 'cmake version 3.25.1', 'ninja': '1.11.1', + 'pkg_config': '1.8.1'}[role] + '\nsecond line\n'} for role, path in M.BUILD_TOOLS.items()} return {'build_tools': {'schema_version': 1, 'tools': entries}, 'tools': {role: entry['version'].splitlines()[0] for role, entry in entries.items()}} @@ -319,7 +320,8 @@ def test_build_tools_accept_full_stdout_with_or_without_final_newline(self): '1.11.1\n'), ('cmake version 3.25.1\nfull stdout\n', '1.11.1')): installed = self.build_receipt() - for role, version in (('cmake', cmake), ('ninja', ninja)): + for role, version in (('cmake', cmake), ('ninja', ninja), + ('pkg_config', ninja.replace('1.11', '1.8'))): installed['build_tools']['tools'][role]['version'] = version installed['tools'][role] = version.splitlines()[0] reader = M.InstalledInventory(self.fake_image(), P) @@ -344,7 +346,8 @@ def test_build_tools_enforce_shared_512_mib_boundary(self): def test_build_tools_reject_role_grammar_controls_and_del(self): cases = {'cmake': ('3.25.1', 'cmake version 3.25', 'cmake version 3.25.1-rc1'), - 'ninja': ('ninja 1.11.1', 'cmake version 3.25.1', '1.11.1-rc1')} + 'ninja': ('ninja 1.11.1', 'cmake version 3.25.1', '1.11.1-rc1'), + 'pkg_config': ('pkgconf 1.8.1', 'cmake version 3.25.1', '1.8.1-rc1')} for role in M.BUILD_TOOLS: valid = 'cmake version 3.25.1' if role == 'cmake' else '1.11.1' cases[role] += tuple(valid + '\nextra' + chr(code) @@ -370,6 +373,31 @@ def test_build_tools_require_nonroot_executable_before_get(self): M.InstalledInventory(image, P).build_tools(self.build_receipt()) image.docker.request.assert_not_called() + def test_pkg_config_is_mandatory_and_alias_cannot_replace_fixed_pkgconf(self): + self.assertEqual(M.BUILD_TOOLS, {'cmake': '/usr/bin/cmake', 'ninja': '/usr/bin/ninja', + 'pkg_config': '/usr/bin/pkgconf'}) + installed = self.build_receipt() + reader = M.InstalledInventory(self.fake_image(), P) + for role in M.BUILD_TOOLS: + bad = copy.deepcopy(installed) + del bad['build_tools']['tools'][role] + with self.subTest(missing=role), patch.object(reader, '_regular') as read: + with self.assertRaisesRegex(M.InventoryError, 'build-tool-schema'): + reader.build_tools(bad) + read.assert_not_called() + identity = {'sha256': hashlib.sha256(b'tool').hexdigest(), 'size': 4, 'executable': True} + for key in ('path', 'resolved_path'): + bad = copy.deepcopy(installed) + bad['build_tools']['tools']['pkg_config'][key] = '/usr/bin/pkg-config' + with self.subTest(key=key), patch.object(reader, '_regular', return_value=identity) as read: + with self.assertRaisesRegex(M.InventoryError, 'build-tool-path'): + reader.build_tools(bad) + self.assertNotIn('/usr/bin/pkgconf', [call.args[0] for call in read.call_args_list]) + bad = copy.deepcopy(installed) + del bad['tools']['pkg_config'] + with patch.object(reader, '_regular', return_value=identity), self.assertRaisesRegex(M.InventoryError, 'build-tool-identity'): + reader.build_tools(bad) + def test_build_tools_exact_schema_paths_and_streamed_bytes(self): installed = self.build_receipt() reader = M.InstalledInventory(self.fake_image(), P) @@ -377,12 +405,13 @@ def test_build_tools_exact_schema_paths_and_streamed_bytes(self): with patch.object(reader, '_regular', return_value=identity) as read: self.assertEqual(reader.build_tools(installed), installed['build_tools']['tools']) self.assertEqual([call.args[0] for call in read.call_args_list], list(M.BUILD_TOOLS.values())) - for key, value in (('path', '/secret'), ('resolved_path', '/secret'), ('sha256', 'a' * 64), - ('size', True), ('executable', 1), ('version', 'wrong\n')): - bad = copy.deepcopy(installed) - bad['build_tools']['tools']['cmake'][key] = value - with self.subTest(key=key), patch.object(reader, '_regular', return_value=identity), self.assertRaises(M.InventoryError): - reader.build_tools(bad) + for role in M.BUILD_TOOLS: + for key, value in (('path', '/secret'), ('resolved_path', '/secret'), ('sha256', 'a' * 64), + ('size', True), ('executable', 1), ('version', 'wrong\n')): + bad = copy.deepcopy(installed) + bad['build_tools']['tools'][role][key] = value + with self.subTest(role=role, key=key), patch.object(reader, '_regular', return_value=identity), self.assertRaises(M.InventoryError): + reader.build_tools(bad) bad = copy.deepcopy(installed) bad['build_tools']['tools']['extra'] = {} with self.assertRaisesRegex(M.InventoryError, 'build-tool-schema'): From 0307d053ead5d7573c0555c61bd2e0810d61ca31 Mon Sep 17 00:00:00 2001 From: Zoey Rose Date: Sun, 11 Oct 2026 22:25:15 -0500 Subject: [PATCH 08/23] build(portable): bind SDK to merged Classic LLVM source --- portable/contract.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/portable/contract.json b/portable/contract.json index 5d3348b..8fa542f 100644 --- a/portable/contract.json +++ b/portable/contract.json @@ -60,7 +60,7 @@ "shader_manifest": "classic-shader-toolchain.json", "consumer": { "repository": "atrinik/classic", - "commit": "bbaf7785b6f7c5905b3f5c52f079d6198da0d540", + "commit": "e009394e8249b34f44a81aa99b3bdfbe37979d79", "shader_directory": "/opt/atrinik-portable/shaders", "cmake_argument": "ATRINIK_GPU_SHADER_DIRECTORY", "input_verification": "exact-source-commit-and-shader-input-sha256", From a25b5dfa3ad6f447d7e26d983b6d8035ed68cdf8 Mon Sep 17 00:00:00 2001 From: Zoey Rose Date: Sun, 11 Oct 2026 23:37:54 -0500 Subject: [PATCH 09/23] build(portable): bind SDK to merged Classic LLVM repair --- portable/contract.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/portable/contract.json b/portable/contract.json index 8fa542f..bc4b864 100644 --- a/portable/contract.json +++ b/portable/contract.json @@ -60,7 +60,7 @@ "shader_manifest": "classic-shader-toolchain.json", "consumer": { "repository": "atrinik/classic", - "commit": "e009394e8249b34f44a81aa99b3bdfbe37979d79", + "commit": "2ad99b1cf0fb0508d6cef5fd19b64df02ceef62e", "shader_directory": "/opt/atrinik-portable/shaders", "cmake_argument": "ATRINIK_GPU_SHADER_DIRECTORY", "input_verification": "exact-source-commit-and-shader-input-sha256", From 46f3961a9760a29433d9a376f3c064f1b5767af9 Mon Sep 17 00:00:00 2001 From: Zoey Rose Date: Mon, 12 Oct 2026 00:19:43 -0500 Subject: [PATCH 10/23] build(portable): bind SDK to Classic inspector repair --- portable/contract.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/portable/contract.json b/portable/contract.json index bc4b864..841bbfb 100644 --- a/portable/contract.json +++ b/portable/contract.json @@ -60,7 +60,7 @@ "shader_manifest": "classic-shader-toolchain.json", "consumer": { "repository": "atrinik/classic", - "commit": "2ad99b1cf0fb0508d6cef5fd19b64df02ceef62e", + "commit": "345fdccd57f79aea933fbde1bb5ae48517a7bd71", "shader_directory": "/opt/atrinik-portable/shaders", "cmake_argument": "ATRINIK_GPU_SHADER_DIRECTORY", "input_verification": "exact-source-commit-and-shader-input-sha256", From a0f524071cce16e481546c264f95a84a9a069f25 Mon Sep 17 00:00:00 2001 From: Zoey Rose Date: Mon, 12 Oct 2026 01:11:08 -0500 Subject: [PATCH 11/23] fix(sdk): require nonempty collector files by default (cherry picked from commit dc0b45707fdc6e96b3484ea182602d2d7d821456) --- tools/collect-released-sdk-metadata.py | 6 ++- .../test_collect_released_sdk_metadata.py | 42 +++++++++++++++++++ 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/tools/collect-released-sdk-metadata.py b/tools/collect-released-sdk-metadata.py index 06f95df..5fef1c4 100644 --- a/tools/collect-released-sdk-metadata.py +++ b/tools/collect-released-sdk-metadata.py @@ -611,7 +611,7 @@ def pax_times(raw): raw = raw[length:] -def single_file_archive(response, path, expected, *, retain, executable=False): +def single_file_archive(response, path, expected, *, retain, executable=False, allow_empty=False): """Read one regular file; reject all links/extensions/path changes, never extract.""" require(response.chunked or response.length is not None, 'unframed-archive') require(response.getheader('Content-Encoding') in (None, 'identity') and @@ -620,7 +620,9 @@ def single_file_archive(response, path, expected, *, retain, executable=False): stat_matches(expected, path) regular_mode(expected['mode'], executable) limit = METADATA_LIMIT if retain else BINARY_LIMIT - require(0 <= expected['size'] <= limit, 'file-size-limit') + # Only producer-tree inventory reads opt in to legitimate empty source files. + # Curated metadata, notices and canonical tools must always contain bytes. + require((0 if allow_empty else 1) <= expected['size'] <= limit, 'file-size-limit') header = exact_read(response, 512) info = tarfile.TarInfo.frombuf(header, 'utf-8', 'strict') if info.type == tarfile.XHDTYPE: diff --git a/tools/tests/test_collect_released_sdk_metadata.py b/tools/tests/test_collect_released_sdk_metadata.py index a84c1be..9054b23 100644 --- a/tools/tests/test_collect_released_sdk_metadata.py +++ b/tools/tests/test_collect_released_sdk_metadata.py @@ -469,6 +469,33 @@ def test_archive_rejects_links_types_paths_and_extra_members(self): M.single_file_archive(Response(archive(path, raw, **option)), path, file_stat(path, raw), retain=True) + def test_archive_empty_producer_file_requires_explicit_opt_in(self): + path, raw = '/opt/atrinik-portable/sources/empty.txt', b'' + for retain in (True, False): + with self.subTest(retain=retain): + with self.assertRaisesRegex(M.Stop, 'file-size-limit'): + M.single_file_archive(Response(archive(path, raw)), path, + file_stat(path, raw), retain=retain) + identity, actual = M.single_file_archive(Response(archive(path, raw)), path, + file_stat(path, raw), retain=retain, allow_empty=True) + self.assertEqual(identity['size'], 0) + self.assertEqual(identity['sha256'], M.hashlib.sha256(raw).hexdigest()) + self.assertEqual(actual, raw if retain else None) + + def test_empty_opt_in_preserves_archive_permissions_limits_and_link_rejection(self): + path, raw = '/opt/atrinik-portable/sources/empty.txt', b'' + for option in ({'mode': 0o666}, {'kind': tarfile.SYMTYPE}, + {'kind': tarfile.LNKTYPE}, {'extra': True}): + with self.subTest(option=option), self.assertRaises(M.Stop): + M.single_file_archive(Response(archive(path, raw, **option)), path, + file_stat(path, raw, option.get('mode', 0o644)), + retain=False, allow_empty=True) + for retain, limit in ((True, M.METADATA_LIMIT), (False, M.BINARY_LIMIT)): + for size in (-1, limit + 1): + with self.subTest(retain=retain, size=size), self.assertRaises(M.Stop): + M.single_file_archive(Response(archive(path, raw)), path, + dict(file_stat(path, raw), size=size), retain=retain, allow_empty=True) + def test_archive_pax_timestamps_only(self): path, raw = M.LLVM_MANIFEST, b'normal metadata' result, actual = M.single_file_archive(Response(archive(path, raw, pax={'mtime': '123.456'})), @@ -755,6 +782,21 @@ def test_portable_collection_checks_all_tool_roles_against_streamed_bytes(self): self.assertFalse(result['image_code_executed']) self.assertFalse(result['final_portable_cohort']) + def test_collection_rejects_empty_metadata_notices_and_canonical_tools(self): + for target in M.TARGETS: + for path in (M.LLVM_MANIFEST, M.LLVM_NOTICES[0], + M.PREFIX + '/bin/clang-23', M.PREFIX + '/lib/libunwind.a'): + data, binary = self.portable_fixture() + if target != 'portable-preliminary': + data = {name: data.get(name, b'public metadata fixture') + for name in M.COMMON_FILES + M.TARGET_FILES[target]} + data['/usr/local/share/atrinik/llvm-toolchain.json'] = data[M.LLVM_MANIFEST] + (data if path in data else binary)[path] = b'' + with self.subTest(target=target, path=path): + result, _ = self.collect_portable_fixture(data, binary, target) + self.assertEqual(result['status'], 'stopped') + self.assertEqual(result['reason'], 'file-size-limit') + def test_portable_contract_and_compiler_receipt_hashes_are_required(self): base = '/opt/atrinik-portable/' for name in ('contract.json', 'compiler-evidence.json'): From cfb2bebd2acde6d674165fc49ea05ad12687278b Mon Sep 17 00:00:00 2001 From: Zoey Rose Date: Mon, 12 Oct 2026 01:11:44 -0500 Subject: [PATCH 12/23] fix(sdk): verify audio and LLVM source archive closure (cherry picked from commit 288a5137c2ac00d0fdb5979ec556d6a279cbd667) --- tools/released_sdk_inventory.py | 90 ++++++++++++- tools/tests/test_released_sdk_inventory.py | 150 +++++++++++++++++++++ 2 files changed, 236 insertions(+), 4 deletions(-) diff --git a/tools/released_sdk_inventory.py b/tools/released_sdk_inventory.py index 9e846be..6e89cb3 100644 --- a/tools/released_sdk_inventory.py +++ b/tools/released_sdk_inventory.py @@ -29,6 +29,9 @@ MAX_TOTAL = 8 * 1024 ** 3 MAX_SECONDS = 900 MAX_HOPS = 16 +SOURCE_MANIFESTS = ('audio-toolchain.json', 'llvm-toolchain.json') +MAX_MANIFEST = 1024 * 1024 +MAX_MANIFEST_SOURCES = 128 GO_DIRECTORY = 1 << 31 GO_SYMLINK = 1 << 27 @@ -37,6 +40,17 @@ class InventoryError(Exception): """Only fixed failure codes; never hostile archive or daemon contents.""" +class ProducerInventory(dict): + """File identities plus bounded source manifests from the same archive. + + Payloads are private validation inputs, excluded from the serialized inventory + and its digest. Copying only the identities cannot satisfy manifest closure. + """ + def __init__(self): + super().__init__() + self.source_manifests = {} + + def require(ok, code): if not ok: raise InventoryError(code) @@ -143,11 +157,11 @@ def next_member(response, budget, protocol, *, symlink=False): def tree_archive(response, protocol, expected, budget=None): - """Hash a complete Docker archive of ROOT, retaining no file payloads.""" + """Hash ROOT, retaining only the bounded, fixed source-manifest payloads.""" budget = budget or Budget() framing(response) protocol.stat_matches(expected, ROOT, directory=True) - files, entries, directories = {}, set(), set() + files, entries, directories = ProducerInventory(), set(), set() root_name = PurePosixPath(ROOT).name while True: info = next_member(response, budget, protocol) @@ -174,15 +188,23 @@ def tree_archive(response, protocol, expected, budget=None): require(info.type in (tarfile.REGTYPE, tarfile.AREGTYPE) and name != root_name, 'inventory-member-type') protocol.regular_mode(info.mode) + path = name[len(root_name) + 1:] + payload = bytearray() if path in SOURCE_MANIFESTS else None + if payload is not None: + require(0 < info.size <= MAX_MANIFEST, 'inventory-manifest-size') sha = hashlib.sha256() remaining = info.size while remaining: raw = budget.exact(response, min(65536, remaining)) sha.update(raw) + if payload is not None: + payload.extend(raw) remaining -= len(raw) require(not any(budget.exact(response, (-info.size) % 512)), 'inventory-padding') - files[name[len(root_name) + 1:]] = {'sha256': sha.hexdigest(), 'size': info.size, - 'executable': bool(info.mode & 0o111)} + files[path] = {'sha256': sha.hexdigest(), 'size': info.size, + 'executable': bool(info.mode & 0o111)} + if payload is not None: + files.source_manifests[path] = bytes(payload) require(root_name in directories and bool(files), 'inventory-empty') return files @@ -213,6 +235,38 @@ def notice_requests(runtime, debian): return sorted(requests) +def source_manifest(records, reference, expected): + """Parse only fixed manifests authenticated by the producer tree read.""" + require(reference == expected, 'inventory-manifest-path') + payloads = getattr(records, 'source_manifests', {}) + raw = payloads.get(expected) + require(isinstance(raw, bytes) and 0 < len(raw) <= MAX_MANIFEST, + 'inventory-manifest-missing-or-size') + identity = records.get(expected) + require(isinstance(identity, dict) and identity.get('size') == len(raw) + and identity.get('sha256') == hashlib.sha256(raw).hexdigest(), + 'inventory-manifest-bytes-mismatch') + + def unique(pairs): + value = {} + for key, item in pairs: + require(key not in value, 'inventory-manifest-duplicate') + value[key] = item + return value + + def invalid_constant(_): + raise InventoryError('inventory-manifest-json') + + try: + value = json.loads(raw.decode('utf-8'), object_pairs_hook=unique, + parse_constant=invalid_constant) + except (ValueError, UnicodeError, RecursionError): + raise InventoryError('inventory-manifest-json') from None + require(isinstance(value, dict) and type(value.get('schema_version')) is int + and value['schema_version'] == 1, 'inventory-manifest-schema') + return value + + def crosscheck_archives(records, runtime, contract): archives, sources = runtime.get('archives'), contract.get('sources') require(isinstance(archives, dict) and 0 < len(archives) <= MAX_MEMBERS @@ -232,6 +286,34 @@ def add(path, sha): sha = source.get('sha256') require(isinstance(sha, str), 'inventory-source-digest') add('sources/' + sha + '.tar.gz', sha) + if 'audio_manifest' in contract: + audio = source_manifest(records, contract['audio_manifest'], 'audio-toolchain.json') + dependencies = audio.get('dependencies') + require(isinstance(dependencies, list) and 0 < len(dependencies) < MAX_MANIFEST_SOURCES, + 'inventory-audio-sources') + names = set() + for entry in [audio.get('sdl_mixer'), *dependencies]: + require(isinstance(entry, dict), 'inventory-audio-source') + name, source = entry.get('name'), entry.get('source') + require(isinstance(name, str) and re.fullmatch(r'[A-Za-z0-9_.+-]{1,128}', name) + and name not in names and isinstance(source, dict), 'inventory-audio-source') + names.add(name) + sha = source.get('sha256') + require(isinstance(sha, str), 'inventory-source-digest') + add('sources/' + sha + '.tar.gz', sha) + compiler = contract.get('compiler', {}) + require(isinstance(compiler, dict), 'inventory-compiler-schema') + if 'toolchain_manifest' in compiler: + llvm = source_manifest(records, compiler['toolchain_manifest'], 'llvm-toolchain.json') + version, source, metadata = llvm.get('version'), llvm.get('source'), llvm.get('metadata') + require(isinstance(version, str) and re.fullmatch(r'[0-9]{1,4}\.[0-9]{1,4}\.[0-9]{1,4}', version) + and isinstance(source, dict) and isinstance(metadata, dict), 'inventory-llvm-source') + # The installer writes this exact filename, and portable/Dockerfile + # copies its source directory into sources/llvm without flattening it. + archive = 'llvm-project-' + version + '.src.tar.xz' + require(metadata.get('source_archive') == 'share/atrinik/sources/' + archive, + 'inventory-llvm-archive-path') + add('sources/llvm/' + archive, source.get('sha256')) providers = runtime.get('custom_providers', {}) require(isinstance(providers, dict) and len(providers) <= MAX_MEMBERS, 'inventory-custom-providers') for provider in providers.values(): diff --git a/tools/tests/test_released_sdk_inventory.py b/tools/tests/test_released_sdk_inventory.py index faa5f9a..2e13155 100644 --- a/tools/tests/test_released_sdk_inventory.py +++ b/tools/tests/test_released_sdk_inventory.py @@ -233,6 +233,156 @@ def test_archive_crosschecks(self): with self.assertRaises(M.InventoryError): M.crosscheck_archives(records, runtime, contract) + def manifest_fixture(self): + # Use the producer's actual schemas and path declarations, with small + # stand-ins for the source archives so tests never download or execute. + contract = json.loads((ROOT / 'portable/contract.json').read_text()) + audio = json.loads((ROOT / 'audio-toolchain.json').read_text()) + llvm = json.loads((ROOT / 'llvm-toolchain.json').read_text()) + payloads = {'sources/debian/foo.tar.xz': b'debian source'} + runtime = {'archives': {'foo.tar.xz': hashlib.sha256(b'debian source').hexdigest()}} + for entry in contract['sources']: + raw = ('source ' + entry['name']).encode() + entry['sha256'] = hashlib.sha256(raw).hexdigest() + payloads['sources/' + entry['sha256'] + '.tar.gz'] = raw + for entry in [audio['sdl_mixer'], *audio['dependencies']]: + raw = ('source ' + entry['name']).encode() + entry['source']['sha256'] = hashlib.sha256(raw).hexdigest() + payloads['sources/' + entry['source']['sha256'] + '.tar.gz'] = raw + llvm['source']['sha256'] = hashlib.sha256(b'llvm source').hexdigest() + payloads['sources/llvm/' + Path(llvm['metadata']['source_archive']).name] = b'llvm source' + payloads['audio-toolchain.json'] = json.dumps(audio).encode() + payloads['llvm-toolchain.json'] = json.dumps(llvm).encode() + return payloads, runtime, contract + + def producer_fixture(self, payloads): + entries = [DIR] + directories = set() + for name, raw in sorted(payloads.items()): + for parent in reversed(Path(name).parents): + if str(parent) != '.' and str(parent) not in directories: + entries.append(('atrinik-portable/' + str(parent), b'', tarfile.DIRTYPE, {})) + directories.add(str(parent)) + entries.append(('atrinik-portable/' + name, raw, tarfile.REGTYPE, {})) + return self.parse(entries) + + def test_authenticated_manifests_close_audio_and_llvm_sources(self): + payloads, runtime, contract = self.manifest_fixture() + records = self.producer_fixture(payloads) + M.crosscheck_archives(records, runtime, contract) + self.assertEqual(set(records.source_manifests), set(M.SOURCE_MANIFESTS)) + self.assertEqual(M.inventory_digest(records), M.inventory_digest(dict(records))) + # A hash-only inventory cannot manufacture authenticated manifest data. + with self.assertRaisesRegex(M.InventoryError, 'inventory-manifest-missing-or-size'): + M.crosscheck_archives(dict(records), runtime, contract) + + def test_each_declared_audio_and_llvm_source_missing_or_replaced_fails(self): + payloads, runtime, contract = self.manifest_fixture() + audio = json.loads(payloads['audio-toolchain.json']) + paths = ['sources/' + entry['source']['sha256'] + '.tar.gz' + for entry in [audio['sdl_mixer'], *audio['dependencies']]] + paths += [path for path in payloads if path.startswith('sources/llvm/')] + for path in paths: + for missing in (True, False): + bad = dict(payloads) + if missing: + del bad[path] + else: + bad[path] = b'replaced archive' + # Even equal whole-tree inventories cannot bless missing bytes. + records = self.producer_fixture(bad) + self.assertEqual(M.inventory_digest(records), M.inventory_digest(self.producer_fixture(bad))) + with self.subTest(path=path, missing=missing), self.assertRaisesRegex( + M.InventoryError, 'inventory-source-bytes-mismatch'): + M.crosscheck_archives(records, runtime, contract) + + def test_manifest_missing_replaced_identity_and_limits_fail(self): + payloads, runtime, contract = self.manifest_fixture() + for name in M.SOURCE_MANIFESTS: + bad = dict(payloads) + del bad[name] + with self.subTest(name=name), self.assertRaisesRegex( + M.InventoryError, 'inventory-manifest-missing-or-size'): + M.crosscheck_archives(self.producer_fixture(bad), runtime, contract) + records = self.producer_fixture(payloads) + records.source_manifests[name] += b' ' + with self.assertRaisesRegex(M.InventoryError, 'inventory-manifest-bytes-mismatch'): + M.crosscheck_archives(records, runtime, contract) + for raw in (b'', b' ' * (M.MAX_MANIFEST + 1)): + with self.subTest(name=name, size=len(raw)), self.assertRaisesRegex( + M.InventoryError, 'inventory-manifest-size'): + self.producer_fixture({name: raw}) + + def test_manifest_json_duplicates_types_and_depth_fail(self): + payloads, runtime, contract = self.manifest_fixture() + for name in M.SOURCE_MANIFESTS: + for raw in (b'{', b'\xff', b'[]', b'null', b'{"schema_version":true}', + b'{"schema_version":2}', b'{"schema_version":1,"schema_version":1}', + b'{"schema_version":1,"extra":{"sha256":1,"sha256":2}}', + b'{"schema_version":1,"extra":NaN}', b'[' * 2000 + b']' * 2000): + bad = dict(payloads) + bad[name] = raw + with self.subTest(name=name, raw=raw[:80]), self.assertRaises(M.InventoryError): + M.crosscheck_archives(self.producer_fixture(bad), runtime, contract) + + def test_manifest_references_cannot_escape_fixed_boundary(self): + payloads, runtime, contract = self.manifest_fixture() + for reference in ('/etc/passwd', '../audio-toolchain.json', 'sub/audio-toolchain.json', + 'sources/../audio-toolchain.json', 'audio-toolchain.json/extra', + 'audio-toolchain.json\\extra', None, [], {}): + for role in ('audio', 'llvm'): + bad = copy.deepcopy(contract) + if role == 'audio': + bad['audio_manifest'] = reference + else: + bad['compiler']['toolchain_manifest'] = reference + with self.subTest(role=role, reference=reference), self.assertRaisesRegex( + M.InventoryError, 'inventory-manifest-path'): + M.crosscheck_archives(self.producer_fixture(payloads), runtime, bad) + + def test_audio_manifest_invalid_sources_and_duplicate_names_fail(self): + payloads, runtime, contract = self.manifest_fixture() + original = json.loads(payloads['audio-toolchain.json']) + variants = [] + for key, values in (('dependencies', [None, {}, [], [None], [original['dependencies'][0]] * 128]), + ('sdl_mixer', [None, [], {}, {'name': '../escape', 'source': {}}, + {'name': 'mixer', 'source': None}])): + for value in values: + bad = copy.deepcopy(original) + bad[key] = value + variants.append(bad) + bad = copy.deepcopy(original) + bad['dependencies'].append(copy.deepcopy(bad['sdl_mixer'])) + variants.append(bad) + for sha in (None, 1, [], 'A' * 64, '../escape', 'a' * 63): + bad = copy.deepcopy(original) + bad['dependencies'][0]['source']['sha256'] = sha + variants.append(bad) + for value in variants: + bad = dict(payloads) + bad['audio-toolchain.json'] = json.dumps(value).encode() + with self.subTest(value=value), self.assertRaises(M.InventoryError): + M.crosscheck_archives(self.producer_fixture(bad), runtime, contract) + + def test_llvm_manifest_invalid_sources_and_archive_paths_fail(self): + payloads, runtime, contract = self.manifest_fixture() + original = json.loads(payloads['llvm-toolchain.json']) + for key, values in (('version', [None, True, '../escape', '23.1.3/../escape']), + ('source', [None, [], {}, {'sha256': 'A' * 64}]), + ('metadata', [None, [], {}, *[{'source_archive': path} for path in ( + '/etc/passwd', 'share/atrinik/sources/../escape.tar.xz', + 'share/atrinik/sources/nested/llvm-project-23.1.3.src.tar.xz', + 'share/atrinik/sources/llvm-project-23.1.3.src.tar.xz/extra', + 'share/atrinik/sources/llvm-project-23.1.3.src.tar.xz\\extra', + 'share/atrinik/sources/llvm-project-22.1.0.src.tar.xz')]])): + for value in values: + llvm = copy.deepcopy(original) + llvm[key] = value + bad = dict(payloads) + bad['llvm-toolchain.json'] = json.dumps(llvm).encode() + with self.subTest(key=key, value=value), self.assertRaises(M.InventoryError): + M.crosscheck_archives(self.producer_fixture(bad), runtime, contract) + def notice_image(self, links=None, drift=False): links = links or {} From e07ffb5ed32390a5de187bd325df22dd9c93805a Mon Sep 17 00:00:00 2001 From: Zoey Rose Date: Mon, 12 Oct 2026 01:16:24 -0500 Subject: [PATCH 13/23] test(sdk): reject empty inventory notices and build tools (cherry picked from commit a960d517e5908280b72373a25972ef9f21d1e8e8) --- tools/tests/test_released_sdk_inventory.py | 26 +++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/tools/tests/test_released_sdk_inventory.py b/tools/tests/test_released_sdk_inventory.py index 2e13155..52c1faa 100644 --- a/tools/tests/test_released_sdk_inventory.py +++ b/tools/tests/test_released_sdk_inventory.py @@ -156,9 +156,8 @@ def test_limits_and_deadline(self): def test_single_file_zero_and_extra_member(self): raw = tar([('empty', b'', tarfile.REGTYPE, {})]) - record, data = P.single_file_archive(Response(raw), '/empty', regular('/empty', b''), retain=False) - self.assertEqual(record['size'], 0) - self.assertIsNone(data) + with self.assertRaisesRegex(P.Stop, 'file-size-limit'): + P.single_file_archive(Response(raw), '/empty', regular('/empty', b''), retain=False) with self.assertRaises(P.Stop): P.single_file_archive(Response(tar([('a', b'x', tarfile.REGTYPE, {}), ('extra', b'x', tarfile.REGTYPE, {})])), @@ -455,6 +454,27 @@ def test_common_license_exact_fixed_names_and_format(self): for call in image.docker.request.call_args_list] self.assertEqual(set(paths), {M.COMMON + '/' + name for name in M.COMMON_LICENSE_NAMES}) + def test_empty_inventory_notices_and_build_tools_are_rejected(self): + runtime, debian = self.documents() + for path in (M.DOC + '/foo/copyright', M.COMMON + '/BSD', *M.BUILD_TOOLS.values()): + image = self.fake_image() + executable = path in M.BUILD_TOOLS.values() + expected = regular(path, b'') + if executable: + expected['mode'] = 0o755 + image.stat.side_effect = lambda value: expected if value == path else directory(value) + conn = Mock() + image.docker.request.return_value = ( + conn, Response(tar([(Path(path).name, b'', tarfile.REGTYPE, + {'mode': expected['mode']})]), expected)) + reader = M.InstalledInventory(image, P) + with self.subTest(path=path), self.assertRaisesRegex(P.Stop, 'file-size-limit'): + if path.startswith(M.DOC + '/'): + reader.debian_notices(runtime, debian) + else: + reader._regular(path, executable=executable) + conn.close.assert_called_once() + def build_receipt(self): entries = {role: {'path': path, 'resolved_path': path, 'sha256': hashlib.sha256(b'tool').hexdigest(), 'size': 4, 'executable': True, From d18a2de5da2c89c53f0f1dfcf96b108214085c73 Mon Sep 17 00:00:00 2001 From: Zoey Rose Date: Mon, 12 Oct 2026 01:22:52 -0500 Subject: [PATCH 14/23] docs(sdk): clarify retained manifests and sealed tool cohort --- README.md | 2 +- tools/RELEASED_SDK_INVENTORY.md | 14 +++++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 54fd85d..82a24eb 100644 --- a/README.md +++ b/README.md @@ -468,7 +468,7 @@ identities before building and publication. fixed `/usr/bin/cmake`, `/usr/bin/ninja` and `/usr/bin/pkgconf` paths, canonical executable paths, SHA-256 hashes, byte sizes, executable status and complete version stdout. Its first lines agree with the -existing installed tool version strings. The consumer verifies both protected +existing installed tool version strings. The consumer verifies all three protected root-owned executables through open descriptors before any version probe and fences their resolution, bytes and file identity around configure and build. The mandatory `pkg_config` role also supplies the installed tool version string, diff --git a/tools/RELEASED_SDK_INVENTORY.md b/tools/RELEASED_SDK_INVENTORY.md index ac5680a..11b3d1c 100644 --- a/tools/RELEASED_SDK_INVENTORY.md +++ b/tools/RELEASED_SDK_INVENTORY.md @@ -6,8 +6,14 @@ has no CLI, credentials, registry operations, release pins or image execution. It is not yet imported or invoked by `collect-released-sdk-metadata.py`; the three fixed 1.18.0 targets and their producer mapping remain authoritative. -The helper streams `/opt/atrinik-portable` without extraction or retaining file -payloads. It requires a root directory entry, parents before children, unique +The helper streams `/opt/atrinik-portable` without extraction. It retains only +the fixed `audio-toolchain.json` and `llvm-toolchain.json` payloads, each nonempty +and at most 1 MiB, as private validation inputs on `ProducerInventory`. Before +parsing either manifest, it binds the retained bytes to the streamed record's +SHA-256 and size. These payloads are excluded from serialized inventory maps and +their digests; copying the records to a plain dictionary loses the payloads and +fails closed when a contract references a manifest. All other file payloads are +discarded after hashing. It requires a root directory entry, parents before children, unique canonical relative paths, regular files and safe directory/file modes. It rejects links, special/sparse files, path/size PAX overrides, malformed headers, truncated members, missing end markers and nonzero trailing data. Timestamp-only @@ -24,7 +30,9 @@ wrapper's fourteen fixed names. Debian notices are selected only by matching rows. Their records retain the requested absolute path as key and add `resolved_path`. Source archives named by the authenticated runtime and contract documents must match the complete producer inventory, including custom provider -archives. +archives and the audio and LLVM archives declared by their referenced manifests. +Debian notices, common licenses and build-tool executables must be nonempty; +only producer-tree inventory records may describe legitimate zero-byte files. Docker HEAD's `LinkTarget` is a fully resolved result, so it cannot establish a safe link traversal by itself. The notice resolver reads each raw symbolic link From 1c1865ad8b8593c716b017464cae1fbe8b8b3f1b Mon Sep 17 00:00:00 2001 From: Zoey Rose Date: Mon, 12 Oct 2026 06:24:39 -0500 Subject: [PATCH 15/23] fix(sdk): require complete portable source archive declarations (cherry picked from commit 50684ca1c05fc1e6af9c9e3a14538d80722c8057) --- tools/RELEASED_SDK_INVENTORY.md | 28 ++- tools/released_sdk_inventory.py | 124 +++++++---- tools/tests/test_released_sdk_inventory.py | 232 ++++++++++++++++++++- 3 files changed, 337 insertions(+), 47 deletions(-) diff --git a/tools/RELEASED_SDK_INVENTORY.md b/tools/RELEASED_SDK_INVENTORY.md index 11b3d1c..8c4e5c4 100644 --- a/tools/RELEASED_SDK_INVENTORY.md +++ b/tools/RELEASED_SDK_INVENTORY.md @@ -7,12 +7,13 @@ It is not yet imported or invoked by `collect-released-sdk-metadata.py`; the three fixed 1.18.0 targets and their producer mapping remain authoritative. The helper streams `/opt/atrinik-portable` without extraction. It retains only -the fixed `audio-toolchain.json` and `llvm-toolchain.json` payloads, each nonempty +the fixed `audio-toolchain.json`, `llvm-toolchain.json`, `curl-toolchain.json` +and `classic-curl-toolchain.json` payloads, each nonempty and at most 1 MiB, as private validation inputs on `ProducerInventory`. Before -parsing either manifest, it binds the retained bytes to the streamed record's +parsing a manifest, it binds the retained bytes to the streamed record's SHA-256 and size. These payloads are excluded from serialized inventory maps and their digests; copying the records to a plain dictionary loses the payloads and -fails closed when a contract references a manifest. All other file payloads are +fails closed during source-archive validation. All other file payloads are discarded after hashing. It requires a root directory entry, parents before children, unique canonical relative paths, regular files and safe directory/file modes. It rejects links, special/sparse files, path/size PAX overrides, malformed headers, @@ -31,8 +32,27 @@ rows. Their records retain the requested absolute path as key and add `resolved_path`. Source archives named by the authenticated runtime and contract documents must match the complete producer inventory, including custom provider archives and the audio and LLVM archives declared by their referenced manifests. +Every declared archive record requires an integer size greater than zero and at +most 1 GiB, plus the declared SHA-256; even the matching digest of an empty file +cannot satisfy an archive declaration. Debian notices, common licenses and build-tool executables must be nonempty; -only producer-tree inventory records may describe legitimate zero-byte files. +other producer-tree files may legitimately have zero bytes. + +Archive validation requires the schema-2 `portable-final`, `linux/amd64` +contract, including the fixed audio, LLVM and curl manifest references. The +curl manifest must reference the fixed shared Classic curl manifest. Historical +contracts with absent references cannot qualify through this helper. The +contract's source cohort is exactly `openssl`, `sdl3`, `sdl3-image` and +`sdl3-ttf`; the audio cohort is `SDL3_mixer` with `libogg`, `libopus` and +`libopusfile`. Missing, duplicate or renamed components fail validation. +`runtime-sources.json.custom_providers` must contain exactly `libcurl` and +`libcares`, with the fields emitted by `portable/legal.py`. Their complete +archive maps must match the authenticated curl manifests: the four Debian +curl source/signature files under `sources/curl/`, and the digest-named c-ares +archive under `sources/`. Removing an individual archive from a provider's +declaration cannot bypass its byte check. These checks validate declared source +closure; they do not parse Debian `.dsc` dependency semantics or independently +establish completeness of the producer's Debian source-package archive map. Docker HEAD's `LinkTarget` is a fully resolved result, so it cannot establish a safe link traversal by itself. The notice resolver reads each raw symbolic link diff --git a/tools/released_sdk_inventory.py b/tools/released_sdk_inventory.py index 6e89cb3..bfb95cd 100644 --- a/tools/released_sdk_inventory.py +++ b/tools/released_sdk_inventory.py @@ -29,9 +29,9 @@ MAX_TOTAL = 8 * 1024 ** 3 MAX_SECONDS = 900 MAX_HOPS = 16 -SOURCE_MANIFESTS = ('audio-toolchain.json', 'llvm-toolchain.json') +SOURCE_MANIFESTS = ('audio-toolchain.json', 'llvm-toolchain.json', + 'curl-toolchain.json', 'classic-curl-toolchain.json') MAX_MANIFEST = 1024 * 1024 -MAX_MANIFEST_SOURCES = 128 GO_DIRECTORY = 1 << 31 GO_SYMLINK = 1 << 27 @@ -239,11 +239,13 @@ def source_manifest(records, reference, expected): """Parse only fixed manifests authenticated by the producer tree read.""" require(reference == expected, 'inventory-manifest-path') payloads = getattr(records, 'source_manifests', {}) + require(isinstance(payloads, dict), 'inventory-manifest-payloads') raw = payloads.get(expected) require(isinstance(raw, bytes) and 0 < len(raw) <= MAX_MANIFEST, 'inventory-manifest-missing-or-size') identity = records.get(expected) - require(isinstance(identity, dict) and identity.get('size') == len(raw) + require(isinstance(identity, dict) and type(identity.get('size')) is int + and identity['size'] == len(raw) and identity.get('sha256') == hashlib.sha256(raw).hexdigest(), 'inventory-manifest-bytes-mismatch') @@ -268,6 +270,11 @@ def invalid_constant(_): def crosscheck_archives(records, runtime, contract): + require(isinstance(records, dict) and isinstance(runtime, dict) + and isinstance(contract, dict), 'inventory-source-documents') + require(type(contract.get('schema_version')) is int and contract['schema_version'] == 2 + and contract.get('target') == 'portable-final' and contract.get('platform') == 'linux/amd64', + 'inventory-contract-schema') archives, sources = runtime.get('archives'), contract.get('sources') require(isinstance(archives, dict) and 0 < len(archives) <= MAX_MEMBERS and isinstance(sources, list) and 0 < len(sources) <= MAX_MEMBERS, @@ -281,49 +288,90 @@ def add(path, sha): for name, sha in archives.items(): require(isinstance(name, str) and '/' not in name, 'inventory-source-name') add('sources/debian/' + name, sha) + source_names = set() for source in sources: require(isinstance(source, dict), 'inventory-source-row') + name = source.get('name') + require(isinstance(name, str) and name in {'openssl', 'sdl3', 'sdl3-image', 'sdl3-ttf'} + and name not in source_names, 'inventory-source-cohort') + source_names.add(name) sha = source.get('sha256') require(isinstance(sha, str), 'inventory-source-digest') add('sources/' + sha + '.tar.gz', sha) - if 'audio_manifest' in contract: - audio = source_manifest(records, contract['audio_manifest'], 'audio-toolchain.json') - dependencies = audio.get('dependencies') - require(isinstance(dependencies, list) and 0 < len(dependencies) < MAX_MANIFEST_SOURCES, - 'inventory-audio-sources') - names = set() - for entry in [audio.get('sdl_mixer'), *dependencies]: - require(isinstance(entry, dict), 'inventory-audio-source') - name, source = entry.get('name'), entry.get('source') - require(isinstance(name, str) and re.fullmatch(r'[A-Za-z0-9_.+-]{1,128}', name) - and name not in names and isinstance(source, dict), 'inventory-audio-source') - names.add(name) - sha = source.get('sha256') - require(isinstance(sha, str), 'inventory-source-digest') - add('sources/' + sha + '.tar.gz', sha) - compiler = contract.get('compiler', {}) + require(source_names == {'openssl', 'sdl3', 'sdl3-image', 'sdl3-ttf'}, 'inventory-source-cohort') + audio = source_manifest(records, contract.get('audio_manifest'), 'audio-toolchain.json') + dependencies = audio.get('dependencies') + require(isinstance(dependencies, list) and len(dependencies) == 3, + 'inventory-audio-sources') + mixer = audio.get('sdl_mixer') + require(isinstance(mixer, dict) and mixer.get('name') == 'SDL3_mixer', 'inventory-audio-source') + names = set() + for entry in [mixer, *dependencies]: + require(isinstance(entry, dict), 'inventory-audio-source') + name, source = entry.get('name'), entry.get('source') + require(isinstance(name, str) and re.fullmatch(r'[A-Za-z0-9_.+-]{1,128}', name) + and name not in names and isinstance(source, dict), 'inventory-audio-source') + names.add(name) + sha = source.get('sha256') + require(isinstance(sha, str), 'inventory-source-digest') + add('sources/' + sha + '.tar.gz', sha) + require(names == {'SDL3_mixer', 'libogg', 'libopus', 'libopusfile'}, 'inventory-audio-sources') + compiler = contract.get('compiler') require(isinstance(compiler, dict), 'inventory-compiler-schema') - if 'toolchain_manifest' in compiler: - llvm = source_manifest(records, compiler['toolchain_manifest'], 'llvm-toolchain.json') - version, source, metadata = llvm.get('version'), llvm.get('source'), llvm.get('metadata') - require(isinstance(version, str) and re.fullmatch(r'[0-9]{1,4}\.[0-9]{1,4}\.[0-9]{1,4}', version) - and isinstance(source, dict) and isinstance(metadata, dict), 'inventory-llvm-source') - # The installer writes this exact filename, and portable/Dockerfile - # copies its source directory into sources/llvm without flattening it. - archive = 'llvm-project-' + version + '.src.tar.xz' - require(metadata.get('source_archive') == 'share/atrinik/sources/' + archive, - 'inventory-llvm-archive-path') - add('sources/llvm/' + archive, source.get('sha256')) - providers = runtime.get('custom_providers', {}) - require(isinstance(providers, dict) and len(providers) <= MAX_MEMBERS, 'inventory-custom-providers') - for provider in providers.values(): - require(isinstance(provider, dict) and isinstance(provider.get('source_archives'), dict) - and 0 < len(provider['source_archives']) <= MAX_MEMBERS, 'inventory-custom-provider') - for path, sha in provider['source_archives'].items(): - require(isinstance(path, str) and path.startswith('sources/'), 'inventory-source-name') + llvm = source_manifest(records, compiler.get('toolchain_manifest'), 'llvm-toolchain.json') + version, source, metadata = llvm.get('version'), llvm.get('source'), llvm.get('metadata') + require(isinstance(version, str) and re.fullmatch(r'[0-9]{1,4}\.[0-9]{1,4}\.[0-9]{1,4}', version) + and isinstance(source, dict) and isinstance(metadata, dict), 'inventory-llvm-source') + # The installer writes this exact filename, and portable/Dockerfile + # copies its source directory into sources/llvm without flattening it. + archive = 'llvm-project-' + version + '.src.tar.xz' + require(metadata.get('source_archive') == 'share/atrinik/sources/' + archive, + 'inventory-llvm-archive-path') + add('sources/llvm/' + archive, source.get('sha256')) + + # legal.py emits both providers. Bind their complete archive declarations + # to the producer manifests, so dropping a provider or one archive cannot + # shrink the corresponding-source requirements. + curl = source_manifest(records, contract.get('curl_manifest'), 'curl-toolchain.json') + shared = source_manifest(records, curl.get('cares_manifest'), 'classic-curl-toolchain.json') + version, source_version, archives = curl.get('version'), curl.get('source_version'), curl.get('archives') + require(curl.get('source_package') == 'curl' and isinstance(version, str) + and re.fullmatch(r'[0-9]{1,4}\.[0-9]{1,4}\.[0-9]{1,4}', version) + and isinstance(source_version, str) and len(source_version) <= 128 + and re.fullmatch(re.escape(version) + r'-[0-9A-Za-z.+~]+', source_version), + 'inventory-curl-source') + require(isinstance(archives, dict) and set(archives) == { + 'curl_' + source_version + '.debian.tar.xz', 'curl_' + source_version + '.dsc', + 'curl_' + version + '.orig.tar.gz', 'curl_' + version + '.orig.tar.gz.asc'}, + 'inventory-curl-archives') + cares = shared.get('cares') + require(isinstance(cares, dict) and isinstance(cares.get('version'), str) + and re.fullmatch(r'[0-9]{1,4}\.[0-9]{1,4}\.[0-9]{1,4}', cares['version']) + and isinstance(cares.get('sha256'), str), 'inventory-cares-source') + expected_sources = { + 'libcurl': {'sources/curl/' + name: sha for name, sha in archives.items()}, + 'libcares': {'sources/' + cares['sha256'] + '.tar.gz': cares['sha256']}} + providers = runtime.get('custom_providers') + require(isinstance(providers, dict) and set(providers) == set(expected_sources), + 'inventory-custom-providers') + for name, sources in expected_sources.items(): + provider = providers[name] + require(isinstance(provider, dict) and set(provider) == { + 'path', 'sha256', 'version', 'source_archives', 'build_receipt'}, 'inventory-custom-provider') + require(isinstance(provider['path'], str) and re.fullmatch( + r'/usr/local/lib/' + name + r'\.so(?:\.[0-9]+)+', provider['path']) + and isinstance(provider['sha256'], str) and re.fullmatch(r'[0-9a-f]{64}', provider['sha256']) + and provider['version'] == (version if name == 'libcurl' else cares['version']) + and provider['build_receipt'] == 'curl-source-receipt.json' + and isinstance(provider['source_archives'], dict) and provider['source_archives'] == sources, + 'inventory-custom-provider') + for path, sha in sources.items(): add(path, sha) for path, sha in required.items(): - require(path in records and records[path]['sha256'] == sha, 'inventory-source-bytes-mismatch') + record = records.get(path) + require(isinstance(record, dict) and type(record.get('size')) is int + and 0 < record['size'] <= MAX_FILE and record.get('sha256') == sha, + 'inventory-source-bytes-mismatch') class BudgetResponse: diff --git a/tools/tests/test_released_sdk_inventory.py b/tools/tests/test_released_sdk_inventory.py index 52c1faa..6f8ff26 100644 --- a/tools/tests/test_released_sdk_inventory.py +++ b/tools/tests/test_released_sdk_inventory.py @@ -218,12 +218,10 @@ def test_notice_request_selection_and_no_descriptor_path_extension(self): M.notice_requests(runtime, []) def test_archive_crosschecks(self): - runtime, _ = self.documents() - contract = {'sources': [{'sha256': 'b' * 64}]} - records = {'sources/debian/foo.tar.xz': {'sha256': 'a' * 64}, - 'sources/' + 'b' * 64 + '.tar.gz': {'sha256': 'b' * 64}} + payloads, runtime, contract = self.manifest_fixture() + records = self.producer_fixture(payloads) M.crosscheck_archives(records, runtime, contract) - for path in records: + for path in (path for path in records if path.startswith('sources/')): bad = copy.deepcopy(records) bad[path]['sha256'] = 'c' * 64 with self.assertRaisesRegex(M.InventoryError, 'inventory-source-bytes-mismatch'): @@ -238,6 +236,8 @@ def manifest_fixture(self): contract = json.loads((ROOT / 'portable/contract.json').read_text()) audio = json.loads((ROOT / 'audio-toolchain.json').read_text()) llvm = json.loads((ROOT / 'llvm-toolchain.json').read_text()) + curl = json.loads((ROOT / 'portable/curl-toolchain.json').read_text()) + shared = json.loads((ROOT / 'classic-curl-toolchain.json').read_text()) payloads = {'sources/debian/foo.tar.xz': b'debian source'} runtime = {'archives': {'foo.tar.xz': hashlib.sha256(b'debian source').hexdigest()}} for entry in contract['sources']: @@ -252,6 +252,24 @@ def manifest_fixture(self): payloads['sources/llvm/' + Path(llvm['metadata']['source_archive']).name] = b'llvm source' payloads['audio-toolchain.json'] = json.dumps(audio).encode() payloads['llvm-toolchain.json'] = json.dumps(llvm).encode() + provider_sources = {'libcurl': {}} + for name in curl['archives']: + raw = ('source ' + name).encode() + sha = hashlib.sha256(raw).hexdigest() + curl['archives'][name] = sha + payloads['sources/curl/' + name] = raw + provider_sources['libcurl']['sources/curl/' + name] = sha + sha = hashlib.sha256(b'cares source').hexdigest() + shared['cares']['sha256'] = sha + payloads['sources/' + sha + '.tar.gz'] = b'cares source' + provider_sources['libcares'] = {'sources/' + sha + '.tar.gz': sha} + runtime['custom_providers'] = { + name: {'path': '/usr/local/lib/' + name + '.so.1.0.0', + 'sha256': hashlib.sha256(name.encode()).hexdigest(), 'version': version, + 'source_archives': provider_sources[name], 'build_receipt': 'curl-source-receipt.json'} + for name, version in (('libcurl', curl['version']), ('libcares', shared['cares']['version']))} + payloads['curl-toolchain.json'] = json.dumps(curl).encode() + payloads['classic-curl-toolchain.json'] = json.dumps(shared).encode() return payloads, runtime, contract def producer_fixture(self, payloads): @@ -275,6 +293,206 @@ def test_authenticated_manifests_close_audio_and_llvm_sources(self): with self.assertRaisesRegex(M.InventoryError, 'inventory-manifest-missing-or-size'): M.crosscheck_archives(dict(records), runtime, contract) + def test_contract_schema_and_manifest_references_are_mandatory(self): + payloads, runtime, contract = self.manifest_fixture() + for key in ('audio_manifest', 'curl_manifest', 'compiler'): + bad = copy.deepcopy(contract) + del bad[key] + # Removing both the declaration and all its archives cannot reduce + # the qualification requirements to the remaining subset. + partial = {name: raw for name, raw in payloads.items() if not name.startswith('sources/')} + with self.subTest(missing=key), self.assertRaises(M.InventoryError): + M.crosscheck_archives(self.producer_fixture(partial), runtime, bad) + bad = copy.deepcopy(contract) + del bad['compiler']['toolchain_manifest'] + with self.assertRaisesRegex(M.InventoryError, 'inventory-manifest-path'): + M.crosscheck_archives(self.producer_fixture(payloads), runtime, bad) + for key, values in (('schema_version', [None, True, 1, 2.0, '2']), + ('target', [None, 'classic-final']), ('platform', [None, 'linux/arm64']), + ('compiler', [None, [], 'llvm-toolchain.json'])): + for value in values: + bad = copy.deepcopy(contract) + bad[key] = value + with self.subTest(key=key, value=value), self.assertRaises(M.InventoryError): + M.crosscheck_archives(self.producer_fixture(payloads), runtime, bad) + + def test_contract_source_cohort_cannot_be_shortened_or_duplicated(self): + payloads, runtime, contract = self.manifest_fixture() + for index in range(len(contract['sources'])): + bad = copy.deepcopy(contract) + removed = bad['sources'].pop(index) + partial = dict(payloads) + del partial['sources/' + removed['sha256'] + '.tar.gz'] + with self.subTest(removed=removed['name']), self.assertRaisesRegex( + M.InventoryError, 'inventory-source-cohort'): + M.crosscheck_archives(self.producer_fixture(partial), runtime, bad) + for rows in ([*contract['sources'], contract['sources'][0]], + [{**row, 'name': 'other'} for row in contract['sources']], [], None, {}): + bad = {**contract, 'sources': rows} + with self.subTest(rows=rows), self.assertRaises(M.InventoryError): + M.crosscheck_archives(self.producer_fixture(payloads), runtime, bad) + + def test_audio_cohort_cannot_be_shortened_or_renamed(self): + payloads, runtime, contract = self.manifest_fixture() + original = json.loads(payloads['audio-toolchain.json']) + for index in range(len(original['dependencies'])): + audio = copy.deepcopy(original) + removed = audio['dependencies'].pop(index) + partial = dict(payloads) + del partial['sources/' + removed['source']['sha256'] + '.tar.gz'] + partial['audio-toolchain.json'] = json.dumps(audio).encode() + with self.subTest(removed=removed['name']), self.assertRaises(M.InventoryError): + M.crosscheck_archives(self.producer_fixture(partial), runtime, contract) + for role in ('sdl_mixer', 'dependency'): + audio = copy.deepcopy(original) + entry = audio['sdl_mixer'] if role == 'sdl_mixer' else audio['dependencies'][0] + entry['name'] = 'other' + partial = {**payloads, 'audio-toolchain.json': json.dumps(audio).encode()} + with self.subTest(role=role), self.assertRaises(M.InventoryError): + M.crosscheck_archives(self.producer_fixture(partial), runtime, contract) + + def test_both_custom_providers_and_all_fields_are_mandatory(self): + payloads, runtime, contract = self.manifest_fixture() + variants = [{}] + for value in (None, [], {}, {'other': runtime['custom_providers']['libcurl']}): + variants.append({**runtime, 'custom_providers': value}) + variants.append({key: value for key, value in runtime.items() if key != 'custom_providers'}) + for name in runtime['custom_providers']: + bad = copy.deepcopy(runtime) + del bad['custom_providers'][name] + variants.append(bad) + for field in runtime['custom_providers'][name]: + bad = copy.deepcopy(runtime) + del bad['custom_providers'][name][field] + variants.append(bad) + for field, value in (('path', '/etc/passwd'), ('path', []), ('sha256', True), + ('version', None), ('build_receipt', '../curl-source-receipt.json'), + ('source_archives', None), ('source_archives', []), ('source_archives', {})): + bad = copy.deepcopy(runtime) + bad['custom_providers'][name][field] = value + variants.append(bad) + bad = copy.deepcopy(runtime) + bad['custom_providers']['other'] = copy.deepcopy(bad['custom_providers']['libcurl']) + variants.append(bad) + for bad in variants: + with self.subTest(runtime=bad), self.assertRaises(M.InventoryError): + M.crosscheck_archives(self.producer_fixture(payloads), bad, contract) + for name, provider in runtime['custom_providers'].items(): + bad = copy.deepcopy(runtime) + del bad['custom_providers'][name] + partial = {path: raw for path, raw in payloads.items() if path not in provider['source_archives']} + with self.subTest(missing_provider_and_archives=name), self.assertRaisesRegex( + M.InventoryError, 'inventory-custom-providers'): + M.crosscheck_archives(self.producer_fixture(partial), bad, contract) + + def test_custom_provider_archives_cannot_be_omitted_replaced_or_substituted(self): + payloads, runtime, contract = self.manifest_fixture() + for name, provider in runtime['custom_providers'].items(): + for path in provider['source_archives']: + for change in ('missing', 'replaced', 'omitted-declaration'): + partial, bad = dict(payloads), copy.deepcopy(runtime) + if change == 'replaced': + partial[path] = b'replaced source' + else: + del partial[path] + if change == 'omitted-declaration': + del bad['custom_providers'][name]['source_archives'][path] + with self.subTest(path=path, change=change), self.assertRaises(M.InventoryError): + M.crosscheck_archives(self.producer_fixture(partial), bad, contract) + bad = copy.deepcopy(runtime) + sha = contract['sources'][0]['sha256'] + bad['custom_providers'][name]['source_archives'] = {'sources/' + sha + '.tar.gz': sha} + with self.subTest(substituted=name), self.assertRaisesRegex(M.InventoryError, 'inventory-custom-provider'): + M.crosscheck_archives(self.producer_fixture(payloads), bad, contract) + + def test_curl_manifests_require_complete_archive_set_and_reference_chain(self): + payloads, runtime, contract = self.manifest_fixture() + original = json.loads(payloads['curl-toolchain.json']) + for name in original['archives']: + curl, bad = copy.deepcopy(original), copy.deepcopy(runtime) + del curl['archives'][name] + del bad['custom_providers']['libcurl']['source_archives']['sources/curl/' + name] + partial = {**payloads, 'curl-toolchain.json': json.dumps(curl).encode()} + del partial['sources/curl/' + name] + with self.subTest(archive=name), self.assertRaisesRegex(M.InventoryError, 'inventory-curl-archives'): + M.crosscheck_archives(self.producer_fixture(partial), bad, contract) + for key, value in (('cares_manifest', None), ('cares_manifest', '../classic-curl-toolchain.json'), + ('cares_manifest', '/etc/passwd'), ('source_package', 'other'), + ('source_version', None), ('source_version', '../7.88.1'), + ('version', True), ('archives', [])): + curl = {**original, key: value} + partial = {**payloads, 'curl-toolchain.json': json.dumps(curl).encode()} + with self.subTest(key=key, value=value), self.assertRaises(M.InventoryError): + M.crosscheck_archives(self.producer_fixture(partial), runtime, contract) + shared = json.loads(payloads['classic-curl-toolchain.json']) + for value in (None, [], {}, {'version': True, 'sha256': 'a' * 64}, + {'version': '1.34.6', 'sha256': None}, {'version': '1.34.6', 'sha256': '../escape'}): + bad = {**shared, 'cares': value} + partial = {**payloads, 'classic-curl-toolchain.json': json.dumps(bad).encode()} + with self.subTest(cares=value), self.assertRaises(M.InventoryError): + M.crosscheck_archives(self.producer_fixture(partial), runtime, contract) + + def test_source_document_types_fail_with_fixed_errors(self): + payloads, runtime, contract = self.manifest_fixture() + records = self.producer_fixture(payloads) + for index in range(3): + for value in (None, [], True, 'document'): + args = [records, runtime, contract] + args[index] = value + with self.subTest(index=index, value=value), self.assertRaisesRegex( + M.InventoryError, 'inventory-source-documents'): + M.crosscheck_archives(*args) + + def test_all_declared_archive_records_require_typed_nonempty_bounded_bytes(self): + payloads, runtime, contract = self.manifest_fixture() + records = self.producer_fixture(payloads) + for path in (path for path in records if path.startswith('sources/')): + original = records[path] + malformed = [None, [], 'record', True, {}, {'sha256': original['sha256']}] + malformed += [{**original, 'size': size} for size in (None, False, True, 0, -1, '1', 1.0, M.MAX_FILE + 1)] + malformed += [{**original, 'sha256': sha} for sha in (None, [], 'A' * 64, '0' * 64)] + for record in malformed: + bad = copy.deepcopy(records) + bad[path] = record + with self.subTest(path=path, record=record), self.assertRaisesRegex( + M.InventoryError, 'inventory-source-bytes-mismatch'): + M.crosscheck_archives(bad, runtime, contract) + + def test_matching_empty_archive_digest_is_rejected_for_every_source_family(self): + empty = hashlib.sha256(b'').hexdigest() + for family in ('debian', 'contract', 'audio', 'llvm', 'libcurl', 'libcares'): + payloads, runtime, contract = self.manifest_fixture() + if family == 'debian': + path = 'sources/debian/foo.tar.xz' + runtime['archives']['foo.tar.xz'] = empty + elif family == 'contract': + contract['sources'][0]['sha256'] = empty + path = 'sources/' + empty + '.tar.gz' + elif family in ('audio', 'llvm'): + name = 'audio-toolchain.json' if family == 'audio' else 'llvm-toolchain.json' + manifest = json.loads(payloads[name]) + source = manifest['sdl_mixer']['source'] if family == 'audio' else manifest['source'] + source['sha256'] = empty + payloads[name] = json.dumps(manifest).encode() + path = ('sources/' + empty + '.tar.gz' if family == 'audio' + else 'sources/llvm/' + Path(manifest['metadata']['source_archive']).name) + elif family == 'libcurl': + curl = json.loads(payloads['curl-toolchain.json']) + name = next(iter(curl['archives'])) + curl['archives'][name] = empty + payloads['curl-toolchain.json'] = json.dumps(curl).encode() + path = 'sources/curl/' + name + runtime['custom_providers'][family]['source_archives'][path] = empty + else: + shared = json.loads(payloads['classic-curl-toolchain.json']) + shared['cares']['sha256'] = empty + payloads['classic-curl-toolchain.json'] = json.dumps(shared).encode() + path = 'sources/' + empty + '.tar.gz' + runtime['custom_providers'][family]['source_archives'] = {path: empty} + payloads[path] = b'' + with self.subTest(family=family), self.assertRaisesRegex(M.InventoryError, 'inventory-source-bytes-mismatch'): + M.crosscheck_archives(self.producer_fixture(payloads), runtime, contract) + def test_each_declared_audio_and_llvm_source_missing_or_replaced_fails(self): payloads, runtime, contract = self.manifest_fixture() audio = json.loads(payloads['audio-toolchain.json']) @@ -305,6 +523,10 @@ def test_manifest_missing_replaced_identity_and_limits_fail(self): M.crosscheck_archives(self.producer_fixture(bad), runtime, contract) records = self.producer_fixture(payloads) records.source_manifests[name] += b' ' + with self.assertRaisesRegex(M.InventoryError, 'inventory-manifest-bytes-mismatch'): + M.crosscheck_archives(records, runtime, contract) + records = self.producer_fixture(payloads) + records[name]['size'] = float(records[name]['size']) with self.assertRaisesRegex(M.InventoryError, 'inventory-manifest-bytes-mismatch'): M.crosscheck_archives(records, runtime, contract) for raw in (b'', b' ' * (M.MAX_MANIFEST + 1)): From 502c672344f67d4b76a06598856e8a247f90c64e Mon Sep 17 00:00:00 2001 From: Zoey Rose Date: Mon, 12 Oct 2026 07:09:34 -0500 Subject: [PATCH 16/23] ci: retain scrubbed dependency review job summaries --- .github/workflows/dependency-review.yml | 10 ++++++++++ AGENTS.md | 4 ++++ 2 files changed, 14 insertions(+) diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index 2090a50..31d1282 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -215,3 +215,13 @@ jobs: if __name__ == '__main__': main() + - name: Retain dependency review job summary + if: ${{ always() }} + continue-on-error: true + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: dependency-review-job-summary-${{ github.event.pull_request.head.sha }}-${{ github.run_id }}-${{ github.run_attempt }} + # Retain only the runner's masked copies used for the actual job summary. + path: ${{ runner.temp }}/_runner_file_commands/step_summary_*-scrubbed + retention-days: 7 + if-no-files-found: warn diff --git a/AGENTS.md b/AGENTS.md index 46b48a8..fcd77ca 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -120,6 +120,10 @@ exists; fix actionable introduced risk or record explicit maintainer-accepted existing risk. License metadata does not certify compatibility; existing repository license policies apply. + For programmatic access, read every runner-scrubbed report in the + `dependency-review-job-summary---` artifact after + verifying its digest and latest current-head run/attempt identity; an absent + or unreadable actual report remains unknown, not clean. - Commits and pull-request titles use Conventional Commits. Preserve unrelated work and finish with `git diff --check`. - Update this `AGENTS.md` in the same change when major rework alters image From ebe1b46560aa0b225294e5173680aa4e5262e27f Mon Sep 17 00:00:00 2001 From: Zoey Rose Date: Mon, 12 Oct 2026 09:44:10 -0500 Subject: [PATCH 17/23] build: retarget portable consumer to merged Classic revision --- portable/contract.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/portable/contract.json b/portable/contract.json index 841bbfb..12251d5 100644 --- a/portable/contract.json +++ b/portable/contract.json @@ -60,7 +60,7 @@ "shader_manifest": "classic-shader-toolchain.json", "consumer": { "repository": "atrinik/classic", - "commit": "345fdccd57f79aea933fbde1bb5ae48517a7bd71", + "commit": "bc10e27404319aff3c793aaffcfc57972449deb5", "shader_directory": "/opt/atrinik-portable/shaders", "cmake_argument": "ATRINIK_GPU_SHADER_DIRECTORY", "input_verification": "exact-source-commit-and-shader-input-sha256", From cc6b6edcb6b2eaeb1b3c731140e1b4163860e8d2 Mon Sep 17 00:00:00 2001 From: Zoey Rose Date: Mon, 12 Oct 2026 10:23:57 -0500 Subject: [PATCH 18/23] fix(sdk): bind provider bytes and protect tool ownership (cherry picked from commit e9b6b87796624c09840cc750085bceb446c9ca48) --- tools/collect-released-sdk-metadata.py | 3 +- tools/released_sdk_inventory.py | 84 ++++++- tools/tests/test_released_sdk_inventory.py | 254 ++++++++++++++++++--- 3 files changed, 296 insertions(+), 45 deletions(-) diff --git a/tools/collect-released-sdk-metadata.py b/tools/collect-released-sdk-metadata.py index 5fef1c4..1b96d8f 100644 --- a/tools/collect-released-sdk-metadata.py +++ b/tools/collect-released-sdk-metadata.py @@ -611,7 +611,7 @@ def pax_times(raw): raw = raw[length:] -def single_file_archive(response, path, expected, *, retain, executable=False, allow_empty=False): +def single_file_archive(response, path, expected, *, retain, executable=False, allow_empty=False, root_owned=False): """Read one regular file; reject all links/extensions/path changes, never extract.""" require(response.chunked or response.length is not None, 'unframed-archive') require(response.getheader('Content-Encoding') in (None, 'identity') and @@ -636,6 +636,7 @@ def single_file_archive(response, path, expected, *, retain, executable=False, a regular_mode(info.mode, executable) require(info.mode == expected['mode'] and info.uid >= 0 and info.gid >= 0, 'archive-metadata-mismatch') + require(not root_owned or info.uid == 0, 'archive-owner-mismatch') sha = hashlib.sha256() saved = bytearray() if retain else None remaining = info.size diff --git a/tools/released_sdk_inventory.py b/tools/released_sdk_inventory.py index bfb95cd..afc0697 100644 --- a/tools/released_sdk_inventory.py +++ b/tools/released_sdk_inventory.py @@ -6,6 +6,7 @@ and its archive protocol. This foundation is deliberately not wired to a release target: final producer/consumer/index/platform/config pins are still required. """ +from abc import ABC, abstractmethod import hashlib import json from pathlib import PurePosixPath @@ -40,6 +41,19 @@ class InventoryError(Exception): """Only fixed failure codes; never hostile archive or daemon contents.""" +class ProviderReader(ABC): + """Authenticated byte reader supplied by the source-inventory caller. + + Implementations own admission to their stopped image or protected filesystem, + and fence actual file/ancestor identities around every read. This interface + is not authentication: callers must use the same source for the producer + inventory, metadata and provider bytes, never an advisory metadata adapter. + """ + @abstractmethod + def read_provider(self, path): + """Read a fixed-namespace library; return sha256, size and executable.""" + + class ProducerInventory(dict): """File identities plus bounded source manifests from the same archive. @@ -123,6 +137,10 @@ def header(raw): # slashes before its normalization can hide a noncanonical spelling. require(not (info.isdir() and raw[:100].split(b'\0', 1)[0].endswith(b'//')), 'inventory-path-invalid') + # Moby archives GET / with a raw '/' directory entry. TarInfo strips + # its slash to empty; preserve only this exact canonical root spelling. + if info.isdir() and info.name == '' and raw[:100].split(b'\0', 1)[0] == b'/': + info.name = '/' return info except (tarfile.TarError, ValueError, UnicodeError): raise InventoryError('inventory-header-invalid') from None @@ -269,7 +287,8 @@ def invalid_constant(_): return value -def crosscheck_archives(records, runtime, contract): +def crosscheck_archives(records, runtime, contract, *, reader=None): + """Validate source closure and bind provider bytes using the same image reader.""" require(isinstance(records, dict) and isinstance(runtime, dict) and isinstance(contract, dict), 'inventory-source-documents') require(type(contract.get('schema_version')) is int and contract['schema_version'] == 2 @@ -358,7 +377,7 @@ def add(path, sha): provider = providers[name] require(isinstance(provider, dict) and set(provider) == { 'path', 'sha256', 'version', 'source_archives', 'build_receipt'}, 'inventory-custom-provider') - require(isinstance(provider['path'], str) and re.fullmatch( + require(isinstance(provider['path'], str) and len(provider['path']) <= MAX_PATH and re.fullmatch( r'/usr/local/lib/' + name + r'\.so(?:\.[0-9]+)+', provider['path']) and isinstance(provider['sha256'], str) and re.fullmatch(r'[0-9a-f]{64}', provider['sha256']) and provider['version'] == (version if name == 'libcurl' else cares['version']) @@ -372,6 +391,15 @@ def add(path, sha): require(isinstance(record, dict) and type(record.get('size')) is int and 0 < record['size'] <= MAX_FILE and record.get('sha256') == sha, 'inventory-source-bytes-mismatch') + require(isinstance(reader, ProviderReader), 'inventory-provider-reader-required') + for provider in providers.values(): + observed = reader.read_provider(provider['path']) + require(isinstance(observed, dict) and set(observed) == {'sha256', 'size', 'executable'} + and isinstance(observed['sha256'], str) + and re.fullmatch(r'[0-9a-f]{64}', observed['sha256']) + and type(observed['size']) is int and 0 < observed['size'] <= MAX_FILE + and type(observed['executable']) is bool, 'inventory-provider-identity') + require(observed['sha256'] == provider['sha256'], 'inventory-provider-bytes-mismatch') class BudgetResponse: @@ -386,7 +414,7 @@ def read(self, size): return self.budget.read(self.response, size) -class InstalledInventory: +class InstalledInventory(ProviderReader): """Use only with an authenticated StoppedImage; does not grant release trust.""" def __init__(self, image, protocol): self.image, self.protocol = image, protocol @@ -397,14 +425,39 @@ def _fence(self): self.image.fence() self.budget.check() - def _parents(self, path): + def _directory_owner(self, path, expected): + """Read only the directory header, never its recursive archive contents. + + Docker HEAD omits UID/GID. The first member of its authenticated GET + supplies ownership; close immediately to bound even the root archive. + """ + conn, response = self._get(path) + try: + require(self.protocol.path_stat(response) == expected, 'inventory-path-drift') + framing(response) + info = next_member(response, self.budget, self.protocol) + names = {'/'} if path == '/' else {PurePosixPath(path).name} + require(info is not None and info.name in names and info.isdir() + and info.size == 0 and info.mode == expected['mode'] & 0o777, + 'inventory-directory-entry') + require(info.uid == 0, 'inventory-directory-owner') + finally: + conn.close() + + def _parents(self, path, *, protected=False): result = {} for parent in reversed(PurePosixPath(path).parents): - if str(parent) == '/': + if str(parent) == '/' and not protected: continue self.budget.check() value = self.image.stat(str(parent)) - self.protocol.stat_matches(value, str(parent), directory=True) + if str(parent) == '/': + require(value.get('name') == '/', 'inventory-directory-entry') + self.protocol.stat_matches({**value, 'name': ''}, '/', directory=True) + else: + self.protocol.stat_matches(value, str(parent), directory=True) + if protected: + self._directory_owner(str(parent), value) result[str(parent)] = value return result @@ -434,28 +487,37 @@ def producer(self): self._recheck(observations) return result + def read_provider(self, path): + return self._regular(path, provider=True) + def common_licenses(self): return {name: self._regular(COMMON + '/' + name) for name in COMMON_LICENSE_NAMES} - def _regular(self, path, *, executable=False): - # Internal only: callers are the fixed licenses and verified doc resolver. - require(path in {COMMON + '/' + name for name in COMMON_LICENSE_NAMES} + def _regular(self, path, *, executable=False, provider=False): + # Provider paths have already passed source-manifest validation; enforce + # the fixed library namespace again at the actual archive request. + provider_path = isinstance(path, str) and len(path) <= MAX_PATH and re.fullmatch( + r'/usr/local/lib/lib(?:curl|cares)\.so(?:\.[0-9]+)+', path) is not None + require(not provider or provider_path, 'inventory-file-not-allowed') + require((provider and provider_path) or path in {COMMON + '/' + name for name in COMMON_LICENSE_NAMES} or path in BUILD_TOOLS.values() or path.startswith(DOC + '/'), 'inventory-file-not-allowed') if path.startswith(DOC + '/'): under_doc(path) self._fence() - observations = self._parents(path) + protected = path in BUILD_TOOLS.values() or provider expected = self.image.stat(path) self.protocol.stat_matches(expected, path) self.protocol.regular_mode(expected['mode'], executable=executable) + require(0 < expected['size'] <= self.protocol.BINARY_LIMIT, 'file-size-limit') + observations = self._parents(path, protected=protected) observations[path] = expected self.budget.member(expected['size']) conn, response = self._get(path) try: require(self.protocol.path_stat(response) == expected, 'inventory-path-drift') identity, _ = self.protocol.single_file_archive( - BudgetResponse(response, self.budget), path, expected, retain=False, executable=executable) + BudgetResponse(response, self.budget), path, expected, retain=False, executable=executable, root_owned=protected) self.budget.check() finally: conn.close() diff --git a/tools/tests/test_released_sdk_inventory.py b/tools/tests/test_released_sdk_inventory.py index 6f8ff26..7b2487a 100644 --- a/tools/tests/test_released_sdk_inventory.py +++ b/tools/tests/test_released_sdk_inventory.py @@ -42,7 +42,7 @@ def getheader(self, name, default=None): def directory(path): - return {'name': Path(path).name, 'mode': M.GO_DIRECTORY | 0o755, + return {'name': Path(path).name if path != '/' else '/', 'mode': M.GO_DIRECTORY | 0o755, 'size': 4096, 'mtime': 'fixed', 'linkTarget': ''} @@ -169,6 +169,40 @@ def fake_image(self): image.stat.side_effect = lambda path: directory(path) return image + def installed_image(self, payloads=None, metadata=None, missing=()): + """Exercise real archive readers; HEAD intentionally has no UID/GID.""" + payloads = payloads if payloads is not None else { + '/usr/local/lib/' + name + '.so.1.0.0': name.encode() + for name in ('libcurl', 'libcares')} + metadata = metadata or {} + image = self.fake_image() + connections = [] + def stat(path): + if path in missing: + raise P.Stop('docker-http-status') + if path in payloads: + return {**regular(path, payloads[path]), 'mode': metadata.get(path, {}).get('mode', 0o755)} + return directory(path) + def request(method, route, headers): + self.assertEqual(method, 'GET') + path = urllib.parse.parse_qs(urllib.parse.urlsplit(route).query)['path'][0] + expected = stat(path) + conn = Mock() + connections.append(conn) + raw = payloads.get(path, b'') + kind = tarfile.REGTYPE if path in payloads else tarfile.DIRTYPE + extra = {'mode': expected['mode'] & 0o777, **metadata.get(path, {})} + name = Path(path).name if path != '/' else '/' + return conn, Response(tar([(name, raw, kind, extra)]), expected) + image.stat.side_effect = stat + image.docker.request.side_effect = request + image.connections = connections + return image + + def crosscheck(self, records, runtime, contract, *, image=None): + reader = M.InstalledInventory(image if image is not None else self.installed_image(), P) + return M.crosscheck_archives(records, runtime, contract, reader=reader) + def test_created_image_fences_and_connection_close(self): image = self.fake_image() conn = Mock() @@ -220,15 +254,15 @@ def test_notice_request_selection_and_no_descriptor_path_extension(self): def test_archive_crosschecks(self): payloads, runtime, contract = self.manifest_fixture() records = self.producer_fixture(payloads) - M.crosscheck_archives(records, runtime, contract) + self.crosscheck(records, runtime, contract) for path in (path for path in records if path.startswith('sources/')): bad = copy.deepcopy(records) bad[path]['sha256'] = 'c' * 64 with self.assertRaisesRegex(M.InventoryError, 'inventory-source-bytes-mismatch'): - M.crosscheck_archives(bad, runtime, contract) + self.crosscheck(bad, runtime, contract) runtime['archives'] = {'../secret': 'a' * 64} with self.assertRaises(M.InventoryError): - M.crosscheck_archives(records, runtime, contract) + self.crosscheck(records, runtime, contract) def manifest_fixture(self): # Use the producer's actual schemas and path declarations, with small @@ -286,12 +320,12 @@ def producer_fixture(self, payloads): def test_authenticated_manifests_close_audio_and_llvm_sources(self): payloads, runtime, contract = self.manifest_fixture() records = self.producer_fixture(payloads) - M.crosscheck_archives(records, runtime, contract) + self.crosscheck(records, runtime, contract) self.assertEqual(set(records.source_manifests), set(M.SOURCE_MANIFESTS)) self.assertEqual(M.inventory_digest(records), M.inventory_digest(dict(records))) # A hash-only inventory cannot manufacture authenticated manifest data. with self.assertRaisesRegex(M.InventoryError, 'inventory-manifest-missing-or-size'): - M.crosscheck_archives(dict(records), runtime, contract) + self.crosscheck(dict(records), runtime, contract) def test_contract_schema_and_manifest_references_are_mandatory(self): payloads, runtime, contract = self.manifest_fixture() @@ -302,11 +336,11 @@ def test_contract_schema_and_manifest_references_are_mandatory(self): # the qualification requirements to the remaining subset. partial = {name: raw for name, raw in payloads.items() if not name.startswith('sources/')} with self.subTest(missing=key), self.assertRaises(M.InventoryError): - M.crosscheck_archives(self.producer_fixture(partial), runtime, bad) + self.crosscheck(self.producer_fixture(partial), runtime, bad) bad = copy.deepcopy(contract) del bad['compiler']['toolchain_manifest'] with self.assertRaisesRegex(M.InventoryError, 'inventory-manifest-path'): - M.crosscheck_archives(self.producer_fixture(payloads), runtime, bad) + self.crosscheck(self.producer_fixture(payloads), runtime, bad) for key, values in (('schema_version', [None, True, 1, 2.0, '2']), ('target', [None, 'classic-final']), ('platform', [None, 'linux/arm64']), ('compiler', [None, [], 'llvm-toolchain.json'])): @@ -314,7 +348,7 @@ def test_contract_schema_and_manifest_references_are_mandatory(self): bad = copy.deepcopy(contract) bad[key] = value with self.subTest(key=key, value=value), self.assertRaises(M.InventoryError): - M.crosscheck_archives(self.producer_fixture(payloads), runtime, bad) + self.crosscheck(self.producer_fixture(payloads), runtime, bad) def test_contract_source_cohort_cannot_be_shortened_or_duplicated(self): payloads, runtime, contract = self.manifest_fixture() @@ -325,12 +359,12 @@ def test_contract_source_cohort_cannot_be_shortened_or_duplicated(self): del partial['sources/' + removed['sha256'] + '.tar.gz'] with self.subTest(removed=removed['name']), self.assertRaisesRegex( M.InventoryError, 'inventory-source-cohort'): - M.crosscheck_archives(self.producer_fixture(partial), runtime, bad) + self.crosscheck(self.producer_fixture(partial), runtime, bad) for rows in ([*contract['sources'], contract['sources'][0]], [{**row, 'name': 'other'} for row in contract['sources']], [], None, {}): bad = {**contract, 'sources': rows} with self.subTest(rows=rows), self.assertRaises(M.InventoryError): - M.crosscheck_archives(self.producer_fixture(payloads), runtime, bad) + self.crosscheck(self.producer_fixture(payloads), runtime, bad) def test_audio_cohort_cannot_be_shortened_or_renamed(self): payloads, runtime, contract = self.manifest_fixture() @@ -342,14 +376,14 @@ def test_audio_cohort_cannot_be_shortened_or_renamed(self): del partial['sources/' + removed['source']['sha256'] + '.tar.gz'] partial['audio-toolchain.json'] = json.dumps(audio).encode() with self.subTest(removed=removed['name']), self.assertRaises(M.InventoryError): - M.crosscheck_archives(self.producer_fixture(partial), runtime, contract) + self.crosscheck(self.producer_fixture(partial), runtime, contract) for role in ('sdl_mixer', 'dependency'): audio = copy.deepcopy(original) entry = audio['sdl_mixer'] if role == 'sdl_mixer' else audio['dependencies'][0] entry['name'] = 'other' partial = {**payloads, 'audio-toolchain.json': json.dumps(audio).encode()} with self.subTest(role=role), self.assertRaises(M.InventoryError): - M.crosscheck_archives(self.producer_fixture(partial), runtime, contract) + self.crosscheck(self.producer_fixture(partial), runtime, contract) def test_both_custom_providers_and_all_fields_are_mandatory(self): payloads, runtime, contract = self.manifest_fixture() @@ -376,14 +410,14 @@ def test_both_custom_providers_and_all_fields_are_mandatory(self): variants.append(bad) for bad in variants: with self.subTest(runtime=bad), self.assertRaises(M.InventoryError): - M.crosscheck_archives(self.producer_fixture(payloads), bad, contract) + self.crosscheck(self.producer_fixture(payloads), bad, contract) for name, provider in runtime['custom_providers'].items(): bad = copy.deepcopy(runtime) del bad['custom_providers'][name] partial = {path: raw for path, raw in payloads.items() if path not in provider['source_archives']} with self.subTest(missing_provider_and_archives=name), self.assertRaisesRegex( M.InventoryError, 'inventory-custom-providers'): - M.crosscheck_archives(self.producer_fixture(partial), bad, contract) + self.crosscheck(self.producer_fixture(partial), bad, contract) def test_custom_provider_archives_cannot_be_omitted_replaced_or_substituted(self): payloads, runtime, contract = self.manifest_fixture() @@ -398,12 +432,12 @@ def test_custom_provider_archives_cannot_be_omitted_replaced_or_substituted(self if change == 'omitted-declaration': del bad['custom_providers'][name]['source_archives'][path] with self.subTest(path=path, change=change), self.assertRaises(M.InventoryError): - M.crosscheck_archives(self.producer_fixture(partial), bad, contract) + self.crosscheck(self.producer_fixture(partial), bad, contract) bad = copy.deepcopy(runtime) sha = contract['sources'][0]['sha256'] bad['custom_providers'][name]['source_archives'] = {'sources/' + sha + '.tar.gz': sha} with self.subTest(substituted=name), self.assertRaisesRegex(M.InventoryError, 'inventory-custom-provider'): - M.crosscheck_archives(self.producer_fixture(payloads), bad, contract) + self.crosscheck(self.producer_fixture(payloads), bad, contract) def test_curl_manifests_require_complete_archive_set_and_reference_chain(self): payloads, runtime, contract = self.manifest_fixture() @@ -415,7 +449,7 @@ def test_curl_manifests_require_complete_archive_set_and_reference_chain(self): partial = {**payloads, 'curl-toolchain.json': json.dumps(curl).encode()} del partial['sources/curl/' + name] with self.subTest(archive=name), self.assertRaisesRegex(M.InventoryError, 'inventory-curl-archives'): - M.crosscheck_archives(self.producer_fixture(partial), bad, contract) + self.crosscheck(self.producer_fixture(partial), bad, contract) for key, value in (('cares_manifest', None), ('cares_manifest', '../classic-curl-toolchain.json'), ('cares_manifest', '/etc/passwd'), ('source_package', 'other'), ('source_version', None), ('source_version', '../7.88.1'), @@ -423,14 +457,14 @@ def test_curl_manifests_require_complete_archive_set_and_reference_chain(self): curl = {**original, key: value} partial = {**payloads, 'curl-toolchain.json': json.dumps(curl).encode()} with self.subTest(key=key, value=value), self.assertRaises(M.InventoryError): - M.crosscheck_archives(self.producer_fixture(partial), runtime, contract) + self.crosscheck(self.producer_fixture(partial), runtime, contract) shared = json.loads(payloads['classic-curl-toolchain.json']) for value in (None, [], {}, {'version': True, 'sha256': 'a' * 64}, {'version': '1.34.6', 'sha256': None}, {'version': '1.34.6', 'sha256': '../escape'}): bad = {**shared, 'cares': value} partial = {**payloads, 'classic-curl-toolchain.json': json.dumps(bad).encode()} with self.subTest(cares=value), self.assertRaises(M.InventoryError): - M.crosscheck_archives(self.producer_fixture(partial), runtime, contract) + self.crosscheck(self.producer_fixture(partial), runtime, contract) def test_source_document_types_fail_with_fixed_errors(self): payloads, runtime, contract = self.manifest_fixture() @@ -441,7 +475,7 @@ def test_source_document_types_fail_with_fixed_errors(self): args[index] = value with self.subTest(index=index, value=value), self.assertRaisesRegex( M.InventoryError, 'inventory-source-documents'): - M.crosscheck_archives(*args) + self.crosscheck(*args) def test_all_declared_archive_records_require_typed_nonempty_bounded_bytes(self): payloads, runtime, contract = self.manifest_fixture() @@ -456,7 +490,7 @@ def test_all_declared_archive_records_require_typed_nonempty_bounded_bytes(self) bad[path] = record with self.subTest(path=path, record=record), self.assertRaisesRegex( M.InventoryError, 'inventory-source-bytes-mismatch'): - M.crosscheck_archives(bad, runtime, contract) + self.crosscheck(bad, runtime, contract) def test_matching_empty_archive_digest_is_rejected_for_every_source_family(self): empty = hashlib.sha256(b'').hexdigest() @@ -491,7 +525,7 @@ def test_matching_empty_archive_digest_is_rejected_for_every_source_family(self) runtime['custom_providers'][family]['source_archives'] = {path: empty} payloads[path] = b'' with self.subTest(family=family), self.assertRaisesRegex(M.InventoryError, 'inventory-source-bytes-mismatch'): - M.crosscheck_archives(self.producer_fixture(payloads), runtime, contract) + self.crosscheck(self.producer_fixture(payloads), runtime, contract) def test_each_declared_audio_and_llvm_source_missing_or_replaced_fails(self): payloads, runtime, contract = self.manifest_fixture() @@ -511,7 +545,7 @@ def test_each_declared_audio_and_llvm_source_missing_or_replaced_fails(self): self.assertEqual(M.inventory_digest(records), M.inventory_digest(self.producer_fixture(bad))) with self.subTest(path=path, missing=missing), self.assertRaisesRegex( M.InventoryError, 'inventory-source-bytes-mismatch'): - M.crosscheck_archives(records, runtime, contract) + self.crosscheck(records, runtime, contract) def test_manifest_missing_replaced_identity_and_limits_fail(self): payloads, runtime, contract = self.manifest_fixture() @@ -520,15 +554,15 @@ def test_manifest_missing_replaced_identity_and_limits_fail(self): del bad[name] with self.subTest(name=name), self.assertRaisesRegex( M.InventoryError, 'inventory-manifest-missing-or-size'): - M.crosscheck_archives(self.producer_fixture(bad), runtime, contract) + self.crosscheck(self.producer_fixture(bad), runtime, contract) records = self.producer_fixture(payloads) records.source_manifests[name] += b' ' with self.assertRaisesRegex(M.InventoryError, 'inventory-manifest-bytes-mismatch'): - M.crosscheck_archives(records, runtime, contract) + self.crosscheck(records, runtime, contract) records = self.producer_fixture(payloads) records[name]['size'] = float(records[name]['size']) with self.assertRaisesRegex(M.InventoryError, 'inventory-manifest-bytes-mismatch'): - M.crosscheck_archives(records, runtime, contract) + self.crosscheck(records, runtime, contract) for raw in (b'', b' ' * (M.MAX_MANIFEST + 1)): with self.subTest(name=name, size=len(raw)), self.assertRaisesRegex( M.InventoryError, 'inventory-manifest-size'): @@ -544,7 +578,7 @@ def test_manifest_json_duplicates_types_and_depth_fail(self): bad = dict(payloads) bad[name] = raw with self.subTest(name=name, raw=raw[:80]), self.assertRaises(M.InventoryError): - M.crosscheck_archives(self.producer_fixture(bad), runtime, contract) + self.crosscheck(self.producer_fixture(bad), runtime, contract) def test_manifest_references_cannot_escape_fixed_boundary(self): payloads, runtime, contract = self.manifest_fixture() @@ -559,7 +593,7 @@ def test_manifest_references_cannot_escape_fixed_boundary(self): bad['compiler']['toolchain_manifest'] = reference with self.subTest(role=role, reference=reference), self.assertRaisesRegex( M.InventoryError, 'inventory-manifest-path'): - M.crosscheck_archives(self.producer_fixture(payloads), runtime, bad) + self.crosscheck(self.producer_fixture(payloads), runtime, bad) def test_audio_manifest_invalid_sources_and_duplicate_names_fail(self): payloads, runtime, contract = self.manifest_fixture() @@ -583,7 +617,7 @@ def test_audio_manifest_invalid_sources_and_duplicate_names_fail(self): bad = dict(payloads) bad['audio-toolchain.json'] = json.dumps(value).encode() with self.subTest(value=value), self.assertRaises(M.InventoryError): - M.crosscheck_archives(self.producer_fixture(bad), runtime, contract) + self.crosscheck(self.producer_fixture(bad), runtime, contract) def test_llvm_manifest_invalid_sources_and_archive_paths_fail(self): payloads, runtime, contract = self.manifest_fixture() @@ -602,7 +636,7 @@ def test_llvm_manifest_invalid_sources_and_archive_paths_fail(self): bad = dict(payloads) bad['llvm-toolchain.json'] = json.dumps(llvm).encode() with self.subTest(key=key, value=value), self.assertRaises(M.InventoryError): - M.crosscheck_archives(self.producer_fixture(bad), runtime, contract) + self.crosscheck(self.producer_fixture(bad), runtime, contract) def notice_image(self, links=None, drift=False): @@ -690,12 +724,12 @@ def test_empty_inventory_notices_and_build_tools_are_rejected(self): conn, Response(tar([(Path(path).name, b'', tarfile.REGTYPE, {'mode': expected['mode']})]), expected)) reader = M.InstalledInventory(image, P) - with self.subTest(path=path), self.assertRaisesRegex(P.Stop, 'file-size-limit'): + with self.subTest(path=path), self.assertRaisesRegex(M.InventoryError, 'file-size-limit'): if path.startswith(M.DOC + '/'): reader.debian_notices(runtime, debian) else: reader._regular(path, executable=executable) - conn.close.assert_called_once() + image.docker.request.assert_not_called() def build_receipt(self): entries = {role: {'path': path, 'resolved_path': path, 'sha256': hashlib.sha256(b'tool').hexdigest(), @@ -706,6 +740,160 @@ def build_receipt(self): return {'build_tools': {'schema_version': 1, 'tools': entries}, 'tools': {role: entry['version'].splitlines()[0] for role, entry in entries.items()}} + def test_provider_identity_requires_authenticated_reader_and_actual_bytes(self): + payloads, runtime, contract = self.manifest_fixture() + records = self.producer_fixture(payloads) + with self.assertRaisesRegex(M.InventoryError, 'inventory-provider-reader-required'): + M.crosscheck_archives(records, runtime, contract) + image = self.installed_image() + self.crosscheck(records, runtime, contract, image=image) + requests = [urllib.parse.parse_qs(urllib.parse.urlsplit(call.args[1]).query)['path'][0] + for call in image.docker.request.call_args_list] + self.assertEqual(set(requests), {'/', '/usr', '/usr/local', '/usr/local/lib', + *[item['path'] for item in runtime['custom_providers'].values()]}) + for conn in image.connections: + conn.close.assert_called_once() + for name, item in runtime['custom_providers'].items(): + with self.subTest(provider=name): + bad = copy.deepcopy(runtime) + bad['custom_providers'][name]['sha256'] = 'a' * 64 + with self.assertRaisesRegex(M.InventoryError, 'inventory-provider-bytes-mismatch'): + self.crosscheck(records, bad, contract) + absent = copy.deepcopy(runtime) + absent['custom_providers'][name]['path'] += '.123' + with self.assertRaisesRegex(P.Stop, 'docker-http-status'): + self.crosscheck(records, absent, contract, + image=self.installed_image(missing=(item['path'] + '.123',))) + data = {entry['path']: key.encode() for key, entry in runtime['custom_providers'].items()} + data[item['path']] = b'replaced installed binary' + with self.assertRaisesRegex(M.InventoryError, 'inventory-provider-bytes-mismatch'): + self.crosscheck(records, runtime, contract, image=self.installed_image(data)) + + def test_provider_reader_boundary_rejects_missing_reader_and_invalid_identities(self): + payloads, runtime, contract = self.manifest_fixture() + records = self.producer_fixture(payloads) + for reader in (None, {}, {'sha256': 'a' * 64}, object()): + with self.subTest(reader=reader), self.assertRaisesRegex( + M.InventoryError, 'inventory-provider-reader-required'): + M.crosscheck_archives(records, runtime, contract, reader=reader) + class InvalidReader(M.ProviderReader): + def __init__(self, value): + self.value = value + def read_provider(self, path): + return self.value + valid = {'sha256': hashlib.sha256(b'libcurl').hexdigest(), 'size': 7, 'executable': True} + for value in (None, [], {}, {**valid, 'extra': True}, {**valid, 'size': True}, + {**valid, 'size': 0}, {**valid, 'size': M.MAX_FILE + 1}, + {**valid, 'sha256': 'A' * 64}, {**valid, 'executable': 1}): + with self.subTest(value=value), self.assertRaisesRegex(M.InventoryError, 'inventory-provider-identity'): + M.crosscheck_archives(records, runtime, contract, reader=InvalidReader(value)) + + def test_provider_reader_keeps_fixed_namespace_and_no_link_defenses(self): + for path in ('/etc/passwd', '/usr/local/lib/libcurl.so', '/usr/local/lib/libcurl.so.1/extra', + '/usr/local/lib/libcares.so.1/../elsewhere', '/usr/local/lib/libcurl.so.' + '1' * M.MAX_PATH): + image = self.installed_image() + with self.subTest(path=path), self.assertRaisesRegex(M.InventoryError, 'inventory-file-not-allowed'): + M.InstalledInventory(image, P).read_provider(path) + image.docker.request.assert_not_called() + payloads, runtime, contract = self.manifest_fixture() + records = self.producer_fixture(payloads) + for provider in runtime['custom_providers'].values(): + image = self.installed_image(metadata={provider['path']: {'type': tarfile.SYMTYPE, + 'linkname': 'other.so'}}) + with self.subTest(path=provider['path']), self.assertRaisesRegex(P.Stop, 'unexpected-archive-entry'): + self.crosscheck(records, runtime, contract, image=image) + + def test_provider_files_and_ancestors_are_root_owned(self): + payloads, runtime, contract = self.manifest_fixture() + records = self.producer_fixture(payloads) + paths = ('/', '/usr', '/usr/local', '/usr/local/lib', + *[entry['path'] for entry in runtime['custom_providers'].values()]) + for path in paths: + image = self.installed_image(metadata={path: {'uid': 1000, 'gid': 1000}}) + with self.subTest(path=path), self.assertRaisesRegex( + (M.InventoryError, P.Stop), 'inventory-directory-owner|archive-owner-mismatch'): + self.crosscheck(records, runtime, contract, image=image) + for conn in image.connections: + conn.close.assert_called_once() + + def test_build_tools_real_reader_protects_file_and_all_ancestors(self): + data = {path: b'tool' for path in M.BUILD_TOOLS.values()} + installed = self.build_receipt() + image = self.installed_image(data) + self.assertEqual(M.InstalledInventory(image, P).build_tools(installed), installed['build_tools']['tools']) + for conn in image.connections: + conn.close.assert_called_once() + for path in ('/', '/usr', '/usr/bin', *M.BUILD_TOOLS.values()): + image = self.installed_image(data, {path: {'uid': 1000, 'gid': 1000}}) + with self.subTest(path=path), self.assertRaisesRegex( + (M.InventoryError, P.Stop), 'inventory-directory-owner|archive-owner-mismatch'): + M.InstalledInventory(image, P).build_tools(installed) + for conn in image.connections: + conn.close.assert_called_once() + # The portable consumer requires UID 0; GID 0 is not part of that contract. + image = self.installed_image(data, {path: {'gid': 1000} for path in ('/', '/usr', '/usr/bin', *data)}) + self.assertEqual(M.InstalledInventory(image, P).build_tools(installed), installed['build_tools']['tools']) + for role, path in M.BUILD_TOOLS.items(): + changed = {**data, path: b'changed tool'} + with self.subTest(role=role), self.assertRaisesRegex(M.InventoryError, 'build-tool-bytes-mismatch'): + M.InstalledInventory(self.installed_image(changed), P).build_tools(installed) + + def test_protected_directory_header_rejects_ownership_overrides_and_path_drift(self): + data = {path: b'tool' for path in M.BUILD_TOOLS.values()} + for path in ('/', '/usr', '/usr/bin'): + for extra in ({'mode': 0o777}, {'type': tarfile.SYMTYPE, 'linkname': 'elsewhere'}, + {'name': '../escape'}, {'pax_headers': {'uid': '0'}, 'uid': 1000}, + {'pax_headers': {'gid': '0'}}, {'uid': -1}): + image = self.installed_image(data, {path: extra}) + with self.subTest(path=path, extra=extra), self.assertRaises((M.InventoryError, P.Stop)): + M.InstalledInventory(image, P).build_tools(self.build_receipt()) + for conn in image.connections: + conn.close.assert_called_once() + image = self.installed_image(data) + request = image.docker.request.side_effect + def drift(method, route, headers): + conn, response = request(method, route, headers) + if urllib.parse.parse_qs(urllib.parse.urlsplit(route).query)['path'] == ['/usr']: + response.headers['X-Docker-Container-Path-Stat'] = base64.b64encode( + json.dumps({**directory('/usr'), 'mtime': 'changed'}).encode()).decode() + return conn, response + image.docker.request.side_effect = drift + with self.assertRaisesRegex(M.InventoryError, 'inventory-path-drift'): + M.InstalledInventory(image, P).build_tools(self.build_receipt()) + + def test_root_directory_requires_exact_moby_archive_name(self): + data = {path: b'tool' for path in M.BUILD_TOOLS.values()} + for name in ('.', './', '//', '///', 'usr'): + image = self.installed_image(data, {'/': {'name': name}}) + with self.subTest(name=name), self.assertRaises(M.InventoryError): + M.InstalledInventory(image, P).build_tools(self.build_receipt()) + raw = bytearray(tar([('/', b'', tarfile.DIRTYPE, {})])) + raw[:100] = bytes(100) + raw[148:156] = b' ' + raw[148:156] = ('%06o\0 ' % sum(raw[:512])).encode() + image = self.fake_image() + image.docker.request.return_value = Mock(), Response(bytes(raw), directory('/')) + with self.assertRaisesRegex(M.InventoryError, 'inventory-directory-entry'): + M.InstalledInventory(image, P)._directory_owner('/', directory('/')) + image = self.installed_image(data, {'/': {'name': '/'}}) + self.assertEqual(M.InstalledInventory(image, P).build_tools(self.build_receipt()), + self.build_receipt()['build_tools']['tools']) + + def test_directory_ownership_read_is_bounded_to_header(self): + image = self.fake_image() + # Simulate a huge recursive directory response. Do not request body bytes. + response = Response(tar([('usr', b'', tarfile.DIRTYPE, {})]), directory('/usr')) + original = response.read + def read(size): + self.assertEqual(response.tell(), 0) + self.assertEqual(size, 512) + return original(size) + response.read = read + conn = Mock() + image.docker.request.return_value = conn, response + M.InstalledInventory(image, P)._directory_owner('/usr', directory('/usr')) + conn.close.assert_called_once() + def test_build_tools_accept_full_stdout_with_or_without_final_newline(self): for cmake, ninja in (('cmake version 3.25.1', '1.11'), ('cmake version 3.25.1\n\nCMake suite maintained and supported by Kitware.', From 90cac14c58226ad2506ef35e240434d9076231d8 Mon Sep 17 00:00:00 2001 From: Zoey Rose Date: Mon, 12 Oct 2026 11:26:50 -0500 Subject: [PATCH 19/23] fix(sdk): fence published Linux image identity (cherry picked from commit 612147d0f8d0d6e86e40ca127a978e5c8b97b0e4) --- .github/workflows/publish-linux.yml | 107 +++- AGENTS.md | 14 + tools/tests/test_verify_published_identity.py | 533 ++++++++++++++++++ tools/tests/test_workflow_contracts.py | 118 +++- tools/verify_published_identity.py | 362 ++++++++++++ 5 files changed, 1112 insertions(+), 22 deletions(-) create mode 100644 tools/tests/test_verify_published_identity.py create mode 100644 tools/verify_published_identity.py diff --git a/.github/workflows/publish-linux.yml b/.github/workflows/publish-linux.yml index 37456a2..5192f1d 100644 --- a/.github/workflows/publish-linux.yml +++ b/.github/workflows/publish-linux.yml @@ -24,6 +24,8 @@ jobs: publish: runs-on: ubuntu-26.04 timeout-minutes: 360 + env: + CANDIDATE_ONLY: ${{ inputs.candidate_only }} steps: - name: Check out image sources uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -55,6 +57,9 @@ jobs: { echo "image=${image}" + prefix= + if [[ "${CANDIDATE_ONLY}" == true ]]; then prefix=candidate-; fi + echo "publish_tag=${image}:${prefix}sha-${GITHUB_SHA}" echo 'tags<> "${GITHUB_STEP_SUMMARY}" + set -euo pipefail + images=(linux-build classic-build classic-portable-build) + platforms=(ubuntu-26.04 ubuntu-26.04 debian-12) + digests=("${BROAD_DIGEST}" "${CLASSIC_DIGEST}" "${PORTABLE_DIGEST}") + for digest in "${digests[@]}"; do [[ "${digest}" =~ ^sha256:[0-9a-f]{64}$ ]]; done + for index in 0 1 2; do + image="ghcr.io/${GITHUB_REPOSITORY_OWNER,,}/${images[index]}" + tags=(--tag "${image}:latest" --tag "${image}:${platforms[index]}") + if [[ "${GITHUB_REF_TYPE}" == tag ]]; then + [[ "${GITHUB_REF_NAME}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] + tags+=(--tag "${image}:${GITHUB_REF_NAME#v}") + fi + if ! docker buildx imagetools create "${tags[@]}" "${image}@${digests[index]}"; then + echo "Alias promotion incomplete. Rerun the failed job in this workflow run to converge all aliases." >&2 + exit 1 + fi + printf '%s@%s\n' "${image}" "${digests[index]}" >> "${GITHUB_STEP_SUMMARY}" + done + + - name: Retain failed identity evidence + if: ${{ failure() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: linux-published-identity-failure-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/linux-published-identity/${{ github.run_id }}-${{ github.run_attempt }} + if-no-files-found: error + retention-days: 14 diff --git a/AGENTS.md b/AGENTS.md index fcd77ca..758075a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -85,6 +85,20 @@ `candidate-sha-`. Windows publishes `candidate-sha-` and `classic-check-candidate-sha-` with cross-build and native Windows tests intact. Neither path may move rolling, platform, or version aliases. +- The Linux publisher captures each loaded image's immutable Docker ID and raw + inspection before runtime tests. Curl/TLS, Classic client/server, portable + public-CA and offline consumer checks must execute those captured IDs. After + all runtime gates, publish only SHA candidates, then verify the actual pushed + OCI index, Linux/amd64 manifest and config against the captured RootFS and + complete execution config with `tools/verify_published_identity.py`. Only + exact source/revision publication labels, creation/history and BuildKit inline + cache metadata may differ; unknown raw config and execution fields fail closed. + Keep bounded metadata-only authenticated acquisition confined to the + same-repository hosted publisher. Retain raw inspection, OCI objects, + descriptor relationships and hashed reports as mandatory artifacts before + promoting any release aliases. Rerun the failed job in the same workflow run + to recover partial promotion; preserve historical released-SDK collector + targets and final-cohort gates. - `llvm-toolchain.json`, `llvm-bootstrap-packages.lock`, and the shared LLVM installer/validator own LLVM 23.1.3 Clang/LLD/compiler-rt/libunwind source, licensing, installed metadata, and target contracts. Every affected input diff --git a/tools/tests/test_verify_published_identity.py b/tools/tests/test_verify_published_identity.py new file mode 100644 index 0000000..16e674c --- /dev/null +++ b/tools/tests/test_verify_published_identity.py @@ -0,0 +1,533 @@ +# Copyright (c) 2026 Atrinik contributors +# SPDX-License-Identifier: MIT +"""Offline tests for the Linux publisher's loaded-to-published identity fence.""" +import copy +import importlib.util +import io +import json +import os +from pathlib import Path +import tempfile +import unittest +from unittest.mock import Mock, patch + + +ROOT = Path(__file__).resolve().parents[2] +SPEC = importlib.util.spec_from_file_location( + "published_identity", ROOT / "tools/verify_published_identity.py") +M = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(M) +REVISION = "1" * 40 +INDEX_DIGEST = "sha256:" + "9" * 64 +BEARER = "FAKE_SCOPED_PULL_BEARER" + + +def encoded(value): + return json.dumps(value, separators=(",", ":")).encode() + + +def descriptor(raw, media_type): + return {"digest": M.R.digest(raw), "size": len(raw), "mediaType": media_type} + + +def runtime(labels=None): + return { + "Hostname": "", + "Domainname": "", + "User": "1000:1000", + "AttachStdin": False, + "AttachStdout": False, + "AttachStderr": False, + "Tty": False, + "OpenStdin": False, + "StdinOnce": False, + "Env": ["PATH=/usr/bin", "LANG=C.UTF-8"], + "Cmd": ["/bin/true"], + "Entrypoint": None, + "Image": "", + "Labels": labels, + "Volumes": None, + "WorkingDir": "/work", + "NetworkDisabled": False, + "MacAddress": "", + "OnBuild": None, + "StopSignal": "SIGTERM", + "StopTimeout": 10, + "Shell": ["/bin/sh", "-c"], + "ArgsEscaped": False, + "ExposedPorts": {"443/tcp": {}}, + "Healthcheck": {"Test": ["CMD", "/bin/true"], "Interval": 1, + "Timeout": 2, "StartPeriod": 3, "StartInterval": 4, + "Retries": 5}, + } + + +def fixture(): + layers = ["sha256:" + "a" * 64, "sha256:" + "b" * 64] + local = { + "Id": "sha256:" + "c" * 64, + "Os": "linux", + "Architecture": "amd64", + "Variant": "", + "Author": "Atrinik", + "RootFS": {"Type": "layers", "Layers": list(layers)}, + "Config": runtime({"fixture": "same"}), + } + published = { + "architecture": "amd64", + "os": "linux", + "author": "Atrinik", + "created": "2026-10-12T00:00:00Z", + "history": [], + "moby.buildkit.cache.v0": None, + "rootfs": {"type": "layers", "diff_ids": list(layers)}, + "config": runtime({ + "fixture": "same", + "org.opencontainers.image.source": M.R.SOURCE, + "org.opencontainers.image.revision": REVISION, + }), + } + return local, published + + +def context_env(**changes): + ref = changes.pop("GITHUB_REF", "refs/tags/v1.2.3") + env = { + "GITHUB_ACTIONS": "true", + "RUNNER_ENVIRONMENT": "github-hosted", + "GITHUB_REPOSITORY": M.R.REPOSITORY, + "GITHUB_EVENT_NAME": "push", + "GITHUB_REF": ref, + "GITHUB_WORKFLOW_REF": M.R.REPOSITORY + "/" + M.WORKFLOW + "@" + ref, + "GITHUB_SHA": REVISION, + "GITHUB_ACTOR": "trusted-actor", + "GITHUB_RUN_ID": "123", + "GITHUB_RUN_ATTEMPT": "2", + } + env.update(changes) + return env + + +def registry_fixture(config=None): + if config is None: + _, config = fixture() + raw_config = encoded(config) + layers = [ + {"digest": "sha256:" + "d" * 64, "size": 101, + "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip"}, + {"digest": "sha256:" + "e" * 64, "size": 102, + "mediaType": "application/vnd.oci.image.layer.v1.tar+zstd"}, + ] + manifest = {"schemaVersion": 2, "mediaType": M.R.MANIFEST, + "config": descriptor(raw_config, M.R.CONFIG), "layers": layers} + raw_manifest = encoded(manifest) + selected = dict(descriptor(raw_manifest, M.R.MANIFEST), + platform={"os": "linux", "architecture": "amd64"}) + attestation = dict( + descriptor(encoded({"attestation": True}), M.R.MANIFEST), + platform={"os": "unknown", "architecture": "unknown"}, + annotations={"vnd.docker.reference.type": "attestation-manifest", + "vnd.docker.reference.digest": selected["digest"]}) + index = {"schemaVersion": 2, "mediaType": M.R.INDEX, + "manifests": [selected, attestation]} + raw_index = encoded(index) + objects = {INDEX_DIGEST: raw_index, selected["digest"]: raw_manifest, + manifest["config"]["digest"]: raw_config} + return objects, config, manifest, index + + +class Response(io.BytesIO): + def __init__(self, raw): + super().__init__(raw) + self.status = 200 + self.length = len(raw) + + +class OfflineTests(unittest.TestCase): + def setUp(self): + blockers = ( + patch.object(M.R, "https_get", side_effect=AssertionError("network forbidden")), + patch.object(M.R.LocalConnection, "connect", + side_effect=AssertionError("Docker socket forbidden")), + patch.object(M.R.subprocess, "run", + side_effect=AssertionError("subprocess forbidden")), + ) + for blocker in blockers: + blocker.start() + self.addCleanup(blocker.stop) + + def assert_stop(self, code, call, *args): + with self.assertRaisesRegex(M.Stop, code): + call(*args) + + def test_exact_runtime_config_rootfs_and_publication_labels_match(self): + local, published = fixture() + result = M.compare(local, published, REVISION) + self.assertEqual(result["loaded_image_id"], local["Id"]) + self.assertTrue(result["rootfs_equal"]) + self.assertTrue(result["execution_config_equal"]) + self.assertEqual(result["docker_absent_defaults"], M.DEFAULTS) + + def test_docker_absent_defaults_are_exact_but_null_is_not_empty_array(self): + local, published = fixture() + for key in M.DEFAULTS: + for direction in ("loaded-materialized", "published-materialized"): + left, right = copy.deepcopy(local), copy.deepcopy(published) + if key == "Labels": + if direction == "loaded-materialized": + left["Config"][key] = None + else: + left["Config"].pop(key, None) + right["config"]["Labels"] = { + "org.opencontainers.image.source": M.R.SOURCE, + "org.opencontainers.image.revision": REVISION, + } + elif direction == "loaded-materialized": + left["Config"][key] = copy.deepcopy(M.DEFAULTS[key]) + right["config"].pop(key, None) + else: + left["Config"].pop(key, None) + right["config"][key] = copy.deepcopy(M.DEFAULTS[key]) + with self.subTest(key=key, direction=direction): + M.compare(left, right, REVISION) + for key in ("Cmd", "Entrypoint", "Env", "OnBuild"): + with self.subTest(null_vs_array=key): + left, right = copy.deepcopy(local), copy.deepcopy(published) + left["Config"][key] = None + right["config"][key] = [] + self.assert_stop("published-runtime-config-mismatch", M.compare, + left, right, REVISION) + + def test_environment_order_scalar_types_and_rootfs_order_are_bound(self): + local, published = fixture() + published["config"]["Env"].reverse() + self.assert_stop("published-runtime-config-mismatch", M.compare, + local, published, REVISION) + local, published = fixture() + local["Config"]["StopTimeout"] = True + published["config"]["StopTimeout"] = True + self.assert_stop("invalid-runtime-timeout", M.compare, local, published, REVISION) + local, published = fixture() + published["rootfs"]["diff_ids"].reverse() + self.assert_stop("published-rootfs-mismatch", M.compare, local, published, REVISION) + + def test_unknown_runtime_and_top_level_config_fields_are_rejected(self): + for location, code in (("runtime", "unknown-runtime-config-field"), + ("top", "unknown-image-config-field")): + local, published = fixture() + if location == "runtime": + local["Config"]["Unknown"] = "same" + published["config"]["Unknown"] = "same" + else: + published["unknown"] = "value" + with self.subTest(location=location): + self.assert_stop(code, M.compare, local, published, REVISION) + + def test_runtime_field_types_maps_and_healthcheck_are_strict(self): + cases = [ + ("AttachStdin", 0, "invalid-runtime-boolean"), + ("User", None, "invalid-runtime-string"), + ("Env", [True], "invalid-runtime-array"), + ("Labels", {"x": 1}, "invalid-runtime-map-value"), + ("Volumes", {"/data": None}, "invalid-runtime-map-value"), + ("ExposedPorts", {"80/tcp": []}, "invalid-runtime-map-value"), + ("StopTimeout", -1, "invalid-runtime-timeout"), + ("Healthcheck", {"Unknown": 1}, "invalid-healthcheck"), + ("Healthcheck", {"Test": [1]}, "invalid-healthcheck-value"), + ("Healthcheck", {"Retries": True}, "invalid-healthcheck-value"), + ] + for key, value, code in cases: + local, published = fixture() + local["Config"][key] = value + published["config"][key] = copy.deepcopy(value) + with self.subTest(key=key, value=value): + self.assert_stop(code, M.compare, local, published, REVISION) + + def test_loaded_and_published_platform_fields_are_exact(self): + for key, value in (("OsVersion", "6.8"), ("OsFeatures", ["feature"])): + local, published = fixture() + local[key] = value + with self.subTest(loaded=key): + self.assert_stop("loaded-platform-mismatch", M.compare, + local, published, REVISION) + for key, value in (("os.version", "6.8"), ("os.features", []), + ("variant", "v8")): + local, published = fixture() + published[key] = value + with self.subTest(published=key): + self.assert_stop("published-platform-mismatch", M.compare, + local, published, REVISION) + + def test_author_revision_and_source_are_exact(self): + for mutation, code in ( + (lambda local, published: published.update(author="other"), + "image-author-mismatch"), + (lambda local, published: published["config"]["Labels"].update( + {"org.opencontainers.image.revision": "2" * 40}), + "producer-label-mismatch"), + (lambda local, published: published["config"]["Labels"].update( + {"org.opencontainers.image.source": "https://example.invalid"}), + "producer-label-mismatch")): + local, published = fixture() + mutation(local, published) + self.assert_stop(code, M.compare, local, published, REVISION) + + def test_only_declared_top_level_build_metadata_is_exempt_from_comparison(self): + local, published = fixture() + published.update(created={"opaque": "builder value"}, history=[{"created_by": "x"}], + **{"moby.buildkit.cache.v0": {"arbitrary": [1, True, None]}}) + self.assertTrue(M.compare(local, published, REVISION)["execution_config_equal"]) + + def test_decode_rejects_empty_oversized_duplicate_and_nonfinite_json(self): + self.assert_stop("invalid-json-size", M.decode, b"") + self.assert_stop("invalid-json-size", M.decode, b" " * (M.R.LIMIT + 1)) + self.assert_stop("duplicate-json-key", M.decode, b'{"x":1,"x":2}') + for raw in (b'{"x":NaN}', b'{"x":Infinity}', b'{"x":-Infinity}'): + with self.subTest(raw=raw): + self.assert_stop("invalid-json-value", M.decode, raw) + + def test_registry_binds_complete_descriptor_chain_and_platform(self): + objects, config, manifest, index = registry_fixture() + calls = [] + def get(package, kind, digest, bearer): + calls.append((package, kind, digest, bearer)) + return objects[digest] + with patch.object(M.R, "registry_object", side_effect=get): + blobs, relation, actual = M.registry("atrinik/linux-build", INDEX_DIGEST, BEARER) + self.assertEqual(actual, config) + self.assertEqual(set(blobs), {"index.json", "manifest.json", "config.json"}) + self.assertEqual(relation["platform"], "linux/amd64") + self.assertEqual(relation["manifest_descriptor"], index["manifests"][0]) + self.assertEqual(relation["config_descriptor"], manifest["config"]) + self.assertEqual([item[1] for item in calls], ["manifests", "manifests", "blobs"]) + + def test_registry_rejects_descriptor_size_digest_platform_and_type_confusion(self): + mutations = [ + ("manifest-size", "manifest-size-mismatch", + lambda o, c, m, i: i["manifests"][0].update(size=1)), + ("config-size", "config-size-mismatch", + lambda o, c, m, i: m["config"].update(size=1)), + ("manifest-digest", "invalid-oci-descriptor", + lambda o, c, m, i: i["manifests"][0].update(digest="bad")), + ("descriptor-type", "invalid-oci-descriptor", + lambda o, c, m, i: i["manifests"][0].update(size=True)), + ("platform-extra", "unexpected-platform-metadata", + lambda o, c, m, i: i["manifests"][0]["platform"].update(variant="v8")), + ("schema-bool", "invalid-oci-index", + lambda o, c, m, i: i.update(schemaVersion=True)), + ("manifest-media", "invalid-image-manifest", + lambda o, c, m, i: m.update(mediaType=M.R.INDEX)), + ("config-media", "invalid-oci-descriptor", + lambda o, c, m, i: m["config"].update(mediaType=M.R.MANIFEST)), + ("layer-media", "invalid-oci-descriptor", + lambda o, c, m, i: m["layers"][0].update(mediaType=M.R.CONFIG)), + ] + for name, code, mutate in mutations: + objects, config, manifest, index = registry_fixture() + mutate(objects, config, manifest, index) + raw_config = encoded(config) + if name not in ("config-size", "config-media"): + manifest["config"] = descriptor(raw_config, M.R.CONFIG) + raw_manifest = encoded(manifest) + if name not in ("manifest-size", "manifest-digest", "descriptor-type"): + index["manifests"][0].update(descriptor(raw_manifest, M.R.MANIFEST)) + index["manifests"][1]["annotations"]["vnd.docker.reference.digest"] = \ + index["manifests"][0]["digest"] + raw_index = encoded(index) + objects = {INDEX_DIGEST: raw_index, + index["manifests"][0]["digest"]: raw_manifest, + manifest["config"]["digest"]: raw_config} + with self.subTest(name=name), patch.object( + M.R, "registry_object", side_effect=lambda p, k, d, b, data=objects: data[d]), \ + self.assertRaisesRegex(M.Stop, code): + M.registry("atrinik/linux-build", INDEX_DIGEST, BEARER) + + def test_registry_rejects_ambiguous_platform_attestation_and_layer_count(self): + for name, code in (("ambiguous", "ambiguous-linux-amd64-manifest"), + ("attestation-subject", "attestation-subject-mismatch"), + ("layer-count", "layer-count-mismatch")): + objects, config, manifest, index = registry_fixture() + if name == "ambiguous": + index["manifests"].append(copy.deepcopy(index["manifests"][0])) + elif name == "attestation-subject": + index["manifests"][1]["annotations"]["vnd.docker.reference.digest"] = "sha256:" + "f" * 64 + else: + config["rootfs"]["diff_ids"].pop() + raw_config = encoded(config) + manifest["config"] = descriptor(raw_config, M.R.CONFIG) + raw_manifest = encoded(manifest) + for item in index["manifests"]: + if item["platform"]["os"] == "linux": + item.update(descriptor(raw_manifest, M.R.MANIFEST)) + if name != "attestation-subject": + index["manifests"][1]["annotations"]["vnd.docker.reference.digest"] = \ + index["manifests"][0]["digest"] + raw_index = encoded(index) + data = {INDEX_DIGEST: raw_index, M.R.digest(raw_manifest): raw_manifest, + M.R.digest(raw_config): raw_config} + with self.subTest(name=name), patch.object( + M.R, "registry_object", side_effect=lambda p, k, d, b, data=data: data[d]), \ + self.assertRaisesRegex(M.Stop, code): + M.registry("atrinik/linux-build", INDEX_DIGEST, BEARER) + + def test_context_binds_host_repository_event_ref_workflow_and_source(self): + ctx = M.context(context_env()) + self.assertEqual(ctx["head"], REVISION) + self.assertFalse(ctx["candidate_only"]) + dispatch = context_env( + GITHUB_EVENT_NAME="workflow_dispatch", GITHUB_REF="refs/heads/feature/safe", + GITHUB_WORKFLOW_REF=M.R.REPOSITORY + "/" + M.WORKFLOW + "@refs/heads/feature/safe", + CANDIDATE_ONLY="true") + self.assertTrue(M.context(dispatch)["candidate_only"]) + bad = ( + {"RUNNER_ENVIRONMENT": "self-hosted"}, + {"GITHUB_REPOSITORY": "other/repo"}, + {"GITHUB_EVENT_NAME": "pull_request"}, + {"GITHUB_REF": "refs/heads/main"}, + {"GITHUB_WORKFLOW_REF": "other/workflow@refs/tags/v1.2.3"}, + {"GITHUB_SHA": "not-a-revision"}, + {"GITHUB_ACTOR": "bad actor"}, + {"OPENSSL_CONF": "/tmp/hostile"}, + ) + for changes in bad: + with self.subTest(changes=changes), self.assertRaises(M.Stop): + M.context(context_env(**changes)) + + def test_evidence_files_reject_names_links_modes_and_oversize(self): + with tempfile.TemporaryDirectory() as temporary: + folder = Path(temporary) + os.chmod(folder, 0o700) + for name in ("", ".", "..", "sub/file"): + with self.subTest(save=name), self.assertRaises(M.Stop): + M.save(folder, name, b"x") + with self.subTest(read=name), self.assertRaises(M.Stop): + M.read(folder, name) + path = folder / "evidence" + path.write_bytes(b"x") + os.chmod(path, 0o644) + self.assert_stop("unsafe-evidence-file", M.read, folder, "evidence") + os.chmod(path, 0o600) + hardlink = folder / "hardlink" + os.link(path, hardlink) + self.assert_stop("unsafe-evidence-file", M.read, folder, "evidence") + hardlink.unlink() + path.unlink() + path.symlink_to("missing") + with self.assertRaises(OSError): + M.read(folder, "evidence") + self.assert_stop("invalid-evidence-file", M.save, folder, "large", b"x" * (M.R.LIMIT + 1)) + + def test_directory_requires_absolute_owned_private_nonsymlink_path(self): + ctx = M.context(context_env()) + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + os.chmod(root, 0o700) + with patch.dict(os.environ, {"RUNNER_TEMP": str(root)}): + folder = M.directory(ctx, "broad", True) + self.assertEqual(folder.stat().st_mode & 0o777, 0o700) + os.chmod(root, 0o777) + with patch.dict(os.environ, {"RUNNER_TEMP": str(root)}): + self.assert_stop("unsafe-runner-temp", M.directory, ctx, "broad", False) + with patch.dict(os.environ, {"RUNNER_TEMP": "relative"}): + self.assert_stop("unsafe-runner-temp", M.directory, ctx, "broad", False) + + def capture_fixture(self, folder, output): + local, _ = fixture() + raw = encoded(local) + response = Response(raw) + connection = Mock() + docker = Mock() + docker.negotiate.return_value = {"docker_api": "1.51", "docker_version": "28.5.1"} + docker.request.return_value = (connection, response) + with patch.object(M.R, "runner_docker", return_value=docker), \ + patch.dict(os.environ, {"GITHUB_OUTPUT": str(output)}): + M.capture(M.context(context_env()), "broad", folder) + connection.close.assert_called_once_with() + requested = docker.request.call_args.args[1] + self.assertIn("platform=%7B%22os%22%3A%22linux%22%2C%22architecture%22%3A%22amd64%22%7D", requested) + return local + + def test_capture_writes_bound_evidence_and_only_validated_digest_output(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + folder, output = root / "evidence", root / "output" + folder.mkdir(mode=0o700) + output.touch(mode=0o600) + local = self.capture_fixture(folder, output) + capture = M.decode((folder / "capture.json").read_bytes()) + self.assertEqual(capture["image_id"], local["Id"]) + self.assertEqual(capture["target"], "broad") + self.assertEqual(output.read_text(), "image_id=" + local["Id"] + "\n") + + def test_capture_rejects_output_symlink_hardlink_writable_mode_and_relative_path(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + target = root / "target" + target.touch(mode=0o600) + hardlink = root / "hardlink" + os.link(target, hardlink) + symlink = root / "symlink" + symlink.symlink_to(target) + writable = root / "writable" + writable.touch(mode=0o666) + os.chmod(writable, 0o666) + for output in (symlink, hardlink, writable, Path("relative-output")): + folder = root / ("case-" + str(len(list(root.iterdir())))) + folder.mkdir(mode=0o700) + with self.subTest(output=output), self.assertRaises((M.Stop, OSError)): + self.capture_fixture(folder, output) + + def test_verify_binds_capture_context_image_and_registry_relation(self): + ctx = M.context(context_env()) + local, published = fixture() + raw = encoded(local) + with tempfile.TemporaryDirectory() as temporary: + folder = Path(temporary) + os.chmod(folder, 0o700) + inspection = M.save(folder, "loaded-inspect.json", raw) + M.save(folder, "capture.json", M.encoded({ + "context": ctx, "target": "broad", "docker_api": "1.51", + "docker_version": "28.5.1", "image_id": local["Id"], + "inspection": inspection, + })) + relation = {"package": M.TARGETS["broad"][0], "index_digest": INDEX_DIGEST, + "manifest_digest": "sha256:" + "d" * 64, + "config_digest": "sha256:" + "e" * 64, "platform": "linux/amd64", + "manifest_descriptor": {}, "config_descriptor": {}} + blobs = {"index.json": b"index", "manifest.json": b"manifest", "config.json": b"config"} + with patch.object(M.R, "scoped_bearer", return_value=BEARER) as bearer, \ + patch.object(M, "registry", return_value=(blobs, relation, published)) as registry, \ + patch("builtins.print"): + M.verify(ctx, "broad", folder, INDEX_DIGEST, "ORIGINAL_TOKEN") + bearer.assert_called_once_with(M.TARGETS["broad"][0], "ORIGINAL_TOKEN", ctx["actor"]) + registry.assert_called_once_with(M.TARGETS["broad"][0], INDEX_DIGEST, BEARER) + report = M.decode((folder / "report.json").read_bytes()) + self.assertEqual(report["status"], "verified-published-identity") + self.assertEqual(report["loaded_image_id"], local["Id"]) + self.assertEqual(report["context"], ctx) + + def test_verify_rejects_capture_context_and_image_drift_before_registry(self): + ctx = M.context(context_env()) + local, _ = fixture() + raw = encoded(local) + for change, code in (("context", "capture-context-mismatch"), + ("image", "capture-image-mismatch")): + with tempfile.TemporaryDirectory() as temporary: + folder = Path(temporary) + os.chmod(folder, 0o700) + inspection = M.save(folder, "loaded-inspect.json", raw) + captured = {"context": ctx, "target": "broad", "image_id": local["Id"], + "inspection": inspection} + if change == "context": + captured["context"] = dict(ctx, run_attempt="3") + else: + captured["image_id"] = "sha256:" + "f" * 64 + M.save(folder, "capture.json", M.encoded(captured)) + with self.subTest(change=change), patch.object( + M.R, "scoped_bearer") as bearer, self.assertRaisesRegex(M.Stop, code): + M.verify(ctx, "broad", folder, INDEX_DIGEST, "ORIGINAL_TOKEN") + bearer.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/tests/test_workflow_contracts.py b/tools/tests/test_workflow_contracts.py index 50d4823..a561eb9 100644 --- a/tools/tests/test_workflow_contracts.py +++ b/tools/tests/test_workflow_contracts.py @@ -21,13 +21,18 @@ def step_script(name, step): start = text.index(f' - name: {step}\n') block = text[start:].split('\n - name: ', 1)[0] script = block.split(' run: |\n', 1)[1] - return '\n'.join(line[10:] for line in script.splitlines()) + '\n' + lines = [] + for line in script.splitlines(): + if line.strip() and not line.startswith(' '): + break + lines.append(line[10:]) + return '\n'.join(lines) + '\n' class WorkflowContracts(unittest.TestCase): def test_portable_public_ca_command_and_failure_are_mandatory(self): for name, image in (('validate.yml', 'atrinik-classic-portable:validation'), - ('publish-linux.yml', 'atrinik-classic-portable:release-validation')): + ('publish-linux.yml', 'sha256:' + 'a' * 64)): for docker_exit in (0, 17): with self.subTest(workflow=name, docker_exit=docker_exit), tempfile.TemporaryDirectory() as directory: root = Path(directory) @@ -40,7 +45,7 @@ def test_portable_public_ca_command_and_failure_are_mandatory(self): docker.chmod(0o755) env = dict(os.environ, PATH=directory + ':' + os.environ['PATH'], GITHUB_WORKSPACE=directory, DOCKER_CALL=str(root / 'docker-call.json'), - DOCKER_EXIT=str(docker_exit), HTTPS_PROXY='http://fixture-user:fixture-password@invalid') + DOCKER_EXIT=str(docker_exit), IMAGE_ID=image, HTTPS_PROXY='http://fixture-user:fixture-password@invalid') result = subprocess.run(['bash', '-euo', 'pipefail', '-c', step_script(name, 'Verify portable candidate public CA')], cwd=directory, env=env, capture_output=True, text=True) @@ -78,7 +83,7 @@ def test_portable_public_ca_gate_precedes_consumer_and_publication(self): self.assertIn('set -euo pipefail', block) if name == 'publish-linux.yml': self.assertLess(gate, text.index(' - name: Publish immutable portable candidate')) - self.assertLess(gate, text.index(' - name: Promote validated portable aliases')) + self.assertLess(gate, text.index(' - name: Promote validated Linux aliases')) def run_step(self, name, step, **values): with tempfile.TemporaryDirectory() as directory: @@ -133,7 +138,7 @@ def test_windows_candidate_and_release_tag_names(self): def test_candidate_mode_keeps_qualification_and_fences_promotion(self): linux = workflow('publish-linux.yml') self.assertEqual(linux.count('if: ${{ !inputs.candidate_only }}'), 1) - self.assertIn('Promote validated portable aliases\n if: ${{ !inputs.candidate_only }}', linux) + self.assertIn('Promote validated Linux aliases\n if: ${{ !inputs.candidate_only }}', linux) for target in ('target: classic-validation', 'target: validation', 'target: portable-validation'): self.assertIn(target, linux) @@ -145,6 +150,109 @@ def test_candidate_mode_keeps_qualification_and_fences_promotion(self): self.assertNotIn('candidate_only', native) self.assertIn('run-classic-native-tests.ps1', native) + def test_linux_publish_uses_only_sha_tags_then_fences_every_alias(self): + text = workflow('publish-linux.yml') + for step, output in (('Build and publish Linux image', 'metadata.outputs.publish_tag'), + ('Build and publish Classic image', 'classic-metadata.outputs.publish_tag'), + ('Publish immutable portable candidate', 'portable-metadata.outputs.tag')): + block = text.split(' - name: ' + step + '\n')[1].split('\n - name: ')[0] + self.assertIn('tags: ${{ steps.' + output + ' }}', block) + self.assertNotIn('outputs.tags', block) + fence = text.index(' - name: Verify all published image identities') + upload = text.index(' - name: Retain published identity evidence') + promote = text.index(' - name: Promote validated Linux aliases') + self.assertLess(text.index(' - name: Publish immutable portable candidate'), fence) + self.assertLess(fence, upload) + self.assertLess(upload, promote) + for block in (text[fence:upload], text[upload:promote]): + self.assertNotIn('continue-on-error', block) + self.assertNotIn(' if:', block) + self.assertIn('if-no-files-found: error', text[upload:promote]) + for candidate in ('false', 'true'): + for step, image in (('Select image tags', 'linux-build'), + ('Select Classic image tags', 'classic-build')): + result = self.run_step('publish-linux.yml', step, IMAGE_NAME=image, + CANDIDATE_ONLY=candidate) + prefix = 'candidate-' if candidate == 'true' else '' + self.assertIn(f'publish_tag=ghcr.io/atrinik/{image}:{prefix}sha-' + 'a' * 40, result) + + def test_linux_runtime_checks_use_captured_ids(self): + text = workflow('publish-linux.yml') + checks = { + 'classic': ['Qualify Classic cancellation and public TLS before publication', + 'Run Classic client and server checks before publication'], + 'broad': ['Qualify broad Linux cancellation and public TLS before publication'], + 'portable': ['Verify portable candidate public CA', + 'Validate real portable consumer before publication'], + } + for target, names in checks.items(): + captured = text.index(' - name: Capture loaded ' + target + ' image identity') + for name in names: + start = text.index(' - name: ' + name + '\n') + self.assertLess(captured, start) + block = text[start:].split('\n - name: ')[0] + self.assertIn('IMAGE_ID: ${{ steps.' + target + '-identity.outputs.image_id }}', block) + self.assertIn('"${IMAGE_ID}"', block) + self.assertNotIn(':qualification', block) + self.assertNotIn(':release-validation', block) + + def test_published_fence_stops_on_first_failed_identity(self): + for failed in ('none', 'broad', 'classic', 'portable'): + with self.subTest(failed=failed), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + command = root / 'python3' + command.write_text('#!/usr/bin/env python3\nimport os, sys\n' + 'from pathlib import Path\n' + 'target = sys.argv[sys.argv.index("--target") + 1]\n' + 'with Path(os.environ["CALLS"]).open("a") as out: out.write(target + "\\n")\n' + 'sys.exit(19 if target == os.environ["FAIL_TARGET"] else 0)\n') + # Avoid recursively selecting the fake python interpreter. + command.write_text(command.read_text().replace('#!/usr/bin/env python3', '#!/usr/bin/python3')) + command.chmod(0o755) + env = dict(os.environ, PATH=directory + ':' + os.environ['PATH'], + CALLS=str(root / 'calls'), FAIL_TARGET=failed, + BROAD_DIGEST='sha256:' + 'a' * 64, CLASSIC_DIGEST='sha256:' + 'b' * 64, + PORTABLE_DIGEST='sha256:' + 'c' * 64) + result = subprocess.run(['bash', '-euo', 'pipefail', '-c', + step_script('publish-linux.yml', 'Verify all published image identities')], + env=env, capture_output=True, text=True) + self.assertEqual(result.returncode, 0 if failed == 'none' else 19) + targets = ['broad', 'classic', 'portable'] + expected = targets if failed == 'none' else targets[:targets.index(failed) + 1] + self.assertEqual((root / 'calls').read_text().splitlines(), expected) + + def test_linux_alias_promotion_uses_verified_digests_and_stops_on_failure(self): + for fail_call in (0, 2): + with self.subTest(fail_call=fail_call), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + docker = root / 'docker' + docker.write_text('#!/usr/bin/python3\nimport json, os, sys\n' + 'from pathlib import Path\n' + 'path = Path(os.environ["CALLS"])\n' + 'calls = json.loads(path.read_text()) if path.exists() else []\n' + 'calls.append(sys.argv[1:]); path.write_text(json.dumps(calls))\n' + 'sys.exit(17 if len(calls) == int(os.environ["FAIL_CALL"]) else 0)\n') + docker.chmod(0o755) + digests = ['sha256:' + char * 64 for char in 'abc'] + env = dict(os.environ, PATH=directory + ':' + os.environ['PATH'], + CALLS=str(root / 'calls'), FAIL_CALL=str(fail_call), + BROAD_DIGEST=digests[0], CLASSIC_DIGEST=digests[1], PORTABLE_DIGEST=digests[2], + GITHUB_REPOSITORY_OWNER='Atrinik', GITHUB_REF_TYPE='tag', GITHUB_REF_NAME='v9.8.7', + GITHUB_STEP_SUMMARY=str(root / 'summary')) + result = subprocess.run(['bash', '-euo', 'pipefail', '-c', + step_script('publish-linux.yml', 'Promote validated Linux aliases')], + env=env, capture_output=True, text=True) + self.assertEqual(result.returncode, 0 if fail_call == 0 else 1, result.stderr) + calls = json.loads((root / 'calls').read_text()) + self.assertEqual(len(calls), 3 if fail_call == 0 else 2) + for index, call in enumerate(calls): + image = 'ghcr.io/atrinik/' + ['linux-build', 'classic-build', 'classic-portable-build'][index] + platform = ['ubuntu-26.04', 'ubuntu-26.04', 'debian-12'][index] + self.assertEqual(call, ['buildx', 'imagetools', 'create', '--tag', image + ':latest', + '--tag', image + ':' + platform, '--tag', image + ':9.8.7', image + '@' + digests[index]]) + if fail_call: + self.assertIn('Rerun the failed job in this workflow run', result.stderr) + def test_shared_llvm_inputs_select_every_image(self): script = step_script('validate.yml', 'Detect image changes') inputs = ('llvm-toolchain.json', 'llvm-bootstrap-packages.lock', diff --git a/tools/verify_published_identity.py b/tools/verify_published_identity.py new file mode 100644 index 0000000..cab8104 --- /dev/null +++ b/tools/verify_published_identity.py @@ -0,0 +1,362 @@ +#!/usr/bin/python3 -I +# Copyright (c) 2026 Atrinik contributors +# SPDX-License-Identifier: MIT +"""Bind Linux publisher runtime-tested image IDs to actual pushed OCI metadata. + +No registry layers are downloaded and no image code is executed here. Only the +same-repository hosted publisher may capture the local daemon or acquire metadata. +The historical released-SDK collector's target/revision/admission stays unchanged. +""" +import argparse +import importlib.util +import json +import os +from pathlib import Path +import re +import resource +import signal +import stat +import sys +import urllib.parse + +_SPEC = importlib.util.spec_from_file_location( + 'registry_transport', Path(__file__).with_name('collect-released-sdk-metadata.py')) +R = importlib.util.module_from_spec(_SPEC) +_SPEC.loader.exec_module(R) +require = R.require +Stop = R.Stop +TARGETS = { + 'broad': ('atrinik/linux-build', 'atrinik-linux-curl:qualification'), + 'classic': ('atrinik/classic-build', 'atrinik-classic-curl:qualification'), + 'portable': ('atrinik/classic-portable-build', 'atrinik-classic-portable:release-validation'), +} +WORKFLOW = '.github/workflows/publish-linux.yml' +# Docker API 1.49–1.51 may materialize these absent fields. Never equate empty +# arrays/objects with null, nor normalize non-default values or unknown keys. +DEFAULTS = {'Cmd': None, 'Entrypoint': None, 'Env': None, 'Labels': None, + 'OnBuild': None, 'Volumes': None, 'User': '', 'WorkingDir': ''} +CONFIG_KEYS = {'architecture', 'os', 'variant', 'os.version', 'os.features', + 'config', 'rootfs', 'author', 'created', 'history', 'moby.buildkit.cache.v0'} +STRING_FIELDS = {'Hostname', 'Domainname', 'User', 'Image', 'WorkingDir', 'MacAddress', 'StopSignal'} +BOOL_FIELDS = {'AttachStdin', 'AttachStdout', 'AttachStderr', 'Tty', 'OpenStdin', 'StdinOnce', + 'ArgsEscaped', 'NetworkDisabled'} +ARRAY_FIELDS = {'Env', 'Cmd', 'Entrypoint', 'OnBuild', 'Shell'} +MAP_FIELDS = {'Labels', 'Volumes', 'ExposedPorts'} +RUNTIME_FIELDS = STRING_FIELDS | BOOL_FIELDS | ARRAY_FIELDS | MAP_FIELDS | {'Healthcheck', 'StopTimeout'} +PUBLICATION_LABELS = {'org.opencontainers.image.source', 'org.opencontainers.image.revision'} + + +def decode(raw): + require(isinstance(raw, bytes) and 0 < len(raw) <= R.LIMIT, 'invalid-json-size') + value = R.decode(raw) + # Reject non-finite numbers accepted by Python's default parser; JSON from + # Docker/OCI cannot contain these. This also bounds hostile deep documents. + try: + json.dumps(value, allow_nan=False) + except (ValueError, RecursionError): + raise Stop('invalid-json-value') from None + return value + + +def context(env): + require(env.get('GITHUB_ACTIONS') == 'true' and + env.get('RUNNER_ENVIRONMENT') == 'github-hosted' and sys.platform == 'linux', + 'hosted-runner-required') + require(env.get('GITHUB_REPOSITORY') == R.REPOSITORY, 'repository-not-allowed') + ref, event = env.get('GITHUB_REF', ''), env.get('GITHUB_EVENT_NAME') + candidate = env.get('CANDIDATE_ONLY') == 'true' + release = re.fullmatch(r'refs/tags/v[0-9]+\.[0-9]+\.[0-9]+', ref) is not None + require((event == 'push' and release and not candidate) or + (event == 'workflow_dispatch' and + (release or (ref == 'refs/heads/main') or + (candidate and re.fullmatch(r'refs/heads/[A-Za-z0-9._/-]+', ref)))), + 'untrusted-workflow-event') + require(env.get('GITHUB_WORKFLOW_REF') == R.REPOSITORY + '/' + WORKFLOW + '@' + ref, + 'workflow-identity-mismatch') + head, actor = env.get('GITHUB_SHA', ''), env.get('GITHUB_ACTOR', '') + require(re.fullmatch(r'[0-9a-f]{40}', head) is not None, 'invalid-workflow-head') + require(re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9-]{0,38}(?:\[bot\])?', actor) is not None, + 'invalid-workflow-actor') + for key in ('GITHUB_RUN_ID', 'GITHUB_RUN_ATTEMPT'): + require(re.fullmatch(r'[1-9][0-9]{0,19}', env.get(key, '')) is not None, 'invalid-workflow-run') + require(not any(env.get(key) for key in ('OPENSSL_CONF', 'OPENSSL_MODULES', 'SSLKEYLOGFILE')), + 'tls-override-not-allowed') + return {'repository': R.REPOSITORY, 'workflow': WORKFLOW, 'ref': ref, 'event': event, + 'head': head, 'actor': actor, 'candidate_only': candidate, + 'run_id': env['GITHUB_RUN_ID'], 'run_attempt': env['GITHUB_RUN_ATTEMPT']} + + +def loaded_identity(local): + require(isinstance(local, dict) and R.valid_digest(local.get('Id')), 'invalid-loaded-image') + require(local.get('Os') == 'linux' and local.get('Architecture') == 'amd64' and + local.get('Variant', '') == '' and not local.get('OsVersion') and + not local.get('OsFeatures'), 'loaded-platform-mismatch') + rootfs = local.get('RootFS') + require(isinstance(rootfs, dict) and set(rootfs) == {'Type', 'Layers'} and + rootfs['Type'] == 'layers' and isinstance(rootfs['Layers'], list) and rootfs['Layers'] and + all(R.valid_digest(item) for item in rootfs['Layers']), 'invalid-loaded-rootfs') + validate_runtime(local.get('Config')) + return local['Id'] + + +def validate_runtime(config): + require(isinstance(config, dict) and set(config) <= RUNTIME_FIELDS, 'unknown-runtime-config-field') + for key, value in config.items(): + if key in STRING_FIELDS: + require(isinstance(value, str), 'invalid-runtime-string') + elif key in BOOL_FIELDS: + require(type(value) is bool, 'invalid-runtime-boolean') + elif key in ARRAY_FIELDS: + require(value is None or (isinstance(value, list) and + all(isinstance(item, str) for item in value)), 'invalid-runtime-array') + elif key in MAP_FIELDS: + require(value is None or isinstance(value, dict), 'invalid-runtime-map') + for item in (value or {}).values(): + require(isinstance(item, str) if key == 'Labels' else item == {}, 'invalid-runtime-map-value') + elif key == 'StopTimeout': + require(type(value) is int and value >= 0, 'invalid-runtime-timeout') + elif key == 'Healthcheck': + require(value is None or (isinstance(value, dict) and set(value) <= { + 'Test', 'Interval', 'Timeout', 'StartPeriod', 'StartInterval', 'Retries'}), + 'invalid-healthcheck') + for field, item in (value or {}).items(): + require((isinstance(item, list) and all(isinstance(part, str) for part in item)) + if field == 'Test' else (type(item) is int and item >= 0), 'invalid-healthcheck-value') + + +def same_json(left, right): + # Python compares True == 1; use JSON equality to retain scalar types. + return json.dumps(left, sort_keys=True, allow_nan=False) == json.dumps(right, sort_keys=True, allow_nan=False) + + +def compare(local, config, revision): + loaded_identity(local) + require(isinstance(config, dict) and set(config) <= CONFIG_KEYS, 'unknown-image-config-field') + require(config.get('os') == 'linux' and config.get('architecture') == 'amd64' and + config.get('variant', '') == '' and 'os.version' not in config and + 'os.features' not in config, 'published-platform-mismatch') + require(config.get('author', '') == local.get('Author', ''), 'image-author-mismatch') + rootfs = config.get('rootfs') + require(isinstance(rootfs, dict) and set(rootfs) == {'type', 'diff_ids'} and + rootfs['type'] == 'layers' and rootfs['diff_ids'] == local['RootFS']['Layers'], + 'published-rootfs-mismatch') + expected, actual = dict(local['Config']), config.get('config') + validate_runtime(actual) + actual = dict(actual) + labels = actual.get('Labels') + require(isinstance(labels, dict) and + labels.get('org.opencontainers.image.source') == R.SOURCE and + labels.get('org.opencontainers.image.revision') == revision, 'producer-label-mismatch') + for cfg in (expected, actual): + labels = cfg.get('Labels') + require(labels is None or (isinstance(labels, dict) and + all(isinstance(k, str) and isinstance(v, str) for k, v in labels.items())), + 'invalid-runtime-labels') + # Only these exact, validated publication labels may differ. + cfg['Labels'] = {k: v for k, v in (labels or {}).items() if k not in PUBLICATION_LABELS} + for key, default in DEFAULTS.items(): + cfg.setdefault(key, default) + require(same_json(expected, actual), 'published-runtime-config-mismatch') + return {'loaded_image_id': local['Id'], 'rootfs_equal': True, 'execution_config_equal': True, + 'publication_exceptions': ['config.Labels.org.opencontainers.image.source', + 'config.Labels.org.opencontainers.image.revision', 'created', 'history', + 'moby.buildkit.cache.v0'], 'docker_absent_defaults': DEFAULTS} + + +def registry(package, index_digest, bearer): + # Reuse only bounded HTTPS/auth/digest primitives, never the historical + # collector's revision gate, pulls, containers, files, or acceptance claims. + index_raw = R.registry_object(package, 'manifests', index_digest, bearer) + index = decode(index_raw) + require(isinstance(index, dict) and type(index.get('schemaVersion')) is int and + index['schemaVersion'] == 2 and index.get('mediaType') == R.INDEX and + isinstance(index.get('manifests'), list), 'invalid-oci-index') + candidates = [] + for item in index['manifests']: + R.descriptor(item, {R.MANIFEST}) + platform, annotations = item.get('platform'), item.get('annotations', {}) + require(isinstance(platform, dict) and isinstance(annotations, dict), 'invalid-index-platform') + if platform.get('os') == 'linux' and platform.get('architecture') == 'amd64': + require(platform == {'os': 'linux', 'architecture': 'amd64'} and + annotations.get('vnd.docker.reference.type') != 'attestation-manifest', + 'unexpected-platform-metadata') + candidates.append(item) + else: + require(platform == {'os': 'unknown', 'architecture': 'unknown'} and + annotations.get('vnd.docker.reference.type') == 'attestation-manifest', + 'unexpected-index-platform') + require(len(candidates) == 1, 'ambiguous-linux-amd64-manifest') + selected = candidates[0] + for item in index['manifests']: + if item is not selected: + require(item['annotations'].get('vnd.docker.reference.digest') == selected['digest'], + 'attestation-subject-mismatch') + manifest_raw = R.registry_object(package, 'manifests', selected['digest'], bearer) + require(len(manifest_raw) == selected['size'], 'manifest-size-mismatch') + manifest = decode(manifest_raw) + require(isinstance(manifest, dict) and type(manifest.get('schemaVersion')) is int and + manifest['schemaVersion'] == 2 and manifest.get('mediaType') == R.MANIFEST and + not manifest.get('artifactType') and not manifest.get('subject'), 'invalid-image-manifest') + desc = R.descriptor(manifest.get('config'), {R.CONFIG}) + require(desc['size'] <= R.LIMIT and isinstance(manifest.get('layers'), list) and + manifest['layers'], 'invalid-image-layers') + for layer in manifest['layers']: + R.descriptor(layer, R.LAYERS) + config_raw = R.registry_object(package, 'blobs', desc['digest'], bearer) + require(len(config_raw) == desc['size'], 'config-size-mismatch') + config = decode(config_raw) + require(isinstance(config, dict) and isinstance(config.get('rootfs'), dict) and + isinstance(config['rootfs'].get('diff_ids'), list) and + len(config['rootfs']['diff_ids']) == len(manifest['layers']), 'layer-count-mismatch') + return {'index.json': index_raw, 'manifest.json': manifest_raw, 'config.json': config_raw}, { + 'package': package, 'index_digest': index_digest, 'manifest_digest': selected['digest'], + 'config_digest': desc['digest'], 'platform': 'linux/amd64', + 'manifest_descriptor': selected, 'config_descriptor': desc}, config + + +def safe_directory(path): + info = path.lstat() + require(stat.S_ISDIR(info.st_mode) and info.st_uid == os.getuid() and + info.st_mode & 0o077 == 0, 'unsafe-evidence-directory') + + +def directory(ctx, target, create): + runner_temp = Path(os.environ['RUNNER_TEMP']) + require(runner_temp.is_absolute() and runner_temp.resolve() == runner_temp, 'unsafe-runner-temp') + info = runner_temp.stat() + require(stat.S_ISDIR(info.st_mode) and info.st_uid == os.getuid() and + not info.st_mode & 0o022, 'unsafe-runner-temp') + root = runner_temp / 'linux-published-identity' + if create: + root.mkdir(mode=0o700, exist_ok=True) + safe_directory(root) + run = root / (ctx['run_id'] + '-' + ctx['run_attempt']) + if create: + run.mkdir(mode=0o700, exist_ok=True) + safe_directory(run) + folder = run / target + if create: + folder.mkdir(mode=0o700) + safe_directory(folder) + return folder + + +def save(folder, name, raw): + require('/' not in name and name not in ('', '.', '..') and len(raw) <= R.LIMIT, + 'invalid-evidence-file') + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW + fd = os.open(folder / name, flags, 0o600) + with os.fdopen(fd, 'wb') as output: + output.write(raw) + return {'file': name, 'digest': R.digest(raw), 'size': len(raw)} + + +def read(folder, name): + require('/' not in name and name not in ('', '.', '..'), 'invalid-evidence-file') + fd = os.open(folder / name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + with os.fdopen(fd, 'rb') as source: + info = os.fstat(source.fileno()) + require(stat.S_ISREG(info.st_mode) and info.st_uid == os.getuid() and info.st_nlink == 1 and + info.st_mode & 0o077 == 0 and 0 < info.st_size <= R.LIMIT, 'unsafe-evidence-file') + raw = source.read(R.LIMIT + 1) + require(len(raw) == info.st_size, 'evidence-file-changed') + return raw + + +def encoded(value): + return (json.dumps(value, sort_keys=True, indent=2, allow_nan=False) + '\n').encode() + + +def capture(ctx, target, folder): + docker = R.runner_docker() + daemon = docker.negotiate() + query = urllib.parse.urlencode({'platform': '{"os":"linux","architecture":"amd64"}'}) + path = '/images/' + urllib.parse.quote(TARGETS[target][1], safe='') + '/json?' + query + conn, response = docker.request('GET', path) + try: + raw = R.bounded_read(response) + finally: + conn.close() + local = decode(raw) + image_id = loaded_identity(local) + artifact = save(folder, 'loaded-inspect.json', raw) + save(folder, 'capture.json', encoded({'context': ctx, 'target': target, **daemon, + 'image_id': image_id, 'inspection': artifact})) + # GitHub owns this output path; it may be readable but never writable by + # other users. Only append the validated digest to the existing regular file. + output_path = Path(os.environ['GITHUB_OUTPUT']) + require(output_path.is_absolute() and output_path.resolve() == output_path, + 'unsafe-github-output') + fd = os.open(output_path, os.O_WRONLY | os.O_APPEND | os.O_NOFOLLOW | os.O_NONBLOCK) + with os.fdopen(fd, 'w', encoding='utf-8') as output: + info = os.fstat(output.fileno()) + require(stat.S_ISREG(info.st_mode) and info.st_uid == os.getuid() and + info.st_nlink == 1 and not info.st_mode & 0o022, 'unsafe-github-output') + output.write('image_id=' + image_id + '\n') + + +def verify(ctx, target, folder, index_digest, original): + require(R.valid_digest(index_digest), 'invalid-published-digest') + require(0 < len(original) < 16384 and all(32 < ord(c) < 127 for c in original), + 'workflow-token-unavailable') + raw = read(folder, 'loaded-inspect.json') + captured = decode(read(folder, 'capture.json')) + require(isinstance(captured, dict) and captured.get('context') == ctx and + captured.get('target') == target and captured.get('inspection') == { + 'file': 'loaded-inspect.json', 'digest': R.digest(raw), 'size': len(raw)}, + 'capture-context-mismatch') + local = decode(raw) + require(captured.get('image_id') == loaded_identity(local), 'capture-image-mismatch') + package = TARGETS[target][0] + bearer = R.scoped_bearer(package, original, ctx['actor']) + blobs, relation, config = registry(package, index_digest, bearer) + artifacts = [save(folder, name, content) for name, content in blobs.items()] + result = compare(local, config, ctx['head']) + report = {'schema_version': 1, 'context': ctx, 'target': target, + 'status': 'verified-published-identity', **relation, **result, + 'artifacts': [captured['inspection'], *artifacts], + 'verifier_digest': R.digest(Path(__file__).read_bytes())} + raw_report = encoded(report) + save(folder, 'report.json', raw_report) + save(folder, 'report.sha256', (R.digest(raw_report).removeprefix('sha256:') + ' report.json\n').encode()) + print(json.dumps({'target': target, 'status': report['status'], 'report_digest': R.digest(raw_report)})) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('operation', choices=('capture', 'verify')) + parser.add_argument('--target', required=True, choices=tuple(TARGETS)) + parser.add_argument('--digest') + args = parser.parse_args() + original = os.environ.pop('GHCR_TOKEN', '') + for key in ('GH_TOKEN', 'GITHUB_TOKEN', 'GH_CONFIG_DIR', 'XDG_CONFIG_HOME', 'GH_DEBUG'): + os.environ.pop(key, None) + def stop(_signum, _frame): + raise Stop('operation-interrupted') + try: + require(sys.flags.isolated == 1, 'isolated-python-required') + os.umask(0o077) + resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) + signal.signal(signal.SIGALRM, stop) + signal.signal(signal.SIGTERM, stop) + signal.signal(signal.SIGINT, stop) + signal.alarm(300) + ctx = context(os.environ) + R.verify_checkout(ctx) + folder = directory(ctx, args.target, args.operation == 'capture') + if args.operation == 'capture': + require(args.digest is None, 'unexpected-capture-digest') + capture(ctx, args.target, folder) + else: + verify(ctx, args.target, folder, args.digest, original) + return 0 + except BaseException: + # No raw daemon/registry/auth/error bodies or signed redirect URLs in logs. + print('{"status":"stopped","reason":"published-identity-fence-failed"}') + return 1 + finally: + signal.alarm(0) + + +if __name__ == '__main__': + sys.exit(main()) From 0a971445fe3864792177130c2fff7483832f2968 Mon Sep 17 00:00:00 2001 From: Zoey Rose Date: Mon, 12 Oct 2026 18:11:42 -0500 Subject: [PATCH 20/23] fix(linux): install pinned gh-stack without API discovery --- linux/Dockerfile | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/linux/Dockerfile b/linux/Dockerfile index f3831e8..64cddfa 100644 --- a/linux/Dockerfile +++ b/linux/Dockerfile @@ -418,12 +418,31 @@ RUN install -d -o "${USERNAME}" -g "${USERNAME}" "/home/${USERNAME}/.codex" \ > "/etc/sudoers.d/${USERNAME}" \ && chmod 0440 "/etc/sudoers.d/${USERNAME}" -# gh-stack v0.1.0 is a non-immutable release tag, so retain the supported pin -# while independently enforcing its attested linux-amd64 asset digest. The +# Fetch the pinned asset directly: gh extension install still discovers the +# latest release through the API even with --pin, which can hit anonymous limits. +# gh-stack v0.1.0 is a non-immutable release tag, so independently enforce its +# attested linux-amd64 asset digest before installation. The # source commit is the verified release-workflow signer/source digest for # github/gh-stack/.github/workflows/release.yml at refs/tags/v0.1.0. -RUN HOME="/home/${USERNAME}" runuser -u "${USERNAME}" -- \ - gh extension install github/gh-stack --pin "v${GH_STACK_VERSION}" \ +RUN curl --fail --location --silent --show-error \ + "https://github.com/github/gh-stack/releases/download/v${GH_STACK_VERSION}/linux-amd64" \ + --output /tmp/gh-stack-linux-amd64 \ + && echo "${GH_STACK_SHA256_AMD64} /tmp/gh-stack-linux-amd64" | sha256sum -c - \ + && install -d -m 0755 -o "${USERNAME}" -g "${USERNAME}" \ + "/home/${USERNAME}/.local" "/home/${USERNAME}/.local/share" \ + "/home/${USERNAME}/.local/share/gh" \ + "/home/${USERNAME}/.local/share/gh/extensions" \ + "/home/${USERNAME}/.local/share/gh/extensions/gh-stack" \ + && install -m 0755 -o "${USERNAME}" -g "${USERNAME}" /tmp/gh-stack-linux-amd64 \ + "/home/${USERNAME}/.local/share/gh/extensions/gh-stack/gh-stack" \ + && printf '%s\n' 'owner: github' 'name: gh-stack' 'host: github.com' \ + "tag: v${GH_STACK_VERSION}" 'ispinned: true' \ + "path: /home/${USERNAME}/.local/share/gh/extensions/gh-stack/gh-stack" \ + > "/home/${USERNAME}/.local/share/gh/extensions/gh-stack/manifest.yml" \ + && chown "${USERNAME}:${USERNAME}" \ + "/home/${USERNAME}/.local/share/gh/extensions/gh-stack/manifest.yml" \ + && chmod 0644 "/home/${USERNAME}/.local/share/gh/extensions/gh-stack/manifest.yml" \ + && rm /tmp/gh-stack-linux-amd64 \ && echo "${GH_STACK_SHA256_AMD64} /home/${USERNAME}/.local/share/gh/extensions/gh-stack/gh-stack" \ | sha256sum -c - \ && runuser -u "${USERNAME}" -- grep -Fx 'owner: github' \ From abcbc50af65d975928563bf051dacdda38446246 Mon Sep 17 00:00:00 2001 From: Zoey Rose Date: Mon, 12 Oct 2026 19:44:49 -0500 Subject: [PATCH 21/23] fix(build): preserve Docker 1.49 image identity checks (cherry picked from commit ace7f90ef52eb232c8d3c00d856e1015f006aeb6) --- tools/tests/test_verify_published_identity.py | 94 +++++++++++++++++++ tools/verify_published_identity.py | 11 ++- 2 files changed, 102 insertions(+), 3 deletions(-) diff --git a/tools/tests/test_verify_published_identity.py b/tools/tests/test_verify_published_identity.py index 16e674c..b942f66 100644 --- a/tools/tests/test_verify_published_identity.py +++ b/tools/tests/test_verify_published_identity.py @@ -20,6 +20,11 @@ REVISION = "1" * 40 INDEX_DIGEST = "sha256:" + "9" * 64 BEARER = "FAKE_SCOPED_PULL_BEARER" +DOCKER_149_DEFAULTS = { + "Hostname": "", "Domainname": "", "Image": "", + "AttachStdin": False, "AttachStdout": False, "AttachStderr": False, + "Tty": False, "OpenStdin": False, "StdinOnce": False, +} def encoded(value): @@ -90,6 +95,37 @@ def fixture(): return local, published +def public_ubuntu_transport_fixture(api): + """Project actual public Ubuntu inspect/OCI shapes, with synthetic labels. + + Docker 29.1.3 was queried at API 1.49, 1.50 and 1.51 for the cached + ubuntu@sha256:678c6550cc43645e08669028bc177f50be4e7c5b8cca677067b1914d4afc7a03. + The public index/amd64 manifest/config chain was separately hash-verified; + its OCI config SHA256 is 86a1a31fdd84f2dc79bd6d92272100d4369b085790753bc521069eab663074e9. + Config below retains the captured non-label fields and values. Only labels + use synthetic test data; no fixture claims an Atrinik-published Ubuntu image. + """ + local, published = fixture() + config = { + "Env": ["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"], + "Cmd": ["/bin/bash"], + "Labels": {"fixture": "same"}, + } + local["Config"] = copy.deepcopy(config) + local["Config"].update({"Entrypoint": None, "OnBuild": None, "Volumes": None, + "User": "", "WorkingDir": ""}) + if api == "1.49": + local["Config"].update(DOCKER_149_DEFAULTS) + elif api not in {"1.50", "1.51"}: + raise ValueError("fixture API not captured") + published["config"] = config + published["config"]["Labels"].update({ + "org.opencontainers.image.source": M.R.SOURCE, + "org.opencontainers.image.revision": REVISION, + }) + return local, published + + def context_env(**changes): ref = changes.pop("GITHUB_REF", "refs/tags/v1.2.3") env = { @@ -168,6 +204,64 @@ def test_exact_runtime_config_rootfs_and_publication_labels_match(self): self.assertTrue(result["execution_config_equal"]) self.assertEqual(result["docker_absent_defaults"], M.DEFAULTS) + def test_public_ubuntu_inspect_and_oci_config_shapes_match_all_supported_apis(self): + for api in ("1.49", "1.50", "1.51"): + with self.subTest(api=api): + local, published = public_ubuntu_transport_fixture(api) + before = copy.deepcopy((local, published)) + result = M.compare(local, published, REVISION) + self.assertEqual((local, published), before) + self.assertTrue(result["rootfs_equal"]) + self.assertTrue(result["execution_config_equal"]) + for key, value in DOCKER_149_DEFAULTS.items(): + self.assertIs(type(result["docker_absent_defaults"][key]), type(value)) + self.assertEqual(result["docker_absent_defaults"][key], value) + + def test_each_docker149_nondefault_differs_from_omitted_or_zero(self): + for key, zero in DOCKER_149_DEFAULTS.items(): + for side in ("loaded", "published"): + for peer in ("omitted", "zero"): + with self.subTest(key=key, side=side, peer=peer): + local, published = public_ubuntu_transport_fixture("1.49") + changed, unchanged = ((local["Config"], published["config"]) + if side == "loaded" else (published["config"], local["Config"])) + changed[key] = True if type(zero) is bool else "changed" + if peer == "omitted": + unchanged.pop(key, None) + else: + unchanged[key] = zero + self.assert_stop("published-runtime-config-mismatch", M.compare, + local, published, REVISION) + + def test_each_docker149_default_rejects_wrong_types_before_normalizing(self): + for key, zero in DOCKER_149_DEFAULTS.items(): + code = "invalid-runtime-boolean" if type(zero) is bool else "invalid-runtime-string" + wrong_values = (0, 1, "false", None, [], {}) if type(zero) is bool else (0, False, None, [], {}) + for side in ("loaded", "published"): + for peer in ("omitted", "zero"): + for wrong in wrong_values: + with self.subTest(key=key, side=side, peer=peer, wrong=wrong): + local, published = public_ubuntu_transport_fixture("1.49") + changed, unchanged = ((local["Config"], published["config"]) + if side == "loaded" else (published["config"], local["Config"])) + changed[key] = wrong + if peer == "omitted": + unchanged.pop(key, None) + else: + unchanged[key] = zero + self.assert_stop(code, M.compare, local, published, REVISION) + + def test_captured_config_shape_still_rejects_unknown_zero_fields(self): + for side in ("loaded", "published", "both"): + local, published = public_ubuntu_transport_fixture("1.49") + if side in ("loaded", "both"): + local["Config"]["UnknownDockerDefault"] = False + if side in ("published", "both"): + published["config"]["UnknownDockerDefault"] = False + with self.subTest(side=side): + self.assert_stop("unknown-runtime-config-field", M.compare, + local, published, REVISION) + def test_docker_absent_defaults_are_exact_but_null_is_not_empty_array(self): local, published = fixture() for key in M.DEFAULTS: diff --git a/tools/verify_published_identity.py b/tools/verify_published_identity.py index cab8104..2e6df18 100644 --- a/tools/verify_published_identity.py +++ b/tools/verify_published_identity.py @@ -31,10 +31,15 @@ 'portable': ('atrinik/classic-portable-build', 'atrinik-classic-portable:release-validation'), } WORKFLOW = '.github/workflows/publish-linux.yml' -# Docker API 1.49–1.51 may materialize these absent fields. Never equate empty -# arrays/objects with null, nor normalize non-default values or unknown keys. +# Docker API 1.49 also materializes nine zero-valued container-only fields; +# API 1.50 removes them. All supported APIs through 1.51 materialize the other +# absence defaults below. Never equate empty arrays/objects with null, nor +# normalize non-default values or unknown keys. DEFAULTS = {'Cmd': None, 'Entrypoint': None, 'Env': None, 'Labels': None, - 'OnBuild': None, 'Volumes': None, 'User': '', 'WorkingDir': ''} + 'OnBuild': None, 'Volumes': None, 'User': '', 'WorkingDir': '', + 'Hostname': '', 'Domainname': '', 'Image': '', + 'AttachStdin': False, 'AttachStdout': False, 'AttachStderr': False, + 'Tty': False, 'OpenStdin': False, 'StdinOnce': False} CONFIG_KEYS = {'architecture', 'os', 'variant', 'os.version', 'os.features', 'config', 'rootfs', 'author', 'created', 'history', 'moby.buildkit.cache.v0'} STRING_FIELDS = {'Hostname', 'Domainname', 'User', 'Image', 'WorkingDir', 'MacAddress', 'StopSignal'} From 7a69f75afbb1c7138210db3f8b210f6521174ecc Mon Sep 17 00:00:00 2001 From: Zoey Rose Date: Mon, 12 Oct 2026 20:09:22 -0500 Subject: [PATCH 22/23] fix(ci): select Linux images for publication identity changes (cherry picked from commit 0885eadcb937221f3e793ed0cb498ce188f27e31) --- .github/workflows/validate.yml | 6 ++++ tools/tests/test_workflow_contracts.py | 39 ++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 54fd342..be82ac3 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -39,6 +39,8 @@ jobs: case "${path}" in .dockerignore | llvm-toolchain.json | llvm-bootstrap-packages.lock | \ tools/install_llvm_toolchain.py | tools/validate_llvm_toolchain.py | \ + tools/verify_published_identity.py | tools/tests/test_verify_published_identity.py | \ + tools/collect-released-sdk-metadata.py | tools/tests/test_collect_released_sdk_metadata.py | \ tools/tests/test_install_llvm_toolchain.py | tools/tests/test_workflow_contracts.py | .github/actionlint.yaml | .github/workflows/* | \ portable/* | classic-packages.lock | classic-curl-toolchain.json | \ classic-shader-toolchain.json | classic-shader-toolchain.spdx.json | \ @@ -51,6 +53,8 @@ jobs: case "${path}" in .dockerignore | llvm-toolchain.json | llvm-bootstrap-packages.lock | \ tools/install_llvm_toolchain.py | tools/validate_llvm_toolchain.py | \ + tools/verify_published_identity.py | tools/tests/test_verify_published_identity.py | \ + tools/collect-released-sdk-metadata.py | tools/tests/test_collect_released_sdk_metadata.py | \ tools/tests/test_install_llvm_toolchain.py | tools/tests/test_workflow_contracts.py | .github/actionlint.yaml | \ .github/workflows/* | linux/* | \ audio-toolchain.json | audio-toolchain.spdx.json | \ @@ -72,6 +76,8 @@ jobs: case "${path}" in .dockerignore | llvm-toolchain.json | llvm-bootstrap-packages.lock | \ tools/install_llvm_toolchain.py | tools/validate_llvm_toolchain.py | \ + tools/verify_published_identity.py | tools/tests/test_verify_published_identity.py | \ + tools/collect-released-sdk-metadata.py | tools/tests/test_collect_released_sdk_metadata.py | \ tools/tests/test_install_llvm_toolchain.py | tools/tests/test_workflow_contracts.py | .github/workflows/publish-linux.yml | \ .github/workflows/validate.yml | linux/* | \ audio-toolchain.json | audio-toolchain.spdx.json | \ diff --git a/tools/tests/test_workflow_contracts.py b/tools/tests/test_workflow_contracts.py index a561eb9..ff5fce6 100644 --- a/tools/tests/test_workflow_contracts.py +++ b/tools/tests/test_workflow_contracts.py @@ -273,6 +273,45 @@ def test_shared_llvm_inputs_select_every_image(self): self.assertEqual(set(output.read_text().splitlines()), {'portable=true', 'classic=true', 'linux=true', 'windows=true'}) + def test_publication_identity_inputs_select_linux_images(self): + script = step_script('validate.yml', 'Detect image changes') + cases = ( + ('tools/verify_published_identity.py', True, False), + ('tools/tests/test_verify_published_identity.py', True, True), + ('tools/collect-released-sdk-metadata.py', True, False), + ('tools/tests/test_collect_released_sdk_metadata.py', True, True), + ('windows/Dockerfile', False, True), + ('README.md', False, False), + ('tools/unrelated-helper.py', False, False), + ) + for changed, linux_selected, windows_selected in cases: + with self.subTest(changed=changed), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + # Real isolated Git history exercises the workflow's exact diff command. + def git(*args): + return subprocess.run(['git', *args], cwd=root, check=True, + capture_output=True, text=True).stdout.strip() + git('init', '-q') + git('-c', 'user.name=Fixture', '-c', 'user.email=fixture@example.invalid', + '-c', 'commit.gpgsign=false', 'commit', '-qm', 'base', '--allow-empty') + base = git('rev-parse', 'HEAD') + source = root / changed + source.parent.mkdir(parents=True, exist_ok=True) + source.write_text('isolated selector input\n') + git('add', '--', changed) + git('-c', 'user.name=Fixture', '-c', 'user.email=fixture@example.invalid', + '-c', 'commit.gpgsign=false', 'commit', '-qm', 'change') + output = root / 'output' + env = dict(os.environ, GITHUB_OUTPUT=str(output), + BASE_SHA=base, HEAD_SHA=git('rev-parse', 'HEAD')) + subprocess.run(['bash', '-euo', 'pipefail', '-c', script], cwd=root, + env=env, check=True, capture_output=True, text=True) + selected = 'true' if linux_selected else 'false' + windows = 'true' if windows_selected else 'false' + self.assertEqual(set(output.read_text().splitlines()), + {f'portable={selected}', f'classic={selected}', + f'linux={selected}', f'windows={windows}'}) + def test_pr_workflow_remains_private_package_free_and_strict(self): text = workflow('validate.yml') self.assertNotIn('packages:', text) From 0de3f509c9cf4d47252d44ef15d13c35fd8244db Mon Sep 17 00:00:00 2001 From: Zoey Rose Date: Mon, 12 Oct 2026 20:11:26 -0500 Subject: [PATCH 23/23] build(portable): pin current Classic consumer revision (cherry picked from commit 42d56ad8e2430dabbaef66d3b5ee7430e8f4fdbd) --- portable/contract.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/portable/contract.json b/portable/contract.json index 12251d5..3e79dfe 100644 --- a/portable/contract.json +++ b/portable/contract.json @@ -60,7 +60,7 @@ "shader_manifest": "classic-shader-toolchain.json", "consumer": { "repository": "atrinik/classic", - "commit": "bc10e27404319aff3c793aaffcfc57972449deb5", + "commit": "bd5317a03cdc82f029105c5dc26428b09fcd54b3", "shader_directory": "/opt/atrinik-portable/shaders", "cmake_argument": "ATRINIK_GPU_SHADER_DIRECTORY", "input_verification": "exact-source-commit-and-shader-input-sha256",