diff --git a/.github/workflows/qualify-released-linux-sdk.yml b/.github/workflows/qualify-released-linux-sdk.yml index 583de7b..4729572 100644 --- a/.github/workflows/qualify-released-linux-sdk.yml +++ b/.github/workflows/qualify-released-linux-sdk.yml @@ -10,6 +10,11 @@ on: - .github/workflows/qualify-released-linux-sdk.yml - tools/collect-released-sdk-metadata.py - tools/tests/test_collect_released_sdk_metadata.py + - tools/released_sdk_inventory.py + - tools/scan-released-sdk-filesystem.py + - tools/tests/test_released_sdk_inventory.py + - tools/tests/test_released_sdk_filesystem.py + - tools/RELEASED_SDK_INVENTORY.md workflow_dispatch: permissions: @@ -20,7 +25,9 @@ concurrency: cancel-in-progress: false # Private packages stay on trusted same-repository push/dispatch. This workflow -# never runs pull-request code, starts image code, publishes, or changes aliases. +# never runs pull-request code, publishes, or changes aliases. Historical targets +# never start image code. Only a future reviewed fixed final target can execute +# the separately receipted read-only filesystem scanner after created-only reads. jobs: collect: name: Collect ${{ matrix.target }} installed identity @@ -33,11 +40,13 @@ jobs: contents: read packages: read runs-on: ubuntu-26.04 - timeout-minutes: 35 + timeout-minutes: 65 + env: + PYTHONDONTWRITEBYTECODE: "1" strategy: fail-fast: false matrix: - target: [broad, classic, portable-preliminary] + target: [broad, classic, portable-preliminary, portable-final] steps: - name: Check out reviewed collector uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -45,7 +54,7 @@ jobs: persist-credentials: false - name: Test offline collection boundaries - run: python3 -I -m unittest discover -s tools/tests -p 'test_collect_released_sdk_metadata.py' -v + run: python3 -I -B -m unittest discover -s tools/tests -p 'test*released*sdk*.py' -v - name: Collect fixed released image metadata env: @@ -55,7 +64,7 @@ jobs: run: | set -euo pipefail env -u SSLKEYLOGFILE -u OPENSSL_CONF -u OPENSSL_MODULES \ - python3 -I tools/collect-released-sdk-metadata.py --target "${TARGET}" + python3 -I -B tools/collect-released-sdk-metadata.py --target "${TARGET}" - name: Retain bounded identity receipts if: ${{ always() && !cancelled() }} diff --git a/AGENTS.md b/AGENTS.md index 758075a..527abf7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -120,6 +120,18 @@ candidates before promoting rolling/version aliases, and recover partial alias promotion by rerunning the failed job from the same workflow run. Do not create manual release tags as a substitute for semantic-release. +- Released SDK identity collection admits only code-pinned immutable targets on + trusted hosted push/dispatch. Preserve the original three 1.18.0 identities. + Final portable inventory qualification uses two distinct receipts: an + unstarted acquisition container and a separate non-root, read-only, + network-disabled filesystem scanner. Complete descriptor-relative enumeration + must match all producer, Debian notice and common-license records before + wrapper pins are adopted; Docker directory tar framing alone cannot prove + completeness. No caller-supplied image, revision or filesystem-path overrides. + The fixed `portable-final` target pins the reviewed 1.22.0 producer, consumer + and index/platform/config tuple. Its hosted acquisition and filesystem + qualification must succeed before wrapper metadata/inventory pins are adopted; + release publication alone does not establish those inventories. - Validate Dockerfiles with `docker build --check`. Build and smoke-test each affected image, including `classic-validation` before `classic-final` and the Windows `classic-check` target plus native bundle; note that a cold diff --git a/tools/RELEASED_SDK_INVENTORY.md b/tools/RELEASED_SDK_INVENTORY.md index 8c4e5c4..a028712 100644 --- a/tools/RELEASED_SDK_INVENTORY.md +++ b/tools/RELEASED_SDK_INVENTORY.md @@ -1,10 +1,13 @@ -# Released portable SDK inventory foundation +# Released portable SDK inventory qualification `released_sdk_inventory.py` is an internal acquisition helper for the existing collector's authenticated, task-owned `StoppedImage` and archive protocol. It has no CLI, credentials, registry operations, release pins or image execution. -It is not yet imported or invoked by `collect-released-sdk-metadata.py`; the -three fixed 1.18.0 targets and their producer mapping remain authoritative. +The collector imports it through an explicit same-checkout path under isolated +Python. Only its internal final-target path invokes inventory and execution; +`portable-final` pins the reviewed 1.22.0 release. The three fixed 1.18.0 +targets and their producer mapping remain unchanged; final targets cannot use +the historical producer-policy fallback. The helper streams `/opt/atrinik-portable` without extraction. It retains only the fixed `audio-toolchain.json`, `llvm-toolchain.json`, `curl-toolchain.json` @@ -81,19 +84,83 @@ field sets, full producer version strings and legacy first-line compatibility, then compares streamed bytes, size and executable status. It never executes a tool to obtain a version. LLVM compiler evidence retains its separate roles. -Final wiring requires actual merged Classic consumer/shader commit M, final -producer P and released image D. A subsequent reviewed change must add fixed -index, linux/amd64 manifest and config digests plus per-target producer revision; -validate M/P and shader metadata against those fixed constants; bind each saved -metadata file to the producer inventory; emit all three complete maps and their -canonical digests; and invoke the helper inside the collector's existing -cleanup-protected acquisition. Do not add caller-supplied pins, paths or trust -extensions. Original broad/classic/preliminary identities must remain unchanged. -Wrong final M/D fixtures and actual hosted unstarted-image collection remain -required at that integration point. Cleanup failure must still invalidate the -receipt. Even successful acquisition retains `installed-identity-only`, -`image_code_executed: false` and `final_portable_cohort: false`; it does not prove -runtime, export, gameplay or audible behavior. +The collector's internal final path now binds every retained portable metadata +file to the producer map, checks corresponding source and shader-input hashes, +passes that same authenticated `InstalledInventory` reader to source/provider +closure validation, checks the consumer in both contract and shader record, retains all twelve LLVM +roles, and acquires the three build-tool identities. It writes complete +`inventories/{producer,debian_notices,common_licenses}.json` artifacts, canonical +map digests and seven metadata hashes. Saved acquisition receipts retain +`installed-identity-only`, `image_code_executed: false` and +`final_portable_cohort: false`; those fields describe its created-only acquisition +container. + +`scan-released-sdk-filesystem.py` independently enumerates the installed tree +through descriptor-relative `scandir`, `stat`, `open` and reads. Before complete +producer enumeration it retains only the four fixed audio, LLVM, portable curl +and shared curl manifests, each nonempty and at most 1 MiB, through the same +descriptor identity fences. +The resulting `ProducerInventory` binds those private payloads to the enumerated +hash and size before archive checks; payloads are excluded from serialized maps. +Producer-tree zero-byte files remain valid. Notices, licenses and single-file +metadata reads remain nonempty. Unlike directory +tar generation, any unreadable entry fails the scan. It inventories the same +fixed roots and seven files, resolves and fences notice links, and implements +the shared `ProviderReader` interface for actual libcurl/libcares bytes. Provider +reads admit only versioned canonical regular paths under `/usr/local/lib`, require +UID 0 for the file and every ancestor including `/`, and fence UID/GID with the +other descriptor identities before and after reading. The scanner hashes the three +build tools and twelve LLVM roles, and checks final filesystem identities. +`ScannerImage` uses a separate task-owned container: UID/GID 65534, read-only +root and reviewed scanner/helper mounts, no network, dropped capabilities, no +privileges or credentials, fixed `/usr/bin/python3 -I -B` entrypoint and bounded +wait/log output. Host-side image, command, source, mount and ownership fences +surround execution and cleanup. Only this owned executed scanner may be stopped +on failure; the acquisition container remains unstarted. + +Its separate `filesystem-qualification.json` receipt explicitly records +`image_code_executed: true`, the exact index/platform/config/producer/consumer, +scanner/helper hashes, run identity, its own container and cleanup outcome. +Success requires exact equality of all three maps, every metadata/tool field and +canonical digests. An omitted unrelated producer file, missing notice, changed +resolved path, malformed report, source drift or cleanup failure stops the final +collection. This is trusted-producer filesystem qualification; image Python and +OS remain trusted. It does not establish runtime, export, gameplay or audible +acceptance, and does not claim independence from a malicious image OS. + +The fixed `portable-final` row admits only `atrinik/classic-portable-build:1.22.0` +with this authenticated release identity: + +| Field | Immutable value | +| --- | --- | +| Producer revision | `eb1f6af78216adb4375826cdb70ef78a61ab403a` | +| Classic consumer | `bd5317a03cdc82f029105c5dc26428b09fcd54b3` | +| OCI index | `sha256:74761e930f5d57face2d81efb0dc50dd4ce32ce1204810e56580bf68a5972ebc` | +| Linux/amd64 manifest | `sha256:5e30957e54277bdaadeeffa42246258dc9df2086f3fb41f2125d041735ac72aa` | +| OCI config | `sha256:c35dbeac2679d79dab524a13c9a3b74a8187878ecd3b3dff52bd2892418d7a79` | + +Release publication and tested-image identity have been reviewed separately. +This row enables collection; it does not certify the final installed inventories +or filesystem equality. After source review, CI and merge, dispatch +`qualify-released-linux-sdk.yml` on reviewed `main` and inspect the actual +`released-linux-sdk-portable-final--` artifact. Both its acquisition +and filesystem qualification receipts must pass on this exact tuple before +wrapper metadata/inventory pins are adopted. Later semantic releases do not +retarget this row. No CLI, environment or workflow input can supply image pins, +producer identities or paths. A missing final policy still fails closed, and no +historical target is relabeled. Export, runtime and user hardware qualification +remain separate. + +The filesystem scanner adapts descriptor-relative enumeration and byte-recording +from the MIT-licensed wrapper at +[`a9661a2bd978107fe4e01e45b2e22fcd1e2d095c`](https://github.com/atrinik/atrinik/tree/a9661a2bd978107fe4e01e45b2e22fcd1e2d095c): +`atrinik_workspace/linux_export.py` (`_identity`, `_inventory`, `_open_root`, +`_open_beneath`) and `atrinik_workspace/linux_portable.py` (`byte_record`, +`verified_tree_inventory`). Copyright (c) 2026 Atrinik contributors; the MIT +permission and warranty notice are retained through this repository's `LICENSE`. +This reuses contemporaneously MIT-licensed infrastructure, not historical Classic +material or a historical relicensing grant. Adaptations add fixed roots, shared +budgets, independent notice resolution and a bounded machine-readable report. Offline validation: diff --git a/tools/collect-released-sdk-metadata.py b/tools/collect-released-sdk-metadata.py index 1b96d8f..d337aa0 100644 --- a/tools/collect-released-sdk-metadata.py +++ b/tools/collect-released-sdk-metadata.py @@ -15,6 +15,7 @@ import base64 import hashlib import http.client +import importlib.util import json import os from pathlib import Path, PurePosixPath @@ -30,6 +31,12 @@ import tarfile import urllib.parse import uuid +from types import SimpleNamespace + +_INVENTORY_SPEC = importlib.util.spec_from_file_location('released_sdk_inventory', + Path(__file__).resolve().with_name('released_sdk_inventory.py')) +INVENTORY = importlib.util.module_from_spec(_INVENTORY_SPEC) +_INVENTORY_SPEC.loader.exec_module(INVENTORY) REVISION = '4af130578df18a1944880cd279c7f23afe9cc668' SOURCE = 'https://github.com/atrinik/devcontainer' @@ -43,7 +50,22 @@ 'sha256:86d3c4182dc786db8ace56d9d8dc57ff4f14c1c4f517b728547626f576280449'), 'portable-preliminary': ('atrinik/classic-portable-build', '1.18.0', 'sha256:ece5d3892db0ae7c8a13141720fdc62775b02cf67cf2b6df15bff9ccdb039527'), + 'portable-final': ('atrinik/classic-portable-build', '1.22.0', + 'sha256:74761e930f5d57face2d81efb0dc50dd4ce32ce1204810e56580bf68a5972ebc'), +} +# Final entries must never inherit the historical producer-policy fallback. +HISTORICAL_TARGETS = frozenset({'broad', 'classic', 'portable-preliminary'}) +FINAL_TARGETS = { + 'portable-final': { + 'producer_revision': 'eb1f6af78216adb4375826cdb70ef78a61ab403a', + 'consumer_commit': 'bd5317a03cdc82f029105c5dc26428b09fcd54b3', + 'manifest_digest': 'sha256:5e30957e54277bdaadeeffa42246258dc9df2086f3fb41f2125d041735ac72aa', + 'config_digest': 'sha256:c35dbeac2679d79dab524a13c9a3b74a8187878ecd3b3dff52bd2892418d7a79', + }, } +SCANNER_FILES = ('scan-released-sdk-filesystem.py', 'released_sdk_inventory.py') +SCANNER_LIMIT = 64 * 1024 * 1024 + INDEX = 'application/vnd.oci.image.index.v1+json' MANIFEST = 'application/vnd.oci.image.manifest.v1+json' CONFIG = 'application/vnd.oci.image.config.v1+json' @@ -206,8 +228,8 @@ def valid_digest(value): class LocalConnection(http.client.HTTPConnection): - def __init__(self, path, identity): - super().__init__('localhost', timeout=90) + def __init__(self, path, identity, timeout=90): + super().__init__('localhost', timeout=timeout) self.path, self.identity = path, identity def connect(self): @@ -306,7 +328,10 @@ def descriptor(value, media_types): return value -def verified_registry(package, index_digest, bearer): +def verified_registry(package, index_digest, bearer, *, target=None): + policy = target_policy(target) if target is not None else {'producer_revision': REVISION} + if target is not None: + require(TARGETS[target][0] == package and TARGETS[target][2] == index_digest, 'fixed-target-mismatch') index_raw = registry_object(package, 'manifests', index_digest, bearer) index = decode(index_raw) require(isinstance(index, dict) and index.get('schemaVersion') == 2 and @@ -338,7 +363,10 @@ def verified_registry(package, index_digest, bearer): config = decode(config_raw) require(config.get('os') == 'linux' and config.get('architecture') == 'amd64', 'config-platform-mismatch') labels = config.get('config', {}).get('Labels', {}) - require(labels.get('org.opencontainers.image.revision') == REVISION and + if target in FINAL_TARGETS: + require(selected['digest'] == policy['manifest_digest'] and + cfg_descriptor['digest'] == policy['config_digest'], 'final-image-coordinate-mismatch') + require(labels.get('org.opencontainers.image.revision') == policy['producer_revision'] and labels.get('org.opencontainers.image.source') == SOURCE, 'producer-label-mismatch') rootfs = config.get('rootfs', {}) require(rootfs.get('type') == 'layers' and isinstance(rootfs.get('diff_ids'), list) and @@ -387,12 +415,13 @@ def __init__(self, path, identity): self.path, self.identity = path, identity self.api = '' - def request(self, method, path, headers=None, body=None, expected=200): - conn = LocalConnection(self.path, self.identity) + def request(self, method, path, headers=None, body=None, expected=200, timeout=90): + conn = LocalConnection(self.path, self.identity, timeout=timeout) try: conn.request(method, self.api + path, body=body, headers=headers or {}) response = conn.getresponse() - require(response.status == expected, 'docker-request-denied') + statuses = expected if isinstance(expected, tuple) else (expected,) + require(response.status in statuses, 'docker-request-denied') return conn, response except BaseException: conn.close() @@ -459,19 +488,44 @@ def selected_manifest_matches(value, verified): def local_config_matches(local, expected): - if not isinstance(local, dict) or not isinstance(expected, dict): + if type(local) is not dict or type(expected) is not dict: return False + + def same_json(left, right): + # Python equates False, 0 and 0.0, including inside lists and mappings. + # Compare complete JSON values without coercing keys or non-JSON types. + if type(left) is not type(right): + return False + if type(left) is dict: + return (all(type(key) is str for key in left) and + all(type(key) is str for key in right) and left.keys() == right.keys() and + all(same_json(value, right[key]) for key, value in left.items())) + if type(left) is list: + return len(left) == len(right) and all(same_json(a, b) for a, b in zip(left, right)) + if type(left) in (type(None), bool, int, str): + return left == right + if type(left) is float: + return json.dumps(left, allow_nan=False) == json.dumps(right, allow_nan=False) + return False + local = dict(local) - # API <=1.51 emits these empty defaults; API 1.52 omits unset fields: + # API 1.49 also emits nine zero-valued container fields; 1.50 removes them: + # https://docs.docker.com/reference/api/engine/version-history/#v150-api-changes + # APIs through 1.51 emit the other empty defaults; 1.52 omits unset fields: # https://docs.docker.com/reference/api/engine/version-history/#v152-api-changes # Normalize only API-side absence defaults; preserve every raw-declared key. defaults = {'Cmd': None, 'Entrypoint': None, 'Env': None, 'Labels': None, - 'OnBuild': None, 'Volumes': None, 'User': '', 'WorkingDir': ''} - for key, default in defaults.items(): - if (key not in expected and key in local and type(local[key]) is type(default) - and local[key] == default): - del local[key] - return local == expected + 'OnBuild': None, 'Volumes': None, 'User': '', 'WorkingDir': '', + 'Hostname': '', 'Domainname': '', 'Image': '', + 'AttachStdin': False, 'AttachStdout': False, 'AttachStderr': False, + 'Tty': False, 'OpenStdin': False, 'StdinOnce': False} + try: + for key, default in defaults.items(): + if key not in expected and key in local and same_json(local[key], default): + del local[key] + return same_json(local, expected) + except (ValueError, RecursionError): + return False def verify_local(local, reference, sha, verified): @@ -731,7 +785,7 @@ def parents(self, path): self.directories.add(str(parent)) def read(self, path, *, retain, executable=False): - allowed = set(COMMON_FILES + TARGET_FILES[self.target]) if retain else { + allowed = set(COMMON_FILES + (PORTABLE_FILES if self.target in FINAL_TARGETS else TARGET_FILES[self.target])) if retain else { PREFIX + '/' + pair[1] for pair in TOOL_FILES.values()} require(path in allowed, 'image-path-not-allowed') self.fence() @@ -857,10 +911,238 @@ def validate_metadata(target, data): return compiler + +def canonical_json(value): + return (json.dumps(value, sort_keys=True, separators=(',', ':'), allow_nan=False) + '\n').encode() + + +def target_policy(target): + require(target in TARGETS, 'target-not-allowed') + if target not in FINAL_TARGETS: + require(target in HISTORICAL_TARGETS, 'final-target-policy-missing') + return {'producer_revision': REVISION} + value = FINAL_TARGETS[target] + require(isinstance(value, dict) and set(value) == { + 'producer_revision', 'consumer_commit', 'manifest_digest', 'config_digest'} + and TARGETS[target][0] == 'atrinik/classic-portable-build' + and all(isinstance(value[key], str) and re.fullmatch(r'[0-9a-f]{40}', value[key]) + for key in ('producer_revision', 'consumer_commit')) + and all(valid_digest(value[key]) for key in ('manifest_digest', 'config_digest')), + 'final-target-policy-incomplete') + return value + + +def collect_final_inventory(container, data, file_records, llvm_tools, policy): + protocol = SimpleNamespace(BINARY_LIMIT=BINARY_LIMIT, stat_matches=stat_matches, regular_mode=regular_mode, + path_stat=path_stat, single_file_archive=single_file_archive, pax_times=pax_times) + reader = INVENTORY.InstalledInventory(container, protocol) + raw = {path.removeprefix(INVENTORY.ROOT + '/'): value for path, value in data.items() + if path.startswith(INVENTORY.ROOT + '/')} + records = {name: file_records[INVENTORY.ROOT + '/' + name] for name in raw} + documents = {name: decode(raw[name]) for name in INVENTORY.METADATA_FILES} + contract, shader = documents['contract.json'], documents['shader-generation.json'] + require(contract.get('consumer', {}).get('commit') == policy['consumer_commit'] and + contract['consumer'].get('repository') == 'atrinik/classic' and + shader.get('source_commit') == policy['consumer_commit'], 'final-consumer-mismatch') + maps = {'producer': reader.producer()} + INVENTORY.bind_metadata(maps['producer'], raw, records) + INVENTORY.crosscheck_archives(maps['producer'], documents['runtime-sources.json'], contract, reader=reader) + inputs = shader.get('source_inputs') + require(isinstance(inputs, dict) and 0 < len(inputs) <= INVENTORY.MAX_MEMBERS, 'final-shader-inputs') + for path, sha in inputs.items(): + INVENTORY.relative_path(path) + item = maps['producer'].get('sources/classic-shaders/' + path) + require(isinstance(item, dict) and item.get('sha256') == sha, 'final-shader-input-bytes') + maps['debian_notices'] = reader.debian_notices(documents['runtime-sources.json'], documents['debian-sources.json']) + maps['common_licenses'] = reader.common_licenses() + tools = reader.build_tools(documents['installed.json']) + require(set(llvm_tools) == set(TOOL_FILES) == set(INVENTORY.LLVM_TOOL_FILES), 'final-llvm-roles') + llvm = {role: {key: item[key] for key in ('path', 'canonical_path', 'sha256', 'size')} + | {'executable': bool(int(item['mode'], 8) & 0o111)} for role, item in llvm_tools.items()} + return INVENTORY.expected_scan(maps, {name: records[name] for name in INVENTORY.METADATA_FILES}, tools, llvm) + + +def scanner_sources(): + result = {} + for name in SCANNER_FILES: + path = Path(__file__).resolve().with_name(name) + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + try: + before = os.fstat(fd) + require(stat.S_ISREG(before.st_mode) and before.st_uid == os.getuid() + and before.st_mode & 0o022 == 0 and 0 < before.st_size <= 1024 * 1024, + 'unsafe-scanner-source') + raw = os.read(fd, before.st_size + 1) + def identity(value): + return (value.st_dev, value.st_ino, value.st_mode, value.st_uid, value.st_gid, + value.st_size, value.st_mtime_ns, value.st_ctime_ns) + require(len(raw) == before.st_size and identity(os.fstat(fd)) == identity(before) + and identity(path.stat(follow_symlinks=False)) == identity(before), 'scanner-source-changed') + result[name] = {'path': str(path), 'sha256': hashlib.sha256(raw).hexdigest(), 'size': len(raw)} + finally: + os.close(fd) + return result + + +def scanner_logs(response): + require(response.chunked or response.length is not None, 'scanner-logs-unframed') + result = bytearray() + while True: + header = response.read(8) + if not header: + break + require(len(header) == 8 and header[:4] == b'\x01\0\0\0', 'scanner-logs-invalid') + size = struct.unpack('>I', header[4:])[0] + require(0 < size <= 1024 * 1024 and len(result) + size <= SCANNER_LIMIT, 'scanner-logs-limit') + result.extend(exact_read(response, size)) + require(response.length in (None, 0) and result, 'scanner-logs-incomplete') + return decode(bytes(result)) + + +class ScannerImage(StoppedImage): + """Separate executed qualification; never weakens StoppedImage's fence.""" + def __init__(self, docker, target, context, verified, sources): + require(target in FINAL_TARGETS, 'final-target-required') + policy = target_policy(target) + require(verified['manifest_digest'] == policy['manifest_digest'] and + verified['config_digest'] == policy['config_digest'], 'final-image-coordinate-mismatch') + require(set(sources) == set(SCANNER_FILES), 'scanner-source-set') + super().__init__(docker, target, context, verified) + self.name = 'atrinik-sdk-filesystem-' + self.owner + self.sources = sources + self.entrypoint = ['/usr/bin/python3', '-I', '-B', '/scan/scan-released-sdk-filesystem.py'] + self.mounts = [{'Type': 'bind', 'Source': item['path'], 'Target': '/scan/' + name, + 'ReadOnly': True} for name, item in sources.items()] + + def create(self): + require(not self.verified['config_object'].get('config', {}).get('Volumes'), 'image-volumes-not-allowed') + body = {'Image': self.reference, 'Entrypoint': self.entrypoint, 'Cmd': [], 'Env': [], + 'Labels': {OWNER_LABEL: self.owner}, 'User': '65534:65534', 'WorkingDir': '/', + 'NetworkDisabled': True, 'Healthcheck': {'Test': ['NONE']}, + 'HostConfig': {'NetworkMode': 'none', 'ReadonlyRootfs': True, 'CapDrop': ['ALL'], + 'SecurityOpt': ['no-new-privileges'], 'RestartPolicy': {'Name': 'no'}, + 'Mounts': self.mounts, 'LogConfig': {'Type': 'local', 'Config': {'max-size': '128m', 'max-file': '1'}}}} + self.attempted = True + conn, response = self.docker.request('POST', '/containers/create?' + urllib.parse.urlencode({ + 'name': self.name, 'platform': 'linux/amd64'}), {'Content-Type': 'application/json'}, + json.dumps(body).encode(), expected=201) + try: + value = decode(bounded_read(response)) + finally: + conn.close() + identifier = value.get('Id') + require(isinstance(identifier, str) and re.fullmatch(r'[0-9a-f]{64}', identifier), 'invalid-created-container') + self.identifier = identifier + self.fence() + + def fence(self, states=('created',)): + require(self.identifier is not None, 'container-not-created') + value = self.docker.get('/containers/' + self.identifier + '/json') + config, host = value.get('Config', {}), value.get('HostConfig', {}) + state = value.get('State', {}) + require(value.get('Id') == self.identifier and value.get('Name') == '/' + self.name and + config.get('Labels', {}).get(OWNER_LABEL) == self.owner and config.get('Image') == self.reference, + 'scanner-ownership-mismatch') + require(config.get('Entrypoint') == self.entrypoint and config.get('Cmd') in (None, []) + and config.get('User') == '65534:65534' and config.get('WorkingDir') == '/', + 'scanner-command-mismatch') + require(state.get('Status') in states and state.get('Running') is (state.get('Status') == 'running') + and (state.get('Status') == 'running' or state.get('Pid') == 0), 'scanner-state-mismatch') + require(host.get('ReadonlyRootfs') is True and host.get('NetworkMode') == 'none' + and not host.get('Privileged') and not host.get('Binds') + and host.get('CapDrop') == ['ALL'] and host.get('SecurityOpt') == ['no-new-privileges'] + and host.get('RestartPolicy', {}).get('Name') == 'no', 'scanner-isolation-mismatch') + mounts = value.get('Mounts') + require(isinstance(mounts, list) and len(mounts) == len(self.mounts), 'scanner-mount-mismatch') + actual = {(item.get('Source'), item.get('Destination')) for item in mounts + if item.get('Type') == 'bind' and item.get('RW') is False} + require(actual == {(item['Source'], item['Target']) for item in self.mounts}, 'scanner-mount-mismatch') + image = value.get('Image') + require(image in (self.verified['config_digest'], self.index_digest, self.verified['manifest_digest']), + 'container-image-mismatch') + selected = value.get('ImageManifestDescriptor') + if selected is not None or image != self.verified['config_digest']: + require(selected_manifest_matches(selected, self.verified), 'container-manifest-mismatch') + return value + + def execute(self): + self.fence() + conn, _ = self.docker.request('POST', '/containers/' + self.identifier + '/start', expected=204) + conn.close() + conn, response = self.docker.request('POST', '/containers/' + self.identifier + '/wait?condition=not-running', + timeout=INVENTORY.MAX_SECONDS + 30) + try: + result = decode(bounded_read(response)) + require(type(result.get('StatusCode')) is int and result['StatusCode'] == 0 and + not result.get('Error'), 'scanner-process-failed') + finally: + conn.close() + self.fence(('exited',)) + conn, response = self.docker.request('GET', '/containers/' + self.identifier + '/logs?stdout=1&stderr=1') + try: + return scanner_logs(response) + finally: + conn.close() + + def cleanup(self): + if not self.attempted: + return + if self.identifier is None: + # Resolve only our unique name after an ambiguous create response. + value = self.docker.get('/containers/' + self.name + '/json') + identifier = value.get('Id') + require(isinstance(identifier, str) and re.fullmatch(r'[0-9a-f]{64}', identifier), 'cleanup-identity-invalid') + self.identifier = identifier + value = self.fence(('created', 'running', 'exited')) + if value['State']['Status'] == 'running': + conn, _ = self.docker.request('POST', '/containers/' + self.identifier + '/stop?t=10', + expected=(204, 304)) + conn.close() + self.fence(('exited',)) + conn, _ = self.docker.request('DELETE', '/containers/' + self.identifier, expected=204) + conn.close() + self.attempted = False + + +def qualify_filesystem(docker, target, context, verified, expected): + policy = target_policy(target) + require(target in FINAL_TARGETS, 'final-target-required') + receipt = {**context, 'schema_version': 1, 'claim': 'trusted-producer-filesystem-equality', + 'image_code_executed': True, 'final_portable_cohort': False, 'target': target, + 'index_digest': TARGETS[target][2], 'manifest_digest': verified['manifest_digest'], + 'config_digest': verified['config_digest'], **policy, 'status': 'started'} + scanner = None + try: + require(verified['manifest_digest'] == policy['manifest_digest'] and + verified['config_digest'] == policy['config_digest'], 'final-image-coordinate-mismatch') + sources = scanner_sources() + receipt['scanner_sources'] = {name: {key: item[key] for key in ('sha256', 'size')} + for name, item in sources.items()} + scanner = ScannerImage(docker, target, context, verified, sources) + scanner.create() + receipt['container_id'] = scanner.identifier + observed = scanner.execute() + require(scanner_sources() == sources, 'scanner-source-changed') + INVENTORY.compare_scan(observed, expected) + receipt['scan'] = observed + receipt['status'] = 'verified-filesystem-equality' + except BaseException: + receipt.update(status='stopped', reason='filesystem-qualification-failed') + finally: + if scanner is not None: + try: + signal.alarm(120) + scanner.cleanup() + receipt['owned_container_removed'] = True + except BaseException: + receipt.update(status='stopped', reason='owned-cleanup-failed', owned_container_removed=False) + return receipt + def collect(target, context, original, folder): package, tag, sha = TARGETS[target] + policy = target_policy(target) receipt = {**context, 'schema_version': 1, 'target': target, 'package': package, - 'display_tag': tag, 'index_digest': sha, 'producer_revision': REVISION, + 'display_tag': tag, 'index_digest': sha, 'producer_revision': policy['producer_revision'], 'source': SOURCE, 'platform': 'linux/amd64', 'status': 'started', 'claim': 'installed-identity-only', 'image_code_executed': False, 'final_portable_cohort': False, 'artifacts': [], 'tool_bytes': {}} @@ -869,7 +1151,7 @@ def collect(target, context, original, folder): docker = runner_docker() receipt.update(docker.negotiate()) bearer = scoped_bearer(package, original, context['actor']) - verified = verified_registry(package, sha, bearer) + verified = verified_registry(package, sha, bearer, target=target) for name in ('index', 'manifest', 'config'): receipt['artifacts'].append(save(folder, 'registry/' + name + '.json', verified[name])) receipt.update({'manifest_digest': verified['manifest_digest'], @@ -880,10 +1162,12 @@ def collect(target, context, original, folder): container = StoppedImage(docker, target, context, verified) container.create() receipt['container_id'] = container.identifier - data = {} - for path in COMMON_FILES + TARGET_FILES[target]: - _, raw = container.read(path, retain=True) + data, file_records = {}, {} + for path in COMMON_FILES + (PORTABLE_FILES if target in FINAL_TARGETS else TARGET_FILES[target]): + observed_file, raw = container.read(path, retain=True) data[path] = raw + file_records[path] = {'sha256': observed_file['sha256'], 'size': observed_file['size'], + 'executable': bool(int(observed_file['mode'], 8) & 0o111)} compiler = validate_metadata(target, data) for path, raw in data.items(): receipt['artifacts'].append(save(folder, 'files' + path, raw)) @@ -906,6 +1190,17 @@ def collect(target, context, original, folder): identity['producer_descriptor_verified'] = False receipt['tool_bytes'][role] = identity container.fence() + if target in FINAL_TARGETS: + final_inventory = collect_final_inventory(container, data, file_records, receipt['tool_bytes'], policy) + for name, records in final_inventory['inventories'].items(): + receipt['artifacts'].append(save(folder, 'inventories/' + name + '.json', canonical_json(records))) + receipt['inventory_sha256'] = final_inventory['inventory_sha256'] + receipt['metadata_sha256'] = {name: item['sha256'] for name, item in final_inventory['metadata'].items()} + receipt['build_tools'] = final_inventory['build_tools'] + # Keep acquisition and execution claims separate, including failure. + qualification = qualify_filesystem(docker, target, context, verified, final_inventory) + receipt['artifacts'].append(save(folder, 'filesystem-qualification.json', canonical_json(qualification))) + require(qualification['status'] == 'verified-filesystem-equality', 'filesystem-qualification-failed') receipt['status'] = 'verified-installed-identity' except Stop as error: receipt.update({'status': 'stopped', 'reason': str(error)}) @@ -945,7 +1240,7 @@ def stop(_signum, _frame): signal.signal(signal.SIGALRM, stop) signal.signal(signal.SIGTERM, stop) signal.signal(signal.SIGINT, stop) - signal.alarm(1800) + signal.alarm(3600 if args.target in FINAL_TARGETS else 1800) context = runner_context(os.environ) root = verify_checkout(context) require(isinstance(original, str) and 0 < len(original) < 16384 and @@ -954,6 +1249,8 @@ def stop(_signum, _frame): receipt = collect(args.target, context, original, folder) del original receipt['collector_sha256'] = hashlib.sha256(Path(__file__).read_bytes()).hexdigest() + receipt['inventory_helper_sha256'] = hashlib.sha256( + Path(__file__).with_name('released_sdk_inventory.py').read_bytes()).hexdigest() save(folder, 'receipt.json', (json.dumps(receipt, indent=2, sort_keys=True) + '\n').encode()) # The receipt itself is bound by a separate hash; never print raw registry, # archive, daemon, auth or exception content. diff --git a/tools/released_sdk_inventory.py b/tools/released_sdk_inventory.py index afc0697..0240278 100644 --- a/tools/released_sdk_inventory.py +++ b/tools/released_sdk_inventory.py @@ -33,6 +33,25 @@ SOURCE_MANIFESTS = ('audio-toolchain.json', 'llvm-toolchain.json', 'curl-toolchain.json', 'classic-curl-toolchain.json') MAX_MANIFEST = 1024 * 1024 +METADATA_FILES = ('contract.json', 'installed.json', 'compiler-evidence.json', + 'runtime-abi.json', 'runtime-sources.json', 'debian-sources.json', + 'shader-generation.json') +LLVM_PREFIX = '/opt/llvm-23.1.3' +LLVM_TOOL_FILES = { + 'compiler': ('bin/clang', 'bin/clang-23'), + 'cxx_compiler': ('bin/clang++', 'bin/clang-23'), + 'linker': ('bin/ld.lld', 'bin/lld'), + 'readelf': ('bin/llvm-readelf', 'bin/llvm-readobj'), + 'archiver': ('bin/llvm-ar', 'bin/llvm-ar'), + 'ranlib': ('bin/llvm-ranlib', 'bin/llvm-ar'), + 'nm': ('bin/llvm-nm', 'bin/llvm-nm'), + 'strip': ('bin/llvm-strip', 'bin/llvm-objcopy'), + 'objcopy': ('bin/llvm-objcopy', 'bin/llvm-objcopy'), + 'objdump': ('bin/llvm-objdump', 'bin/llvm-objdump'), + 'runtime': ('lib/clang/23/lib/linux/libclang_rt.builtins-x86_64.a', + 'lib/clang/23/lib/linux/libclang_rt.builtins-x86_64.a'), + 'unwinder': ('lib/libunwind.a', 'lib/libunwind.a'), +} GO_DIRECTORY = 1 << 31 GO_SYMLINK = 1 << 27 @@ -624,33 +643,73 @@ def debian_notices(self, runtime, debian): def build_tools(self, installed): - require(isinstance(installed, dict), 'build-tool-schema') - receipt = installed.get('build_tools') - require(isinstance(receipt, dict) and set(receipt) == {'schema_version', 'tools'} - and type(receipt['schema_version']) is int and receipt['schema_version'] == 1 - and isinstance(receipt['tools'], dict) and set(receipt['tools']) == set(BUILD_TOOLS), - 'build-tool-schema') - result = {} - for role, path in BUILD_TOOLS.items(): - entry = receipt['tools'][role] - require(isinstance(entry, dict) and set(entry) == { - 'path', 'resolved_path', 'sha256', 'size', 'executable', 'version'}, 'build-tool-fields') - require(entry['path'] == path and entry['resolved_path'] == path, - 'build-tool-path') - require(isinstance(entry['sha256'], str) and re.fullmatch(r'[0-9a-f]{64}', entry['sha256']) - and type(entry['size']) is int and 0 < entry['size'] <= MAX_BUILD_TOOL - and entry['executable'] is True and isinstance(entry['version'], str) - and 0 < len(entry['version']) <= 65536 - and not any((ord(char) < 32 and char != '\n') or ord(char) == 127 - for char in entry['version']) - and re.fullmatch(r'cmake version [0-9]+\.[0-9]+\.[0-9]+' if role == 'cmake' - else r'[0-9]+\.[0-9]+(?:\.[0-9]+)?', - entry['version'].splitlines()[0]) is not None - and isinstance(installed.get('tools'), dict) - and installed['tools'].get(role) == entry['version'].splitlines()[0], - 'build-tool-identity') - observed = self._regular(path, executable=True) - require(all(observed[key] == entry[key] for key in ('sha256', 'size', 'executable')), - 'build-tool-bytes-mismatch') - result[role] = dict(entry) - return result + return build_tool_records(installed, lambda path: self._regular(path, executable=True)) + + +def build_tool_records(installed, read): + require(isinstance(installed, dict), 'build-tool-schema') + receipt = installed.get('build_tools') + require(isinstance(receipt, dict) and set(receipt) == {'schema_version', 'tools'} + and type(receipt['schema_version']) is int and receipt['schema_version'] == 1 + and isinstance(receipt['tools'], dict) and set(receipt['tools']) == set(BUILD_TOOLS), + 'build-tool-schema') + result = {} + for role, path in BUILD_TOOLS.items(): + entry = receipt['tools'][role] + require(isinstance(entry, dict) and set(entry) == { + 'path', 'resolved_path', 'sha256', 'size', 'executable', 'version'}, 'build-tool-fields') + require(entry['path'] == path and entry['resolved_path'] == path, + 'build-tool-path') + require(isinstance(entry['sha256'], str) and re.fullmatch(r'[0-9a-f]{64}', entry['sha256']) + and type(entry['size']) is int and 0 < entry['size'] <= MAX_BUILD_TOOL + and entry['executable'] is True and isinstance(entry['version'], str) + and 0 < len(entry['version']) <= 65536 + and not any((ord(char) < 32 and char != '\n') or ord(char) == 127 + for char in entry['version']) + and re.fullmatch(r'cmake version [0-9]+\.[0-9]+\.[0-9]+' if role == 'cmake' + else r'[0-9]+\.[0-9]+(?:\.[0-9]+)?', + entry['version'].splitlines()[0]) is not None + and isinstance(installed.get('tools'), dict) + and installed['tools'].get(role) == entry['version'].splitlines()[0], + 'build-tool-identity') + observed = read(path) + require(all(observed[key] == entry[key] for key in ('sha256', 'size', 'executable')), + 'build-tool-bytes-mismatch') + result[role] = dict(entry) + return result + + +def bind_metadata(producer, raw, records): + """Bind every retained producer file and require the seven-file handoff.""" + require(isinstance(producer, dict) and isinstance(raw, dict) and isinstance(records, dict) + and set(METADATA_FILES) <= set(raw) and set(records) == set(raw), + 'inventory-metadata-files') + for name, value in raw.items(): + relative_path(name) + require(isinstance(value, bytes) and len(value) <= 4 * 1024 * 1024, + 'inventory-metadata-size') + record = records[name] + require(isinstance(record, dict) and set(record) == {'sha256', 'size', 'executable'} + and type(record['size']) is int and type(record['executable']) is bool + and record['size'] == len(value) and record['sha256'] == hashlib.sha256(value).hexdigest() + and name in producer and producer[name] == record, 'inventory-metadata-binding') + + +def expected_scan(inventories, metadata, build_tools, llvm_tools): + require(set(inventories) == {'producer', 'debian_notices', 'common_licenses'} + and set(metadata) == set(METADATA_FILES) and set(build_tools) == set(BUILD_TOOLS) + and set(llvm_tools) == set(LLVM_TOOL_FILES), 'inventory-report-roles') + return {'schema_version': 1, 'claim': 'trusted-producer-filesystem-inventory', + 'image_code_executed': True, 'final_portable_cohort': False, + 'inventories': inventories, + 'inventory_sha256': {key: inventory_digest(value) for key, value in inventories.items()}, + 'metadata': metadata, 'build_tools': build_tools, 'llvm_tools': llvm_tools} + + +def compare_scan(observed, expected): + # Compare complete JSON structures, not a claimed digest or Python's bool/int + # equality. This catches omitted unrelated files and every role/field change. + require(isinstance(observed, dict) and set(observed) == set(expected), 'scanner-report-shape') + require(json.dumps(observed, sort_keys=True, separators=(',', ':'), allow_nan=False) == + json.dumps(expected, sort_keys=True, separators=(',', ':'), allow_nan=False), + 'scanner-inventory-mismatch') diff --git a/tools/scan-released-sdk-filesystem.py b/tools/scan-released-sdk-filesystem.py new file mode 100644 index 0000000..b4c9145 --- /dev/null +++ b/tools/scan-released-sdk-filesystem.py @@ -0,0 +1,321 @@ +#!/usr/bin/python3 -I +# Copyright (c) 2026 Atrinik contributors +# SPDX-License-Identifier: MIT +"""Fixed-root trusted-producer qualification; no image or path CLI overrides. + +Descriptor-relative inventory/byte recording adapted from MIT-licensed +atrinik/atrinik a9661a2bd978107fe4e01e45b2e22fcd1e2d095c: +atrinik_workspace/linux_export.py (_identity, _inventory, _open_root, +_open_beneath), linux_portable.py (byte_record, verified_tree_inventory). +See RELEASED_SDK_INVENTORY.md for provenance and execution-claim boundaries. +""" +import hashlib +import importlib.util +import json +import os +import re +from pathlib import Path +import signal +import stat +import sys +import time + +_SPEC = importlib.util.spec_from_file_location('inventory_contract', + Path(__file__).resolve().with_name('released_sdk_inventory.py')) +I = importlib.util.module_from_spec(_SPEC) +_SPEC.loader.exec_module(I) +MAX_REPORT = 64 * 1024 * 1024 +MAX_DEPTH = 128 + + +def unique(items): + result = {} + for key, value in items: + I.require(key not in result, 'scanner-duplicate-json-key') + result[key] = value + return result + + +def decode(raw): + try: + return json.loads(raw, object_pairs_hook=unique, + parse_constant=lambda _: (_ for _ in ()).throw(ValueError())) + except (ValueError, UnicodeError, RecursionError): + raise I.InventoryError('scanner-invalid-json') from None + + +def identity(info): + return (info.st_dev, info.st_ino, info.st_mode, info.st_size, info.st_uid, info.st_gid, + info.st_mtime_ns, info.st_ctime_ns) + + +class Filesystem(I.ProviderReader): + """All paths are fixed by scan(); root_fd exists only for offline fixtures.""" + def __init__(self, root_fd): + self.root_fd = root_fd + self.deadline = time.monotonic() + I.MAX_SECONDS + self.total = 0 + self.observed = {} + self.links = {} + + def check(self): + I.require(time.monotonic() < self.deadline, 'scanner-deadline') + + def remember(self, path, info): + self.check() + value = identity(info) + I.require(path not in self.observed or self.observed[path] == value, 'scanner-path-drift') + self.observed[path] = value + I.require(len(self.observed) <= I.MAX_MEMBERS, 'scanner-member-limit') + + def directory(self, path, *, protected=False): + I.require(path.startswith('/'), 'scanner-path') + if path != '/': + I.relative_path(path[1:]) + fd = os.dup(self.root_fd) + current = '' + try: + root = os.fstat(fd) + I.require(not protected or (root.st_uid == 0 and root.st_mode & 0o022 == 0), + 'scanner-provider-owner') + self.remember('/', root) + for part in path.split('/')[1:]: + if not part: + continue + self.check() + child = os.open(part, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=fd) + os.close(fd) + fd = child + current += '/' + part + info = os.fstat(fd) + I.require(stat.S_ISDIR(info.st_mode) and info.st_mode & 0o022 == 0, + 'scanner-unsafe-directory') + I.require(not protected or info.st_uid == 0, 'scanner-provider-owner') + self.remember(current, info) + return fd + except BaseException: + os.close(fd) + raise + + def record(self, path, *, retain=False, executable=False, allow_empty=False, max_size=None, protected=False): + parent, name = path.rsplit('/', 1) + I.relative_path(name) + directory = self.directory(parent, protected=protected) + fd = None + try: + before = os.stat(name, dir_fd=directory, follow_symlinks=False) + I.require(stat.S_ISREG(before.st_mode) and before.st_mode & 0o6022 == 0 + and before.st_mode & 0o444 == 0o444, 'scanner-unsafe-file') + I.require(not protected or before.st_uid == 0, 'scanner-provider-owner') + if executable: + I.require(before.st_mode & 0o111 == 0o111, 'scanner-tool-not-executable') + limit = 4 * 1024 * 1024 if retain else I.MAX_FILE + if max_size is not None: + limit = min(limit, max_size) + I.require((0 if allow_empty else 1) <= before.st_size <= limit, 'scanner-file-limit') + self.total += before.st_size + I.require(self.total <= I.MAX_TOTAL, 'scanner-total-limit') + fd = os.open(name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=directory) + I.require(identity(os.fstat(fd)) == identity(before), 'scanner-path-drift') + self.remember(path, before) + digest = hashlib.sha256() + data = bytearray() if retain else None + count = 0 + while True: + self.check() + raw = os.read(fd, min(65536, before.st_size - count + 1)) + if not raw: + break + count += len(raw) + I.require(count <= before.st_size, 'scanner-file-grew') + digest.update(raw) + if data is not None: + data.extend(raw) + I.require(count == before.st_size and identity(os.fstat(fd)) == identity(before) + and identity(os.stat(name, dir_fd=directory, follow_symlinks=False)) == identity(before), + 'scanner-file-drift') + return {'sha256': digest.hexdigest(), 'size': count, + 'executable': bool(before.st_mode & 0o111)}, (bytes(data) if retain else None) + finally: + if fd is not None: + os.close(fd) + os.close(directory) + + def read_provider(self, path): + I.require(isinstance(path, str) and len(path) <= I.MAX_PATH and re.fullmatch( + r'/usr/local/lib/lib(?:curl|cares)\.so(?:\.[0-9]+)+', path), + 'scanner-provider-path') + self.fence() + record, _ = self.record(path, protected=True) + self.fence() + return record + + def tree(self): + # Retain only the four fixed source manifests, with bounds before reads. + # record() and remember() fence descriptors and tree enumeration together. + manifests = {name: self.record(I.ROOT + '/' + name, retain=True, + max_size=I.MAX_MANIFEST) + for name in I.SOURCE_MANIFESTS} + directory = self.directory(I.ROOT) + def enumerate_tree(fd, prefix, depth): + self.check() + I.require(depth <= MAX_DEPTH, 'scanner-depth-limit') + result = {} + # scandir/open/stat errors propagate: no tar producer can silently + # omit an unreadable file or directory from this traversal. + with os.scandir(fd) as entries: + for entry in entries: + self.check() + path = prefix + '/' + entry.name + I.relative_path(path[1:]) + info = os.stat(entry.name, dir_fd=fd, follow_symlinks=False) + self.remember(path, info) + result[path] = identity(info) + if stat.S_ISDIR(info.st_mode): + child = os.open(entry.name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=fd) + try: + I.require(identity(os.fstat(child)) == identity(info), 'scanner-path-drift') + result.update(enumerate_tree(child, path, depth + 1)) + finally: + os.close(child) + else: + I.require(stat.S_ISREG(info.st_mode), 'scanner-tree-special-or-link') + return result + try: + before = enumerate_tree(directory, I.ROOT, 0) + records = I.ProducerInventory() + for path, item in sorted(before.items()): + if stat.S_ISREG(item[2]): + records[path[len(I.ROOT) + 1:]], _ = self.record(path, allow_empty=True) + I.require(enumerate_tree(directory, I.ROOT, 0) == before, 'scanner-tree-drift') + I.require(bool(records), 'scanner-empty-tree') + for name, (record, payload) in manifests.items(): + I.require(records.get(name) == record and len(payload) == record['size'] + and hashlib.sha256(payload).hexdigest() == record['sha256'], + 'scanner-manifest-bytes-mismatch') + records.source_manifests[name] = payload + return records + finally: + os.close(directory) + + def resolve(self, path, boundary): + """Resolve every raw component, including links before parent hops.""" + I.require(path.startswith(boundary + '/'), 'scanner-alias-boundary') + I.relative_path(path[1:]) + boundary_parts = boundary.split('/')[1:] + current = list(boundary_parts) + pending = path[len(boundary) + 1:].split('/') + visited = set() + while pending: + self.check() + component = pending.pop(0) + I.require(component != '', 'scanner-alias-path') + if component == '.': + continue + if component == '..': + I.require(len(current) > len(boundary_parts), 'scanner-alias-boundary') + current.pop() + continue + parent = '/' + '/'.join(current) + fd = self.directory(parent) + try: + candidate = parent + '/' + component + info = os.stat(component, dir_fd=fd, follow_symlinks=False) + self.remember(candidate, info) + if stat.S_ISLNK(info.st_mode): + I.require(candidate not in visited and len(visited) < I.MAX_HOPS, + 'scanner-alias-cycle-or-limit') + visited.add(candidate) + raw = os.readlink(component, dir_fd=fd) + I.require(0 < len(raw) <= I.MAX_PATH and '\\' not in raw + and all(32 < ord(c) < 127 for c in raw), 'scanner-alias-path') + self.links[candidate] = raw + if raw.startswith('/'): + I.require(raw.startswith(boundary + '/'), 'scanner-alias-boundary') + current = list(boundary_parts) + raw = raw[len(boundary) + 1:] + pending = raw.split('/') + pending + I.require(sum(len(part) + 1 for part in current + pending) <= I.MAX_PATH, + 'scanner-alias-path') + else: + I.require(stat.S_ISDIR(info.st_mode) if pending else stat.S_ISREG(info.st_mode), + 'scanner-alias-type') + current.append(component) + finally: + os.close(fd) + return '/' + '/'.join(current) + + def fence(self): + for path, expected in list(self.observed.items()): + self.check() + if path == '/': + actual = os.fstat(self.root_fd) + else: + parent, name = path.rsplit('/', 1) + fd = self.directory(parent or '/') + try: + actual = os.stat(name, dir_fd=fd, follow_symlinks=False) + if path in self.links: + I.require(os.readlink(name, dir_fd=fd) == self.links[path], 'scanner-alias-drift') + finally: + os.close(fd) + I.require(identity(actual) == expected, 'scanner-path-drift') + + +def scan(root_fd): + fs = Filesystem(root_fd) + raw, metadata_records = {}, {} + for name in I.METADATA_FILES: + metadata_records[name], raw[name] = fs.record(I.ROOT + '/' + name, retain=True) + documents = {name: decode(value) for name, value in raw.items()} + maps = {'producer': fs.tree()} + I.bind_metadata(maps['producer'], raw, metadata_records) + I.crosscheck_archives(maps['producer'], documents['runtime-sources.json'], documents['contract.json'], reader=fs) + maps['debian_notices'] = {} + for requested in I.notice_requests(documents['runtime-sources.json'], documents['debian-sources.json']): + resolved = fs.resolve(requested, I.DOC) + record, _ = fs.record(resolved) + maps['debian_notices'][requested] = {**record, 'resolved_path': resolved} + maps['common_licenses'] = {name: fs.record(I.COMMON + '/' + name)[0] for name in I.COMMON_LICENSE_NAMES} + build_tools = I.build_tool_records(documents['installed.json'], + lambda path: fs.record(path, executable=True)[0]) + llvm_tools = {} + for role, (declared, canonical) in I.LLVM_TOOL_FILES.items(): + path, resolved = I.LLVM_PREFIX + '/' + declared, I.LLVM_PREFIX + '/' + canonical + I.require(fs.resolve(path, I.LLVM_PREFIX) == resolved, 'scanner-llvm-alias') + record, _ = fs.record(resolved, executable=role not in ('runtime', 'unwinder')) + llvm_tools[role] = {'path': path, 'canonical_path': resolved, **record} + fs.fence() + return {'schema_version': 1, 'claim': 'trusted-producer-filesystem-inventory', + 'image_code_executed': True, 'final_portable_cohort': False, + 'inventories': maps, 'inventory_sha256': {key: I.inventory_digest(value) for key, value in maps.items()}, + 'metadata': metadata_records, 'build_tools': build_tools, 'llvm_tools': llvm_tools} + + +def main(): + def stop(_number, _frame): + raise I.InventoryError('scanner-interrupted') + try: + I.require(sys.flags.isolated == 1 and len(sys.argv) == 1 and os.geteuid() != 0, + 'scanner-execution-boundary') + signal.signal(signal.SIGALRM, stop) + signal.signal(signal.SIGTERM, stop) + signal.alarm(I.MAX_SECONDS) + fd = os.open('/', os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + try: + result = scan(fd) + finally: + os.close(fd) + encoded = (json.dumps(result, sort_keys=True, separators=(',', ':')) + '\n').encode() + I.require(len(encoded) <= MAX_REPORT, 'scanner-report-limit') + sys.stdout.buffer.write(encoded) + return 0 + except BaseException: + print('{"status":"stopped","reason":"filesystem-scan-failed"}') + return 1 + finally: + signal.alarm(0) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools/tests/test_collect_released_sdk_metadata.py b/tools/tests/test_collect_released_sdk_metadata.py index 9054b23..9f6cbcd 100644 --- a/tools/tests/test_collect_released_sdk_metadata.py +++ b/tools/tests/test_collect_released_sdk_metadata.py @@ -3,6 +3,7 @@ """Offline trust-boundary tests; never read real credentials or access Docker.""" import base64 import copy +from decimal import Decimal import importlib.util import io import json @@ -22,6 +23,11 @@ PACKAGE, TAG, INDEX_DIGEST = M.TARGETS['broad'] SECRET = 'FAKE_ORIGINAL_TOKEN_NEVER_RETAIN' BEARER = 'FAKE_SCOPED_PULL_BEARER' +DOCKER_149_DEFAULTS = { + 'Hostname': '', 'Domainname': '', 'Image': '', + 'AttachStdin': False, 'AttachStdout': False, 'AttachStderr': False, + 'Tty': False, 'OpenStdin': False, 'StdinOnce': False, +} def encoded(value): @@ -116,21 +122,55 @@ def verified(self, config=None, manifest=None, index=None): def test_fixed_released_coordinates_and_target_file_sets(self): self.assertEqual(M.REVISION, '4af130578df18a1944880cd279c7f23afe9cc668') - self.assertEqual(M.TARGETS, { + self.assertEqual({name: M.TARGETS[name] for name in M.HISTORICAL_TARGETS}, { 'broad': ('atrinik/linux-build', '1.18.0', 'sha256:613cf43ab07634db005a28462a009c4c28c90dccb1b5175e13e8072b6dfe8d72'), 'classic': ('atrinik/classic-build', '1.18.0', 'sha256:86d3c4182dc786db8ace56d9d8dc57ff4f14c1c4f517b728547626f576280449'), 'portable-preliminary': ('atrinik/classic-portable-build', '1.18.0', 'sha256:ece5d3892db0ae7c8a13141720fdc62775b02cf67cf2b6df15bff9ccdb039527')}) - self.assertEqual(set(M.TARGET_FILES), set(M.TARGETS)) - for target in M.TARGETS: + self.assertEqual(M.HISTORICAL_TARGETS, {'broad', 'classic', 'portable-preliminary'}) + self.assertEqual(set(M.TARGETS), set(M.HISTORICAL_TARGETS) | {'portable-final'}) + self.assertEqual(set(M.TARGET_FILES), set(M.HISTORICAL_TARGETS)) + for target in M.HISTORICAL_TARGETS: files = M.COMMON_FILES + M.TARGET_FILES[target] self.assertEqual(len(files), len(set(files))) self.assertTrue(set(M.LLVM_NOTICES) <= set(files)) self.assertNotIn('/usr/local/share/atrinik/toolchains.json', M.CLASSIC_FILES) self.assertNotIn('/opt/atrinik-portable/installed.json', M.CLASSIC_FILES) + def test_actual_final_release_coordinates_and_missing_policy_fail_closed(self): + self.assertEqual(M.TARGETS['portable-final'], ( + 'atrinik/classic-portable-build', '1.22.0', + 'sha256:74761e930f5d57face2d81efb0dc50dd4ce32ce1204810e56580bf68a5972ebc')) + self.assertEqual(set(M.FINAL_TARGETS), {'portable-final'}) + self.assertEqual(M.target_policy('portable-final'), { + 'producer_revision': 'eb1f6af78216adb4375826cdb70ef78a61ab403a', + 'consumer_commit': 'bd5317a03cdc82f029105c5dc26428b09fcd54b3', + 'manifest_digest': 'sha256:5e30957e54277bdaadeeffa42246258dc9df2086f3fb41f2125d041735ac72aa', + 'config_digest': 'sha256:c35dbeac2679d79dab524a13c9a3b74a8187878ecd3b3dff52bd2892418d7a79'}) + with patch.dict(M.FINAL_TARGETS, {}, clear=True), self.assertRaisesRegex( + M.Stop, 'final-target-policy-missing'): + M.target_policy('portable-final') + + def test_actual_final_release_rejects_incomplete_or_malformed_policy(self): + original = M.target_policy('portable-final') + for field in original: + missing = dict(original) + del missing[field] + with self.subTest(missing=field), patch.dict( + M.FINAL_TARGETS, {'portable-final': missing}), self.assertRaisesRegex( + M.Stop, 'final-target-policy-incomplete'): + M.target_policy('portable-final') + malformed = dict(original, **{field: 'invalid'}) + with self.subTest(malformed=field), patch.dict( + M.FINAL_TARGETS, {'portable-final': malformed}), self.assertRaisesRegex( + M.Stop, 'final-target-policy-incomplete'): + M.target_policy('portable-final') + with patch.dict(M.TARGETS, {'portable-final': M.TARGETS['broad']}), self.assertRaisesRegex( + M.Stop, 'final-target-policy-incomplete'): + M.target_policy('portable-final') + def test_native_targets_collect_raw_oci_notices_and_all_twelve_roles(self): for target in ('broad', 'classic'): data, binary = self.portable_fixture() @@ -783,7 +823,7 @@ def test_portable_collection_checks_all_tool_roles_against_streamed_bytes(self): self.assertFalse(result['final_portable_cohort']) def test_collection_rejects_empty_metadata_notices_and_canonical_tools(self): - for target in M.TARGETS: + for target in M.HISTORICAL_TARGETS: for path in (M.LLVM_MANIFEST, M.LLVM_NOTICES[0], M.PREFIX + '/bin/clang-23', M.PREFIX + '/lib/libunwind.a'): data, binary = self.portable_fixture() @@ -984,6 +1024,96 @@ def test_local_config_documented_api_absence_defaults_only(self): reference, INDEX_DIGEST, verified) self.assertEqual(set(actual), set(defaults) | set(local['Config'])) + def test_captured_public_ubuntu_config_matches_supported_docker_apis(self): + # Captured Docker 29.1.3 APIs 1.49/1.50/1.51, public Ubuntu index + # sha256:678c6550cc43645e08669028bc177f50be4e7c5b8cca677067b1914d4afc7a03; + # verified OCI config 86a1a31fdd84f2dc79bd6d92272100d4369b085790753bc521069eab663074e9. + # Preserve captured non-label Config fields; fixture labels/rootfs and + # resulting descriptors are synthetic, not a released SDK identity. + config, manifest, index = registry_fixture() + config['config'].update({ + 'Env': ['PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'], + 'Cmd': ['/bin/bash'], + }) + verified = self.verified(config, manifest, index) + reference = 'ghcr.io/' + PACKAGE + '@' + INDEX_DIGEST + selected = dict(descriptor(verified['manifest'], M.MANIFEST), + platform={'os': 'linux', 'architecture': 'amd64'}) + for api in ('1.49', '1.50', '1.51'): + actual = dict(copy.deepcopy(config['config']), Entrypoint=None, + OnBuild=None, Volumes=None, User='', WorkingDir='') + if api == '1.49': + actual.update(DOCKER_149_DEFAULTS) + local = {'Os': 'linux', 'Architecture': 'amd64', 'RepoDigests': [reference], + 'RootFS': {'Type': 'layers', 'Layers': config['rootfs']['diff_ids']}, + 'Config': actual, 'Descriptor': selected} + for identity, kind in ((verified['config_digest'], 'config-digest'), + (verified['manifest_digest'], 'platform-manifest-digest')): + local['Id'] = identity + before = copy.deepcopy((local, verified)) + with self.subTest(api=api, identity=kind): + self.assertEqual(M.verify_local(local, reference, INDEX_DIGEST, verified), kind) + self.assertEqual((local, verified), before) + + def test_docker149_defaults_preserve_raw_declared_values(self): + for key, zero in DOCKER_149_DEFAULTS.items(): + nondefault = True if type(zero) is bool else 'declared' + with self.subTest(key=key): + self.assertTrue(M.local_config_matches({key: zero}, {})) + self.assertFalse(M.local_config_matches({}, {key: zero})) + for value in (zero, nondefault): + expected = {key: value} + actual = copy.deepcopy(expected) + self.assertTrue(M.local_config_matches(actual, expected)) + self.assertEqual(actual, expected) + + def test_docker149_nondefault_and_wrong_types_reject_absent_or_zero_peer(self): + for key, zero in DOCKER_149_DEFAULTS.items(): + invalid = ((True, 0, 0.0, 1, 'false', None, [], {}) if type(zero) is bool + else ('changed', 0, False, None, [], {})) + for value in invalid: + for peer in ({}, {key: zero}): + for actual, expected in (({key: value}, peer), (peer, {key: value})): + with self.subTest(key=key, actual=actual, expected=expected): + self.assertFalse(M.local_config_matches(actual, expected)) + + def test_local_config_unknown_zero_differences_are_not_normalized(self): + for value in (None, '', False, 0, [], {}): + for actual, expected in (({'UnknownDockerDefault': value}, {}), + ({}, {'UnknownDockerDefault': value})): + with self.subTest(actual=actual, expected=expected): + self.assertFalse(M.local_config_matches(actual, expected)) + + def test_local_config_complete_comparison_retains_nested_json_types(self): + shapes = (lambda value: {'ArgsEscaped': value}, + lambda value: {'Healthcheck': {'Retries': value}}, + lambda value: {'Labels': {'fixture': [value]}}, + lambda value: {'Unknown': {'nested': [value]}}) + for shape in shapes: + for value in (False, 0, 0.0, True, 1, 1.0): + expected = shape(value) + self.assertTrue(M.local_config_matches(copy.deepcopy(expected), expected)) + for changed in (False, 0, 0.0, True, 1, 1.0): + if type(value) is type(changed) and value == changed: + continue + with self.subTest(expected=expected, actual=shape(changed)): + self.assertFalse(M.local_config_matches(shape(changed), expected)) + expected = {'Healthcheck': {'Retries': 0, 'Test': ['CMD', '/bin/true']}, 'ArgsEscaped': False} + reordered = {'ArgsEscaped': False, 'Healthcheck': {'Test': ['CMD', '/bin/true'], 'Retries': 0}} + self.assertTrue(M.local_config_matches(reordered, expected)) + + def test_local_config_non_json_values_fail_closed(self): + cyclic = [] + cyclic.append(cyclic) + for value in (float('nan'), float('inf'), float('-inf'), Decimal('0'), + (0,), {0: 'coerced key'}, {0}, b'bytes', cyclic): + for key in ('Healthcheck', 'Labels', 'Unknown'): + actual = {key: {'nested': value}} + with self.subTest(key=key, value_type=type(value).__name__): + self.assertFalse(M.local_config_matches(actual, actual)) + self.assertFalse(M.local_config_matches(actual, {})) + self.assertFalse(M.local_config_matches({}, actual)) + def test_local_config_broad_shape_preserves_declared_fields(self): # Retained broad OCI config has exactly Cmd, Env and Labels. Use that # shape here; the private replay separately checks the actual raw bytes. @@ -1060,13 +1190,13 @@ def test_workflow_is_same_repo_push_dispatch_read_only_and_pinned(self): self.assertNotIn('build/llvm-released-sdk-verification-final', text) self.assertIn("github.event_name == 'push' && github.ref == '" + M.BRANCH + "'", text) self.assertIn("github.ref == 'refs/heads/main' || github.ref == '" + M.BRANCH + "'", text) - self.assertIn('target: [broad, classic, portable-preliminary]', text) + self.assertIn('target: [broad, classic, portable-preliminary, portable-final]', text) self.assertIn(' workflow_dispatch:', text) self.assertIn("github.repository == 'atrinik/devcontainer'", text) self.assertIn(' packages: read', text) self.assertNotIn(': write', text) self.assertIn('persist-credentials: false', text) - self.assertIn('python3 -I tools/collect-released-sdk-metadata.py', text) + self.assertIn('python3 -I -B tools/collect-released-sdk-metadata.py', text) self.assertIn('env -u SSLKEYLOGFILE -u OPENSSL_CONF -u OPENSSL_MODULES', text) for line in text.splitlines(): if 'uses:' in line: diff --git a/tools/tests/test_released_sdk_filesystem.py b/tools/tests/test_released_sdk_filesystem.py new file mode 100644 index 0000000..455761e --- /dev/null +++ b/tools/tests/test_released_sdk_filesystem.py @@ -0,0 +1,810 @@ +# Copyright (c) 2026 Atrinik contributors +# SPDX-License-Identifier: MIT +"""Offline exact-filesystem versus Docker-stream acceptance fixtures.""" +import copy +import hashlib +import importlib.util +import io +import json +import os +from pathlib import Path +import signal +import stat +import struct +import tempfile +import unittest +from types import SimpleNamespace +from unittest.mock import Mock, patch + +ROOT = Path(__file__).resolve().parents[2] + + +def load(name, relative): + spec = importlib.util.spec_from_file_location(name, ROOT / relative) + result = importlib.util.module_from_spec(spec) + spec.loader.exec_module(result) + return result + + +C = load('collector_for_scanner', 'tools/collect-released-sdk-metadata.py') +S = load('filesystem_scanner', 'tools/scan-released-sdk-filesystem.py') +I = S.I +TARGET = 'portable-final-fixture' +POLICY = {'producer_revision': 'a' * 40, 'consumer_commit': 'b' * 40, + 'manifest_digest': 'sha256:' + 'c' * 64, 'config_digest': 'sha256:' + 'd' * 64} + + +class Response(io.BytesIO): + chunked = True + length = None + + +def encoded(value): + return json.dumps(value, sort_keys=True, separators=(',', ':')).encode() + + +class FilesystemTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + # Fixture permissions remain real/non-root; emulate root-owned image + # stat ownership only, with explicit wrong-owner/drift controls below. + self.stat_uid = 0 + self.stat_gid = 0 + for name in ('stat', 'fstat'): + original = getattr(S.os, name) + def owned_stat(*args, _original=original, **kwargs): + info = _original(*args, **kwargs) + fields = {key: getattr(info, key) for key in dir(info) if key.startswith('st_')} + return SimpleNamespace(**(fields | {'st_uid': self.stat_uid, 'st_gid': self.stat_gid})) + operation = patch.object(S.os, name, side_effect=owned_stat) + operation.start() + self.addCleanup(operation.stop) + self.root_fd = os.open(self.root, os.O_RDONLY | os.O_DIRECTORY) + self.addCleanup(os.close, self.root_fd) + for operation in (patch.object(C, 'https_get', side_effect=AssertionError('network forbidden')), + patch.object(C.LocalConnection, 'connect', side_effect=AssertionError('Docker forbidden'))): + operation.start() + self.addCleanup(operation.stop) + + def put(self, path, raw, mode=0o644): + result = self.root / path.removeprefix('/') + result.parent.mkdir(parents=True, exist_ok=True, mode=0o755) + result.write_bytes(raw) + result.chmod(mode) + return result + + def fixture(self): + source = b'fixture corresponding source' + sha = hashlib.sha256(source).hexdigest() + # Reuse actual producer schemas and complete declared archive families. + archive_fixtures = load('scanner_source_fixtures', 'tools/tests/test_released_sdk_inventory.py') + payloads, runtime, contract = archive_fixtures.InventoryTests().manifest_fixture() + for path, raw in payloads.items(): + self.put(I.ROOT + '/' + path, raw) + tools = {} + for role, path in I.BUILD_TOOLS.items(): + raw = ('fixture ' + role).encode() + self.put(path, raw, 0o755) + tools[role] = {'path': path, 'resolved_path': path, 'sha256': hashlib.sha256(raw).hexdigest(), + 'size': len(raw), 'executable': True, + 'version': 'cmake version 3.25.1\n' if role == 'cmake' else '1.11.1'} + for declared, canonical in I.LLVM_TOOL_FILES.values(): + self.put(I.LLVM_PREFIX + '/' + canonical, b'llvm fixture', 0o755 if '/bin/' in '/' + canonical else 0o644) + if declared != canonical: + link = self.root / (I.LLVM_PREFIX + '/' + declared).removeprefix('/') + if not link.exists(): + link.symlink_to(Path(canonical).name) + docs = {name: {} for name in I.METADATA_FILES} + docs['contract.json'] = contract + docs['contract.json']['consumer']['commit'] = POLICY['consumer_commit'] + docs['installed.json'] = {'build_tools': {'schema_version': 1, 'tools': tools}, + 'tools': {role: item['version'].splitlines()[0] for role, item in tools.items()}} + docs['runtime-sources.json'] = {**runtime, 'source_packages': {'foo': '1'}} + for role, provider in runtime['custom_providers'].items(): + self.put(provider['path'], role.encode()) + docs['debian-sources.json'] = [{'package': 'foo', 'source': 'foo', 'source_version': '1', + 'notice_directory': I.DOC + '/foo'}] + docs['shader-generation.json'] = {'source_commit': POLICY['consumer_commit'], + 'source_inputs': {'client/shaders/source': sha}} + self.put(I.ROOT + '/sources/classic-shaders/client/shaders/source', source) + for name, data in docs.items(): + self.put(I.ROOT + '/' + name, encoded(data)) + self.put(I.ROOT + '/unrelated-empty', b'') + self.put(I.DOC + '/foo/copyright', b'copyright') + for name in I.COMMON_LICENSE_NAMES: + self.put(I.COMMON + '/' + name, name.encode()) + return docs + + def test_complete_scan_and_independent_omitted_file_detection(self): + self.fixture() + actual = S.scan(self.root_fd) + self.assertEqual(set(actual['metadata']), set(I.METADATA_FILES)) + self.assertEqual(len(actual['build_tools']), 3) + self.assertEqual(len(actual['llvm_tools']), 12) + self.assertTrue(actual['image_code_executed']) + I.compare_scan(actual, actual) + omitted = copy.deepcopy(actual) + del omitted['inventories']['producer']['unrelated-empty'] + omitted['inventory_sha256']['producer'] = I.inventory_digest(omitted['inventories']['producer']) + with self.assertRaisesRegex(I.InventoryError, 'scanner-inventory-mismatch'): + I.compare_scan(actual, omitted) + for section in ('producer', 'debian_notices', 'common_licenses'): + altered = copy.deepcopy(actual) + del altered['inventories'][section][next(iter(altered['inventories'][section]))] + with self.subTest(section=section), self.assertRaises(I.InventoryError): + I.compare_scan(actual, altered) + + def test_source_manifests_retained_and_bound_without_serializing_payloads(self): + self.fixture() + producer = S.Filesystem(self.root_fd).tree() + self.assertIsInstance(producer, I.ProducerInventory) + self.assertEqual(set(producer.source_manifests), set(I.SOURCE_MANIFESTS)) + for name, raw in producer.source_manifests.items(): + self.assertEqual(producer[name]['size'], len(raw)) + self.assertEqual(producer[name]['sha256'], hashlib.sha256(raw).hexdigest()) + self.assertNotIn('source_manifests', json.loads(encoded(producer))) + self.assertEqual(producer['unrelated-empty']['size'], 0) + + def test_manifest_missing_empty_and_oversize_fail_before_payload_read(self): + self.fixture() + for name in I.SOURCE_MANIFESTS: + path = self.root / (I.ROOT + '/' + name).removeprefix('/') + saved = path.read_bytes() + for size in (None, 0, I.MAX_MANIFEST + 1): + with self.subTest(name=name, size=size): + if size is None: + path.unlink() + else: + path.write_bytes(b'x' * size) + fs = S.Filesystem(self.root_fd) + record = fs.record + def bounded(path, **kwargs): + if path == I.ROOT + '/' + name: + with patch.object(S.os, 'read', side_effect=AssertionError('payload read forbidden')): + return record(path, **kwargs) + return record(path, **kwargs) + with patch.object(fs, 'record', side_effect=bounded): + with self.assertRaises((I.InventoryError, FileNotFoundError)): + fs.tree() + path.write_bytes(saved) + + def test_manifest_replacement_between_preread_and_tree_is_rejected(self): + self.fixture() + for name in I.SOURCE_MANIFESTS: + fs = S.Filesystem(self.root_fd) + original = fs.record + def record(path, **kwargs): + result = original(path, **kwargs) + if path == I.ROOT + '/' + name and kwargs.get('retain'): + actual = self.root / path.removeprefix('/') + replacement = actual.with_suffix('.replacement') + replacement.write_bytes(actual.read_bytes()) + replacement.replace(actual) + return result + with self.subTest(name=name), patch.object(fs, 'record', side_effect=record): + with self.assertRaises(I.InventoryError): + fs.tree() + + def test_retained_manifest_payload_hash_and_size_must_match_tree(self): + self.fixture() + for name in I.SOURCE_MANIFESTS: + for mismatch in ('hash', 'size'): + fs = S.Filesystem(self.root_fd) + original = fs.record + def record(path, **kwargs): + identity, raw = original(path, **kwargs) + if path == I.ROOT + '/' + name and kwargs.get('retain'): + raw = bytes([raw[0] ^ 1]) + raw[1:] if mismatch == 'hash' else b'' + return identity, raw + with self.subTest(name=name, mismatch=mismatch), patch.object(fs, 'record', side_effect=record): + with self.assertRaisesRegex(I.InventoryError, 'scanner-manifest-bytes-mismatch'): + fs.tree() + + def test_every_declared_archive_missing_empty_or_changed_fails_scan(self): + self.fixture() + fixtures = load('scanner_archive_negatives', 'tools/tests/test_released_sdk_inventory.py') + payloads, _, _ = fixtures.InventoryTests().manifest_fixture() + for name, saved in payloads.items(): + if not name.startswith('sources/'): + continue + path = self.root / (I.ROOT + '/' + name).removeprefix('/') + for raw in (None, b'', b'changed source archive'): + with self.subTest(name=name, raw=raw): + if raw is None: + path.unlink() + else: + path.write_bytes(raw) + with self.assertRaisesRegex(I.InventoryError, 'inventory-source-bytes-mismatch'): + S.scan(self.root_fd) + path.write_bytes(saved) + + def test_scanner_rejects_missing_mandatory_manifest_and_provider_declarations(self): + docs = self.fixture() + for document, key in (('contract.json', 'audio_manifest'), ('contract.json', 'compiler'), + ('contract.json', 'curl_manifest'), ('runtime-sources.json', 'custom_providers')): + bad = copy.deepcopy(docs[document]) + del bad[key] + self.put(I.ROOT + '/' + document, encoded(bad)) + with self.subTest(document=document, key=key), self.assertRaises(I.InventoryError): + S.scan(self.root_fd) + self.put(I.ROOT + '/' + document, encoded(docs[document])) + for role in ('libcurl', 'libcares'): + bad = copy.deepcopy(docs['runtime-sources.json']) + del bad['custom_providers'][role] + self.put(I.ROOT + '/runtime-sources.json', encoded(bad)) + with self.subTest(role=role), self.assertRaisesRegex(I.InventoryError, 'inventory-custom-providers'): + S.scan(self.root_fd) + self.put(I.ROOT + '/runtime-sources.json', encoded(docs['runtime-sources.json'])) + + def test_provider_namespace_ownership_and_uid_gid_fences(self): + docs = self.fixture() + path = docs['runtime-sources.json']['custom_providers']['libcurl']['path'] + for bad in ('/usr/local/lib/libcurl.so', '/usr/lib/libcurl.so.1', + '/usr/local/lib/libother.so.1', '/usr/local/lib/../lib/libcurl.so.1'): + with self.subTest(path=bad), self.assertRaisesRegex(I.InventoryError, 'scanner-provider-path'): + S.Filesystem(self.root_fd).read_provider(bad) + for target in ('/', '/usr', '/usr/local', '/usr/local/lib', path): + fs = S.Filesystem(self.root_fd) + # Inject each ancestor/file owner at the stat boundary, before guards. + original_stat, original_fstat = S.os.stat, S.os.fstat + def wrong_owner(*args, _call, **kwargs): + info = _call(*args, **kwargs) + fd = args[0] if _call is original_fstat else kwargs.get('dir_fd') + actual = os.readlink('/proc/self/fd/' + str(fd)) if fd is not None else str(args[0]) + if _call is original_stat and fd is not None: + actual += '/' + str(args[0]) + if actual == str(self.root) + ('' if target == '/' else target): + info.st_uid = 1000 + return info + with self.subTest(owner=target), patch.object(S.os, 'stat', side_effect=lambda *a, **k: wrong_owner(*a, _call=original_stat, **k)), \ + patch.object(S.os, 'fstat', side_effect=lambda *a, **k: wrong_owner(*a, _call=original_fstat, **k)): + with self.assertRaisesRegex(I.InventoryError, 'scanner-provider-owner'): + fs.read_provider(path) + for field in ('stat_uid', 'stat_gid'): + fs = S.Filesystem(self.root_fd) + fs.read_provider(path) + setattr(self, field, 17) + with self.subTest(field=field), self.assertRaises(I.InventoryError): + fs.fence() + setattr(self, field, 0) + + def test_provider_read_fences_gid_drift_and_rejects_writable_root(self): + docs = self.fixture() + path = docs['runtime-sources.json']['custom_providers']['libcurl']['path'] + fs = S.Filesystem(self.root_fd) + original = fs.record + def drift(*args, **kwargs): + value = original(*args, **kwargs) + self.stat_gid = 17 + return value + with patch.object(fs, 'record', side_effect=drift), self.assertRaises(I.InventoryError): + fs.read_provider(path) + self.stat_gid = 0 + self.root.chmod(0o777) + self.addCleanup(self.root.chmod, 0o700) + with self.assertRaisesRegex(I.InventoryError, 'scanner-provider-owner'): + S.Filesystem(self.root_fd).read_provider(path) + + def test_provider_missing_changed_empty_symlink_and_unsafe_mode_fail(self): + docs = self.fixture() + for role, provider in docs['runtime-sources.json']['custom_providers'].items(): + actual = self.root / provider['path'].removeprefix('/') + for state in ('missing', 'changed', 'empty', 'symlink', 'writable'): + saved = actual.read_bytes() + actual.unlink() + if state == 'symlink': + actual.symlink_to('another-library') + elif state != 'missing': + actual.write_bytes(b'' if state == 'empty' else b'wrong' if state == 'changed' else saved) + actual.chmod(0o666 if state == 'writable' else 0o644) + with self.subTest(role=role, state=state), self.assertRaises((I.InventoryError, FileNotFoundError)): + S.scan(self.root_fd) + if actual.exists() or actual.is_symlink(): + actual.unlink() + actual.write_bytes(saved) + actual.chmod(0o644) + + def test_notices_and_common_licenses_remain_nonempty(self): + self.fixture() + for path in (I.DOC + '/foo/copyright', I.COMMON + '/' + I.COMMON_LICENSE_NAMES[0]): + actual = self.root / path.removeprefix('/') + original = actual.read_bytes() + actual.write_bytes(b'') + with self.subTest(path=path), self.assertRaisesRegex(I.InventoryError, 'scanner-file-limit'): + S.scan(self.root_fd) + actual.write_bytes(original) + + def test_unreadable_directory_fails_instead_of_silent_omission(self): + self.fixture() + denied = self.root / (I.ROOT + '/unreadable').removeprefix('/') + denied.mkdir() + denied.chmod(0) + self.addCleanup(denied.chmod, 0o755) + # Real kernel permission failure under the required native non-root UID. + self.assertNotEqual(os.geteuid(), 0) + with self.assertRaises(PermissionError): + S.scan(self.root_fd) + + def test_special_link_tree_and_unreadable_regular_fail(self): + self.fixture() + extra = self.root / (I.ROOT + '/unexpected').removeprefix('/') + extra.symlink_to('/etc/passwd') + with self.assertRaisesRegex(I.InventoryError, 'scanner-tree-special-or-link'): + S.scan(self.root_fd) + extra.unlink() + os.mkfifo(extra) + with self.assertRaises(I.InventoryError): + S.scan(self.root_fd) + extra.unlink() + extra.write_bytes(b'unreadable') + extra.chmod(0) + with self.assertRaises(I.InventoryError): + S.scan(self.root_fd) + + def test_tree_directory_replaced_with_link_is_rejected(self): + self.fixture() + directory = self.root / (I.ROOT + '/sources/debian').removeprefix('/') + directory.rename(directory.with_name('moved')) + directory.symlink_to('moved', target_is_directory=True) + with self.assertRaises(I.InventoryError): + S.scan(self.root_fd) + + def test_limits_and_post_read_identity_fence(self): + self.fixture() + for name, limit in (('MAX_MEMBERS', 2), ('MAX_TOTAL', 1), ('MAX_FILE', 1), ('MAX_SECONDS', 0)): + with self.subTest(name=name), patch.object(I, name, limit), self.assertRaises(I.InventoryError): + S.scan(self.root_fd) + fs = S.Filesystem(self.root_fd) + fs.record(I.ROOT + '/unrelated-empty', allow_empty=True) + self.put(I.ROOT + '/unrelated-empty', b'changed') + with self.assertRaisesRegex(I.InventoryError, 'scanner-path-drift'): + fs.fence() + + def test_notice_alias_resolution_and_escape_cycle_drift(self): + self.fixture() + copyright_path = self.root / (I.DOC + '/foo/copyright').removeprefix('/') + copyright_path.unlink() + self.put(I.DOC + '/bar/copyright', b'copyright') + copyright_path.symlink_to('../bar/copyright') + actual = S.scan(self.root_fd) + self.assertEqual(actual['inventories']['debian_notices'][I.DOC + '/foo/copyright']['resolved_path'], + I.DOC + '/bar/copyright') + for raw in ('../../../etc/passwd', 'copyright', '/etc/passwd'): + copyright_path.unlink() + copyright_path.symlink_to(raw) + with self.subTest(raw=raw), self.assertRaises(I.InventoryError): + S.scan(self.root_fd) + copyright_path.unlink() + copyright_path.symlink_to('../bar/copyright') + fs = S.Filesystem(self.root_fd) + fs.resolve(I.DOC + '/foo/copyright', I.DOC) + copyright_path.unlink() + copyright_path.symlink_to('../bar/other') + with self.assertRaises(I.InventoryError): + fs.fence() + + def test_metadata_tools_and_report_exact_field_mismatches(self): + self.fixture() + actual = S.scan(self.root_fd) + for edit in ('metadata', 'build_tools', 'llvm_tools', 'inventory_sha256', 'bool-type', 'extra'): + altered = copy.deepcopy(actual) + if edit == 'bool-type': + altered['image_code_executed'] = 1 + elif edit == 'extra': + altered['unexpected'] = True + else: + del altered[edit][next(iter(altered[edit]))] + with self.subTest(edit=edit), self.assertRaises(I.InventoryError): + I.compare_scan(altered, actual) + for role, path in I.BUILD_TOOLS.items(): + with self.subTest(role=role): + original = (self.root / path[1:]).read_bytes() + self.put(path, b'changed', 0o755) + with self.assertRaisesRegex(I.InventoryError, 'build-tool-bytes-mismatch'): + S.scan(self.root_fd) + self.put(path, original, 0o755) + + def test_collector_inventory_binding_consumer_and_shader_checks(self): + docs = self.fixture() + actual = S.scan(self.root_fd) + raw = {I.ROOT + '/' + name: encoded(value) for name, value in docs.items()} + records = {I.ROOT + '/' + name: value for name, value in actual['metadata'].items()} + llvm = {role: {**item, 'mode': '0o755' if item['executable'] else '0o644'} + for role, item in actual['llvm_tools'].items()} + reader = C.INVENTORY.InstalledInventory(Mock(), Mock()) + reader.producer = Mock() + reader.debian_notices = Mock() + reader.common_licenses = Mock() + reader.build_tools = Mock() + fixtures = load('collector_provider_archives', 'tools/tests/test_released_sdk_inventory.py') + image = fixtures.InventoryTests().installed_image() + reader.producer.return_value = actual['inventories']['producer'] + reader.debian_notices.return_value = actual['inventories']['debian_notices'] + reader.common_licenses.return_value = actual['inventories']['common_licenses'] + reader.build_tools.return_value = actual['build_tools'] + # The combined dependency test uses the real provider-reader interface. + source_check = C.INVENTORY.crosscheck_archives + def reader_check(producer, runtime, contract, *, reader): + self.assertIs(reader, expected_reader) + return source_check(producer, runtime, contract, reader=reader) + expected_reader = reader + reader_class = C.INVENTORY.InstalledInventory + def new_reader(cls, *args): + return expected_reader + with patch.object(reader_class, '__new__', new_reader), \ + patch.object(C.INVENTORY, 'crosscheck_archives', side_effect=reader_check) as crosscheck: + self.assertEqual(C.collect_final_inventory(image, raw, records, llvm, POLICY), actual) + self.assertIs(crosscheck.call_args.kwargs['reader'], reader) + # Real _regular/read_provider and the collector's actual protocol + # stream root/ancestor headers and both binary payloads. No helper + # provider-read method is mocked, so a missing adapter field fails here. + routes = [call.args[1] for call in image.docker.request.call_args_list] + self.assertTrue(any('path=%2F' == route.split('?')[1] for route in routes)) + for role in ('libcurl', 'libcares'): + self.assertTrue(any(role + '.so.1.0.0' in route for route in routes)) + for connection in image.connections: + connection.close.assert_called_once() + self.assertGreaterEqual(image.fence.call_count, 4) + for path in ('/', '/usr', '/usr/local', '/usr/local/lib', + '/usr/local/lib/libcurl.so.1.0.0', '/usr/local/lib/libcares.so.1.0.0'): + wrong_owner = fixtures.InventoryTests().installed_image(metadata={path: {'uid': 1000}}) + with self.subTest(owner=path), self.assertRaisesRegex( + (C.INVENTORY.InventoryError, C.Stop), + 'inventory-directory-owner|archive-owner-mismatch'): + C.collect_final_inventory(wrong_owner, raw, records, llvm, POLICY) + for connection in wrong_owner.connections: + connection.close.assert_called_once() + payloads = {'/usr/local/lib/' + role + '.so.1.0.0': role.encode() + for role in ('libcurl', 'libcares')} + for role in ('libcurl', 'libcares'): + bad_bytes = {**payloads, '/usr/local/lib/' + role + '.so.1.0.0': b'changed'} + wrong_bytes = fixtures.InventoryTests().installed_image(payloads=bad_bytes) + with self.subTest(bytes=role), self.assertRaisesRegex( + C.INVENTORY.InventoryError, 'inventory-provider-bytes-mismatch'): + C.collect_final_inventory(wrong_bytes, raw, records, llvm, POLICY) + reader.debian_notices.reset_mock() + reader.common_licenses.reset_mock() + reader.build_tools.reset_mock() + with patch.object(C.INVENTORY, 'crosscheck_archives', side_effect=C.INVENTORY.InventoryError( + 'inventory-provider-bytes-mismatch')): + with self.assertRaisesRegex(C.INVENTORY.InventoryError, 'inventory-provider-bytes-mismatch'): + C.collect_final_inventory(image, raw, records, llvm, POLICY) + reader.debian_notices.assert_not_called() + reader.common_licenses.assert_not_called() + reader.build_tools.assert_not_called() + for name in ('contract.json', 'shader-generation.json'): + changed = copy.deepcopy(docs) + if name == 'contract.json': + changed[name]['consumer']['commit'] = '1' * 40 + else: + changed[name]['source_commit'] = '1' * 40 + bad = {**raw, I.ROOT + '/' + name: encoded(changed[name])} + with self.subTest(name=name), self.assertRaisesRegex(C.Stop, 'final-consumer-mismatch'): + C.collect_final_inventory(image, bad, records, llvm, POLICY) + changed = copy.deepcopy(actual['inventories']['producer']) + changed['sources/classic-shaders/client/shaders/source']['sha256'] = '0' * 64 + reader.producer.return_value = changed + with self.assertRaisesRegex(C.Stop, 'final-shader-input-bytes'): + C.collect_final_inventory(image, raw, records, llvm, POLICY) + + def test_duplicate_metadata_json_rejected(self): + self.fixture() + self.put(I.ROOT + '/runtime-abi.json', b'{"a":1,"a":2}') + with self.assertRaisesRegex(I.InventoryError, 'scanner-duplicate-json-key'): + S.scan(self.root_fd) + + def test_metadata_bind_rejects_missing_fields_and_different_bytes(self): + raw = {name: b'{}' for name in I.METADATA_FILES} + records = {name: {'sha256': hashlib.sha256(value).hexdigest(), 'size': len(value), 'executable': False} + for name, value in raw.items()} + I.bind_metadata(records, raw, records) + for edit in ('missing', 'size', 'bool-type'): + changed = copy.deepcopy(records) + if edit == 'missing': + del changed['contract.json'] + elif edit == 'size': + changed['contract.json']['size'] = 1 + else: + changed['contract.json']['executable'] = 0 + with self.subTest(edit=edit), self.assertRaises(I.InventoryError): + I.bind_metadata(records, raw, changed) + + +class QualificationTests(unittest.TestCase): + def setUp(self): + for operation in ( + patch.dict(C.TARGETS, {TARGET: ('atrinik/classic-portable-build', 'fixture-only', 'sha256:' + 'e' * 64)}), + patch.dict(C.FINAL_TARGETS, {TARGET: POLICY}), + patch.object(C.LocalConnection, 'connect', side_effect=AssertionError('Docker forbidden')), + patch.object(C.signal, 'alarm')): + operation.start() + self.addCleanup(operation.stop) + self.context = {'run_id': '1', 'run_attempt': '1'} + self.verified = {'config_digest': POLICY['config_digest'], 'manifest_digest': POLICY['manifest_digest'], + 'config_object': {'config': {}}} + + def scanner(self): + sources = C.scanner_sources() + docker = Mock() + scanner = C.ScannerImage(docker, TARGET, self.context, self.verified, sources) + scanner.identifier, scanner.attempted = 'f' * 64, True + value = {'Id': scanner.identifier, 'Name': '/' + scanner.name, + 'Config': {'Labels': {C.OWNER_LABEL: scanner.owner}, 'Image': scanner.reference, + 'Entrypoint': scanner.entrypoint, 'Cmd': [], 'User': '65534:65534', 'WorkingDir': '/'}, + 'HostConfig': {'ReadonlyRootfs': True, 'NetworkMode': 'none', 'Privileged': False, + 'CapDrop': ['ALL'], 'SecurityOpt': ['no-new-privileges'], 'RestartPolicy': {'Name': 'no'}}, + 'Mounts': [{'Source': row['Source'], 'Destination': row['Target'], 'Type': 'bind', 'RW': False} + for row in scanner.mounts], + 'State': {'Status': 'created', 'Running': False, 'Pid': 0}, 'Image': POLICY['config_digest']} + docker.get.return_value = value + return scanner, value + + def test_actual_final_target_is_separate_and_old_rows_unchanged(self): + originals = {'broad', 'classic', 'portable-preliminary'} + self.assertEqual(set(C.TARGETS) - {TARGET}, originals | {'portable-final'}) + self.assertEqual(set(C.FINAL_TARGETS), {TARGET, 'portable-final'}) + self.assertNotIn('portable-final', C.HISTORICAL_TARGETS) + for target in originals: + self.assertEqual(C.target_policy(target), {'producer_revision': C.REVISION}) + with self.assertRaisesRegex(C.Stop, 'final-target-required'): + C.ScannerImage(Mock(), target, self.context, self.verified, C.scanner_sources()) + self.assertEqual(C.TOOL_FILES, C.INVENTORY.LLVM_TOOL_FILES) + + def test_actual_release_scanner_binds_fixed_index_platform_and_config(self): + policy = C.target_policy('portable-final') + verified = {'manifest_digest': policy['manifest_digest'], + 'config_digest': policy['config_digest'], 'config_object': {'config': {}}} + docker = Mock() + scanner = C.ScannerImage(docker, 'portable-final', self.context, verified, C.scanner_sources()) + self.assertEqual(scanner.reference, + 'ghcr.io/atrinik/classic-portable-build@sha256:' + '74761e930f5d57face2d81efb0dc50dd4ce32ce1204810e56580bf68a5972ebc') + self.assertEqual(scanner.entrypoint, ['/usr/bin/python3', '-I', '-B', + '/scan/scan-released-sdk-filesystem.py']) + for field in ('manifest_digest', 'config_digest'): + changed = dict(verified, **{field: 'sha256:' + '0' * 64}) + with self.subTest(field=field), self.assertRaisesRegex( + C.Stop, 'final-image-coordinate-mismatch'): + C.ScannerImage(docker, 'portable-final', self.context, changed, C.scanner_sources()) + docker.request.assert_not_called() + docker.get.assert_not_called() + + def test_wrong_final_platform_config_and_policy_rejected_before_create(self): + for field in ('manifest_digest', 'config_digest'): + bad = copy.deepcopy(self.verified) + bad[field] = 'sha256:' + '1' * 64 + with self.subTest(field=field), self.assertRaisesRegex(C.Stop, 'final-image-coordinate-mismatch'): + C.ScannerImage(Mock(), TARGET, self.context, bad, C.scanner_sources()) + with patch.dict(C.FINAL_TARGETS, {TARGET: {}}), self.assertRaises(C.Stop): + C.target_policy(TARGET) + + def test_scanner_create_exact_readonly_command_mounts_and_no_credentials(self): + scanner, value = self.scanner() + scanner.identifier, scanner.attempted = None, False + value['Id'] = 'a' * 64 + scanner.docker.request.return_value = (Mock(), Response(encoded({'Id': value['Id']}))) + scanner.create() + call = scanner.docker.request.call_args + body = json.loads(call.args[3]) + self.assertEqual(body['Entrypoint'], ['/usr/bin/python3', '-I', '-B', '/scan/scan-released-sdk-filesystem.py']) + self.assertEqual(body['Env'], []) + self.assertEqual(body['User'], '65534:65534') + self.assertTrue(body['HostConfig']['ReadonlyRootfs']) + self.assertEqual({row['Target'] for row in body['HostConfig']['Mounts']}, + {'/scan/' + name for name in C.SCANNER_FILES}) + self.assertTrue(all(row['ReadOnly'] for row in body['HostConfig']['Mounts'])) + self.assertNotIn('X-Registry-Auth', call.args[2]) + + def test_fences_reject_wrong_owner_write_mount_network_and_image(self): + changes = [('Config', 'User', '0'), ('Config', 'Entrypoint', ['/bin/sh']), + ('HostConfig', 'ReadonlyRootfs', False), ('HostConfig', 'NetworkMode', 'bridge')] + for section, key, bad in changes: + scanner, value = self.scanner() + value[section][key] = bad + with self.subTest(key=key), self.assertRaises(C.Stop): + scanner.fence() + scanner, value = self.scanner() + value['Mounts'][0]['RW'] = True + with self.assertRaises(C.Stop): + scanner.fence() + value['Mounts'][0]['RW'] = False + value['Config']['Labels'][C.OWNER_LABEL] = 'foreign' + with self.assertRaises(C.Stop): + scanner.cleanup() + scanner.docker.request.assert_not_called() + + def test_logs_complete_framing_stderr_extra_and_limit(self): + raw = encoded({'schema_version': 1}) + frame = b'\x01\0\0\0' + struct.pack('>I', len(raw)) + raw + self.assertEqual(C.scanner_logs(Response(frame)), {'schema_version': 1}) + for bad in (frame[:-1], frame + b'x', b'\x02' + frame[1:], frame + frame): + with self.subTest(raw=bad), self.assertRaises(C.Stop): + C.scanner_logs(Response(bad)) + with patch.object(C, 'SCANNER_LIMIT', 1), self.assertRaises(C.Stop): + C.scanner_logs(Response(frame)) + + def test_execute_waits_for_success_and_parses_bounded_output(self): + scanner, value = self.scanner() + payload = encoded({'observed': True}) + frame = b'\x01\0\0\0' + struct.pack('>I', len(payload)) + payload + def request(method, path, **kwargs): + if path.endswith('/start'): + value['State'] = {'Status': 'running', 'Running': True, 'Pid': 123} + return Mock(), Response() + if '/wait?' in path: + value['State'] = {'Status': 'exited', 'Running': False, 'Pid': 0} + self.assertEqual(kwargs['timeout'], I.MAX_SECONDS + 30) + return Mock(), Response(encoded({'StatusCode': 0})) + if '/logs?' in path: + return Mock(), Response(frame) + self.fail('unexpected Docker method') + scanner.docker.request.side_effect = request + self.assertEqual(scanner.execute(), {'observed': True}) + self.assertEqual(scanner.docker.request.call_count, 3) + + def test_missing_policy_wrong_target_and_registry_coordinate_fail_before_fetch(self): + with patch.dict(C.FINAL_TARGETS, {}, clear=True), self.assertRaisesRegex(C.Stop, 'final-target-policy-missing'): + C.target_policy(TARGET) + with patch.object(C, 'registry_object') as fetch, self.assertRaisesRegex(C.Stop, 'fixed-target-mismatch'): + C.verified_registry('atrinik/classic-portable-build', 'sha256:' + '0' * 64, + 'fixture-bearer', target=TARGET) + fetch.assert_not_called() + + def test_qualification_identity_comparison_and_cleanup_failure(self): + expected = {'fixture': True} + for failure in ('none', 'compare', 'cleanup', 'source'): + scanner = Mock() + scanner.identifier = 'f' * 64 + scanner.execute.return_value = expected if failure != 'compare' else {'fixture': False} + if failure == 'cleanup': + scanner.cleanup.side_effect = RuntimeError('hostile secret') + sources = C.scanner_sources() + readings = [sources, {} if failure == 'source' else sources] + with self.subTest(failure=failure), patch.object(C, 'ScannerImage', return_value=scanner), \ + patch.object(C, 'scanner_sources', side_effect=readings): + result = C.qualify_filesystem(Mock(), TARGET, self.context, self.verified, expected) + self.assertEqual(result['status'], 'verified-filesystem-equality' if failure == 'none' else 'stopped') + self.assertTrue(result['image_code_executed']) + self.assertFalse(result['final_portable_cohort']) + self.assertEqual(result['index_digest'], C.TARGETS[TARGET][2]) + self.assertEqual(result['producer_revision'], POLICY['producer_revision']) + scanner.cleanup.assert_called_once() + self.assertNotIn('hostile secret', encoded(result).decode()) + + def test_final_collection_retains_separate_receipt_and_cleanup_failure_stops(self): + baseline = load('historical_collector_fixtures', 'tools/tests/test_collect_released_sdk_metadata.py') + data, binaries = baseline.OfflineTests().portable_fixture() + verified = dict(self.verified, index=b'{}', manifest=b'{}', config=b'{}') + context = dict(self.context, actor='fixture-actor') + owned = Mock() + owned.identifier = 'f' * 64 + owned.docker.negotiate.return_value = {} + owned.docker.inspect.return_value = {} + owned.docker.pull.return_value = 1 + def read(path, *, retain, executable=False): + raw = data[path] if retain else binaries[path] + return {'sha256': hashlib.sha256(raw).hexdigest(), 'size': len(raw), + 'mode': '0o755' if executable else '0o644'}, raw if retain else None + owned.read.side_effect = read + report = {'inventories': {'producer': {}, 'debian_notices': {}, 'common_licenses': {}}, + 'inventory_sha256': {'producer': 'fixture', 'debian_notices': 'fixture', 'common_licenses': 'fixture'}, + 'metadata': {name: {'sha256': hashlib.sha256(data[I.ROOT + '/' + name]).hexdigest()} + for name in I.METADATA_FILES}, 'build_tools': {}} + for failure in ('none', 'qualification', 'cleanup'): + qualification = {'status': 'stopped' if failure == 'qualification' else 'verified-filesystem-equality', + 'image_code_executed': True, 'final_portable_cohort': False} + owned.cleanup.side_effect = RuntimeError('private detail') if failure == 'cleanup' else None + with ( + tempfile.TemporaryDirectory() as temp, + self.subTest(failure=failure), + patch.object(C, 'runner_docker', return_value=owned.docker), + patch.object(C, 'scoped_bearer', return_value='fixture-bearer'), + patch.object(C, 'verified_registry', return_value=verified), + patch.object(C, 'StoppedImage', return_value=owned), + patch.object(C, 'collect_final_inventory', return_value=report), + patch.object(C, 'qualify_filesystem', return_value=qualification), + ): + receipt = C.collect(TARGET, context, 'fixture-original', Path(temp)) + self.assertEqual(receipt['status'], 'verified-installed-identity' if failure == 'none' else 'stopped') + self.assertFalse(receipt['image_code_executed']) + self.assertFalse(receipt['final_portable_cohort']) + retained = json.loads((Path(temp) / 'filesystem-qualification.json').read_bytes()) + self.assertTrue(retained['image_code_executed']) + self.assertEqual(set(receipt['metadata_sha256']), set(I.METADATA_FILES)) + self.assertEqual(len(receipt['tool_bytes']), 12) + self.assertEqual({p.name for p in (Path(temp) / 'inventories').iterdir()}, + {'producer.json', 'debian_notices.json', 'common_licenses.json'}) + self.assertNotIn('private detail', json.dumps(receipt)) + + def test_cleanup_stops_only_owned_running_scanner_then_deletes(self): + scanner, value = self.scanner() + value['State'] = {'Status': 'running', 'Running': True, 'Pid': 123} + def request(method, path, **kwargs): + if '/stop?' in path: + value['State'] = {'Status': 'exited', 'Running': False, 'Pid': 0} + return Mock(), Response() + scanner.docker.request.side_effect = request + scanner.cleanup() + self.assertEqual([call.args[1] for call in scanner.docker.request.call_args_list], + ['/containers/' + scanner.identifier + '/stop?t=10', '/containers/' + scanner.identifier]) + self.assertFalse(scanner.attempted) + + + def scanner_cleanup_http(self, status, drift=None): + scanner, value = self.scanner() + value['State'] = {'Status': 'running', 'Running': True, 'Pid': 123} + scanner.docker = C.Docker('/fixture-only', (1, 2, 3)) + calls = [] + connections = [] + def connection(*args, **kwargs): + conn = Mock() + connections.append(conn) + current = {} + def request(method, path, **kwargs): + current.update(method=method, path=path) + calls.append((method, path)) + def getresponse(): + response_status = 200 + raw = b'' + if current['method'] == 'GET': + raw = json.dumps(value).encode() + elif current['method'] == 'POST': + value['State'] = {'Status': 'exited', 'Running': False, 'Pid': 0} + if drift: + drift(value) + response_status = status + elif current['method'] == 'DELETE': + response_status = 204 + response = Response(raw) + response.status = response_status + return response + conn.request.side_effect = request + conn.getresponse.side_effect = getresponse + return conn + return scanner, calls, connections, connection + + def test_scanner_cleanup_actual_http_already_stopped_deletes_after_refence(self): + for status in (204, 304): + scanner, calls, connections, connection = self.scanner_cleanup_http(status) + with self.subTest(status=status), patch.object(C, 'LocalConnection', side_effect=connection): + scanner.cleanup() + path = '/containers/' + scanner.identifier + self.assertEqual(calls, [('GET', path + '/json'), ('POST', path + '/stop?t=10'), + ('GET', path + '/json'), ('DELETE', path)]) + self.assertFalse(scanner.attempted) + for conn in connections: + conn.close.assert_called_once() + + def test_scanner_cleanup_actual_http_rejects_other_statuses_without_delete(self): + for status in (200, 201, 202, 400, 404, 409, 500): + scanner, calls, connections, connection = self.scanner_cleanup_http(status) + with self.subTest(status=status), patch.object(C, 'LocalConnection', side_effect=connection), \ + self.assertRaisesRegex(C.Stop, 'docker-request-denied'): + scanner.cleanup() + self.assertTrue(scanner.attempted) + self.assertEqual([method for method, _ in calls], ['GET', 'POST']) + for conn in connections: + conn.close.assert_called_once() + + def test_scanner_cleanup_actual_http_already_stopped_rechecks_identity_before_delete(self): + edits = { + 'owner': lambda value: value['Config']['Labels'].update({C.OWNER_LABEL: 'foreign'}), + 'image': lambda value: value.update(Image='sha256:' + '0' * 64), + 'isolation': lambda value: value['HostConfig'].update(NetworkMode='host'), + 'state': lambda value: value['State'].update(Status='running', Running=True, Pid=123), + } + for name, edit in edits.items(): + scanner, calls, connections, connection = self.scanner_cleanup_http(304, edit) + with self.subTest(drift=name), patch.object(C, 'LocalConnection', side_effect=connection), \ + self.assertRaises(C.Stop): + scanner.cleanup() + self.assertTrue(scanner.attempted) + self.assertEqual([method for method, _ in calls], ['GET', 'POST', 'GET']) + for conn in connections: + conn.close.assert_called_once() + + +if __name__ == '__main__': + unittest.main()