diff --git a/.github/workflows/core-protocol.yml b/.github/workflows/core-protocol.yml index 0d69736a..ec0fa05e 100644 --- a/.github/workflows/core-protocol.yml +++ b/.github/workflows/core-protocol.yml @@ -50,7 +50,7 @@ jobs: test "$(docker exec "$POSTGRES_CONTAINER" psql -X -U postgres -d postgres -v ON_ERROR_STOP=1 \ -Atqc 'SELECT pg_reload_conf()')" = t # pg_reload_conf sends SIGHUP; read the actual setting until applied. - for attempt in {1..50}; do + for _ in {1..50}; do docker exec "$POSTGRES_CONTAINER" psql -X -U postgres -d postgres -v ON_ERROR_STOP=1 \ -Atqc "SELECT setting, pending_restart FROM pg_settings WHERE name = 'autovacuum'" \ > "$RUNNER_TEMP/core-postgres-offline-settings.txt" diff --git a/.github/workflows/linux-packages.yml b/.github/workflows/linux-packages.yml index 4c00cfbc..98aa117a 100644 --- a/.github/workflows/linux-packages.yml +++ b/.github/workflows/linux-packages.yml @@ -82,6 +82,7 @@ jobs: git rev-parse HEAD | tee out/evidence/revision.txt nfpm --version | tee out/evidence/nfpm-version.txt node --test scripts/workflow-topology.test.mjs + python3 -B packaging/linux/test_extore_receipts.py python3 -B packaging/linux/test_native.py -v 2>&1 | tee out/evidence/native-tests.log go run github.com/rhysd/actionlint/cmd/actionlint@v1.7.12 -color .github/workflows/linux-packages.yml - name: Build the three product binaries for this architecture @@ -99,6 +100,7 @@ jobs: docker buildx build --platform "linux/$ARCH" -f deployment/docker/extensions.Dockerfile \ --target binaries-musl --output type=local,dest=out/extensions-musl . - name: Build verified source archives and native packages + id: packages run: | revision=$(git rev-parse HEAD) SOURCE_DATE_EPOCH=$(git show -s --format=%ct HEAD) @@ -109,6 +111,7 @@ jobs: done python3 scripts/build-distribution.py build --component extensions \ --version 0.1.0-preview --revision "$revision" --platform "linux-musl-$ARCH" --input out/extensions-musl + package_paths=() for component in core extensions cli; do package="lmm-$component" if [[ "$component" == cli ]]; then package=lmm; fi @@ -118,12 +121,19 @@ jobs: archive="out/distributions/${package}-0.1.0-preview-${platform}.tar.gz" digest=$(sha256sum "$archive" | cut -d ' ' -f 1) for release in 1 2; do - python3 packaging/linux/build.py --archive "$archive" --sha256 "$digest" \ - --revision "$revision" --component "$component" --format "$format" --release "$release" + built=$(python3 packaging/linux/build.py --archive "$archive" --sha256 "$digest" \ + --revision "$revision" --component "$component" --format "$format" --release "$release") + package_paths+=("$built" "$built.sha256" "$built.buildinfo.json") done done done (cd out/linux-packages && sha256sum --check ./*.sha256) + for path in "${package_paths[@]}"; do test -s "$path"; done + { + echo 'paths<> "$GITHUB_OUTPUT" - name: Install, upgrade, run and remove in disposable distribution containers run: | set -o pipefail @@ -159,23 +169,39 @@ jobs: -f packages/extore/tests/native_plugin_alpine.Dockerfile --target fixture \ --build-arg "EXTORE_RUST_TARGET=$RUST_TARGET" -t lmm-extore-alpine-fixture packages/extore docker image inspect lmm-extore-alpine-fixture --format '{{.Id}}' > out/evidence/extore-fixture-image.txt + umask 077 + staging=$(mktemp -d "$RUNNER_TEMP/extore-receipts.XXXXXXXX") + report=extore-native-alpine.json + touch "$staging/$report" "$staging/$report.package.json" timeout 300 docker run --rm --network none --add-host localhost.localdomain:127.0.0.1 \ --cap-add NET_ADMIN --cpus 1 --memory 768m --pids-limit 256 \ --security-opt no-new-privileges:true -e LMM_PACKAGING_ISOLATED=1 \ -v "$PWD/out/linux-packages:/packages:ro" -v "$PWD/out/native-probe:/probe:ro" \ - -v "$PWD/out/evidence:/evidence" \ + -v "$staging/$report:/receipts/$report" \ + -v "$staging/$report.package.json:/receipts/$report.package.json" \ -v "$PWD/packaging/linux/extore-plugin-smoke.py:/package-probe.py:ro" \ --entrypoint python3 lmm-extore-alpine-fixture /package-probe.py \ --arch "$ARCH" --revision "$(git rev-parse HEAD)" \ --parent-netns "$(readlink /proc/self/ns/net)" \ - --report /evidence/extore-native-alpine.json 2>&1 | tee out/evidence/extore-native-alpine.log + --report "/receipts/$report" > "$staging/fixture.log" 2>&1 + for receipt in "$staging/$report" "$staging/$report.package.json"; do + test -s "$receipt" + test -O "$receipt" + done + mv -- "$staging/$report" "$staging/$report.package.json" out/evidence/ - name: Preserve preview packages and scoped evidence, never publish if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a with: name: native-linux-${{ matrix.arch }}-${{ github.sha }} path: | - out/linux-packages/ - out/evidence/ + ${{ steps.packages.outputs.paths }} + out/evidence/revision.txt + out/evidence/nfpm-version.txt + out/evidence/install-matrix.txt + out/evidence/image-digests.txt + out/evidence/extore-fixture-image.txt + out/evidence/extore-native-alpine.json + out/evidence/extore-native-alpine.json.package.json if-no-files-found: error retention-days: 7 diff --git a/packaging/linux/extore-plugin-smoke.py b/packaging/linux/extore-plugin-smoke.py index 35fc2a06..22d97a4d 100644 --- a/packaging/linux/extore-plugin-smoke.py +++ b/packaging/linux/extore-plugin-smoke.py @@ -14,6 +14,34 @@ import urllib.request +def probe_receipt(report, host_sha256, plugin_sha256): + """Export only reviewed acceptance facts from the private probe report.""" + def confirmed(actual, expected): + for key, value in expected.items(): + if type(actual.get(key)) is not type(value) or actual[key] != value: + raise ValueError(f"unconfirmed native acceptance field: {key}") + return {key: actual[key] for key in expected} + + receipt = confirmed(report, { + "status": "passed", "consumer_exit_code": 0, + "host_library_sha256": host_sha256, "plugin_library_sha256": plugin_sha256, + "actual_rust_health": True, "isolated_network_namespace": True, + "tls_certificate_verification": True, "authorized_catalog": True, + "cross_shop_http_detail_denied": True, "revoked_token_denied": True, + "catalog_calls_did_not_issue_cards": True, "actual_sqlite_grant_usage_recorded": True, + }) + receipt["consumer"] = confirmed(report["consumer"], { + "status": "passed", "host_abi": 1, "interface": "extore.commerce", + "real_extore_https_sqlite": True, "native_digest_path_binding": True, + "native_repeated_open_call_release_close": True, "store_sync_persisted": True, + "http_catalog_fallback_calls": 0, "secret_values_logged": False, + "production_accounts_created": False, + "store_repository": "transactional memory test repository", + "core_authority": "existing Store test double; independent CoreSession acceptance not run", + }) + return receipt + + def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--arch", choices=("amd64", "arm64"), required=True) @@ -104,7 +132,12 @@ def main(): time.sleep(0.1) if not ready: raise RuntimeError("installed extensions did not start without native configuration") - if os.readlink(f"/proc/{process.pid}/exe") != "/usr/bin/lmm-extensions": + # Procfs checks credentials; container root cannot inspect another UID. + identity = subprocess.run( + ["readlink", "-n", f"/proc/{process.pid}/exe"], + check=True, stdout=subprocess.PIPE, timeout=5, user=account.pw_uid, + group=account.pw_gid, extra_groups=[]) + if identity.stdout != b"/usr/bin/lmm-extensions": raise RuntimeError("default process is not the installed program") finally: process.terminate() @@ -127,15 +160,19 @@ def main(): "service_uid": account.pw_uid, "service_gid": account.pw_gid, "default_process_stopped": True, } + with tempfile.TemporaryDirectory(prefix="extore-private-probe-") as directory: + private_report = Path(directory) / "probe.json" + command = [sys.executable, "/work/tests/native_plugin_probe.py", + "--inside-netns", "--parent-netns", args.parent_netns, + "--binary", "/opt/extore/extore-server", + "--host-library", host, "--plugin-library", plugin, + "--report", str(private_report), "--consumer", str(args.consumer), + "-test.run", "^TestNativePluginCatalogStore$", "-test.v"] + subprocess.run(command, check=True) + accepted = probe_receipt(json.loads(private_report.read_text()), + info["files"][host]["sha256"], info["files"][plugin]["sha256"]) + args.report.write_text(json.dumps(accepted, indent=2) + "\n") Path(str(args.report) + ".package.json").write_text(json.dumps(receipt, indent=2) + "\n") - command = [sys.executable, "/work/tests/native_plugin_probe.py", - "--inside-netns", "--parent-netns", args.parent_netns, - "--binary", "/opt/extore/extore-server", - "--host-library", "/usr/lib/lmm/extensions/extore/libextore_plugin_host.so", - "--plugin-library", "/usr/lib/lmm/extensions/extore/libextore_commerce_plugin.so", - "--report", str(args.report), "--consumer", str(args.consumer), - "-test.run", "^TestNativePluginCatalogStore$", "-test.v"] - subprocess.run(command, check=True) if __name__ == "__main__": diff --git a/packaging/linux/test_extore_receipts.py b/packaging/linux/test_extore_receipts.py new file mode 100644 index 00000000..ebb8f6c8 --- /dev/null +++ b/packaging/linux/test_extore_receipts.py @@ -0,0 +1,47 @@ +"""Check the upload boundary with synthetic private probe output.""" +import runpy +import unittest +from pathlib import Path + + +probe_receipt = runpy.run_path(str(Path(__file__).with_name("extore-plugin-smoke.py")))["probe_receipt"] + + +class ProbeReceiptTests(unittest.TestCase): + def setUp(self): + self.host, self.plugin = "a" * 64, "b" * 64 + self.consumer = { + "status": "passed", "host_abi": 1, "interface": "extore.commerce", + "real_extore_https_sqlite": True, "native_digest_path_binding": True, + "native_repeated_open_call_release_close": True, "store_sync_persisted": True, + "http_catalog_fallback_calls": 0, "secret_values_logged": False, + "production_accounts_created": False, + "store_repository": "transactional memory test repository", + "core_authority": "existing Store test double; independent CoreSession acceptance not run", + } + self.report = { + "status": "passed", "consumer_exit_code": 0, "consumer": self.consumer, + "host_library_sha256": self.host, "plugin_library_sha256": self.plugin, + "actual_rust_health": True, "isolated_network_namespace": True, + "tls_certificate_verification": True, "authorized_catalog": True, + "cross_shop_http_detail_denied": True, "revoked_token_denied": True, + "catalog_calls_did_not_issue_cards": True, "actual_sqlite_grant_usage_recorded": True, + } + + def test_only_reviewed_fields_cross_the_boundary(self): + private = dict(self.report, stdout="private fixture output", + consumer=dict(self.consumer, credentials={"test": "private"})) + self.assertEqual(probe_receipt(private, self.host, self.plugin), self.report) + + def test_incomplete_failed_or_mistyped_facts_cannot_be_exported(self): + missing = self.report.copy() + del missing["consumer_exit_code"] + for raw in [missing, dict(self.report, status="failed"), dict(self.report, consumer_exit_code=False), + dict(self.report, host_library_sha256=self.plugin), + dict(self.report, consumer=dict(self.consumer, core_authority="unreviewed text"))]: + with self.subTest(raw=raw), self.assertRaises(ValueError): + probe_receipt(raw, self.host, self.plugin) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/workflow-topology.test.mjs b/scripts/workflow-topology.test.mjs index 8ae95281..6332a6d7 100644 --- a/scripts/workflow-topology.test.mjs +++ b/scripts/workflow-topology.test.mjs @@ -83,7 +83,7 @@ test('workflows initialize only their required pinned submodules', () => { lmm: ['apps/cli'], 'microservice-distribution': ['apps/cli', 'apps/cli', 'packages/extore'], 'linux-packages': ['apps/cli', 'packages/extore'], - 'core-protocol': ['apps/cli'], + 'core-protocol': ['apps/cli', 'packages/extore'], deploy: ['packages/extore'], }; for (const [name, paths] of Object.entries(dependencies)) {