diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7883b184..fbd4aa77 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -51,7 +51,8 @@ permissions: env: RELEASE_PLATFORM_PROFILE: ${{ inputs.platform_profile || 'full' }} RELEASE_CHANNEL: ${{ inputs.channel || (startsWith(github.ref, 'refs/tags/rust-v') && 'beta') || 'candidate' }} - EMULEBB_BUILD_REF: a508474e81f641e2a1ba1a6f7c8535e8ab358ddf + EMULEBB_BUILD_REF: 7664330e9381e6ac4f10a72d39f293a4b886b5b9 + EMULEBB_LINUX_PREPARED_IMAGE: ghcr.io/emulebb/emulebb-build-ci/ubuntu-rust@sha256:e006429d3a240a6f4e43b1d8f05581bcd963ca681059e2661465ebb359f9118e EMULEBB_BUILD_TESTS_REF: 86e32b1d547ca68c294350ad45d70d79b987d2bb EMULEBB_TOOLING_REF: d406cfd8e6f3e3b6f04e377ac469c413293f81d9 MINIUPNP_REF: 4e7a10948004ca2085ac8103e14fc77edb898b4c @@ -202,6 +203,7 @@ jobs: ref: ${{ env.EMULEBB_TOOLING_REF }} path: .ci/emulebb-tooling - name: Install pinned Rust + if: matrix.os != 'linux' uses: dtolnay/rust-toolchain@ce678459e9fc7500d337468f904b95f1b5c10b5e # 1.98.1 - name: Set up Python uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 @@ -210,44 +212,15 @@ jobs: - name: Install pinned workspace CLI dependencies run: python -m pip install -r tools/requirements-ci.txt click==8.4.1 pydantic==2.14.3 - name: Set up Node.js + if: matrix.os != 'linux' uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: node-version: "26.13.0" package-manager-cache: false - name: Install pinned npm + if: matrix.os != 'linux' run: npm install --global npm@11.20.0 - - name: Install Linux build dependencies - if: matrix.os == 'linux' - timeout-minutes: 6 - working-directory: .ci/emulebb-build - run: >- - python -B -m emule_workspace provision-ubuntu-dependencies-ci - --profile rust-native-release --deadline-seconds 300 - --network-timeout-seconds 20 --attempts 3 - --report "$RUNNER_TEMP/emulebb-rust-out/reports/native-linux-provisioning.json" - - name: Install verified AppImage builder - if: matrix.os == 'linux' - shell: bash - run: | - if [[ '${{ matrix.arch }}' == 'ARM64' ]]; then - tool_arch=aarch64 - expected=f0837e7448a0c1e4e650a93bb3e85802546e60654ef287576f46c71c126a9158 - else - tool_arch=x86_64 - expected=ed4ce84f0d9caff66f50bcca6ff6f35aae54ce8135408b3fa33abfc3cb384eb0 - fi - tool="$RUNNER_TEMP/appimagetool-${tool_arch}.AppImage" - curl --fail --location --silent --show-error \ - "https://github.com/AppImage/appimagetool/releases/download/1.9.1/appimagetool-${tool_arch}.AppImage" \ - --output "$tool" - echo "$expected $tool" | sha256sum --check - chmod 755 "$tool" - { - echo "APPIMAGETOOL=$tool" - echo "APPIMAGE_EXTRACT_AND_RUN=1" - echo "ARCH=$tool_arch" - } >> "$GITHUB_ENV" - name: Assert release build excludes test-only features run: python tools/check_rust_client_policy.py - name: Prepare inherited build output directory @@ -255,6 +228,7 @@ jobs: CARGO_TARGET_DIR: ${{ runner.temp }}/emulebb-rust-out/builds/rust/target run: python -c "from pathlib import Path; import os; Path(os.environ['CARGO_TARGET_DIR']).mkdir(parents=True, exist_ok=True)" - name: Build and package native release asset + if: matrix.os != 'linux' working-directory: .ci/emulebb-build env: EMULEBB_WORKSPACE_OUTPUT_ROOT: ${{ runner.temp }}/emulebb-rust-out @@ -264,6 +238,28 @@ jobs: EMULEBB_SOURCE_REVISION: ${{ needs.prepare.outputs.source_sha }} RUSTUP_TOOLCHAIN: "1.98.1" run: python -m emule_workspace package-emulebb-rust-ci --release-version ${{ env.RELEASE_VERSION }} --target-os ${{ matrix.os }} --platform ${{ matrix.arch }} + - name: Set up pinned Docker for prepared Linux packaging + if: matrix.os == 'linux' + uses: docker/setup-docker-action@7ec28a010822b4626af4874f9448a4b05ac6f3e5 # v5.6.0 + with: + version: v29.6.1 + daemon-config: '{"features":{"containerd-snapshotter":true}}' + - name: Build and package Linux release asset with verified prepared builder + if: matrix.os == 'linux' + timeout-minutes: 40 + working-directory: .ci/emulebb-build + env: + EMULEBB_WORKSPACE_OUTPUT_ROOT: ${{ runner.temp }}/emulebb-rust-out + CARGO_TARGET_DIR: ${{ runner.temp }}/emulebb-rust-out/builds/rust/target + EMULEBB_RELEASE_VERSION: ${{ env.RELEASE_VERSION }} + EMULEBB_RELEASE_CHANNEL: ${{ env.RELEASE_CHANNEL }} + EMULEBB_SOURCE_REVISION: ${{ needs.prepare.outputs.source_sha }} + RUSTUP_TOOLCHAIN: "1.98.1" + PREPARED_IMAGE: ${{ env.EMULEBB_LINUX_PREPARED_IMAGE }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: >- + python -B -m emule_workspace package-emulebb-rust-prepared-ci + --image "$PREPARED_IMAGE" --release-version "$RELEASE_VERSION" --platform ${{ matrix.arch }} - name: Retain build diagnostics if: always() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 @@ -271,7 +267,8 @@ jobs: name: emulebb-rust-build-logs-${{ matrix.os }}-${{ matrix.arch }} path: | ${{ runner.temp }}/emulebb-rust-out/logs/builds/**/*.log - ${{ runner.temp }}/emulebb-rust-out/reports/native-linux-provisioning.json + ${{ runner.temp }}/emulebb-rust-out/reports/prepared-native/**/execution.json + ${{ runner.temp }}/emulebb-rust-out/reports/prepared-native/**/package.log if-no-files-found: warn retention-days: 30 - name: Retain package and provenance @@ -490,6 +487,7 @@ jobs: CARGO_TARGET_DIR: ${{ runner.temp }}/emulebb-rust-out/builds/rust/target run: >- python -B -m emule_workspace package-emulebb-rust-linux-ci + --use-ci-image-mirrors --libc musl --architecture ${{ matrix.architecture }} --release-version "$RELEASE_VERSION" --test-scope full --smoke --smoke-report-path "${{ runner.temp }}/emulebb-rust-out/evidence/emulebb-rust-v${{ env.RELEASE_VERSION }}-package-smoke-${{ matrix.slug }}.json" @@ -710,6 +708,7 @@ jobs: EMULEBB_WORKSPACE_OUTPUT_ROOT: ${{ runner.temp }}/emulebb-rust-out run: >- python -B -m emule_workspace package-emulebb-rust-image-ci + --use-ci-image-mirrors --variant ${{ matrix.variant }} --release-version "$RELEASE_VERSION" --assets-dir "${{ runner.temp }}/emulebb-rust-out/image-inputs" --control-rust-repo "${{ github.workspace }}/.ci/control" diff --git a/packaging/docker/Dockerfile.alpine b/packaging/docker/Dockerfile.alpine index da3dbee5..ca2f6ed5 100644 --- a/packaging/docker/Dockerfile.alpine +++ b/packaging/docker/Dockerfile.alpine @@ -1,6 +1,7 @@ # syntax=docker/dockerfile:1@sha256:4edf897a3ffa55b89f906fc8cc78afdb3f1834cc9c7083565e611a8a7d5fe99e ARG VERSION=0.1.0-beta.2 -FROM docker.io/library/alpine:3.24@sha256:3f4d6b32e26703c6082839234f43769d5046b3ac7c910033911d1e71d8cd630a AS unpack +ARG ALPINE_UNPACK_IMAGE=docker.io/library/alpine:3.24@sha256:3f4d6b32e26703c6082839234f43769d5046b3ac7c910033911d1e71d8cd630a +FROM ${ALPINE_UNPACK_IMAGE} AS unpack ARG VERSION ARG TARGETARCH SHELL ["/bin/ash", "-eo", "pipefail", "-c"] diff --git a/tools/tests/test_release_linux_provisioning.py b/tools/tests/test_release_linux_provisioning.py index a4df6e50..3e3f92f3 100644 --- a/tools/tests/test_release_linux_provisioning.py +++ b/tools/tests/test_release_linux_provisioning.py @@ -1,6 +1,7 @@ -"""Native release setup must use bounded, pinned Ubuntu provisioning.""" +"""Linux release packaging must use the verified prepared Builder pipeline.""" from pathlib import Path +import importlib.util import shlex import unittest @@ -8,6 +9,10 @@ ROOT = Path(__file__).resolve().parents[2] +SPEC = importlib.util.spec_from_file_location("release_publish_linux_provisioning", ROOT / "tools/release_publish.py") +assert SPEC is not None and SPEC.loader is not None +PUBLISH = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(PUBLISH) class ReleaseLinuxProvisioningTests(unittest.TestCase): @@ -15,50 +20,136 @@ def setUp(self): self.workflow = yaml.safe_load((ROOT / ".github/workflows/release.yml").read_text(encoding="utf-8")) self.job = self.workflow["jobs"]["native-package"] self.steps = self.job["steps"] - self.provision = next(step for step in self.steps - if "provision-ubuntu-dependencies-ci" in step.get("run", "")) - - def test_linux_setup_uses_the_shared_bounded_provisioner(self): - self.assertEqual(self.provision["if"], "matrix.os == 'linux'") - self.assertEqual(shlex.split(self.provision["run"]), [ - "python", "-B", "-m", "emule_workspace", "provision-ubuntu-dependencies-ci", - "--profile", "rust-native-release", "--deadline-seconds", "300", - "--network-timeout-seconds", "20", "--attempts", "3", - "--report", "$RUNNER_TEMP/emulebb-rust-out/reports/native-linux-provisioning.json", + self.prepared = next(step for step in self.steps + if "package-emulebb-rust-prepared-ci" in step.get("run", "")) + self.host = next(step for step in self.steps + if "package-emulebb-rust-ci --release-version" in step.get("run", "")) + + def test_linux_uses_prepared_pipeline_without_host_provisioning_or_fallback(self): + self.assertEqual(self.prepared["if"], "matrix.os == 'linux'") + self.assertEqual(self.host["if"], "matrix.os != 'linux'") + self.assertEqual(shlex.split(self.prepared["run"]), [ + "python", "-B", "-m", "emule_workspace", "package-emulebb-rust-prepared-ci", + "--image", "$PREPARED_IMAGE", "--release-version", "$RELEASE_VERSION", + "--platform", "${{", "matrix.arch", "}}", ]) - self.assertFalse(any("apt-get" in step.get("run", "") for step in self.steps), - "APT source selection and retries belong to the pinned Builder") + commands = "\n".join(step.get("run", "") for step in self.steps) + for forbidden in ("apt-get", "provision-ubuntu-dependencies-ci", "appimagetool-", "APPIMAGE_EXTRACT_AND_RUN"): + self.assertNotIn(forbidden, commands) + self.assertNotIn("||", self.prepared["run"], "An unverified image must never select a host fallback") - def test_setup_has_a_short_outer_deadline_and_failure_remains_blocking(self): - self.assertEqual(self.provision["timeout-minutes"], 6) + def test_prepared_source_identity_output_and_host_verification_are_explicit(self): + self.assertEqual(self.prepared["env"], { + **self.host["env"], + "PREPARED_IMAGE": "${{ env.EMULEBB_LINUX_PREPARED_IMAGE }}", + "GH_TOKEN": "${{ secrets.GITHUB_TOKEN }}", + }) + self.assertEqual(self.prepared["env"]["EMULEBB_SOURCE_REVISION"], "${{ needs.prepare.outputs.source_sha }}") + self.assertEqual(self.prepared["timeout-minutes"], 40) self.assertEqual(self.job["timeout-minutes"], 45) - self.assertNotIn("continue-on-error", self.provision) + self.assertNotIn("continue-on-error", self.prepared) self.assertNotIn("continue-on-error", self.job) - self.assertNotIn("env", self.provision, - "Dependency setup must not repair inherited workspace inputs") - def test_setup_uses_the_checked_out_packaging_owner_before_packaging(self): + def test_prepared_execution_identity_and_failure_log_are_retained(self): + diagnostics = next(step for step in self.steps + if step.get("name") == "Retain build diagnostics") + self.assertEqual(diagnostics["if"], "always()") + self.assertEqual(set(diagnostics["with"]["path"].splitlines()), { + "${{ runner.temp }}/emulebb-rust-out/logs/builds/**/*.log", + "${{ runner.temp }}/emulebb-rust-out/reports/prepared-native/**/execution.json", + "${{ runner.temp }}/emulebb-rust-out/reports/prepared-native/**/package.log", + }) + + def test_prepared_packaging_uses_pinned_owner_and_native_docker_before_upload(self): checkout = next(step for step in self.steps if step.get("with", {}).get("repository") == "emulebb/emulebb-build") self.assertEqual(checkout["with"]["ref"], "${{ env.EMULEBB_BUILD_REF }}") self.assertRegex(self.workflow["env"]["EMULEBB_BUILD_REF"], r"^[0-9a-f]{40}$") - self.assertEqual(self.provision["working-directory"], checkout["with"]["path"]) - provision_index = self.steps.index(self.provision) - self.assertLess(self.steps.index(checkout), provision_index) - self.assertTrue(any("pip install" in step.get("run", "") - and "click==8.4.1" in step["run"] for step in self.steps[:provision_index])) - appimage = next(step for step in self.steps if step.get("name") == "Install verified AppImage builder") - package = next(step for step in self.steps if "package-emulebb-rust-ci" in step.get("run", "")) - self.assertLess(provision_index, self.steps.index(appimage)) - self.assertLess(provision_index, self.steps.index(package)) - self.assertIs(self.workflow["jobs"]["optional-macos-package"]["steps"], self.steps) - - def test_provisioning_evidence_is_retained_even_if_packaging_fails(self): - diagnostics = next(step for step in self.steps if step.get("name") == "Retain build diagnostics") - self.assertEqual(diagnostics["if"], "always()") - self.assertIn("${{ runner.temp }}/emulebb-rust-out/reports/native-linux-provisioning.json", - diagnostics["with"]["path"]) - self.assertIn("${{ runner.temp }}/emulebb-rust-out/logs/builds/**/*.log", diagnostics["with"]["path"]) + self.assertRegex( + self.workflow["env"]["EMULEBB_LINUX_PREPARED_IMAGE"], + r"^ghcr\.io/emulebb/emulebb-build-ci/ubuntu-rust@sha256:[0-9a-f]{64}$", + ) + self.assertEqual(self.prepared["working-directory"], checkout["with"]["path"]) + docker = next(step for step in self.steps if step.get("uses", "").startswith("docker/setup-docker-action@")) + self.assertEqual(docker["if"], "matrix.os == 'linux'") + self.assertEqual(docker["with"]["version"], "v29.6.1") + self.assertRegex(docker["uses"], r"@[0-9a-f]{40}$") + index = self.steps.index(self.prepared) + self.assertLess(self.steps.index(checkout), index) + self.assertLess(self.steps.index(docker), index) + upload = next(step for step in self.steps if step.get("with", {}).get("name") == + "${{ env.NATIVE_PAYLOAD_PREFIX }}-${{ matrix.slug }}") + self.assertLess(index, self.steps.index(upload)) + self.assertEqual(upload["with"]["if-no-files-found"], "error") + + def test_profiles_keep_native_smoke_and_optional_artifact_graph(self): + matrix = {"include": "${{ fromJSON(needs.prepare.outputs.native_matrix) }}"} + smoke = self.workflow["jobs"]["native-package-smoke"] + self.assertEqual(self.job["strategy"]["matrix"], matrix) + self.assertEqual(smoke["strategy"]["matrix"], matrix) + for profile, expected in ( + ("full", {"windows-x64", "windows-arm64", "linux-x64", "linux-arm64", "macos-x64", "macos-arm64"}), + ("windows-linux", {"windows-x64", "windows-arm64", "linux-x64", "linux-arm64"}), + ): + with self.subTest(profile=profile): + self.assertEqual({row["slug"] for row in PUBLISH.native_matrix(profile)}, expected) + self.assertEqual(self.job["env"]["NATIVE_PAYLOAD_PREFIX"], "emulebb-rust-native-payload") + self.assertEqual(smoke["env"]["NATIVE_PAYLOAD_PREFIX"], "emulebb-rust-native-payload") + optional = self.workflow["jobs"]["optional-macos-package"] + optional_smoke = self.workflow["jobs"]["optional-macos-smoke"] + self.assertEqual(optional["if"], "${{ needs.prepare.outputs.platform_profile == 'windows-linux' }}") + self.assertEqual(optional_smoke["needs"], ["prepare", "optional-macos-package"]) + self.assertEqual(optional["env"]["NATIVE_PAYLOAD_PREFIX"], "emulebb-rust-optional-native-payload") + self.assertEqual(optional_smoke["env"]["NATIVE_PAYLOAD_PREFIX"], "emulebb-rust-optional-native-payload") + self.assertEqual(optional_smoke["steps"], smoke["steps"]) + self.assertEqual(self.workflow["jobs"]["native-package-smoke"]["needs"], ["prepare", "native-package"]) + self.assertEqual(self.host["run"], + "python -m emule_workspace package-emulebb-rust-ci --release-version ${{ env.RELEASE_VERSION }} " + "--target-os ${{ matrix.os }} --platform ${{ matrix.arch }}") + + def test_linux_image_owns_toolchains_while_host_keeps_python_control(self): + setup = [step for step in self.steps + if step.get("uses", "").startswith(("dtolnay/rust-toolchain@", "actions/setup-node@")) + or step.get("run") == "npm install --global npm@11.20.0"] + self.assertEqual(len(setup), 3) + for step in setup: + self.assertEqual(step["if"], "matrix.os != 'linux'") + python = next(step for step in self.steps if step.get("uses", "").startswith("actions/setup-python@")) + dependencies = next(step for step in self.steps + if "pip install -r tools/requirements-ci.txt" in step.get("run", "")) + policy = next(step for step in self.steps if step.get("run") == "python tools/check_rust_client_policy.py") + for step in (python, dependencies, policy): + self.assertNotIn("if", step) + self.assertLess(self.steps.index(step), self.steps.index(self.prepared)) + + def test_native_smoke_proves_identity_and_owns_profile_contention_scratch(self): + for name in ("native-package-smoke", "optional-macos-smoke"): + job = self.workflow["jobs"][name] + smoke = next(step for step in job["steps"] + if "smoke-rust-release-package.py" in step.get("run", "")) + self.assertEqual(smoke["env"]["EMULEBB_WORKSPACE_OUTPUT_ROOT"], + "${{ runner.temp }}/emulebb-rust-out") + self.assertIn("--expected-source-sha", smoke["run"]) + self.assertIn('--expected-channel "${{ env.RELEASE_CHANNEL }}"', smoke["run"]) + self.assertIn("${{ matrix.os != 'macos' && '--verify-profile-ownership' || '' }}", smoke["run"]) + self.assertIn("${{ matrix.os == 'windows' && matrix.arch == 'x64' && '--render-webui' || '' }}", smoke["run"]) + self.assertNotIn("continue-on-error", smoke) + + def test_musl_and_image_consumers_select_mirrors_without_changing_runtime_base(self): + for job, command in (("musl-package", "package-emulebb-rust-linux-ci"), + ("image-candidate", "package-emulebb-rust-image-ci")): + package = next(step for step in self.workflow["jobs"][job]["steps"] + if command in step.get("run", "")) + self.assertIn("--use-ci-image-mirrors", shlex.split(package["run"])) + self.assertNotIn("continue-on-error", package) + recipe = (ROOT / "packaging/docker/Dockerfile.alpine").read_text(encoding="utf-8").splitlines() + self.assertIn("ARG ALPINE_UNPACK_IMAGE=docker.io/library/alpine:3.24@sha256:" + "3f4d6b32e26703c6082839234f43769d5046b3ac7c910033911d1e71d8cd630a", recipe) + self.assertEqual([line for line in recipe if line.startswith("FROM ")], [ + "FROM ${ALPINE_UNPACK_IMAGE} AS unpack", + "FROM ghcr.io/linuxserver/baseimage-alpine:3.24@sha256:" + "7315b19a32460a84b4474993ef33065483ffec792fbc70dbe12eab918aa2bbf4", + ]) def test_musl_host_cache_preflight_inherits_an_existing_owned_target(self): steps = self.workflow["jobs"]["musl-package"]["steps"]