diff --git a/.github/scripts/ci_changes.py b/.github/scripts/ci_changes.py index 683bf0062..6f39db9e4 100644 --- a/.github/scripts/ci_changes.py +++ b/.github/scripts/ci_changes.py @@ -87,6 +87,7 @@ RELEASE_SECURITY_WORKFLOWS = frozenset( { + ".github/workflows/evidence-dev.yml", ".github/workflows/release.yml", ".github/workflows/release-candidate.yml", ".github/workflows/release-repeatability.yml", diff --git a/.github/workflows/evidence-dev.yml b/.github/workflows/evidence-dev.yml new file mode 100644 index 000000000..c9c0c18ba --- /dev/null +++ b/.github/workflows/evidence-dev.yml @@ -0,0 +1,377 @@ +name: Registry Evidence Development Build + +on: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: registry-evidence-development-build + cancel-in-progress: false + +env: + CARGO_TERM_COLOR: always + CARGO_INCREMENTAL: "0" + +jobs: + validate: + name: Validate protected-main development source + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + actions: read + contents: read + outputs: + source_sha: ${{ steps.identity.outputs.source_sha }} + version: ${{ steps.identity.outputs.version }} + tag: ${{ steps.identity.outputs.tag }} + steps: + - name: Checkout exact workflow source + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v4.2.2 + with: + ref: ${{ github.sha }} + fetch-depth: 0 + persist-credentials: false + submodules: false + + - name: Validate manual source and successful CI + id: identity + env: + GH_TOKEN: ${{ github.token }} + shell: bash + run: | + set -euo pipefail + if [[ "${GITHUB_EVENT_NAME}" != workflow_dispatch || + "${GITHUB_REF}" != refs/heads/main ]]; then + echo "Evidence development builds must be dispatched from main" >&2 + exit 1 + fi + if [[ ! "${GITHUB_SHA}" =~ ^[0-9a-f]{40}$ ]]; then + echo "GitHub did not supply an exact source commit" >&2 + exit 1 + fi + git fetch --force origin refs/heads/main:refs/remotes/origin/main + if [[ "$(git rev-parse refs/remotes/origin/main)" != "${GITHUB_SHA}" ]]; then + echo "Development source must be the current protected-main commit" >&2 + exit 1 + fi + ci_run="$( + gh api "repos/${GITHUB_REPOSITORY}/actions/workflows/ci.yml/runs?head_sha=${GITHUB_SHA}&status=success&per_page=100" \ + | jq -c --arg sha "${GITHUB_SHA}" \ + '[.workflow_runs[] | select( + .head_sha == $sha and + .conclusion == "success" and + .event == "push" + )] | sort_by(.updated_at) | last' + )" + if [[ "${ci_run}" == null ]]; then + echo "Protected-main CI has no successful push run for this source" >&2 + exit 1 + fi + version="$( + cargo metadata --locked --no-deps --format-version 1 \ + | jq -er '.packages[] + | select(.name == "registry-evidence") + | .version' + )" + if [[ ! "${version}" =~ ^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$ ]]; then + echo "Evidence workspace version is not canonical semantic version text" >&2 + exit 1 + fi + tag="v${version}-dev.${GITHUB_RUN_ID}.${GITHUB_RUN_ATTEMPT}" + if git ls-remote --exit-code --tags origin "refs/tags/${tag}" \ + >/dev/null 2>&1; then + echo "Development tag ${tag} already exists" >&2 + exit 1 + else + tag_lookup_status=$? + if [[ "${tag_lookup_status}" -ne 2 ]]; then + echo "Cannot prove development tag ${tag} is absent" >&2 + exit 1 + fi + fi + release_response="${RUNNER_TEMP}/evidence-dev-release-response" + if gh api --include --silent \ + "repos/${GITHUB_REPOSITORY}/releases/tags/${tag}" \ + >"${release_response}" 2>&1; then + release_status=200 + else + release_status="$( + python3 release/scripts/release_workflow_guard.py http-status \ + --response "${release_response}" + )" + fi + if [[ "${release_status}" != 404 ]]; then + echo "Development release destination is not absent" >&2 + exit 1 + fi + { + echo "source_sha=${GITHUB_SHA}" + echo "version=${version}" + echo "tag=${tag}" + } >> "${GITHUB_OUTPUT}" + + build: + name: Build Evidence dev toolset for ${{ matrix.asset }} + needs: validate + runs-on: ${{ matrix.runner }} + timeout-minutes: 40 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + include: + - runner: ubuntu-24.04 + target: x86_64-unknown-linux-gnu + asset: linux-amd64 + - runner: ubuntu-24.04-arm + target: aarch64-unknown-linux-gnu + asset: linux-arm64 + - runner: macos-14 + target: aarch64-apple-darwin + asset: macos-arm64 + steps: + - name: Checkout exact development source + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v4.2.2 + with: + ref: ${{ needs.validate.outputs.source_sha }} + fetch-depth: 1 + persist-credentials: false + submodules: false + + - name: Restore Evidence development Cargo cache + uses: actions/cache@2c8a9bd7457de244a408f35966fab2fb45fda9c8 # v4.2.3 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: registry-evidence-dev-${{ matrix.asset }}-${{ hashFiles('rust-toolchain.toml', 'Cargo.lock') }} + restore-keys: | + registry-evidence-dev-${{ matrix.asset }}- + + - name: Build native Evidence development binaries + shell: bash + run: | + set -euo pipefail + rustup toolchain install 1.95.0 \ + --profile minimal --target "${{ matrix.target }}" + cargo build --release --locked \ + -p registry-evidence \ + -p registry-evidencectl \ + -p registry-mint \ + --target "${{ matrix.target }}" + mkdir -p development-platform + for binary in evidence evidencectl mint; do + asset="${binary}-${{ needs.validate.outputs.tag }}-${{ matrix.asset }}" + cp "target/${{ matrix.target }}/release/${binary}" \ + "development-platform/${asset}" + chmod 0755 "development-platform/${asset}" + observed="$("development-platform/${asset}" --version)" + test "${observed}" = "${binary} ${{ needs.validate.outputs.version }}" + done + + - name: Upload native Evidence development binaries + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: evidence-dev-${{ matrix.asset }}-${{ github.run_id }}-${{ github.run_attempt }} + path: development-platform + if-no-files-found: error + retention-days: 2 + + assemble: + name: Assemble and smoke the curl-installable toolset + needs: + - validate + - build + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + actions: read + contents: read + steps: + - name: Checkout exact development source + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v4.2.2 + with: + ref: ${{ needs.validate.outputs.source_sha }} + fetch-depth: 1 + persist-credentials: false + submodules: false + + - name: Download exact native binaries + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + pattern: evidence-dev-*-${{ github.run_id }}-${{ github.run_attempt }} + path: development-inputs + merge-multiple: true + + - name: Assemble development assets and checksums + shell: bash + run: | + set -euo pipefail + tag="${{ needs.validate.outputs.tag }}" + mkdir -p development-assets + cp development-inputs/* development-assets/ + installer="evidencectl-${tag}-install.sh" + awk -v version="${tag}" ' + $0 == "default_version=\"\"" { + print "default_version=\"" version "\"" + rendered = 1 + next + } + { print } + END { if (!rendered) exit 1 } + ' crates/registry-evidencectl/install.sh \ + > "development-assets/${installer}" + chmod 0755 "development-assets/${installer}" + cp "development-assets/${installer}" \ + development-assets/evidencectl-install.sh + chmod 0755 development-assets/evidencectl-install.sh + jq -n \ + --arg repository "${GITHUB_REPOSITORY}" \ + --arg source_sha "${{ needs.validate.outputs.source_sha }}" \ + --arg version "${{ needs.validate.outputs.version }}" \ + --arg tag "${tag}" \ + --argjson run_id "${GITHUB_RUN_ID}" \ + --argjson run_attempt "${GITHUB_RUN_ATTEMPT}" \ + '{ + schema: "registry.evidence-development-build/v1", + repository: $repository, + source_sha: $source_sha, + workspace_version: $version, + tag: $tag, + run_id: $run_id, + run_attempt: $run_attempt + }' > "development-assets/registry-evidence-${tag}-source.json" + checksum_file="${RUNNER_TEMP}/evidence-dev-SHA256SUMS" + ( + cd development-assets + find . -maxdepth 1 -type f ! -name SHA256SUMS -printf '%f\0' \ + | sort -z \ + | xargs -0 sha256sum -- + ) > "${checksum_file}" + mv "${checksum_file}" development-assets/SHA256SUMS + + - name: Smoke the development installer before publication + shell: bash + run: | + set -euo pipefail + install_dir="${RUNNER_TEMP}/evidence-dev-install" + EVIDENCECTL_ASSET_DIR="${GITHUB_WORKSPACE}/development-assets" \ + EVIDENCECTL_INSTALL_DIR="${install_dir}" \ + bash development-assets/evidencectl-install.sh + for binary in evidence evidencectl mint; do + observed="$("${install_dir}/${binary}" --version)" + test "${observed}" = "${binary} ${{ needs.validate.outputs.version }}" + done + + - name: Upload exact Evidence development assets + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: evidence-dev-assets-${{ github.run_id }}-${{ github.run_attempt }} + path: development-assets + if-no-files-found: error + retention-days: 8 + + publish: + name: Publish unique Evidence development prerelease + needs: + - validate + - assemble + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + actions: read + contents: write + steps: + - name: Download exact assembled development assets + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: evidence-dev-assets-${{ github.run_id }}-${{ github.run_attempt }} + path: development-assets + + - name: Reverify the closed development asset roster + shell: bash + run: | + set -euo pipefail + tag="${{ needs.validate.outputs.tag }}" + for platform in linux-amd64 linux-arm64 macos-arm64; do + for binary in evidence evidencectl mint; do + echo "${binary}-${tag}-${platform}" + done + done > "${RUNNER_TEMP}/expected-assets" + { + echo "evidencectl-${tag}-install.sh" + echo evidencectl-install.sh + echo "registry-evidence-${tag}-source.json" + echo SHA256SUMS + } >> "${RUNNER_TEMP}/expected-assets" + sort -o "${RUNNER_TEMP}/expected-assets" "${RUNNER_TEMP}/expected-assets" + find development-assets -maxdepth 1 -type f -exec basename {} \; \ + | sort > "${RUNNER_TEMP}/actual-assets" + diff -u "${RUNNER_TEMP}/expected-assets" "${RUNNER_TEMP}/actual-assets" + if find development-assets -type l -print -quit | grep -q .; then + echo "Development assets must not contain symbolic links" >&2 + exit 1 + fi + ( + cd development-assets + sha256sum --check --strict SHA256SUMS + ) + jq -e \ + --arg repository "${GITHUB_REPOSITORY}" \ + --arg source_sha "${{ needs.validate.outputs.source_sha }}" \ + --arg tag "${tag}" \ + --argjson run_id "${GITHUB_RUN_ID}" \ + --argjson run_attempt "${GITHUB_RUN_ATTEMPT}" \ + '.schema == "registry.evidence-development-build/v1" and + .repository == $repository and + .source_sha == $source_sha and + .tag == $tag and + .run_id == $run_id and + .run_attempt == $run_attempt' \ + "development-assets/registry-evidence-${tag}-source.json" \ + >/dev/null + + - name: Publish unique development prerelease + env: + GH_TOKEN: ${{ github.token }} + shell: bash + run: | + set -euo pipefail + tag="${{ needs.validate.outputs.tag }}" + source_sha="${{ needs.validate.outputs.source_sha }}" + install_url="https://github.com/${GITHUB_REPOSITORY}/releases/download/${tag}/evidencectl-install.sh" + notes="${RUNNER_TEMP}/evidence-development-release-notes.md" + { + echo "Development build from protected-main source \`${source_sha}\`." + echo + echo "This is an unsupported prerelease for development and evaluation." + echo "Its checksums are not signed and it is not a Registry Stack release." + echo + echo '```sh' + echo "curl -fsSL \"${install_url}\" | bash" + echo '```' + echo + echo "Workflow run: https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" + } > "${notes}" + gh release create "${tag}" development-assets/* \ + --repo "${GITHUB_REPOSITORY}" \ + --target "${source_sha}" \ + --title "Registry Evidence development build ${tag}" \ + --notes-file "${notes}" \ + --prerelease \ + --latest=false + { + echo '## Install this development build' + echo + echo '```sh' + echo "curl -fsSL \"${install_url}\" | bash" + echo '```' + echo + echo "Source: \`${source_sha}\`" + echo + echo "This prerelease is unsupported and its checksums are not signed." + } >> "${GITHUB_STEP_SUMMARY}" diff --git a/crates/registry-evidencectl/install.sh b/crates/registry-evidencectl/install.sh index 4ffb41f4b..757c2554b 100644 --- a/crates/registry-evidencectl/install.sh +++ b/crates/registry-evidencectl/install.sh @@ -3,12 +3,12 @@ set -euo pipefail repo="registrystack/registry-stack" binaries=(evidence evidencectl mint) -# Release packaging replaces this empty value with the asset's canonical tag. +# Publication packaging replaces this empty value with the asset's canonical tag. default_version="" script_name="${BASH_SOURCE[0]:-}" script_name="${script_name##*/}" filename_version="" -if [[ "$script_name" =~ ^evidencectl-(v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*))-install\.sh$ ]]; then +if [[ "$script_name" =~ ^evidencectl-(v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-dev\.([1-9][0-9]*)\.([1-9][0-9]*))?)-install\.sh$ ]]; then filename_version="${BASH_REMATCH[1]}" fi if [ -n "$default_version" ] && @@ -45,7 +45,7 @@ set to the verified directory: https://github.com/${repo}/blob//release/VERIFY.md Environment: - EVIDENCECTL_VERSION Release tag to install. A released installer + EVIDENCECTL_VERSION Toolset tag to install. A published installer embeds its tag and refuses a different override. EVIDENCECTL_INSTALL_DIR Install directory. Defaults to ~/.local/bin. EVIDENCECTL_ASSET_DIR Read already-downloaded release assets from this @@ -66,17 +66,21 @@ need() { } if [ -z "$version" ]; then - echo "No release is pinned for this installer copy." >&2 + echo "No toolset tag is pinned for this installer copy." >&2 echo "Evidence binaries ship with releases that include them; set" >&2 - echo "EVIDENCECTL_VERSION to a pinned vMAJOR.MINOR.PATCH tag or run the" >&2 - echo "versioned evidencectl--install.sh asset from a release." >&2 + echo "EVIDENCECTL_VERSION to a pinned vMAJOR.MINOR.PATCH tag, or run a" >&2 + echo "published evidencectl--install.sh asset." >&2 exit 1 fi -if [[ ! "$version" =~ ^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$ ]]; then - echo "Refusing non-canonical release tag '$version'." >&2 - echo "Set EVIDENCECTL_VERSION to a pinned vMAJOR.MINOR.PATCH tag." >&2 +if [[ ! "$version" =~ ^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-dev\.([1-9][0-9]*)\.([1-9][0-9]*))?$ ]]; then + echo "Refusing non-canonical Evidence toolset tag '$version'." >&2 + echo "Use vMAJOR.MINOR.PATCH or a workflow-produced vMAJOR.MINOR.PATCH-dev.RUN.ATTEMPT tag." >&2 exit 1 fi +development_build=0 +if [[ "$version" =~ -dev\.[1-9][0-9]*\.[1-9][0-9]*$ ]]; then + development_build=1 +fi need uname if [ -z "$asset_dir" ]; then @@ -191,7 +195,16 @@ for binary in "${binaries[@]}"; do verify_asset "${binary}-${version}-${os_label}-${arch_label}" done printf 'Integrity checks passed: %s binaries matched SHA256SUMS.\n' "${#binaries[@]}" -cat < Result { + tokio::time::Instant::now() + .checked_add( + Duration::from_millis(u64::from(timeout_ms)) + .saturating_add(crate::rhai_worker::WORKER_STARTUP_GRACE), + ) + .ok_or(OfflineFixtureError::ExecutionContractViolation) +} + +/// Classify a worker failure that reached the fixture runner without the host +/// having recorded a terminal error of its own. +/// +/// A timeout is the harness running out of wall clock, not the compiled plan +/// being violated. Reporting it as a plan violation sends an author looking for +/// a defect in a fixture that is correct. +fn offline_worker_error(error: WorkerError) -> OfflineFixtureError { + match error { + WorkerError::TimedOut => OfflineFixtureError::SourceDeadlineExceeded, + _ => OfflineFixtureError::ExecutionContractViolation, + } +} + fn run_rhai_worker( request: &WorkerRequest, host: &mut OfflineRhaiHost<'_>, @@ -1335,7 +1363,7 @@ fn run_rhai_worker( .build() .map_err(|_| OfflineFixtureError::ExecutionContractViolation)? .block_on(worker.evaluate_with_host(request, host, hard_deadline)) - .map_err(|_| OfflineFixtureError::ExecutionContractViolation) + .map_err(offline_worker_error) } fn rhai_output(value: RhaiTypedValue) -> Result { @@ -2959,6 +2987,30 @@ mod tests { assert!(host.terminal_error.is_none()); } + #[test] + fn the_offline_deadline_leaves_the_authored_budget_for_the_source() { + let authored = Duration::from_millis(8_000); + let granted = offline_hard_deadline(8_000) + .expect("offline hard deadline is representable") + .saturating_duration_since(tokio::time::Instant::now()); + assert!( + granted > authored, + "process start must not be charged to the authored source deadline" + ); + } + + #[test] + fn an_offline_worker_timeout_is_not_reported_as_a_plan_violation() { + assert_eq!( + offline_worker_error(WorkerError::TimedOut), + OfflineFixtureError::SourceDeadlineExceeded + ); + assert_eq!( + offline_worker_error(WorkerError::ContractViolation), + OfflineFixtureError::ExecutionContractViolation + ); + } + #[test] fn snapshot_match_projects_only_declared_physical_fields() { let fields = serde_json::Map::from_iter([ diff --git a/crates/registry-relay/src/rhai_worker.rs b/crates/registry-relay/src/rhai_worker.rs index 118d22b61..8160bd3b2 100644 --- a/crates/registry-relay/src/rhai_worker.rs +++ b/crates/registry-relay/src/rhai_worker.rs @@ -70,7 +70,7 @@ const MAX_OUTPUT_SCHEMA_NODES: usize = 256; const MAX_OUTPUT_SCHEMA_EXPANDED_NODES: usize = 4_096; const MAX_OUTPUT_OBJECT_FIELDS: usize = 32; const MAX_OUTPUT_ARRAY_ITEMS: usize = 256; -const WORKER_STARTUP_GRACE: Duration = if cfg!(debug_assertions) { +pub(crate) const WORKER_STARTUP_GRACE: Duration = if cfg!(debug_assertions) { Duration::from_secs(10) } else { Duration::from_secs(2) diff --git a/products/evidence/README.md b/products/evidence/README.md index abeaa0433..d4bc8db3b 100644 --- a/products/evidence/README.md +++ b/products/evidence/README.md @@ -191,11 +191,32 @@ tag, then rerun the installer with `EVIDENCECTL_ASSET_DIR` pointed at that verified directory. Three environment variables configure the installer: `EVIDENCECTL_VERSION` -names a `vMAJOR.MINOR.PATCH` release for a copy that carries none, +names a `vMAJOR.MINOR.PATCH` release or workflow-produced development tag for +a copy that carries none, `EVIDENCECTL_INSTALL_DIR` sets the install directory (default `~/.local/bin`), and `EVIDENCECTL_ASSET_DIR` installs from a locally verified asset directory instead of downloading. +### Manual development build + +To test the current protected `main` revision before the next release, run the +**Registry Evidence Development Build** workflow manually from the `main` +branch. It requires successful protected-main CI, builds the same three-binary +toolset for Linux amd64, Linux arm64, and macOS arm64, and creates a unique +prerelease named `v-dev..`. + +The workflow summary and prerelease notes contain the exact install command: + +```sh +curl -fsSL "https://github.com/registrystack/registry-stack/releases/download//evidencectl-install.sh" | bash +``` + +Development prereleases use unique source-bound tags, and the workflow never +overwrites them. They are unsupported. Their installer checks each binary +against the included `SHA256SUMS`; those checksums are not signed, and the +prerelease is not a Registry Stack release. Use a normal released version for +production or release verification. + To build the toolset from source instead: ```sh diff --git a/release/scripts/check-gates-inventory.py b/release/scripts/check-gates-inventory.py index 5de2bd7d4..6c38248c7 100644 --- a/release/scripts/check-gates-inventory.py +++ b/release/scripts/check-gates-inventory.py @@ -38,6 +38,10 @@ "Release workflow change classification", '".github/workflows/release.yml",', ), + ( + "Evidence development workflow change classification", + '".github/workflows/evidence-dev.yml",', + ), ( "Release candidate workflow change classification", '".github/workflows/release-candidate.yml",', @@ -363,6 +367,7 @@ RELEASE_SECURITY_POLICY_PATHS = ( ".github/workflows/docs-pages.yml", + ".github/workflows/evidence-dev.yml", ".github/workflows/release.yml", ".github/workflows/release-candidate.yml", ".github/workflows/release-canary.yml", @@ -375,6 +380,23 @@ # The compact v2 release contract is the active release inventory. REQUIRED_RELEASE_SECURITY_GATES = ( + ( + "Protected-main Evidence development prerelease", + ".github/workflows/evidence-dev.yml", + ( + "workflow_dispatch:", + '"${GITHUB_REF}" != refs/heads/main', + "name: Validate manual source and successful CI", + "actions/workflows/ci.yml/runs?head_sha=${GITHUB_SHA}&status=success", + 'tag="v${version}-dev.${GITHUB_RUN_ID}.${GITHUB_RUN_ATTEMPT}"', + "name: Smoke the development installer before publication", + "name: Reverify the closed development asset roster", + "name: Publish unique development prerelease", + '--target "${source_sha}"', + "--prerelease", + "--latest=false", + ), + ), ( "Protected-main candidate-bound annotated tag promotion", ".github/workflows/release.yml", @@ -560,6 +582,12 @@ ) ORDERED_RELEASE_SECURITY_GATES = ( + ( + "Evidence development smoke before publication permission", + ".github/workflows/evidence-dev.yml", + "name: Smoke the development installer before publication", + "publish:\n name: Publish unique Evidence development prerelease", + ), ( "Latest release recheck immediately before docs deployment", ".github/workflows/docs-pages.yml", @@ -635,6 +663,25 @@ ) FORBIDDEN_RELEASE_SECURITY_GATES = ( + ( + "Evidence development publication cannot mutate an existing release or use branch workflow code", + ".github/workflows/evidence-dev.yml", + ( + "push:", + "pull_request:", + "schedule:", + "repository_dispatch:", + "gh release upload", + "gh release delete", + "--clobber", + "git push", + "git update-ref", + "/git/refs", + "packages: write", + "id-token: write", + "attestations: write", + ), + ), ( "Promotion cannot rebuild product bytes or write refs", ".github/workflows/release.yml", diff --git a/release/scripts/test_check_gates_inventory.py b/release/scripts/test_check_gates_inventory.py index cee3502b5..3f5010428 100644 --- a/release/scripts/test_check_gates_inventory.py +++ b/release/scripts/test_check_gates_inventory.py @@ -574,6 +574,10 @@ def test_missing_executable_release_image_oci_smoke_is_reported(self) -> None: def test_missing_release_workflow_classification_is_reported(self) -> None: workflows = ( + ( + ".github/workflows/evidence-dev.yml", + "Evidence development workflow change classification", + ), ( ".github/workflows/release.yml", "Release workflow change classification", diff --git a/release/scripts/test_release_workflow_structure.py b/release/scripts/test_release_workflow_structure.py index f5e6bda45..e540a4657 100644 --- a/release/scripts/test_release_workflow_structure.py +++ b/release/scripts/test_release_workflow_structure.py @@ -48,6 +48,135 @@ def verify_latest_release_fixture(metadata: dict, expected_tag: str) -> subproce ) +class EvidenceDevelopmentWorkflowStructureTest(unittest.TestCase): + def test_is_manual_main_only_with_one_narrow_publication_job(self) -> None: + text, document = workflow("evidence-dev.yml") + trigger = text.split("permissions:", 1)[0] + self.assertIn("workflow_dispatch:", trigger) + self.assertNotIn("push:", trigger) + self.assertNotIn("pull_request:", trigger) + self.assertNotIn("schedule:", trigger) + self.assertEqual( + list(document["jobs"]), + ["validate", "build", "assemble", "publish"], + ) + self.assertEqual( + document["jobs"]["validate"]["permissions"], + {"actions": "read", "contents": "read"}, + ) + self.assertEqual( + document["jobs"]["assemble"]["permissions"], + {"actions": "read", "contents": "read"}, + ) + self.assertEqual( + document["jobs"]["publish"]["permissions"], + {"actions": "read", "contents": "write"}, + ) + self.assertEqual(text.count("contents: write"), 1) + publish_uses = { + step.get("uses", "") for step in document["jobs"]["publish"]["steps"] + } + self.assertFalse( + any(action.startswith("actions/checkout@") for action in publish_uses) + ) + + def test_binds_a_unique_dev_tag_to_successful_protected_main(self) -> None: + _, document = workflow("evidence-dev.yml") + validation = step_run( + document, + "validate", + "Validate manual source and successful CI", + ) + self.assertIn('"${GITHUB_REF}" != refs/heads/main', validation) + self.assertIn( + '"$(git rev-parse refs/remotes/origin/main)" != "${GITHUB_SHA}"', + validation, + ) + self.assertIn("actions/workflows/ci.yml/runs?head_sha=${GITHUB_SHA}", validation) + self.assertIn('.event == "push"', validation) + self.assertIn( + 'tag="v${version}-dev.${GITHUB_RUN_ID}.${GITHUB_RUN_ATTEMPT}"', + validation, + ) + self.assertIn("Cannot prove development tag ${tag} is absent", validation) + for job_name in ("build", "assemble"): + checkout = next( + step + for step in document["jobs"][job_name]["steps"] + if step.get("uses", "").startswith("actions/checkout@") + ) + self.assertEqual( + checkout["with"]["ref"], + "${{ needs.validate.outputs.source_sha }}", + ) + self.assertFalse(checkout["with"]["persist-credentials"]) + + def test_builds_and_smokes_the_released_toolset_shape(self) -> None: + _, document = workflow("evidence-dev.yml") + matrix = document["jobs"]["build"]["strategy"]["matrix"]["include"] + self.assertEqual( + {(entry["target"], entry["asset"]) for entry in matrix}, + { + ("x86_64-unknown-linux-gnu", "linux-amd64"), + ("aarch64-unknown-linux-gnu", "linux-arm64"), + ("aarch64-apple-darwin", "macos-arm64"), + }, + ) + build = step_run( + document, + "build", + "Build native Evidence development binaries", + ) + for package in ("registry-evidence", "registry-evidencectl", "registry-mint"): + self.assertIn(f"-p {package}", build) + self.assertIn("cargo build --release --locked", build) + self.assertIn("for binary in evidence evidencectl mint", build) + + assemble = step_run( + document, + "assemble", + "Assemble development assets and checksums", + ) + smoke = step_run( + document, + "assemble", + "Smoke the development installer before publication", + ) + self.assertIn('$0 == "default_version=\\\"\\\""', assemble) + self.assertIn("registry.evidence-development-build/v1", assemble) + self.assertIn("sha256sum --", assemble) + self.assertIn("EVIDENCECTL_ASSET_DIR", smoke) + self.assertIn("bash development-assets/evidencectl-install.sh", smoke) + + def test_publishes_one_unique_prerelease_and_prints_its_curl_command(self) -> None: + text, document = workflow("evidence-dev.yml") + verify = step_run( + document, + "publish", + "Reverify the closed development asset roster", + ) + publish = step_run( + document, + "publish", + "Publish unique development prerelease", + ) + self.assertIn("diff -u", verify) + self.assertIn("sha256sum --check --strict SHA256SUMS", verify) + self.assertIn("gh release create", publish) + self.assertIn('--target "${source_sha}"', publish) + self.assertIn("--prerelease", publish) + self.assertIn("--latest=false", publish) + self.assertIn("releases/download/${tag}/evidencectl-install.sh", publish) + for forbidden in ( + "gh release upload", + "gh release delete", + "--clobber", + "git push", + "git update-ref", + ): + self.assertNotIn(forbidden, text) + + class CandidateWorkflowStructureTest(unittest.TestCase): def test_current_release_pipeline_has_no_retired_notary_surface(self) -> None: paths = (