From 5774edbf9e5aebf7f60b6301095c6821c5a42620 Mon Sep 17 00:00:00 2001 From: quality Date: Fri, 2 Oct 2026 17:00:58 -0400 Subject: [PATCH] test: assert release.yml and pre-release.yml stay one pipeline and attach the artifacts docs/releasing.md lists Signed-off-by: quality --- __tests__/releaseWorkflows.test.ts | 121 +++++++++++++++++++++++++++++ __tests__/utils/workflowYaml.ts | 31 ++++++++ __tests__/workflows.test.ts | 28 +------ 3 files changed, 155 insertions(+), 25 deletions(-) create mode 100644 __tests__/releaseWorkflows.test.ts create mode 100644 __tests__/utils/workflowYaml.ts diff --git a/__tests__/releaseWorkflows.test.ts b/__tests__/releaseWorkflows.test.ts new file mode 100644 index 0000000..998ba99 --- /dev/null +++ b/__tests__/releaseWorkflows.test.ts @@ -0,0 +1,121 @@ +import type { Job, Step, Workflow } from './utils/workflowYaml' +import { describe, expect, it } from 'vitest' + +import { expression, loadYaml, read } from './utils/workflowYaml' + +// release.yml (tag push) and pre-release.yml (manual) are two copies of one pipeline: verify the +// bundle, generate and sign an SBOM, attest provenance, publish a GitHub Release with three +// artifacts. Nothing shares code between them, so every edit to one has to be mirrored by hand; +// this file fails when the copies drift apart, or apart from what docs/releasing.md lists. + +const releasePath = '.github/workflows/release.yml' +const preReleasePath = '.github/workflows/pre-release.yml' +const releasingDoc = 'docs/releasing.md' + +const release = loadYaml(releasePath) +const preRelease = loadYaml(preReleasePath) + +function steps(job: Job): Step[] { + return job.steps ?? [] +} + +function runOf(job: Job, name: string): string { + const found = steps(job).find(s => s.name === name) + expect(found?.run, `no step named '${name}' with a run:`).toBeDefined() + return found?.run as string +} + +function usesOf(job: Job): string[] { + return steps(job).flatMap(s => (s.uses ? [s.uses] : [])) +} + +function releaseStep(job: Job): Step { + const found = steps(job).find(s => s.uses?.startsWith('softprops/action-gh-release@')) + expect(found, 'no softprops/action-gh-release step').toBeDefined() + return found as Step +} + +const sharedScripts = ['Install waybill', 'Generate SBOM (SPDX 2.3)', 'Sign SBOM', 'Rename provenance bundle'] + +describe('release.yml and pre-release.yml are the same pipeline', () => { + const [r, p] = [release.jobs['github-release'], preRelease.jobs['github-release']] + + it('pin the same waybill version and sha256', () => { + expect(release.env?.WAYBILL_VERSION).toMatch(/^v\d+\.\d+\.\d+$/) + expect(release.env?.WAYBILL_SHA256).toMatch(/^[0-9a-f]{64}$/) + expect(preRelease.env?.WAYBILL_VERSION).toBe(release.env?.WAYBILL_VERSION) + expect(preRelease.env?.WAYBILL_SHA256).toBe(release.env?.WAYBILL_SHA256) + }) + + it('pin the same action shas in the same order in both jobs', () => { + expect(usesOf(preRelease.jobs.verify)).toEqual(usesOf(release.jobs.verify)) + expect(usesOf(p)).toEqual(usesOf(r)) + }) + + it.each(sharedScripts)('run an identical \'%s\' script', (name) => { + expect(runOf(p, name)).toBe(runOf(r, name)) + }) + + it('attest the same subjects', () => { + const attest = (job: Job) => steps(job).find(s => s.uses?.startsWith('actions/attest-build-provenance@')) + expect(attest(p)?.with).toEqual(attest(r)?.with) + }) + + it('run the same verify job, apart from the pre-release input check and the ::error:: wording', () => { + const strip = (job: Job) => steps(job) + .filter(s => s.name !== 'Validate version input') + .map(s => ({ uses: s.uses, with: s.with, run: s.run?.split('\n').filter(line => !line.includes('::error::')).join('\n') })) + expect(strip(preRelease.jobs.verify)).toEqual(strip(release.jobs.verify)) + }) +}) + +describe.each([ + [releasePath, release], + [preReleasePath, preRelease], +])('%s', (_file, workflow) => { + const publish = workflow.jobs['github-release'] + + it('reads only at the top level and grants the release job exactly what signing and publishing need', () => { + expect(workflow.permissions).toEqual({ contents: 'read' }) + expect(workflow.jobs.verify.permissions).toBeUndefined() + expect(publish.permissions).toEqual({ 'contents': 'write', 'id-token': 'write', 'attestations': 'write' }) + }) + + it('checks the waybill sha256 before unpacking the tarball', () => { + const install = runOf(publish, 'Install waybill') + expect(install).toContain('sha256sum -c') + expect(install.indexOf('sha256sum')).toBeLessThan(install.indexOf('tar -xzf')) + }) + + it('attaches exactly the artifacts docs/releasing.md lists', () => { + const documented = [...read(releasingDoc).matchAll(/`prow-github-actions-([^`]+)`/g)].map(m => m[1]) + expect(documented.length).toBeGreaterThan(0) + const attached = String(releaseStep(publish).with?.files).trim().split('\n').map(s => s.trim()) + expect(attached).toEqual(documented.map(suffix => `prow-github-actions-${expression('env.VERSION')}${suffix}`)) + }) +}) + +describe('pre-release.yml version input', () => { + const validate = runOf(preRelease.jobs.verify, 'Validate version input') + const pattern = /grep -Eq '(\^.*\$)'/.exec(validate)?.[1] + + it('is checked by a grep pattern in the first verify step', () => { + expect(steps(preRelease.jobs.verify)[0]?.name).toBe('Validate version input') + expect(pattern, 'no grep -Eq pattern in the validation step').toBeDefined() + }) + + it.each([ + ['2.1.0-rc.1', true], + ['3.0.0-alpha.0', true], + ['10.20.30-beta.12', true], + ['2.1.0', false], + ['v2.1.0-rc.1', false], + ['2.1.0-rc', false], + ['2.1.0-rc.1.2', false], + ['2.1-rc.1', false], + ['2.1.0-dev.1', false], + ['2.1.0-rc.1 ', false], + ])('%s accepted: %s', (version, accepted) => { + expect(new RegExp(pattern as string).test(version)).toBe(accepted) + }) +}) diff --git a/__tests__/utils/workflowYaml.ts b/__tests__/utils/workflowYaml.ts new file mode 100644 index 0000000..feac1c4 --- /dev/null +++ b/__tests__/utils/workflowYaml.ts @@ -0,0 +1,31 @@ +import { readFileSync } from 'node:fs' +import * as path from 'node:path' +import * as yaml from 'js-yaml' + +// the shape of a GitHub Actions workflow as the workflow tests read it, and the readers they share + +export const root = path.resolve(__dirname, '..', '..') + +export type Mapping = Record +export interface Step { name?: string, id?: string, if?: string, uses?: string, with?: Mapping, run?: string, env?: Mapping } +export interface Job { name?: string, needs?: string, if?: string, uses?: string, with?: Mapping, permissions?: Mapping, env?: Mapping, steps?: Step[] } +export interface Workflow { + on: Mapping + permissions?: Mapping + concurrency?: Mapping + env?: Mapping + jobs: Record +} + +/** `${{ inner }}`, an Actions expression */ +export function expression(inner: string): string { + return `$\{{ ${inner} }}` +} + +export function read(file: string): string { + return readFileSync(path.join(root, file), 'utf8') +} + +export function loadYaml(file: string): T { + return yaml.load(read(file)) as T +} diff --git a/__tests__/workflows.test.ts b/__tests__/workflows.test.ts index e73dd52..3239f34 100644 --- a/__tests__/workflows.test.ts +++ b/__tests__/workflows.test.ts @@ -1,14 +1,14 @@ +import type { Mapping, Workflow } from './utils/workflowYaml' import { existsSync, readdirSync, readFileSync } from 'node:fs' import * as path from 'node:path' -import * as yaml from 'js-yaml' -import { describe, expect, it } from 'vitest' +import { describe, expect, it } from 'vitest' import { fixedLabelCommands } from '../src/labels/fixed' import { prefixedLabelCommands } from '../src/labels/prefixed' import { mergeProwConfig, parseProwConfig } from '../src/utils/config' import { builtinLabelDefaults, desiredLabels } from '../src/utils/labelCatalog' +import { expression, loadYaml, read, root } from './utils/workflowYaml' -const root = path.resolve(__dirname, '..') const { version } = JSON.parse(readFileSync(path.join(root, 'package.json'), 'utf8')) as { version: string } const reusableWorkflowPath = '.github/workflows/prow.yml' const reusableWorkflowRef = `cncf/prow-github-actions/.github/workflows/prow.yml@v${version}` @@ -29,28 +29,6 @@ const requiredTriggers: Record = { push: [], } -type Mapping = Record -interface Step { uses?: string, with?: Mapping, run?: string } -interface Job { if?: string, uses?: string, with?: Mapping, permissions?: Mapping, steps?: Step[] } -interface Workflow { - on: Mapping - permissions?: Mapping - concurrency?: Mapping - jobs: Record -} - -function expression(inner: string): string { - return `$\{{ ${inner} }}` -} - -function read(file: string): string { - return readFileSync(path.join(root, file), 'utf8') -} - -function loadYaml(file: string): T { - return yaml.load(read(file)) as T -} - const actionInputs = Object.keys((loadYaml<{ inputs: Mapping }>('action.yml')).inputs) const reusable = loadYaml(reusableWorkflowPath) const workflowCall = reusable.on.workflow_call as { inputs: Mapping, secrets: Mapping }