From 85b1c3746c68a50008df00c4a3ccc6566b4d4668 Mon Sep 17 00:00:00 2001 From: askalf <263217947+askalf@users.noreply.github.com> Date: Thu, 12 Nov 2026 08:39:49 -0500 Subject: [PATCH] tree-tested: the pull request's job must have run on the same runner labels --- CHANGELOG.md | 4 ++++ scripts/tree-tested/check.mjs | 22 +++++++++++++++++++++- scripts/tree-tested/check.test.mjs | 16 +++++++++++++--- 3 files changed, 38 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 571857d..6493469 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,10 @@ names the commit's date and the pull request it came from. ## November 2026 +- `actions/tree-tested` also requires the pull request's job to have asked for the same runner + labels as the job asking. A workflow whose `runs-on` depends on who opened the pull request (a + bot's or a fork's pull request on a hosted runner, its push on a self-hosted one) tested on + another machine than the push would, so that push runs in full. - A new action, `actions/tree-tested`: on a push to the default branch it says whether the pull request merged as that commit already passed the same workflow, with every job and every step green, at a head whose tree is the pushed tree byte for byte, and whether the job asking diff --git a/scripts/tree-tested/check.mjs b/scripts/tree-tested/check.mjs index 3ae97bf..ad6b069 100644 --- a/scripts/tree-tested/check.mjs +++ b/scripts/tree-tested/check.mjs @@ -32,7 +32,10 @@ // 5. This job, the one asking (found in this run by its runner's name), has a name no other job // of this run has, and exactly one job of that run has it. A job the push runs and the pull // request did not is not there, and two jobs under one display name (a matrix whose entries -// share a name) cannot be told apart, so neither is taken. +// share a name) cannot be told apart, so neither is taken. That job asked for the same runner +// labels as this one: a job whose runs-on depends on who opened the pull request (a fork's or +// a bot's pull request on a hosted runner, its push on a self-hosted one) tested on another +// machine than the push would, and is not taken. // 6. That job's log shows what its checkout made. It has exactly one actions/checkout step, and // exactly one "HEAD is now at" line in the whole log, and that line is inside the checkout // step and reads "Merge into ": GitHub's merge of the pull request. The head @@ -137,6 +140,21 @@ export function thisJob(jobs, runnerName) { return mine.length === 1 ? mine[0] : null; } +/** The runner labels a job asked for (its runs-on), lower case and sorted, or null when none are listed. */ +const labelsOf = (j) => (Array.isArray(j?.labels) && j.labels.length + ? [...new Set(j.labels.map((l) => String(l).toLowerCase()))].sort() : null); + +/** + * Why the pull request's job ran on another kind of runner than the job asking, or ''. Labels are + * compared as a set, ignoring case, as GitHub matches them. Pure. + */ +export function runnerProblem(me, twin) { + const mine = labelsOf(me), theirs = labelsOf(twin); + if (!mine || !theirs) return `the runner labels of "${me?.name ?? '?'}" are not listed for both runs`; + if (mine.join('\n') !== theirs.join('\n')) return `"${me?.name ?? '?'}" there ran on [${theirs.join(', ')}], this job on [${mine.join(', ')}]: another machine`; + return ''; +} + const STEP_START = /##\[group\]Run (.*)$/; const CHECKOUT_STEP = /^actions\/checkout@/; const HEAD_LINE = /HEAD is now at [0-9a-f]{7,40} /; @@ -243,6 +261,8 @@ export async function decide(env, io = { fetch: globalThis.fetch }) { if (namesakes !== 1) return { tested: false, why: `${namesakes} jobs of this run are named "${me.name}", so the one the pull request ran cannot be told apart` }; const twins = theirs.jobs.filter((j) => j?.name === me.name); if (twins.length !== 1) return { tested: false, why: `the job "${me.name}" is ${twins.length ? 'not one job' : 'not a job'} in ${run.html_url ?? run.id}: the pull request did not run it` }; + const elsewhere = runnerProblem(me, twins[0]); + if (elsewhere) return { tested: false, why: `${elsewhere} (${run.html_url ?? run.id})` }; // What that job checked out. const seen = checkoutMerge(await getLog(io, env, f.repo, twins[0].id)); if (seen.why) return { tested: false, why: `in the log of "${me.name}" there, ${seen.why}; what it tested is not known` }; diff --git a/scripts/tree-tested/check.test.mjs b/scripts/tree-tested/check.test.mjs index 2e88e93..452bf13 100644 --- a/scripts/tree-tested/check.test.mjs +++ b/scripts/tree-tested/check.test.mjs @@ -4,7 +4,7 @@ import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { factsOf, notEligible, mergedAs, runAtHead, jobsProblem, contains, thisJob, checkoutMerge, decide, main } from './check.mjs'; +import { factsOf, notEligible, mergedAs, runAtHead, jobsProblem, contains, thisJob, runnerProblem, checkoutMerge, decide, main } from './check.mjs'; let pass = 0; let fail = 0; @@ -30,8 +30,9 @@ const pr = (o = {}) => ({ number: 7, merged_at: '2026-11-12T00:00:00Z', merge_co const run = (o = {}) => ({ id: 900, event: 'pull_request', status: 'completed', conclusion: 'success', head_sha: HEAD, created_at: '2026-11-11T23:00:00Z', html_url: 'https://x/run/900', pull_requests: [], ...o }); const step = (name, conclusion = 'success') => ({ name, conclusion, status: 'completed' }); let jobIds = 1000; -const job = (name, conclusion = 'success', steps = [step('Set up job'), step('Test'), step('Complete job')]) => ({ id: jobIds++, name, conclusion, status: 'completed', steps }); -const mine = (name, runner = 'pool-e3') => ({ id: 1, name, status: 'in_progress', runner_name: runner }); +const LABELS = ['self-hosted', 'pool-exec']; +const job = (name, conclusion = 'success', steps = [step('Set up job'), step('Test'), step('Complete job')], labels = LABELS) => ({ id: jobIds++, name, conclusion, status: 'completed', steps, labels }); +const mine = (name, runner = 'pool-e3', labels = LABELS) => ({ id: 1, name, status: 'in_progress', runner_name: runner, labels }); // A job log as GitHub writes it: each step opens with "##[group]Run ". const T = '2026-11-12T02:58:14.1195074Z '; const checkoutStep = (head = HEAD, base = BASE) => [`${T}##[group]Run actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1`, `${T}with:`, `${T}##[endgroup]`, @@ -104,6 +105,12 @@ console.log('\n the pure rules'); check('this job is the one in progress on this runner, and only when there is one', thisJob([mine('a', 'other'), mine('b'), { name: 'c', status: 'queued', runner_name: 'pool-e3' }], 'pool-e3')?.name === 'b' && thisJob([mine('a'), mine('b')], 'pool-e3') === null && thisJob([], 'pool-e3') === null); + check('the same runner labels, in any order or case, are the same runner', + runnerProblem(mine('a'), job('a', 'success', undefined, ['POOL-EXEC', 'self-hosted', 'pool-exec'])) === ''); + check('a hosted runner there and a self-hosted one here are not', + /ran on \[ubuntu-latest\], this job on \[pool-exec, self-hosted\]: another machine/.test(runnerProblem(mine('a'), job('a', 'success', undefined, ['ubuntu-latest'])))); + check('a job with no labels listed, on either side, is not taken', + /not listed for both runs/.test(runnerProblem(mine('a', 'pool-e3', []), job('a'))) && /not listed for both runs/.test(runnerProblem(mine('a'), { name: 'a' }))); check('the merge the one checkout made is read from its step', JSON.stringify(checkoutMerge(checkoutLog())) === JSON.stringify({ head: HEAD, base: BASE })); const lines = (...parts) => parts.flat().join('\n'); // A merge checkout, then a step that checks out an older ordinary commit before the tests. @@ -148,6 +155,9 @@ for (const [name, over, re, envOver] of [ ['two jobs of this run share the display name of the job asking (a wider push matrix under name: test)', { [`/repos/${REPO}/actions/runs/${RUN_ID}/jobs`]: { total_count: 2, jobs: [mine('unit'), { id: 3, name: 'unit', status: 'queued' }] } }, /2 jobs of this run are named "unit"/], ['the caller has not declared the job the same on push', {}, /has not declared/, { SAME_ON_PULL_REQUEST: '' }], + // A bot's or a fork's pull request runs on a hosted runner; the push after it runs on ours. + ['the pull request\'s job ran on another runner than this push does', + { [`/repos/${REPO}/actions/runs/900/jobs`]: { total_count: 2, jobs: [job('unit', 'success', undefined, ['ubuntu-latest']), job('e2e')] } }, /"unit" there ran on \[ubuntu-latest\], this job on \[pool-exec, self-hosted\]/], ['the log shows a merge of another head', { LOG: { text: checkoutLog(OTHER, BASE) } }, /merge of ddddddd, not of #7's head/], // Tested against a release branch, then retargeted to main with no new run: the base it merged // is not in the head, so the job tested the head plus that branch, not the pushed tree.