Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,10 @@ names the commit's date and the pull request it came from.

## November 2026

- `actions/tree-tested` also requires the pull request's job to have asked for the same runner
labels as the job asking. A workflow whose `runs-on` depends on who opened the pull request (a
bot's or a fork's pull request on a hosted runner, its push on a self-hosted one) tested on
another machine than the push would, so that push runs in full.
- A new action, `actions/tree-tested`: on a push to the default branch it says whether the
pull request merged as that commit already passed the same workflow, with every job and every
step green, at a head whose tree is the pushed tree byte for byte, and whether the job asking
Expand Down
22 changes: 21 additions & 1 deletion scripts/tree-tested/check.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,10 @@
// 5. This job, the one asking (found in this run by its runner's name), has a name no other job
// of this run has, and exactly one job of that run has it. A job the push runs and the pull
// request did not is not there, and two jobs under one display name (a matrix whose entries
// share a name) cannot be told apart, so neither is taken.
// share a name) cannot be told apart, so neither is taken. That job asked for the same runner
// labels as this one: a job whose runs-on depends on who opened the pull request (a fork's or
// a bot's pull request on a hosted runner, its push on a self-hosted one) tested on another
// machine than the push would, and is not taken.
// 6. That job's log shows what its checkout made. It has exactly one actions/checkout step, and
// exactly one "HEAD is now at" line in the whole log, and that line is inside the checkout
// step and reads "Merge <head> into <base>": GitHub's merge of the pull request. The head
Expand Down Expand Up @@ -137,6 +140,21 @@ export function thisJob(jobs, runnerName) {
return mine.length === 1 ? mine[0] : null;
}

/** The runner labels a job asked for (its runs-on), lower case and sorted, or null when none are listed. */
const labelsOf = (j) => (Array.isArray(j?.labels) && j.labels.length
? [...new Set(j.labels.map((l) => String(l).toLowerCase()))].sort() : null);

/**
* Why the pull request's job ran on another kind of runner than the job asking, or ''. Labels are
* compared as a set, ignoring case, as GitHub matches them. Pure.
*/
export function runnerProblem(me, twin) {
const mine = labelsOf(me), theirs = labelsOf(twin);
if (!mine || !theirs) return `the runner labels of "${me?.name ?? '?'}" are not listed for both runs`;
if (mine.join('\n') !== theirs.join('\n')) return `"${me?.name ?? '?'}" there ran on [${theirs.join(', ')}], this job on [${mine.join(', ')}]: another machine`;
return '';
}

const STEP_START = /##\[group\]Run (.*)$/;
const CHECKOUT_STEP = /^actions\/checkout@/;
const HEAD_LINE = /HEAD is now at [0-9a-f]{7,40} /;
Expand Down Expand Up @@ -243,6 +261,8 @@ export async function decide(env, io = { fetch: globalThis.fetch }) {
if (namesakes !== 1) return { tested: false, why: `${namesakes} jobs of this run are named "${me.name}", so the one the pull request ran cannot be told apart` };
const twins = theirs.jobs.filter((j) => j?.name === me.name);
if (twins.length !== 1) return { tested: false, why: `the job "${me.name}" is ${twins.length ? 'not one job' : 'not a job'} in ${run.html_url ?? run.id}: the pull request did not run it` };
const elsewhere = runnerProblem(me, twins[0]);
if (elsewhere) return { tested: false, why: `${elsewhere} (${run.html_url ?? run.id})` };
// What that job checked out.
const seen = checkoutMerge(await getLog(io, env, f.repo, twins[0].id));
if (seen.why) return { tested: false, why: `in the log of "${me.name}" there, ${seen.why}; what it tested is not known` };
Expand Down
16 changes: 13 additions & 3 deletions scripts/tree-tested/check.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
import { mkdtempSync, readFileSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { factsOf, notEligible, mergedAs, runAtHead, jobsProblem, contains, thisJob, checkoutMerge, decide, main } from './check.mjs';
import { factsOf, notEligible, mergedAs, runAtHead, jobsProblem, contains, thisJob, runnerProblem, checkoutMerge, decide, main } from './check.mjs';

let pass = 0;
let fail = 0;
Expand All @@ -30,8 +30,9 @@ const pr = (o = {}) => ({ number: 7, merged_at: '2026-11-12T00:00:00Z', merge_co
const run = (o = {}) => ({ id: 900, event: 'pull_request', status: 'completed', conclusion: 'success', head_sha: HEAD, created_at: '2026-11-11T23:00:00Z', html_url: 'https://x/run/900', pull_requests: [], ...o });
const step = (name, conclusion = 'success') => ({ name, conclusion, status: 'completed' });
let jobIds = 1000;
const job = (name, conclusion = 'success', steps = [step('Set up job'), step('Test'), step('Complete job')]) => ({ id: jobIds++, name, conclusion, status: 'completed', steps });
const mine = (name, runner = 'pool-e3') => ({ id: 1, name, status: 'in_progress', runner_name: runner });
const LABELS = ['self-hosted', 'pool-exec'];
const job = (name, conclusion = 'success', steps = [step('Set up job'), step('Test'), step('Complete job')], labels = LABELS) => ({ id: jobIds++, name, conclusion, status: 'completed', steps, labels });
const mine = (name, runner = 'pool-e3', labels = LABELS) => ({ id: 1, name, status: 'in_progress', runner_name: runner, labels });
// A job log as GitHub writes it: each step opens with "##[group]Run <step>".
const T = '2026-11-12T02:58:14.1195074Z ';
const checkoutStep = (head = HEAD, base = BASE) => [`${T}##[group]Run actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1`, `${T}with:`, `${T}##[endgroup]`,
Expand Down Expand Up @@ -104,6 +105,12 @@ console.log('\n the pure rules');
check('this job is the one in progress on this runner, and only when there is one',
thisJob([mine('a', 'other'), mine('b'), { name: 'c', status: 'queued', runner_name: 'pool-e3' }], 'pool-e3')?.name === 'b'
&& thisJob([mine('a'), mine('b')], 'pool-e3') === null && thisJob([], 'pool-e3') === null);
check('the same runner labels, in any order or case, are the same runner',
runnerProblem(mine('a'), job('a', 'success', undefined, ['POOL-EXEC', 'self-hosted', 'pool-exec'])) === '');
check('a hosted runner there and a self-hosted one here are not',
/ran on \[ubuntu-latest\], this job on \[pool-exec, self-hosted\]: another machine/.test(runnerProblem(mine('a'), job('a', 'success', undefined, ['ubuntu-latest']))));
check('a job with no labels listed, on either side, is not taken',
/not listed for both runs/.test(runnerProblem(mine('a', 'pool-e3', []), job('a'))) && /not listed for both runs/.test(runnerProblem(mine('a'), { name: 'a' })));
check('the merge the one checkout made is read from its step', JSON.stringify(checkoutMerge(checkoutLog())) === JSON.stringify({ head: HEAD, base: BASE }));
const lines = (...parts) => parts.flat().join('\n');
// A merge checkout, then a step that checks out an older ordinary commit before the tests.
Expand Down Expand Up @@ -148,6 +155,9 @@ for (const [name, over, re, envOver] of [
['two jobs of this run share the display name of the job asking (a wider push matrix under name: test)',
{ [`/repos/${REPO}/actions/runs/${RUN_ID}/jobs`]: { total_count: 2, jobs: [mine('unit'), { id: 3, name: 'unit', status: 'queued' }] } }, /2 jobs of this run are named "unit"/],
['the caller has not declared the job the same on push', {}, /has not declared/, { SAME_ON_PULL_REQUEST: '' }],
// A bot's or a fork's pull request runs on a hosted runner; the push after it runs on ours.
['the pull request\'s job ran on another runner than this push does',
{ [`/repos/${REPO}/actions/runs/900/jobs`]: { total_count: 2, jobs: [job('unit', 'success', undefined, ['ubuntu-latest']), job('e2e')] } }, /"unit" there ran on \[ubuntu-latest\], this job on \[pool-exec, self-hosted\]/],
['the log shows a merge of another head', { LOG: { text: checkoutLog(OTHER, BASE) } }, /merge of ddddddd, not of #7's head/],
// Tested against a release branch, then retargeted to main with no new run: the base it merged
// is not in the head, so the job tested the head plus that branch, not the pushed tree.
Expand Down
Loading