From 85b1c3746c68a50008df00c4a3ccc6566b4d4668 Mon Sep 17 00:00:00 2001
From: askalf <263217947+askalf@users.noreply.github.com>
Date: Thu, 12 Nov 2026 08:39:49 -0500
Subject: [PATCH] tree-tested: the pull request's job must have run on the same
runner labels
---
CHANGELOG.md | 4 ++++
scripts/tree-tested/check.mjs | 22 +++++++++++++++++++++-
scripts/tree-tested/check.test.mjs | 16 +++++++++++++---
3 files changed, 38 insertions(+), 4 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 571857d..6493469 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -13,6 +13,10 @@ names the commit's date and the pull request it came from.
## November 2026
+- `actions/tree-tested` also requires the pull request's job to have asked for the same runner
+ labels as the job asking. A workflow whose `runs-on` depends on who opened the pull request (a
+ bot's or a fork's pull request on a hosted runner, its push on a self-hosted one) tested on
+ another machine than the push would, so that push runs in full.
- A new action, `actions/tree-tested`: on a push to the default branch it says whether the
pull request merged as that commit already passed the same workflow, with every job and every
step green, at a head whose tree is the pushed tree byte for byte, and whether the job asking
diff --git a/scripts/tree-tested/check.mjs b/scripts/tree-tested/check.mjs
index 3ae97bf..ad6b069 100644
--- a/scripts/tree-tested/check.mjs
+++ b/scripts/tree-tested/check.mjs
@@ -32,7 +32,10 @@
// 5. This job, the one asking (found in this run by its runner's name), has a name no other job
// of this run has, and exactly one job of that run has it. A job the push runs and the pull
// request did not is not there, and two jobs under one display name (a matrix whose entries
-// share a name) cannot be told apart, so neither is taken.
+// share a name) cannot be told apart, so neither is taken. That job asked for the same runner
+// labels as this one: a job whose runs-on depends on who opened the pull request (a fork's or
+// a bot's pull request on a hosted runner, its push on a self-hosted one) tested on another
+// machine than the push would, and is not taken.
// 6. That job's log shows what its checkout made. It has exactly one actions/checkout step, and
// exactly one "HEAD is now at" line in the whole log, and that line is inside the checkout
// step and reads "Merge
into ": GitHub's merge of the pull request. The head
@@ -137,6 +140,21 @@ export function thisJob(jobs, runnerName) {
return mine.length === 1 ? mine[0] : null;
}
+/** The runner labels a job asked for (its runs-on), lower case and sorted, or null when none are listed. */
+const labelsOf = (j) => (Array.isArray(j?.labels) && j.labels.length
+ ? [...new Set(j.labels.map((l) => String(l).toLowerCase()))].sort() : null);
+
+/**
+ * Why the pull request's job ran on another kind of runner than the job asking, or ''. Labels are
+ * compared as a set, ignoring case, as GitHub matches them. Pure.
+ */
+export function runnerProblem(me, twin) {
+ const mine = labelsOf(me), theirs = labelsOf(twin);
+ if (!mine || !theirs) return `the runner labels of "${me?.name ?? '?'}" are not listed for both runs`;
+ if (mine.join('\n') !== theirs.join('\n')) return `"${me?.name ?? '?'}" there ran on [${theirs.join(', ')}], this job on [${mine.join(', ')}]: another machine`;
+ return '';
+}
+
const STEP_START = /##\[group\]Run (.*)$/;
const CHECKOUT_STEP = /^actions\/checkout@/;
const HEAD_LINE = /HEAD is now at [0-9a-f]{7,40} /;
@@ -243,6 +261,8 @@ export async function decide(env, io = { fetch: globalThis.fetch }) {
if (namesakes !== 1) return { tested: false, why: `${namesakes} jobs of this run are named "${me.name}", so the one the pull request ran cannot be told apart` };
const twins = theirs.jobs.filter((j) => j?.name === me.name);
if (twins.length !== 1) return { tested: false, why: `the job "${me.name}" is ${twins.length ? 'not one job' : 'not a job'} in ${run.html_url ?? run.id}: the pull request did not run it` };
+ const elsewhere = runnerProblem(me, twins[0]);
+ if (elsewhere) return { tested: false, why: `${elsewhere} (${run.html_url ?? run.id})` };
// What that job checked out.
const seen = checkoutMerge(await getLog(io, env, f.repo, twins[0].id));
if (seen.why) return { tested: false, why: `in the log of "${me.name}" there, ${seen.why}; what it tested is not known` };
diff --git a/scripts/tree-tested/check.test.mjs b/scripts/tree-tested/check.test.mjs
index 2e88e93..452bf13 100644
--- a/scripts/tree-tested/check.test.mjs
+++ b/scripts/tree-tested/check.test.mjs
@@ -4,7 +4,7 @@
import { mkdtempSync, readFileSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
-import { factsOf, notEligible, mergedAs, runAtHead, jobsProblem, contains, thisJob, checkoutMerge, decide, main } from './check.mjs';
+import { factsOf, notEligible, mergedAs, runAtHead, jobsProblem, contains, thisJob, runnerProblem, checkoutMerge, decide, main } from './check.mjs';
let pass = 0;
let fail = 0;
@@ -30,8 +30,9 @@ const pr = (o = {}) => ({ number: 7, merged_at: '2026-11-12T00:00:00Z', merge_co
const run = (o = {}) => ({ id: 900, event: 'pull_request', status: 'completed', conclusion: 'success', head_sha: HEAD, created_at: '2026-11-11T23:00:00Z', html_url: 'https://x/run/900', pull_requests: [], ...o });
const step = (name, conclusion = 'success') => ({ name, conclusion, status: 'completed' });
let jobIds = 1000;
-const job = (name, conclusion = 'success', steps = [step('Set up job'), step('Test'), step('Complete job')]) => ({ id: jobIds++, name, conclusion, status: 'completed', steps });
-const mine = (name, runner = 'pool-e3') => ({ id: 1, name, status: 'in_progress', runner_name: runner });
+const LABELS = ['self-hosted', 'pool-exec'];
+const job = (name, conclusion = 'success', steps = [step('Set up job'), step('Test'), step('Complete job')], labels = LABELS) => ({ id: jobIds++, name, conclusion, status: 'completed', steps, labels });
+const mine = (name, runner = 'pool-e3', labels = LABELS) => ({ id: 1, name, status: 'in_progress', runner_name: runner, labels });
// A job log as GitHub writes it: each step opens with "##[group]Run ".
const T = '2026-11-12T02:58:14.1195074Z ';
const checkoutStep = (head = HEAD, base = BASE) => [`${T}##[group]Run actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1`, `${T}with:`, `${T}##[endgroup]`,
@@ -104,6 +105,12 @@ console.log('\n the pure rules');
check('this job is the one in progress on this runner, and only when there is one',
thisJob([mine('a', 'other'), mine('b'), { name: 'c', status: 'queued', runner_name: 'pool-e3' }], 'pool-e3')?.name === 'b'
&& thisJob([mine('a'), mine('b')], 'pool-e3') === null && thisJob([], 'pool-e3') === null);
+ check('the same runner labels, in any order or case, are the same runner',
+ runnerProblem(mine('a'), job('a', 'success', undefined, ['POOL-EXEC', 'self-hosted', 'pool-exec'])) === '');
+ check('a hosted runner there and a self-hosted one here are not',
+ /ran on \[ubuntu-latest\], this job on \[pool-exec, self-hosted\]: another machine/.test(runnerProblem(mine('a'), job('a', 'success', undefined, ['ubuntu-latest']))));
+ check('a job with no labels listed, on either side, is not taken',
+ /not listed for both runs/.test(runnerProblem(mine('a', 'pool-e3', []), job('a'))) && /not listed for both runs/.test(runnerProblem(mine('a'), { name: 'a' })));
check('the merge the one checkout made is read from its step', JSON.stringify(checkoutMerge(checkoutLog())) === JSON.stringify({ head: HEAD, base: BASE }));
const lines = (...parts) => parts.flat().join('\n');
// A merge checkout, then a step that checks out an older ordinary commit before the tests.
@@ -148,6 +155,9 @@ for (const [name, over, re, envOver] of [
['two jobs of this run share the display name of the job asking (a wider push matrix under name: test)',
{ [`/repos/${REPO}/actions/runs/${RUN_ID}/jobs`]: { total_count: 2, jobs: [mine('unit'), { id: 3, name: 'unit', status: 'queued' }] } }, /2 jobs of this run are named "unit"/],
['the caller has not declared the job the same on push', {}, /has not declared/, { SAME_ON_PULL_REQUEST: '' }],
+ // A bot's or a fork's pull request runs on a hosted runner; the push after it runs on ours.
+ ['the pull request\'s job ran on another runner than this push does',
+ { [`/repos/${REPO}/actions/runs/900/jobs`]: { total_count: 2, jobs: [job('unit', 'success', undefined, ['ubuntu-latest']), job('e2e')] } }, /"unit" there ran on \[ubuntu-latest\], this job on \[pool-exec, self-hosted\]/],
['the log shows a merge of another head', { LOG: { text: checkoutLog(OTHER, BASE) } }, /merge of ddddddd, not of #7's head/],
// Tested against a release branch, then retargeted to main with no new run: the base it merged
// is not in the head, so the job tested the head plus that branch, not the pushed tree.