From 8c5e5dff210dceef4753d8b73ae96afcee474c78 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Isaac=20Rold=C3=A1n?= <isaac.roldan@shopify.com>
Date: Wed, 12 Aug 2026 17:00:16 +0200
Subject: [PATCH] Retry E2E tests once in CI and report flaky results

With retries: 0, a single flaky interaction fails the whole shard. CI
now retries each failing test once; a test that passes on retry keeps
the shard green.

Retries are only honest if flake stays visible, so the JSON reporter is
enabled in CI and scripts/report-flaky.js surfaces every
failed-then-passed test as a grep-able FLAKY: log line and in the GitHub
job summary. maxFailures goes from 3 to 5 because first attempts of
flaky tests count toward it, and the old limit could stop a shard
before retries prove tests flaky rather than broken.

Local runs keep retries: 0 so flake fails fast during development.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 .github/workflows/tests-pr.yml       |  4 ++
 packages/e2e/playwright.config.ts    |  8 ++--
 packages/e2e/scripts/report-flaky.js | 57 ++++++++++++++++++++++++++++
 3 files changed, 66 insertions(+), 3 deletions(-)
 create mode 100644 packages/e2e/scripts/report-flaky.js

diff --git a/.github/workflows/tests-pr.yml b/.github/workflows/tests-pr.yml
index 3165c915e66..c375d1a6a72 100644
--- a/.github/workflows/tests-pr.yml
+++ b/.github/workflows/tests-pr.yml
@@ -260,6 +260,10 @@ jobs:
           E2E_ORG_ID: ${{ secrets.E2E_ORG_ID }}
           E2E_LOADTEST_HEADER: ${{ secrets.E2E_LOADTEST_HEADER }}
         run: pnpm exec playwright test --shard ${{ matrix.shard }}
+      - name: Report flaky tests
+        if: ${{ !cancelled() }}
+        working-directory: packages/e2e
+        run: node scripts/report-flaky.js
       - name: Upload Playwright report
         uses: actions/upload-artifact@v4
         if: ${{ !cancelled() }}
diff --git a/packages/e2e/playwright.config.ts b/packages/e2e/playwright.config.ts
index e10b229cd3d..19edd3990b2 100644
--- a/packages/e2e/playwright.config.ts
+++ b/packages/e2e/playwright.config.ts
@@ -12,10 +12,12 @@ export default defineConfig({
   testDir: './tests',
   fullyParallel: true,
   forbidOnly: isCI,
-  retries: 0,
+  retries: isCI ? 1 : 0, // One retry in CI; flaky-on-retry is reported, not ignored (scripts/report-flaky.js)
   workers: 10,
-  maxFailures: isCI ? 3 : 0, // Stop early in CI after 3 failures
-  reporter: isCI ? [['html', {open: 'never'}], ['list']] : [['list']],
+  maxFailures: isCI ? 5 : 0, // Stop early in CI; first attempts of flaky tests count, so leave room for retries
+  reporter: isCI
+    ? [['html', {open: 'never'}], ['list'], ['json', {outputFile: 'test-results/results.json'}]]
+    : [['list']],
   timeout: TEST_TIMEOUT.default, // Heavy tests override via test.setTimeout()
   globalTimeout: 20 * 60 * 1000,
 
diff --git a/packages/e2e/scripts/report-flaky.js b/packages/e2e/scripts/report-flaky.js
new file mode 100644
index 00000000000..621f73c7b6e
--- /dev/null
+++ b/packages/e2e/scripts/report-flaky.js
@@ -0,0 +1,57 @@
+/* eslint-disable no-console */
+
+// Surfaces tests that failed and then passed on retry ("flaky" in Playwright's
+// JSON report). Retries keep flake from failing the shard; this keeps it from
+// disappearing. Each flaky test is printed as a grep-able `FLAKY:` log line
+// and listed in the GitHub job summary.
+
+import {appendFileSync, existsSync, readFileSync} from 'node:fs'
+import * as path from 'node:path'
+import {fileURLToPath} from 'node:url'
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url))
+const resultsPath = path.join(__dirname, '../test-results/results.json')
+
+if (!existsSync(resultsPath)) {
+  console.log('no results.json found, skipping flaky report')
+  process.exit(0)
+}
+
+const report = JSON.parse(readFileSync(resultsPath, 'utf8'))
+
+const flakyTests = []
+function walkSuite(suite, titlePath) {
+  for (const spec of suite.specs ?? []) {
+    for (const test of spec.tests ?? []) {
+      if (test.status === 'flaky') {
+        flakyTests.push({title: [...titlePath, spec.title].join(' › '), file: spec.file})
+      }
+    }
+  }
+  for (const child of suite.suites ?? []) {
+    walkSuite(child, [...titlePath, child.title])
+  }
+}
+for (const suite of report.suites ?? []) {
+  // Root suites are titled with the file name, which spec.file already carries.
+  walkSuite(suite, [])
+}
+
+if (flakyTests.length === 0) {
+  console.log('no flaky tests in this shard')
+  process.exit(0)
+}
+
+for (const test of flakyTests) {
+  console.log(`FLAKY: ${test.file} › ${test.title}`)
+}
+
+if (process.env.GITHUB_STEP_SUMMARY) {
+  const summary = [
+    '## Flaky tests (failed, then passed on retry)',
+    '',
+    ...flakyTests.map((test) => `- \`${test.file}\` › ${test.title}`),
+    '',
+  ].join('\n')
+  appendFileSync(process.env.GITHUB_STEP_SUMMARY, `${summary}\n`)
+}