diff --git a/CHANGELOG.md b/CHANGELOG.md index e5e9c5a..d6a5fc9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ Notable changes to Testfold are documented here. The project follows Semantic Ve ## Unreleased +## 0.4.4 — 2026-09-07 + +- Count Playwright tests that pass on retry as successful tests in suite totals instead of silently omitting flaky outcomes from the summary. + ## 0.4.2 — 2026-09-06 - Update the full dependency tree, including ESLint 10, Jest 30, TypeScript 6, and Zod 4, while preserving the Node.js 22.18.0 runtime contract. diff --git a/package-lock.json b/package-lock.json index 4dde105..d9ff4ef 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "testfold", - "version": "0.4.3", + "version": "0.4.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "testfold", - "version": "0.4.3", + "version": "0.4.4", "license": "MIT", "dependencies": { "dotenv": "^17.4.2", diff --git a/package.json b/package.json index d87d02f..e099616 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "testfold", - "version": "0.4.3", + "version": "0.4.4", "packageManager": "npm@11.19.1", "description": "Unified test runner with Jest and Playwright support, parallel execution, and detailed failure reporting", "type": "module", diff --git a/src/parsers/playwright.ts b/src/parsers/playwright.ts index a41b08c..83033a5 100644 --- a/src/parsers/playwright.ts +++ b/src/parsers/playwright.ts @@ -149,7 +149,11 @@ export class PlaywrightParser implements Parser { this.collectResults(data.suites, '', failures, testResults); return { - passed: data.stats.expected, + // Playwright reports tests that passed only after a retry under `flaky`, + // not `expected`. They are still terminally successful tests and must be + // included in the passed/total counts; otherwise summary totals silently + // lose executed specs while collectResults correctly marks them passed. + passed: data.stats.expected + data.stats.flaky, failed: data.stats.unexpected, skipped: data.stats.skipped, duration: data.stats.duration, diff --git a/tests/fixtures/playwright/flaky.json b/tests/fixtures/playwright/flaky.json new file mode 100644 index 0000000..a6d6fc3 --- /dev/null +++ b/tests/fixtures/playwright/flaky.json @@ -0,0 +1,39 @@ +{ + "config": {}, + "suites": [ + { + "title": "Retry Tests", + "file": "tests/e2e/retry.spec.ts", + "specs": [ + { + "title": "passes immediately", + "ok": true, + "tests": [{ + "title": "passes immediately", + "results": [{ "status": "passed", "duration": 10, "attachments": [] }] + }] + }, + { + "title": "passes on retry", + "ok": true, + "tests": [{ + "title": "passes on retry", + "results": [ + { "status": "failed", "duration": 20, "error": { "message": "first attempt" }, "attachments": [] }, + { "status": "passed", "duration": 15, "attachments": [] } + ] + }] + } + ], + "suites": [] + } + ], + "stats": { + "startTime": "2026-09-07T00:00:00.000Z", + "duration": 45, + "expected": 1, + "unexpected": 0, + "skipped": 0, + "flaky": 1 + } +} diff --git a/tests/unit/parsers/playwright.test.ts b/tests/unit/parsers/playwright.test.ts index 8300238..9dc1847 100644 --- a/tests/unit/parsers/playwright.test.ts +++ b/tests/unit/parsers/playwright.test.ts @@ -18,6 +18,17 @@ describe('PlaywrightParser', () => { expect(result.success).toBe(true); }); + it('counts a test that passes on retry as passed instead of dropping it from totals', async () => { + const result = await parser.parse(resolve(fixturesDir, 'flaky.json')); + + expect(result.passed).toBe(2); + expect(result.failed).toBe(0); + expect(result.skipped).toBe(0); + expect(result.success).toBe(true); + expect(result.testResults).toHaveLength(2); + expect(result.testResults?.every((test) => test.status === 'passed')).toBe(true); + }); + it('should extract duration from stats', async () => { const result = await parser.parse(resolve(fixturesDir, 'success.json'));