Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,10 @@ Notable changes to Testfold are documented here. The project follows Semantic Ve

## Unreleased

## 0.4.4 — 2026-09-07

- Count Playwright tests that pass on retry as successful tests in suite totals instead of silently omitting flaky outcomes from the summary.

## 0.4.2 — 2026-09-06

- Update the full dependency tree, including ESLint 10, Jest 30, TypeScript 6, and Zod 4, while preserving the Node.js 22.18.0 runtime contract.
Expand Down
4 changes: 2 additions & 2 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "testfold",
"version": "0.4.3",
"version": "0.4.4",
"packageManager": "npm@11.19.1",
"description": "Unified test runner with Jest and Playwright support, parallel execution, and detailed failure reporting",
"type": "module",
Expand Down
6 changes: 5 additions & 1 deletion src/parsers/playwright.ts
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,11 @@ export class PlaywrightParser implements Parser {
this.collectResults(data.suites, '', failures, testResults);

return {
passed: data.stats.expected,
// Playwright reports tests that passed only after a retry under `flaky`,
// not `expected`. They are still terminally successful tests and must be
// included in the passed/total counts; otherwise summary totals silently
// lose executed specs while collectResults correctly marks them passed.
passed: data.stats.expected + data.stats.flaky,
failed: data.stats.unexpected,
skipped: data.stats.skipped,
duration: data.stats.duration,
Expand Down
39 changes: 39 additions & 0 deletions tests/fixtures/playwright/flaky.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
{
"config": {},
"suites": [
{
"title": "Retry Tests",
"file": "tests/e2e/retry.spec.ts",
"specs": [
{
"title": "passes immediately",
"ok": true,
"tests": [{
"title": "passes immediately",
"results": [{ "status": "passed", "duration": 10, "attachments": [] }]
}]
},
{
"title": "passes on retry",
"ok": true,
"tests": [{
"title": "passes on retry",
"results": [
{ "status": "failed", "duration": 20, "error": { "message": "first attempt" }, "attachments": [] },
{ "status": "passed", "duration": 15, "attachments": [] }
]
}]
}
],
"suites": []
}
],
"stats": {
"startTime": "2026-09-07T00:00:00.000Z",
"duration": 45,
"expected": 1,
"unexpected": 0,
"skipped": 0,
"flaky": 1
}
}
11 changes: 11 additions & 0 deletions tests/unit/parsers/playwright.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,17 @@ describe('PlaywrightParser', () => {
expect(result.success).toBe(true);
});

it('counts a test that passes on retry as passed instead of dropping it from totals', async () => {
const result = await parser.parse(resolve(fixturesDir, 'flaky.json'));

expect(result.passed).toBe(2);
expect(result.failed).toBe(0);
expect(result.skipped).toBe(0);
expect(result.success).toBe(true);
expect(result.testResults).toHaveLength(2);
expect(result.testResults?.every((test) => test.status === 'passed')).toBe(true);
});

it('should extract duration from stats', async () => {
const result = await parser.parse(resolve(fixturesDir, 'success.json'));

Expand Down