diff --git a/CHANGELOG.md b/CHANGELOG.md index d6a5fc9..6290c3e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ Notable changes to Testfold are documented here. The project follows Semantic Ve ## Unreleased +## 0.4.5 — 2026-09-24 + +- Fail the run with exit code 2 and name each `--file` filter that selected no test file in any selected suite, instead of silently dropping it when another filter matched. A filter that matches a file in another selected suite is not an error. + ## 0.4.4 — 2026-09-07 - Count Playwright tests that pass on retry as successful tests in suite totals instead of silently omitting flaky outcomes from the summary. diff --git a/CLAUDE.md b/CLAUDE.md index 7b7f2d1..aca65b7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -65,7 +65,7 @@ testfold -r console,json,timing-text # Multiple reporters testfold -r ./my-reporter.ts # Custom reporter from file testfold -g "auth" # Filter by test name testfold --grep-invert "slow" # Exclude tests by pattern -testfold -f auth.test.ts # Filter by file +testfold -f auth.test.ts # Filter by file (a filter matching no test in any selected suite fails the run, exit 2) testfold --dry-run # Preview commands without running testfold --dry-run unit # Preview only unit suite diff --git a/README.md b/README.md index 7dc5834..567c940 100644 --- a/README.md +++ b/README.md @@ -199,6 +199,18 @@ export default { Returning `void` or `undefined` is treated as success (backward compatible). +### File Filters + +`--file` (`-f`) values are passed to Jest and Playwright as positional test-path filters, which the frameworks treat as regular expressions searched in each test file's path; Playwright also accepts a `:line` suffix. The frameworks combine several filters as a union, so testfold checks each requested filter after the run: + +- A filter must select at least one test file that one of the selected suites ran. A filter that matches a file in any selected suite is satisfied; it does not have to match every suite. +- Each filter that selected no test file in any selected suite is recorded as an infrastructure failure named `Unmatched File Filter: `, attached to the first selected suite. The run fails with exit code `2`, even when other filters matched and their tests passed. +- The check applies only when every selected suite ran and reported the test files it executed. When a suite was skipped by fail-fast, stopped by a `beforeSuite` guard, timed out, produced no result file, or uses a custom parser that reports neither `testFiles` nor a complete `testResults` list, the check is skipped because the filter may belong to that suite; such a run already fails or depends on that suite's own result. + +With `--file`, reporters receive each suite's result after all selected suites finish, so a suite is never shown as passed before the check. + +Custom parsers can return `testFiles` (the test files the framework ran) in their `ParseResult` to take part in the check. + ### Path Prefix Resolution Pass partial test file names and testfold resolves them to full paths: diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index f09ff8b..5924ccb 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -36,7 +36,7 @@ testfold is a unified test runner that orchestrates multiple test frameworks (Je ### Core Layer (`src/core/`) - **runner.ts** - `TestRunner` class, main entry point. Creates reporters (built-in + custom from file paths) -- **orchestrator.ts** - Coordinates suite execution (parallel/sequential), guard-aware hooks, filter flag passing +- **orchestrator.ts** - Coordinates suite execution (parallel/sequential), guard-aware hooks, filter flag passing, and `--file` coverage: a requested file that selected no test in any selected suite fails the run with exit code 2 - **executor.ts** - Spawns child processes with `detached: true`, captures output. SIGKILL escalation on timeout, workers→maxWorkers mapping, progress streaming via onOutput callback ### Parsers (`src/parsers/`) @@ -63,6 +63,7 @@ testfold is a unified test runner that orchestrates multiple test frameworks (Je - **ansi.ts** - Strip ANSI escape codes - **sanitize.ts** - Sanitize strings for filenames - **files.ts** - Directory cleanup, file writing +- **file-filters.ts** - Matches `--file` filters against the test files each suite ran; used by the orchestrator to fail runs where a requested file selected no test ### CLI (`src/cli/`) diff --git a/package-lock.json b/package-lock.json index d9ff4ef..128e729 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "testfold", - "version": "0.4.4", + "version": "0.4.5", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "testfold", - "version": "0.4.4", + "version": "0.4.5", "license": "MIT", "dependencies": { "dotenv": "^17.4.2", diff --git a/package.json b/package.json index e099616..2b7d177 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "testfold", - "version": "0.4.4", + "version": "0.4.5", "packageManager": "npm@11.19.1", "description": "Unified test runner with Jest and Playwright support, parallel execution, and detailed failure reporting", "type": "module", diff --git a/src/cli/index.ts b/src/cli/index.ts index 1881f8e..3df4a50 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -136,7 +136,9 @@ Options: Custom: path to reporter module (e.g., ./my-reporter.ts) --grep, -g Filter tests by name pattern --grep-invert Exclude tests matching pattern - --file, -f Filter by test file (repeat for multiple files) + --file, -f Filter by test file (repeat for multiple files); + a file matching no test in any selected suite + fails the run with exit code 2 --dry-run Print planned commands without executing --no-parallel Run suites sequentially --fail-fast Stop on first failure @@ -146,7 +148,7 @@ Options: Exit Codes: 0 All tests passed 1 One or more tests failed - 2 Infrastructure error (config, parse, spawn failure) + 2 Infrastructure error (config, parse, spawn failure, unmatched --file) 3 Suite killed by timeout Pass-through Arguments: diff --git a/src/config/types.ts b/src/config/types.ts index 35dd653..bd08458 100644 --- a/src/config/types.ts +++ b/src/config/types.ts @@ -119,6 +119,8 @@ export interface SuiteResult { resultFile: string; /** Individual test results for timing analysis */ testResults?: TestResult[]; + /** Test files the framework ran, when the parser could determine them */ + testFiles?: string[]; /** Error category for semantic exit code determination */ errorCategory?: ErrorCategory; } diff --git a/src/core/orchestrator.ts b/src/core/orchestrator.ts index 0849b61..0f18884 100644 --- a/src/core/orchestrator.ts +++ b/src/core/orchestrator.ts @@ -14,6 +14,7 @@ import { CustomParserLoader } from '../parsers/custom.js'; import { executeCommand } from './executor.js'; import { readEnvFileContent } from '../config/env-loader.js'; import { createProgressCallback } from '../utils/progress.js'; +import { findUnmatchedFileFilters, normalizeFileFilters } from '../utils/file-filters.js'; export interface OrchestratorOptions { config: ValidatedConfig; @@ -83,10 +84,26 @@ export class Orchestrator { await this.config.hooks.beforeAll(); } + // With --file filters, suite results are reported only after every suite has + // finished, because an unmatched filter can be judged only across all of them. + const fileFilters = normalizeFileFilters(this.file); + const notifyPerSuite = fileFilters.length === 0; + // Execute suites const results = this.config.parallel - ? await this.runParallel(suitesToRun) - : await this.runSequential(suitesToRun); + ? await this.runParallel(suitesToRun, notifyPerSuite) + : await this.runSequential(suitesToRun, notifyPerSuite); + + if (!notifyPerSuite) { + this.applyFileFilterCoverage(fileFilters, suitesToRun, results); + results.forEach((result, index) => { + const suite = suitesToRun[index]; + if (!suite) return; + for (const reporter of this.reporters) { + reporter.onSuiteComplete(suite, result); + } + }); + } // Aggregate results const aggregated = this.aggregate(results); @@ -104,16 +121,57 @@ export class Orchestrator { return aggregated; } - private async runParallel(suites: Suite[]): Promise { - const promises = suites.map((suite) => this.runSuite(suite)); + /** + * Fail the run when a requested --file filter selected no test file in any + * selected suite. Frameworks combine positional filters as a union, so without + * this check a mistyped path is silently dropped while other paths pass. + * + * The check applies only when every selected suite ran and reported the test + * files it executed. Otherwise the run already fails for that suite's own + * reason, and an unmatched filter could belong to the suite whose files are + * unknown. Unmatched filters are recorded as infrastructure failures on the + * first selected suite and name every suite that was searched. + */ + private applyFileFilterCoverage( + filters: string[], + suites: Suite[], + results: SuiteResult[], + ): void { + const target = results[0]; + if (!target || results.length < suites.length) return; + + const unmatched = findUnmatchedFileFilters( + filters, + results.map((result) => result.testFiles), + this.cwd, + ); + if (!unmatched || unmatched.length === 0) return; + + const searched = suites.map((suite) => suite.name).join(', '); + for (const filter of unmatched) { + target.failures.push({ + testName: `Unmatched File Filter: ${filter}`, + filePath: '', + error: `--file ${filter} matched no test file in the selected suites (${searched}). No test from this path ran.`, + }); + target.failed += 1; + } + target.success = false; + if (target.errorCategory !== 'timeout') { + target.errorCategory = 'infra_error'; + } + } + + private async runParallel(suites: Suite[], notify: boolean): Promise { + const promises = suites.map((suite) => this.runSuite(suite, notify)); return Promise.all(promises); } - private async runSequential(suites: Suite[]): Promise { + private async runSequential(suites: Suite[], notify: boolean): Promise { const results: SuiteResult[] = []; for (const suite of suites) { - const result = await this.runSuite(suite); + const result = await this.runSuite(suite, notify); results.push(result); if (this.config.failFast && !result.success) { @@ -124,7 +182,7 @@ export class Orchestrator { return results; } - private async runSuite(suite: Suite): Promise { + private async runSuite(suite: Suite, notify: boolean): Promise { // Run before hook and check guard result if (this.config.hooks?.beforeSuite) { const guardResult = await this.config.hooks.beforeSuite(suite); @@ -142,8 +200,10 @@ export class Orchestrator { resultFile: '', errorCategory: 'infra_error', }; - for (const reporter of this.reporters) { - reporter.onSuiteComplete(suite, result); + if (notify) { + for (const reporter of this.reporters) { + reporter.onSuiteComplete(suite, result); + } } return result; } @@ -258,6 +318,7 @@ export class Orchestrator { logFile, resultFile, testResults: parseResult.testResults, + testFiles: errorCategory === 'timeout' ? undefined : this.getTestFiles(parseResult), errorCategory, }; @@ -276,13 +337,28 @@ export class Orchestrator { } // Notify reporters (after guard check so reporters see final state) - for (const reporter of this.reporters) { - reporter.onSuiteComplete(suite, result); + if (notify) { + for (const reporter of this.reporters) { + reporter.onSuiteComplete(suite, result); + } } return result; } + /** + * Test files the suite ran. Parsers that do not report them (custom parsers) + * fall back to the files of their test results, but only when those results + * account for every counted test; a partial list cannot prove a filter missed. + */ + private getTestFiles(parseResult: ParseResult): string[] | undefined { + if (parseResult.testFiles) return parseResult.testFiles; + const tests = parseResult.testResults; + const counted = parseResult.passed + parseResult.failed + parseResult.skipped; + if (!tests || tests.length !== counted) return undefined; + return [...new Set(tests.map((test) => test.file))]; + } + private getParser(suite: Suite) { switch (suite.type) { case 'jest': diff --git a/src/parsers/jest.ts b/src/parsers/jest.ts index 7669288..f1ce696 100644 --- a/src/parsers/jest.ts +++ b/src/parsers/jest.ts @@ -134,6 +134,7 @@ export class JestParser implements Parser { success: data.success && crashedSuites === 0, failures, testResults, + testFiles: data.testResults.map((fileResult) => fileResult.name), }; } diff --git a/src/parsers/playwright.ts b/src/parsers/playwright.ts index 83033a5..5856ed8 100644 --- a/src/parsers/playwright.ts +++ b/src/parsers/playwright.ts @@ -4,11 +4,12 @@ import { readFile } from 'node:fs/promises'; import { existsSync } from 'node:fs'; +import { isAbsolute, resolve } from 'node:path'; import type { Parser, ParseResult, TestResult } from './types.js'; import type { FailureDetail } from '../config/types.js'; interface PlaywrightResult { - config: unknown; + config?: { rootDir?: string }; suites: PlaywrightSuite[]; stats: { startTime: string; @@ -160,9 +161,28 @@ export class PlaywrightParser implements Parser { success: data.stats.unexpected === 0, failures, testResults, + testFiles: this.collectTestFiles(data.suites, data.config?.rootDir), }; } + /** + * Playwright reports spec files relative to its rootDir. Resolve them so they + * can be compared with the paths requested through `--file`. + */ + private collectTestFiles(suites: PlaywrightSuite[], rootDir?: string): string[] { + const files = new Set(); + const visit = (items: PlaywrightSuite[]) => { + for (const suite of items) { + if (suite.file) { + files.add(rootDir && !isAbsolute(suite.file) ? resolve(rootDir, suite.file) : suite.file); + } + if (suite.suites) visit(suite.suites); + } + }; + visit(suites); + return [...files]; + } + private collectResults( suites: PlaywrightSuite[], parentTitle: string, diff --git a/src/parsers/types.ts b/src/parsers/types.ts index 48b841d..693fccf 100644 --- a/src/parsers/types.ts +++ b/src/parsers/types.ts @@ -18,6 +18,13 @@ export interface ParseResult { errorCategory?: ErrorCategory; /** Raw test results for timing analysis */ testResults?: TestResult[]; + /** + * Every test file the framework ran, including files that crashed before + * reporting a test. Used to confirm that each requested `--file` filter + * selected at least one test file. Custom parsers may omit it; `testResults` + * files are then used instead. + */ + testFiles?: string[]; } export interface Parser { diff --git a/src/utils/file-filters.ts b/src/utils/file-filters.ts new file mode 100644 index 0000000..caf99df --- /dev/null +++ b/src/utils/file-filters.ts @@ -0,0 +1,81 @@ +/** + * File filter coverage. + * + * `--file` values are forwarded to Jest and Playwright as positional test-path + * filters. Both frameworks combine positional filters as a union, so a filter + * that matches nothing is silently ignored whenever another filter matches. + * These helpers check each requested filter against the test files the + * selected suites actually ran. + */ + +import { isAbsolute, resolve } from 'node:path'; + +/** Normalize the `file` option into a list of non-empty filters. */ +export function normalizeFileFilters(file?: string | string[]): string[] { + if (file === undefined) return []; + const values = Array.isArray(file) ? file : [file]; + return values.filter((value) => value.trim() !== ''); +} + +function toPosix(value: string): string { + return value.replace(/\\/g, '/'); +} + +/** + * Check whether a requested filter selects a test file. + * + * Jest and Playwright treat a positional filter as a regular expression that is + * searched in the absolute test-file path; Playwright also accepts a + * `:line[:column]` suffix. Literal path comparisons are applied as well, so a + * filter that is not a valid regular expression, or a test file reported + * relative to a framework root, is still recognized. + */ +export function matchesFileFilter(filter: string, testFile: string, cwd: string): boolean { + const pattern = filter.replace(/:\d+(?::\d+)?$/, ''); + const absoluteFile = isAbsolute(testFile) ? testFile : resolve(cwd, testFile); + const candidates = [ + ...new Set([testFile, absoluteFile, toPosix(testFile), toPosix(absoluteFile)]), + ]; + const posixPattern = toPosix(pattern); + + let regex: RegExp | null = null; + try { + regex = new RegExp(pattern); + } catch { + regex = null; + } + + if (regex && candidates.some((candidate) => regex.test(candidate))) { + return true; + } + + if ( + candidates.some((candidate) => candidate.includes(posixPattern) || candidate.includes(pattern)) + ) { + return true; + } + + if (toPosix(resolve(cwd, pattern)) === toPosix(absoluteFile)) { + return true; + } + + // Test file reported relative to a framework root that is unknown here. + return !isAbsolute(testFile) && posixPattern.endsWith(`/${toPosix(testFile)}`); +} + +/** + * Return the filters that match no test file in any of the given inventories. + * Returns `null` when at least one inventory is unknown, because an unmatched + * filter could then belong to the suite whose files are not known. + */ +export function findUnmatchedFileFilters( + filters: string[], + inventories: Array, + cwd: string, +): string[] | null { + if (filters.length === 0) return []; + if (inventories.some((inventory) => inventory === undefined)) return null; + + const files = inventories.flatMap((inventory) => inventory ?? []); + return filters.filter((filter) => !files.some((file) => matchesFileFilter(filter, file, cwd))); +} diff --git a/tests/unit/core/orchestrator-file-filters.test.ts b/tests/unit/core/orchestrator-file-filters.test.ts new file mode 100644 index 0000000..e77cab0 --- /dev/null +++ b/tests/unit/core/orchestrator-file-filters.test.ts @@ -0,0 +1,195 @@ +import { resolve, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { mkdir, writeFile, rm } from 'node:fs/promises'; +import { Orchestrator } from '../../../src/core/orchestrator.js'; +import type { ValidatedConfig } from '../../../src/config/schema.js'; +import type { Suite, SuiteResult } from '../../../src/config/types.js'; +import type { Reporter } from '../../../src/reporters/types.js'; +import { matchesFileFilter } from '../../../src/utils/file-filters.js'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const tempDir = resolve(__dirname, '../../fixtures/temp-file-filters'); +const projectDir = resolve(tempDir, 'project'); + +function jestResult(files: string[]): string { + return JSON.stringify({ + numPassedTests: files.length, + numFailedTests: 0, + numFailedTestSuites: 0, + numPendingTests: 0, + numTotalTests: files.length, + success: true, + testResults: files.map((file) => ({ + name: resolve(projectDir, file), + status: 'passed', + message: '', + startTime: 0, + endTime: 10, + assertionResults: [ + { + ancestorTitles: ['suite'], + title: 'works', + status: 'passed', + duration: 1, + failureMessages: [], + }, + ], + })), + }); +} + +function jestSuite(name: string, resultFile: string): Suite { + return { name, type: 'jest', command: 'echo ran', resultFile }; +} + +function createRecordingReporter(): Reporter & { completed: SuiteResult[] } { + const completed: SuiteResult[] = []; + return { + completed, + onStart: () => {}, + onSuiteComplete: (_suite, result) => { + completed.push({ ...result, failures: [...result.failures] }); + }, + onComplete: async () => {}, + }; +} + +async function run(suites: Suite[], file: string[]) { + const reporter = createRecordingReporter(); + const config: ValidatedConfig = { + suites, + artifactsDir: tempDir, + parallel: false, + failFast: false, + }; + const orchestrator = new Orchestrator({ config, reporters: [reporter], cwd: projectDir, file }); + const results = await orchestrator.run(); + return { results, reporter }; +} + +describe('--file filter coverage', () => { + beforeAll(async () => { + await mkdir(projectDir, { recursive: true }); + await writeFile(resolve(tempDir, 'unit.json'), jestResult(['tests/unit/labels.test.ts'])); + await writeFile( + resolve(tempDir, 'integration.json'), + jestResult(['tests/integration/api.test.ts']), + ); + await writeFile( + resolve(tempDir, 'e2e.json'), + JSON.stringify({ + config: { rootDir: resolve(projectDir, 'tests/e2e') }, + stats: { startTime: '', duration: 5, expected: 1, unexpected: 0, skipped: 0, flaky: 0 }, + suites: [ + { + title: 'login.spec.ts', + file: 'login.spec.ts', + specs: [ + { + title: 'logs in', + ok: true, + tests: [{ title: 'logs in', results: [{ status: 'passed', duration: 5 }] }], + }, + ], + }, + ], + }), + ); + }); + + afterAll(async () => { + await rm(tempDir, { recursive: true, force: true }); + }); + + it('fails and names a requested file that matched no test while another file passed', async () => { + const { results, reporter } = await run( + [jestSuite('unit', 'unit.json')], + ['tests/unit/labels.test.ts', 'tests/unit/does-not-exist.test.ts'], + ); + + expect(results.success).toBe(false); + expect(results.exitCode).toBe(2); + expect(results.totals.passed).toBe(1); + expect(results.totals.failed).toBe(1); + const unit = results.suites[0]; + expect(unit?.errorCategory).toBe('infra_error'); + expect(unit?.failures.map((f) => f.testName)).toEqual([ + 'Unmatched File Filter: tests/unit/does-not-exist.test.ts', + ]); + expect(unit?.failures[0]?.error).toContain('tests/unit/does-not-exist.test.ts'); + // Reporters receive the suite only after the check, so no suite is shown as passed. + expect(reporter.completed).toHaveLength(1); + expect(reporter.completed[0]?.success).toBe(false); + }); + + it('passes when every requested file matched a test', async () => { + const { results, reporter } = await run( + [jestSuite('unit', 'unit.json')], + ['tests/unit/labels.test.ts', 'labels'], + ); + + expect(results.success).toBe(true); + expect(results.exitCode).toBe(0); + expect(reporter.completed[0]?.success).toBe(true); + }); + + it('accepts a file that belongs to another selected suite', async () => { + const { results } = await run( + [jestSuite('unit', 'unit.json'), jestSuite('integration', 'integration.json')], + ['tests/unit/labels.test.ts', 'tests/integration/api.test.ts'], + ); + + expect(results.success).toBe(true); + expect(results.exitCode).toBe(0); + }); + + it('reports a file missing from every selected suite once, naming the searched suites', async () => { + const { results } = await run( + [jestSuite('unit', 'unit.json'), jestSuite('integration', 'integration.json')], + ['tests/unit/labels.test.ts', 'tests/unit/typo.test.ts'], + ); + + expect(results.exitCode).toBe(2); + expect(results.totals.failed).toBe(1); + expect(results.suites[0]?.failures[0]?.error).toContain('(unit, integration)'); + expect(results.suites[1]?.failures).toEqual([]); + }); + + it('matches Playwright spec files reported relative to rootDir, including line suffixes', async () => { + const { results } = await run( + [{ name: 'e2e', type: 'playwright', command: 'echo ran', resultFile: 'e2e.json' }], + ['tests/e2e/login.spec.ts:12'], + ); + + expect(results.success).toBe(true); + expect(results.exitCode).toBe(0); + }); + + it('does not claim a missing file when a selected suite reported no file inventory', async () => { + const { results } = await run( + [jestSuite('unit', 'unit.json'), jestSuite('broken', 'absent.json')], + ['tests/unit/labels.test.ts', 'tests/broken/some.test.ts'], + ); + + expect(results.exitCode).toBe(2); + const testNames = results.suites.flatMap((s) => s.failures.map((f) => f.testName)); + expect(testNames).toEqual(['Missing Result File']); + }); +}); + +describe('matchesFileFilter', () => { + const cwd = '/repo'; + + it.each([ + ['tests/unit/a.test.ts', '/repo/tests/unit/a.test.ts', true], + ['./tests/unit/a.test.ts', '/repo/tests/unit/a.test.ts', true], + ['/repo/tests/unit/a.test.ts', '/repo/tests/unit/a.test.ts', true], + ['a.test', '/repo/tests/unit/a.test.ts', true], + ['tests/unit/b.test.ts', '/repo/tests/unit/a.test.ts', false], + ['tests/unit/a.test.ts:10:2', '/repo/tests/unit/a.test.ts', true], + ['tests/e2e/login.spec.ts', 'login.spec.ts', true], + ['tests/(unclosed', '/repo/tests/(unclosed.test.ts', true], + ])('%s against %s -> %s', (filter, file, expected) => { + expect(matchesFileFilter(filter, file, cwd)).toBe(expected); + }); +});