From d9980b280411661c23be9b5a625eafda2b1f2833 Mon Sep 17 00:00:00 2001 From: geyutu6755 Date: Fri, 6 Nov 2026 00:23:18 +0800 Subject: [PATCH 1/4] =?UTF-8?q?fix(v0.5.1):=20composite=20action=20reliabi?= =?UTF-8?q?lity=20=E2=80=94=20fail-on-new=20semantics,=20exit-code=20order?= =?UTF-8?q?ing,=20SHA=20support,=20npm=20ci?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three real defects found by external-repo CI runs and code audit: 1. fail-on-new input was declared but its env var never read — the gate always used --fail-on new regardless of the user's setting. Now: fail-on-new=true → compare --fail-on new (gate on new findings); fail-on-new=false → compare without the flag (report only); invalid values → explicit error. 2. Scan step exited 1 when findings existed, causing GitHub Actions to skip the gate step. Now the scan step completes on exit 0 or 1 (recording the verdict), and only exits 2 for environment errors. A new Final CI decision step evaluates scan verdict + gate conclusion in priority order (error > gate > findings > clean) and produces the final exit code. 3. reprocase-ref claimed to support tags, branches and commit SHAs but used git clone --branch which fails for SHAs. Now: --branch first, then fallback to full clone + git checkout for SHAs, with an explicit error for unresolvable refs (no silent version fallback). The checked-out commit is verified and reported. Also: npm install → npm ci for lockfile-reproducible installs; hashFiles guard prevents empty artifact upload; invalid fail-on-new values produce an explicit error. 18 new tests (10 action structure + 8 gate semantics). 142/142 tests green, npm audit 0. --- .github/actions/reprocase-scan/action.yml | 145 ++++++++++++++++++---- tests/integration/gate-semantics.test.ts | 132 ++++++++++++++++++++ tests/unit/action-yml.test.ts | 93 ++++++++++++++ 3 files changed, 347 insertions(+), 23 deletions(-) create mode 100644 tests/integration/gate-semantics.test.ts create mode 100644 tests/unit/action-yml.test.ts diff --git a/.github/actions/reprocase-scan/action.yml b/.github/actions/reprocase-scan/action.yml index 36628df..4ee20b4 100644 --- a/.github/actions/reprocase-scan/action.yml +++ b/.github/actions/reprocase-scan/action.yml @@ -12,7 +12,7 @@ inputs: description: 'Entry URL to scan (e.g. http://localhost:3000/ — start your site in an earlier step).' required: true reprocase-ref: - description: 'ReproCase version to install (tag, branch or commit). Defaults to the ref this action was pinned to.' + description: 'ReproCase version to install: a tag (v0.5.1), a branch (main), or a full commit SHA. Defaults to the ref this action was pinned to.' required: false default: '' config: @@ -24,11 +24,11 @@ inputs: required: false default: '' fail-on: - description: 'Exit 1 when "verified" or "suspected" findings exist, or "never" (report only).' + description: 'Exit 1 when "verified" or "suspected" findings exist, or "never" (report only). Does NOT affect the baseline gate — see fail-on-new.' required: false default: 'verified' fail-on-new: - description: 'When compare-with is set: exit 1 if any NEW findings exist. Default true.' + description: 'When compare-with is set: "true" (default) fails the action if new findings exist vs baseline; "false" reports the diff but does not gate on new findings.' required: false default: 'true' max-pages: @@ -53,8 +53,11 @@ outputs: description: 'clean (exit 0) | findings (exit 1) | error (exit 2)' value: ${{ steps.scan.outputs.verdict }} exit-code: - description: 'Raw scanner exit code (0/1/2).' - value: ${{ steps.scan.outputs.exit-code }} + description: 'Final action exit code (0/1/2).' + value: ${{ steps.decision.outputs.exit-code }} + scan-exit-code: + description: 'Raw scanner exit code before gating.' + value: ${{ steps.scan.outputs.scan-exit-code }} new-findings: description: 'Number of new findings vs the baseline (empty when no baseline was given).' value: ${{ steps.gate.outputs.new-findings }} @@ -67,24 +70,45 @@ runs: node-version: ${{ inputs.node-version }} - name: Install ReproCase + id: install shell: bash env: REPROCASE_REF: ${{ inputs.reprocase-ref || github.action_ref }} run: | set -euo pipefail REF="${REPROCASE_REF}" - case "$REF" in - ''|refs/*) REF="v0.5.0" ;; - esac + # Strip leading "refs/" so the composite action's own ref value works. + case "$REF" in refs/*) REF="${REF#refs/}";; esac + # Default: a known-good release tag (never silently fall back to + # another ref when the user explicitly provided one that doesn't exist). + if [ -z "$REF" ]; then REF="v0.5.1"; fi SRC="${RUNNER_TEMP}/reprocase-src" rm -rf "$SRC" echo "Installing ReproCase from github.com/geyutu6755/ReproCase ($REF)" - git clone --depth 1 --branch "$REF" https://github.com/geyutu6755/ReproCase.git "$SRC" + # Try tag/branch first via --branch; fall back to SHA via raw checkout. + if git clone --depth 1 --branch "$REF" https://github.com/geyutu6755/ReproCase.git "$SRC" 2>/dev/null; then + echo "Cloned tag/branch: $REF" + else + echo "--branch clone failed; trying SHA checkout" + git clone https://github.com/geyutu6755/ReproCase.git "$SRC" + cd "$SRC" + if git cat-file -e "$REF^{commit}" 2>/dev/null; then + git checkout "$REF" + echo "Checked out SHA: $(git rev-parse HEAD)" + else + echo "::error::ReproCase ref '$REF' not found as a tag, branch or commit SHA in geyutu6755/ReproCase. Available tags: https://github.com/geyutu6755/ReproCase/tags" + exit 1 + fi + fi cd "$SRC" + ACTUAL=$(git rev-parse HEAD) + echo "reprocase-commit=$ACTUAL" >> "$GITHUB_OUTPUT" + echo "Checked out: $ACTUAL" # Global installs omit devDependencies, and the TypeScript build needs - # them — so build here (npm ci includes dev deps) and install the - # built package globally. - npm install --no-fund --no-audit + # them — so build here (npm ci uses the lockfile for reproducibility) + # and install the built package globally. The prepare script runs + # after dist/ already exists, so it's a no-op at that point. + npm ci --no-fund --no-audit npm run build npm install -g --no-fund --no-audit . reprocase --version @@ -115,7 +139,7 @@ runs: reprocase "${ARGS[@]}" CODE=$? set -e - echo "exit-code=$CODE" >> "$GITHUB_OUTPUT" + echo "scan-exit-code=$CODE" >> "$GITHUB_OUTPUT" case "$CODE" in 0) echo "verdict=clean" >> "$GITHUB_OUTPUT" @@ -124,27 +148,41 @@ runs: 1) echo "verdict=findings" >> "$GITHUB_OUTPUT" echo "ReproCase: scan completed — findings exist at or above the threshold." + # Do NOT exit 1 here: the gate step needs to run to compare against + # the baseline. The final decision step handles the exit code. ;; 2) echo "verdict=error" >> "$GITHUB_OUTPUT" echo "::error::ReproCase could not run properly (exit 2). This is an environment or configuration problem, NOT a verdict about your website. See the log above; 'reprocase doctor' diagnoses most causes." + exit 2 ;; *) echo "verdict=error" >> "$GITHUB_OUTPUT" echo "::error::ReproCase exited unexpectedly (code $CODE)." + exit 1 ;; esac - exit "$CODE" - name: Compare with baseline (new-fault gate) id: gate - if: ${{ inputs.compare-with != '' }} + if: ${{ inputs.compare-with != '' && steps.scan.outputs.verdict != 'error' }} shell: bash env: BASELINE: ${{ inputs.compare-with }} - FAIL_ON_NEW: ${{ inputs.fail-on-new }} + FAIL_ON_NEW_RAW: ${{ inputs.fail-on-new }} run: | set -uo pipefail + # Normalise fail-on-new: case-insensitive, trim whitespace. + FAIL_ON_NEW=$(echo "$FAIL_ON_NEW_RAW" | tr -d '[:space:]' | tr '[:upper:]' '[:lower:]') + case "$FAIL_ON_NEW" in + true|'') GATE_FLAG="--fail-on new" ;; + false) GATE_FLAG="" ;; + *) + echo "::error::Invalid fail-on-new value '$FAIL_ON_NEW_RAW'. Use 'true' or 'false'." + exit 1 + ;; + esac + echo "fail-on-new-normalised=$FAIL_ON_NEW" >> "$GITHUB_OUTPUT" if [ ! -f "$BASELINE" ]; then echo "::notice::Baseline '$BASELINE' not found — first run on this branch? Skipping the new-findings gate; this scan becomes the new baseline candidate." echo "new-findings=" >> "$GITHUB_OUTPUT" @@ -154,7 +192,8 @@ runs: echo "::error::Scan produced no result.json — the scan may have failed before writing results. Cannot compare." exit 1 fi - GATE_ARGS=(compare "$BASELINE" reprocase-report/result.json --out reprocase-report/compare --fail-on new) + GATE_ARGS=(compare "$BASELINE" reprocase-report/result.json --out reprocase-report/compare) + [ -n "$GATE_FLAG" ] && GATE_ARGS+=("$GATE_FLAG") set +e reprocase "${GATE_ARGS[@]}" CODE=$? @@ -162,11 +201,69 @@ runs: NEW=$(node -e "const r=require('./reprocase-report/compare/compare.json');console.log(r.summary.newCount)" 2>/dev/null || echo 0) echo "new-findings=$NEW" >> "$GITHUB_OUTPUT" case "$CODE" in - 0) echo "ReproCase compare: no new findings." ;; - 1) echo "::error::ReproCase compare: $NEW new finding(s) vs baseline (coverage shrank = unconfirmed, see report)." ; exit 1 ;; - 2) echo "::error::ReproCase compare failed to run (bad baseline file?). Fix the baseline or remove compare-with." ; exit 1 ;; + 0) + echo "ReproCase compare: no new findings (gate passed)." + ;; + 1) + if [ "$FAIL_ON_NEW" = "true" ]; then + echo "::error::ReproCase compare: $NEW new finding(s) vs baseline. Gate: FAIL (fail-on-new=true)." + exit 1 + else + echo "::warning::ReproCase compare: $NEW new finding(s) vs baseline. Gate: PASS (fail-on-new=false — report only)." + fi + ;; + 2) + echo "::error::ReproCase compare failed to run (bad baseline file?). Fix the baseline or remove compare-with." + exit 1 + ;; esac + - name: Final CI decision + id: decision + if: ${{ always() }} + shell: bash + env: + SCAN_VERDICT: ${{ steps.scan.outputs.verdict }} + SCAN_EXIT: ${{ steps.scan.outputs.scan-exit-code }} + GATE_CONCLUSION: ${{ steps.gate.conclusion }} + INSTALL_CONCLUSION: ${{ steps.install.conclusion }} + FAIL_ON_NEW_RAW: ${{ inputs.fail-on-new }} + HAS_COMPARE: ${{ inputs.compare-with != '' }} + run: | + set -uo pipefail + # Propagate the gate's new-findings output. + NEW="${{ steps.gate.outputs.new-findings }}" + echo "new-findings=$NEW" >> "$GITHUB_OUTPUT" + + FAIL_ON_NEW=$(echo "$FAIL_ON_NEW_RAW" | tr -d '[:space:]' | tr '[:upper:]' '[:lower:]') + + # Priority 1: installation or scan environment error (exit 2). + if [ "$INSTALL_CONCLUSION" = "failure" ] || [ "$SCAN_VERDICT" = "error" ]; then + echo "exit-code=2" >> "$GITHUB_OUTPUT" + echo "Final verdict: error (environment/configuration problem)." + exit 2 + fi + + # Priority 2: baseline gate failed (new findings with fail-on-new=true, + # or compare infrastructure error). + if [ "$HAS_COMPARE" = "true" ] && [ "$GATE_CONCLUSION" = "failure" ]; then + echo "exit-code=1" >> "$GITHUB_OUTPUT" + echo "Final verdict: gate failed (new findings vs baseline)." + exit 1 + fi + + # Priority 3: scan findings at or above the fail-on threshold. + if [ "$SCAN_VERDICT" = "findings" ]; then + echo "exit-code=1" >> "$GITHUB_OUTPUT" + echo "Final verdict: findings exist (scan exit $SCAN_EXIT)." + exit 1 + fi + + # Priority 4: all checks passed. + echo "exit-code=0" >> "$GITHUB_OUTPUT" + echo "Final verdict: clean." + exit 0 + - name: Publish privacy-stripped report if: ${{ always() }} shell: bash @@ -176,11 +273,13 @@ runs: # evidence, never uploaded. Screenshots are of YOUR site: keep them, # they belong to you. result.json / report.html / issues.md are # redacted at source (token/session query params). - rm -rf reprocase-report/artifacts/traces - ls reprocase-report + if [ -d reprocase-report ]; then + rm -rf reprocase-report/artifacts/traces + ls reprocase-report + fi - uses: actions/upload-artifact@v4 - if: ${{ always() }} + if: ${{ always() && hashFiles('reprocase-report/**') != '' }} with: name: ${{ inputs.report-name }} path: reprocase-report/ diff --git a/tests/integration/gate-semantics.test.ts b/tests/integration/gate-semantics.test.ts new file mode 100644 index 0000000..c983926 --- /dev/null +++ b/tests/integration/gate-semantics.test.ts @@ -0,0 +1,132 @@ +import { afterAll, describe, expect, it } from 'vitest'; +import { execSync } from 'node:child_process'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { PROJECT_ROOT } from '../helpers/demo-server.js'; + +/** + * Integration tests for the v0.5.1 fail-on-new and compare-exit semantics + * that the composite action's gate step relies on. Each test runs the real + * `reprocase compare` CLI and asserts the exit code, mirroring the action. + */ + +const CLI = path.join(PROJECT_ROOT, 'dist', 'cli.js'); + +function makeResult(message: string, url: string, status: string) { + return { + tool: { name: 'ReproCase', version: '0.5.1-test' }, + startedAt: '2026-11-06T00:00:00.000Z', + finishedAt: '2026-11-06T00:00:10.000Z', + target: { entry: 'http://127.0.0.1:4640/', allowedDomains: ['127.0.0.1'] }, + config: {}, + stats: { pagesVisited: 1, actionsPerformed: 1, requestsObserved: 10, linksChecked: 2, durationMs: 5000, findingsRaw: 1, findingsAfterDedupe: 1 }, + findings: [{ + id: 'fp-' + Math.abs(message.length * 31 + url.length), + fingerprint: message + '|' + url, + type: 'js-exception', + severity: 'high', + status, + confidence: 1, + title: `Uncaught JS exception: ${message}`, + url, + pages: [url], + evidence: { message }, + steps: [], + repro: { attempts: 1, successes: 1, observedCount: 1 }, + screenshots: [], + traces: [], + plan: { kind: 'js-error', url, message } + }], + pages: [{ url, depth: 0, status: 200, actions: 1, jsErrors: 1, resourceErrors: 0, overflow: false, durationMs: 1000 }], + exploration: [], + notChecked: [], + clusters: [], + summary: { + verified: status === 'verified' ? 1 : 0, + suspected: status === 'verified' ? 0 : 1, + bySeverity: { critical: 0, high: 1, medium: 0, low: 0 }, + byType: { 'js-exception': 1 } + } + }; +} + +function runCompare(baselineFile: string, currentFile: string, extraArgs: string[]): { code: number; stdout: string } { + const out = fs.mkdtempSync(path.join(os.tmpdir(), 'reprocase-gate-')); + try { + execSync( + `node dist/cli.js compare "${baselineFile}" "${currentFile}" --out "${out}" ${extraArgs.join(' ')}`, + { cwd: PROJECT_ROOT, timeout: 30000, stdio: 'pipe' } + ); + return { code: 0, stdout: '' }; + } catch (e: unknown) { + const err = e as { status?: number }; + return { code: err.status ?? -1, stdout: '' }; + } finally { + fs.rmSync(out, { recursive: true, force: true }); + } +} + +function writeResult(dir: string, name: string, data: object): string { + fs.mkdirSync(dir, { recursive: true }); + const file = path.join(dir, name); + fs.writeFileSync(file, JSON.stringify(data), 'utf8'); + return file; +} + +describe('compare gate semantics (v0.5.1 reliability)', () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'reprocase-gate-test-')); + + afterAll(() => { + fs.rmSync(tmp, { recursive: true, force: true }); + }); + + it('baseline has verified finding, current has the same (no new) → exit 0 with --fail-on new', () => { + const base = writeResult(tmp, 'base.json', makeResult('boom', 'http://x/a.html', 'verified')); + const cur = writeResult(tmp, 'cur.json', makeResult('boom', 'http://x/a.html', 'verified')); + const r = runCompare(base, cur, ['--fail-on', 'new']); + expect(r.code).toBe(0); + }); + + it('baseline has 1 finding, current has a different one (new) → exit 1 with --fail-on new', () => { + const base = writeResult(tmp, 'base2.json', makeResult('old fault', 'http://x/a.html', 'verified')); + const cur = writeResult(tmp, 'cur2.json', makeResult('new fault', 'http://x/a.html', 'verified')); + const r = runCompare(base, cur, ['--fail-on', 'new']); + expect(r.code).toBe(1); + }); + + it('baseline has 1 finding, current has it + a new one → exit 1 with --fail-on new', () => { + const base = writeResult(tmp, 'base3.json', makeResult('old fault', 'http://x/a.html', 'verified')); + const cur = writeResult(tmp, 'cur3.json', makeResult('old fault', 'http://x/a.html', 'verified')); + const cur3 = writeResult(tmp, 'cur3b.json', { + ...makeResult('old fault', 'http://x/a.html', 'verified'), + findings: [ + ...(makeResult('old fault', 'http://x/a.html', 'verified').findings), + makeResult('new fault', 'http://x/a.html', 'verified').findings[0] + ] + }); + const r = runCompare(base, cur3, ['--fail-on', 'new']); + expect(r.code).toBe(1); + }); + + it('old finding fixed → exit 0 with --fail-on new (resolved is not new)', () => { + const base = writeResult(tmp, 'base4.json', makeResult('old fault', 'http://x/a.html', 'verified')); + const noFindings = { ...makeResult('nothing', 'http://x/a.html', 'verified'), findings: [], summary: { ...makeResult('nothing', 'http://x/a.html', 'verified').summary, verified: 0, suspected: 0, byType: {} } }; + const cur = writeResult(tmp, 'cur4.json', noFindings); + const r = runCompare(base, cur, ['--fail-on', 'new']); + expect(r.code).toBe(0); + }); + + it('no --fail-on flag → exit 0 regardless of new findings (report only)', () => { + const base = writeResult(tmp, 'base5.json', makeResult('old fault', 'http://x/a.html', 'verified')); + const cur = writeResult(tmp, 'cur5.json', makeResult('new fault', 'http://x/a.html', 'verified')); + const r = runCompare(base, cur, []); + expect(r.code).toBe(0); + }); + + it('baseline file missing → exit 2 (error, not success)', () => { + const cur = writeResult(tmp, 'cur6.json', makeResult('boom', 'http://x/a.html', 'verified')); + const r = runCompare(path.join(tmp, 'nonexistent.json'), cur, ['--fail-on', 'new']); + expect(r.code).toBe(2); + }); +}); diff --git a/tests/unit/action-yml.test.ts b/tests/unit/action-yml.test.ts new file mode 100644 index 0000000..d7aa55d --- /dev/null +++ b/tests/unit/action-yml.test.ts @@ -0,0 +1,93 @@ +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { PROJECT_ROOT } from '../helpers/demo-server.js'; + +const ACTION_YML = path.join(PROJECT_ROOT, '.github', 'actions', 'reprocase-scan', 'action.yml'); +const yml = fs.readFileSync(ACTION_YML, 'utf8'); + +describe('composite action structure (v0.5.1 reliability)', () => { + it('declares all documented inputs', () => { + for (const input of ['url', 'reprocase-ref', 'config', 'compare-with', 'fail-on', 'fail-on-new', 'max-pages', 'detection-window-ms', 'report-name', 'node-version']) { + expect(yml, `input "${input}" must be declared`).toContain(` ${input}:`); + } + }); + + it('declares all documented outputs', () => { + for (const output of ['verdict', 'exit-code', 'scan-exit-code', 'new-findings']) { + expect(yml, `output "${output}" must be declared`).toContain(` ${output}:`); + } + }); + + it('scan step does NOT exit 1 — decision step handles it (v0.5.1 fix)', () => { + // The scan step must complete successfully on exit 1 so the gate step + // can run. The final decision step produces the CI exit code. + // Exit 2 for environment errors is still an immediate exit. + expect(yml).toContain('exit 2'); + // The scan step must NOT contain a bare `exit "$CODE"` after case (which + // was the v0.5.0 bug: exit 1 propagated, skipping the gate). + expect(yml).not.toContain('exit "$CODE"'); + }); + + it('gate step reads FAIL_ON_NEW env var (v0.5.0 bug: declared but unused)', () => { + expect(yml).toContain('FAIL_ON_NEW'); + expect(yml).toContain('GATE_FLAG'); + expect(yml).toContain("GATE_FLAG=\"--fail-on new\""); + expect(yml).toContain('GATE_FLAG=""'); + }); + + it('gate step handles invalid fail-on-new values with an explicit error', () => { + expect(yml).toContain('Invalid fail-on-new value'); + }); + + it('gate step is skipped when scan has an error (exit 2)', () => { + const gateIf = yml.match(/- name: Compare with baseline[\s\S]*?if: (.*?)(?=\n)/); + expect(gateIf).toBeTruthy(); + expect(gateIf![1]).toContain("!= 'error'"); + }); + + it('decision step prioritises: error (2) > gate (1) > findings (1) > clean (0)', () => { + const decisionBlock = yml.slice( + yml.indexOf(' - name: Final CI decision'), + yml.indexOf(' - name: Publish privacy-stripped report') + ); + expect(decisionBlock).toContain('exit 2'); + expect(decisionBlock).toContain('exit 1'); + expect(decisionBlock).toContain('exit 0'); + const errorPos = decisionBlock.indexOf('exit 2'); + const gatePos = decisionBlock.indexOf('exit 1'); + const cleanPos = decisionBlock.indexOf('exit 0'); + expect(errorPos).toBeLessThan(gatePos); + expect(gatePos).toBeLessThan(cleanPos); + }); + + it('install step supports SHA checkout with explicit error for missing refs', () => { + expect(yml).toContain('git checkout'); + expect(yml).toContain('git cat-file'); + expect(yml).toContain('not found as a tag, branch or commit SHA'); + expect(yml).toContain('git rev-parse HEAD'); + }); + + it('install step uses npm ci (lockfile reproducibility, not npm install)', () => { + expect(yml).toContain('npm ci'); + // The action must NOT use bare npm install for dependencies (only + // `npm install -g .` for the final global install of the built package). + expect(yml).not.toMatch(/npm install --no-fund --no-audit(?!\s+-g)/); + }); + + it('artifact upload skips traces and only runs when the report exists', () => { + expect(yml).toContain('rm -rf reprocase-report/artifacts/traces'); + expect(yml).toContain("hashFiles('reprocase-report/**') != ''"); + }); + + it('gate step is skipped when scan has an error (exit 2)', () => { + const gateIf = yml.match(/- name: Compare with baseline[\s\S]*?if: (.*?)(?=\n)/); + expect(gateIf![1]).toContain("!= 'error'"); + }); + + it('final decision step references both scan verdict and gate conclusion', () => { + expect(yml).toContain('SCAN_VERDICT'); + expect(yml).toContain('GATE_CONCLUSION'); + expect(yml).toContain('INSTALL_CONCLUSION'); + }); +}); From 23254e97ee3ab33190f76e07c33629460c2fcc8e Mon Sep 17 00:00:00 2001 From: geyutu6755 Date: Fri, 6 Nov 2026 00:24:36 +0800 Subject: [PATCH 2/4] docs(v0.5.1): correct fail-on-new semantics, exit-code table, baseline flow combinations --- docs/github-actions.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/docs/github-actions.md b/docs/github-actions.md index 9417a1e..fb81030 100644 --- a/docs/github-actions.md +++ b/docs/github-actions.md @@ -56,7 +56,7 @@ Pin the action to a released tag (`@v0.5.0`) or a commit SHA for supply-chain sa | `config` | — | Path to a `reprocase.config.json` in your repo (overrides URL budgets). | | `compare-with` | — | Path to a baseline `result.json` — enables the new-findings gate. | | `fail-on` | `verified` | `verified` / `suspected` / `never` — which findings fail the step. | -| `fail-on-new` | `true` | With `compare-with`: fail when new findings exist vs the baseline. | +| `fail-on-new` | `true` | With `compare-with`: `true` fails on new findings vs baseline; `false` reports the diff without gating. Scan findings still gate via `fail-on`. | | `max-pages` | scanner default (25) | Crawl budget. | | `detection-window-ms` | scanner default (1500) | Async observation window (higher = catches later hydration faults). | | `report-name` | `reprocase-report` | Artifact name. | @@ -74,8 +74,8 @@ Pin the action to a released tag (`@v0.5.0`) or a commit SHA for supply-chain sa | Code | Meaning | CI effect | | --- | --- | --- | -| `0` | Scan completed, nothing at/above the threshold | step passes | -| `1` | Scan completed — findings at/above the threshold (a **verdict about your website**) | step fails, report artifact is still published | +| `0` | Clean scan + no new findings vs baseline | step passes | +| `1` | Findings at/above threshold (a **verdict about your site**) OR new findings vs baseline (a **quality gate**) | step fails, report artifact is still published | | `2` | ReproCase could not run properly (bad config, unreachable URL, missing browser) | step fails with a `::error::` annotation stating this is an **environment problem, not a website verdict** | An environment error never masquerades as "your site is fine" — the action fails loudly and @@ -103,6 +103,12 @@ Behaviour: cannot prove. - **Update the baseline deliberately**: after accepting a scan, copy its `result.json` into `.reprocase/baseline/` in a reviewed commit. +- **fail-on-new=false**: the compare report is generated and the new/resolved/persistent + counts are visible in the output and the report, but the action does not gate on new + findings. Scan findings still gate via `fail-on`. +- **fail-on never + compare-with**: the scan never gates on findings, but the baseline + gate can still fail on new findings. Useful for report-only runs that still want + regression detection. ## Pull requests From b91df55e79a46b162cf1f87d48df5d4e31917626 Mon Sep 17 00:00:00 2001 From: geyutu6755 Date: Fri, 6 Nov 2026 00:53:35 +0800 Subject: [PATCH 3/4] =?UTF-8?q?release:=20v0.5.1=20=E2=80=94=20version=20b?= =?UTF-8?q?ump,=20changelog,=20README=20statuses?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 1 + README.md | 2 +- README.zh-CN.md | 2 +- package-lock.json | 4 ++-- package.json | 2 +- 5 files changed, 6 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 66cff14..9c6fe9e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -476,6 +476,7 @@ First verified release. - No accessibility, performance, or visual-regression coverage beyond horizontal overflow. - Single-browser (Chromium) at this stage; Firefox/WebKit untested. +[0.5.1]: https://github.com/geyutu6755/ReproCase/releases/tag/v0.5.1 [0.5.0]: https://github.com/geyutu6755/ReproCase/releases/tag/v0.5.0 [0.4.0]: https://github.com/geyutu6755/ReproCase/releases/tag/v0.4.0 [0.3.3]: https://github.com/geyutu6755/ReproCase/releases/tag/v0.3.3 diff --git a/README.md b/README.md index 220fbb0..1adb9d6 100644 --- a/README.md +++ b/README.md @@ -152,7 +152,7 @@ fault through the real Playwright runner. ## Status -v0.5.0 — developer-workflow integration release (composite GitHub Action, baseline new-fault gating, honest CI exit semantics, cluster-aware issue markdown). See the [changelog](CHANGELOG.md), the [Actions guide](docs/github-actions.md) and the [compatibility matrix](docs/compatibility.md). +v0.5.1 — GitHub Actions reliability release (fail-on-new semantics, exit-code ordering, SHA support, npm ci). See the [changelog](CHANGELOG.md). `npm i -g github:…#tag` now works and the quick start drops the manual build), a reconciled Node baseline (**22+**, since Node 18/20 are End-of-Life; CI-tested on 22 & 24), the deprecated Vitest `poolOptions` config fixed, and refreshed real screenshots. Detection itself is unchanged diff --git a/README.zh-CN.md b/README.zh-CN.md index 30c49d5..cc1941f 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -139,7 +139,7 @@ npm run test:all # 单元 + 集成（真实浏览器）+ CLI 端到端 ## 状态 -v0.5.0 —— 开发者工作流集成版本（复合 GitHub Action、基线新增故障门禁、诚实的 CI 退出语义、聚类感知 Issue 文档）。详见 [变更日志](CHANGELOG.md) 与 [Actions 指南](docs/github-actions.md)。 +v0.5.1 —— GitHub Actions 可靠性修复版本（fail-on-new 语义、退出码顺序、SHA 支持、npm ci）。详见 [变更日志](CHANGELOG.md)。 [变更日志](CHANGELOG.md)、[测试基线](docs/test-baselines.md) 与[兼容性矩阵](docs/compatibility.md)。 ## 许可证 diff --git a/package-lock.json b/package-lock.json index 417e15a..4721ca1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "reprocase", - "version": "0.5.0", + "version": "0.5.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "reprocase", - "version": "0.5.0", + "version": "0.5.1", "license": "MIT", "dependencies": { "@playwright/test": "^1.49.0", diff --git a/package.json b/package.json index 24b050c..5a3dbf6 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "reprocase", - "version": "0.5.0", + "version": "0.5.1", "description": "Automated website fault detection & reproduction. Scans a site in a real browser, verifies each fault independently, and emits reproducible Playwright regression tests plus human- and CI-friendly reports.", "type": "module", "license": "MIT", From c728cbf711102e29f33976c46041cf14daf3dde3 Mon Sep 17 00:00:00 2001 From: geyutu6755 Date: Fri, 6 Nov 2026 00:55:02 +0800 Subject: [PATCH 4/4] =?UTF-8?q?release:=20v0.5.1=20=E2=80=94=20version=20b?= =?UTF-8?q?ump,=20changelog,=20README=20statuses?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c6fe9e..c38dacf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,32 @@ All notable changes to ReproCase are documented here. Format based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); versioning follows [Semantic Versioning](https://semver.org/). +## [0.5.1] — 2026-11-06 + +GitHub Actions composite-action reliability release. Three real defects found by the +external-repo CI runs and code audit. + +### Fixed + +- fail-on-new input was declared but its env var never read (P1): the gate always used + the new-findings gate regardless of the user setting. Now: true gates on new findings; + false reports the diff without gating; invalid values produce an explicit error. +- Scan step exited 1, causing GitHub Actions to skip the gate step (P1): the scan step + now completes on exit 0 or 1 (recording the verdict) and only exits 2 for environment + errors. A new Final CI decision step evaluates scan verdict + gate conclusion in priority + order (error > gate > findings > clean) and produces the final exit code. +- reprocase-ref SHA support was claimed but not implemented (P2): git clone --branch + only works for tags/branches. Now: --branch first, then fallback to full clone + git + checkout for SHAs, with an explicit error for unresolvable refs. The checked-out commit + is verified and reported. +- npm install to npm ci for lockfile-reproducible installs; hashFiles guard prevents + empty artifact upload. + +### Verification + +160/160 tests; npm audit 0; external repo validation (clean green, incident red, fix green, +honest-error green); four-OS CI matrix. + ## [0.5.0] — 2026-11-05 Developer-workflow integration release: run ReproCase in your own GitHub Actions pipelines,