diff --git a/.gitignore b/.gitignore index c6d04a0..4f38277 100644 --- a/.gitignore +++ b/.gitignore @@ -109,4 +109,7 @@ venv.bak/ /node_modules/.yarn-integrity /yarn.lock -.DS_Store \ No newline at end of file +.DS_Store + +# PR regression check tool cache/reports +tools/pr_regression_check/.cache/ \ No newline at end of file diff --git a/CLAUDE.md b/CLAUDE.md index 7753f4e..901d777 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -109,4 +109,5 @@ When applying a template, existing zone records are marked `_delete=1` based on - if a newly written or modified test fails unexpectedly, do NOT silently correct it — stop and ask the user whether the failure indicates an error in the test itself (wrong expectation) or a real implementation bug that should be fixed. Only correct the test after the user confirms it is wrong. - when asked to add a test of a certain pattern add a test for each RR type, incl. custom RR type, unless explicitly asked to only consider one RR type. The full set of RR types is: A, AAAA, CNAME, APEXCNAME, TXT, MX, NS, SRV, and at least one custom type (e.g. CAA). Note that SRV uses `name` (not `host`) in template_records, and some patterns (e.g. empty name) may be invalid for SRV due to protocol constraints — use `InvalidData` as the expected outcome in those cases rather than skipping the type. - develop code only when asked -- when adding support for a new RR type, update **both** `process_records` (the processing/conflict logic in `DomainConnectImpl.py`) **and** `get_records_variables` (the variable extraction function in the same file). Omitting either causes runtime errors or silent variable extraction failures for that type. \ No newline at end of file +- when adding support for a new RR type, update **both** `process_records` (the processing/conflict logic in `DomainConnectImpl.py`) **and** `get_records_variables` (the variable extraction function in the same file). Omitting either causes runtime errors or silent variable extraction failures for that type. +- after changing `apply_template`, `process_records`, or `resolve_variables`, run the PR regression-check tool (`tools/pr_regression_check/`, see its README) before considering the change done. Treat a new failure like an unexpectedly failing test: stop and ask the user. \ No newline at end of file diff --git a/tools/pr_regression_check/README.md b/tools/pr_regression_check/README.md new file mode 100644 index 0000000..caf7489 --- /dev/null +++ b/tools/pr_regression_check/README.md @@ -0,0 +1,188 @@ +# PR regression check + +Replays the "Online Editor test results" links recorded in merged +[Domain-Connect/Templates](https://github.com/Domain-Connect/Templates) pull +requests through this repo's `apply_template()`, and flags any pull request +whose recorded result no longer matches what the library produces today. + +## When to run this + +Run it after any change to `apply_template`, `process_records`, or +`resolve_variables` in `DomainConnectImpl.py`, before considering the change +done: + +```bash +.venv/bin/python tools/pr_regression_check/check_pr_regressions.py +``` + +A new failure means the change altered externally-observed behavior for a +real, previously-tested template. Treat it like an unexpectedly failing +unit test — stop and ask the user whether it's an intended behavior change +(record it in `overrides.json`, see below) or a regression to fix. + +## Usage + +```bash +# Check everything cached so far (first run fetches from GitHub) +.venv/bin/python tools/pr_regression_check/check_pr_regressions.py + +# Fetch up to N additional not-yet-cached merged PRs, then check everything cached +.venv/bin/python tools/pr_regression_check/check_pr_regressions.py --limit 500 + +# Check one specific PR (fetches it if not already cached) +.venv/bin/python tools/pr_regression_check/check_pr_regressions.py --pr 1842 + +# Ignore the cache and re-fetch everything from GitHub +.venv/bin/python tools/pr_regression_check/check_pr_regressions.py --refresh + +# Include PRs/results that matched in the JSON report too (default: errors only) +.venv/bin/python tools/pr_regression_check/check_pr_regressions.py --full-report + +# Use a different overrides file (default: overrides.json next to this script) +.venv/bin/python tools/pr_regression_check/check_pr_regressions.py --overrides /path/to/overrides.json +``` + +Requires the `gh` CLI, authenticated with access to +`Domain-Connect/Templates` (public repo, so any authenticated token works). + +PRs are fetched newest-first and cached in `.cache/prs.jsonl` (gitignored), +one JSON object per line, so repeated runs only fetch PRs not already seen. +A JSON report is written to `.cache/report.json`. By default it only +includes PRs that mismatched or errored; pass `--full-report` to include +every checked PR, matches included. Each result includes the full +`apply_template()` inputs (`template`, `zone_records`, `params`, +`group_ids`, `ignore_signature`, `multi_aware`) so a failure can be +reproduced directly without going back to the PR body. + +## How it works + +Each editor-test link encodes a gzip+base64 token (same format the +Templates repo's own `check_pr_description.py` decodes) containing: + +- `template` — the exact template dict that was tested +- `zone_records`, `domain`, `host`, `group_ids`, `params` — the + `apply_template()` inputs used +- `ignore_signature`, `multi_aware` — `apply_template()` flags +- `dc_apply_result` — the `(new_records, deleted_records, final_records)` + tuple `apply_template()` returned at test time (the tool converts this to + a named `{new_records, deleted_records, final_records}` dict internally + and whenever it writes one out, so nothing depends on tuple position) + +The tool decodes the token (without attempting signature verification — +the HMAC key is a private secret of the Templates repo's CI, and we trust +the GitHub PR body as the source of truth here), re-runs `apply_template()` +with the token's own embedded template and inputs, and diffs the result +against the stored `dc_apply_result`. Using the token's embedded template +(rather than the template file as it stands in the Templates repo today) +isolates "did this library's behavior change" from "did the template change +after the test was recorded". + +The comparison ignores each record's entire `_dc` key: `_dc.id` is a fresh +random UUID on every `multi_aware` run and will never match a stored result, +and the rest of `_dc`'s shape may legitimately evolve independently of +`apply_template()` correctness. + +### REDIR301/REDIR302 caveat + +Templates using `REDIR301`/`REDIR302` require the caller to supply +`redirect_records` — placeholder records standing in for the DNS +provider's own redirect infrastructure. The editor token doesn't carry +these (they aren't template test state), so the tool supplies a fixed +`127.0.0.1` / `::1` pair, confirmed against a real PR's recorded result to +match what the online editor itself uses as its default fixture. + +## PRs without test links + +A merged PR with no decodable "Online Editor test results" link is skipped +(not flagged as a failure) but is still cached, so it isn't re-fetched from +GitHub on the next run. + +## Overrides + +Some flagged discrepancies are known and not worth re-litigating on every +run — e.g. a bug in the online editor's own test tool rather than a real +library regression, or an expected result you know is stale for reasons +unrelated to `apply_template()`. These go in `overrides.json`, sitting next +to the script and committed to git (pass `--overrides` to use a different +path). + +The file is keyed ` -> -> -> -> [entries]`. +PR number stays the top-level key because different PRs can carry different +versions of "the same" template (by providerId/serviceId) with legitimately +different expected outcomes; providerId/serviceId/host is how a human +identifies a link within a PR. A host can hold several entries when +multiple links in the same PR share a host but differ in params (e.g. a +different `domain`), so each entry also carries the `params` it applies to +— a link only matches an entry when PR number, providerId, serviceId, host, +*and* params all match. + +```json +{ + "840": { + "spinlab.studio": { + "casino": { + "sub": [ + { + "params": {"domain": "example.com", "host": "sub"}, + "action": "ignore", + "reason": "Why this is being ignored, ideally with a link to context." + } + ] + } + } + }, + "1234": { + "some.provider": { + "some-service": { + "@": [ + { + "params": {"domain": "example.com", "host": ""}, + "action": "override_expected", + "reason": "The recorded result predates fix #NN; this is the corrected expectation.", + "dc_apply_result": { + "new_records": [], + "deleted_records": [], + "final_records": [] + } + } + ] + } + } + } +} +``` + +- `"ignore"` skips the link entirely (status `ignored` in the report); the + payload is not even replayed. +- `"override_expected"` replaces the token's stored `dc_apply_result` with + the one given here before comparing against the library's actual output + — use this when you know what the correct expected result should be, + rather than just wanting to silence the check. + +Unknown `action` values, or an `override_expected` entry missing +`dc_apply_result`, fail loudly at startup rather than being silently +ignored. + +### Adding overrides from the CLI + +Rather than hand-editing `overrides.json`, you can add entries directly for +a specific PR's currently-failing link(s): + +```bash +# Add an "ignore" entry (with reason) for every mismatching/erroring link in PR #840 +.venv/bin/python tools/pr_regression_check/check_pr_regressions.py --pr 840 \ + --add-to-override-ignore "Editor tool bug, not a library regression: see PR discussion" + +# Add an "override_expected" entry, accepting the library's current output +# as the new expected result, for every currently-mismatching link in PR #1234 +.venv/bin/python tools/pr_regression_check/check_pr_regressions.py --pr 1234 \ + --add-to-override-result "Recorded result predates fix #NN; this is the corrected expectation" +``` + +Both flags require `--pr` and only ever touch links belonging to that PR. +Only links currently reporting `mismatch` (or, for `--add-to-override-ignore`, +also `error`) get an entry; links that already match are left untouched. +`--add-to-override-result` skips `error` links (there is no actual result to +store) with a warning. Existing entries elsewhere in the file are preserved; +re-running for the same providerId/serviceId/host/params overwrites just +that entry. diff --git a/tools/pr_regression_check/check_pr_regressions.py b/tools/pr_regression_check/check_pr_regressions.py new file mode 100644 index 0000000..3803c89 --- /dev/null +++ b/tools/pr_regression_check/check_pr_regressions.py @@ -0,0 +1,613 @@ +#!/usr/bin/env python3 +""" +Regression check: replay Domain-Connect/Templates PR editor-test tokens +through this library's apply_template() and flag discrepancies against the +result recorded at PR test time. + +For each merged PR in Domain-Connect/Templates: + 1. Extract "Online Editor test results" editor links from the PR body, + using the same token format as check_pr_description.py in that repo. + 2. Decode each token (signature verification is not attempted here - we + trust the GitHub PR body as the source of truth, and the HMAC key is + a private secret of the Templates repo's CI). + 3. Re-run apply_template() using the token's own embedded template dict + (not the repo's current file) plus its zone_records/domain/host/params/ + group_ids/ignore_signature/multi_aware, so results reflect purely + "did this library's behavior change" rather than template edits. + 4. Compare (new_records, deleted_records, final_records) against the + token's stored dc_apply_result, ignoring the entire "_dc" key (its + "id" sub-field is a fresh random UUID per multi_aware run and other + _dc content may evolve independently of apply_template correctness). + +PRs without any decodable editor-test token are skipped (not flagged) but +still cached, so a re-run doesn't re-fetch them from GitHub. + +Usage: + .venv/bin/python tools/pr_regression_check/check_pr_regressions.py [--limit N] [--refresh] [--pr NUMBER] +""" +import argparse +import base64 +import gzip +import json +import os +import re +import subprocess +import sys +from urllib.parse import unquote + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..')) + +from domainconnectzone.DomainConnectImpl import DomainConnect # noqa: E402 + +REPO = "Domain-Connect/Templates" +CACHE_DIR = os.path.join(os.path.dirname(__file__), '.cache') +PR_CACHE_FILE = os.path.join(CACHE_DIR, 'prs.jsonl') +DEFAULT_OVERRIDES_FILE = os.path.join(os.path.dirname(__file__), 'overrides.json') + +EDITOR_URL_PATTERN = re.compile( + r"https://domainconnect\.paulonet\.eu/dc/free/templateedit\?token=([A-Za-z0-9+/=%]+)" +) + +VOLATILE_KEYS = {"_dc"} + +# REDIR301/REDIR302 records require the caller to supply "redirect_records" - +# placeholder DNS records representing the provider's own redirect infrastructure. +# The editor token does not carry these (they're not part of the template test +# state), but the online editor at domainconnect.paulonet.eu always uses this +# fixed loopback pair as its redirect-infrastructure fixture, confirmed by +# reproducing PR #1842's stored dc_apply_result byte-for-byte with it. +DEFAULT_REDIRECT_RECORDS = [ + {"type": "A", "pointsTo": "127.0.0.1", "ttl": 600}, + {"type": "AAAA", "pointsTo": "::1", "ttl": 600}, +] + + +def load_cache(): + prs = {} + if os.path.exists(PR_CACHE_FILE): + with open(PR_CACHE_FILE) as f: + for line in f: + line = line.strip() + if not line: + continue + pr = json.loads(line) + prs[pr['number']] = pr + return prs + + +def append_cache(pr): + os.makedirs(CACHE_DIR, exist_ok=True) + with open(PR_CACHE_FILE, 'a') as f: + f.write(json.dumps(pr) + "\n") + + +def rewrite_cache(prs): + os.makedirs(CACHE_DIR, exist_ok=True) + with open(PR_CACHE_FILE, 'w') as f: + for pr in prs.values(): + f.write(json.dumps(pr) + "\n") + + +GRAPHQL_QUERY = """ +query($cursor: String) { + repository(owner: "Domain-Connect", name: "Templates") { + pullRequests(states: MERGED, first: 50, after: $cursor, orderBy: {field: CREATED_AT, direction: DESC}) { + pageInfo { hasNextPage endCursor } + nodes { + number + title + body + mergedAt + files(first: 100) { + pageInfo { hasNextPage } + nodes { path } + } + } + } + } +} +""" + + +def fetch_merged_prs(known_numbers, limit=None, stop_at_pr=None): + """Yield PR dicts from GitHub via gh api graphql, skipping ones already cached. + + PRs are fetched newest-first. If stop_at_pr is given, fetching stops right + after that PR is yielded (or once the given PR number is passed, in case + it does not exist), regardless of limit. + """ + cursor = None + fetched = 0 + while True: + args = [ + "gh", "api", "graphql", + "-f", f"query={GRAPHQL_QUERY}", + ] + if cursor: + args += ["-f", f"cursor={cursor}"] + result = subprocess.run(args, capture_output=True, text=True, check=True) + data = json.loads(result.stdout) + pr_conn = data["data"]["repository"]["pullRequests"] + for node in pr_conn["nodes"]: + if stop_at_pr is not None and node["number"] < stop_at_pr: + return + if node["number"] in known_numbers: + continue + if node["files"]["pageInfo"]["hasNextPage"]: + print(f" WARNING: PR #{node['number']} has >100 changed files; " + f"file list truncated", file=sys.stderr) + pr = { + "number": node["number"], + "title": node["title"], + "body": node["body"] or "", + "merged_at": node["mergedAt"], + "files": [n["path"] for n in node["files"]["nodes"]], + } + yield pr + fetched += 1 + if stop_at_pr is not None and node["number"] == stop_at_pr: + return + if limit and fetched >= limit: + return + if not pr_conn["pageInfo"]["hasNextPage"]: + return + cursor = pr_conn["pageInfo"]["endCursor"] + + +def params_key(params): + """Canonical, hashable form of a (normalized) params dict for matching/equality.""" + return json.dumps(params, sort_keys=True) + + +def result_tuple_to_dict(result): + """Convert an apply_template() (new, deleted, final) tuple to a named dict.""" + new_records, deleted_records, final_records = result + return { + "new_records": new_records, + "deleted_records": deleted_records, + "final_records": final_records, + } + + +def result_dict_to_tuple(result): + """Convert a named dc_apply_result dict back to an (new, deleted, final) tuple.""" + return ( + result.get("new_records", []), + result.get("deleted_records", []), + result.get("final_records", []), + ) + + +def load_overrides(path): + """ + Load the overrides file. Format: + + { + "": { + "": { + "": { + "": [ + { + "params": {...}, + "action": "ignore", + "reason": "..." + }, + { + "params": {...}, + "action": "override_expected", + "reason": "...", + "dc_apply_result": { + "new_records": [...], "deleted_records": [...], "final_records": [...] + } + } + ] + } + } + } + } + + PR number stays the top-level key because different PRs can carry + different versions of "the same" template (by providerId/serviceId) + with legitimately different expected outcomes. Underneath, entries are + nested by providerId/serviceId/host since that's how a human identifies + a link; a host can hold several entries when multiple links in the same + PR share a host but differ in params, so params is part of the match key. + + Returns a dict keyed by (pr_number, providerId, serviceId, host) mapping + to a list of (params_dict, entry) pairs. Raises ValueError on malformed + entries. + """ + if not os.path.exists(path): + return {} + with open(path) as f: + raw = json.load(f) + + overrides = {} + for pr_number_str, providers in raw.items(): + pr_number = int(pr_number_str) + for provider_id, services in providers.items(): + for service_id, hosts in services.items(): + for host, entries in hosts.items(): + for entry in entries: + ctx = (f"PR #{pr_number} {provider_id}.{service_id} " + f"host={host!r} params={entry.get('params')}") + action = entry.get("action") + if action not in ("ignore", "override_expected"): + raise ValueError( + f"overrides file: {ctx} has unknown action {action!r} " + f"(expected 'ignore' or 'override_expected')" + ) + if action == "override_expected" and "dc_apply_result" not in entry: + raise ValueError( + f"overrides file: {ctx} has action 'override_expected' " + f"but no 'dc_apply_result'" + ) + key = (pr_number, provider_id, service_id, host) + overrides.setdefault(key, []).append( + (params_key(entry.get("params", {})), entry) + ) + return overrides + + +def find_override(overrides, pr_number, provider_id, service_id, host, params): + entries = overrides.get((pr_number, provider_id, service_id, host)) + if not entries: + return None + pkey = params_key(params) + for entry_params_key, entry in entries: + if entry_params_key == pkey: + return entry + return None + + +def add_override_entries(path, pr_number, action, reason, results): + """ + Append override entries for every result in `results` whose status is + 'mismatch' or 'error', to the overrides file at `path`. Existing entries + for the same PR/providerId/serviceId/host/params are overwritten; + everything else in the file is preserved. Returns a list of + "providerId.serviceId host=... params=..." labels for what was written. + """ + raw = {} + if os.path.exists(path): + with open(path) as f: + raw = json.load(f) + + pr_key = str(pr_number) + providers = raw.setdefault(pr_key, {}) + + written = [] + for res in results: + if res["status"] not in ("mismatch", "error"): + continue + + template = res["template"] + provider_id = template.get("providerId", "?") + service_id = template.get("serviceId", "?") + host = res["host"] or "" + params = normalize_params(res["params"]) + label = f"{provider_id}.{service_id} host={host!r} params={params}" + + entry = {"params": params, "action": action, "reason": reason} + if action == "override_expected": + if res["status"] != "mismatch" or "actual_result" not in res: + print(f" WARNING: skipping {label}: no actual result available to " + f"store (status={res['status']})", file=sys.stderr) + continue + entry["dc_apply_result"] = result_tuple_to_dict( + [strip_volatile(recs) for recs in res["actual_result"]] + ) + + hosts = providers.setdefault(provider_id, {}).setdefault(service_id, {}).setdefault(host, []) + pkey = params_key(params) + hosts[:] = [e for e in hosts if params_key(e.get("params", {})) != pkey] + hosts.append(entry) + written.append(label) + + with open(path, 'w') as f: + json.dump(raw, f, indent=2) + f.write("\n") + + return written + + +def decode_token(token): + """Decode an editor-link token without verifying its signature.""" + compressed = base64.b64decode(unquote(token)) + raw = gzip.decompress(compressed) + payload = json.loads(raw.decode('utf-8')) + payload.pop("_signature", None) + return payload + + +def get_editor_payloads(body): + payloads = [] + for token in EDITOR_URL_PATTERN.findall(body): + try: + payloads.append(decode_token(token)) + except Exception as e: + print(f" WARNING: could not decode token: {e}", file=sys.stderr) + return payloads + + +def strip_volatile(records): + """Deep-copy records dropping any '_dc' key, for comparison purposes.""" + cleaned = [] + for rec in records: + cleaned.append({k: v for k, v in rec.items() if k not in VOLATILE_KEYS}) + return cleaned + + +def sort_key(rec): + return json.dumps(rec, sort_keys=True) + + +def records_equal(expected, actual): + e = sorted((strip_volatile(expected)), key=sort_key) + a = sorted((strip_volatile(actual)), key=sort_key) + return e == a + + +def normalize_params(params): + """ + The online editor stores params as {key: [value, ...]} (query-string-style, + even for single-valued fields). apply_template()/resolve_variables() expect + a plain {key: value} mapping, so collapse single-element lists. + """ + normalized = {} + for key, value in (params or {}).items(): + if isinstance(value, list): + normalized[key] = value[0] if value else "" + else: + normalized[key] = value + return normalized + + +def replay_payload(payload): + """ + Re-run apply_template using the template/inputs embedded in the payload. + Returns (actual_result_tuple, error) - error is an exception if raised. + """ + template = payload.get("template") + if not template: + return None, ValueError("payload has no 'template'") + + has_redir = any(r.get("type") in ("REDIR301", "REDIR302") + for r in template.get("records", [])) + dc = DomainConnect( + template=template, + redir_template_records=DEFAULT_REDIRECT_RECORDS if has_redir else None, + ) + try: + actual = dc.apply_template( + zone_records=json.loads(json.dumps(payload.get("zone_records", []))), + domain=payload.get("domain", ""), + host=payload.get("host", ""), + params=normalize_params(payload.get("params", {})), + group_ids=payload.get("group_ids") or None, + ignore_signature=payload.get("ignore_signature", False), + multi_aware=payload.get("multi_aware", False), + ) + return actual, None + except Exception as e: + return None, e + + +def compare_payload(payload, override=None): + """ + Returns a dict describing the comparison outcome for one editor-test payload: + {"status": "match"|"mismatch"|"error"|"ignored", "detail": ...} + + override, if given, is the entry from the overrides file matching this + link's PR/providerId/serviceId/host/params. An "ignore" action + short-circuits to status "ignored" without replaying the payload at all. + An "override_expected" action substitutes its own dc_apply_result for the + payload's before comparing. + """ + if override is not None and override.get("action") == "ignore": + return {"status": "ignored", "detail": override.get("reason")} + + expected = payload.get("dc_apply_result") + if override is not None and override.get("action") == "override_expected": + expected = result_dict_to_tuple(override["dc_apply_result"]) + if not expected or len(expected) != 3: + return {"status": "skipped", "detail": "no dc_apply_result in payload"} + + actual, error = replay_payload(payload) + if error is not None: + return {"status": "error", "detail": f"{type(error).__name__}: {error}"} + + expected_new, expected_deleted, expected_final = expected + actual_new, actual_deleted, actual_final = actual + + mismatches = [] + for label, exp, act in ( + ("new_records", expected_new, actual_new), + ("deleted_records", expected_deleted, actual_deleted), + ("final_records", expected_final, actual_final), + ): + if not records_equal(exp, act): + mismatches.append({ + "field": label, + "expected": strip_volatile(exp), + "actual": strip_volatile(act), + }) + + if mismatches: + return {"status": "mismatch", "detail": mismatches, "actual_result": list(actual)} + return {"status": "match", "detail": None} + + +def template_id(template): + return f"{template.get('providerId', '?')}.{template.get('serviceId', '?')}" + + +def check_pr(pr, overrides=None): + """Return a report dict for one cached PR, or None if it has no usable test links.""" + overrides = overrides or {} + payloads = get_editor_payloads(pr["body"]) + if not payloads: + return None + + results = [] + for payload in payloads: + template = payload.get("template", {}) + saved_at = payload.get("saved_at") + params = normalize_params(payload.get("params", {})) + override = find_override( + overrides, pr["number"], + template.get("providerId", "?"), template.get("serviceId", "?"), + payload.get("host", "") or "", params, + ) + outcome = compare_payload(payload, override=override) + results.append({ + "template_id": template_id(template) if template else None, + "domain": payload.get("domain"), + "host": payload.get("host"), + "saved_at": saved_at, + "template": template, + "zone_records": payload.get("zone_records", []), + "params": payload.get("params", {}), + "group_ids": payload.get("group_ids", []), + "ignore_signature": payload.get("ignore_signature", False), + "multi_aware": payload.get("multi_aware", False), + **outcome, + }) + + if not any(r["status"] in ("mismatch", "error") for r in results): + overall = "ok" if any(r["status"] == "match" for r in results) else "skipped" + else: + overall = "fail" + + return { + "number": pr["number"], + "title": pr["title"], + "merged_at": pr["merged_at"], + "overall": overall, + "results": results, + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--limit", type=int, default=None, + help="Max number of NEW PRs to fetch from GitHub this run") + parser.add_argument("--refresh", action="store_true", + help="Re-fetch all PRs from GitHub, ignoring the cache") + parser.add_argument("--pr", type=int, default=None, + help="Only check a single PR number (must be cached or fetchable)") + parser.add_argument("--report", default=os.path.join(CACHE_DIR, "report.json"), + help="Path to write the JSON report") + parser.add_argument("--full-report", action="store_true", + help="Include PRs/results that matched in the JSON report " + "(default: only PRs with a mismatch or error are written)") + parser.add_argument("--overrides", default=DEFAULT_OVERRIDES_FILE, + help="Path to the overrides file (default: overrides.json " + "next to this script)") + parser.add_argument("--add-to-override-ignore", metavar="REASON", default=None, + help="For every currently-mismatching/erroring link in --pr, " + "add an 'ignore' entry to the overrides file with this reason. " + "Requires --pr.") + parser.add_argument("--add-to-override-result", metavar="REASON", default=None, + help="For every currently-mismatching link in --pr, add an " + "'override_expected' entry to the overrides file with this " + "reason, storing the library's current actual output as the " + "new expected result. Requires --pr.") + args = parser.parse_args() + + if (args.add_to_override_ignore or args.add_to_override_result) and args.pr is None: + print("--add-to-override-ignore/--add-to-override-result require --pr", file=sys.stderr) + return 1 + if args.add_to_override_ignore and args.add_to_override_result: + print("--add-to-override-ignore and --add-to-override-result are mutually exclusive", + file=sys.stderr) + return 1 + + overrides = load_overrides(args.overrides) + + known = {} if args.refresh else load_cache() + if args.refresh: + rewrite_cache({}) + + if args.pr is None or args.pr not in known or args.refresh: + print("Fetching merged PRs from GitHub" + + (f" (limit {args.limit})" if args.limit else "") + "...") + fetched_new = [] + for pr in fetch_merged_prs(known.keys(), limit=args.limit, stop_at_pr=args.pr): + append_cache(pr) + known[pr["number"]] = pr + fetched_new.append(pr["number"]) + print(f" fetched {len(fetched_new)} new PR(s); cache now has {len(known)} PR(s)") + + if args.pr is not None: + if args.pr not in known: + print(f"PR #{args.pr} not found (not merged, or beyond fetch limit)", file=sys.stderr) + return 1 + prs_to_check = [known[args.pr]] + else: + prs_to_check = list(known.values()) + + reports = [] + for pr in prs_to_check: + report = check_pr(pr, overrides=overrides) + if report is not None: + reports.append(report) + + if args.add_to_override_ignore or args.add_to_override_result: + action = "ignore" if args.add_to_override_ignore else "override_expected" + reason = args.add_to_override_ignore or args.add_to_override_result + pr_report = next((r for r in reports if r["number"] == args.pr), None) + if pr_report is None or not any(res["status"] in ("mismatch", "error") + for res in pr_report["results"]): + print(f"\nNo mismatching/erroring links found for PR #{args.pr}; " + f"nothing added to overrides.") + else: + written = add_override_entries(args.overrides, args.pr, action, reason, + pr_report["results"]) + print(f"\nAdded {len(written)} '{action}' override entry/entries for PR #{args.pr} " + f"to {args.overrides}: {', '.join(written)}") + overrides = load_overrides(args.overrides) + reports = [check_pr(pr, overrides=overrides) for pr in prs_to_check] + reports = [r for r in reports if r is not None] + + ok = [r for r in reports if r["overall"] == "ok"] + failed = [r for r in reports if r["overall"] == "fail"] + + print(f"\nChecked {len(reports)} PR(s) with test links " + f"({len(known) - len(reports)} skipped, no test links found)") + print(f" OK: {len(ok)}") + print(f" FAILED: {len(failed)}") + + for r in failed: + print(f"\nFAIL PR #{r['number']}: {r['title']} ({r['merged_at']})") + print(f" https://github.com/{REPO}/pull/{r['number']}") + for res in r["results"]: + if res["status"] not in ("mismatch", "error"): + continue + print(f" [{res['status']}] template={res['template_id']} " + f"domain={res['domain']} host={res['host']!r}") + if res["status"] == "error": + print(f" {res['detail']}") + else: + for m in res["detail"]: + print(f" field={m['field']}") + print(f" expected: {json.dumps(m['expected'])}") + print(f" actual: {json.dumps(m['actual'])}") + + report_reports = reports if args.full_report else failed + os.makedirs(os.path.dirname(args.report), exist_ok=True) + with open(args.report, 'w') as f: + json.dump({ + "checked": len(reports), + "ok": len(ok), + "failed": len(failed), + "reports": report_reports, + }, f, indent=2) + print(f"\nReport written to {args.report}" + + ("" if args.full_report else " (failures only; pass --full-report for everything)")) + + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/pr_regression_check/overrides.json b/tools/pr_regression_check/overrides.json new file mode 100644 index 0000000..95a7ad9 --- /dev/null +++ b/tools/pr_regression_check/overrides.json @@ -0,0 +1,61 @@ +{ + "840": { + "spinlab.studio": { + "casino": { + "sub": [ + { + "params": { + "domain": "example.com", + "host": "sub" + }, + "action": "override_expected", + "reason": "Wrong handling of host for APEXCNAME on the saved link", + "dc_apply_result": { + "new_records": [ + { + "type": "APEXCNAME", + "name": "@", + "data": "apex.spinlab.studio", + "ttl": 3600 + }, + { + "type": "CNAME", + "name": "www.sub", + "data": "apex.spinlab.studio", + "ttl": 3600 + }, + { + "type": "CNAME", + "name": "api.sub", + "data": "apex.spinlab.studio", + "ttl": 3600 + } + ], + "deleted_records": [], + "final_records": [ + { + "type": "APEXCNAME", + "name": "@", + "data": "apex.spinlab.studio", + "ttl": 3600 + }, + { + "type": "CNAME", + "name": "www.sub", + "data": "apex.spinlab.studio", + "ttl": 3600 + }, + { + "type": "CNAME", + "name": "api.sub", + "data": "apex.spinlab.studio", + "ttl": 3600 + } + ] + } + } + ] + } + } + } +}