From e63bfc0b5edcd3f0f0d59225c60c0699f57910ed Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 16 Sep 2026 14:49:25 +0000 Subject: [PATCH] Fail CI when Helm, Stack, or PipelineRun representations drift Record live M17.4 Results and M17.11 graph/GUI Newman evidence, then add a static-quality gate that compares Helm CRD copies to operator CRDs, converts every Stack/Team YAML, and checks Go/Python builders against pipeline params. Stop merge PipelineRuns from sending undeclared cache-repo. Co-authored-by: jmjava --- .github/workflows/static-quality.yml | 3 + docs/REGRESSION.md | 1 + .../tests/fixtures/pipelineruns/merge.json | 4 - .../tests/test_m17_representation_sync.py | 28 +++ .../tests/test_m17_static_quality.py | 1 + milestones/milestone-17.md | 58 ++--- operator/internal/pipeline/builder.go | 1 - .../pipeline/testdata/golden/merge.json | 4 - orchestrator/pipelinerun_builder.py | 1 - scripts/check-representation-sync.py | 235 ++++++++++++++++++ scripts/check-representation-sync.sh | 14 ++ scripts/run-regression.sh | 4 + 12 files changed, 311 insertions(+), 43 deletions(-) create mode 100644 libs/tekton-dag-common/tests/test_m17_representation_sync.py create mode 100755 scripts/check-representation-sync.py create mode 100755 scripts/check-representation-sync.sh diff --git a/.github/workflows/static-quality.yml b/.github/workflows/static-quality.yml index de956e2..9dcccf9 100644 --- a/.github/workflows/static-quality.yml +++ b/.github/workflows/static-quality.yml @@ -171,3 +171,6 @@ jobs: - name: Package and render chart run: bash scripts/check-helm-chart.sh + + - name: Check representation sync + run: bash scripts/check-representation-sync.sh diff --git a/docs/REGRESSION.md b/docs/REGRESSION.md index 2d2f9f1..e00b42d 100644 --- a/docs/REGRESSION.md +++ b/docs/REGRESSION.md @@ -48,6 +48,7 @@ traffic artifact as verification. | **E — Results + DB** | [run-full-test-and-verify-results.sh](../scripts/run-full-test-and-verify-results.sh) | **Auto** if `tekton-results-api` exists; **forced** with `--with-results-verify`; **off** with `--skip-results-verify`; strict weekly/dispatch automation uses `run-regression-agent-full.sh` | | **F — GUI Postman** | [management-gui-tests.json](../tests/postman/management-gui-tests.json) vs a live Flask backend | `--gui-newman` or `scripts/run-gui-newman.sh`; weekly/dispatch in `graph-gui-newman.yml` | | **F — Graph Postman** | [graph-tests.json](../tests/postman/graph-tests.json) vs orchestrator + Neo4j | `run-cluster-ci.sh --with-graph`; weekly/dispatch in `graph-gui-newman.yml` | +| **F — Representation sync** | Helm CRD copies, Stack/Team conversion, and PipelineRun params | `scripts/check-representation-sync.sh` in `static-quality.yml` | | **G — Full Kind E2E** | [run-all-setup-and-test.sh](../scripts/run-all-setup-and-test.sh) | `--kind-e2e` | | **H — Intercept product E2E** | [run-product-intercept-e2e.sh](../scripts/run-product-intercept-e2e.sh) via authenticated orchestrator API | Weekly/dispatch matrix in `intercept-e2e.yml`; requires repository secret `E2E_GIT_SSH_PRIVATE_KEY` with read access to application repos | diff --git a/libs/tekton-dag-common/tests/fixtures/pipelineruns/merge.json b/libs/tekton-dag-common/tests/fixtures/pipelineruns/merge.json index 9095661..b91617f 100644 --- a/libs/tekton-dag-common/tests/fixtures/pipelineruns/merge.json +++ b/libs/tekton-dag-common/tests/fixtures/pipelineruns/merge.json @@ -31,10 +31,6 @@ "name": "image-registry", "value": "localhost:5000" }, - { - "name": "cache-repo", - "value": "localhost:5000/kaniko-cache" - }, { "name": "max-retries", "value": "2" diff --git a/libs/tekton-dag-common/tests/test_m17_representation_sync.py b/libs/tekton-dag-common/tests/test_m17_representation_sync.py new file mode 100644 index 0000000..c22c9a8 --- /dev/null +++ b/libs/tekton-dag-common/tests/test_m17_representation_sync.py @@ -0,0 +1,28 @@ +"""Static acceptance checks for M17.12 representation synchronization.""" + +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[3] + + +def test_static_quality_runs_representation_sync(): + workflow = (ROOT / ".github/workflows/static-quality.yml").read_text() + gate = (ROOT / "scripts/check-helm-chart.sh").read_text() + + assert "bash scripts/check-representation-sync.sh" in workflow + assert "bash scripts/check-helm-chart.sh" in workflow + assert "helm package" in gate + assert gate.count("--include-crds") == 2 + + +def test_representation_sync_script_covers_required_surfaces(): + script = (ROOT / "scripts/check-representation-sync.py").read_text() + + assert "tektondag.io_stackruns.yaml" in script + assert "operator" in script and "helm" in script + assert "stack_yaml_to_cr" in script + assert "team_yaml_to_cr" in script + assert "BuildPR" in script + assert "build_pr_pipelinerun" in script + assert "stack-pr-test" in script + assert "stack-merge-release" in script diff --git a/libs/tekton-dag-common/tests/test_m17_static_quality.py b/libs/tekton-dag-common/tests/test_m17_static_quality.py index 8bf35e8..559e4ff 100644 --- a/libs/tekton-dag-common/tests/test_m17_static_quality.py +++ b/libs/tekton-dag-common/tests/test_m17_static_quality.py @@ -18,6 +18,7 @@ def test_static_quality_workflow_covers_required_domains(): assert "npm run lint" in workflow assert "npm run build" in workflow assert "bash scripts/check-helm-chart.sh" in workflow + assert "bash scripts/check-representation-sync.sh" in workflow def test_quality_tool_downloads_and_actions_are_immutable(): diff --git a/milestones/milestone-17.md b/milestones/milestone-17.md index d1422f9..518003f 100644 --- a/milestones/milestone-17.md +++ b/milestones/milestone-17.md @@ -1,6 +1,6 @@ # Milestone 17 — End-to-end quality and production-readiness closure -**Status:** Paused after M17.10; resume with M17.4 live acceptance, then M17.11 +**Status:** In progress after M17.11; next is M17.12 representation sync This milestone converts the September 2026 end-to-end audit into an executable backlog. Work is ordered by production risk, not by subsystem. A checkbox is @@ -22,35 +22,25 @@ For each slice: The milestone is complete only when every P0–P3 item is checked and the final regression criteria in `docs/AGENT-REGRESSION.md` are satisfied. -## Resume checkpoint — 2026-09-15 - -Work is intentionally paused to preserve the remaining implementation budget. - -- Completed with recorded acceptance: M17.1–M17.3 and M17.5–M17.10. -- Still open: M17.4 and M17.11–M17.21. -- M17.10 landed through - [PR #60](https://github.com/jmjava/tekton-dag/pull/60); its final revision - passed all 17 reported checks, including Kind Phase 2, authoritative Newman, - both intercept backends, static quality, regression, and supply-chain scans. -- The first resume action is still M17.4 live acceptance. Scheduled run - `35107095845` already executed on `main` and failed because the workflow - lacked JDK 21. Merge the Results toolchain fix to `main`, then confirm a - green scheduled or manually dispatched `results-regression` run before - marking M17.4 complete. -- After M17.4, continue in numeric order from M17.11. Do not skip directly to - maintainability work because M17.11–M17.14 establish the test evidence needed +## Resume checkpoint — 2026-09-16 + +- Completed with recorded acceptance: M17.1–M17.11. +- Still open: M17.12–M17.21. +- M17.4 live Results run `35108434664` on [PR #63](https://github.com/jmjava/tekton-dag/pull/63) exited 0 after the Java 21 toolchain fix. +- M17.11 live graph + GUI Newman run `35109309784` on [PR #65](https://github.com/jmjava/tekton-dag/pull/65) reported zero failed assertions. +- Continue in numeric order from M17.12. Do not skip directly to + maintainability work because M17.12–M17.14 establish the test evidence needed to refactor safely. -- Before resuming, fetch the latest `cursor/close-e2e-audit-gaps-fc5f` and +- Before starting a slice, fetch the latest `cursor/close-e2e-audit-gaps-fc5f` and create a fresh `cursor/-fc5f` branch. Do not reuse merged slice branches. | Resume order | Work | Terminal condition | |---|---|---| -| 1 | M17.4 Results live acceptance | Scheduled/manual strict workflow green and evidence recorded | -| 2 | M17.11–M17.14 test depth | Optional suites, representation sync, runner branches, and compatibility matrix enforced | -| 3 | M17.15–M17.19 maintainability | Duplication, ownership, legacy surface, errors, and config contracts consolidated behind green tests | -| 4 | M17.20–M17.21 docs/release | Canonical docs, governance, and reproducible release automation complete | -| 5 | Final verification | Full prescribed regression satisfies `docs/AGENT-REGRESSION.md` | +| 1 | M17.12–M17.14 test depth | Representation sync, runner branches, and compatibility matrix enforced | +| 2 | M17.15–M17.19 maintainability | Duplication, ownership, legacy surface, errors, and config contracts consolidated behind green tests | +| 3 | M17.20–M17.21 docs/release | Canonical docs, governance, and reproducible release automation complete | +| 4 | Final verification | Full prescribed regression satisfies `docs/AGENT-REGRESSION.md` | ## P0 — Security and execution truth @@ -75,14 +65,12 @@ Work is intentionally paused to preserve the remaining implementation budget. described as continuously verified without current evidence. - Acceptance: artifacts retain PipelineRun/TaskRun logs and traffic evidence. -- [ ] **M17.4 Add scheduled Tekton Results verification** +- [x] **M17.4 Add scheduled Tekton Results verification** - Install Results/Postgres and run the strict Results DB verification. - Acceptance: `run-regression-agent-full.sh` equivalent exits zero and uploads diagnostic artifacts on failure. - - Evidence: scheduled run `35107095845` on `main` failed compiling - `baggage-spring-boot-starter` (`invalid target release: 21`) because the - workflow did not install JDK 21. The workflow now provisions Java 21, PHP - DOM, and Go before the strict regression. Live green run still required. + - Evidence: run `35108434664` installed Java 21, compiled both Maven modules, + passed Phase 2 and Newman, verified Results, and exited 0. ## P1 — CI gates and operator assurance @@ -139,17 +127,19 @@ Work is intentionally paused to preserve the remaining implementation budget. StackRun reconciliation gate and required bootstrap `fetch-source` execution checkpoint. -- [ ] **M17.11 Exercise optional graph and GUI API suites** +- [x] **M17.11 Exercise optional graph and GUI API suites** - Run Neo4j graph Newman on a scheduled cadence and GUI Newman against a live backend. - Acceptance: both collections report zero failed assertions in CI. - - Evidence: `graph-gui-newman.yml` now schedules GUI Flask Newman and Kind - `--with-graph` Newman. Live green jobs still required. + - Evidence: run `35109309784` passed GUI Newman (54 assertions) and graph + Newman (38 + 36 assertions) with zero failures. - [ ] **M17.12 Test Helm and representation synchronization** - Test chart packaging/rendering, CRD copies, Stack YAML→CR conversion, and parameter compatibility across StackRun, operator builders, and Pipelines. - Acceptance: drift in any duplicated representation fails PR CI. + - Evidence: `check-representation-sync` is wired into static-quality. Live + green static-quality run still required. - [ ] **M17.13 Test embedded Task shell and stack test runners** - Add shell-level fixtures for malformed input and exercise Newman, @@ -218,5 +208,7 @@ Work is intentionally paused to preserve the remaining implementation budget. | 2026-09-15 | M17.9 demo validation | Git LFS recordings; stream, A/V drift, narration, and OCR checks | Run 34975666059 passed | | 2026-09-15 | M17.10 Newman execution truth | Current-run StackRun reconciliation and bootstrap fetch-source checkpoint | Run 34989095946 passed; final PR revision passed all checks | | 2026-09-15 | Pause checkpoint | Completed M17.1–M17.3 and M17.5–M17.10; open M17.4 and M17.11–M17.21 | Resume instructions recorded above | -| 2026-09-16 | M17.11 graph and GUI Newman automation | Scheduled/manual/PR-path workflow; live Flask GUI runner; cluster `--with-graph`; list endpoints tolerate missing kubeconfig | Automation added; first live graph + GUI Newman run still required | +| 2026-09-16 | M17.4 Results live acceptance | Java 21 toolchain; Phase 2; Newman; Results DB | Run 35108434664 passed | +| 2026-09-16 | M17.11 graph and GUI Newman | Live Flask GUI collection; Kind Neo4j graph collection | Run 35109309784 passed, zero assertion failures | +| 2026-09-16 | M17.12 representation sync automation | CRD copy, Stack/Team conversion, and PipelineRun param drift gate | Automation added; first live static-quality run still required | diff --git a/operator/internal/pipeline/builder.go b/operator/internal/pipeline/builder.go index 5348c6b..f0f61a9 100644 --- a/operator/internal/pipeline/builder.go +++ b/operator/internal/pipeline/builder.go @@ -288,7 +288,6 @@ func BuildMerge(opt Options) (*unstructured.Unstructured, error) { param("stack-file", opt.StackFile), param("changed-app", opt.ChangedApp), param("image-registry", opt.ImageRegistry), - param("cache-repo", opt.CacheRepo), } obj := map[string]any{ "apiVersion": TektonAPIVersion, diff --git a/operator/internal/pipeline/testdata/golden/merge.json b/operator/internal/pipeline/testdata/golden/merge.json index 9095661..b91617f 100644 --- a/operator/internal/pipeline/testdata/golden/merge.json +++ b/operator/internal/pipeline/testdata/golden/merge.json @@ -31,10 +31,6 @@ "name": "image-registry", "value": "localhost:5000" }, - { - "name": "cache-repo", - "value": "localhost:5000/kaniko-cache" - }, { "name": "max-retries", "value": "2" diff --git a/orchestrator/pipelinerun_builder.py b/orchestrator/pipelinerun_builder.py index 75fdda6..5f4b02a 100644 --- a/orchestrator/pipelinerun_builder.py +++ b/orchestrator/pipelinerun_builder.py @@ -233,7 +233,6 @@ def build_merge_pipelinerun( {"name": "stack-file", "value": stack_file}, {"name": "changed-app", "value": changed_app}, {"name": "image-registry", "value": image_registry}, - {"name": "cache-repo", "value": cache_repo}, ], "workspaces": [ { diff --git a/scripts/check-representation-sync.py b/scripts/check-representation-sync.py new file mode 100755 index 0000000..feb421f --- /dev/null +++ b/scripts/check-representation-sync.py @@ -0,0 +1,235 @@ +#!/usr/bin/env python3 +"""Fail when duplicated Helm, CRD, Stack, or PipelineRun representations drift.""" + +from __future__ import annotations + +import argparse +import filecmp +import re +import sys +from pathlib import Path + +import yaml + +from tekton_dag_common.stack_cr import iter_stack_files, stack_yaml_to_cr +from tekton_dag_common.team_cr import iter_team_files, team_yaml_to_cr + +CRD_NAMES = ( + "tektondag.io_stackruns.yaml", + "tektondag.io_stacks.yaml", + "tektondag.io_teams.yaml", +) + +GO_BUILDERS = { + "BuildPR": "stack-pr-test", + "BuildBootstrap": "stack-bootstrap", + "BuildMerge": "stack-merge-release", + "BuildPromote": "stack-promote", + "BuildPRContinue": "stack-pr-continue", +} + +PYTHON_BUILDERS = { + "build_pr_pipelinerun": "stack-pr-test", + "build_bootstrap_pipelinerun": "stack-bootstrap", + "build_merge_pipelinerun": "stack-merge-release", + "build_promote_pipelinerun": "stack-promote", +} + +PARAM_RE = re.compile(r'param\("([^"]+)"') +PY_PARAM_RE = re.compile(r'\{"name": "([^"]+)", "value":') +FUNC_RE = re.compile(r"^func (Build[A-Za-z]+)\(") +PY_FUNC_RE = re.compile(r"^def (build_[a-z_]+)\(") + + +def _repo_root() -> Path: + return Path(__file__).resolve().parents[1] + + +def check_crd_copies(root: Path) -> list[str]: + errors: list[str] = [] + generated = root / "operator" / "config" / "crd" / "bases" + packaged = root / "helm" / "tekton-dag" / "crds" + for name in CRD_NAMES: + left = generated / name + right = packaged / name + if not left.is_file(): + errors.append(f"missing generated CRD: {left}") + continue + if not right.is_file(): + errors.append(f"missing Helm CRD copy: {right}") + continue + if not filecmp.cmp(left, right, shallow=False): + errors.append(f"CRD drift: {left} != {right}") + return errors + + +def check_stack_and_team_conversion(root: Path) -> list[str]: + errors: list[str] = [] + stacks = root / "stacks" + converted = 0 + for path in iter_stack_files(stacks): + data = yaml.safe_load(path.read_text()) or {} + if not isinstance(data.get("apps"), list): + continue + cr = stack_yaml_to_cr( + data, + namespace="tekton-pipelines", + stack_file=f"stacks/{path.name}", + git_url="https://github.com/jmjava/tekton-dag.git", + ) + if cr.get("kind") != "Stack": + errors.append(f"{path}: conversion did not produce kind Stack") + continue + if not cr.get("metadata", {}).get("name"): + errors.append(f"{path}: Stack CR is missing metadata.name") + continue + if not cr.get("spec", {}).get("apps"): + errors.append(f"{path}: Stack CR has no spec.apps") + continue + converted += 1 + if converted == 0: + errors.append("no stack YAML files converted to Stack CRs") + + teams = 0 + for team_name, path in iter_team_files(root / "teams"): + data = yaml.safe_load(path.read_text()) or {} + cr = team_yaml_to_cr(data, team_dir_name=team_name, namespace="tekton-pipelines") + if cr.get("kind") != "Team" or not cr.get("metadata", {}).get("name"): + errors.append(f"{path}: Team conversion failed") + continue + teams += 1 + if teams == 0: + errors.append("no team YAML files converted to Team CRs") + return errors + + +def _pipeline_params(path: Path) -> tuple[set[str], set[str]]: + docs = list(yaml.safe_load_all(path.read_text())) + pipeline = next( + ( + doc + for doc in docs + if isinstance(doc, dict) and doc.get("kind") == "Pipeline" + ), + None, + ) + if pipeline is None: + return set(), set() + required: set[str] = set() + declared: set[str] = set() + for param in pipeline.get("spec", {}).get("params") or []: + name = param.get("name") + if not name: + continue + declared.add(name) + if "default" not in param: + required.add(name) + return required, declared + + +def _go_builder_params(source: str) -> dict[str, set[str]]: + current = None + params: dict[str, set[str]] = {name: set() for name in GO_BUILDERS} + for line in source.splitlines(): + match = FUNC_RE.match(line) + if match: + current = match.group(1) if match.group(1) in GO_BUILDERS else None + continue + if current is None: + continue + params[current].update(PARAM_RE.findall(line)) + if line.startswith("func "): + current = None + return params + + +def _python_builder_params(source: str) -> dict[str, set[str]]: + current = None + params: dict[str, set[str]] = {name: set() for name in PYTHON_BUILDERS} + for line in source.splitlines(): + match = PY_FUNC_RE.match(line) + if match: + current = match.group(1) if match.group(1) in PYTHON_BUILDERS else None + continue + if current is None: + continue + if line.startswith("def "): + current = None + continue + params[current].update(PY_PARAM_RE.findall(line)) + return params + + +def check_pipeline_param_compatibility(root: Path) -> list[str]: + errors: list[str] = [] + pipelines = { + "stack-pr-test": root / "pipeline" / "stack-pr-pipeline.yaml", + "stack-bootstrap": root / "pipeline" / "stack-bootstrap-pipeline.yaml", + "stack-merge-release": root / "pipeline" / "stack-merge-pipeline.yaml", + "stack-promote": root / "pipeline" / "stack-promote-pipeline.yaml", + "stack-pr-continue": root / "pipeline" / "stack-pr-continue-pipeline.yaml", + } + declared: dict[str, tuple[set[str], set[str]]] = {} + for name, path in pipelines.items(): + if not path.is_file(): + errors.append(f"missing pipeline: {path}") + continue + declared[name] = _pipeline_params(path) + + go_params = _go_builder_params((root / "operator" / "internal" / "pipeline" / "builder.go").read_text()) + py_params = _python_builder_params((root / "orchestrator" / "pipelinerun_builder.py").read_text()) + + for builder, pipeline in GO_BUILDERS.items(): + required, all_declared = declared.get(pipeline, (set(), set())) + emitted = go_params.get(builder, set()) + missing = required - emitted + extra = emitted - all_declared + if missing: + errors.append( + f"{builder} does not set required {pipeline} params: {sorted(missing)}" + ) + if extra: + errors.append( + f"{builder} sets unknown {pipeline} params: {sorted(extra)}" + ) + + for builder, pipeline in PYTHON_BUILDERS.items(): + go_builder = next(name for name, target in GO_BUILDERS.items() if target == pipeline) + # Compile-image overrides are optional and only emitted when provided. + comparable = py_params[builder] - { + name for name in py_params[builder] if name.startswith("compile-image-") + } + go_comparable = go_params[go_builder] - { + name for name in go_params[go_builder] if name.startswith("compile-image-") + } + if comparable != go_comparable: + errors.append( + f"{builder} params {sorted(comparable)} != {go_builder} params {sorted(go_comparable)}" + ) + return errors + + +def run_checks(root: Path | None = None) -> list[str]: + repo = root or _repo_root() + errors = [] + errors.extend(check_crd_copies(repo)) + errors.extend(check_stack_and_team_conversion(repo)) + errors.extend(check_pipeline_param_compatibility(repo)) + return errors + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.parse_args(argv) + errors = run_checks() + if errors: + print("ERROR: representation drift detected", file=sys.stderr) + for error in errors: + print(f" - {error}", file=sys.stderr) + return 1 + print("OK: Helm CRDs, Stack/Team conversion, and PipelineRun params are in sync") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/check-representation-sync.sh b/scripts/check-representation-sync.sh new file mode 100755 index 0000000..ce933f6 --- /dev/null +++ b/scripts/check-representation-sync.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# Fail when Helm CRDs, Stack/Team conversion, or PipelineRun params drift. +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=common.sh +source "$SCRIPT_DIR/common.sh" + +if [[ -x "$REPO_ROOT/.venv/bin/python3" ]]; then + export PATH="$REPO_ROOT/.venv/bin:$PATH" +fi + +need python3 +export PYTHONPATH="${REPO_ROOT}/libs/tekton-dag-common${PYTHONPATH:+:$PYTHONPATH}" +exec python3 "$SCRIPT_DIR/check-representation-sync.py" "$@" diff --git a/scripts/run-regression.sh b/scripts/run-regression.sh index 4600a13..4092c23 100755 --- a/scripts/run-regression.sh +++ b/scripts/run-regression.sh @@ -126,6 +126,10 @@ ensure_mikefarah_yq need yq bash "$SCRIPT_DIR/verify-dag-phase1.sh" +echo "" +echo ">>> Representation sync — scripts/check-representation-sync.sh" +bash "$SCRIPT_DIR/check-representation-sync.sh" + run_pytest_dir() { local name="$1" local dir="$2"