Test run-stack-tests Newman, Playwright, and Artillery runners

libs/tekton-dag-common/tests/test_m17_stack_test_runners.py

@@ -0,0 +1,35 @@
+"""Static acceptance checks for M17.13 stack test runners."""
+
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[3]
+
+
+def test_task_sources_extracted_runner_script():
+    task = (ROOT / "tasks/run-stack-tests.yaml").read_text()
+    script = (ROOT / "scripts/run-stack-tests-runners.sh").read_text()
+
+    assert "scripts/run-stack-tests-runners.sh" in task
+    assert "ERROR: run-stack-tests runners script missing" in task
+    assert "PHASE 2: Per-app tests" not in task
+    assert "run_newman" in script
+    assert "run_playwright" in script
+    assert "run_artillery" in script
+    assert "stack-json is not valid JSON" in script
+
+
+def test_runner_fixture_suite_covers_success_and_failure():
+    tests = (ROOT / "libs/tekton-dag-common/tests/test_run_stack_tests_runners.py").read_text()
+
+    assert "test_malformed_stack_json_fails" in tests
+    assert '"{not-json"' in tests
+    assert "test_runner_success_and_failure_paths" in tests
+    assert '"newman"' in tests
+    assert '"playwright"' in tests
+    assert '"artillery"' in tests
+    assert "[postman] PASS" in tests
+    assert "[postman] FAIL" in tests
+    assert "[playwright] PASS" in tests
+    assert "[playwright] FAIL" in tests
+    assert "[artillery] PASS" in tests
+    assert "[artillery] FAIL" in tests

libs/tekton-dag-common/tests/test_run_stack_tests_runners.py

@@ -0,0 +1,213 @@
+"""Fixture coverage for run-stack-tests Newman, Playwright, and Artillery branches."""
+
+from __future__ import annotations
+
+import json
+import os
+import stat
+import subprocess
+from pathlib import Path
+
+import pytest
+
+ROOT = Path(__file__).resolve().parents[3]
+RUNNERS = ROOT / "scripts" / "run-stack-tests-runners.sh"
+
+
+def _write_executable(path: Path, body: str) -> None:
+    path.write_text(body)
+    path.chmod(path.stat().st_mode | stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR)
+
+
+def _stub_bin(
+    tmp_path: Path,
+    *,
+    newman: int = 0,
+    playwright: int = 0,
+    artillery: int = 0,
+) -> Path:
+    bindir = tmp_path / "bin"
+    bindir.mkdir(parents=True)
+    _write_executable(
+        bindir / "newman",
+        f"#!/bin/sh\necho newman-stub \"$@\"\nexit {newman}\n",
+    )
+    _write_executable(
+        bindir / "artillery",
+        f"#!/bin/sh\necho artillery-stub \"$@\"\nexit {artillery}\n",
+    )
+    _write_executable(
+        bindir / "npx",
+        "#!/bin/sh\n"
+        'if [ "$1" = "playwright" ]; then\n'
+        f'  echo playwright-stub "$@"\n  exit {playwright}\n'
+        "fi\n"
+        "exit 0\n",
+    )
+    _write_executable(bindir / "npm", "#!/bin/sh\nexit 0\n")
+    _write_executable(
+        bindir / "curl",
+        "#!/bin/sh\n"
+        'echo \'{"frontend":"ok","api":"ok","sess-1":"present"}\'\n',
+    )
+    return bindir
+
+
+def _stack(*apps: dict) -> str:
+    return json.dumps(
+        {
+            "propagation": {"baggage-key": "dev-session"},
+            "defaults": {"namespace": "staging", "service-port": "80"},
+            "apps": list(apps),
+        }
+    )
+
+
+def _app(name: str, **tests: str) -> dict:
+    payload: dict = {"name": name, "namespace": "staging", "service-port": "80"}
+    if tests:
+        payload["tests"] = tests
+    return payload
+
+
+def _prepare_runner_files(source: Path, kind: str) -> None:
+    if kind == "newman":
+        collection = source / "tests" / "postman" / "api.json"
+        collection.parent.mkdir(parents=True, exist_ok=True)
+        collection.write_text('{"info":{"name":"api"},"item":[]}\n')
+        return
+    if kind == "playwright":
+        suite = source / "tests" / "playwright"
+        suite.mkdir(parents=True, exist_ok=True)
+        (source / "tests" / "package.json").write_text('{"name":"api-tests"}\n')
+        (suite / "example.spec.js").write_text("test('ok', () => {});\n")
+        return
+    if kind == "artillery":
+        script = source / "tests" / "artillery" / "load.yml"
+        script.parent.mkdir(parents=True, exist_ok=True)
+        script.write_text("config:\n  target: http://localhost\n")
+        return
+    raise AssertionError(kind)
+
+
+def _run(
+    root: Path,
+    stack_json: str,
+    *,
+    newman: int = 0,
+    playwright: int = 0,
+    artillery: int = 0,
+    extra_env: dict[str, str] | None = None,
+    prepare: str | None = None,
+) -> tuple[subprocess.CompletedProcess[str], Path]:
+    source = root / "source"
+    source.mkdir(parents=True)
+    if prepare:
+        _prepare_runner_files(source, prepare)
+    summary = root / "summary.json"
+    bindir = _stub_bin(
+        root, newman=newman, playwright=playwright, artillery=artillery
+    )
+    env = os.environ.copy()
+    env.update(
+        {
+            "STACK_JSON": stack_json,
+            "APP_LIST": "frontend api",
+            "ENTRY_APP": "frontend",
+            "CHAIN": "frontend api",
+            "BUILD_APPS": "api",
+            "INTERCEPT": "x-dev-session:sess-1",
+            "DEFAULT_NS": "staging",
+            "TESTS_TO_RUN": "",
+            "UNMAPPED_AREA": "",
+            "APPS_TO_TEST": "",
+            "TEST_SOURCE": str(source),
+            "TEST_SUMMARY_PATH": str(summary),
+            "PATH": f"{bindir}{os.pathsep}{env['PATH']}",
+        }
+    )
+    if extra_env:
+        env.update(extra_env)
+    completed = subprocess.run(
+        ["sh", str(RUNNERS)],
+        cwd=source,
+        env=env,
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    return completed, summary
+
+
+def test_malformed_stack_json_fails(tmp_path: Path) -> None:
+    completed, _summary = _run(tmp_path, "{not-json")
+    assert completed.returncode == 1
+    assert "stack-json is not valid JSON" in completed.stderr
+
+
+def test_unmapped_area_exits_without_runners(tmp_path: Path) -> None:
+    completed, summary = _run(
+        tmp_path,
+        _stack(_app("frontend"), _app("api")),
+        extra_env={"UNMAPPED_AREA": "billing"},
+    )
+    assert completed.returncode == 0
+    payload = json.loads(summary.read_text())
+    assert payload["unmapped-area"] == "billing"
+    assert "newman-stub" not in completed.stdout
+    assert "playwright-stub" not in completed.stdout
+    assert "artillery-stub" not in completed.stdout
+
+
+@pytest.mark.parametrize(
+    ("kind", "tests", "pass_token", "fail_token"),
+    [
+        (
+            "newman",
+            {"postman": "tests/postman/api.json"},
+            "[postman] PASS",
+            "[postman] FAIL",
+        ),
+        (
+            "playwright",
+            {"playwright": "tests/playwright"},
+            "[playwright] PASS",
+            "[playwright] FAIL",
+        ),
+        (
+            "artillery",
+            {"artillery": "tests/artillery/load.yml"},
+            "[artillery] PASS",
+            "[artillery] FAIL",
+        ),
+    ],
+)
+def test_runner_success_and_failure_paths(
+    tmp_path: Path,
+    kind: str,
+    tests: dict[str, str],
+    pass_token: str,
+    fail_token: str,
+) -> None:
+    stack = _stack(_app("frontend"), _app("api", **tests))
+
+    passed, passed_summary = _run(
+        tmp_path / "success",
+        stack,
+        prepare=kind,
+    )
+    assert passed.returncode == 0, passed.stdout + passed.stderr
+    assert pass_token in passed.stdout
+    assert json.loads(passed_summary.read_text())["api"] == "pass"
+
+    failed, failed_summary = _run(
+        tmp_path / "failure",
+        stack,
+        prepare=kind,
+        newman=1 if kind == "newman" else 0,
+        playwright=1 if kind == "playwright" else 0,
+        artillery=1 if kind == "artillery" else 0,
+    )
+    assert failed.returncode == 1, failed.stdout + failed.stderr
+    assert fail_token in failed.stdout
+    assert json.loads(failed_summary.read_text())["api"] == "fail"

milestones/milestone-17.md

@@ -1,6 +1,6 @@
 # Milestone 17 — End-to-end quality and production-readiness closure
 
-**Status:** In progress after M17.11; next is M17.12 representation sync
+**Status:** In progress after M17.12; next is M17.13 stack test runners
 
 This milestone converts the September 2026 end-to-end audit into an executable
 backlog. Work is ordered by production risk, not by subsystem. A checkbox is
@@ -24,20 +24,21 @@ regression criteria in `docs/AGENT-REGRESSION.md` are satisfied.
 
 ## Resume checkpoint — 2026-09-16
 
-- Completed with recorded acceptance: M17.1–M17.11.
-- Still open: M17.12–M17.21.
+- Completed with recorded acceptance: M17.1–M17.12.
+- Still open: M17.13–M17.21.
 - M17.4 live Results run `35108434664` on [PR #63](https://github.com/jmjava/tekton-dag/pull/63) exited 0 after the Java 21 toolchain fix.
 - M17.11 live graph + GUI Newman run `35109309784` on [PR #65](https://github.com/jmjava/tekton-dag/pull/65) reported zero failed assertions.
-- Continue in numeric order from M17.12. Do not skip directly to
-  maintainability work because M17.12–M17.14 establish the test evidence needed
+- M17.12 live static-quality run `35111800721` on [PR #67](https://github.com/jmjava/tekton-dag/pull/67) passed Ruff, Go lint/vet, ShellCheck, both frontend builds, Helm package/render, and representation sync.
+- Continue in numeric order from M17.13. Do not skip directly to
+  maintainability work because M17.13–M17.14 establish the test evidence needed
   to refactor safely.
 - Before starting a slice, fetch the latest `cursor/close-e2e-audit-gaps-fc5f` and
   create a fresh `cursor/<slice>-fc5f` branch. Do not reuse merged slice
   branches.
 
 | Resume order | Work | Terminal condition |
 |---|---|---|
-| 1 | M17.12–M17.14 test depth | Representation sync, runner branches, and compatibility matrix enforced |
+| 1 | M17.13–M17.14 test depth | Runner branches and compatibility matrix enforced |
 | 2 | M17.15–M17.19 maintainability | Duplication, ownership, legacy surface, errors, and config contracts consolidated behind green tests |
 | 3 | M17.20–M17.21 docs/release | Canonical docs, governance, and reproducible release automation complete |
 | 4 | Final verification | Full prescribed regression satisfies `docs/AGENT-REGRESSION.md` |
@@ -134,12 +135,12 @@ regression criteria in `docs/AGENT-REGRESSION.md` are satisfied.
   - Evidence: run `35109309784` passed GUI Newman (54 assertions) and graph
     Newman (38 + 36 assertions) with zero failures.
 
-- [ ] **M17.12 Test Helm and representation synchronization**
+- [x] **M17.12 Test Helm and representation synchronization**
   - Test chart packaging/rendering, CRD copies, Stack YAML→CR conversion, and
     parameter compatibility across StackRun, operator builders, and Pipelines.
   - Acceptance: drift in any duplicated representation fails PR CI.
-  - Evidence: `check-representation-sync` is wired into static-quality. Live
-    green static-quality run still required.
+  - Evidence: run `35111800721` passed representation sync plus the rest of
+    static-quality after the Ruff import-order hotfix.
 
 - [ ] **M17.13 Test embedded Task shell and stack test runners**
   - Add shell-level fixtures for malformed input and exercise Newman,
@@ -210,5 +211,6 @@ regression criteria in `docs/AGENT-REGRESSION.md` are satisfied.
 | 2026-09-15 | Pause checkpoint | Completed M17.1–M17.3 and M17.5–M17.10; open M17.4 and M17.11–M17.21 | Resume instructions recorded above |
 | 2026-09-16 | M17.4 Results live acceptance | Java 21 toolchain; Phase 2; Newman; Results DB | Run 35108434664 passed |
 | 2026-09-16 | M17.11 graph and GUI Newman | Live Flask GUI collection; Kind Neo4j graph collection | Run 35109309784 passed, zero assertion failures |
-| 2026-09-16 | M17.12 representation sync automation | CRD copy, Stack/Team conversion, and PipelineRun param drift gate | Automation added; first live static-quality run still required |
+| 2026-09-16 | M17.12 representation sync acceptance | CRD copy, Stack/Team conversion, PipelineRun param drift, and static-quality | Run 35111800721 passed |
+| 2026-09-16 | M17.13 stack test runner fixtures | Extracted `run-stack-tests` Newman/Playwright/Artillery runners; malformed JSON plus success/failure fixtures | Automation added; first live local-regression run still required |