From 7d4ef49db1a891551ecb13f69c8bbf50bc64127a Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 16 Sep 2026 15:01:16 +0000 Subject: [PATCH] Extract run-stack-tests runners for fixture coverage Move Newman, Playwright, and Artillery branches out of the embedded Task script so malformed stack-json and each runner's success and failure paths can be exercised without a cluster. Record M17.12 acceptance from static-quality run 35111800721. Co-authored-by: jmjava --- .../tests/test_m17_stack_test_runners.py | 35 ++ .../tests/test_run_stack_tests_runners.py | 213 +++++++++++ milestones/milestone-17.md | 22 +- scripts/run-stack-tests-runners.sh | 358 ++++++++++++++++++ tasks/run-stack-tests.yaml | 344 +---------------- 5 files changed, 632 insertions(+), 340 deletions(-) create mode 100644 libs/tekton-dag-common/tests/test_m17_stack_test_runners.py create mode 100644 libs/tekton-dag-common/tests/test_run_stack_tests_runners.py create mode 100755 scripts/run-stack-tests-runners.sh diff --git a/libs/tekton-dag-common/tests/test_m17_stack_test_runners.py b/libs/tekton-dag-common/tests/test_m17_stack_test_runners.py new file mode 100644 index 0000000..35e4efe --- /dev/null +++ b/libs/tekton-dag-common/tests/test_m17_stack_test_runners.py @@ -0,0 +1,35 @@ +"""Static acceptance checks for M17.13 stack test runners.""" + +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[3] + + +def test_task_sources_extracted_runner_script(): + task = (ROOT / "tasks/run-stack-tests.yaml").read_text() + script = (ROOT / "scripts/run-stack-tests-runners.sh").read_text() + + assert "scripts/run-stack-tests-runners.sh" in task + assert "ERROR: run-stack-tests runners script missing" in task + assert "PHASE 2: Per-app tests" not in task + assert "run_newman" in script + assert "run_playwright" in script + assert "run_artillery" in script + assert "stack-json is not valid JSON" in script + + +def test_runner_fixture_suite_covers_success_and_failure(): + tests = (ROOT / "libs/tekton-dag-common/tests/test_run_stack_tests_runners.py").read_text() + + assert "test_malformed_stack_json_fails" in tests + assert '"{not-json"' in tests + assert "test_runner_success_and_failure_paths" in tests + assert '"newman"' in tests + assert '"playwright"' in tests + assert '"artillery"' in tests + assert "[postman] PASS" in tests + assert "[postman] FAIL" in tests + assert "[playwright] PASS" in tests + assert "[playwright] FAIL" in tests + assert "[artillery] PASS" in tests + assert "[artillery] FAIL" in tests diff --git a/libs/tekton-dag-common/tests/test_run_stack_tests_runners.py b/libs/tekton-dag-common/tests/test_run_stack_tests_runners.py new file mode 100644 index 0000000..b480aff --- /dev/null +++ b/libs/tekton-dag-common/tests/test_run_stack_tests_runners.py @@ -0,0 +1,213 @@ +"""Fixture coverage for run-stack-tests Newman, Playwright, and Artillery branches.""" + +from __future__ import annotations + +import json +import os +import stat +import subprocess +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[3] +RUNNERS = ROOT / "scripts" / "run-stack-tests-runners.sh" + + +def _write_executable(path: Path, body: str) -> None: + path.write_text(body) + path.chmod(path.stat().st_mode | stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR) + + +def _stub_bin( + tmp_path: Path, + *, + newman: int = 0, + playwright: int = 0, + artillery: int = 0, +) -> Path: + bindir = tmp_path / "bin" + bindir.mkdir(parents=True) + _write_executable( + bindir / "newman", + f"#!/bin/sh\necho newman-stub \"$@\"\nexit {newman}\n", + ) + _write_executable( + bindir / "artillery", + f"#!/bin/sh\necho artillery-stub \"$@\"\nexit {artillery}\n", + ) + _write_executable( + bindir / "npx", + "#!/bin/sh\n" + 'if [ "$1" = "playwright" ]; then\n' + f' echo playwright-stub "$@"\n exit {playwright}\n' + "fi\n" + "exit 0\n", + ) + _write_executable(bindir / "npm", "#!/bin/sh\nexit 0\n") + _write_executable( + bindir / "curl", + "#!/bin/sh\n" + 'echo \'{"frontend":"ok","api":"ok","sess-1":"present"}\'\n', + ) + return bindir + + +def _stack(*apps: dict) -> str: + return json.dumps( + { + "propagation": {"baggage-key": "dev-session"}, + "defaults": {"namespace": "staging", "service-port": "80"}, + "apps": list(apps), + } + ) + + +def _app(name: str, **tests: str) -> dict: + payload: dict = {"name": name, "namespace": "staging", "service-port": "80"} + if tests: + payload["tests"] = tests + return payload + + +def _prepare_runner_files(source: Path, kind: str) -> None: + if kind == "newman": + collection = source / "tests" / "postman" / "api.json" + collection.parent.mkdir(parents=True, exist_ok=True) + collection.write_text('{"info":{"name":"api"},"item":[]}\n') + return + if kind == "playwright": + suite = source / "tests" / "playwright" + suite.mkdir(parents=True, exist_ok=True) + (source / "tests" / "package.json").write_text('{"name":"api-tests"}\n') + (suite / "example.spec.js").write_text("test('ok', () => {});\n") + return + if kind == "artillery": + script = source / "tests" / "artillery" / "load.yml" + script.parent.mkdir(parents=True, exist_ok=True) + script.write_text("config:\n target: http://localhost\n") + return + raise AssertionError(kind) + + +def _run( + root: Path, + stack_json: str, + *, + newman: int = 0, + playwright: int = 0, + artillery: int = 0, + extra_env: dict[str, str] | None = None, + prepare: str | None = None, +) -> tuple[subprocess.CompletedProcess[str], Path]: + source = root / "source" + source.mkdir(parents=True) + if prepare: + _prepare_runner_files(source, prepare) + summary = root / "summary.json" + bindir = _stub_bin( + root, newman=newman, playwright=playwright, artillery=artillery + ) + env = os.environ.copy() + env.update( + { + "STACK_JSON": stack_json, + "APP_LIST": "frontend api", + "ENTRY_APP": "frontend", + "CHAIN": "frontend api", + "BUILD_APPS": "api", + "INTERCEPT": "x-dev-session:sess-1", + "DEFAULT_NS": "staging", + "TESTS_TO_RUN": "", + "UNMAPPED_AREA": "", + "APPS_TO_TEST": "", + "TEST_SOURCE": str(source), + "TEST_SUMMARY_PATH": str(summary), + "PATH": f"{bindir}{os.pathsep}{env['PATH']}", + } + ) + if extra_env: + env.update(extra_env) + completed = subprocess.run( + ["sh", str(RUNNERS)], + cwd=source, + env=env, + capture_output=True, + text=True, + check=False, + ) + return completed, summary + + +def test_malformed_stack_json_fails(tmp_path: Path) -> None: + completed, _summary = _run(tmp_path, "{not-json") + assert completed.returncode == 1 + assert "stack-json is not valid JSON" in completed.stderr + + +def test_unmapped_area_exits_without_runners(tmp_path: Path) -> None: + completed, summary = _run( + tmp_path, + _stack(_app("frontend"), _app("api")), + extra_env={"UNMAPPED_AREA": "billing"}, + ) + assert completed.returncode == 0 + payload = json.loads(summary.read_text()) + assert payload["unmapped-area"] == "billing" + assert "newman-stub" not in completed.stdout + assert "playwright-stub" not in completed.stdout + assert "artillery-stub" not in completed.stdout + + +@pytest.mark.parametrize( + ("kind", "tests", "pass_token", "fail_token"), + [ + ( + "newman", + {"postman": "tests/postman/api.json"}, + "[postman] PASS", + "[postman] FAIL", + ), + ( + "playwright", + {"playwright": "tests/playwright"}, + "[playwright] PASS", + "[playwright] FAIL", + ), + ( + "artillery", + {"artillery": "tests/artillery/load.yml"}, + "[artillery] PASS", + "[artillery] FAIL", + ), + ], +) +def test_runner_success_and_failure_paths( + tmp_path: Path, + kind: str, + tests: dict[str, str], + pass_token: str, + fail_token: str, +) -> None: + stack = _stack(_app("frontend"), _app("api", **tests)) + + passed, passed_summary = _run( + tmp_path / "success", + stack, + prepare=kind, + ) + assert passed.returncode == 0, passed.stdout + passed.stderr + assert pass_token in passed.stdout + assert json.loads(passed_summary.read_text())["api"] == "pass" + + failed, failed_summary = _run( + tmp_path / "failure", + stack, + prepare=kind, + newman=1 if kind == "newman" else 0, + playwright=1 if kind == "playwright" else 0, + artillery=1 if kind == "artillery" else 0, + ) + assert failed.returncode == 1, failed.stdout + failed.stderr + assert fail_token in failed.stdout + assert json.loads(failed_summary.read_text())["api"] == "fail" diff --git a/milestones/milestone-17.md b/milestones/milestone-17.md index 518003f..1040841 100644 --- a/milestones/milestone-17.md +++ b/milestones/milestone-17.md @@ -1,6 +1,6 @@ # Milestone 17 — End-to-end quality and production-readiness closure -**Status:** In progress after M17.11; next is M17.12 representation sync +**Status:** In progress after M17.12; next is M17.13 stack test runners This milestone converts the September 2026 end-to-end audit into an executable backlog. Work is ordered by production risk, not by subsystem. A checkbox is @@ -24,12 +24,13 @@ regression criteria in `docs/AGENT-REGRESSION.md` are satisfied. ## Resume checkpoint — 2026-09-16 -- Completed with recorded acceptance: M17.1–M17.11. -- Still open: M17.12–M17.21. +- Completed with recorded acceptance: M17.1–M17.12. +- Still open: M17.13–M17.21. - M17.4 live Results run `35108434664` on [PR #63](https://github.com/jmjava/tekton-dag/pull/63) exited 0 after the Java 21 toolchain fix. - M17.11 live graph + GUI Newman run `35109309784` on [PR #65](https://github.com/jmjava/tekton-dag/pull/65) reported zero failed assertions. -- Continue in numeric order from M17.12. Do not skip directly to - maintainability work because M17.12–M17.14 establish the test evidence needed +- M17.12 live static-quality run `35111800721` on [PR #67](https://github.com/jmjava/tekton-dag/pull/67) passed Ruff, Go lint/vet, ShellCheck, both frontend builds, Helm package/render, and representation sync. +- Continue in numeric order from M17.13. Do not skip directly to + maintainability work because M17.13–M17.14 establish the test evidence needed to refactor safely. - Before starting a slice, fetch the latest `cursor/close-e2e-audit-gaps-fc5f` and create a fresh `cursor/-fc5f` branch. Do not reuse merged slice @@ -37,7 +38,7 @@ regression criteria in `docs/AGENT-REGRESSION.md` are satisfied. | Resume order | Work | Terminal condition | |---|---|---| -| 1 | M17.12–M17.14 test depth | Representation sync, runner branches, and compatibility matrix enforced | +| 1 | M17.13–M17.14 test depth | Runner branches and compatibility matrix enforced | | 2 | M17.15–M17.19 maintainability | Duplication, ownership, legacy surface, errors, and config contracts consolidated behind green tests | | 3 | M17.20–M17.21 docs/release | Canonical docs, governance, and reproducible release automation complete | | 4 | Final verification | Full prescribed regression satisfies `docs/AGENT-REGRESSION.md` | @@ -134,12 +135,12 @@ regression criteria in `docs/AGENT-REGRESSION.md` are satisfied. - Evidence: run `35109309784` passed GUI Newman (54 assertions) and graph Newman (38 + 36 assertions) with zero failures. -- [ ] **M17.12 Test Helm and representation synchronization** +- [x] **M17.12 Test Helm and representation synchronization** - Test chart packaging/rendering, CRD copies, Stack YAML→CR conversion, and parameter compatibility across StackRun, operator builders, and Pipelines. - Acceptance: drift in any duplicated representation fails PR CI. - - Evidence: `check-representation-sync` is wired into static-quality. Live - green static-quality run still required. + - Evidence: run `35111800721` passed representation sync plus the rest of + static-quality after the Ruff import-order hotfix. - [ ] **M17.13 Test embedded Task shell and stack test runners** - Add shell-level fixtures for malformed input and exercise Newman, @@ -210,5 +211,6 @@ regression criteria in `docs/AGENT-REGRESSION.md` are satisfied. | 2026-09-15 | Pause checkpoint | Completed M17.1–M17.3 and M17.5–M17.10; open M17.4 and M17.11–M17.21 | Resume instructions recorded above | | 2026-09-16 | M17.4 Results live acceptance | Java 21 toolchain; Phase 2; Newman; Results DB | Run 35108434664 passed | | 2026-09-16 | M17.11 graph and GUI Newman | Live Flask GUI collection; Kind Neo4j graph collection | Run 35109309784 passed, zero assertion failures | -| 2026-09-16 | M17.12 representation sync automation | CRD copy, Stack/Team conversion, and PipelineRun param drift gate | Automation added; first live static-quality run still required | +| 2026-09-16 | M17.12 representation sync acceptance | CRD copy, Stack/Team conversion, PipelineRun param drift, and static-quality | Run 35111800721 passed | +| 2026-09-16 | M17.13 stack test runner fixtures | Extracted `run-stack-tests` Newman/Playwright/Artillery runners; malformed JSON plus success/failure fixtures | Automation added; first live local-regression run still required | diff --git a/scripts/run-stack-tests-runners.sh b/scripts/run-stack-tests-runners.sh new file mode 100755 index 0000000..203c661 --- /dev/null +++ b/scripts/run-stack-tests-runners.sh @@ -0,0 +1,358 @@ +#!/bin/sh +# Newman / Playwright / Artillery runner used by tasks/run-stack-tests. +# Source from the Task after Tekton params are exported, or execute locally +# with the same environment variables. +# shellcheck shell=sh +set -eu + +STACK_JSON="${STACK_JSON:-}" +APP_LIST="${APP_LIST:-}" +ENTRY_APP="${ENTRY_APP:-}" +CHAIN="${CHAIN:-}" +BUILD_APPS="${BUILD_APPS:-}" +INTERCEPT="${INTERCEPT:-}" +DEFAULT_NS="${DEFAULT_NS:-staging}" +TESTS_TO_RUN="${TESTS_TO_RUN:-}" +UNMAPPED_AREA="${UNMAPPED_AREA:-}" +APPS_TO_TEST="${APPS_TO_TEST:-}" +TEST_SOURCE="${TEST_SOURCE:-.}" +TEST_SUMMARY_PATH="${TEST_SUMMARY_PATH:-/tmp/run-stack-tests-summary.json}" + +cd "$TEST_SOURCE" + +echo '{}' > /tmp/test-summary.json +OVERALL_PASS=true + +if [ -n "$UNMAPPED_AREA" ] && [ -z "$TESTS_TO_RUN" ]; then + echo "" + echo "####################################################" + echo " NO MAPPED REGRESSION for area: $UNMAPPED_AREA" + echo " This area needs regression tests to be built." + echo "####################################################" + echo "" + jq --arg area "$UNMAPPED_AREA" \ + '. + {"unmapped-area": $area, "message": "No mapped regression; area needs tests"}' \ + /tmp/test-summary.json > /tmp/test-summary-tmp.json + mv /tmp/test-summary-tmp.json /tmp/test-summary.json + tee "$TEST_SUMMARY_PATH" < /tmp/test-summary.json + exit 0 +fi + +if ! printf '%s' "$STACK_JSON" | jq -e 'type == "object"' >/dev/null 2>&1; then + echo "ERROR: stack-json is not valid JSON" >&2 + exit 1 +fi + +HEADER_NAME="" +HEADER_VAL="" +BAGGAGE_KEY=$(printf '%s' "$STACK_JSON" | jq -r '.propagation."baggage-key" // "dev-session"') +if [ -n "$INTERCEPT" ]; then + HEADER_NAME=$(printf '%s' "$INTERCEPT" | cut -d: -f1) + HEADER_VAL=$(printf '%s' "$INTERCEPT" | cut -d: -f2-) +fi + +FILTERED=false +HAS_E2E=false +if [ -n "$TESTS_TO_RUN" ]; then + FILTERED=true + echo "" + echo "####################################################" + echo " FILTERED TEST RUN (from test-plan graph)" + echo " Tests: $TESTS_TO_RUN" + echo " Apps: $APPS_TO_TEST" + echo "####################################################" + echo "" + if printf '%s' "$TESTS_TO_RUN" | grep -q "e2e/"; then + HAS_E2E=true + fi +fi + +app_in_test_plan() { + [ "$FILTERED" = "false" ] && return 0 + [ -z "$APPS_TO_TEST" ] && return 0 + printf '%s' ",$APPS_TO_TEST," | grep -q ",$1," +} + +run_newman() { + collection="$1" + base_url="$2" + label="$3" + echo " [$label] Running: $collection" + if [ -n "$HEADER_NAME" ]; then + newman run "$collection" --env-var "baseUrl=$base_url" --reporters cli,json \ + --global-var "${HEADER_NAME}=${HEADER_VAL}" + else + newman run "$collection" --env-var "baseUrl=$base_url" --reporters cli,json + fi +} + +run_playwright() { + suite_dir="$1" + base_url="$2" + label="$3" + echo " [$label] Running: $suite_dir" + package_dir=$(dirname "$suite_dir") + if [ ! -f "${package_dir}/package.json" ]; then + echo " [$label] No package.json found (skipping)" + return 0 + fi + cd "$package_dir" + npm install --legacy-peer-deps 2>/dev/null || true + BASE_URL="$base_url" + export BASE_URL + if [ -n "$HEADER_NAME" ]; then + INTERCEPT_HEADER_NAME="$HEADER_NAME" + INTERCEPT_HEADER_VALUE="$HEADER_VAL" + export INTERCEPT_HEADER_NAME + export INTERCEPT_HEADER_VALUE + fi + rel_path=$(printf '%s' "$suite_dir" | sed "s|^${package_dir}/||") + status=0 + npx playwright test "$rel_path" --reporter=list 2>/dev/null || status=$? + cd "$TEST_SOURCE" + return "$status" +} + +run_artillery() { + script_path="$1" + base_url="$2" + label="$3" + echo " [$label] Running: $script_path" + if [ -n "$HEADER_NAME" ]; then + cat > /tmp/artillery-override.yml < /tmp/test-summary-tmp.json + mv /tmp/test-summary-tmp.json /tmp/test-summary.json +else + echo "" + echo "####################################################" + echo " PHASE 1: E2E test through entry point ($ENTRY_APP)" + echo "####################################################" + echo "" + echo " Chain: $CHAIN" + echo " Intercepted: $BUILD_APPS" + echo " Header: $HEADER_NAME=$HEADER_VAL" + echo "" + + ENTRY_NS=$(printf '%s' "$STACK_JSON" | jq -r --arg a "$ENTRY_APP" --arg d "$DEFAULT_NS" \ + '(.defaults.namespace // $d) as $dn | .apps[]|select(.name==$a)|.namespace // $dn') + ENTRY_SPORT=$(printf '%s' "$STACK_JSON" | jq -r --arg a "$ENTRY_APP" \ + '(.defaults."service-port" // "80") as $dp | .apps[]|select(.name==$a)|."service-port" // $dp') + ENTRY_URL="http://${ENTRY_APP}.${ENTRY_NS}.svc.cluster.local:${ENTRY_SPORT}" + + echo " Entry URL: $ENTRY_URL" + echo "" + + E2E_PASS=true + E2E_RESULT=$(curl -s \ + -H "${HEADER_NAME}: ${HEADER_VAL}" \ + -H "baggage: ${BAGGAGE_KEY}=${HEADER_VAL}" \ + --connect-timeout 15 --max-time 60 \ + "$ENTRY_URL" 2>/dev/null || echo '{"error":"unreachable"}') + + echo " Response:" + printf '%s\n' "$E2E_RESULT" | jq '.' 2>/dev/null || echo " $E2E_RESULT" + echo "" + + HOP=0 + for APP in $CHAIN; do + HOP=$((HOP + 1)) + IS_INTERCEPTED=false + for BA in $BUILD_APPS; do + [ "$BA" = "$APP" ] && IS_INTERCEPTED=true + done + + MARKER="" + if [ "$IS_INTERCEPTED" = "true" ]; then + MARKER=" [INTERCEPTED → PR build]" + fi + + if printf '%s' "$E2E_RESULT" | grep -qi "$APP"; then + echo " HOP $HOP ($APP)${MARKER}: REACHED" + else + echo " HOP $HOP ($APP)${MARKER}: NOT CONFIRMED" + fi + done + + if [ -n "$HEADER_VAL" ] && printf '%s' "$E2E_RESULT" | grep -qi "$HEADER_VAL"; then + echo "" + echo " Header value '$HEADER_VAL' found in response" + fi + + echo "" + if [ "$E2E_PASS" = "true" ]; then + echo " E2E: PASS" + else + echo " E2E: FAIL" + OVERALL_PASS=false + fi + + jq --arg s "$([ "$E2E_PASS" = "true" ] && echo pass || echo fail)" \ + '. + {"e2e-entry-point": $s}' \ + /tmp/test-summary.json > /tmp/test-summary-tmp.json + mv /tmp/test-summary-tmp.json /tmp/test-summary.json + + ENTRY_POSTMAN=$(printf '%s' "$STACK_JSON" | jq -r --arg a "$ENTRY_APP" \ + '.apps[]|select(.name==$a)|.tests.postman // ""') + ENTRY_PLAYWRIGHT=$(printf '%s' "$STACK_JSON" | jq -r --arg a "$ENTRY_APP" \ + '.apps[]|select(.name==$a)|.tests.playwright // ""') + + if [ -n "$ENTRY_POSTMAN" ] && [ -f "$ENTRY_POSTMAN" ]; then + echo "" + if run_newman "$ENTRY_POSTMAN" "$ENTRY_URL" "e2e/postman"; then + echo " [e2e/postman] PASS" + else + echo " [e2e/postman] FAIL" + OVERALL_PASS=false + fi + fi + + if [ -n "$ENTRY_PLAYWRIGHT" ] && [ -d "$ENTRY_PLAYWRIGHT" ]; then + echo "" + if run_playwright "$ENTRY_PLAYWRIGHT" "$ENTRY_URL" "e2e/playwright"; then + echo " [e2e/playwright] PASS" + else + echo " [e2e/playwright] FAIL" + OVERALL_PASS=false + fi + fi +fi + +# =========================================================== +# Phase 2: Per-app tests +# =========================================================== +echo "" +echo "####################################################" +echo " PHASE 2: Per-app tests" +echo "####################################################" + +for APP in $APP_LIST; do + if [ "$APP" = "$ENTRY_APP" ]; then + echo "" + echo " Skipping $APP (already tested in e2e phase)" + continue + fi + + if ! app_in_test_plan "$APP"; then + echo "" + echo " Skipping $APP (not in test plan)" + jq --arg a "$APP" '. + {($a): "skipped (not in plan)"}' \ + /tmp/test-summary.json > /tmp/test-summary-tmp.json + mv /tmp/test-summary-tmp.json /tmp/test-summary.json + continue + fi + + echo "" + echo "=========================================" + echo " Testing: $APP" + + IS_INTERCEPTED=false + for BA in $BUILD_APPS; do + [ "$BA" = "$APP" ] && IS_INTERCEPTED=true + done + if [ "$IS_INTERCEPTED" = "true" ]; then + echo " (INTERCEPTED — hitting PR build via Telepresence)" + else + echo " (normal cluster deployment)" + fi + + echo "=========================================" + + NS=$(printf '%s' "$STACK_JSON" | jq -r --arg a "$APP" --arg d "$DEFAULT_NS" \ + '(.defaults.namespace // $d) as $dn | .apps[]|select(.name==$a)|.namespace // $dn') + SPORT=$(printf '%s' "$STACK_JSON" | jq -r --arg a "$APP" \ + '(.defaults."service-port" // "80") as $dp | .apps[]|select(.name==$a)|."service-port" // $dp') + SERVICE_URL="http://${APP}.${NS}.svc.cluster.local:${SPORT}" + + POSTMAN=$(printf '%s' "$STACK_JSON" | jq -r --arg a "$APP" \ + '.apps[]|select(.name==$a)|.tests.postman // ""') + PLAYWRIGHT=$(printf '%s' "$STACK_JSON" | jq -r --arg a "$APP" \ + '.apps[]|select(.name==$a)|.tests.playwright // ""') + ARTILLERY=$(printf '%s' "$STACK_JSON" | jq -r --arg a "$APP" \ + '.apps[]|select(.name==$a)|.tests.artillery // ""') + + APP_PASS=true + + if [ -n "$POSTMAN" ] && [ -f "$POSTMAN" ]; then + if run_newman "$POSTMAN" "$SERVICE_URL" "postman"; then + echo " [postman] PASS" + else + echo " [postman] FAIL" + APP_PASS=false + fi + elif [ -n "$POSTMAN" ]; then + echo " [postman] Collection not found: $POSTMAN (skipping)" + fi + + if [ -n "$PLAYWRIGHT" ] && [ -d "$PLAYWRIGHT" ]; then + if run_playwright "$PLAYWRIGHT" "$SERVICE_URL" "playwright"; then + echo " [playwright] PASS" + else + echo " [playwright] FAIL" + APP_PASS=false + fi + elif [ -n "$PLAYWRIGHT" ]; then + echo " [playwright] Test dir not found: $PLAYWRIGHT (skipping)" + fi + + if [ -n "$ARTILLERY" ] && [ -f "$ARTILLERY" ]; then + if run_artillery "$ARTILLERY" "$SERVICE_URL" "artillery"; then + echo " [artillery] PASS" + else + echo " [artillery] FAIL" + APP_PASS=false + fi + elif [ -n "$ARTILLERY" ]; then + echo " [artillery] Script not found: $ARTILLERY (skipping)" + fi + + STATUS="pass" + if [ "$APP_PASS" = "false" ]; then + STATUS="fail" + OVERALL_PASS=false + fi + + jq --arg a "$APP" --arg s "$STATUS" '. + {($a): $s}' \ + /tmp/test-summary.json > /tmp/test-summary-tmp.json + mv /tmp/test-summary-tmp.json /tmp/test-summary.json +done + +echo "" +echo "=========================================" +echo " Test Summary" +echo "=========================================" +jq '.' /tmp/test-summary.json +tee "$TEST_SUMMARY_PATH" < /tmp/test-summary.json + +if [ "$OVERALL_PASS" = "false" ]; then + echo "" + echo "FAILED: One or more test phases failed." + exit 1 +fi diff --git a/tasks/run-stack-tests.yaml b/tasks/run-stack-tests.yaml index cc39f72..f3ebbc0 100644 --- a/tasks/run-stack-tests.yaml +++ b/tasks/run-stack-tests.yaml @@ -22,6 +22,9 @@ spec: Artillery) with the intercept header attached. For intercepted apps this validates the PR build directly. For non-intercepted apps this validates they still work with the header present. + + Runner logic lives in scripts/run-stack-tests-runners.sh in the + platform repo (cloned into the test-source workspace). params: - name: stack-json type: string @@ -79,338 +82,19 @@ spec: BUILD_APPS="$(params.build-apps)" INTERCEPT="$(params.intercept-header-value)" DEFAULT_NS="$(params.default-namespace)" - - HEADER_NAME="" - HEADER_VAL="" - BAGGAGE_KEY=$(echo "$STACK_JSON" | jq -r '.propagation."baggage-key" // "dev-session"') - if [ -n "$INTERCEPT" ]; then - HEADER_NAME=$(echo "$INTERCEPT" | cut -d: -f1) - HEADER_VAL=$(echo "$INTERCEPT" | cut -d: -f2-) - fi - TESTS_TO_RUN="$(params.tests-to-run)" UNMAPPED_AREA="$(params.unmapped-area)" APPS_TO_TEST="$(params.apps-to-test)" - - echo '{}' > /tmp/test-summary.json - OVERALL_PASS=true - - if [ -n "$UNMAPPED_AREA" ] && [ -z "$TESTS_TO_RUN" ]; then - echo "" - echo "####################################################" - echo " NO MAPPED REGRESSION for area: $UNMAPPED_AREA" - echo " This area needs regression tests to be built." - echo "####################################################" - echo "" - jq --arg area "$UNMAPPED_AREA" \ - '. + {"unmapped-area": $area, "message": "No mapped regression; area needs tests"}' \ - /tmp/test-summary.json > /tmp/test-summary-tmp.json - mv /tmp/test-summary-tmp.json /tmp/test-summary.json - cat /tmp/test-summary.json | tee $(results.test-summary.path) - exit 0 - fi - - FILTERED=false - HAS_E2E=false - if [ -n "$TESTS_TO_RUN" ]; then - FILTERED=true - echo "" - echo "####################################################" - echo " FILTERED TEST RUN (from test-plan graph)" - echo " Tests: $TESTS_TO_RUN" - echo " Apps: $APPS_TO_TEST" - echo "####################################################" - echo "" - if echo "$TESTS_TO_RUN" | grep -q "e2e/"; then - HAS_E2E=true - fi - fi - - app_in_test_plan() { - [ "$FILTERED" = "false" ] && return 0 - [ -z "$APPS_TO_TEST" ] && return 0 - echo ",$APPS_TO_TEST," | grep -q ",$1," - } - - # =========================================================== - # Phase 1: E2E through entry point - # =========================================================== - if [ "$FILTERED" = "true" ] && [ "$HAS_E2E" = "false" ]; then - echo "" - echo "####################################################" - echo " PHASE 1: SKIPPED (no e2e tests in test plan)" - echo "####################################################" - jq '. + {"e2e-entry-point": "skipped (no e2e in plan)"}' \ - /tmp/test-summary.json > /tmp/test-summary-tmp.json - mv /tmp/test-summary-tmp.json /tmp/test-summary.json - else - echo "" - echo "####################################################" - echo " PHASE 1: E2E test through entry point ($ENTRY_APP)" - echo "####################################################" - echo "" - echo " Chain: $CHAIN" - echo " Intercepted: $BUILD_APPS" - echo " Header: $HEADER_NAME=$HEADER_VAL" - echo "" - - ENTRY_NS=$(echo "$STACK_JSON" | jq -r --arg a "$ENTRY_APP" --arg d "$DEFAULT_NS" \ - '(.defaults.namespace // $d) as $dn | .apps[]|select(.name==$a)|.namespace // $dn') - ENTRY_SPORT=$(echo "$STACK_JSON" | jq -r --arg a "$ENTRY_APP" \ - '(.defaults."service-port" // "80") as $dp | .apps[]|select(.name==$a)|."service-port" // $dp') - ENTRY_URL="http://${ENTRY_APP}.${ENTRY_NS}.svc.cluster.local:${ENTRY_SPORT}" - - echo " Entry URL: $ENTRY_URL" - echo "" - - # Send request through the full chain with the intercept header - E2E_PASS=true - E2E_RESULT=$(curl -s \ - -H "${HEADER_NAME}: ${HEADER_VAL}" \ - -H "baggage: ${BAGGAGE_KEY}=${HEADER_VAL}" \ - --connect-timeout 15 --max-time 60 \ - "$ENTRY_URL" 2>/dev/null || echo '{"error":"unreachable"}') - - echo " Response:" - echo "$E2E_RESULT" | jq '.' 2>/dev/null || echo " $E2E_RESULT" - echo "" - - # Verify each hop in the chain was reached - HOP=0 - for APP in $CHAIN; do - HOP=$((HOP + 1)) - IS_INTERCEPTED=false - for BA in $BUILD_APPS; do - [ "$BA" = "$APP" ] && IS_INTERCEPTED=true - done - - MARKER="" - $IS_INTERCEPTED && MARKER=" [INTERCEPTED → PR build]" - - if echo "$E2E_RESULT" | grep -qi "$APP"; then - echo " HOP $HOP ($APP)${MARKER}: REACHED" - else - echo " HOP $HOP ($APP)${MARKER}: NOT CONFIRMED" - fi - done - - # Verify the intercept header propagated - if echo "$E2E_RESULT" | grep -qi "$HEADER_VAL"; then - echo "" - echo " Header value '$HEADER_VAL' found in response" - fi - - echo "" - if [ "$E2E_PASS" = "true" ]; then - echo " E2E: PASS" - else - echo " E2E: FAIL" - OVERALL_PASS=false - fi - - jq --arg s "$([ "$E2E_PASS" = "true" ] && echo pass || echo fail)" \ - '. + {"e2e-entry-point": $s}' \ - /tmp/test-summary.json > /tmp/test-summary-tmp.json - mv /tmp/test-summary-tmp.json /tmp/test-summary.json - - # Also run entry app's Postman/Playwright tests through the full chain - ENTRY_POSTMAN=$(echo "$STACK_JSON" | jq -r --arg a "$ENTRY_APP" \ - '.apps[]|select(.name==$a)|.tests.postman // ""') - ENTRY_PLAYWRIGHT=$(echo "$STACK_JSON" | jq -r --arg a "$ENTRY_APP" \ - '.apps[]|select(.name==$a)|.tests.playwright // ""') - - if [ -n "$ENTRY_POSTMAN" ] && [ -f "$ENTRY_POSTMAN" ]; then - echo "" - echo " [e2e/postman] Running entry-point collection: $ENTRY_POSTMAN" - NEWMAN_ARGS="newman run $ENTRY_POSTMAN --env-var baseUrl=$ENTRY_URL --reporters cli,json" - if [ -n "$HEADER_NAME" ]; then - NEWMAN_ARGS="$NEWMAN_ARGS --global-var ${HEADER_NAME}=${HEADER_VAL}" - fi - if eval $NEWMAN_ARGS; then - echo " [e2e/postman] PASS" - else - echo " [e2e/postman] FAIL" - OVERALL_PASS=false - fi - fi - - if [ -n "$ENTRY_PLAYWRIGHT" ] && [ -d "$ENTRY_PLAYWRIGHT" ]; then - echo "" - echo " [e2e/playwright] Running entry-point e2e tests: $ENTRY_PLAYWRIGHT" - PACKAGE_DIR=$(dirname "$ENTRY_PLAYWRIGHT") - if [ -f "${PACKAGE_DIR}/package.json" ]; then - cd "$PACKAGE_DIR" - npm install --legacy-peer-deps 2>/dev/null || true - export BASE_URL="$ENTRY_URL" - if [ -n "$HEADER_NAME" ]; then - export INTERCEPT_HEADER_NAME="$HEADER_NAME" - export INTERCEPT_HEADER_VALUE="$HEADER_VAL" - fi - REL_PATH=$(echo "$ENTRY_PLAYWRIGHT" | sed "s|^${PACKAGE_DIR}/||") - if npx playwright test "$REL_PATH" --reporter=list 2>/dev/null; then - echo " [e2e/playwright] PASS" - else - echo " [e2e/playwright] FAIL" - OVERALL_PASS=false - fi - cd "$(workspaces.test-source.path)" - fi - fi - - fi - - # =========================================================== - # Phase 2: Per-app tests - # =========================================================== - echo "" - echo "####################################################" - echo " PHASE 2: Per-app tests" - echo "####################################################" - - for APP in $APP_LIST; do - if [ "$APP" = "$ENTRY_APP" ]; then - echo "" - echo " Skipping $APP (already tested in e2e phase)" - continue - fi - - if ! app_in_test_plan "$APP"; then - echo "" - echo " Skipping $APP (not in test plan)" - jq --arg a "$APP" '. + {($a): "skipped (not in plan)"}' \ - /tmp/test-summary.json > /tmp/test-summary-tmp.json - mv /tmp/test-summary-tmp.json /tmp/test-summary.json - continue - fi - - echo "" - echo "=========================================" - echo " Testing: $APP" - - IS_INTERCEPTED=false - for BA in $BUILD_APPS; do - [ "$BA" = "$APP" ] && IS_INTERCEPTED=true - done - $IS_INTERCEPTED && echo " (INTERCEPTED — hitting PR build via Telepresence)" - $IS_INTERCEPTED || echo " (normal cluster deployment)" - - echo "=========================================" - - NS=$(echo "$STACK_JSON" | jq -r --arg a "$APP" --arg d "$DEFAULT_NS" \ - '(.defaults.namespace // $d) as $dn | .apps[]|select(.name==$a)|.namespace // $dn') - SPORT=$(echo "$STACK_JSON" | jq -r --arg a "$APP" \ - '(.defaults."service-port" // "80") as $dp | .apps[]|select(.name==$a)|."service-port" // $dp') - SERVICE_URL="http://${APP}.${NS}.svc.cluster.local:${SPORT}" - - POSTMAN=$(echo "$STACK_JSON" | jq -r --arg a "$APP" \ - '.apps[]|select(.name==$a)|.tests.postman // ""') - PLAYWRIGHT=$(echo "$STACK_JSON" | jq -r --arg a "$APP" \ - '.apps[]|select(.name==$a)|.tests.playwright // ""') - ARTILLERY=$(echo "$STACK_JSON" | jq -r --arg a "$APP" \ - '.apps[]|select(.name==$a)|.tests.artillery // ""') - - APP_PASS=true - - # --- Postman / Newman --- - if [ -n "$POSTMAN" ] && [ -f "$POSTMAN" ]; then - echo " [postman] Running: $POSTMAN" - NEWMAN_ARGS="newman run $POSTMAN --env-var baseUrl=$SERVICE_URL --reporters cli,json" - if [ -n "$HEADER_NAME" ]; then - NEWMAN_ARGS="$NEWMAN_ARGS --global-var ${HEADER_NAME}=${HEADER_VAL}" - fi - if eval $NEWMAN_ARGS; then - echo " [postman] PASS" - else - echo " [postman] FAIL" - APP_PASS=false - fi - elif [ -n "$POSTMAN" ]; then - echo " [postman] Collection not found: $POSTMAN (skipping)" - fi - - # --- Playwright --- - if [ -n "$PLAYWRIGHT" ] && [ -d "$PLAYWRIGHT" ]; then - echo " [playwright] Running: $PLAYWRIGHT" - PACKAGE_DIR=$(dirname "$PLAYWRIGHT") - if [ -f "${PACKAGE_DIR}/package.json" ]; then - cd "$PACKAGE_DIR" - npm install --legacy-peer-deps 2>/dev/null || true - export BASE_URL="$SERVICE_URL" - if [ -n "$HEADER_NAME" ]; then - export INTERCEPT_HEADER_NAME="$HEADER_NAME" - export INTERCEPT_HEADER_VALUE="$HEADER_VAL" - fi - REL_PATH=$(echo "$PLAYWRIGHT" | sed "s|^${PACKAGE_DIR}/||") - if npx playwright test "$REL_PATH" --reporter=list 2>/dev/null; then - echo " [playwright] PASS" - else - echo " [playwright] FAIL" - APP_PASS=false - fi - cd "$(workspaces.test-source.path)" - else - echo " [playwright] No package.json found (skipping)" - fi - elif [ -n "$PLAYWRIGHT" ]; then - echo " [playwright] Test dir not found: $PLAYWRIGHT (skipping)" - fi - - # --- Artillery --- - if [ -n "$ARTILLERY" ] && [ -f "$ARTILLERY" ]; then - echo " [artillery] Running: $ARTILLERY" - ARTILLERY_ARGS="" - if [ -n "$HEADER_NAME" ]; then - cat > /tmp/artillery-override.yml < /tmp/test-summary-tmp.json - mv /tmp/test-summary-tmp.json /tmp/test-summary.json - done - - # =========================================================== - # Summary - # =========================================================== - echo "" - echo "=========================================" - echo " Test Summary" - echo "=========================================" - cat /tmp/test-summary.json | jq '.' - cat /tmp/test-summary.json | tee $(results.test-summary.path) - - if [ "$OVERALL_PASS" = "false" ]; then - echo "" - echo "FAILED: One or more test phases failed." + TEST_SOURCE="$(workspaces.test-source.path)" + TEST_SUMMARY_PATH="$(results.test-summary.path)" + export STACK_JSON APP_LIST ENTRY_APP CHAIN BUILD_APPS INTERCEPT \ + DEFAULT_NS TESTS_TO_RUN UNMAPPED_AREA APPS_TO_TEST \ + TEST_SOURCE TEST_SUMMARY_PATH + + RUNNERS="${TEST_SOURCE}/scripts/run-stack-tests-runners.sh" + if [ ! -f "$RUNNERS" ]; then + echo "ERROR: run-stack-tests runners script missing: $RUNNERS" >&2 exit 1 fi + # shellcheck disable=SC1090 + . "$RUNNERS"