diff --git a/.github/workflows/graph-gui-newman.yml b/.github/workflows/graph-gui-newman.yml new file mode 100644 index 0000000..0212de1 --- /dev/null +++ b/.github/workflows/graph-gui-newman.yml @@ -0,0 +1,127 @@ +name: Graph and GUI Newman + +on: + pull_request: + paths: + - ".github/workflows/graph-gui-newman.yml" + - "scripts/run-gui-newman.sh" + - "scripts/run-cluster-ci.sh" + - "scripts/install-neo4j-kind.sh" + - "scripts/run-orchestrator-tests.sh" + - "tests/postman/graph-tests.json" + - "tests/postman/management-gui-tests.json" + - "management-gui/backend/**" + - "orchestrator/**" + workflow_dispatch: + schedule: + # Weekly graph + GUI API evidence on the default branch. + - cron: "11 8 * * 4" + +permissions: + contents: read + +concurrency: + group: graph-gui-newman-${{ github.ref }} + cancel-in-progress: false + +jobs: + gui-api: + name: GUI Newman vs live backend + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 + with: + python-version: "3.12" + + - name: Set up Node + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: "22" + + - name: Bootstrap Python venv + run: ./scripts/bootstrap-regression-venv.sh + + - name: Install Newman + run: npm install -g newman@6.2.1 + + - name: Run GUI Newman + run: | + set -o pipefail + bash scripts/run-gui-newman.sh 2>&1 | tee gui-newman.log + + - name: Upload GUI Newman evidence + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: gui-newman-${{ github.sha }} + path: gui-newman.log + if-no-files-found: warn + retention-days: 30 + + graph-api: + name: Graph Newman vs Neo4j + runs-on: ubuntu-latest + timeout-minutes: 90 + env: + KIND_VERSION: v0.27.0 + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 + with: + python-version: "3.12" + + - name: Set up Node + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: "22" + + - name: Install Kind and Newman + run: | + curl -fsSL -o /tmp/kind "https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-linux-amd64" + echo "a6875aaea358acf0ac07786b1a6755d08fd640f4c79b7a2e46681cc13f49a04b /tmp/kind" | sha256sum -c - + chmod +x /tmp/kind + sudo mv /tmp/kind /usr/local/bin/kind + npm install -g newman@6.2.1 + ./scripts/bootstrap-regression-venv.sh + + - name: Run graph Newman + env: + CLUSTER_CI_GIT_REVISION: ${{ github.sha }} + CLUSTER_CI_GIT_URL: https://github.com/${{ github.repository }}.git + run: | + set -o pipefail + bash scripts/run-cluster-ci.sh --skip-isolation --with-graph 2>&1 | tee graph-newman.log + + - name: Collect graph failure diagnostics + if: failure() + run: | + kubectl logs deployment/tekton-dag-orchestrator -n tekton-pipelines \ + --all-containers --tail=-1 > orchestrator.log 2>&1 || true + kubectl get pods,svc -n tekton-pipelines -l app=graph-db -o yaml \ + > graph-db.yaml 2>&1 || true + kubectl logs -n tekton-pipelines -l app=graph-db \ + --all-containers=true --prefix=true \ + > graph-db.log 2>&1 || true + + - name: Upload graph Newman evidence + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: graph-newman-${{ github.sha }} + path: | + graph-newman.log + orchestrator.log + graph-db.yaml + graph-db.log + if-no-files-found: warn + retention-days: 30 diff --git a/docs/REGRESSION.md b/docs/REGRESSION.md index d57e2e6..2d2f9f1 100644 --- a/docs/REGRESSION.md +++ b/docs/REGRESSION.md @@ -46,7 +46,8 @@ traffic artifact as verification. | **C — Tekton DAG pipeline** | [scripts/verify-dag-phase2.sh](../scripts/verify-dag-phase2.sh) — **`stack-dag-verify`** to **Succeeded** | **Auto** if `kubectl` works and `Pipeline/stack-dag-verify` exists in `NAMESPACE`. **Skipped** when [run-full-test-and-verify-results.sh](../scripts/run-full-test-and-verify-results.sh) will run (it already includes Phase 2). **Forced failure if missing** with `--require-dag-verify`. **Off** with `--skip-dag-verify` or `REGRESSION_DAG_VERIFY=skip`. | | **D — Cluster API** | Newman via [run-orchestrator-tests.sh](../scripts/run-orchestrator-tests.sh) `--all` | **Auto** if orchestrator `Service` exists and `newman` on `PATH`; **required** with `--cluster` | | **E — Results + DB** | [run-full-test-and-verify-results.sh](../scripts/run-full-test-and-verify-results.sh) | **Auto** if `tekton-results-api` exists; **forced** with `--with-results-verify`; **off** with `--skip-results-verify`; strict weekly/dispatch automation uses `run-regression-agent-full.sh` | -| **F — GUI Postman** | [management-gui-tests.json](../tests/postman/management-gui-tests.json) vs `http://localhost:5000` | `--gui-newman` | +| **F — GUI Postman** | [management-gui-tests.json](../tests/postman/management-gui-tests.json) vs a live Flask backend | `--gui-newman` or `scripts/run-gui-newman.sh`; weekly/dispatch in `graph-gui-newman.yml` | +| **F — Graph Postman** | [graph-tests.json](../tests/postman/graph-tests.json) vs orchestrator + Neo4j | `run-cluster-ci.sh --with-graph`; weekly/dispatch in `graph-gui-newman.yml` | | **G — Full Kind E2E** | [run-all-setup-and-test.sh](../scripts/run-all-setup-and-test.sh) | `--kind-e2e` | | **H — Intercept product E2E** | [run-product-intercept-e2e.sh](../scripts/run-product-intercept-e2e.sh) via authenticated orchestrator API | Weekly/dispatch matrix in `intercept-e2e.yml`; requires repository secret `E2E_GIT_SSH_PRIVATE_KEY` with read access to application repos | diff --git a/libs/tekton-dag-common/tests/test_m17_graph_gui_newman.py b/libs/tekton-dag-common/tests/test_m17_graph_gui_newman.py new file mode 100644 index 0000000..76b3219 --- /dev/null +++ b/libs/tekton-dag-common/tests/test_m17_graph_gui_newman.py @@ -0,0 +1,42 @@ +"""Static acceptance checks for the M17.11 graph and GUI Newman suites.""" + +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[3] + + +def test_graph_gui_workflow_is_scheduled_and_retains_diagnostics(): + workflow = (ROOT / ".github/workflows/graph-gui-newman.yml").read_text() + + assert "pull_request:" in workflow + assert '".github/workflows/graph-gui-newman.yml"' in workflow + assert '"scripts/run-gui-newman.sh"' in workflow + assert '"scripts/install-neo4j-kind.sh"' in workflow + assert '"tests/postman/graph-tests.json"' in workflow + assert '"tests/postman/management-gui-tests.json"' in workflow + assert "workflow_dispatch:" in workflow + assert "schedule:" in workflow + assert "run-gui-newman.sh" in workflow + assert "run-cluster-ci.sh --skip-isolation --with-graph" in workflow + assert "if: failure()" in workflow + assert "graph-db.log" in workflow + assert "if: always()" in workflow + assert "actions/upload-artifact@" in workflow + + +def test_gui_newman_script_starts_live_backend(): + script = (ROOT / "scripts/run-gui-newman.sh").read_text() + + assert "management-gui-tests.json" in script + assert "python3 app.py" in script + assert "/api/health" in script + assert "apiMutationToken=$API_MUTATION_TOKEN" in script + assert "die \"management GUI backend did not become healthy" in script + + +def test_cluster_ci_can_require_graph_newman(): + script = (ROOT / "scripts/run-cluster-ci.sh").read_text() + + assert "--with-graph)" in script + assert "install-neo4j-kind.sh" in script + assert "newman_args+=(--all)" in script diff --git a/management-gui/backend/k8s_client.py b/management-gui/backend/k8s_client.py index 200e4a1..76878d8 100644 --- a/management-gui/backend/k8s_client.py +++ b/management-gui/backend/k8s_client.py @@ -60,8 +60,8 @@ def list_configmap_names(context, namespace): def list_pipelineruns(context, namespace, limit=50, label_selector=""): """List recent PipelineRuns in a namespace.""" - api = get_api(context) try: + api = get_api(context) result = api.list_namespaced_custom_object( group="tekton.dev", version="v1", @@ -74,6 +74,9 @@ def list_pipelineruns(context, namespace, limit=50, label_selector=""): except ApiException as e: logger.error("Failed to list PipelineRuns: %s", e.reason) return [] + except Exception as e: + logger.debug("list_pipelineruns: %s", e) + return [] def get_pipelinerun(context, namespace, name): @@ -134,8 +137,8 @@ def create_stackrun(context, namespace, manifest): def list_stackruns(context, namespace, limit=50, label_selector=""): """List recent StackRuns in a namespace.""" - api = get_api(context) try: + api = get_api(context) result = api.list_namespaced_custom_object( group="tektondag.io", version="v1alpha1", @@ -148,6 +151,9 @@ def list_stackruns(context, namespace, limit=50, label_selector=""): except ApiException as e: logger.error("Failed to list StackRuns: %s", e.reason) return [] + except Exception as e: + logger.debug("list_stackruns: %s", e) + return [] def get_stackrun(context, namespace, name): diff --git a/management-gui/backend/routes/stacks.py b/management-gui/backend/routes/stacks.py index 77a51c8..dab7df8 100644 --- a/management-gui/backend/routes/stacks.py +++ b/management-gui/backend/routes/stacks.py @@ -41,10 +41,10 @@ def get_dag(team, stack_file): return jsonify({"error": f"Unknown team: {team}"}), 404 allowed = team_cfg.get("stacks", []) + resolver = current_app.config["STACK_RESOLVER"] if allowed and stack_file not in allowed: - return jsonify({"error": f"Stack {stack_file} not allowed for team {team}"}), 403 + return jsonify({"error": f"Stack not found: {stack_file}"}), 404 - resolver = current_app.config["STACK_RESOLVER"] dag = resolver.get_dag(stack_file) if dag is None: return jsonify({"error": f"Stack not found: {stack_file}"}), 404 diff --git a/management-gui/backend/tests/test_k8s_client.py b/management-gui/backend/tests/test_k8s_client.py index e913888..926e1c6 100644 --- a/management-gui/backend/tests/test_k8s_client.py +++ b/management-gui/backend/tests/test_k8s_client.py @@ -38,6 +38,13 @@ def test_list_pipelineruns_api_error(mock_get_api): assert result == [] +@patch("k8s_client.get_api") +def test_list_pipelineruns_returns_empty_when_client_setup_fails(mock_get_api): + mock_get_api.side_effect = k8s_client.config.ConfigException("no kubeconfig") + + assert k8s_client.list_pipelineruns("ctx", "ns") == [] + + @patch("k8s_client.get_api") def test_get_pipelinerun(mock_get_api): mock_api = MagicMock() @@ -260,6 +267,13 @@ def test_create_stackrun_reraises_api_error(mock_get_api): assert exc_info.value is error +@patch("k8s_client.get_api") +def test_list_stackruns_returns_empty_when_client_setup_fails(mock_get_api): + mock_get_api.side_effect = RuntimeError("client setup failed") + + assert k8s_client.list_stackruns("ctx", "apps") == [] + + @patch("k8s_client.get_api") def test_list_stackruns_returns_empty_on_api_error(mock_get_api): from kubernetes.client.rest import ApiException diff --git a/management-gui/backend/tests/test_routes.py b/management-gui/backend/tests/test_routes.py index 7c4de0e..a2daea8 100644 --- a/management-gui/backend/tests/test_routes.py +++ b/management-gui/backend/tests/test_routes.py @@ -121,6 +121,12 @@ def test_get_dag(client): assert "apps" in data or "nodes" in data or "name" in data +def test_get_dag_unknown_stack_returns_404(client): + resp = client.get("/api/teams/default/stacks/stacks/nonexistent.yaml/dag") + assert resp.status_code == 404 + assert "error" in resp.get_json() + + @patch("k8s_client.list_configmap_names") @patch("k8s_client.list_secret_names") def test_injection_status(mock_secrets, mock_cms, client): diff --git a/milestones/milestone-17.md b/milestones/milestone-17.md index 224b58c..3a09265 100644 --- a/milestones/milestone-17.md +++ b/milestones/milestone-17.md @@ -140,6 +140,8 @@ Work is intentionally paused to preserve the remaining implementation budget. - Run Neo4j graph Newman on a scheduled cadence and GUI Newman against a live backend. - Acceptance: both collections report zero failed assertions in CI. + - Evidence: `graph-gui-newman.yml` now schedules GUI Flask Newman and Kind + `--with-graph` Newman. Live green jobs still required. - [ ] **M17.12 Test Helm and representation synchronization** - Test chart packaging/rendering, CRD copies, Stack YAML→CR conversion, and @@ -213,4 +215,5 @@ Work is intentionally paused to preserve the remaining implementation budget. | 2026-09-15 | M17.9 demo validation | Git LFS recordings; stream, A/V drift, narration, and OCR checks | Run 34975666059 passed | | 2026-09-15 | M17.10 Newman execution truth | Current-run StackRun reconciliation and bootstrap fetch-source checkpoint | Run 34989095946 passed; final PR revision passed all checks | | 2026-09-15 | Pause checkpoint | Completed M17.1–M17.3 and M17.5–M17.10; open M17.4 and M17.11–M17.21 | Resume instructions recorded above | +| 2026-09-16 | M17.11 graph and GUI Newman automation | Scheduled/manual/PR-path workflow; live Flask GUI runner; cluster `--with-graph`; list endpoints tolerate missing kubeconfig | Automation added; first live graph + GUI Newman run still required | diff --git a/scripts/run-cluster-ci.sh b/scripts/run-cluster-ci.sh index 777b25d..d3e774b 100755 --- a/scripts/run-cluster-ci.sh +++ b/scripts/run-cluster-ci.sh @@ -10,6 +10,7 @@ # Usage: # ./scripts/run-cluster-ci.sh # ./scripts/run-cluster-ci.sh --skip-newman +# ./scripts/run-cluster-ci.sh --skip-isolation --with-graph # ./scripts/run-cluster-ci.sh --with-operator # ./scripts/run-cluster-ci.sh --skip-operator # ./scripts/run-cluster-ci.sh --isolation-repeats 3 diff --git a/scripts/run-gui-newman.sh b/scripts/run-gui-newman.sh new file mode 100755 index 0000000..4de51e6 --- /dev/null +++ b/scripts/run-gui-newman.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +# Start the management GUI Flask backend and run Newman against it. +# +# Usage: +# ./scripts/run-gui-newman.sh +# +# Env: +# API_MUTATION_TOKEN Bearer token for mutation routes (generated if unset) +# GUI_NEWMAN_PORT Local Flask port (default 5000) +# TEAM_NAME Team filter for the backend (default *) +set -euo pipefail +[ -z "${BASH_VERSION:-}" ] && exec bash "$0" "$@" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=common.sh +source "$SCRIPT_DIR/common.sh" +cd "$REPO_ROOT" + +need newman +need python3 +need curl + +if [[ -x "$REPO_ROOT/.venv/bin/python3" ]]; then + export PATH="$REPO_ROOT/.venv/bin:$PATH" +fi + +API_MUTATION_TOKEN="${API_MUTATION_TOKEN:-$(openssl rand -hex 32)}" +export API_MUTATION_TOKEN +PORT="${GUI_NEWMAN_PORT:-5000}" +export PORT +export TEAM_NAME="${TEAM_NAME:-*}" +COLLECTION="${REPO_ROOT}/tests/postman/management-gui-tests.json" + +GUI_PID="" +cleanup() { + if [[ -n "$GUI_PID" ]] && kill -0 "$GUI_PID" 2>/dev/null; then + kill "$GUI_PID" 2>/dev/null || true + wait "$GUI_PID" 2>/dev/null || true + fi +} +trap cleanup EXIT + +echo "==============================================" +echo " Management GUI Newman" +echo " baseUrl=http://127.0.0.1:${PORT}" +echo "==============================================" + +free_tcp_port "$PORT" 1 +( + cd "$REPO_ROOT/management-gui/backend" + exec python3 app.py +) & +GUI_PID=$! + +ready=false +for _ in $(seq 1 30); do + if ! kill -0 "$GUI_PID" 2>/dev/null; then + die "management GUI backend exited before becoming healthy" + fi + if curl -sf "http://127.0.0.1:${PORT}/api/health" | grep -q '"ok"'; then + ready=true + break + fi + sleep 1 +done +[[ "$ready" == "true" ]] || die "management GUI backend did not become healthy on port ${PORT}" + +echo "=== Running Newman: management-gui-tests.json ===" +newman run "$COLLECTION" \ + --env-var "baseUrl=http://127.0.0.1:${PORT}" \ + --env-var "apiMutationToken=$API_MUTATION_TOKEN" \ + --reporters cli \ + --color on + +echo "=== GUI Newman passed ===" diff --git a/scripts/run-regression.sh b/scripts/run-regression.sh index b487211..4600a13 100755 --- a/scripts/run-regression.sh +++ b/scripts/run-regression.sh @@ -82,7 +82,6 @@ fi if [[ "$LOCAL_ONLY" == "true" ]]; then SKIP_CLUSTER=true SKIP_PLAYWRIGHT=true - RUN_GUI_NEWMAN=false RUN_KIND_E2E=false RESULTS_VERIFY_MODE=skip DAG_VERIFY_MODE=skip