ci: narrow benchmark checks and add documentation validation gate

.github/scripts/ci_changes.py

@@ -15,10 +15,28 @@
     "shell",
     "wheels",
     "benchmarks",
+    "docs",
 )
 
 CPP_SUFFIXES = (".c", ".cc", ".cpp", ".h", ".hpp")
 
+BENCHMARK_PYTHON_PATHS = frozenset(
+    f"benchmarks/{name}.py"
+    for name in (
+        "build_indexes",
+        "download_datasets",
+        "install_baselines",
+        "plot_homepage",
+        "plot_results",
+        "query_index",
+        "reproduce",
+        "run",
+        "run_kmeans",
+        "run_sweeps",
+        "update_indexing",
+    )
+)
+
 # Keep broad pull-request checks for routing edits, then avoid repeating
 # their builds after a CI-only merge into an already released main branch.
 CI_ROUTING_PATHS = frozenset(
@@ -34,10 +52,15 @@
 def classify(paths):
     selected = set()
     for path in paths:
-        if path.startswith("docs/") or path.endswith(".md"):
+        if path.startswith("docs/"):
+            selected.add("docs")
+            continue
+        if path.endswith(".md"):
+            continue
+        if path == ".github/workflows/docs.yml":
+            selected.update(("docs", "python_quality"))
             continue
         if path.startswith(".github/ISSUE_TEMPLATE/") or path in (
-            ".github/workflows/docs.yml",
             ".github/release.yml",
             ".github/CODEOWNERS",
             "LICENSE",
@@ -56,7 +79,7 @@ def classify(paths):
                 selected.add("cpp_format")
             elif path.endswith(".py"):
                 selected.add("python_quality")
-        elif path == "benchmarks/plot_homepage.py":
+        elif path in BENCHMARK_PYTHON_PATHS:
             selected.add("python_quality")
         elif path in (
             "sample/run_cpp.sh",
@@ -113,6 +136,8 @@ def classify(paths):
                 selected.add("cpp_format")
         elif path == "tests/python/test_cpp_clustering_examples.py":
             selected.update(("cpp", "python_quality"))
+        elif path == "tests/python/test_benchmark_protocol.py":
+            selected.add("python_quality")
         elif path.startswith("tests/python/"):
             selected.update(("python_build", "wheels"))
             if path == "tests/python/conftest.py":
@@ -194,7 +219,7 @@ def detect(event_name, event, ref_name):
             )
             # Reuse normal path rules: accompanying lint/support edits must keep
             # their checks, while build-related or unknown inputs remain broad.
-            if companions <= {"python_quality", "shell"}:
+            if companions <= {"python_quality", "shell", "docs"}:
                 return companions | {"python_quality"}
     return selected
 
@@ -208,6 +233,4 @@ def detect(event_name, event, ref_name):
     with open(os.environ["GITHUB_OUTPUT"], "a") as output:
         for group in GROUPS:
             print(f"{group}={str(group in selected).lower()}", file=output)
-    print(
-        "Selected checks:", ", ".join(sorted(selected)) or "none (documentation only)"
-    )
+    print("Selected checks:", ", ".join(sorted(selected)) or "none")

.github/scripts/test_ci_aggregates.py

@@ -28,6 +28,7 @@ def test_matrix_verdicts_and_dependencies(self):
                 "python_build",
                 "PYTHON_SELECTED",
             ),
+            ("docs.yml", "docs-result", "build", "docs", "DOCS_SELECTED"),
         ):
             workflow = (WORKFLOWS / filename).read_text()
             block = re.search(

.github/scripts/test_ci_changes.py

@@ -51,10 +51,13 @@ def test_docs_only_skip_builds(self):
                     "docs/mkdocs.yml",
                     "docs/requirements.txt",
                     "docs/docs/assets/img/plot.png",
-                    ".github/workflows/docs.yml",
                 ]
             ),
-            set(),
+            {"docs"},
+        )
+        self.assertEqual(classify(["README.md", "sample/python/README.md"]), set())
+        self.assertEqual(
+            classify([".github/workflows/docs.yml"]), {"docs", "python_quality"}
         )
 
     def test_known_support_paths_do_not_select_unrelated_builds(self):
@@ -259,7 +262,7 @@ def test_ci_routing_changes_run_full_pr_ci_but_not_repeat_on_tagged_main(self):
                 set(GROUPS),
             )
         for changed_paths, tags, expected in (
-            (paths, "v0.3.3", {"python_quality"}),
+            (paths, "v0.3.3", {"python_quality", "docs"}),
             (paths, "", set(GROUPS)),
             (paths + ["src/simd/dispatch.cpp"], "v0.3.3", set(GROUPS)),
         ):
@@ -291,6 +294,7 @@ def test_tagged_main_routing_changes_preserve_lightweight_companion_checks(self)
             (["scripts/check-python.sh"], {"python_quality", "shell"}),
             (["sample/run_cpp.sh"], {"python_quality", "shell"}),
             (["benchmarks/plot_homepage.py"], {"python_quality"}),
+            (["benchmarks/build_indexes.py"], {"python_quality"}),
             ([".github/ISSUE_TEMPLATE/config.yml"], {"python_quality"}),
             (["src/core/api.cpp"], set(GROUPS)),
             (["python_bindings/__init__.py"], set(GROUPS)),
@@ -305,7 +309,7 @@ def test_tagged_main_routing_changes_preserve_lightweight_companion_checks(self)
         ):
             changed = "\0".join(routing + companions) + "\0"
             for event, ref, tags, result in (
-                ("push", "main", "v0.7.0", expected),
+                ("push", "main", "v0.7.0", expected | {"docs"}),
                 ("push", "main", "", set(GROUPS)),
                 ("push", "master", None, set(GROUPS)),
                 ("pull_request", "148/merge", None, set(GROUPS)),

.github/scripts/test_ci_workflow_contracts.py

@@ -0,0 +1,127 @@
+"""Check lightweight CI coverage and workflow scheduling contracts."""
+
+import json
+import os
+import re
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+from ci_changes import GROUPS, classify
+
+ROOT = Path(__file__).resolve().parents[2]
+
+
+class WorkflowContractsTest(unittest.TestCase):
+    def test_benchmark_support_changes_keep_protocol_coverage_without_builds(self):
+        for path in sorted((ROOT / "benchmarks").glob("*.py")):
+            with self.subTest(path=path.name):
+                self.assertEqual(
+                    classify([str(path.relative_to(ROOT))]), {"python_quality"}
+                )
+        self.assertEqual(
+            classify(["tests/python/test_benchmark_protocol.py"]), {"python_quality"}
+        )
+        self.assertEqual(classify(["benchmarks/unknown.py"]), set(GROUPS))
+        self.assertIn(
+            "cpp",
+            classify(["benchmarks/build_indexes.py", "src/core/api.cpp"]),
+        )
+        workflow = (ROOT / ".github/workflows/python.yml").read_text()
+        quality = workflow.split("  python-quality:\n", 1)[1].split(
+            "  python-build:\n", 1
+        )[0]
+        self.assertIn("tests/python/test_benchmark_protocol.py", quality)
+        self.assertIn("numpy", quality)
+        self.assertIn("pytest", quality)
+        self.assertNotIn("pip install .", quality)
+
+    def test_python_checker_covers_every_benchmark_python_file(self):
+        with tempfile.TemporaryDirectory() as directory:
+            helper = Path(directory) / "ruff"
+            log = Path(directory) / "calls.jsonl"
+            helper.write_text(
+                f"#!{sys.executable}\n"
+                "import json, os, sys\n"
+                "if sys.argv[1:] == ['--version']:\n"
+                "    print('ruff 0.16.1')\n"
+                "else:\n"
+                "    with open(os.environ['RUFF_AUDIT_LOG'], 'a') as stream:\n"
+                "        stream.write(json.dumps(sys.argv[1:]) + '\\n')\n"
+            )
+            helper.chmod(0o755)
+            subprocess.run(
+                ["bash", str(ROOT / "scripts/check-python.sh")],
+                env=dict(os.environ, RUFF=str(helper), RUFF_AUDIT_LOG=str(log)),
+                check=True,
+                capture_output=True,
+                text=True,
+            )
+            calls = [json.loads(line) for line in log.read_text().splitlines()]
+        self.assertEqual(len(calls), 2)
+        for call in calls:
+            covered = set()
+            for argument in call:
+                path = ROOT / argument
+                if path.is_dir():
+                    covered.update(path.rglob("*.py"))
+                elif path.suffix == ".py":
+                    covered.add(path)
+            self.assertLessEqual(set((ROOT / "benchmarks").rglob("*.py")), covered)
+
+    def test_concurrency_groups_distinguish_forks_and_reuse_one_pr(self):
+        def group(template, number, sha, workflow):
+            context = {
+                "github.event.pull_request.number": number,
+                "github.head_ref": "main",
+                "github.sha": sha,
+                "github.workflow": workflow,
+            }
+
+            def expand(match):
+                return str(
+                    next(
+                        context[key.strip()]
+                        for key in match.group(1).split("||")
+                        if context[key.strip()]
+                    )
+                )
+
+            return re.sub(r"\$\{\{(.*?)\}\}", expand, template)
+
+        for filename in ("test.yaml", "python.yml"):
+            text = (ROOT / ".github/workflows" / filename).read_text()
+            template = re.search(r"^  group: (.+)$", text, re.MULTILINE).group(1)
+            with self.subTest(workflow=filename):
+                self.assertNotEqual(
+                    group(template, 201, "a", filename),
+                    group(template, 202, "b", filename),
+                )
+                self.assertEqual(
+                    group(template, 201, "a", filename),
+                    group(template, 201, "b", filename),
+                )
+                self.assertNotEqual(
+                    group(template, 0, "a", filename),
+                    group(template, 0, "b", filename),
+                )
+
+    def test_documentation_summary_is_available_for_non_doc_prs(self):
+        text = (ROOT / ".github/workflows/docs.yml").read_text()
+        triggers = text.split("concurrency:", 1)[0]
+        self.assertNotIn("paths:", triggers)
+        self.assertNotIn("paths-ignore:", triggers)
+        self.assertIn("  pull_request:", triggers)
+        self.assertIn("  workflow_dispatch:", triggers)
+        self.assertIn("uses: ./.github/workflows/changes.yml", text)
+        build = text.split("  build:\n", 1)[1].split("  docs-result:\n", 1)[0]
+        self.assertIn("needs: changes", build)
+        self.assertIn("needs.changes.outputs.docs == 'true'", build)
+        self.assertIn("needs.changes.result != 'success'", build)
+        self.assertEqual(classify(["docs/docs/quick_start.md"]), {"docs"})
+
+
+if __name__ == "__main__":
+    unittest.main()

.github/workflows/changes.yml

@@ -5,6 +5,8 @@ on:
     outputs:
       benchmarks:
         value: ${{ jobs.detect.outputs.benchmarks }}
+      docs:
+        value: ${{ jobs.detect.outputs.docs }}
       cpp:
         value: ${{ jobs.detect.outputs.cpp }}
       cpp_format:
@@ -29,6 +31,7 @@ jobs:
     timeout-minutes: 5
     outputs:
       benchmarks: ${{ steps.paths.outputs.benchmarks }}
+      docs: ${{ steps.paths.outputs.docs }}
       cpp: ${{ steps.paths.outputs.cpp }}
       cpp_format: ${{ steps.paths.outputs.cpp_format }}
       cpp_tidy: ${{ steps.paths.outputs.cpp_tidy }}

.github/workflows/docs.yml

@@ -3,13 +3,7 @@ name: Documentation
 on:
   push:
     branches: ["main"]
-    paths:
-      - "docs/**"
-      - ".github/workflows/docs.yml"
   pull_request:
-    paths:
-      - "docs/**"
-      - ".github/workflows/docs.yml"
   workflow_dispatch:
 
 concurrency:
@@ -20,7 +14,12 @@ permissions:
   contents: read
 
 jobs:
+  changes:
+    uses: ./.github/workflows/changes.yml
+
   build:
+    needs: changes
+    if: ${{ !cancelled() && (needs.changes.result != 'success' || needs.changes.outputs.docs == 'true') }}
     name: Build documentation
     runs-on: ubuntu-latest
     timeout-minutes: 10
@@ -36,6 +35,25 @@ jobs:
       - name: Build documentation
         run: mkdocs build --strict --config-file docs/mkdocs.yml
 
+  docs-result:
+    name: Documentation validation
+    needs: [changes, build]
+    if: ${{ always() }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - name: Require selected documentation build
+        env:
+          CHANGES_RESULT: ${{ needs.changes.result }}
+          DOCS_SELECTED: ${{ needs.changes.outputs.docs }}
+          MATRIX_RESULT: ${{ needs.build.result }}
+        run: |
+          if [ "$CHANGES_RESULT" = success ] && [ "$DOCS_SELECTED" = false ]; then
+            test "$MATRIX_RESULT" = skipped || test "$MATRIX_RESULT" = success
+          else
+            test "$MATRIX_RESULT" = success
+          fi
+
   deploy:
     name: Deploy documentation
     if: github.event_name == 'push'

.github/workflows/python.yml

@@ -7,7 +7,7 @@ on:
     branches: ["main", "master"]
 
 concurrency:
-  group: ${{ github.head_ref || github.sha }}-${{ github.workflow }}
+  group: ${{ github.event.pull_request.number || github.sha }}-${{ github.workflow }}
   cancel-in-progress: true
 
 permissions:
@@ -29,16 +29,17 @@ jobs:
         with:
           python-version: "3.11"
           cache: pip
-      - name: Install Ruff
-        run: python -m pip install "ruff==0.16.1"
+      - name: Install validation dependencies
+        run: python -m pip install "ruff==0.16.1" numpy pytest
       - name: Check Python
         run: ./scripts/check-python.sh
       - name: Check release automation
         run: |
           ruff format --check .github/scripts
           ruff check .github/scripts
           python -m unittest discover -s .github/scripts -p 'test_*.py'
-
+      - name: Test benchmark protocols without native builds
+        run: python -m pytest tests/python/test_benchmark_protocol.py --tb=short -q
 
   python-build:
     needs: changes

.github/workflows/test.yaml

@@ -5,7 +5,7 @@ on:
       - main
   pull_request:
 concurrency:
-  group: ${{ github.head_ref || github.sha }}-${{ github.workflow }}
+  group: ${{ github.event.pull_request.number || github.sha }}-${{ github.workflow }}
   cancel-in-progress: true
 permissions:
   contents: read