diff --git a/.github/scripts/ci_changes.py b/.github/scripts/ci_changes.py index 007e965..a4f9ad4 100644 --- a/.github/scripts/ci_changes.py +++ b/.github/scripts/ci_changes.py @@ -15,10 +15,28 @@ "shell", "wheels", "benchmarks", + "docs", ) CPP_SUFFIXES = (".c", ".cc", ".cpp", ".h", ".hpp") +BENCHMARK_PYTHON_PATHS = frozenset( + f"benchmarks/{name}.py" + for name in ( + "build_indexes", + "download_datasets", + "install_baselines", + "plot_homepage", + "plot_results", + "query_index", + "reproduce", + "run", + "run_kmeans", + "run_sweeps", + "update_indexing", + ) +) + # Keep broad pull-request checks for routing edits, then avoid repeating # their builds after a CI-only merge into an already released main branch. CI_ROUTING_PATHS = frozenset( @@ -34,10 +52,15 @@ def classify(paths): selected = set() for path in paths: - if path.startswith("docs/") or path.endswith(".md"): + if path.startswith("docs/"): + selected.add("docs") + continue + if path.endswith(".md"): + continue + if path == ".github/workflows/docs.yml": + selected.update(("docs", "python_quality")) continue if path.startswith(".github/ISSUE_TEMPLATE/") or path in ( - ".github/workflows/docs.yml", ".github/release.yml", ".github/CODEOWNERS", "LICENSE", @@ -56,7 +79,7 @@ def classify(paths): selected.add("cpp_format") elif path.endswith(".py"): selected.add("python_quality") - elif path == "benchmarks/plot_homepage.py": + elif path in BENCHMARK_PYTHON_PATHS: selected.add("python_quality") elif path in ( "sample/run_cpp.sh", @@ -113,6 +136,8 @@ def classify(paths): selected.add("cpp_format") elif path == "tests/python/test_cpp_clustering_examples.py": selected.update(("cpp", "python_quality")) + elif path == "tests/python/test_benchmark_protocol.py": + selected.add("python_quality") elif path.startswith("tests/python/"): selected.update(("python_build", "wheels")) if path == "tests/python/conftest.py": @@ -194,7 +219,7 @@ def detect(event_name, event, ref_name): ) # Reuse normal path rules: accompanying lint/support edits must keep # their checks, while build-related or unknown inputs remain broad. - if companions <= {"python_quality", "shell"}: + if companions <= {"python_quality", "shell", "docs"}: return companions | {"python_quality"} return selected @@ -208,6 +233,4 @@ def detect(event_name, event, ref_name): with open(os.environ["GITHUB_OUTPUT"], "a") as output: for group in GROUPS: print(f"{group}={str(group in selected).lower()}", file=output) - print( - "Selected checks:", ", ".join(sorted(selected)) or "none (documentation only)" - ) + print("Selected checks:", ", ".join(sorted(selected)) or "none") diff --git a/.github/scripts/test_ci_aggregates.py b/.github/scripts/test_ci_aggregates.py index dbebcfb..2d7375f 100644 --- a/.github/scripts/test_ci_aggregates.py +++ b/.github/scripts/test_ci_aggregates.py @@ -28,6 +28,7 @@ def test_matrix_verdicts_and_dependencies(self): "python_build", "PYTHON_SELECTED", ), + ("docs.yml", "docs-result", "build", "docs", "DOCS_SELECTED"), ): workflow = (WORKFLOWS / filename).read_text() block = re.search( diff --git a/.github/scripts/test_ci_changes.py b/.github/scripts/test_ci_changes.py index eff5007..f5b65a1 100644 --- a/.github/scripts/test_ci_changes.py +++ b/.github/scripts/test_ci_changes.py @@ -51,10 +51,13 @@ def test_docs_only_skip_builds(self): "docs/mkdocs.yml", "docs/requirements.txt", "docs/docs/assets/img/plot.png", - ".github/workflows/docs.yml", ] ), - set(), + {"docs"}, + ) + self.assertEqual(classify(["README.md", "sample/python/README.md"]), set()) + self.assertEqual( + classify([".github/workflows/docs.yml"]), {"docs", "python_quality"} ) def test_known_support_paths_do_not_select_unrelated_builds(self): @@ -259,7 +262,7 @@ def test_ci_routing_changes_run_full_pr_ci_but_not_repeat_on_tagged_main(self): set(GROUPS), ) for changed_paths, tags, expected in ( - (paths, "v0.3.3", {"python_quality"}), + (paths, "v0.3.3", {"python_quality", "docs"}), (paths, "", set(GROUPS)), (paths + ["src/simd/dispatch.cpp"], "v0.3.3", set(GROUPS)), ): @@ -291,6 +294,7 @@ def test_tagged_main_routing_changes_preserve_lightweight_companion_checks(self) (["scripts/check-python.sh"], {"python_quality", "shell"}), (["sample/run_cpp.sh"], {"python_quality", "shell"}), (["benchmarks/plot_homepage.py"], {"python_quality"}), + (["benchmarks/build_indexes.py"], {"python_quality"}), ([".github/ISSUE_TEMPLATE/config.yml"], {"python_quality"}), (["src/core/api.cpp"], set(GROUPS)), (["python_bindings/__init__.py"], set(GROUPS)), @@ -305,7 +309,7 @@ def test_tagged_main_routing_changes_preserve_lightweight_companion_checks(self) ): changed = "\0".join(routing + companions) + "\0" for event, ref, tags, result in ( - ("push", "main", "v0.7.0", expected), + ("push", "main", "v0.7.0", expected | {"docs"}), ("push", "main", "", set(GROUPS)), ("push", "master", None, set(GROUPS)), ("pull_request", "148/merge", None, set(GROUPS)), diff --git a/.github/scripts/test_ci_workflow_contracts.py b/.github/scripts/test_ci_workflow_contracts.py new file mode 100644 index 0000000..a147dfe --- /dev/null +++ b/.github/scripts/test_ci_workflow_contracts.py @@ -0,0 +1,127 @@ +"""Check lightweight CI coverage and workflow scheduling contracts.""" + +import json +import os +import re +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +from ci_changes import GROUPS, classify + +ROOT = Path(__file__).resolve().parents[2] + + +class WorkflowContractsTest(unittest.TestCase): + def test_benchmark_support_changes_keep_protocol_coverage_without_builds(self): + for path in sorted((ROOT / "benchmarks").glob("*.py")): + with self.subTest(path=path.name): + self.assertEqual( + classify([str(path.relative_to(ROOT))]), {"python_quality"} + ) + self.assertEqual( + classify(["tests/python/test_benchmark_protocol.py"]), {"python_quality"} + ) + self.assertEqual(classify(["benchmarks/unknown.py"]), set(GROUPS)) + self.assertIn( + "cpp", + classify(["benchmarks/build_indexes.py", "src/core/api.cpp"]), + ) + workflow = (ROOT / ".github/workflows/python.yml").read_text() + quality = workflow.split(" python-quality:\n", 1)[1].split( + " python-build:\n", 1 + )[0] + self.assertIn("tests/python/test_benchmark_protocol.py", quality) + self.assertIn("numpy", quality) + self.assertIn("pytest", quality) + self.assertNotIn("pip install .", quality) + + def test_python_checker_covers_every_benchmark_python_file(self): + with tempfile.TemporaryDirectory() as directory: + helper = Path(directory) / "ruff" + log = Path(directory) / "calls.jsonl" + helper.write_text( + f"#!{sys.executable}\n" + "import json, os, sys\n" + "if sys.argv[1:] == ['--version']:\n" + " print('ruff 0.16.1')\n" + "else:\n" + " with open(os.environ['RUFF_AUDIT_LOG'], 'a') as stream:\n" + " stream.write(json.dumps(sys.argv[1:]) + '\\n')\n" + ) + helper.chmod(0o755) + subprocess.run( + ["bash", str(ROOT / "scripts/check-python.sh")], + env=dict(os.environ, RUFF=str(helper), RUFF_AUDIT_LOG=str(log)), + check=True, + capture_output=True, + text=True, + ) + calls = [json.loads(line) for line in log.read_text().splitlines()] + self.assertEqual(len(calls), 2) + for call in calls: + covered = set() + for argument in call: + path = ROOT / argument + if path.is_dir(): + covered.update(path.rglob("*.py")) + elif path.suffix == ".py": + covered.add(path) + self.assertLessEqual(set((ROOT / "benchmarks").rglob("*.py")), covered) + + def test_concurrency_groups_distinguish_forks_and_reuse_one_pr(self): + def group(template, number, sha, workflow): + context = { + "github.event.pull_request.number": number, + "github.head_ref": "main", + "github.sha": sha, + "github.workflow": workflow, + } + + def expand(match): + return str( + next( + context[key.strip()] + for key in match.group(1).split("||") + if context[key.strip()] + ) + ) + + return re.sub(r"\$\{\{(.*?)\}\}", expand, template) + + for filename in ("test.yaml", "python.yml"): + text = (ROOT / ".github/workflows" / filename).read_text() + template = re.search(r"^ group: (.+)$", text, re.MULTILINE).group(1) + with self.subTest(workflow=filename): + self.assertNotEqual( + group(template, 201, "a", filename), + group(template, 202, "b", filename), + ) + self.assertEqual( + group(template, 201, "a", filename), + group(template, 201, "b", filename), + ) + self.assertNotEqual( + group(template, 0, "a", filename), + group(template, 0, "b", filename), + ) + + def test_documentation_summary_is_available_for_non_doc_prs(self): + text = (ROOT / ".github/workflows/docs.yml").read_text() + triggers = text.split("concurrency:", 1)[0] + self.assertNotIn("paths:", triggers) + self.assertNotIn("paths-ignore:", triggers) + self.assertIn(" pull_request:", triggers) + self.assertIn(" workflow_dispatch:", triggers) + self.assertIn("uses: ./.github/workflows/changes.yml", text) + build = text.split(" build:\n", 1)[1].split(" docs-result:\n", 1)[0] + self.assertIn("needs: changes", build) + self.assertIn("needs.changes.outputs.docs == 'true'", build) + self.assertIn("needs.changes.result != 'success'", build) + self.assertEqual(classify(["docs/docs/quick_start.md"]), {"docs"}) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/changes.yml b/.github/workflows/changes.yml index 6d18267..755ad84 100644 --- a/.github/workflows/changes.yml +++ b/.github/workflows/changes.yml @@ -5,6 +5,8 @@ on: outputs: benchmarks: value: ${{ jobs.detect.outputs.benchmarks }} + docs: + value: ${{ jobs.detect.outputs.docs }} cpp: value: ${{ jobs.detect.outputs.cpp }} cpp_format: @@ -29,6 +31,7 @@ jobs: timeout-minutes: 5 outputs: benchmarks: ${{ steps.paths.outputs.benchmarks }} + docs: ${{ steps.paths.outputs.docs }} cpp: ${{ steps.paths.outputs.cpp }} cpp_format: ${{ steps.paths.outputs.cpp_format }} cpp_tidy: ${{ steps.paths.outputs.cpp_tidy }} diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 0f61021..b9a6fa0 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -3,13 +3,7 @@ name: Documentation on: push: branches: ["main"] - paths: - - "docs/**" - - ".github/workflows/docs.yml" pull_request: - paths: - - "docs/**" - - ".github/workflows/docs.yml" workflow_dispatch: concurrency: @@ -20,7 +14,12 @@ permissions: contents: read jobs: + changes: + uses: ./.github/workflows/changes.yml + build: + needs: changes + if: ${{ !cancelled() && (needs.changes.result != 'success' || needs.changes.outputs.docs == 'true') }} name: Build documentation runs-on: ubuntu-latest timeout-minutes: 10 @@ -36,6 +35,25 @@ jobs: - name: Build documentation run: mkdocs build --strict --config-file docs/mkdocs.yml + docs-result: + name: Documentation validation + needs: [changes, build] + if: ${{ always() }} + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Require selected documentation build + env: + CHANGES_RESULT: ${{ needs.changes.result }} + DOCS_SELECTED: ${{ needs.changes.outputs.docs }} + MATRIX_RESULT: ${{ needs.build.result }} + run: | + if [ "$CHANGES_RESULT" = success ] && [ "$DOCS_SELECTED" = false ]; then + test "$MATRIX_RESULT" = skipped || test "$MATRIX_RESULT" = success + else + test "$MATRIX_RESULT" = success + fi + deploy: name: Deploy documentation if: github.event_name == 'push' diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 08df90e..ab9a9cf 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -7,7 +7,7 @@ on: branches: ["main", "master"] concurrency: - group: ${{ github.head_ref || github.sha }}-${{ github.workflow }} + group: ${{ github.event.pull_request.number || github.sha }}-${{ github.workflow }} cancel-in-progress: true permissions: @@ -29,8 +29,8 @@ jobs: with: python-version: "3.11" cache: pip - - name: Install Ruff - run: python -m pip install "ruff==0.16.1" + - name: Install validation dependencies + run: python -m pip install "ruff==0.16.1" numpy pytest - name: Check Python run: ./scripts/check-python.sh - name: Check release automation @@ -38,7 +38,8 @@ jobs: ruff format --check .github/scripts ruff check .github/scripts python -m unittest discover -s .github/scripts -p 'test_*.py' - + - name: Test benchmark protocols without native builds + run: python -m pytest tests/python/test_benchmark_protocol.py --tb=short -q python-build: needs: changes diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 616d9c0..eb3d2ab 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -5,7 +5,7 @@ on: - main pull_request: concurrency: - group: ${{ github.head_ref || github.sha }}-${{ github.workflow }} + group: ${{ github.event.pull_request.number || github.sha }}-${{ github.workflow }} cancel-in-progress: true permissions: contents: read diff --git a/docs/maintaining.md b/docs/maintaining.md index d88ea96..f5e6644 100644 --- a/docs/maintaining.md +++ b/docs/maintaining.md @@ -10,11 +10,13 @@ only the affected job groups: | Changes | Checks | | --- | --- | -| Documentation and Markdown only | Documentation workflow for `docs/`; no C++ or Python builds | +| Documentation and Markdown only | Documentation build for `docs/`; no C++ or Python builds; the documentation summary is always emitted | | Issue forms/configuration and CODEOWNERS | No C++/Python builds | | `benchmarks/core/`, `benchmarks/hotpaths/` code/build files | Ubuntu benchmark build and bounded smoke tests; C++ formatting or Python lint as applicable, without the wheel/platform matrix for benchmark-only edits | | `.github/workflows/test.yaml` | C++ checks, ShellCheck, Python quality (including summary-job tests), and benchmark smoke | -| `benchmarks/plot_homepage.py` | Python lint, including the plotting script; no index or wheel builds | +| Known root benchmark Python scripts | Python lint and dataset-independent benchmark protocol tests; no index or wheel builds | +| `tests/python/test_benchmark_protocol.py` | Python lint and benchmark protocol tests; no index or wheel builds | +| `.github/workflows/docs.yml` | Documentation build and Python quality checks for workflow regression tests | | `sample/run_cpp.sh`, `sample/run_python.sh` | ShellCheck, including these runners | | `tests/core_consumer/`, `tests/fixtures/core_api_v1/` | C++ checks; the frozen v1 fixture is excluded from formatting | | C++ headers or library sources | C++ formatting, clang-tidy, platform tests, sanitizers, consumer build, and Python builds/wheels | @@ -46,9 +48,9 @@ change detection fails, so a detection error cannot silently waive a check. Release tags and manual wheel builds always run the full wheel build. CI routing edits run the full relevant matrix on pull requests. After an already released version reaches `main`, a push containing routing edits avoids repeated -platform builds only when all accompanying files select at most Python quality -checks and ShellCheck under the normal path rules. Python lint and selector tests -always run in this case, and any accompanying shell checks are retained. For +platform builds only when all accompanying files select at most Python quality, +ShellCheck, and documentation checks under the normal path rules. Python lint and +selector tests always run in this case, and any accompanying shell checks are retained. For example, routing plus `scripts/check-python.sh` runs Python quality and ShellCheck; routing plus library code, build configuration, wheel-related inputs or unknown paths keeps the full selection. PR routing checks remain broad. An untagged version on `main` still forces full C++ and Python CI, @@ -64,9 +66,33 @@ No dataset download, timing threshold, or performance comparison is run. The job also runs for shared library sources/headers, root/CMake build settings, Python bindings/package configuration, and changes to the Test workflow. Other -unrecognized benchmark paths retain conservative full selection. The hot-path -Python scripts are included in Python lint. This is Linux build/functionality -coverage, not a performance result or proof of every SIMD backend executing. +unrecognized benchmark paths retain conservative full selection. All benchmark +Python scripts are included in Python lint. Known root Python drivers and their +protocol tests select the existing `Python formatting and linting` job, which +also runs `tests/python/test_benchmark_protocol.py` with NumPy and pytest. These +tests need no native package build, external dataset, or baseline installation. +This is Linux build/functionality coverage, not a performance result or proof of +every SIMD backend executing. + +## Documentation and concurrency gates + +The documentation workflow runs on every PR and main push. Its changed-file +output selects the strict MkDocs build for documentation changes; manual runs +also build. `Documentation validation` always reports a result, accepting a +skipped build only when change detection succeeded and did not select docs. +Selected build failures, cancellations, and unexpected skips fail this summary. +Only successful push builds can deploy the documentation. + +After the updated workflow is merged and the selected-build and intentional-skip +paths have been verified on GitHub, add `Documentation validation` to main's +required status checks, preserving existing contexts and their app IDs. Until +that remote update is applied, the documentation failure is not a required merge +gate. Do not require the conditional `Build documentation` job or a workflow +filtered out by paths: missing check names can leave unrelated PRs pending. + +Test and Python workflow concurrency groups use the PR number, so unrelated +forks using the same branch name do not cancel each other. Push runs keep their +commit-specific groups, and release serialization is unchanged. ## Matrix summary checks diff --git a/scripts/check-python.sh b/scripts/check-python.sh index f278c22..e74ed42 100755 --- a/scripts/check-python.sh +++ b/scripts/check-python.sh @@ -18,9 +18,6 @@ if [[ "$($ruff --version)" != "ruff 0.16.1" ]]; then fi cd "$repo_root" -paths=(python_bindings sample/python tests/python benchmarks/hotpaths) -if [[ -f benchmarks/plot_homepage.py ]]; then - paths+=(benchmarks/plot_homepage.py) -fi +paths=(python_bindings sample/python tests/python benchmarks) "$ruff" format --check "${paths[@]}" "$ruff" check "${paths[@]}" diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index fe351aa..ff777b9 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -54,7 +54,8 @@ target_link_libraries(rabitq_tests # Discover tests for CTest include(GoogleTest) -gtest_discover_tests(rabitq_tests DISCOVERY_MODE PRE_TEST) +# Allow bounded executable startup time on hosted runners before test enumeration. +gtest_discover_tests(rabitq_tests DISCOVERY_MODE PRE_TEST DISCOVERY_TIMEOUT 30) add_test(NAME CoreModuleBoundary COMMAND "${CMAKE_COMMAND}" "-DRABITQ_SOURCE_DIR=${PROJECT_SOURCE_DIR}"