ci: reduce duplicate validation and consolidate tests

.github/scripts/ci_changes.py

@@ -63,8 +63,17 @@ def classify(paths):
         if path == ".github/workflows/pr-labels.yml":
             selected.add("python_quality")
             continue
-        if path.startswith(".github/ISSUE_TEMPLATE/") or path in (
+        if path in (
             ".github/release.yml",
+            "scripts/check-format.sh",
+            "scripts/check-tidy.sh",
+            "tests/core_consumer/run.cmake",
+        ):
+            # These inputs are exercised by the CI automation regression tests.
+            selected.add("python_quality")
+            if path == ".github/release.yml":
+                continue
+        if path.startswith(".github/ISSUE_TEMPLATE/") or path in (
             ".github/CODEOWNERS",
             "LICENSE",
         ):

.github/scripts/test_ci_aggregates.py

@@ -28,13 +28,6 @@ def test_matrix_verdicts_and_dependencies(self):
                 "python_build",
                 "PYTHON_SELECTED",
             ),
-            (
-                "python.yml",
-                "faiss-comparison",
-                "python-build",
-                "python_build",
-                "PYTHON_SELECTED",
-            ),
             ("docs.yml", "docs-result", "build", "docs", "DOCS_SELECTED"),
         ):
             workflow = (WORKFLOWS / filename).read_text()

.github/scripts/test_ci_changes.py

@@ -65,14 +65,15 @@ def test_known_support_paths_do_not_select_unrelated_builds(self):
             ".github/ISSUE_TEMPLATE/config.yml": set(),
             ".github/ISSUE_TEMPLATE/bug_report.yml": set(),
             ".github/CODEOWNERS": set(),
+            ".github/release.yml": {"python_quality"},
             "benchmarks/plot_homepage.py": {"python_quality"},
             "sample/run_cpp.sh": {"shell"},
             "sample/run_python.sh": {"shell"},
             "example.sh": {"shell"},  # Deleted paths still occur in a push diff.
             "example_py.sh": {"shell"},
             "python/ivf.py": {"python_quality", "python_build", "wheels"},
             "python/utils/io.py": {"python_quality", "python_build", "wheels"},
-            "tests/core_consumer/run.cmake": {"cpp"},
+            "tests/core_consumer/run.cmake": {"cpp", "python_quality"},
             "tests/fixtures/core_api_v1/consumer.cpp": {"cpp"},
         }
         for path, expected in cases.items():
@@ -166,8 +167,8 @@ def test_language_specific_changes(self):
             ".github/scripts/prepare_release.py": {"python_quality"},
             "scripts/check-python.sh": {"shell", "python_quality"},
             "scripts/build-macos-openmp.sh": {"shell", "wheels"},
-            "scripts/check-tidy.sh": {"shell", "cpp_tidy"},
-            "scripts/check-format.sh": {"shell", "cpp_format"},
+            "scripts/check-tidy.sh": {"shell", "cpp_tidy", "python_quality"},
+            "scripts/check-format.sh": {"shell", "cpp_format", "python_quality"},
             "scripts/check-includes.sh": {"shell"},
             ".clang-format": {"cpp_format"},
             ".clang-tidy": {"cpp_tidy"},

.github/scripts/test_ci_workflow_contracts.py

@@ -15,6 +15,59 @@
 
 
 class WorkflowContractsTest(unittest.TestCase):
+    def test_tidy_rejects_empty_scope_and_propagates_analysis_failure(self):
+        with tempfile.TemporaryDirectory() as directory:
+            work = Path(directory)
+            compiler = work / "compiler"
+            compiler.write_text(
+                "#!/usr/bin/env bash\n"
+                "printf '%s\\n' '#include <...> search starts here:' "
+                "' /test/include' 'End of search list.'\n"
+            )
+            compiler.chmod(0o755)
+            runner = work / "runner"
+            marker = work / "analyzed"
+            runner.write_text(
+                f"#!{sys.executable}\n"
+                "import os, sys\n"
+                "from pathlib import Path\n"
+                "Path(os.environ['TIDY_MARKER']).touch()\n"
+                "sys.exit(int(os.environ['ANALYSIS_STATUS']))\n"
+            )
+            runner.chmod(0o755)
+            for entries, status, selected in (
+                ([], 0, False),
+                ([{"directory": str(work), "file": "foreign.cpp"}], 0, False),
+                ([{"directory": str(ROOT), "file": "src/core/api.cpp"}], 0, True),
+                ([{"directory": str(ROOT / "src"), "file": "core/api.cpp"}], 0, True),
+                ([{"directory": str(ROOT), "file": "src/core/api.cpp"}], 7, True),
+            ):
+                with self.subTest(entries=entries, status=status):
+                    (work / "compile_commands.json").write_text(json.dumps(entries))
+                    marker.unlink(missing_ok=True)
+                    result = subprocess.run(
+                        ["bash", str(ROOT / "scripts/check-tidy.sh"), str(work)],
+                        env=dict(
+                            os.environ,
+                            CXX=str(compiler),
+                            RUN_CLANG_TIDY=str(runner),
+                            PYTHON=sys.executable,
+                            TIDY_MARKER=str(marker),
+                            ANALYSIS_STATUS=str(status),
+                        ),
+                        capture_output=True,
+                        text=True,
+                        check=False,
+                    )
+                    self.assertEqual(result.returncode == 0, selected and status == 0)
+                    self.assertEqual(marker.exists(), selected)
+                    if not selected:
+                        self.assertIn(
+                            "no matching first-party translation units", result.stderr
+                        )
+                    if result.returncode:
+                        self.assertNotIn("clang-tidy passed", result.stdout)
+
     def test_format_checker_propagates_file_list_failures(self):
         with tempfile.TemporaryDirectory() as directory:
             root = Path(directory)
@@ -198,6 +251,122 @@ def test_windows_consumers_cover_each_component_and_openmp_mode(self):
             "      - name: Test external core-only integrations (Windows)\n", 1
         )[1].split("      - name: Report compiler cache statistics", 1)[0]
         self.assertIn("-DRABITQ_COMPONENT=${{ matrix.component }}", windows)
+        self.assertIn(
+            "-DRABITQ_RUN_REFERENCE_TESTS=${{ matrix.openmp == 'OFF' && "
+            "matrix.component != 'core' && 'ON' || 'OFF' }}",
+            windows,
+        )
+
+    def test_consumer_reference_ownership_preserves_integration_checks(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            log = root / "calls.bin"
+            for tool in ("cmake", "ctest"):
+                helper = root / tool
+                helper.write_text(
+                    r"""#!/usr/bin/env bash
+set -euo pipefail
+# CMake emits nonempty arguments here; a double NUL separates invocations.
+printf '%s\0' "${0##*/}" "$@" >> "$CONSUMER_TRACE"
+printf '\0' >> "$CONSUMER_TRACE"
+args=("$@")
+for ((i = 0; i + 1 < ${#args[@]}; ++i)); do
+    case "${args[i]}" in
+        --prefix) mkdir -p "${args[i+1]}" ;;
+        -B)
+            case "${args[i+1]##*/}" in
+                invalid-python) printf '%s\n' 'requires RABITQ_BUILD_INDEX=ON' >&2; exit 1 ;;
+                invalid-index) printf '%s\n' 'requires RABITQ_BUILD_FASTSCAN=ON' >&2; exit 1 ;;
+                invalid-openmp) printf '%s\n' 'ANN indexes require RABITQ_USE_OPENMP=ON' >&2; exit 1 ;;
+                missing-index|missing-fastscan) printf '%s\n' 'rabitqlib component not found' >&2; exit 1 ;;
+            esac ;;
+        --test-dir)
+            if [[ "${args[i+1]##*/}" == producer && "${FAIL_REFERENCE:-0}" == 1 ]]; then
+                exit 7
+            fi ;;
+    esac
+done
+"""
+                )
+                helper.chmod(0o755)
+            for index, (openmp, reference, component, expected, fail) in enumerate(
+                [
+                    ("ON", "OFF", None, {"core": "OFF", "fastscan": "OFF"}, False),
+                    ("OFF", "ON", None, {"core": "OFF", "fastscan": "ON"}, False),
+                    ("ON", None, None, {"core": "OFF", "fastscan": "ON"}, False),
+                    ("OFF", "OFF", "core", {"core": "OFF"}, False),
+                    ("ON", None, "core", {"core": "ON"}, False),
+                    ("OFF", "ON", "fastscan", {"fastscan": "ON"}, False),
+                    ("OFF", "ON", "fastscan", {"fastscan": "ON"}, True),
+                ]
+            ):
+                with self.subTest(
+                    openmp=openmp, reference=reference, component=component, fail=fail
+                ):
+                    log.unlink(missing_ok=True)
+                    command = [
+                        "cmake",
+                        f"-DRABITQ_SOURCE_DIR={ROOT}",
+                        f"-DWORK_DIR={root / str(index)}",
+                        f"-DCMAKE_COMMAND={root / 'cmake'}",
+                        f"-DCMAKE_CTEST_COMMAND={root / 'ctest'}",
+                        f"-DRABITQ_USE_OPENMP={openmp}",
+                    ]
+                    if reference is not None:
+                        command.append(f"-DRABITQ_RUN_REFERENCE_TESTS={reference}")
+                    if component is not None:
+                        command.append(f"-DRABITQ_COMPONENT={component}")
+                    result = subprocess.run(
+                        [*command, "-P", str(ROOT / "tests/core_consumer/run.cmake")],
+                        env=dict(
+                            os.environ,
+                            CONSUMER_TRACE=str(log),
+                            FAIL_REFERENCE=str(int(fail)),
+                        ),
+                        capture_output=True,
+                        text=True,
+                        check=False,
+                    )
+                    self.assertEqual(result.returncode == 0, not fail, result.stderr)
+                    if fail:
+                        self.assertIn("Command failed (7)", result.stderr)
+                        continue
+                    calls = [
+                        record.decode().split("\0")
+                        for record in log.read_bytes().split(b"\0\0")
+                        if record
+                    ]
+                    producers = {}
+                    tests = []
+                    for tool, *args in calls:
+                        if tool == "cmake" and "-B" in args:
+                            build = Path(args[args.index("-B") + 1])
+                            if build.name == "producer":
+                                producers[build.parent.name] = next(
+                                    arg.split("=", 1)[1]
+                                    for arg in args
+                                    if arg.startswith("-DRABITQ_BUILD_TESTS=")
+                                )
+                        if tool == "ctest":
+                            tests.append(Path(args[args.index("--test-dir") + 1]))
+                    self.assertEqual(producers, expected)
+                    self.assertEqual(
+                        {path.parent.name for path in tests if path.name == "producer"},
+                        {name for name, enabled in expected.items() if enabled == "ON"},
+                    )
+                    for name in expected:
+                        self.assertEqual(
+                            {
+                                path.name
+                                for path in tests
+                                if path.parent.name == name and path.name != "producer"
+                            },
+                            {"ADD_SUBDIRECTORY", "FETCHCONTENT", "PACKAGE"},
+                        )
+                    self.assertEqual(
+                        any(path.name == "benchmark" for path in tests),
+                        "core" in expected,
+                    )
 
     def test_component_partition_rejects_unknown_or_empty_selection(self):
         with tempfile.TemporaryDirectory() as directory:

.github/workflows/python.yml

@@ -129,23 +129,3 @@ jobs:
           tests/python/test_symqg.py::test_save_load_roundtrip
           tests/python/test_qgkmeans.py::test_train_populates_faiss_style_outputs
           tests/python/test_rabitqkmeans.py::test_flat_parameters_and_outputs
-
-  faiss-comparison:
-    # Keep the required check name; FAISS runs against the 3.11 source installation.
-    name: FAISS comparison script
-    needs: [changes, python-build]
-    if: ${{ always() }}
-    runs-on: ubuntu-latest
-    timeout-minutes: 5
-    steps:
-      - name: Require selected comparison tests
-        env:
-          CHANGES_RESULT: ${{ needs.changes.result }}
-          PYTHON_SELECTED: ${{ needs.changes.outputs.python_build }}
-          MATRIX_RESULT: ${{ needs.python-build.result }}
-        run: |
-          if [ "$CHANGES_RESULT" = success ] && [ "$PYTHON_SELECTED" = false ]; then
-            test "$MATRIX_RESULT" = skipped || test "$MATRIX_RESULT" = success
-          else
-            test "$MATRIX_RESULT" = success
-          fi

.github/workflows/test.yaml

@@ -27,19 +27,13 @@ jobs:
       - name: Check formatting
         run: ./scripts/check-format.sh
 
-  clang-tidy-shard:
+  clang-tidy:
     needs: changes
     if: ${{ !cancelled() && (needs.changes.result != 'success' || needs.changes.outputs.cpp_tidy == 'true') }}
-    name: C++ static analysis shard ${{ matrix.shard }}
+    name: C++ static analysis
     runs-on: ubuntu-24.04
     timeout-minutes: 30
-    strategy:
-      fail-fast: false
-      matrix:
-        shard: [0, 1]
     env:
-      TIDY_SHARD_INDEX: ${{ matrix.shard }}
-      TIDY_SHARD_COUNT: "2"
       CXX: clang++-23
     steps:
       - uses: actions/checkout@v7
@@ -64,17 +58,6 @@ jobs:
       - name: Run clang-tidy
         run: ./scripts/check-tidy.sh build-tidy
 
-  clang-tidy:
-    name: C++ static analysis
-    needs: clang-tidy-shard
-    if: ${{ always() && !cancelled() && needs.clang-tidy-shard.result != 'skipped' }}
-    runs-on: ubuntu-latest
-    steps:
-      - name: Require both shards
-        env:
-          BUILD_RESULT: ${{ needs.clang-tidy-shard.result }}
-        run: test "$BUILD_RESULT" = success
-
   shellcheck:
     needs: changes
     if: ${{ !cancelled() && (needs.changes.result != 'success' || needs.changes.outputs.shell == 'true') }}
@@ -229,6 +212,7 @@ jobs:
           cmake -DRABITQ_SOURCE_DIR="$PWD"
           -DWORK_DIR="$RUNNER_TEMP/rabitq-clang-core"
           -DRABITQ_USE_OPENMP=${{ matrix.openmp }}
+          -DRABITQ_RUN_REFERENCE_TESTS=${{ matrix.openmp == 'OFF' && 'ON' || 'OFF' }}
           -P tests/core_consumer/run.cmake
       - name: Report compiler cache statistics
         if: ${{ always() }}
@@ -293,7 +277,7 @@ jobs:
         run: |
           cmake --version
           ninja --version
-      # Keep assertions/debug symbols while avoiding unoptimized sanitizer loops.
+      # Keep assertions and source locations without full variable debug information.
       - name: Configure
         run: >-
           cmake -S . -B build-sanitizers -G Ninja
@@ -302,7 +286,7 @@ jobs:
           -DRABITQ_ENABLE_NATIVE_OPTIMIZATION=OFF
           -DRABITQ_ENABLE_SANITIZERS=ON
           -DCMAKE_BUILD_TYPE=Debug
-          -DCMAKE_CXX_FLAGS_DEBUG="-O1 -g -fno-optimize-sibling-calls"
+          -DCMAKE_CXX_FLAGS_DEBUG="-O1 -g1 -fno-optimize-sibling-calls"
       - name: Build
         run: cmake --build build-sanitizers
       - name: Test
@@ -335,6 +319,7 @@ jobs:
         run: |
           cmake --version
           ninja --version
+          lscpu
       - name: Build RaBitQ-Library with portable tests
         run: |
           cmake -S . -B build-install -G Ninja \
@@ -360,13 +345,14 @@ jobs:
         run: |
           simd_filter='CpuFeatures.*:FloatDistance.*:MatrixDispatchTest.*:FastScan*.*:BatchEstimatorTest.*'
           simd_filter+=':FhtDispatchTest.*:FlipSignTest.*:KacsWalkTest.*:ExcodeIp.*:MaskIpX0Q.*'
-          simd_filter+=':PortableKernels.*:QueryPreparation.*:ScalarQuantize.*:WarmupIpX0Q.*:PackBinary.*'
+          simd_filter+=':PortableKernels.*:QueryPreparation.*:WarmupIpX0Q.*:PackBinary.*'
           simd_filter+=':ExplicitSimd/RabitqRescaleBackendTest.*:QGEstimatorTest.*'
           simd_filter+=':QGKMeansTest.TrainsAndReturnsAssignmentsForFinalCentroids'
           simd_filter+=':RaBitQKMeansTest.FinalModesKeepFlatLloydAssignmentsAcrossClusterCounts'
           simd_filter+=':IvfPersistenceTest.ReloadsRawAndQuantizedStorage'
           simd_filter+=':HnswSaveTest.SuccessfulSavePreservesSearchAfterLoading'
-          simd_filter+=':IvfAddTest.*:IvfRemoveTest.*:HnswRemoveTest.*'
+          simd_filter+=':HnswPaddingTest.TailDimensionsMatchEveryAvailableSearchBackend'
+          simd_filter+=':IvfLifecycleTest.BatchTailSurvivesAddRemoveAndReload:HnswRemoveTest.*'
           for cpu_model in hsw icl; do
             RABITQ_TEST_CPU="$cpu_model" "$RUNNER_TEMP/sde/sde64" "-$cpu_model" -- \
               build-install/tests/rabitq_tests --gtest_filter="$simd_filter"
@@ -428,13 +414,15 @@ jobs:
           cmake -DRABITQ_SOURCE_DIR="$PWD"
           -DWORK_DIR="$RUNNER_TEMP/rabitq-core"
           -DRABITQ_USE_OPENMP=${{ matrix.openmp }}
+          -DRABITQ_RUN_REFERENCE_TESTS=${{ matrix.openmp == 'OFF' && matrix.component != 'core' && 'ON' || 'OFF' }}
           -P tests/core_consumer/run.cmake
       - name: Test external core-only integrations (macOS)
         if: runner.os == 'macOS'
         run: >-
           cmake -DRABITQ_SOURCE_DIR="$PWD"
           -DWORK_DIR="$RUNNER_TEMP/rabitq-core"
           -DRABITQ_USE_OPENMP=${{ matrix.openmp }}
+          -DRABITQ_RUN_REFERENCE_TESTS=${{ matrix.openmp == 'OFF' && matrix.component != 'core' && 'ON' || 'OFF' }}
           -DOpenMP_ROOT="$(brew --prefix)/opt/libomp"
           -P tests/core_consumer/run.cmake
       - name: Test external core-only integrations (Windows)
@@ -448,6 +436,7 @@ jobs:
           "-DWORK_DIR=$env:RUNNER_TEMP/rabitq-core"
           -DRABITQ_COMPONENT=${{ matrix.component }}
           -DRABITQ_USE_OPENMP=${{ matrix.openmp }}
+          -DRABITQ_RUN_REFERENCE_TESTS=${{ matrix.openmp == 'OFF' && matrix.component != 'core' && 'ON' || 'OFF' }}
           -P tests/core_consumer/run.cmake
       - name: Report compiler cache statistics
         if: ${{ always() && runner.os != 'Windows' }}

benchmarks/hotpaths/CMakeLists.txt

@@ -27,4 +27,6 @@ if(RABITQ_BUILD_INDEX)
         "${CMAKE_CURRENT_BINARY_DIR}/ivf-add-smoke-single" 64 8 256 16 1 single)
     add_test(NAME IvfAddHotpathRouted COMMAND rabitq_ivf_add_hotpaths
         "${CMAKE_CURRENT_BINARY_DIR}/ivf-add-smoke-routed" 64 8 256 16 1 routed)
+    add_test(NAME IvfAddHotpathSmall COMMAND rabitq_ivf_add_hotpaths
+        "${CMAKE_CURRENT_BINARY_DIR}/ivf-add-smoke-small" 64 8 8 1 1 single)
 endif()

benchmarks/hotpaths/README.md

@@ -59,6 +59,8 @@ the deterministic input. Original production indexes are not benchmark fixtures.
 
 ## Timing and correctness contract
 
+- IVF add verifies the expected point counts and samples new points' self-neighbors
+  outside the timed call, in addition to comparing output hashes across repetitions.
 - Inputs use an LCG with seed 42 (query/centroid seed 91 where applicable), metric L2.
   Dimensions for encoding/scoring are multiples of 32. Use 64, 128 and 960 dimensions;
   encoding rows 32/4096 and threads 1/4 retain the cases from the original investigation.