From 344b382147cea802b26c74d0329dc6f79bd57b20 Mon Sep 17 00:00:00 2001 From: gouyt13clear Date: Sun, 25 Oct 2026 21:16:23 +0800 Subject: [PATCH 1/2] ci: simplify validation and consolidate redundant tests --- .github/scripts/ci_changes.py | 11 +- .github/scripts/test_ci_aggregates.py | 7 - .github/scripts/test_ci_changes.py | 7 +- .github/scripts/test_ci_workflow_contracts.py | 169 ++++++++++++++++++ .github/workflows/python.yml | 20 --- .github/workflows/test.yaml | 28 +-- benchmarks/hotpaths/CMakeLists.txt | 2 + benchmarks/hotpaths/README.md | 2 + benchmarks/hotpaths/ivf_add.cpp | 22 ++- docs/maintaining.md | 20 ++- scripts/check-tidy.sh | 44 ++--- tests/CMakeLists.txt | 9 + tests/README.md | 16 +- tests/core_consumer/run.cmake | 17 +- tests/integration/cpp_example_index_check.cpp | 69 +++++++ tests/python/test_cpp_clustering_examples.py | 10 +- tests/python/test_hnsw.py | 32 ++-- tests/python/test_import.py | 79 ++++---- tests/python/test_ivf.py | 42 ++--- tests/python/test_symqg.py | 14 +- .../unit/rabitqlib/fastscan/fastscan_test.cpp | 5 +- .../utils/query_preparation_test.cpp | 43 +++-- tests/unit/rabitqlib/utils/rotator_test.cpp | 94 +++++----- tests/unit/rabitqlib/utils/space_test.cpp | 99 ++-------- 24 files changed, 520 insertions(+), 341 deletions(-) create mode 100644 tests/integration/cpp_example_index_check.cpp diff --git a/.github/scripts/ci_changes.py b/.github/scripts/ci_changes.py index 92fe4526..0bde4365 100644 --- a/.github/scripts/ci_changes.py +++ b/.github/scripts/ci_changes.py @@ -63,8 +63,17 @@ def classify(paths): if path == ".github/workflows/pr-labels.yml": selected.add("python_quality") continue - if path.startswith(".github/ISSUE_TEMPLATE/") or path in ( + if path in ( ".github/release.yml", + "scripts/check-format.sh", + "scripts/check-tidy.sh", + "tests/core_consumer/run.cmake", + ): + # These inputs are exercised by the CI automation regression tests. + selected.add("python_quality") + if path == ".github/release.yml": + continue + if path.startswith(".github/ISSUE_TEMPLATE/") or path in ( ".github/CODEOWNERS", "LICENSE", ): diff --git a/.github/scripts/test_ci_aggregates.py b/.github/scripts/test_ci_aggregates.py index c6eb9500..2d7375fd 100644 --- a/.github/scripts/test_ci_aggregates.py +++ b/.github/scripts/test_ci_aggregates.py @@ -28,13 +28,6 @@ def test_matrix_verdicts_and_dependencies(self): "python_build", "PYTHON_SELECTED", ), - ( - "python.yml", - "faiss-comparison", - "python-build", - "python_build", - "PYTHON_SELECTED", - ), ("docs.yml", "docs-result", "build", "docs", "DOCS_SELECTED"), ): workflow = (WORKFLOWS / filename).read_text() diff --git a/.github/scripts/test_ci_changes.py b/.github/scripts/test_ci_changes.py index 82cae3d9..f1e4213c 100644 --- a/.github/scripts/test_ci_changes.py +++ b/.github/scripts/test_ci_changes.py @@ -65,6 +65,7 @@ def test_known_support_paths_do_not_select_unrelated_builds(self): ".github/ISSUE_TEMPLATE/config.yml": set(), ".github/ISSUE_TEMPLATE/bug_report.yml": set(), ".github/CODEOWNERS": set(), + ".github/release.yml": {"python_quality"}, "benchmarks/plot_homepage.py": {"python_quality"}, "sample/run_cpp.sh": {"shell"}, "sample/run_python.sh": {"shell"}, @@ -72,7 +73,7 @@ def test_known_support_paths_do_not_select_unrelated_builds(self): "example_py.sh": {"shell"}, "python/ivf.py": {"python_quality", "python_build", "wheels"}, "python/utils/io.py": {"python_quality", "python_build", "wheels"}, - "tests/core_consumer/run.cmake": {"cpp"}, + "tests/core_consumer/run.cmake": {"cpp", "python_quality"}, "tests/fixtures/core_api_v1/consumer.cpp": {"cpp"}, } for path, expected in cases.items(): @@ -166,8 +167,8 @@ def test_language_specific_changes(self): ".github/scripts/prepare_release.py": {"python_quality"}, "scripts/check-python.sh": {"shell", "python_quality"}, "scripts/build-macos-openmp.sh": {"shell", "wheels"}, - "scripts/check-tidy.sh": {"shell", "cpp_tidy"}, - "scripts/check-format.sh": {"shell", "cpp_format"}, + "scripts/check-tidy.sh": {"shell", "cpp_tidy", "python_quality"}, + "scripts/check-format.sh": {"shell", "cpp_format", "python_quality"}, "scripts/check-includes.sh": {"shell"}, ".clang-format": {"cpp_format"}, ".clang-tidy": {"cpp_tidy"}, diff --git a/.github/scripts/test_ci_workflow_contracts.py b/.github/scripts/test_ci_workflow_contracts.py index e68dc0ce..dbbee0c3 100644 --- a/.github/scripts/test_ci_workflow_contracts.py +++ b/.github/scripts/test_ci_workflow_contracts.py @@ -15,6 +15,59 @@ class WorkflowContractsTest(unittest.TestCase): + def test_tidy_rejects_empty_scope_and_propagates_analysis_failure(self): + with tempfile.TemporaryDirectory() as directory: + work = Path(directory) + compiler = work / "compiler" + compiler.write_text( + "#!/usr/bin/env bash\n" + "printf '%s\\n' '#include <...> search starts here:' " + "' /test/include' 'End of search list.'\n" + ) + compiler.chmod(0o755) + runner = work / "runner" + marker = work / "analyzed" + runner.write_text( + f"#!{sys.executable}\n" + "import os, sys\n" + "from pathlib import Path\n" + "Path(os.environ['TIDY_MARKER']).touch()\n" + "sys.exit(int(os.environ['ANALYSIS_STATUS']))\n" + ) + runner.chmod(0o755) + for entries, status, selected in ( + ([], 0, False), + ([{"directory": str(work), "file": "foreign.cpp"}], 0, False), + ([{"directory": str(ROOT), "file": "src/core/api.cpp"}], 0, True), + ([{"directory": str(ROOT / "src"), "file": "core/api.cpp"}], 0, True), + ([{"directory": str(ROOT), "file": "src/core/api.cpp"}], 7, True), + ): + with self.subTest(entries=entries, status=status): + (work / "compile_commands.json").write_text(json.dumps(entries)) + marker.unlink(missing_ok=True) + result = subprocess.run( + ["bash", str(ROOT / "scripts/check-tidy.sh"), str(work)], + env=dict( + os.environ, + CXX=str(compiler), + RUN_CLANG_TIDY=str(runner), + PYTHON=sys.executable, + TIDY_MARKER=str(marker), + ANALYSIS_STATUS=str(status), + ), + capture_output=True, + text=True, + check=False, + ) + self.assertEqual(result.returncode == 0, selected and status == 0) + self.assertEqual(marker.exists(), selected) + if not selected: + self.assertIn( + "no matching first-party translation units", result.stderr + ) + if result.returncode: + self.assertNotIn("clang-tidy passed", result.stdout) + def test_format_checker_propagates_file_list_failures(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory) @@ -198,6 +251,122 @@ def test_windows_consumers_cover_each_component_and_openmp_mode(self): " - name: Test external core-only integrations (Windows)\n", 1 )[1].split(" - name: Report compiler cache statistics", 1)[0] self.assertIn("-DRABITQ_COMPONENT=${{ matrix.component }}", windows) + self.assertIn( + "-DRABITQ_RUN_REFERENCE_TESTS=${{ matrix.openmp == 'OFF' && " + "matrix.component != 'core' && 'ON' || 'OFF' }}", + windows, + ) + + def test_consumer_reference_ownership_preserves_integration_checks(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + log = root / "calls.bin" + for tool in ("cmake", "ctest"): + helper = root / tool + helper.write_text( + r"""#!/usr/bin/env bash +set -euo pipefail +# CMake emits nonempty arguments here; a double NUL separates invocations. +printf '%s\0' "${0##*/}" "$@" >> "$CONSUMER_TRACE" +printf '\0' >> "$CONSUMER_TRACE" +args=("$@") +for ((i = 0; i + 1 < ${#args[@]}; ++i)); do + case "${args[i]}" in + --prefix) mkdir -p "${args[i+1]}" ;; + -B) + case "${args[i+1]##*/}" in + invalid-python) printf '%s\n' 'requires RABITQ_BUILD_INDEX=ON' >&2; exit 1 ;; + invalid-index) printf '%s\n' 'requires RABITQ_BUILD_FASTSCAN=ON' >&2; exit 1 ;; + invalid-openmp) printf '%s\n' 'ANN indexes require RABITQ_USE_OPENMP=ON' >&2; exit 1 ;; + missing-index|missing-fastscan) printf '%s\n' 'rabitqlib component not found' >&2; exit 1 ;; + esac ;; + --test-dir) + if [[ "${args[i+1]##*/}" == producer && "${FAIL_REFERENCE:-0}" == 1 ]]; then + exit 7 + fi ;; + esac +done +""" + ) + helper.chmod(0o755) + for index, (openmp, reference, component, expected, fail) in enumerate( + [ + ("ON", "OFF", None, {"core": "OFF", "fastscan": "OFF"}, False), + ("OFF", "ON", None, {"core": "OFF", "fastscan": "ON"}, False), + ("ON", None, None, {"core": "OFF", "fastscan": "ON"}, False), + ("OFF", "OFF", "core", {"core": "OFF"}, False), + ("ON", None, "core", {"core": "ON"}, False), + ("OFF", "ON", "fastscan", {"fastscan": "ON"}, False), + ("OFF", "ON", "fastscan", {"fastscan": "ON"}, True), + ] + ): + with self.subTest( + openmp=openmp, reference=reference, component=component, fail=fail + ): + log.unlink(missing_ok=True) + command = [ + "cmake", + f"-DRABITQ_SOURCE_DIR={ROOT}", + f"-DWORK_DIR={root / str(index)}", + f"-DCMAKE_COMMAND={root / 'cmake'}", + f"-DCMAKE_CTEST_COMMAND={root / 'ctest'}", + f"-DRABITQ_USE_OPENMP={openmp}", + ] + if reference is not None: + command.append(f"-DRABITQ_RUN_REFERENCE_TESTS={reference}") + if component is not None: + command.append(f"-DRABITQ_COMPONENT={component}") + result = subprocess.run( + [*command, "-P", str(ROOT / "tests/core_consumer/run.cmake")], + env=dict( + os.environ, + CONSUMER_TRACE=str(log), + FAIL_REFERENCE=str(int(fail)), + ), + capture_output=True, + text=True, + check=False, + ) + self.assertEqual(result.returncode == 0, not fail, result.stderr) + if fail: + self.assertIn("Command failed (7)", result.stderr) + continue + calls = [ + record.decode().split("\0") + for record in log.read_bytes().split(b"\0\0") + if record + ] + producers = {} + tests = [] + for tool, *args in calls: + if tool == "cmake" and "-B" in args: + build = Path(args[args.index("-B") + 1]) + if build.name == "producer": + producers[build.parent.name] = next( + arg.split("=", 1)[1] + for arg in args + if arg.startswith("-DRABITQ_BUILD_TESTS=") + ) + if tool == "ctest": + tests.append(Path(args[args.index("--test-dir") + 1])) + self.assertEqual(producers, expected) + self.assertEqual( + {path.parent.name for path in tests if path.name == "producer"}, + {name for name, enabled in expected.items() if enabled == "ON"}, + ) + for name in expected: + self.assertEqual( + { + path.name + for path in tests + if path.parent.name == name and path.name != "producer" + }, + {"ADD_SUBDIRECTORY", "FETCHCONTENT", "PACKAGE"}, + ) + self.assertEqual( + any(path.name == "benchmark" for path in tests), + "core" in expected, + ) def test_component_partition_rejects_unknown_or_empty_selection(self): with tempfile.TemporaryDirectory() as directory: diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 4d6775e4..7020879a 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -129,23 +129,3 @@ jobs: tests/python/test_symqg.py::test_save_load_roundtrip tests/python/test_qgkmeans.py::test_train_populates_faiss_style_outputs tests/python/test_rabitqkmeans.py::test_flat_parameters_and_outputs - - faiss-comparison: - # Keep the required check name; FAISS runs against the 3.11 source installation. - name: FAISS comparison script - needs: [changes, python-build] - if: ${{ always() }} - runs-on: ubuntu-latest - timeout-minutes: 5 - steps: - - name: Require selected comparison tests - env: - CHANGES_RESULT: ${{ needs.changes.result }} - PYTHON_SELECTED: ${{ needs.changes.outputs.python_build }} - MATRIX_RESULT: ${{ needs.python-build.result }} - run: | - if [ "$CHANGES_RESULT" = success ] && [ "$PYTHON_SELECTED" = false ]; then - test "$MATRIX_RESULT" = skipped || test "$MATRIX_RESULT" = success - else - test "$MATRIX_RESULT" = success - fi diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 72b9834c..abfd4ffa 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -27,19 +27,13 @@ jobs: - name: Check formatting run: ./scripts/check-format.sh - clang-tidy-shard: + clang-tidy: needs: changes if: ${{ !cancelled() && (needs.changes.result != 'success' || needs.changes.outputs.cpp_tidy == 'true') }} - name: C++ static analysis shard ${{ matrix.shard }} + name: C++ static analysis runs-on: ubuntu-24.04 timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - shard: [0, 1] env: - TIDY_SHARD_INDEX: ${{ matrix.shard }} - TIDY_SHARD_COUNT: "2" CXX: clang++-23 steps: - uses: actions/checkout@v7 @@ -64,17 +58,6 @@ jobs: - name: Run clang-tidy run: ./scripts/check-tidy.sh build-tidy - clang-tidy: - name: C++ static analysis - needs: clang-tidy-shard - if: ${{ always() && !cancelled() && needs.clang-tidy-shard.result != 'skipped' }} - runs-on: ubuntu-latest - steps: - - name: Require both shards - env: - BUILD_RESULT: ${{ needs.clang-tidy-shard.result }} - run: test "$BUILD_RESULT" = success - shellcheck: needs: changes if: ${{ !cancelled() && (needs.changes.result != 'success' || needs.changes.outputs.shell == 'true') }} @@ -229,6 +212,7 @@ jobs: cmake -DRABITQ_SOURCE_DIR="$PWD" -DWORK_DIR="$RUNNER_TEMP/rabitq-clang-core" -DRABITQ_USE_OPENMP=${{ matrix.openmp }} + -DRABITQ_RUN_REFERENCE_TESTS=${{ matrix.openmp == 'OFF' && 'ON' || 'OFF' }} -P tests/core_consumer/run.cmake - name: Report compiler cache statistics if: ${{ always() }} @@ -360,12 +344,13 @@ jobs: run: | simd_filter='CpuFeatures.*:FloatDistance.*:MatrixDispatchTest.*:FastScan*.*:BatchEstimatorTest.*' simd_filter+=':FhtDispatchTest.*:FlipSignTest.*:KacsWalkTest.*:ExcodeIp.*:MaskIpX0Q.*' - simd_filter+=':PortableKernels.*:QueryPreparation.*:ScalarQuantize.*:WarmupIpX0Q.*:PackBinary.*' + simd_filter+=':PortableKernels.*:QueryPreparation.*:WarmupIpX0Q.*:PackBinary.*' simd_filter+=':ExplicitSimd/RabitqRescaleBackendTest.*:QGEstimatorTest.*' simd_filter+=':QGKMeansTest.TrainsAndReturnsAssignmentsForFinalCentroids' simd_filter+=':RaBitQKMeansTest.FinalModesKeepFlatLloydAssignmentsAcrossClusterCounts' simd_filter+=':IvfPersistenceTest.ReloadsRawAndQuantizedStorage' simd_filter+=':HnswSaveTest.SuccessfulSavePreservesSearchAfterLoading' + simd_filter+=':HnswPaddingTest.TailDimensionsMatchEveryAvailableSearchBackend' simd_filter+=':IvfAddTest.*:IvfRemoveTest.*:HnswRemoveTest.*' for cpu_model in hsw icl; do RABITQ_TEST_CPU="$cpu_model" "$RUNNER_TEMP/sde/sde64" "-$cpu_model" -- \ @@ -428,6 +413,7 @@ jobs: cmake -DRABITQ_SOURCE_DIR="$PWD" -DWORK_DIR="$RUNNER_TEMP/rabitq-core" -DRABITQ_USE_OPENMP=${{ matrix.openmp }} + -DRABITQ_RUN_REFERENCE_TESTS=${{ matrix.openmp == 'OFF' && matrix.component != 'core' && 'ON' || 'OFF' }} -P tests/core_consumer/run.cmake - name: Test external core-only integrations (macOS) if: runner.os == 'macOS' @@ -435,6 +421,7 @@ jobs: cmake -DRABITQ_SOURCE_DIR="$PWD" -DWORK_DIR="$RUNNER_TEMP/rabitq-core" -DRABITQ_USE_OPENMP=${{ matrix.openmp }} + -DRABITQ_RUN_REFERENCE_TESTS=${{ matrix.openmp == 'OFF' && matrix.component != 'core' && 'ON' || 'OFF' }} -DOpenMP_ROOT="$(brew --prefix)/opt/libomp" -P tests/core_consumer/run.cmake - name: Test external core-only integrations (Windows) @@ -448,6 +435,7 @@ jobs: "-DWORK_DIR=$env:RUNNER_TEMP/rabitq-core" -DRABITQ_COMPONENT=${{ matrix.component }} -DRABITQ_USE_OPENMP=${{ matrix.openmp }} + -DRABITQ_RUN_REFERENCE_TESTS=${{ matrix.openmp == 'OFF' && matrix.component != 'core' && 'ON' || 'OFF' }} -P tests/core_consumer/run.cmake - name: Report compiler cache statistics if: ${{ always() && runner.os != 'Windows' }} diff --git a/benchmarks/hotpaths/CMakeLists.txt b/benchmarks/hotpaths/CMakeLists.txt index afb26ba6..3feda7b7 100644 --- a/benchmarks/hotpaths/CMakeLists.txt +++ b/benchmarks/hotpaths/CMakeLists.txt @@ -27,4 +27,6 @@ if(RABITQ_BUILD_INDEX) "${CMAKE_CURRENT_BINARY_DIR}/ivf-add-smoke-single" 64 8 256 16 1 single) add_test(NAME IvfAddHotpathRouted COMMAND rabitq_ivf_add_hotpaths "${CMAKE_CURRENT_BINARY_DIR}/ivf-add-smoke-routed" 64 8 256 16 1 routed) + add_test(NAME IvfAddHotpathSmall COMMAND rabitq_ivf_add_hotpaths + "${CMAKE_CURRENT_BINARY_DIR}/ivf-add-smoke-small" 64 8 8 1 1 single) endif() diff --git a/benchmarks/hotpaths/README.md b/benchmarks/hotpaths/README.md index cee2a754..41d5c304 100644 --- a/benchmarks/hotpaths/README.md +++ b/benchmarks/hotpaths/README.md @@ -59,6 +59,8 @@ the deterministic input. Original production indexes are not benchmark fixtures. ## Timing and correctness contract +- IVF add verifies the expected point counts and samples new points' self-neighbors + outside the timed call, in addition to comparing output hashes across repetitions. - Inputs use an LCG with seed 42 (query/centroid seed 91 where applicable), metric L2. Dimensions for encoding/scoring are multiples of 32. Use 64, 128 and 960 dimensions; encoding rows 32/4096 and threads 1/4 retain the cases from the original investigation. diff --git a/benchmarks/hotpaths/ivf_add.cpp b/benchmarks/hotpaths/ivf_add.cpp index 7ecbe11e..de99b388 100644 --- a/benchmarks/hotpaths/ivf_add.cpp +++ b/benchmarks/hotpaths/ivf_add.cpp @@ -1,5 +1,7 @@ // Times one IVF::add() per repetition, each on a freshly loaded copy of the base index. +#include #include +#include #include #include #include @@ -65,11 +67,27 @@ void run(int argc, char** argv) { const double seconds = std::chrono::duration(bench::Clock::now() - start).count(); + if (index.num_stored_points() != base_n + add_n || + index.num_points() != base_n + add_n || + index.max_elements() != base_n + add_n) { + throw std::runtime_error("add() did not append the requested points"); + } + // Validate known self-neighbors outside timing, independently of prior runs. + for (size_t row = 0; row < std::min(add_n, size_t{3}); ++row) { + rabitqlib::PID id = rabitqlib::kPidMax; + float distance = 0; + index.search(add_data.data() + row * dim, 1, clusters, &id, &distance); + if (id != base_n + row || !std::isfinite(distance)) { + throw std::runtime_error("Added point is not searchable with its new ID"); + } + } + uint64_t h = 14695981039346656037ULL; const size_t total = index.max_elements(); h = bench::hash(&total, sizeof(total), h); - std::vector ids(10); - std::vector distances(10); + const size_t k = std::min(size_t{10}, index.num_points()); + std::vector ids(k); + std::vector distances(k); index.search(base_data.data(), ids.size(), clusters, ids.data(), distances.data()); h = bench::hash(ids.data(), ids.size() * sizeof(rabitqlib::PID), h); h = bench::hash(distances.data(), distances.size() * sizeof(float), h); diff --git a/docs/maintaining.md b/docs/maintaining.md index 77485694..5ee9046b 100644 --- a/docs/maintaining.md +++ b/docs/maintaining.md @@ -18,8 +18,10 @@ only the affected job groups: | `tests/python/test_benchmark_protocol.py` | Python lint and benchmark protocol tests; no index or wheel builds | | `.github/workflows/docs.yml` | Documentation build and Python quality checks for workflow regression tests | | `.github/workflows/pr-labels.yml` | Python quality, including metadata-classification regression tests | +| `.github/release.yml` | Python quality, including release-category compatibility tests | | `sample/run_cpp.sh`, `sample/run_python.sh` | ShellCheck, including these runners | | `tests/core_consumer/`, `tests/fixtures/core_api_v1/` | C++ checks; the frozen v1 fixture is excluded from formatting | +| `tests/core_consumer/run.cmake`, `scripts/check-format.sh`, `scripts/check-tidy.sh` | Their usual checks, plus Python quality to run their failure-path regression tests | | C++ headers or library sources | C++ formatting, clang-tidy, platform tests, sanitizers, consumer build, and Python builds/wheels | | C++ tests or examples | C++ formatting and platform tests; no clang-tidy | | `tests/allocation/`, `tests/failures/` | C++ formatting and platform tests, including sanitizer and allocation-failure coverage; no Python builds or wheels | @@ -48,12 +50,21 @@ their base. Renames and deletions are included. Relevant jobs also run if change detection fails, so a detection error cannot silently waive a check. Linux and macOS core/FastScan consumer jobs and the Clang matrix use a job-local -`ccache` through CMake's compiler launcher. Independent producer, installed-package, -`add_subdirectory`, and FetchContent builds and tests are retained; identical +`ccache` through CMake's compiler launcher. Independent producer builds and +installed-package, `add_subdirectory`, and FetchContent consumer checks are retained; identical compilations can reuse compiler output within the job. Cache statistics are printed after testing. Compiler caches are not uploaded or shared between workflow runs. Windows keeps its existing build path without a compiler cache. +Numerical reference suites have one owner per compiler/platform and OpenMP mode. +The full C++ jobs own OpenMP ON references; component consumer jobs retain their +build/install/header/symbol checks without rebuilding those suites. OpenMP OFF +runs the combined core/FastScan reference suite in the FastScan producer only. +Across the component and Clang consumer matrices, this reduces producer reference +builds from 20 to 5 while preserving all 60 consumer integration configurations. +`C++ static analysis` runs the complete configured library/binding compilation +database in one job, replacing the two shards and their summary job. + Release tags and manual wheel builds always run the full wheel build. CI routing edits run the full relevant matrix on pull requests. After an already released version reaches `main`, a push containing routing edits avoids repeated @@ -115,6 +126,11 @@ detection succeeded and explicitly selected no checks for that matrix or an unexpected skip fails the summary. If detection fails, the existing fallback runs the matrix and its success is required. +`Python validation` also covers the FAISS comparison run in the Linux 3.11 job. +When migrating from the older workflow, remove the obsolete `FAISS comparison script` +required context before dropping its duplicate summary job. Keep `Python validation` +and every other required context with their existing app IDs. + After these checks have appeared and been validated on GitHub, add their exact names to main's required status checks. Workflow definitions do not update branch protection automatically. Keep existing protections until that transition is diff --git a/scripts/check-tidy.sh b/scripts/check-tidy.sh index cea25c6c..e57fa49d 100755 --- a/scripts/check-tidy.sh +++ b/scripts/check-tidy.sh @@ -83,35 +83,23 @@ done header_filter+=")$" source_filter="^$repo_root/(src|python_bindings|sample|tests)/" -if [[ -n "${TIDY_SHARD_INDEX:-}" || -n "${TIDY_SHARD_COUNT:-}" ]]; then - source_filter="$(python - "$build_dir/compile_commands.json" "$repo_root" \ - "${TIDY_SHARD_INDEX:-}" "${TIDY_SHARD_COUNT:-}" <<'PYCODE' + +# Use the runner's path normalization and filter before accepting a no-op run. +"${PYTHON:-python3}" - "$build_dir/compile_commands.json" "$source_filter" <<'PYCODE' import json +import os import re import sys -from pathlib import Path - -database, root, index, count = sys.argv[1:] -root = Path(root).resolve() -index, count = int(index), int(count) -if count < 1 or not 0 <= index < count: - raise SystemExit("error: invalid clang-tidy shard index or count") - -entries = json.loads(Path(database).read_text()) -sources = set() -for entry in entries: - path = (Path(entry["directory"]) / entry["file"]).resolve() - if path.is_relative_to(root) and path.relative_to(root).parts[0] in { - "src", "python_bindings", "sample", "tests" - }: - sources.add(path) -shard = sorted(sources)[index::count] -if not shard: - raise SystemExit("error: clang-tidy shard has no translation units") -print("^(" + "|".join(re.escape(str(path)) for path in shard) + ")$") + +with open(sys.argv[1]) as database: + entries = json.load(database) +pattern = re.compile(sys.argv[2]) +if not any( + pattern.search(os.path.abspath(os.path.join(entry["directory"], entry["file"]))) + for entry in entries +): + sys.exit("error: clang-tidy found no matching first-party translation units") PYCODE - )" -fi tidy_output="$(mktemp)" if [[ -z "$tidy_output" || ! -f "$tidy_output" ]]; then @@ -131,8 +119,4 @@ if ! "$tidy_runner" \ exit 1 fi -if [[ -n "${TIDY_SHARD_INDEX:-}" ]]; then - echo "clang-tidy passed for shard $TIDY_SHARD_INDEX of $TIDY_SHARD_COUNT" -else - echo "clang-tidy passed for all configured first-party translation units" -fi +echo "clang-tidy passed for all configured first-party translation units" diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index ff777b92..a8fa0441 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -87,6 +87,15 @@ foreach(mode IN ITEMS dispatch scalar) endforeach() if(RABITQ_BUILD_INDEX) + if(RABITQ_BUILD_SAMPLES) + # The Python CLI integration test validates indexes without Python bindings. + add_executable(cpp_example_index_check integration/cpp_example_index_check.cpp) + target_link_libraries(cpp_example_index_check PRIVATE rabitqlib::index) + rabitq_configure_target(cpp_example_index_check) + set_target_properties(cpp_example_index_check PROPERTIES + RUNTIME_OUTPUT_DIRECTORY "${PROJECT_SOURCE_DIR}/bin") + endif() + # Binding is initialized before main; test it in its own process. add_test(NAME OpenMPBinding COMMAND rabitq_tests --gtest_filter=ThreadCount.BoundParallelForUsesRuntimePlacementAndPropagatesErrors) diff --git a/tests/README.md b/tests/README.md index 8413b300..d5cd29cf 100644 --- a/tests/README.md +++ b/tests/README.md @@ -156,13 +156,17 @@ The [Python example tests](python/test_examples.py) cover RaBitQKMeans and QGKMeans clustering, L2/IP indexing, saved cluster files, and queries without importing FAISS. The [FAISS comparison tests](python/test_compare_with_faiss.py) cover both clustering methods and skip FAISS-dependent cases when `faiss-cpu` -is absent. CI runs them in a dedicated Linux job; wheel tests do not install FAISS. +is absent. CI runs them in the Linux Python 3.11 source-install job, whose result +feeds `Python validation`; wheel tests do not install FAISS. The [C++ clustering example tests](python/test_cpp_clustering_examples.py) check saved vectors, exact labels, and compatibility with the C++ index builders. The Ubuntu C++ job runs them against its sample binaries, with `RABITQ_REQUIRE_CPP_EXAMPLES=1` so missing executables fail. Wheel jobs exclude -these tests. Locally, build the examples first; missing executables otherwise skip. +these tests. With both `RABITQ_BUILD_TESTS=ON` and `RABITQ_BUILD_SAMPLES=ON`, the build +also provides `cpp_example_index_check`, which loads each CLI-produced index and +checks its metadata and self-query results without requiring Python bindings. +Locally, build both the examples and this checker first; missing executables otherwise skip. ## Installed CMake package test @@ -212,12 +216,18 @@ continues to run all tests, including index, clustering, persistence and allocat The external consumer runner copies its CMake project and the standalone core example to separate working directories, then tests both core and FastScan with `add_subdirectory`, local-source `FetchContent`, and relocated installed packages. -It also runs each producer's reference tests (downloading the existing GoogleTest dependency). Pass absolute paths: +By default it runs references once in the FastScan producer, which includes the +core suite. A core-only partition runs its own core references. Pass absolute paths: ```text cmake -DRABITQ_SOURCE_DIR= -DWORK_DIR= -P tests/core_consumer/run.cmake ``` +Use `-DRABITQ_RUN_REFERENCE_TESTS=OFF` for integration-only checks when references +have already run in the matching full build. CI uses this for OpenMP ON; OpenMP OFF +keeps one combined reference run per compiler/platform. Consumer tests, header/symbol +checks, invalid-configuration checks and the core benchmark always remain enabled. + On macOS, also pass `-DOpenMP_ROOT=`. On Windows, set `CMAKE_GENERATOR=Visual Studio 18 2026` and `CMAKE_GENERATOR_PLATFORM=x64` in the environment. The runner builds Release with two jobs and disables native tuning. diff --git a/tests/core_consumer/run.cmake b/tests/core_consumer/run.cmake index 0d428954..ad58ee4e 100644 --- a/tests/core_consumer/run.cmake +++ b/tests/core_consumer/run.cmake @@ -1,5 +1,6 @@ # Run with -DRABITQ_SOURCE_DIR= -DWORK_DIR= -P run.cmake. # Optional -DRABITQ_COMPONENT=core|fastscan selects one independent partition. +# Set RABITQ_RUN_REFERENCE_TESTS=OFF when another job owns the numerical suite. cmake_minimum_required(VERSION 3.20) if(NOT RABITQ_SOURCE_DIR OR NOT WORK_DIR) message(FATAL_ERROR "RABITQ_SOURCE_DIR and WORK_DIR are required") @@ -16,6 +17,9 @@ set(configure_options -DCMAKE_BUILD_TYPE=Release -DRABITQ_ENABLE_NATIVE_OPTIMIZA if(NOT DEFINED RABITQ_USE_OPENMP) set(RABITQ_USE_OPENMP ON) endif() +if(NOT DEFINED RABITQ_RUN_REFERENCE_TESTS) + set(RABITQ_RUN_REFERENCE_TESTS ON) +endif() list(APPEND configure_options "-DRABITQ_USE_OPENMP=${RABITQ_USE_OPENMP}") if(NOT RABITQ_USE_OPENMP) list(APPEND configure_options -DCMAKE_DISABLE_FIND_PACKAGE_OpenMP=TRUE) @@ -36,6 +40,11 @@ if(DEFINED RABITQ_COMPONENT) set(components "${RABITQ_COMPONENT}") endif() foreach(component IN LISTS components) + set(reference_tests "${RABITQ_RUN_REFERENCE_TESTS}") + # The FastScan suite includes every core reference with the same core flags. + if(component STREQUAL "core" AND "fastscan" IN_LIST components) + set(reference_tests OFF) + endif() set(scan OFF) if(component STREQUAL "fastscan") set(scan ON) @@ -50,10 +59,12 @@ foreach(component IN LISTS components) DESTINATION "${work}/consumer") run("${CMAKE_COMMAND}" -S "${RABITQ_SOURCE_DIR}" -B "${work}/producer" ${configure_options} -DRABITQ_BUILD_INDEX=OFF - "-DRABITQ_BUILD_FASTSCAN=${scan}" -DRABITQ_BUILD_TESTS=ON) + "-DRABITQ_BUILD_FASTSCAN=${scan}" "-DRABITQ_BUILD_TESTS=${reference_tests}") run("${CMAKE_COMMAND}" --build "${work}/producer" --config Release --parallel 2) - run("${CMAKE_CTEST_COMMAND}" --test-dir "${work}/producer" -C Release - --output-on-failure --timeout 120) + if(reference_tests) + run("${CMAKE_CTEST_COMMAND}" --test-dir "${work}/producer" -C Release + --output-on-failure --no-tests=error --timeout 120) + endif() # Relocate the initial install; subsequent runs update that same test prefix. if(EXISTS "${work}/relocated") run("${CMAKE_COMMAND}" --install "${work}/producer" --config Release diff --git a/tests/integration/cpp_example_index_check.cpp b/tests/integration/cpp_example_index_check.cpp new file mode 100644 index 00000000..9a613046 --- /dev/null +++ b/tests/integration/cpp_example_index_check.cpp @@ -0,0 +1,69 @@ +#include +#include +#include +#include +#include +#include +#include + +#include "rabitqlib/core/detail/eigen.hpp" +#include "rabitqlib/core/metric.hpp" +#include "rabitqlib/index/hnsw/hnsw.hpp" +#include "rabitqlib/index/ivf/ivf.hpp" +#include "rabitqlib/index/types.hpp" +#include "rabitqlib/utils/io.hpp" + +int main(int argc, char** argv) { + try { + if (argc != 6) { + throw std::invalid_argument("Expected kind, index, data, metric and bits"); + } + const std::string kind(argv[1]); + const auto metric = + std::string(argv[4]) == "ip" ? rabitqlib::METRIC_IP : rabitqlib::METRIC_L2; + const size_t bits = std::stoull(argv[5]); + rabitqlib::RowMajorArray data; + rabitqlib::load_vecs(argv[3], data); + const size_t count = data.rows(); + const size_t dim = data.cols(); + const auto check = [&](const auto& index, const auto& search) { + if (index.dimension() != dim || index.num_stored_points() != count || + index.num_points() != count || index.metric_type() != metric || + index.nbits() != bits) { + throw std::runtime_error("Example index metadata does not match input"); + } + // Unit vectors make self the exact winner for the IP fixture as well. + for (size_t row : std::array{0, count / 2, count - 1}) { + rabitqlib::PID id = rabitqlib::kPidMax; + float distance = 0; + search(data.data() + row * dim, id, distance); + if (id != row || !std::isfinite(distance)) { + throw std::runtime_error("Example index did not retrieve its input"); + } + } + }; + if (kind == "ivf") { + rabitqlib::ivf::IVF index; + index.load(argv[2]); + check(index, [&](const float* query, rabitqlib::PID& id, float& distance) { + index.search(query, 1, index.num_clusters(), &id, &distance); + }); + } else if (kind == "hnsw") { + rabitqlib::hnsw::HierarchicalNSW index; + index.load(argv[2]); + check(index, [&](const float* query, rabitqlib::PID& id, float& distance) { + const auto hits = index.search(query, 1, 1, count, 1); + if (hits.size() != 1 || hits[0].size() != 1) { + throw std::runtime_error("Example index returned no neighbor"); + } + distance = hits[0][0].first; + id = hits[0][0].second; + }); + } else { + throw std::invalid_argument("Expected ivf or hnsw"); + } + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/tests/python/test_cpp_clustering_examples.py b/tests/python/test_cpp_clustering_examples.py index e331d583..bf9860b0 100644 --- a/tests/python/test_cpp_clustering_examples.py +++ b/tests/python/test_cpp_clustering_examples.py @@ -90,7 +90,15 @@ def test_cpp_clustering_file_outputs(tmp_path, method, k, metric): metric, ) assert result.returncode == 0, result.stdout + result.stderr - assert index_path.stat().st_size > 0 + result = run_cpp( + "cpp_example_index_check", + "ivf" if method == "qgkmeans" else "hnsw", + index_path, + source, + metric, + 3 if method == "qgkmeans" else 5, + ) + assert result.returncode == 0, result.stdout + result.stderr @pytest.mark.parametrize("method", ["qgkmeans", "rabitqkmeans"]) diff --git a/tests/python/test_hnsw.py b/tests/python/test_hnsw.py index a0c509f8..f92e23cb 100644 --- a/tests/python/test_hnsw.py +++ b/tests/python/test_hnsw.py @@ -61,13 +61,6 @@ def test_properties(built_hnsw): assert built_hnsw.num_clusters == N_CLUSTERS -def test_fast_quantization_builds(base_data, clusters): - idx = HnswIndex(DIM, N_VECTORS, M=8, ef_construction=50, nbits=4) - centroids, cluster_ids = clusters - idx.build(base_data, centroids, cluster_ids, fast_quantization=True) - assert idx.is_built - - def test_seed_reproduces_serial_build_and_search(tmp_path): rng = np.random.default_rng(47) data = rng.standard_normal((65, 65), dtype=np.float32) @@ -105,13 +98,14 @@ def test_parallel_build_preserves_self_retrieval(base_data, clusters): # ── search output shape and dtype ───────────────────────────────────────────── -def test_search_output_contract(built_hnsw, query_data): - ids, dists = built_hnsw.search(query_data, k=_TOPK, ef=_EF) - assert ids.shape == (N_QUERIES, _TOPK) - assert dists.shape == (N_QUERIES, _TOPK) +@pytest.mark.parametrize("nq,k", [(N_QUERIES, _TOPK), (1, 1)]) +def test_search_output_contract(built_hnsw, query_data, nq, k): + ids, dists = built_hnsw.search(query_data[:nq], k=k, ef=_EF) + assert ids.shape == dists.shape == (nq, k) assert ids.dtype == np.uint32 assert dists.dtype == np.float32 assert np.all(ids < N_VECTORS) + assert np.isfinite(dists).all() assert np.all(dists >= 0) @@ -125,18 +119,20 @@ def test_ef_default(built_hnsw, query_data, k): np.testing.assert_array_equal(distances, expected_distances) -def test_single_query(built_hnsw, query_data): - ids, dists = built_hnsw.search(query_data[:1], k=1, ef=_EF) - assert ids.shape == (1, 1) - - # ── search correctness ──────────────────────────────────────────────────────── -def test_self_retrieval(built_hnsw, base_data): +@pytest.mark.parametrize("fast_quantization", [False, True]) +def test_self_retrieval(built_hnsw, base_data, clusters, fast_quantization): """Each database vector must be its own nearest neighbor at high ef.""" + idx = built_hnsw + if fast_quantization: + idx = HnswIndex(DIM, N_VECTORS, M=8, ef_construction=50, nbits=4) + centroids, cluster_ids = clusters + idx.build(base_data, centroids, cluster_ids, fast_quantization=True) + assert idx.is_built probes = base_data[:10] - ids, _ = built_hnsw.search(probes, k=1, ef=200) + ids, _ = idx.search(probes, k=1, ef=200) for i in range(10): assert i in ids[i], f"Vector {i} not found in its own top-1 result" diff --git a/tests/python/test_import.py b/tests/python/test_import.py index f17e344a..612cb43b 100644 --- a/tests/python/test_import.py +++ b/tests/python/test_import.py @@ -1,57 +1,42 @@ """Tests for module import and class construction.""" import pytest -from rabitqlib import HnswIndex, IvfIndex, QGKMeans, SymqgIndex +from rabitqlib import HnswIndex, IvfIndex, SymqgIndex -# ── import ──────────────────────────────────────────────────────────────────── - -def test_all_classes_importable(): - assert HnswIndex is not None - assert IvfIndex is not None - assert SymqgIndex is not None - assert QGKMeans is not None - - -# ── valid construction ──────────────────────────────────────────────────────── - - -def test_ivf_construct_l2(): - idx = IvfIndex(64, 500, 5, nbits=4, metric="l2") - assert idx.dim == 64 - assert idx.num_clusters == 5 - assert idx.nbits == 4 - assert idx.metric == "l2" - assert not idx.is_built - - -def test_ivf_construct_ip(): - idx = IvfIndex(64, 500, 5, nbits=4, metric="ip") - assert idx.metric == "ip" - - -def test_hnsw_construct_defaults(): - idx = HnswIndex(64, 500) - assert idx.dim == 64 - assert not idx.is_built - - -def test_hnsw_construct_explicit(): - idx = HnswIndex( - 64, 500, M=8, ef_construction=50, nbits=4, metric="ip", random_seed=7 - ) - assert idx.metric == "ip" - assert idx.nbits == 4 - - -def test_symqg_construct(): - idx = SymqgIndex(64, max_degree=32) +@pytest.mark.parametrize( + "cls,args,kwargs,expected", + [ + ( + IvfIndex, + (64, 500, 5), + {"nbits": 4, "metric": "l2"}, + {"num_clusters": 5, "nbits": 4, "metric": "l2"}, + ), + (IvfIndex, (64, 500, 5), {"nbits": 4, "metric": "ip"}, {"metric": "ip"}), + (HnswIndex, (64, 500), {}, {}), + ( + HnswIndex, + (64, 500), + { + "M": 8, + "ef_construction": 50, + "nbits": 4, + "metric": "ip", + "random_seed": 7, + }, + {"metric": "ip", "nbits": 4}, + ), + (SymqgIndex, (64,), {"max_degree": 32}, {"max_degree": 32}), + ], + ids=["ivf-l2", "ivf-ip", "hnsw-default", "hnsw-explicit", "symqg"], +) +def test_constructor_contract(cls, args, kwargs, expected): + idx = cls(*args, **kwargs) assert idx.dim == 64 - assert idx.max_degree == 32 assert not idx.is_built - - -# ── invalid metric ──────────────────────────────────────────────────────────── + for name, value in expected.items(): + assert getattr(idx, name) == value @pytest.mark.parametrize( diff --git a/tests/python/test_ivf.py b/tests/python/test_ivf.py index 8261ba12..595147da 100644 --- a/tests/python/test_ivf.py +++ b/tests/python/test_ivf.py @@ -34,32 +34,23 @@ def test_properties(built_ivf): assert built_ivf.max_elements == N_VECTORS -def test_fast_quantization_builds(base_data, clusters): - idx = IvfIndex(DIM, N_VECTORS, N_CLUSTERS, nbits=4) - centroids, cluster_ids = clusters - idx.build(base_data, centroids, cluster_ids, fast_quantization=True) - assert idx.is_built - - # ── search output shape and dtype ───────────────────────────────────────────── -def test_search_output_contract(built_ivf, query_data): - ids, dists = built_ivf.search(query_data, k=_TOPK, nprobe=_NPROBE_ALL) - assert ids.shape == (N_QUERIES, _TOPK) - assert dists.shape == (N_QUERIES, _TOPK) +@pytest.mark.parametrize( + "nq,k,nprobe", + [(N_QUERIES, _TOPK, _NPROBE_ALL), (1, 1, _NPROBE_ALL), (N_QUERIES, 1, 1)], +) +def test_search_output_contract(built_ivf, query_data, nq, k, nprobe): + ids, dists = built_ivf.search(query_data[:nq], k=k, nprobe=nprobe) + assert ids.shape == dists.shape == (nq, k) assert ids.dtype == np.uint32 assert dists.dtype == np.float32 assert np.all(ids < N_VECTORS) + assert np.isfinite(dists).all() assert np.all(dists >= 0) -def test_single_query(built_ivf, query_data): - ids, dists = built_ivf.search(query_data[:1], k=1, nprobe=_NPROBE_ALL) - assert ids.shape == (1, 1) - assert dists.shape == (1, 1) - - # C++ BatchSearchMatchesIndividualQueries covers every width; exercise binding # batching for sign-only, extra-bit and owned-float storage here. @pytest.mark.parametrize("bits", [1, 4, 32]) @@ -97,12 +88,6 @@ def test_batch_search_matches_individual_queries(bits, metric): assert np.all(np.isinf(distances)) -def test_k_equals_one(built_ivf, query_data): - ids, dists = built_ivf.search(query_data, k=1, nprobe=1) - assert ids.shape == (N_QUERIES, 1) - assert dists.shape == (N_QUERIES, 1) - - def test_unfilled_results_use_sentinels(): data = np.zeros((4, DIM), dtype=np.float32) data[1:] = 100.0 @@ -120,10 +105,17 @@ def test_unfilled_results_use_sentinels(): # ── search correctness ──────────────────────────────────────────────────────── -def test_self_retrieval(built_ivf, base_data): +@pytest.mark.parametrize("fast_quantization", [False, True]) +def test_self_retrieval(built_ivf, base_data, clusters, fast_quantization): """Probing all clusters: each database vector must be its own nearest neighbor.""" + idx = built_ivf + if fast_quantization: + idx = IvfIndex(DIM, N_VECTORS, N_CLUSTERS, nbits=4) + centroids, cluster_ids = clusters + idx.build(base_data, centroids, cluster_ids, fast_quantization=True) + assert idx.is_built probes = base_data[:10] - ids, _ = built_ivf.search(probes, k=1, nprobe=_NPROBE_ALL) + ids, _ = idx.search(probes, k=1, nprobe=_NPROBE_ALL) for i in range(10): assert i in ids[i], f"Vector {i} not found in its own top-1 result" diff --git a/tests/python/test_symqg.py b/tests/python/test_symqg.py index d44d6dfb..43550aa3 100644 --- a/tests/python/test_symqg.py +++ b/tests/python/test_symqg.py @@ -39,21 +39,17 @@ def test_properties(built_symqg): # ── search output shape and dtype ───────────────────────────────────────────── -def test_search_output_contract(built_symqg, query_data): - ids, dists = built_symqg.search(query_data, k=_TOPK, ef=_EF) - assert ids.shape == (N_QUERIES, _TOPK) - assert dists.shape == (N_QUERIES, _TOPK) +@pytest.mark.parametrize("nq,k", [(N_QUERIES, _TOPK), (1, 1)]) +def test_search_output_contract(built_symqg, query_data, nq, k): + ids, dists = built_symqg.search(query_data[:nq], k=k, ef=_EF) + assert ids.shape == dists.shape == (nq, k) assert ids.dtype == np.uint32 assert dists.dtype == np.float32 assert np.all(ids < N_VECTORS) + assert np.isfinite(dists).all() assert np.all(dists >= 0) -def test_single_query(built_symqg, query_data): - ids, dists = built_symqg.search(query_data[:1], k=1, ef=_EF) - assert ids.shape == (1, 1) - - @pytest.mark.parametrize("bits", [0, 4, 8]) @pytest.mark.parametrize("metric", ["l2", "ip"]) def test_batch_search_matches_individual_queries(tmp_path, bits, metric): diff --git a/tests/unit/rabitqlib/fastscan/fastscan_test.cpp b/tests/unit/rabitqlib/fastscan/fastscan_test.cpp index 11240944..7b8af047 100644 --- a/tests/unit/rabitqlib/fastscan/fastscan_test.cpp +++ b/tests/unit/rabitqlib/fastscan/fastscan_test.cpp @@ -71,12 +71,9 @@ void check_lut_subset_sums(void (*pack_fn)(size_t, const T*, T*) = pack_lut) } } -TEST(FastScanLutTest, MatchesOrderedSubsetSumsExactly) { +TEST(FastScanLutTest, EverySupportedBackendMatchesOrderedSubsetSumsExactly) { check_lut_subset_sums(); check_lut_subset_sums(); -} - -TEST(FastScanLutTest, EverySupportedBackendMatchesOrderedSubsetSumsExactly) { check_lut_subset_sums(simd::pack_lut_generic); #if defined(__aarch64__) || defined(_M_ARM64) if (cpu::has_neon()) { diff --git a/tests/unit/rabitqlib/utils/query_preparation_test.cpp b/tests/unit/rabitqlib/utils/query_preparation_test.cpp index ad831d77..f581cf64 100644 --- a/tests/unit/rabitqlib/utils/query_preparation_test.cpp +++ b/tests/unit/rabitqlib/utils/query_preparation_test.cpp @@ -17,6 +17,19 @@ namespace rabitqlib { namespace { template void check_quantization(void (*quantize)(T*, const float*, size_t, float, float)) { + const auto check = + [&](const std::vector& input, size_t dim, float lo, float delta) { + SCOPED_TRACE(dim); + SCOPED_TRACE(delta); + std::vector actual(dim + 2, 99), expected(actual); + const float reciprocal = 1.0F / delta; + for (size_t i = 0; i < dim; ++i) { + expected[i + 1] = + static_cast(std::round((input[i + 1] - lo) * reciprocal)); + } + quantize(actual.data() + 1, input.data() + 1, dim, lo, delta); + EXPECT_EQ(actual, expected); + }; const auto max = static_cast(std::numeric_limits::max()); for (const auto& params : {std::array{0, 1}, {-3.25F, 0.125F}, {-7, 0.1F}}) { const float lo = params[0], delta = params[1]; @@ -44,18 +57,28 @@ void check_quantization(void (*quantize)(T*, const float*, size_t, float, float) 32UL, 33UL, input.size() - 1}) { - SCOPED_TRACE(dim); - SCOPED_TRACE(delta); - std::vector actual(dim + 2, 99), expected(actual); - const float reciprocal = 1.0F / delta; - for (size_t i = 0; i < dim; ++i) { - expected[i + 1] = - static_cast(std::round((input[i + 1] - lo) * reciprocal)); - } - quantize(actual.data() + 1, input.data() + 1, dim, lo, delta); - EXPECT_EQ(actual, expected); + check(input, dim, lo, delta); } } + + // Keep exact half-integers at the old SIMD/tail regression coordinates. + const std::array values{ + 0.5F, + 1.5F, + 2.5F, + 3.5F, + 4.5F, + std::nextafter(2.5F, 0.0F), + std::nextafter(2.5F, std::numeric_limits::infinity()), + 125.5F, + 126.5F, + }; + std::vector input(34); + for (size_t i = 0; i < 33; ++i) { + input[i + 1] = (i == 8 || i == 16 || i == 32) ? 2.5F : values[i % values.size()]; + } + SCOPED_TRACE("Half-integer lane boundaries"); + check(input, 33, 0.0F, 1.0F); } template diff --git a/tests/unit/rabitqlib/utils/rotator_test.cpp b/tests/unit/rabitqlib/utils/rotator_test.cpp index 4f1a7cb6..3e7550ea 100644 --- a/tests/unit/rabitqlib/utils/rotator_test.cpp +++ b/tests/unit/rabitqlib/utils/rotator_test.cpp @@ -50,21 +50,31 @@ TEST_F(RotatorTest, DefaultRotatorType) { TEST_F(RotatorTest, RejectsPaddedDimensionSmallerThanInput) { EXPECT_THROW( - (choose_rotator(dim, RotatorType::FhtKacRotator, dim / 2)), + (std::unique_ptr>( + choose_rotator(dim, RotatorType::FhtKacRotator, dim / 2) + )), std::invalid_argument ); EXPECT_THROW( - (choose_rotator(dim, RotatorType::MatrixRotator, dim / 2)), + (std::unique_ptr>( + choose_rotator(dim, RotatorType::MatrixRotator, dim / 2) + )), std::invalid_argument ); } TEST_F(RotatorTest, RejectsZeroDimension) { EXPECT_THROW( - (choose_rotator(0, RotatorType::FhtKacRotator)), std::invalid_argument + (std::unique_ptr>( + choose_rotator(0, RotatorType::FhtKacRotator) + )), + std::invalid_argument ); EXPECT_THROW( - (choose_rotator(0, RotatorType::MatrixRotator)), std::invalid_argument + (std::unique_ptr>( + choose_rotator(0, RotatorType::MatrixRotator) + )), + std::invalid_argument ); } @@ -93,12 +103,20 @@ TEST(FhtRotatorTest, PadsToThirtyTwoAndAcceptsLegacyPadding) { EXPECT_THROW((rotator_impl::FhtKacRotator(65, 72)), std::invalid_argument); EXPECT_THROW((rotator_impl::FhtKacRotator(65, 80)), std::invalid_argument); EXPECT_THROW( - (choose_rotator(65, RotatorType::FhtKacRotator, 80)), std::invalid_argument + (std::unique_ptr>( + choose_rotator(65, RotatorType::FhtKacRotator, 80) + )), + std::invalid_argument + ); + EXPECT_THROW( + (std::unique_ptr>( + choose_rotator(65, RotatorType::FhtKacRotator, 72) + )), + std::invalid_argument ); EXPECT_THROW( - (choose_rotator(65, RotatorType::FhtKacRotator, 72)), std::invalid_argument + (std::unique_ptr>(choose_rotator(63))), std::invalid_argument ); - EXPECT_THROW((choose_rotator(63)), std::invalid_argument); EXPECT_THROW( (rotator_impl::padding_requirement( std::numeric_limits::max(), RotatorType::FhtKacRotator @@ -140,48 +158,13 @@ TEST(FhtRotatorTest, RejectsUnaddressableDimensionsBeforeAllocation) { std::invalid_argument ); EXPECT_THROW( - (choose_rotator(rotator_impl::FhtKacRotator::kMaxDim + 1)), + (std::unique_ptr>( + choose_rotator(rotator_impl::FhtKacRotator::kMaxDim + 1) + )), std::invalid_argument ); } -uint8_t bitreverse8(uint8_t x) { - x = (((x & 0x55) << 1) | ((x & 0xAA) >> 1)); - x = (((x & 0x33) << 2) | ((x & 0xCC) >> 2)); - x = (((x & 0x0F) << 4) | ((x & 0xF0) >> 4)); - return x; -} -TEST(FlipSignTest, FlipWorks) { - const size_t dim = 128; - float data[dim]; - uint8_t flip[dim / 8]; // 1 bit per float - - // Initialize data and flip pattern - for (size_t i = 0; i < dim; ++i) { - data[i] = static_cast(i + 1); // Example data - } - for (size_t i = 0; i < dim / 8; ++i) { - flip[i] = static_cast(i % 256); // Example flip pattern - } - - // Perform sign flipping - rabitqlib::rotator_impl::flip_sign(flip, data, dim); - - // Output the results - uint8_t signs = 0; - for (size_t i = 0; i < dim; ++i) { - ASSERT_EQ(abs(data[i]), static_cast(i + 1)); - int sign = (data[i] < 0) ? 1 : 0; - signs = (signs << 1) | sign; - if (i % 8 == 7) { - uint8_t expected = flip[i / 8]; - signs = bitreverse8(signs); - ASSERT_EQ(static_cast(signs & 0xFF), expected); - signs = 0; - } - } -} - TEST(FhtDispatchTest, BackendsMatchScalarButterfliesAndPadding) { std::vector backends{ simd::fht_rotate_generic, simd::fht_rotate}; @@ -342,7 +325,7 @@ TEST(MatrixRotatorTest, PreservesOverlappingInputAndOutput) { TEST(FlipSignTest, BackendsPreserveBitsAndUnalignedBuffers) { std::vector backends{ - simd::flip_sign_generic, simd::flip_sign}; + simd::flip_sign_generic, rotator_impl::flip_sign}; #if defined(__aarch64__) || defined(_M_ARM64) if (cpu::has_neon()) { backends.push_back(simd::flip_sign_neon); @@ -358,20 +341,31 @@ TEST(FlipSignTest, BackendsPreserveBitsAndUnalignedBuffers) { const std::array values{ 0, 0x80000000U, 0x7f800000U, 0xff800000U, 0x7fc12345U, 0xffc54321U, 1, 0x3f812345U}; for (size_t dim : {64U, 96U, 128U, 160U}) { - for (unsigned mask = 0; mask < 256; ++mask) { + for (unsigned mask = 0; mask <= 256; ++mask) { + SCOPED_TRACE(::testing::Message() << "dim=" << dim << " mask=" << mask); std::vector flip(dim / 8 + 1); - std::fill(flip.begin(), flip.end(), static_cast(mask)); + // The final pattern retains distinct bytes from the old FlipWorks case. + for (size_t i = 0; i < dim / 8; ++i) { + flip[i + 1] = static_cast(mask == 256 ? i : mask); + } for (auto backend : backends) { std::vector data(dim + 2); + std::vector expected(dim); data.front() = data.back() = 12345; for (size_t i = 0; i < dim; ++i) { - std::memcpy(&data[i + 1], &values[i % 8], sizeof(float)); + if (mask == 256) { + data[i + 1] = static_cast(i + 1); + } else { + std::memcpy(&data[i + 1], &values[i % 8], sizeof(float)); + } + std::memcpy(&expected[i], &data[i + 1], sizeof(uint32_t)); + expected[i] ^= uint32_t{(flip[1 + i / 8] >> (i % 8)) & 1U} << 31; } backend(flip.data() + 1, data.data() + 1, dim); for (size_t i = 0; i < dim; ++i) { uint32_t actual; std::memcpy(&actual, &data[i + 1], sizeof(actual)); - EXPECT_EQ(actual, values[i % 8] ^ (((mask >> (i % 8)) & 1U) << 31)); + EXPECT_EQ(actual, expected[i]); } EXPECT_EQ(data.front(), 12345); EXPECT_EQ(data.back(), 12345); diff --git a/tests/unit/rabitqlib/utils/space_test.cpp b/tests/unit/rabitqlib/utils/space_test.cpp index 9748b1cf..95e5c77c 100644 --- a/tests/unit/rabitqlib/utils/space_test.cpp +++ b/tests/unit/rabitqlib/utils/space_test.cpp @@ -4,7 +4,6 @@ #include #include -#include #include #include #include @@ -269,47 +268,21 @@ TEST(WarmupIpX0Q, RejectsQueryWidthsBeyondByte) { } TEST(Select_IP_Func, returns_stable_function_pointer) { - auto ip_func = select_excode_ipfunc(0); - ASSERT_NE(ip_func, nullptr); - ASSERT_EQ(ip_func, select_excode_ipfunc(0)); - - ip_func = select_excode_ipfunc(1); - ASSERT_NE(ip_func, nullptr); - ASSERT_EQ(ip_func, select_excode_ipfunc(1)); - - ip_func = select_excode_ipfunc(2); - ASSERT_NE(ip_func, nullptr); - ASSERT_EQ(ip_func, select_excode_ipfunc(2)); - - ip_func = select_excode_ipfunc(3); - ASSERT_NE(ip_func, nullptr); - ASSERT_EQ(ip_func, select_excode_ipfunc(3)); - - ip_func = select_excode_ipfunc(4); - ASSERT_NE(ip_func, nullptr); - ASSERT_EQ(ip_func, select_excode_ipfunc(4)); - - ip_func = select_excode_ipfunc(5); - ASSERT_NE(ip_func, nullptr); - ASSERT_EQ(ip_func, select_excode_ipfunc(5)); - - ip_func = select_excode_ipfunc(6); - ASSERT_NE(ip_func, nullptr); - ASSERT_EQ(ip_func, select_excode_ipfunc(6)); - - ip_func = select_excode_ipfunc(7); - ASSERT_NE(ip_func, nullptr); - ASSERT_EQ(ip_func, select_excode_ipfunc(7)); - - ip_func = select_excode_ipfunc(8); - ASSERT_NE(ip_func, nullptr); + for (size_t bits = 0; bits <= 8; ++bits) { + SCOPED_TRACE(bits); + const auto ip_func = select_excode_ipfunc(bits); + ASSERT_NE(ip_func, nullptr); + ASSERT_EQ(ip_func, select_excode_ipfunc(bits)); #if defined(__x86_64__) || defined(_M_X64) - if (cpu::has_avx512_core()) { - ASSERT_EQ(ip_func, simd::excode_ipimpl::ip16_fxu8_avx512); - } else if (cpu::has_avx2()) { - ASSERT_EQ(ip_func, simd::excode_ipimpl::ip16_fxu8_avx2); - } + if (bits == 8) { + if (cpu::has_avx512_core()) { + ASSERT_EQ(ip_func, simd::excode_ipimpl::ip16_fxu8_avx512); + } else if (cpu::has_avx2()) { + ASSERT_EQ(ip_func, simd::excode_ipimpl::ip16_fxu8_avx2); + } + } #endif + } } TEST(Select_IP_Func, zero_ex_bits_contributes_nothing) { @@ -329,52 +302,6 @@ TEST(Select_IP_Func, zero_ex_bits_contributes_nothing) { ASSERT_EQ(ip_func(query.data(), codes.data(), dim), 0.0F); } -constexpr size_t kScalarQuantizeTestDim = 33; - -TEST(ScalarQuantize, HalfValuesMatchScalarAcrossVectorBoundaries) { - constexpr size_t dim = kScalarQuantizeTestDim; - const std::array values{ - 0.5F, - 1.5F, - 2.5F, - 3.5F, - 4.5F, - std::nextafter(2.5F, 0.0F), - std::nextafter(2.5F, std::numeric_limits::infinity()), - 125.5F, - 126.5F, - }; - std::array input{}; - std::array expected8{}; - std::array expected16{}; - for (size_t i = 0; i < dim; ++i) { - input[i] = (i == 8 || i == 16 || i == 32) ? 2.5F : values[i % values.size()]; - expected8[i] = static_cast(std::round(input[i])); - expected16[i] = static_cast(std::round(input[i])); - } - - const auto check = [&](auto quantize8, auto quantize16) { - std::array actual8{}; - std::array actual16{}; - quantize8(actual8.data(), input.data(), dim, 0.0F, 1.0F); - quantize16(actual16.data(), input.data(), dim, 0.0F, 1.0F); - EXPECT_EQ(actual8, expected8); - EXPECT_EQ(actual16, expected16); - }; - - check(simd::scalar_quantize_uint8, simd::scalar_quantize_uint16); -#if defined(__x86_64__) || defined(_M_X64) - if (cpu::has_avx2()) { - check(simd::scalar_quantize_uint8_avx2, simd::scalar_quantize_uint16_avx2); - } -#endif -#if defined(__x86_64__) || defined(_M_X64) - if (cpu::has_avx512_core()) { - check(simd::scalar_quantize_uint8_avx512, simd::scalar_quantize_uint16_avx512); - } -#endif -} - TEST(ip16_fxu1_avx, ip_works) { constexpr size_t dim = 64; float query[dim]; From 55ad615bae358f9b8dd5579246ce20392d3cb8a8 Mon Sep 17 00:00:00 2001 From: gouyt13clear Date: Sun, 25 Oct 2026 21:49:09 +0800 Subject: [PATCH 2/2] ci: reduce sanitizer debug info and bound IVF emulation coverage --- .github/workflows/test.yaml | 7 ++- tests/README.md | 9 ++- tests/unit/rabitqlib/index/ivf_test.cpp | 82 +++++++++++++++++++++++++ 3 files changed, 93 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index abfd4ffa..9c51a716 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -277,7 +277,7 @@ jobs: run: | cmake --version ninja --version - # Keep assertions/debug symbols while avoiding unoptimized sanitizer loops. + # Keep assertions and source locations without full variable debug information. - name: Configure run: >- cmake -S . -B build-sanitizers -G Ninja @@ -286,7 +286,7 @@ jobs: -DRABITQ_ENABLE_NATIVE_OPTIMIZATION=OFF -DRABITQ_ENABLE_SANITIZERS=ON -DCMAKE_BUILD_TYPE=Debug - -DCMAKE_CXX_FLAGS_DEBUG="-O1 -g -fno-optimize-sibling-calls" + -DCMAKE_CXX_FLAGS_DEBUG="-O1 -g1 -fno-optimize-sibling-calls" - name: Build run: cmake --build build-sanitizers - name: Test @@ -319,6 +319,7 @@ jobs: run: | cmake --version ninja --version + lscpu - name: Build RaBitQ-Library with portable tests run: | cmake -S . -B build-install -G Ninja \ @@ -351,7 +352,7 @@ jobs: simd_filter+=':IvfPersistenceTest.ReloadsRawAndQuantizedStorage' simd_filter+=':HnswSaveTest.SuccessfulSavePreservesSearchAfterLoading' simd_filter+=':HnswPaddingTest.TailDimensionsMatchEveryAvailableSearchBackend' - simd_filter+=':IvfAddTest.*:IvfRemoveTest.*:HnswRemoveTest.*' + simd_filter+=':IvfLifecycleTest.BatchTailSurvivesAddRemoveAndReload:HnswRemoveTest.*' for cpu_model in hsw icl; do RABITQ_TEST_CPU="$cpu_model" "$RUNNER_TEMP/sde/sde64" "-$cpu_model" -- \ build-install/tests/rabitq_tests --gtest_filter="$simd_filter" diff --git a/tests/README.md b/tests/README.md index d5cd29cf..503eef3c 100644 --- a/tests/README.md +++ b/tests/README.md @@ -121,8 +121,9 @@ applicable platforms. Passing on an AVX2 machine does not verify AVX-512 executi The installed-consumer job runs the full portable Linux Release suite. The Ubuntu native build runs SIMD and clustering smoke tests; Windows, ARM64, and ASan/UBSan jobs retain their full C++ suites. The sanitizer job uses Debug with -`-O1 -g -fno-optimize-sibling-calls`, retaining assertions, frame pointers, leak -checking, and both ASan/UBSan while avoiding unoptimized clustering loops. +`-O1 -g1 -fno-optimize-sibling-calls`, retaining assertions, source locations, +frame pointers, leak checking, and both ASan/UBSan. Minimal debug information +reduces compilation work while omitting full variable/type information. The portable Linux job also reuses its test binary under pinned [Intel SDE](https://www.intel.com/content/www/us/en/download/684897/intel-software-development-emulator.html) @@ -130,6 +131,10 @@ CPU models: Haswell (`-hsw`, AVX2 only) and Ice Lake (`-icl`, AVX-512 including VPOPCNTDQ). Kernel reference tests and small clustering/index tests exercise real runtime dispatch under emulation. `RABITQ_TEST_CPU` checks the emulated features and prints them; an incorrect CPU model fails instead of silently losing coverage. +IVF emulation uses a 31-to-33-point append/remove/reload case across L2/IP, +sign-only, extra-bit and raw storage, and standard/HACC scans. Full IVF lifecycle +combinations remain in the native Release and sanitizer suites. The job prints +the runner CPU information before applying SDE's emulated CPU models. These are correctness checks, not hardware performance measurements. Source-install Python jobs check imports, index persistence, and both clustering diff --git a/tests/unit/rabitqlib/index/ivf_test.cpp b/tests/unit/rabitqlib/index/ivf_test.cpp index a79862c9..ba13e93f 100644 --- a/tests/unit/rabitqlib/index/ivf_test.cpp +++ b/tests/unit/rabitqlib/index/ivf_test.cpp @@ -606,6 +606,88 @@ Hits search_everything( return hits; } +TEST(IvfLifecycleTest, BatchTailSurvivesAddRemoveAndReload) { + constexpr size_t kOld = 31, kCount = 33; + const auto points = random_rows(kCount, 753); + const auto query = random_rows(1, 754); + const std::vector centroid(kDynDim, 0.0F); + const std::vector assigned(kCount, 0); + const std::array removed{0, kCount - 1}; + const std::string path = ::testing::TempDir() + "rabitq_ivf_lifecycle.index"; + for (auto metric : {METRIC_L2, METRIC_IP}) { + for (size_t bits : {1U, 4U, 9U, 32U}) { + for (bool hacc : {false, true}) { + SCOPED_TRACE(::testing::Message() << metric << "/" << bits << "/" << hacc); + IVF index(kOld, kDynDim, 1, bits, metric); + index.construct(points.data(), centroid.data(), assigned.data(), false, 1); + index.add( + points.data() + kOld * kDynDim, + kCount - kOld, + assigned.data() + kOld, + false, + 1 + ); + ASSERT_EQ(index.num_points(), kCount); + const auto search = [&](const IVF& source) { + Hits hits{ + std::vector(source.num_points()), + std::vector(source.num_points())}; + source.search( + query.data(), + hits.ids.size(), + 1, + hits.ids.data(), + hits.distances.data(), + hacc + ); + return hits; + }; + const auto before = search(index); + auto ids = before.ids; + std::sort(ids.begin(), ids.end()); + for (size_t row = 0; row < kCount; ++row) { + ASSERT_EQ(ids[row], row); + EXPECT_TRUE(std::isfinite(before.distances[row])); + if (bits == 32) { + EXPECT_NEAR( + before.distances[row], + exact_distance( + query.data(), + points.data() + before.ids[row] * kDynDim, + metric + ), + 1e-3 + ); + } + } + ASSERT_EQ(index.remove(removed.data(), removed.size()), removed.size()); + ASSERT_EQ(index.num_points(), kCount - removed.size()); + const auto after = search(index); + size_t next = 0; + for (size_t row = 0; row < kCount; ++row) { + if (before.ids[row] == removed[0] || before.ids[row] == removed[1]) { + continue; + } + ASSERT_LT(next, after.ids.size()); + EXPECT_EQ(after.ids[next], before.ids[row]); + EXPECT_EQ(after.distances[next], before.distances[row]); + ++next; + } + EXPECT_EQ(next, after.ids.size()); + index.save(path.c_str()); + IVF loaded; + loaded.load(path.c_str()); + EXPECT_EQ(loaded.metric_type(), metric); + EXPECT_EQ(loaded.nbits(), bits); + const auto restored = search(loaded); + EXPECT_EQ(restored.ids, after.ids); + EXPECT_EQ(restored.distances, after.distances); + } + } + } + std::remove(path.c_str()); +} + TEST(IvfAddTest, EveryWayOfAddingTheSamePointsGivesByteIdenticalIndexes) { constexpr size_t kOld = 200; constexpr size_t kNew = 200;