diff --git a/AGENTS.md b/AGENTS.md index 17a83263..8f0b56e5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -170,6 +170,9 @@ validated sizes before allocation, checked reads, and a compatibility fixture or path. Review each affected index format; preserve SymphonyQG's versioned quantized format and legacy raw fallback unless a breaking change is authorized. +Frozen release-produced indexes and reference searches live in `tests/fixtures/persistence/`; +keep their provenance and hashes, and do not regenerate them with the current writer. + Use `python_bindings/bindings_common.hpp` for shared conversions. Validate array rank, dimensions, index state, and parameter ranges before entering the core. `py::array::forcecast` permits copies; do not use it where callers expect in-place mutation or pointer identity. diff --git a/docs/docs/quick_start.md b/docs/docs/quick_start.md index b086dbfa..456a5dac 100644 --- a/docs/docs/quick_start.md +++ b/docs/docs/quick_start.md @@ -187,6 +187,16 @@ before starting the program. Index save/load paths are UTF-8 strings on Windows and native path bytes on POSIX in C++; Python paths are Unicode strings on all platforms. +### Index file compatibility + +The compatibility suite loads frozen files written by v0.5.2, compares search +IDs and distances, and verifies save/load migration and rejection of truncated +or unsupported versions. See the +[fixture provenance and coverage](https://github.com/VectorDB-NTU/RaBitQ-Library/tree/main/tests/fixtures/persistence). +Compatibility is format-specific; preserving old readers' ability to open new +files is not guaranteed. Consult each index's format documentation before +downgrading the library. + ### Build the Python bindings from source Source builds require Python 3.11 or newer, a C++17 compiler, CMake 3.20 or diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index e03bcf03..1c8f0eee 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -27,6 +27,9 @@ add_executable(rabitq_tests ${INTEG_TESTS} ) target_include_directories(rabitq_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/common) +set_source_files_properties(unit/rabitqlib/index/historical_persistence_test.cpp + PROPERTIES COMPILE_DEFINITIONS + "RABITQ_TEST_FIXTURE_DIR=\"${CMAKE_CURRENT_SOURCE_DIR}/fixtures/persistence/v0.5.2\"") rabitq_configure_target(rabitq_tests) # Link google test and rabitqlib headers diff --git a/tests/README.md b/tests/README.md index 469c1897..e614c588 100644 --- a/tests/README.md +++ b/tests/README.md @@ -3,6 +3,11 @@ This directory contains the C++ unit and integration tests and the Python binding tests for RaBitQ Library. +Persistence tests include [frozen release-produced indexes](fixtures/persistence/README.md) +and search results, in addition to current round trips. C++ and installed-wheel +tests load the same fixtures, compare search results, and verify save/load migration +and rejection of truncated or unsupported formats. + ## Prerequisites - CMake 3.20 or newer for the `ctest --test-dir` commands below (4.2 or newer diff --git a/tests/fixtures/persistence/.gitattributes b/tests/fixtures/persistence/.gitattributes new file mode 100644 index 00000000..df5ad8ed --- /dev/null +++ b/tests/fixtures/persistence/.gitattributes @@ -0,0 +1,3 @@ +*.index -text +*.expected text eol=lf +manifest.json text eol=lf diff --git a/tests/fixtures/persistence/CMakeLists.txt b/tests/fixtures/persistence/CMakeLists.txt new file mode 100644 index 00000000..34c51b95 --- /dev/null +++ b/tests/fixtures/persistence/CMakeLists.txt @@ -0,0 +1,13 @@ +cmake_minimum_required(VERSION 3.20) +project(rabitq_historical_fixtures LANGUAGES CXX) + +if(NOT IS_DIRECTORY "${RABITQ_HISTORICAL_SOURCE}") + message(FATAL_ERROR "Set RABITQ_HISTORICAL_SOURCE to the pinned historical checkout") +endif() +set(RABITQ_BUILD_TESTS OFF CACHE BOOL "" FORCE) +set(RABITQ_BUILD_SAMPLES OFF CACHE BOOL "" FORCE) +set(RABITQ_BUILD_PYTHON_BINDINGS OFF CACHE BOOL "" FORCE) +set(RABITQ_ENABLE_NATIVE_OPTIMIZATION OFF CACHE BOOL "" FORCE) +add_subdirectory("${RABITQ_HISTORICAL_SOURCE}" historical) +add_executable(generate_fixtures generate.cpp) +target_link_libraries(generate_fixtures PRIVATE rabitqlib::rabitqlib) diff --git a/tests/fixtures/persistence/README.md b/tests/fixtures/persistence/README.md new file mode 100644 index 00000000..e4858802 --- /dev/null +++ b/tests/fixtures/persistence/README.md @@ -0,0 +1,55 @@ +# Historical persistence fixtures + +These files were written by release **v0.5.2**, commit +`d929e30dbb7ded52db1830736a2356fbc7dba99a`, using [generate.cpp](generate.cpp). +Tests load committed bytes; they do not generate indexes with the current writer. +[manifest.json](manifest.json) records the producer and SHA-256 hashes of every +index and its reference search results. Keep existing fixtures unchanged when +adding a new format or release. + +The twelve cases cover L2 and inner product for: + +- IVF four-bit legacy files without a header, and raw-vector IVF v1 files; +- HNSW four-bit files, including persisted removal of point 0; +- SymphonyQG legacy raw files and versioned four/eight-bit files. + +IVF also persists removal of point 0. Each file contains 33 points in 65 original +dimensions and the historical **128-dimensional padded domain**. This differs +from current construction's 96 dimensions and detects accidental regeneration +of rotation or padding on load. The `.expected` files hold the point count, +dimension, query count, k, original queries, and ordered ID/distance pairs from +the historical implementation. + +Tests require identical result IDs and allow distance error of +`2e-4 * max(1, abs(reference))` in C++, or `rtol=atol=2e-4` in Python, to account +for compiler and SIMD reductions. The native formats contain `size_t`: these +fixtures target the supported little-endian, 64-bit platforms, not a portable +interchange format for arbitrary architectures. Python also tests synthetic IVF +v1 headers around the frozen historical payload, unknown versions, and invalid +padding. That header test is distinct from release-produced fixtures. + +The small IVF fixtures use Flat centroid routing. Current round-trip and failure +tests exercise explicit HNSW routing and its sidecar; historical auto-selected +HNSW routing at 20,000 clusters is not represented by these small fixtures. + +## Deliberate regeneration + +Use an isolated copy of the pinned release. Do not point this build at the +current source tree. For example, from the repository root: + +```bash +mkdir -p build/fixtures/v0.5.2 +git archive v0.5.2 | tar -x -C build/fixtures/v0.5.2 +cmake -S tests/fixtures/persistence -B build/fixtures/generator \ + -DRABITQ_HISTORICAL_SOURCE="$PWD/build/fixtures/v0.5.2" \ + -DCMAKE_BUILD_TYPE=Release +cmake --build build/fixtures/generator --config Release --parallel 2 +``` + +Run the resulting `generate_fixtures` executable with a **new output directory**. +On Windows it is under `build/fixtures/generator/Release/`. Review the new +artifacts and record their hashes and compiler/ISA in a separate manifest before +adopting them. Some historical constructors seed rotation from `random_device`, +so rebuilding does not promise byte-identical files; each committed index and its +reference results form a fixed pair. `.gitattributes` prevents newline conversion +from changing fixture hashes on Windows. diff --git a/tests/fixtures/persistence/generate.cpp b/tests/fixtures/persistence/generate.cpp new file mode 100644 index 00000000..548aa5ba --- /dev/null +++ b/tests/fixtures/persistence/generate.cpp @@ -0,0 +1,96 @@ +// Build only against the pinned historical source, never the current checkout. +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "rabitqlib/index/hnsw/hnsw.hpp" +#include "rabitqlib/index/ivf/ivf.hpp" +#include "rabitqlib/index/symqg/qg.hpp" +#include "rabitqlib/index/symqg/qg_builder.hpp" + +int main(int argc, char** argv) { + if (argc != 2) { + throw std::invalid_argument("Pass the fixture output directory"); + } + using namespace rabitqlib; + constexpr size_t kCount = 33, kDim = 65, kQueries = 3, kTopK = 5; + const std::filesystem::path directory(argv[1]); + std::filesystem::create_directories(directory); + uint32_t state = 713; + const auto next = [&state] { + state = state * 1664525U + 1013904223U; + return static_cast(static_cast((state >> 8) % 2001) - 1000) / 256.0F; + }; + std::vector data(kCount * kDim), queries(kQueries * kDim), centroid(kDim); + for (auto& value : data) + value = next(); + for (auto& value : queries) + value = next(); + for (size_t i = 0; i < kCount; ++i) { + for (size_t d = 0; d < kDim; ++d) + centroid[d] += data[i * kDim + d] / kCount; + } + std::vector clusters(kCount, 0); + for (auto metric : {METRIC_L2, METRIC_IP}) { + for (const std::string kind : {"ivf4", "ivf32", "hnsw4", "qg0", "qg4", "qg8"}) { + const std::string name = kind + (metric == METRIC_L2 ? "_l2" : "_ip"); + const auto path = (directory / (name + ".index")).string(); + std::vector ids(kQueries * kTopK); + std::vector distances(ids.size()); + if (kind.substr(0, 3) == "ivf") { + ivf::IVF index(kCount, kDim, 1, kind == "ivf32" ? 32 : 4, metric); + index.construct(data.data(), centroid.data(), clusters.data(), false, 1); + const PID removed = 0; + index.remove(&removed, 1); + index.save(path.c_str()); + index.search_batch( + queries.data(), kQueries, kTopK, 1, ids.data(), distances.data() + ); + } else if (kind == "hnsw4") { + hnsw::HierarchicalNSW index(kCount, kDim, 4, 8, 64, 42, metric); + index.construct( + 1, centroid.data(), kCount, data.data(), clusters.data(), 1 + ); + const PID removed = 0; + index.remove(&removed, 1); + index.save(path.c_str()); + const auto result = index.search(queries.data(), kQueries, kTopK, 64, 1); + for (size_t i = 0; i < kQueries; ++i) { + for (size_t j = 0; j < kTopK; ++j) { + ids[i * kTopK + j] = result[i][j].second; + distances[i * kTopK + j] = result[i][j].first; + } + } + } else { + const size_t bits = kind == "qg0" ? 0 : (kind == "qg4" ? 4 : 8); + symqg::QuantizedGraph index( + kCount, kDim, 32, metric, RotatorType::FhtKacRotator, bits, 42 + ); + symqg::QGBuilder builder( + index, 64, data.data(), 1, symqg::QGInitialization::PiPNN, 42 + ); + builder.build(); + index.save(path.c_str()); + index.set_ef(64); + index.search_batch( + queries.data(), kQueries, kTopK, ids.data(), distances.data(), 1 + ); + } + std::ofstream expected(directory / (name + ".expected"), std::ios::binary); + expected.exceptions(std::ios::failbit | std::ios::badbit); + expected << std::setprecision(std::numeric_limits::max_digits10); + expected << kCount << ' ' << kDim << ' ' << kQueries << ' ' << kTopK << '\n'; + for (size_t i = 0; i < queries.size(); ++i) + expected << queries[i] << (i + 1 == queries.size() ? '\n' : ' '); + for (size_t i = 0; i < ids.size(); ++i) + expected << ids[i] << ' ' << distances[i] << '\n'; + expected.close(); + } + } +} diff --git a/tests/fixtures/persistence/manifest.json b/tests/fixtures/persistence/manifest.json new file mode 100644 index 00000000..635f7aeb --- /dev/null +++ b/tests/fixtures/persistence/manifest.json @@ -0,0 +1,34 @@ +{ + "source_tag": "v0.5.2", + "source_commit": "d929e30dbb7ded52db1830736a2356fbc7dba99a", + "writer": "generate.cpp", + "producer": "Windows x86-64, MSVC 19.51.36257, AVX2, one thread, native optimization OFF", + "byte_order": "little", + "size_t_bytes": 8, + "sha256": { + "v0.5.2/hnsw4_ip.expected": "b4cace8e528f3934167b3a3724d0d5a4a1f6b9aca26317b7f6eed3ae522e4497", + "v0.5.2/hnsw4_ip.index": "6015e29fed514ba7c6b90ccd538218a1719cdfac5c548da837e85af1012818ba", + "v0.5.2/hnsw4_l2.expected": "632a19638aa7f7c716e3310f21b28773d5956d708295ff821ff861084a67020d", + "v0.5.2/hnsw4_l2.index": "8161c900b3b85ba00b6aa89a9c24740fce94541bdee0e7b1dbe23073156cad88", + "v0.5.2/ivf32_ip.expected": "e0d953927d08b619c75b9424a032f136d32d3756d3fde4bf8c6abe0b7b743a28", + "v0.5.2/ivf32_ip.index": "f98eabdaf408569ffeb4480b4c5f12d8a2b893f970646f9c26a7773ff453b93d", + "v0.5.2/ivf32_l2.expected": "a9ba0430f4c8a517168aa1ff75bad72d7f6089161561412a75976fc970400f9c", + "v0.5.2/ivf32_l2.index": "c395ace9210c79ea55fa7f842920413cf9848aba02b6c0f3cbee30538a20c4c2", + "v0.5.2/ivf4_ip.expected": "6cc51bd995a4d6dad7f1467074de13d1d8d129895c90e67bf3ade771b6ad741d", + "v0.5.2/ivf4_ip.index": "34135508210fa39db36b7ccef7f04b029d5d9185d35be702982682d83137ab8a", + "v0.5.2/ivf4_l2.expected": "e7881505994bbb518c75d466dffcba2a5fa4dc81d3782fa6417d0c3ccc965e26", + "v0.5.2/ivf4_l2.index": "477c0f37502efc0a7a99587eb3bc0abe24b8b11d465b35aed87a60eb7a48b0c2", + "v0.5.2/qg0_ip.expected": "f7b9c5d1884c21b7cef507b43ee13c69fc1e5037168d1e432a12aa1e6ccedddb", + "v0.5.2/qg0_ip.index": "3f22cbc4a7daa3fdf9024fa8afb38de8ea26216bb5ebb6d4d3d92d15b83b08cf", + "v0.5.2/qg0_l2.expected": "ddfe995d8c67d4bd7123dbe8b8eef60b713899d5c58b0bfada112b4c965c9319", + "v0.5.2/qg0_l2.index": "50b28977edb17048744dd6c2039963bb550ecc471076d0c98c1bcb47f8d08ea8", + "v0.5.2/qg4_ip.expected": "ea91f5dd31b7c395998b16d72e645708d5d1b523d5a9f68e8c1b760679ee4cd4", + "v0.5.2/qg4_ip.index": "8039c3b58d927cc281bc2e9785b4efc2bb9a046bb06189d8e3040fe87d032227", + "v0.5.2/qg4_l2.expected": "6913d262200c128d20940bb2223354d45b51749d137e6344588091aa039e5495", + "v0.5.2/qg4_l2.index": "77554486d9b0de7beb414d1a4d6201dc0c13d1f1c8afda662bdbc732113b63f0", + "v0.5.2/qg8_ip.expected": "4151968625f09edba7dd0061acef59368094890ab3975a551ffe723b4133193a", + "v0.5.2/qg8_ip.index": "542a8e86cc686349406896c3838d889ee7873f6edc7de1c09b5806176846a5f6", + "v0.5.2/qg8_l2.expected": "7385b1a5396c84e994821a0a512ff039337852c9a4e0d2075c1cbbe3246cef2a", + "v0.5.2/qg8_l2.index": "be5ea3d4fd3d61059d7ec985a79eaadbc3d3e7f744042707a7df0d846e2d1efb" + } +} diff --git a/tests/fixtures/persistence/v0.5.2/hnsw4_ip.expected b/tests/fixtures/persistence/v0.5.2/hnsw4_ip.expected new file mode 100644 index 00000000..b9a79485 --- /dev/null +++ b/tests/fixtures/persistence/v0.5.2/hnsw4_ip.expected @@ -0,0 +1,17 @@ +33 65 3 5 +-1.2578125 -3.078125 -3.328125 0.67578125 3.171875 -0.2578125 -2.796875 0.91015625 -3.859375 0.328125 2.296875 -2.0859375 -0.66015625 -3.3046875 2.19140625 2.1484375 2.328125 3.7890625 3.3515625 0.08203125 1.41796875 1.43359375 -1.1484375 1.4375 2.55078125 2.0703125 -1.5859375 -2.8046875 -2.2890625 1.73828125 -2.41796875 -3.09765625 1.53515625 -3.6953125 -0.66796875 0.67578125 0.96484375 -2.9296875 2.41796875 1.25390625 -1.8984375 3.14453125 2.359375 0.65625 -2.52734375 -0.17578125 -0.80859375 -2.171875 -1.63671875 -0.421875 -3.08203125 -0.1328125 -1.94140625 2.734375 2.1875 -2.1015625 3.828125 3.82421875 -3.61328125 -0.9921875 -3.27734375 2.9765625 2.29296875 0.7421875 3.48828125 -2.58203125 1.0234375 -3.12109375 1.83203125 -2.29296875 1.05859375 2.15234375 -2.5390625 -2.3125 -2.7265625 -3.33203125 1.671875 3.6796875 3.53515625 -1.75 0.6328125 -3.1171875 -2.6875 -0.3671875 -2.640625 -2.52734375 -0.65625 1.89453125 -1.08984375 0.25390625 -3.65234375 1.08203125 -0.55078125 3.6171875 2.80078125 2.265625 2.80078125 0.98828125 -1.03515625 1.0234375 -0.04296875 2.95703125 1.8125 -1.6796875 2.75390625 -3.70703125 1.60546875 0.27734375 -3.2265625 -2.57421875 -0.875 1.40234375 -3.55078125 2.80859375 2.953125 3.37109375 -0.02734375 3.16796875 3.23828125 -0.21875 0.57421875 -2.82421875 3.80078125 1.66796875 -3.30859375 -1.21875 1.05078125 1.0546875 2.2890625 3.84765625 1.58984375 -0.7734375 -0.48046875 1.28125 -1.30859375 -1.27734375 2.67578125 1.19921875 2.3515625 -0.140625 -3.828125 -3.84375 1.0703125 -2.27734375 -2.05859375 3.59375 1.3359375 -3.44140625 -0.40234375 -0.26953125 1.671875 -3.24609375 -3.42578125 0.76953125 -0.67578125 -1.53125 -3.2578125 3.65234375 -2.9921875 3.4609375 0.25390625 1.5390625 -0.78125 2.109375 0.99609375 1.5078125 -0.33203125 -1.4296875 -0.01953125 -0.625 2.609375 0.796875 -1.1875 1.125 3.62890625 -3.23046875 -2.91015625 3.0859375 2.23828125 -3.6015625 -1.38671875 3.0703125 -2.2890625 -3.375 3.0390625 -2.02734375 -0.9140625 1 -0.0546875 0.66796875 -3.52734375 -2.8046875 0.93359375 -3.21875 -2.05859375 +17 -107.63942 +30 -97.97155 +26 -96.9916306 +9 -67.9665985 +12 -56.8626518 +10 -88.5409393 +4 -52.2220459 +24 -42.5000038 +29 -40.7176285 +23 -28.9736595 +8 -71.0609741 +31 -47.7379608 +14 -38.9586029 +26 -37.554657 +19 -36.1280594 diff --git a/tests/fixtures/persistence/v0.5.2/hnsw4_ip.index b/tests/fixtures/persistence/v0.5.2/hnsw4_ip.index new file mode 100644 index 00000000..992ba0c3 Binary files /dev/null and b/tests/fixtures/persistence/v0.5.2/hnsw4_ip.index differ diff --git a/tests/fixtures/persistence/v0.5.2/hnsw4_l2.expected b/tests/fixtures/persistence/v0.5.2/hnsw4_l2.expected new file mode 100644 index 00000000..75dd4c9c --- /dev/null +++ b/tests/fixtures/persistence/v0.5.2/hnsw4_l2.expected @@ -0,0 +1,17 @@ +33 65 3 5 +-1.2578125 -3.078125 -3.328125 0.67578125 3.171875 -0.2578125 -2.796875 0.91015625 -3.859375 0.328125 2.296875 -2.0859375 -0.66015625 -3.3046875 2.19140625 2.1484375 2.328125 3.7890625 3.3515625 0.08203125 1.41796875 1.43359375 -1.1484375 1.4375 2.55078125 2.0703125 -1.5859375 -2.8046875 -2.2890625 1.73828125 -2.41796875 -3.09765625 1.53515625 -3.6953125 -0.66796875 0.67578125 0.96484375 -2.9296875 2.41796875 1.25390625 -1.8984375 3.14453125 2.359375 0.65625 -2.52734375 -0.17578125 -0.80859375 -2.171875 -1.63671875 -0.421875 -3.08203125 -0.1328125 -1.94140625 2.734375 2.1875 -2.1015625 3.828125 3.82421875 -3.61328125 -0.9921875 -3.27734375 2.9765625 2.29296875 0.7421875 3.48828125 -2.58203125 1.0234375 -3.12109375 1.83203125 -2.29296875 1.05859375 2.15234375 -2.5390625 -2.3125 -2.7265625 -3.33203125 1.671875 3.6796875 3.53515625 -1.75 0.6328125 -3.1171875 -2.6875 -0.3671875 -2.640625 -2.52734375 -0.65625 1.89453125 -1.08984375 0.25390625 -3.65234375 1.08203125 -0.55078125 3.6171875 2.80078125 2.265625 2.80078125 0.98828125 -1.03515625 1.0234375 -0.04296875 2.95703125 1.8125 -1.6796875 2.75390625 -3.70703125 1.60546875 0.27734375 -3.2265625 -2.57421875 -0.875 1.40234375 -3.55078125 2.80859375 2.953125 3.37109375 -0.02734375 3.16796875 3.23828125 -0.21875 0.57421875 -2.82421875 3.80078125 1.66796875 -3.30859375 -1.21875 1.05078125 1.0546875 2.2890625 3.84765625 1.58984375 -0.7734375 -0.48046875 1.28125 -1.30859375 -1.27734375 2.67578125 1.19921875 2.3515625 -0.140625 -3.828125 -3.84375 1.0703125 -2.27734375 -2.05859375 3.59375 1.3359375 -3.44140625 -0.40234375 -0.26953125 1.671875 -3.24609375 -3.42578125 0.76953125 -0.67578125 -1.53125 -3.2578125 3.65234375 -2.9921875 3.4609375 0.25390625 1.5390625 -0.78125 2.109375 0.99609375 1.5078125 -0.33203125 -1.4296875 -0.01953125 -0.625 2.609375 0.796875 -1.1875 1.125 3.62890625 -3.23046875 -2.91015625 3.0859375 2.23828125 -3.6015625 -1.38671875 3.0703125 -2.2890625 -3.375 3.0390625 -2.02734375 -0.9140625 1 -0.0546875 0.66796875 -3.52734375 -2.8046875 0.93359375 -3.21875 -2.05859375 +26 453.392426 +17 463.42041 +30 493.784241 +31 524.302551 +9 537.602112 +10 489.697937 +4 576.82843 +24 614.970398 +25 631.867432 +29 639.606873 +31 504.269501 +8 524.510742 +26 542.575867 +25 569.645813 +14 586.446594 diff --git a/tests/fixtures/persistence/v0.5.2/hnsw4_l2.index b/tests/fixtures/persistence/v0.5.2/hnsw4_l2.index new file mode 100644 index 00000000..27860690 Binary files /dev/null and b/tests/fixtures/persistence/v0.5.2/hnsw4_l2.index differ diff --git a/tests/fixtures/persistence/v0.5.2/ivf32_ip.expected b/tests/fixtures/persistence/v0.5.2/ivf32_ip.expected new file mode 100644 index 00000000..a3382ec4 --- /dev/null +++ b/tests/fixtures/persistence/v0.5.2/ivf32_ip.expected @@ -0,0 +1,17 @@ +33 65 3 5 +-1.2578125 -3.078125 -3.328125 0.67578125 3.171875 -0.2578125 -2.796875 0.91015625 -3.859375 0.328125 2.296875 -2.0859375 -0.66015625 -3.3046875 2.19140625 2.1484375 2.328125 3.7890625 3.3515625 0.08203125 1.41796875 1.43359375 -1.1484375 1.4375 2.55078125 2.0703125 -1.5859375 -2.8046875 -2.2890625 1.73828125 -2.41796875 -3.09765625 1.53515625 -3.6953125 -0.66796875 0.67578125 0.96484375 -2.9296875 2.41796875 1.25390625 -1.8984375 3.14453125 2.359375 0.65625 -2.52734375 -0.17578125 -0.80859375 -2.171875 -1.63671875 -0.421875 -3.08203125 -0.1328125 -1.94140625 2.734375 2.1875 -2.1015625 3.828125 3.82421875 -3.61328125 -0.9921875 -3.27734375 2.9765625 2.29296875 0.7421875 3.48828125 -2.58203125 1.0234375 -3.12109375 1.83203125 -2.29296875 1.05859375 2.15234375 -2.5390625 -2.3125 -2.7265625 -3.33203125 1.671875 3.6796875 3.53515625 -1.75 0.6328125 -3.1171875 -2.6875 -0.3671875 -2.640625 -2.52734375 -0.65625 1.89453125 -1.08984375 0.25390625 -3.65234375 1.08203125 -0.55078125 3.6171875 2.80078125 2.265625 2.80078125 0.98828125 -1.03515625 1.0234375 -0.04296875 2.95703125 1.8125 -1.6796875 2.75390625 -3.70703125 1.60546875 0.27734375 -3.2265625 -2.57421875 -0.875 1.40234375 -3.55078125 2.80859375 2.953125 3.37109375 -0.02734375 3.16796875 3.23828125 -0.21875 0.57421875 -2.82421875 3.80078125 1.66796875 -3.30859375 -1.21875 1.05078125 1.0546875 2.2890625 3.84765625 1.58984375 -0.7734375 -0.48046875 1.28125 -1.30859375 -1.27734375 2.67578125 1.19921875 2.3515625 -0.140625 -3.828125 -3.84375 1.0703125 -2.27734375 -2.05859375 3.59375 1.3359375 -3.44140625 -0.40234375 -0.26953125 1.671875 -3.24609375 -3.42578125 0.76953125 -0.67578125 -1.53125 -3.2578125 3.65234375 -2.9921875 3.4609375 0.25390625 1.5390625 -0.78125 2.109375 0.99609375 1.5078125 -0.33203125 -1.4296875 -0.01953125 -0.625 2.609375 0.796875 -1.1875 1.125 3.62890625 -3.23046875 -2.91015625 3.0859375 2.23828125 -3.6015625 -1.38671875 3.0703125 -2.2890625 -3.375 3.0390625 -2.02734375 -0.9140625 1 -0.0546875 0.66796875 -3.52734375 -2.8046875 0.93359375 -3.21875 -2.05859375 +17 -111.448395 +26 -98.9153748 +30 -97.6609802 +9 -70.2647552 +18 -59.205368 +10 -88.1517792 +4 -52.7075806 +24 -42.3019562 +29 -40.4504089 +23 -36.7008667 +8 -65.2822876 +31 -45.8321991 +19 -41.9718628 +26 -35.1358032 +14 -33.196228 diff --git a/tests/fixtures/persistence/v0.5.2/ivf32_ip.index b/tests/fixtures/persistence/v0.5.2/ivf32_ip.index new file mode 100644 index 00000000..8d7b8593 Binary files /dev/null and b/tests/fixtures/persistence/v0.5.2/ivf32_ip.index differ diff --git a/tests/fixtures/persistence/v0.5.2/ivf32_l2.expected b/tests/fixtures/persistence/v0.5.2/ivf32_l2.expected new file mode 100644 index 00000000..1afdb5b4 --- /dev/null +++ b/tests/fixtures/persistence/v0.5.2/ivf32_l2.expected @@ -0,0 +1,17 @@ +33 65 3 5 +-1.2578125 -3.078125 -3.328125 0.67578125 3.171875 -0.2578125 -2.796875 0.91015625 -3.859375 0.328125 2.296875 -2.0859375 -0.66015625 -3.3046875 2.19140625 2.1484375 2.328125 3.7890625 3.3515625 0.08203125 1.41796875 1.43359375 -1.1484375 1.4375 2.55078125 2.0703125 -1.5859375 -2.8046875 -2.2890625 1.73828125 -2.41796875 -3.09765625 1.53515625 -3.6953125 -0.66796875 0.67578125 0.96484375 -2.9296875 2.41796875 1.25390625 -1.8984375 3.14453125 2.359375 0.65625 -2.52734375 -0.17578125 -0.80859375 -2.171875 -1.63671875 -0.421875 -3.08203125 -0.1328125 -1.94140625 2.734375 2.1875 -2.1015625 3.828125 3.82421875 -3.61328125 -0.9921875 -3.27734375 2.9765625 2.29296875 0.7421875 3.48828125 -2.58203125 1.0234375 -3.12109375 1.83203125 -2.29296875 1.05859375 2.15234375 -2.5390625 -2.3125 -2.7265625 -3.33203125 1.671875 3.6796875 3.53515625 -1.75 0.6328125 -3.1171875 -2.6875 -0.3671875 -2.640625 -2.52734375 -0.65625 1.89453125 -1.08984375 0.25390625 -3.65234375 1.08203125 -0.55078125 3.6171875 2.80078125 2.265625 2.80078125 0.98828125 -1.03515625 1.0234375 -0.04296875 2.95703125 1.8125 -1.6796875 2.75390625 -3.70703125 1.60546875 0.27734375 -3.2265625 -2.57421875 -0.875 1.40234375 -3.55078125 2.80859375 2.953125 3.37109375 -0.02734375 3.16796875 3.23828125 -0.21875 0.57421875 -2.82421875 3.80078125 1.66796875 -3.30859375 -1.21875 1.05078125 1.0546875 2.2890625 3.84765625 1.58984375 -0.7734375 -0.48046875 1.28125 -1.30859375 -1.27734375 2.67578125 1.19921875 2.3515625 -0.140625 -3.828125 -3.84375 1.0703125 -2.27734375 -2.05859375 3.59375 1.3359375 -3.44140625 -0.40234375 -0.26953125 1.671875 -3.24609375 -3.42578125 0.76953125 -0.67578125 -1.53125 -3.2578125 3.65234375 -2.9921875 3.4609375 0.25390625 1.5390625 -0.78125 2.109375 0.99609375 1.5078125 -0.33203125 -1.4296875 -0.01953125 -0.625 2.609375 0.796875 -1.1875 1.125 3.62890625 -3.23046875 -2.91015625 3.0859375 2.23828125 -3.6015625 -1.38671875 3.0703125 -2.2890625 -3.375 3.0390625 -2.02734375 -0.9140625 1 -0.0546875 0.66796875 -3.52734375 -2.8046875 0.93359375 -3.21875 -2.05859375 +26 450.112122 +17 458.235657 +30 499.837524 +31 524.303589 +9 532.450073 +10 492.983521 +4 575.742737 +24 616.615479 +25 633.027588 +12 635.89978 +31 497.363464 +8 526.510376 +26 553.711975 +25 572.029541 +14 589.477295 diff --git a/tests/fixtures/persistence/v0.5.2/ivf32_l2.index b/tests/fixtures/persistence/v0.5.2/ivf32_l2.index new file mode 100644 index 00000000..5bcceb64 Binary files /dev/null and b/tests/fixtures/persistence/v0.5.2/ivf32_l2.index differ diff --git a/tests/fixtures/persistence/v0.5.2/ivf4_ip.expected b/tests/fixtures/persistence/v0.5.2/ivf4_ip.expected new file mode 100644 index 00000000..66b6febb --- /dev/null +++ b/tests/fixtures/persistence/v0.5.2/ivf4_ip.expected @@ -0,0 +1,17 @@ +33 65 3 5 +-1.2578125 -3.078125 -3.328125 0.67578125 3.171875 -0.2578125 -2.796875 0.91015625 -3.859375 0.328125 2.296875 -2.0859375 -0.66015625 -3.3046875 2.19140625 2.1484375 2.328125 3.7890625 3.3515625 0.08203125 1.41796875 1.43359375 -1.1484375 1.4375 2.55078125 2.0703125 -1.5859375 -2.8046875 -2.2890625 1.73828125 -2.41796875 -3.09765625 1.53515625 -3.6953125 -0.66796875 0.67578125 0.96484375 -2.9296875 2.41796875 1.25390625 -1.8984375 3.14453125 2.359375 0.65625 -2.52734375 -0.17578125 -0.80859375 -2.171875 -1.63671875 -0.421875 -3.08203125 -0.1328125 -1.94140625 2.734375 2.1875 -2.1015625 3.828125 3.82421875 -3.61328125 -0.9921875 -3.27734375 2.9765625 2.29296875 0.7421875 3.48828125 -2.58203125 1.0234375 -3.12109375 1.83203125 -2.29296875 1.05859375 2.15234375 -2.5390625 -2.3125 -2.7265625 -3.33203125 1.671875 3.6796875 3.53515625 -1.75 0.6328125 -3.1171875 -2.6875 -0.3671875 -2.640625 -2.52734375 -0.65625 1.89453125 -1.08984375 0.25390625 -3.65234375 1.08203125 -0.55078125 3.6171875 2.80078125 2.265625 2.80078125 0.98828125 -1.03515625 1.0234375 -0.04296875 2.95703125 1.8125 -1.6796875 2.75390625 -3.70703125 1.60546875 0.27734375 -3.2265625 -2.57421875 -0.875 1.40234375 -3.55078125 2.80859375 2.953125 3.37109375 -0.02734375 3.16796875 3.23828125 -0.21875 0.57421875 -2.82421875 3.80078125 1.66796875 -3.30859375 -1.21875 1.05078125 1.0546875 2.2890625 3.84765625 1.58984375 -0.7734375 -0.48046875 1.28125 -1.30859375 -1.27734375 2.67578125 1.19921875 2.3515625 -0.140625 -3.828125 -3.84375 1.0703125 -2.27734375 -2.05859375 3.59375 1.3359375 -3.44140625 -0.40234375 -0.26953125 1.671875 -3.24609375 -3.42578125 0.76953125 -0.67578125 -1.53125 -3.2578125 3.65234375 -2.9921875 3.4609375 0.25390625 1.5390625 -0.78125 2.109375 0.99609375 1.5078125 -0.33203125 -1.4296875 -0.01953125 -0.625 2.609375 0.796875 -1.1875 1.125 3.62890625 -3.23046875 -2.91015625 3.0859375 2.23828125 -3.6015625 -1.38671875 3.0703125 -2.2890625 -3.375 3.0390625 -2.02734375 -0.9140625 1 -0.0546875 0.66796875 -3.52734375 -2.8046875 0.93359375 -3.21875 -2.05859375 +17 -109.125069 +26 -93.5928345 +30 -92.5121231 +9 -72.1383209 +18 -54.2376633 +10 -95.452652 +4 -55.3820267 +24 -45.6153336 +29 -43.3235931 +23 -36.7782784 +8 -65.693634 +31 -47.2862206 +19 -38.6103706 +26 -37.5991898 +14 -34.2773666 diff --git a/tests/fixtures/persistence/v0.5.2/ivf4_ip.index b/tests/fixtures/persistence/v0.5.2/ivf4_ip.index new file mode 100644 index 00000000..cde16bf4 Binary files /dev/null and b/tests/fixtures/persistence/v0.5.2/ivf4_ip.index differ diff --git a/tests/fixtures/persistence/v0.5.2/ivf4_l2.expected b/tests/fixtures/persistence/v0.5.2/ivf4_l2.expected new file mode 100644 index 00000000..0ea8b42b --- /dev/null +++ b/tests/fixtures/persistence/v0.5.2/ivf4_l2.expected @@ -0,0 +1,17 @@ +33 65 3 5 +-1.2578125 -3.078125 -3.328125 0.67578125 3.171875 -0.2578125 -2.796875 0.91015625 -3.859375 0.328125 2.296875 -2.0859375 -0.66015625 -3.3046875 2.19140625 2.1484375 2.328125 3.7890625 3.3515625 0.08203125 1.41796875 1.43359375 -1.1484375 1.4375 2.55078125 2.0703125 -1.5859375 -2.8046875 -2.2890625 1.73828125 -2.41796875 -3.09765625 1.53515625 -3.6953125 -0.66796875 0.67578125 0.96484375 -2.9296875 2.41796875 1.25390625 -1.8984375 3.14453125 2.359375 0.65625 -2.52734375 -0.17578125 -0.80859375 -2.171875 -1.63671875 -0.421875 -3.08203125 -0.1328125 -1.94140625 2.734375 2.1875 -2.1015625 3.828125 3.82421875 -3.61328125 -0.9921875 -3.27734375 2.9765625 2.29296875 0.7421875 3.48828125 -2.58203125 1.0234375 -3.12109375 1.83203125 -2.29296875 1.05859375 2.15234375 -2.5390625 -2.3125 -2.7265625 -3.33203125 1.671875 3.6796875 3.53515625 -1.75 0.6328125 -3.1171875 -2.6875 -0.3671875 -2.640625 -2.52734375 -0.65625 1.89453125 -1.08984375 0.25390625 -3.65234375 1.08203125 -0.55078125 3.6171875 2.80078125 2.265625 2.80078125 0.98828125 -1.03515625 1.0234375 -0.04296875 2.95703125 1.8125 -1.6796875 2.75390625 -3.70703125 1.60546875 0.27734375 -3.2265625 -2.57421875 -0.875 1.40234375 -3.55078125 2.80859375 2.953125 3.37109375 -0.02734375 3.16796875 3.23828125 -0.21875 0.57421875 -2.82421875 3.80078125 1.66796875 -3.30859375 -1.21875 1.05078125 1.0546875 2.2890625 3.84765625 1.58984375 -0.7734375 -0.48046875 1.28125 -1.30859375 -1.27734375 2.67578125 1.19921875 2.3515625 -0.140625 -3.828125 -3.84375 1.0703125 -2.27734375 -2.05859375 3.59375 1.3359375 -3.44140625 -0.40234375 -0.26953125 1.671875 -3.24609375 -3.42578125 0.76953125 -0.67578125 -1.53125 -3.2578125 3.65234375 -2.9921875 3.4609375 0.25390625 1.5390625 -0.78125 2.109375 0.99609375 1.5078125 -0.33203125 -1.4296875 -0.01953125 -0.625 2.609375 0.796875 -1.1875 1.125 3.62890625 -3.23046875 -2.91015625 3.0859375 2.23828125 -3.6015625 -1.38671875 3.0703125 -2.2890625 -3.375 3.0390625 -2.02734375 -0.9140625 1 -0.0546875 0.66796875 -3.52734375 -2.8046875 0.93359375 -3.21875 -2.05859375 +26 458.689331 +17 462.833405 +30 500.558167 +31 519.726562 +9 533.570984 +10 494.876587 +4 579.590942 +24 613.134827 +12 626.338196 +25 631.41272 +31 495.197601 +8 533.827515 +26 554.517944 +25 575.35553 +14 595.575134 diff --git a/tests/fixtures/persistence/v0.5.2/ivf4_l2.index b/tests/fixtures/persistence/v0.5.2/ivf4_l2.index new file mode 100644 index 00000000..b7339181 Binary files /dev/null and b/tests/fixtures/persistence/v0.5.2/ivf4_l2.index differ diff --git a/tests/fixtures/persistence/v0.5.2/qg0_ip.expected b/tests/fixtures/persistence/v0.5.2/qg0_ip.expected new file mode 100644 index 00000000..c035a8da --- /dev/null +++ b/tests/fixtures/persistence/v0.5.2/qg0_ip.expected @@ -0,0 +1,17 @@ +33 65 3 5 +-1.2578125 -3.078125 -3.328125 0.67578125 3.171875 -0.2578125 -2.796875 0.91015625 -3.859375 0.328125 2.296875 -2.0859375 -0.66015625 -3.3046875 2.19140625 2.1484375 2.328125 3.7890625 3.3515625 0.08203125 1.41796875 1.43359375 -1.1484375 1.4375 2.55078125 2.0703125 -1.5859375 -2.8046875 -2.2890625 1.73828125 -2.41796875 -3.09765625 1.53515625 -3.6953125 -0.66796875 0.67578125 0.96484375 -2.9296875 2.41796875 1.25390625 -1.8984375 3.14453125 2.359375 0.65625 -2.52734375 -0.17578125 -0.80859375 -2.171875 -1.63671875 -0.421875 -3.08203125 -0.1328125 -1.94140625 2.734375 2.1875 -2.1015625 3.828125 3.82421875 -3.61328125 -0.9921875 -3.27734375 2.9765625 2.29296875 0.7421875 3.48828125 -2.58203125 1.0234375 -3.12109375 1.83203125 -2.29296875 1.05859375 2.15234375 -2.5390625 -2.3125 -2.7265625 -3.33203125 1.671875 3.6796875 3.53515625 -1.75 0.6328125 -3.1171875 -2.6875 -0.3671875 -2.640625 -2.52734375 -0.65625 1.89453125 -1.08984375 0.25390625 -3.65234375 1.08203125 -0.55078125 3.6171875 2.80078125 2.265625 2.80078125 0.98828125 -1.03515625 1.0234375 -0.04296875 2.95703125 1.8125 -1.6796875 2.75390625 -3.70703125 1.60546875 0.27734375 -3.2265625 -2.57421875 -0.875 1.40234375 -3.55078125 2.80859375 2.953125 3.37109375 -0.02734375 3.16796875 3.23828125 -0.21875 0.57421875 -2.82421875 3.80078125 1.66796875 -3.30859375 -1.21875 1.05078125 1.0546875 2.2890625 3.84765625 1.58984375 -0.7734375 -0.48046875 1.28125 -1.30859375 -1.27734375 2.67578125 1.19921875 2.3515625 -0.140625 -3.828125 -3.84375 1.0703125 -2.27734375 -2.05859375 3.59375 1.3359375 -3.44140625 -0.40234375 -0.26953125 1.671875 -3.24609375 -3.42578125 0.76953125 -0.67578125 -1.53125 -3.2578125 3.65234375 -2.9921875 3.4609375 0.25390625 1.5390625 -0.78125 2.109375 0.99609375 1.5078125 -0.33203125 -1.4296875 -0.01953125 -0.625 2.609375 0.796875 -1.1875 1.125 3.62890625 -3.23046875 -2.91015625 3.0859375 2.23828125 -3.6015625 -1.38671875 3.0703125 -2.2890625 -3.375 3.0390625 -2.02734375 -0.9140625 1 -0.0546875 0.66796875 -3.52734375 -2.8046875 0.93359375 -3.21875 -2.05859375 +17 -111.448395 +26 -98.9153748 +30 -97.6609802 +9 -70.2647552 +18 -59.205368 +10 -88.1517792 +0 -87.2659454 +4 -52.7075806 +24 -42.3019562 +29 -40.4504089 +8 -65.2822876 +31 -45.8321991 +19 -41.9718628 +26 -35.1358032 +14 -33.196228 diff --git a/tests/fixtures/persistence/v0.5.2/qg0_ip.index b/tests/fixtures/persistence/v0.5.2/qg0_ip.index new file mode 100644 index 00000000..b1684776 Binary files /dev/null and b/tests/fixtures/persistence/v0.5.2/qg0_ip.index differ diff --git a/tests/fixtures/persistence/v0.5.2/qg0_l2.expected b/tests/fixtures/persistence/v0.5.2/qg0_l2.expected new file mode 100644 index 00000000..c49fec5b --- /dev/null +++ b/tests/fixtures/persistence/v0.5.2/qg0_l2.expected @@ -0,0 +1,17 @@ +33 65 3 5 +-1.2578125 -3.078125 -3.328125 0.67578125 3.171875 -0.2578125 -2.796875 0.91015625 -3.859375 0.328125 2.296875 -2.0859375 -0.66015625 -3.3046875 2.19140625 2.1484375 2.328125 3.7890625 3.3515625 0.08203125 1.41796875 1.43359375 -1.1484375 1.4375 2.55078125 2.0703125 -1.5859375 -2.8046875 -2.2890625 1.73828125 -2.41796875 -3.09765625 1.53515625 -3.6953125 -0.66796875 0.67578125 0.96484375 -2.9296875 2.41796875 1.25390625 -1.8984375 3.14453125 2.359375 0.65625 -2.52734375 -0.17578125 -0.80859375 -2.171875 -1.63671875 -0.421875 -3.08203125 -0.1328125 -1.94140625 2.734375 2.1875 -2.1015625 3.828125 3.82421875 -3.61328125 -0.9921875 -3.27734375 2.9765625 2.29296875 0.7421875 3.48828125 -2.58203125 1.0234375 -3.12109375 1.83203125 -2.29296875 1.05859375 2.15234375 -2.5390625 -2.3125 -2.7265625 -3.33203125 1.671875 3.6796875 3.53515625 -1.75 0.6328125 -3.1171875 -2.6875 -0.3671875 -2.640625 -2.52734375 -0.65625 1.89453125 -1.08984375 0.25390625 -3.65234375 1.08203125 -0.55078125 3.6171875 2.80078125 2.265625 2.80078125 0.98828125 -1.03515625 1.0234375 -0.04296875 2.95703125 1.8125 -1.6796875 2.75390625 -3.70703125 1.60546875 0.27734375 -3.2265625 -2.57421875 -0.875 1.40234375 -3.55078125 2.80859375 2.953125 3.37109375 -0.02734375 3.16796875 3.23828125 -0.21875 0.57421875 -2.82421875 3.80078125 1.66796875 -3.30859375 -1.21875 1.05078125 1.0546875 2.2890625 3.84765625 1.58984375 -0.7734375 -0.48046875 1.28125 -1.30859375 -1.27734375 2.67578125 1.19921875 2.3515625 -0.140625 -3.828125 -3.84375 1.0703125 -2.27734375 -2.05859375 3.59375 1.3359375 -3.44140625 -0.40234375 -0.26953125 1.671875 -3.24609375 -3.42578125 0.76953125 -0.67578125 -1.53125 -3.2578125 3.65234375 -2.9921875 3.4609375 0.25390625 1.5390625 -0.78125 2.109375 0.99609375 1.5078125 -0.33203125 -1.4296875 -0.01953125 -0.625 2.609375 0.796875 -1.1875 1.125 3.62890625 -3.23046875 -2.91015625 3.0859375 2.23828125 -3.6015625 -1.38671875 3.0703125 -2.2890625 -3.375 3.0390625 -2.02734375 -0.9140625 1 -0.0546875 0.66796875 -3.52734375 -2.8046875 0.93359375 -3.21875 -2.05859375 +26 450.112122 +17 458.235657 +30 499.837524 +31 524.303589 +9 532.450073 +10 492.983521 +0 498.270233 +4 575.742737 +24 616.615479 +25 633.027588 +31 497.363464 +8 526.510376 +26 553.711975 +25 572.029541 +14 589.477295 diff --git a/tests/fixtures/persistence/v0.5.2/qg0_l2.index b/tests/fixtures/persistence/v0.5.2/qg0_l2.index new file mode 100644 index 00000000..30b50e9d Binary files /dev/null and b/tests/fixtures/persistence/v0.5.2/qg0_l2.index differ diff --git a/tests/fixtures/persistence/v0.5.2/qg4_ip.expected b/tests/fixtures/persistence/v0.5.2/qg4_ip.expected new file mode 100644 index 00000000..993b6d7e --- /dev/null +++ b/tests/fixtures/persistence/v0.5.2/qg4_ip.expected @@ -0,0 +1,17 @@ +33 65 3 5 +-1.2578125 -3.078125 -3.328125 0.67578125 3.171875 -0.2578125 -2.796875 0.91015625 -3.859375 0.328125 2.296875 -2.0859375 -0.66015625 -3.3046875 2.19140625 2.1484375 2.328125 3.7890625 3.3515625 0.08203125 1.41796875 1.43359375 -1.1484375 1.4375 2.55078125 2.0703125 -1.5859375 -2.8046875 -2.2890625 1.73828125 -2.41796875 -3.09765625 1.53515625 -3.6953125 -0.66796875 0.67578125 0.96484375 -2.9296875 2.41796875 1.25390625 -1.8984375 3.14453125 2.359375 0.65625 -2.52734375 -0.17578125 -0.80859375 -2.171875 -1.63671875 -0.421875 -3.08203125 -0.1328125 -1.94140625 2.734375 2.1875 -2.1015625 3.828125 3.82421875 -3.61328125 -0.9921875 -3.27734375 2.9765625 2.29296875 0.7421875 3.48828125 -2.58203125 1.0234375 -3.12109375 1.83203125 -2.29296875 1.05859375 2.15234375 -2.5390625 -2.3125 -2.7265625 -3.33203125 1.671875 3.6796875 3.53515625 -1.75 0.6328125 -3.1171875 -2.6875 -0.3671875 -2.640625 -2.52734375 -0.65625 1.89453125 -1.08984375 0.25390625 -3.65234375 1.08203125 -0.55078125 3.6171875 2.80078125 2.265625 2.80078125 0.98828125 -1.03515625 1.0234375 -0.04296875 2.95703125 1.8125 -1.6796875 2.75390625 -3.70703125 1.60546875 0.27734375 -3.2265625 -2.57421875 -0.875 1.40234375 -3.55078125 2.80859375 2.953125 3.37109375 -0.02734375 3.16796875 3.23828125 -0.21875 0.57421875 -2.82421875 3.80078125 1.66796875 -3.30859375 -1.21875 1.05078125 1.0546875 2.2890625 3.84765625 1.58984375 -0.7734375 -0.48046875 1.28125 -1.30859375 -1.27734375 2.67578125 1.19921875 2.3515625 -0.140625 -3.828125 -3.84375 1.0703125 -2.27734375 -2.05859375 3.59375 1.3359375 -3.44140625 -0.40234375 -0.26953125 1.671875 -3.24609375 -3.42578125 0.76953125 -0.67578125 -1.53125 -3.2578125 3.65234375 -2.9921875 3.4609375 0.25390625 1.5390625 -0.78125 2.109375 0.99609375 1.5078125 -0.33203125 -1.4296875 -0.01953125 -0.625 2.609375 0.796875 -1.1875 1.125 3.62890625 -3.23046875 -2.91015625 3.0859375 2.23828125 -3.6015625 -1.38671875 3.0703125 -2.2890625 -3.375 3.0390625 -2.02734375 -0.9140625 1 -0.0546875 0.66796875 -3.52734375 -2.8046875 0.93359375 -3.21875 -2.05859375 +17 -111.787849 +30 -102.768173 +26 -96.29216 +9 -70.8524017 +29 -61.8877907 +0 -89.5730743 +10 -88.9742966 +4 -48.0263214 +24 -40.3652878 +23 -35.4038467 +8 -62.934021 +31 -50.3188286 +19 -46.4694748 +25 -34.6623192 +26 -32.6777267 diff --git a/tests/fixtures/persistence/v0.5.2/qg4_ip.index b/tests/fixtures/persistence/v0.5.2/qg4_ip.index new file mode 100644 index 00000000..92033ed3 Binary files /dev/null and b/tests/fixtures/persistence/v0.5.2/qg4_ip.index differ diff --git a/tests/fixtures/persistence/v0.5.2/qg4_l2.expected b/tests/fixtures/persistence/v0.5.2/qg4_l2.expected new file mode 100644 index 00000000..1a254669 --- /dev/null +++ b/tests/fixtures/persistence/v0.5.2/qg4_l2.expected @@ -0,0 +1,17 @@ +33 65 3 5 +-1.2578125 -3.078125 -3.328125 0.67578125 3.171875 -0.2578125 -2.796875 0.91015625 -3.859375 0.328125 2.296875 -2.0859375 -0.66015625 -3.3046875 2.19140625 2.1484375 2.328125 3.7890625 3.3515625 0.08203125 1.41796875 1.43359375 -1.1484375 1.4375 2.55078125 2.0703125 -1.5859375 -2.8046875 -2.2890625 1.73828125 -2.41796875 -3.09765625 1.53515625 -3.6953125 -0.66796875 0.67578125 0.96484375 -2.9296875 2.41796875 1.25390625 -1.8984375 3.14453125 2.359375 0.65625 -2.52734375 -0.17578125 -0.80859375 -2.171875 -1.63671875 -0.421875 -3.08203125 -0.1328125 -1.94140625 2.734375 2.1875 -2.1015625 3.828125 3.82421875 -3.61328125 -0.9921875 -3.27734375 2.9765625 2.29296875 0.7421875 3.48828125 -2.58203125 1.0234375 -3.12109375 1.83203125 -2.29296875 1.05859375 2.15234375 -2.5390625 -2.3125 -2.7265625 -3.33203125 1.671875 3.6796875 3.53515625 -1.75 0.6328125 -3.1171875 -2.6875 -0.3671875 -2.640625 -2.52734375 -0.65625 1.89453125 -1.08984375 0.25390625 -3.65234375 1.08203125 -0.55078125 3.6171875 2.80078125 2.265625 2.80078125 0.98828125 -1.03515625 1.0234375 -0.04296875 2.95703125 1.8125 -1.6796875 2.75390625 -3.70703125 1.60546875 0.27734375 -3.2265625 -2.57421875 -0.875 1.40234375 -3.55078125 2.80859375 2.953125 3.37109375 -0.02734375 3.16796875 3.23828125 -0.21875 0.57421875 -2.82421875 3.80078125 1.66796875 -3.30859375 -1.21875 1.05078125 1.0546875 2.2890625 3.84765625 1.58984375 -0.7734375 -0.48046875 1.28125 -1.30859375 -1.27734375 2.67578125 1.19921875 2.3515625 -0.140625 -3.828125 -3.84375 1.0703125 -2.27734375 -2.05859375 3.59375 1.3359375 -3.44140625 -0.40234375 -0.26953125 1.671875 -3.24609375 -3.42578125 0.76953125 -0.67578125 -1.53125 -3.2578125 3.65234375 -2.9921875 3.4609375 0.25390625 1.5390625 -0.78125 2.109375 0.99609375 1.5078125 -0.33203125 -1.4296875 -0.01953125 -0.625 2.609375 0.796875 -1.1875 1.125 3.62890625 -3.23046875 -2.91015625 3.0859375 2.23828125 -3.6015625 -1.38671875 3.0703125 -2.2890625 -3.375 3.0390625 -2.02734375 -0.9140625 1 -0.0546875 0.66796875 -3.52734375 -2.8046875 0.93359375 -3.21875 -2.05859375 +26 455.358521 +17 457.556763 +30 489.623047 +31 529.379272 +9 531.274658 +10 491.338501 +0 493.655945 +4 585.105225 +24 620.488831 +12 631.73877 +31 488.390228 +8 531.206909 +26 558.628113 +25 567.648071 +19 590.280762 diff --git a/tests/fixtures/persistence/v0.5.2/qg4_l2.index b/tests/fixtures/persistence/v0.5.2/qg4_l2.index new file mode 100644 index 00000000..dc6838cb Binary files /dev/null and b/tests/fixtures/persistence/v0.5.2/qg4_l2.index differ diff --git a/tests/fixtures/persistence/v0.5.2/qg8_ip.expected b/tests/fixtures/persistence/v0.5.2/qg8_ip.expected new file mode 100644 index 00000000..84b6a4b1 --- /dev/null +++ b/tests/fixtures/persistence/v0.5.2/qg8_ip.expected @@ -0,0 +1,17 @@ +33 65 3 5 +-1.2578125 -3.078125 -3.328125 0.67578125 3.171875 -0.2578125 -2.796875 0.91015625 -3.859375 0.328125 2.296875 -2.0859375 -0.66015625 -3.3046875 2.19140625 2.1484375 2.328125 3.7890625 3.3515625 0.08203125 1.41796875 1.43359375 -1.1484375 1.4375 2.55078125 2.0703125 -1.5859375 -2.8046875 -2.2890625 1.73828125 -2.41796875 -3.09765625 1.53515625 -3.6953125 -0.66796875 0.67578125 0.96484375 -2.9296875 2.41796875 1.25390625 -1.8984375 3.14453125 2.359375 0.65625 -2.52734375 -0.17578125 -0.80859375 -2.171875 -1.63671875 -0.421875 -3.08203125 -0.1328125 -1.94140625 2.734375 2.1875 -2.1015625 3.828125 3.82421875 -3.61328125 -0.9921875 -3.27734375 2.9765625 2.29296875 0.7421875 3.48828125 -2.58203125 1.0234375 -3.12109375 1.83203125 -2.29296875 1.05859375 2.15234375 -2.5390625 -2.3125 -2.7265625 -3.33203125 1.671875 3.6796875 3.53515625 -1.75 0.6328125 -3.1171875 -2.6875 -0.3671875 -2.640625 -2.52734375 -0.65625 1.89453125 -1.08984375 0.25390625 -3.65234375 1.08203125 -0.55078125 3.6171875 2.80078125 2.265625 2.80078125 0.98828125 -1.03515625 1.0234375 -0.04296875 2.95703125 1.8125 -1.6796875 2.75390625 -3.70703125 1.60546875 0.27734375 -3.2265625 -2.57421875 -0.875 1.40234375 -3.55078125 2.80859375 2.953125 3.37109375 -0.02734375 3.16796875 3.23828125 -0.21875 0.57421875 -2.82421875 3.80078125 1.66796875 -3.30859375 -1.21875 1.05078125 1.0546875 2.2890625 3.84765625 1.58984375 -0.7734375 -0.48046875 1.28125 -1.30859375 -1.27734375 2.67578125 1.19921875 2.3515625 -0.140625 -3.828125 -3.84375 1.0703125 -2.27734375 -2.05859375 3.59375 1.3359375 -3.44140625 -0.40234375 -0.26953125 1.671875 -3.24609375 -3.42578125 0.76953125 -0.67578125 -1.53125 -3.2578125 3.65234375 -2.9921875 3.4609375 0.25390625 1.5390625 -0.78125 2.109375 0.99609375 1.5078125 -0.33203125 -1.4296875 -0.01953125 -0.625 2.609375 0.796875 -1.1875 1.125 3.62890625 -3.23046875 -2.91015625 3.0859375 2.23828125 -3.6015625 -1.38671875 3.0703125 -2.2890625 -3.375 3.0390625 -2.02734375 -0.9140625 1 -0.0546875 0.66796875 -3.52734375 -2.8046875 0.93359375 -3.21875 -2.05859375 +17 -111.539009 +26 -98.6100464 +30 -97.4931488 +9 -70.3984756 +18 -59.0720825 +10 -88.0714569 +0 -87.1107178 +4 -52.4683838 +24 -42.1601105 +29 -40.2988243 +8 -65.4174194 +31 -45.9204292 +19 -41.9337997 +26 -34.6534576 +28 -33.3085632 diff --git a/tests/fixtures/persistence/v0.5.2/qg8_ip.index b/tests/fixtures/persistence/v0.5.2/qg8_ip.index new file mode 100644 index 00000000..7aa0d073 Binary files /dev/null and b/tests/fixtures/persistence/v0.5.2/qg8_ip.index differ diff --git a/tests/fixtures/persistence/v0.5.2/qg8_l2.expected b/tests/fixtures/persistence/v0.5.2/qg8_l2.expected new file mode 100644 index 00000000..d775427c --- /dev/null +++ b/tests/fixtures/persistence/v0.5.2/qg8_l2.expected @@ -0,0 +1,17 @@ +33 65 3 5 +-1.2578125 -3.078125 -3.328125 0.67578125 3.171875 -0.2578125 -2.796875 0.91015625 -3.859375 0.328125 2.296875 -2.0859375 -0.66015625 -3.3046875 2.19140625 2.1484375 2.328125 3.7890625 3.3515625 0.08203125 1.41796875 1.43359375 -1.1484375 1.4375 2.55078125 2.0703125 -1.5859375 -2.8046875 -2.2890625 1.73828125 -2.41796875 -3.09765625 1.53515625 -3.6953125 -0.66796875 0.67578125 0.96484375 -2.9296875 2.41796875 1.25390625 -1.8984375 3.14453125 2.359375 0.65625 -2.52734375 -0.17578125 -0.80859375 -2.171875 -1.63671875 -0.421875 -3.08203125 -0.1328125 -1.94140625 2.734375 2.1875 -2.1015625 3.828125 3.82421875 -3.61328125 -0.9921875 -3.27734375 2.9765625 2.29296875 0.7421875 3.48828125 -2.58203125 1.0234375 -3.12109375 1.83203125 -2.29296875 1.05859375 2.15234375 -2.5390625 -2.3125 -2.7265625 -3.33203125 1.671875 3.6796875 3.53515625 -1.75 0.6328125 -3.1171875 -2.6875 -0.3671875 -2.640625 -2.52734375 -0.65625 1.89453125 -1.08984375 0.25390625 -3.65234375 1.08203125 -0.55078125 3.6171875 2.80078125 2.265625 2.80078125 0.98828125 -1.03515625 1.0234375 -0.04296875 2.95703125 1.8125 -1.6796875 2.75390625 -3.70703125 1.60546875 0.27734375 -3.2265625 -2.57421875 -0.875 1.40234375 -3.55078125 2.80859375 2.953125 3.37109375 -0.02734375 3.16796875 3.23828125 -0.21875 0.57421875 -2.82421875 3.80078125 1.66796875 -3.30859375 -1.21875 1.05078125 1.0546875 2.2890625 3.84765625 1.58984375 -0.7734375 -0.48046875 1.28125 -1.30859375 -1.27734375 2.67578125 1.19921875 2.3515625 -0.140625 -3.828125 -3.84375 1.0703125 -2.27734375 -2.05859375 3.59375 1.3359375 -3.44140625 -0.40234375 -0.26953125 1.671875 -3.24609375 -3.42578125 0.76953125 -0.67578125 -1.53125 -3.2578125 3.65234375 -2.9921875 3.4609375 0.25390625 1.5390625 -0.78125 2.109375 0.99609375 1.5078125 -0.33203125 -1.4296875 -0.01953125 -0.625 2.609375 0.796875 -1.1875 1.125 3.62890625 -3.23046875 -2.91015625 3.0859375 2.23828125 -3.6015625 -1.38671875 3.0703125 -2.2890625 -3.375 3.0390625 -2.02734375 -0.9140625 1 -0.0546875 0.66796875 -3.52734375 -2.8046875 0.93359375 -3.21875 -2.05859375 +26 450.722748 +17 458.054413 +30 500.173096 +31 524.416077 +9 532.182556 +10 493.144135 +0 498.580658 +4 576.22113 +24 616.89917 +25 632.544312 +31 497.186981 +8 526.240112 +26 554.676636 +25 571.910217 +14 589.628906 diff --git a/tests/fixtures/persistence/v0.5.2/qg8_l2.index b/tests/fixtures/persistence/v0.5.2/qg8_l2.index new file mode 100644 index 00000000..564296ac Binary files /dev/null and b/tests/fixtures/persistence/v0.5.2/qg8_l2.index differ diff --git a/tests/python/test_historical_persistence.py b/tests/python/test_historical_persistence.py new file mode 100644 index 00000000..09304b01 --- /dev/null +++ b/tests/python/test_historical_persistence.py @@ -0,0 +1,108 @@ +"""Frozen historical writer outputs must remain searchable with current wheels.""" + +import hashlib +import json +import struct +from pathlib import Path + +import numpy as np +import pytest +from rabitqlib import HnswIndex, IvfIndex, SymqgIndex + +FIXTURES = Path(__file__).resolve().parents[1] / "fixtures" / "persistence" +NAMES = [ + f"{kind}_{metric}" + for kind in ("ivf4", "ivf32", "hnsw4", "qg0", "qg4", "qg8") + for metric in ("l2", "ip") +] + + +def reference(name): + path = FIXTURES / "v0.5.2" / f"{name}.index" + lines = path.with_suffix(".expected").read_text().splitlines() + count, dim, nq, k = map(int, lines[0].split()) + queries = np.fromstring(lines[1], sep=" ", dtype=np.float32).reshape(nq, dim) + results = np.loadtxt(lines[2:]) + ids = results[:, 0].astype(np.uint32).reshape(nq, k) + distances = results[:, 1].astype(np.float32).reshape(nq, k) + cls = ( + IvfIndex + if name.startswith("ivf") + else (HnswIndex if name.startswith("hnsw") else SymqgIndex) + ) + options = {"nprobe": 1} if name.startswith("ivf") else {"ef": 64} + return path, cls, count, queries, ids, distances, options + + +def test_fixture_hashes_match_pinned_manifest(): + manifest = json.loads((FIXTURES / "manifest.json").read_text()) + for filename, digest in manifest["sha256"].items(): + assert hashlib.sha256((FIXTURES / filename).read_bytes()).hexdigest() == digest + + +@pytest.mark.parametrize("name", NAMES) +def test_historical_load_search_and_resave(name, tmp_path): + path, cls, count, queries, ids, distances, options = reference(name) + original = cls.load(str(path)) + destination = tmp_path / "migrated.index" + original.save(str(destination)) + for index in (original, cls.load(str(destination))): + assert index.dim == 65 + assert index.metric == name.rsplit("_", 1)[1] + assert (index.max_elements if cls is IvfIndex else index.num_points) == count + actual_ids, actual_distances = index.search(queries, 5, **options) + np.testing.assert_array_equal(actual_ids, ids) + # Distances may differ across AVX2, AVX-512, NEON, and compiler reductions. + np.testing.assert_allclose(actual_distances, distances, rtol=2e-4, atol=2e-4) + if cls in (IvfIndex, HnswIndex): + assert not np.any(actual_ids == 0) # Historical removal markers survive. + damaged = tmp_path / "truncated.index" + damaged.write_bytes(path.read_bytes()[:-1]) + with pytest.raises(RuntimeError): + cls.load(str(damaged)) + + +@pytest.mark.parametrize("name", ["ivf32_l2", "qg4_l2", "qg8_l2"]) +def test_unknown_historical_version_is_rejected(name, tmp_path): + path, cls, *_ = reference(name) + damaged = bytearray(path.read_bytes()) + struct.pack_into(" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "rabitqlib/index/hnsw/hnsw.hpp" +#include "rabitqlib/index/ivf/ivf.hpp" +#include "rabitqlib/index/symqg/qg.hpp" + +namespace rabitqlib { +namespace { +class HistoricalPersistenceTest : public ::testing::TestWithParam {}; + +TEST_P(HistoricalPersistenceTest, LoadsSearchesResavesAndPreservesStateAfterFailedLoad) { + const std::filesystem::path directory(RABITQ_TEST_FIXTURE_DIR); + const auto name = GetParam(); + const auto path = (directory / (name + ".index")).string(); + std::ifstream expected(directory / (name + ".expected")); + ASSERT_TRUE(expected.is_open()); + size_t count = 0, dim = 0, nq = 0, k = 0; + expected >> count >> dim >> nq >> k; + ASSERT_EQ(count, 33U); + ASSERT_EQ(dim, 65U); + ASSERT_EQ(nq, 3U); + ASSERT_EQ(k, 5U); + std::vector queries(nq * dim), distances(nq * k), reference_distances(nq * k); + std::vector ids(nq * k), reference_ids(nq * k); + for (auto& value : queries) + expected >> value; + for (size_t i = 0; i < ids.size(); ++i) + expected >> reference_ids[i] >> reference_distances[i]; + ASSERT_TRUE(expected.good()); + const auto output = + std::filesystem::temp_directory_path() / ("rabitq_historical_" + name); + const auto compare = [&] { + EXPECT_EQ(ids, reference_ids); + for (size_t i = 0; i < ids.size(); ++i) { + EXPECT_NEAR( + distances[i], + reference_distances[i], + 2e-4F * std::max(1.0F, std::abs(reference_distances[i])) + ); + } + }; + const auto exercise = [&](auto& index, const auto& search) { + index.load(path.c_str()); + EXPECT_EQ(index.dimension(), dim); + const auto metric = name.find("_ip") != std::string::npos ? METRIC_IP : METRIC_L2; + EXPECT_EQ(index.metric_type(), metric); + search(); + compare(); + index.save(output.string().c_str()); + index.load(output.string().c_str()); + search(); + compare(); + std::filesystem::resize_file(output, std::filesystem::file_size(output) - 1); + EXPECT_THROW(index.load(output.string().c_str()), std::runtime_error); + search(); + compare(); + std::filesystem::remove(output); + }; + if (name.find("ivf") == 0) { + ivf::IVF index; + exercise(index, [&] { + EXPECT_EQ(index.padded_dim(), 128U); + EXPECT_EQ(index.initializer_type(), ivf::InitializerType::Flat); + index.search_batch(queries.data(), nq, k, 1, ids.data(), distances.data()); + }); + } else if (name.find("hnsw") == 0) { + hnsw::HierarchicalNSW index; + exercise(index, [&] { + const auto results = index.search(queries.data(), nq, k, 64, 1); + for (size_t i = 0; i < nq; ++i) { + ASSERT_EQ(results[i].size(), k); + for (size_t j = 0; j < k; ++j) { + ids[i * k + j] = results[i][j].second; + distances[i * k + j] = results[i][j].first; + } + } + }); + } else { + symqg::QuantizedGraph index; + exercise(index, [&] { + EXPECT_EQ(index.padded_dim(), 128U); + index.set_ef(64); + index.search_batch(queries.data(), nq, k, ids.data(), distances.data(), 1); + }); + } +} + +INSTANTIATE_TEST_SUITE_P( + V052, + HistoricalPersistenceTest, + ::testing::Values( + "ivf4_l2", + "ivf4_ip", + "ivf32_l2", + "ivf32_ip", + "hnsw4_l2", + "hnsw4_ip", + "qg0_l2", + "qg0_ip", + "qg4_l2", + "qg4_ip", + "qg8_l2", + "qg8_ip" + ), + [](const auto& info) { return info.param; } +); +} // namespace +} // namespace rabitqlib