From cd61c2106ea02546a0a64614325034b6baabcef0 Mon Sep 17 00:00:00 2001 From: Guillaume de Rouville Date: Sat, 15 Aug 2026 23:39:06 -0700 Subject: [PATCH 01/17] Add: Repeatable Levenshtein search queries Every implementation needs to receive the same queries. Add a small generator whose output is fixed by the dictionary, query count, mode, and seed. Mixed files evenly interleave exact words, one-edit changes, two-edit changes, and five-character extensions. Those labels only describe how a query was made. They are not treated as expected answers because another dictionary entry may still match. Byte and UTF-8 modes follow the same rules. UTF-8 edits operate on decoded characters, so one generated edit remains one edit. Signed-off-by: Guillaume de Rouville --- levenshtein/queries.cpp | 153 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100644 levenshtein/queries.cpp diff --git a/levenshtein/queries.cpp b/levenshtein/queries.cpp new file mode 100644 index 0000000..ea19fed --- /dev/null +++ b/levenshtein/queries.cpp @@ -0,0 +1,153 @@ +/** + * @brief Deterministic query generator for immutable-dictionary Levenshtein benchmarks. + * + * Generates exact, one-edit, two-edit, five-symbol length-extended, or mixed queries by sampling lines from an + * existing dictionary. The extension is guaranteed beyond four edits from its sampled source, not necessarily from + * every other dictionary entry. Mutations use bytes by default or validated Unicode codepoints when + * `SZ_LEVENSHTEIN_UTF8` is set. + */ +#include + +#include +#include +#include +#include +#include +#include +#include + +static std::vector load_lines(char const *path) { + std::ifstream input(path); + if (!input) throw std::runtime_error(std::string("failed to open ") + path); + std::vector lines; + std::string line; + while (std::getline(input, line)) { + if (!line.empty() && line.back() == '\r') line.pop_back(); + lines.push_back(std::move(line)); + } + return lines; +} + +static std::uint64_t splitmix64(std::uint64_t &state) { + std::uint64_t value = (state += 0x9E3779B97F4A7C15ull); + value = (value ^ (value >> 30)) * 0xBF58476D1CE4E5B9ull; + value = (value ^ (value >> 27)) * 0x94D049BB133111EBull; + return value ^ (value >> 31); +} + +static std::size_t substitute(std::string &query, std::uint64_t random) { + if (query.empty()) { + query.push_back('~'); + return 0; + } + std::size_t const position = random % query.size(); + query[position] = query[position] == '~' ? '^' : '~'; + return position; +} + +static bool decode_utf8(std::string const &encoded, std::vector &decoded) { + decoded.clear(); + char const *position = encoded.data(), *end = position + encoded.size(); + while (position != end) { + sz_rune_t rune = 0; + sz_rune_length_t const length = sz_rune_decode(position, end, &rune); + if (length == sz_rune_invalid_k) return false; + decoded.push_back(rune); + position += length; + } + return true; +} + +static std::string encode_utf8(std::vector const &decoded) { + std::string encoded; + encoded.reserve(decoded.size() * 3); + for (sz_rune_t rune : decoded) { + sz_u8_t bytes[4]; + sz_rune_length_t const length = sz_rune_encode(rune, bytes); + if (length == sz_rune_invalid_k) throw std::runtime_error("invalid Unicode codepoint"); + encoded.append(reinterpret_cast(bytes), length); + } + return encoded; +} + +static std::size_t substitute(std::vector &query, std::uint64_t random) { + if (query.empty()) { + query.push_back('~'); + return 0; + } + std::size_t const position = random % query.size(); + query[position] = query[position] == '~' ? '^' : '~'; + return position; +} + +int main(int argc, char **argv) { + if (argc != 6) { + std::cerr << "usage: levenshtein_index_queries DICTIONARY OUTPUT COUNT exact|edit1|edit2|reject|mixed SEED\n"; + return 2; + } + auto const dictionary = load_lines(argv[1]); + if (dictionary.empty()) { + std::cerr << "dictionary is empty\n"; + return 3; + } + std::ofstream output(argv[2]); + if (!output) { + std::cerr << "failed to create " << argv[2] << '\n'; + return 4; + } + std::size_t const count = std::stoull(argv[3]); + std::string_view const requested_mode = argv[4]; + std::uint64_t random_state = std::stoull(argv[5]); + bool const utf8 = std::getenv("SZ_LEVENSHTEIN_UTF8") != nullptr; + for (std::size_t query_index = 0; query_index != count; ++query_index) { + std::uint64_t const sample_random = splitmix64(random_state); + std::string query = dictionary[sample_random % dictionary.size()]; + std::string_view mode = requested_mode; + if (mode == "mixed") { + static constexpr std::string_view modes[] = {"exact", "edit1", "edit2", "reject"}; + mode = modes[query_index % 4]; + } + if (utf8) { + std::vector decoded; + if (!decode_utf8(query, decoded)) { + std::cerr << "invalid UTF-8 dictionary entry\n"; + return 6; + } + if (mode == "edit1" || mode == "edit2") { + std::size_t const first_position = substitute(decoded, splitmix64(random_state)); + if (mode == "edit2") { + if (decoded.size() > 1) { + std::size_t second_position = splitmix64(random_state) % (decoded.size() - 1); + if (second_position >= first_position) ++second_position; + decoded[second_position] = decoded[second_position] == '~' ? '^' : '~'; + } + else + decoded.push_back('~'); + } + } + if (mode == "reject") decoded.insert(decoded.begin(), 5, '~'); + query = encode_utf8(decoded); + } + else { + if (mode == "edit1" || mode == "edit2") { + std::size_t const first_position = substitute(query, splitmix64(random_state)); + if (mode == "edit2") { + if (query.size() > 1) { + std::size_t second_position = splitmix64(random_state) % (query.size() - 1); + if (second_position >= first_position) ++second_position; + query[second_position] = query[second_position] == '~' ? '^' : '~'; + } + else + query.push_back('~'); + } + } + if (mode == "reject") query.insert(0, "~~~~~"); // Beyond four edits from the sampled source word. + } + if (mode != "exact" && mode != "edit1" && mode != "edit2" && mode != "reject") { + std::cerr << "unknown mode: " << mode << '\n'; + return 5; + } + output << query << '\n'; + } + return output.good() ? 0 : 6; +} From 787383bac0f238f9e0f1deedb6618ac3f73a5fad Mon Sep 17 00:00:00 2001 From: Guillaume de Rouville Date: Sun, 16 Aug 2026 01:46:49 -0700 Subject: [PATCH 02/17] Add: Public StringZilla dictionary search benchmark Add the StringZilla side of repeated fuzzy search over a dictionary that is built once and then left unchanged. The runner calls the public C batch API rather than the internal C++ index used while the algorithm was being developed. Dictionary construction and one output-sizing pass stay outside the query timer. The three sparse result arrays are then reused, while finding and writing every query and dictionary match remains inside the timer. Result sorting and file output stay outside it. The same runner covers byte and UTF-8 behavior, one or several CPU cores, cache eviction, repeated batches, and bounds one through four. It validates every returned query ID, dictionary ID, and distance before writing the shared result format. On the local Intel AVX2 run with 213,557 words and 10,000 queries, one core took 7.45 ms at k=1 and 66.60 ms at k=2. Four physical cores took 2.11 and 17.99 ms. The complete serial and parallel files matched RapidFuzz. Signed-off-by: Guillaume de Rouville --- levenshtein/stringzilla.cpp | 246 ++++++++++++++++++++++++++++++++++++ 1 file changed, 246 insertions(+) create mode 100644 levenshtein/stringzilla.cpp diff --git a/levenshtein/stringzilla.cpp b/levenshtein/stringzilla.cpp new file mode 100644 index 0000000..d11693d --- /dev/null +++ b/levenshtein/stringzilla.cpp @@ -0,0 +1,246 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static std::vector load_lines(char const *path, std::size_t limit = 0) { + std::ifstream input(path); + if (!input) throw std::runtime_error(std::string("failed to open ") + path); + std::vector lines; + std::string line; + while ((!limit || lines.size() != limit) && std::getline(input, line)) { + if (!line.empty() && line.back() == '\r') line.pop_back(); + lines.push_back(std::move(line)); + } + return lines; +} + +static std::vector make_views(std::vector const &strings) { + std::vector views; + views.reserve(strings.size()); + for (auto const &string : strings) views.push_back({string.data(), string.size()}); + return views; +} + +struct sparse_matches_t { + std::vector query_ids; + std::vector dictionary_ids; + std::vector distances; + + void resize(std::size_t count) { + query_ids.resize(count); + dictionary_ids.resize(count); + distances.resize(count); + } +}; + +static bool dump_matches(std::size_t dictionary_size, std::size_t queries_size, std::uint8_t bound, + sparse_matches_t const &matches, std::string const &path) { + std::vector>> grouped(queries_size); + for (std::size_t match = 0; match != matches.query_ids.size(); ++match) { + if (matches.query_ids[match] >= queries_size || matches.dictionary_ids[match] >= dictionary_size || + matches.distances[match] > bound) + return false; + grouped[matches.query_ids[match]].push_back({matches.dictionary_ids[match], matches.distances[match]}); + } + for (auto &query_matches : grouped) std::sort(query_matches.begin(), query_matches.end()); + + std::ofstream output(path, std::ios::binary); + if (!output) return false; + char const magic[8] = {'S', 'Z', 'L', 'E', 'V', '0', '0', '1'}; + std::uint64_t const dictionary_size_u64 = dictionary_size; + std::uint64_t const queries_size_u64 = queries_size; + output.write(magic, sizeof(magic)); + output.write(reinterpret_cast(&dictionary_size_u64), sizeof(dictionary_size_u64)); + output.write(reinterpret_cast(&queries_size_u64), sizeof(queries_size_u64)); + output.write(reinterpret_cast(&bound), sizeof(bound)); + for (auto const &query_matches : grouped) { + std::uint64_t const matches_size = query_matches.size(); + output.write(reinterpret_cast(&matches_size), sizeof(matches_size)); + for (auto const &match : query_matches) { + output.write(reinterpret_cast(&match.first), sizeof(match.first)); + output.write(reinterpret_cast(&match.second), sizeof(match.second)); + } + } + return output.good(); +} + +template +static sz_status_t index_init(sz_sequence_t const *dictionary, sz_size_t max_distance, void **index, + char const **error) { + if constexpr (utf8_) + return szs_levenshtein_index_utf8_init(dictionary, max_distance, NULL, sz_caps_sp_k, index, error); + else return szs_levenshtein_index_init(dictionary, max_distance, NULL, sz_caps_sp_k, index, error); +} + +template +static sz_status_t index_find(void *index, szs_device_scope_t device, sz_sequence_t const *queries, + sz_size_t bound, sparse_matches_t &matches, sz_size_t *matches_found, + char const **error) { + sz_size_t const capacity = matches.query_ids.size(); + sz_u64_t *query_ids = capacity ? matches.query_ids.data() : NULL; + sz_u32_t *dictionary_ids = capacity ? matches.dictionary_ids.data() : NULL; + sz_u8_t *distances = capacity ? matches.distances.data() : NULL; + if constexpr (utf8_) + return szs_levenshtein_index_utf8_find(index, device, queries, bound, query_ids, dictionary_ids, distances, + capacity, matches_found, error); + else + return szs_levenshtein_index_find(index, device, queries, bound, query_ids, dictionary_ids, distances, + capacity, matches_found, error); +} + +template +static void index_free(void *index) { + if constexpr (utf8_) szs_levenshtein_index_utf8_free(index); + else szs_levenshtein_index_free(index); +} + +template +static int run(std::vector const &dictionary, std::vector const &queries, + std::vector const &max_distances, std::string const &dump_prefix, + int query_repeats, std::size_t query_threads, std::size_t batches_per_repeat, + std::size_t cache_evict_bytes) { + std::vector dictionary_views = make_views(dictionary); + std::vector query_views = make_views(queries); + sz_sequence_t dictionary_sequence, query_sequence; + sz_sequence_from_string_views(dictionary_views.data(), dictionary_views.size(), &dictionary_sequence); + sz_sequence_from_string_views(query_views.data(), query_views.size(), &query_sequence); + + char const *error = NULL; + szs_device_scope_t device = NULL; + if (szs_device_scope_init_cpu_cores(query_threads, &device, &error) != sz_success_k) { + std::cerr << "device initialization failed: " << (error ? error : "unknown error") << '\n'; + return 3; + } + + std::vector cache_evict_buffer(cache_evict_bytes); + std::uint64_t cache_evict_checksum = 0; + for (std::uint8_t max_distance : max_distances) { + void *index = NULL; + auto const build_start = std::chrono::steady_clock::now(); + if (sz_status_t status = index_init(&dictionary_sequence, max_distance, &index, &error); + status != sz_success_k) { + std::cerr << "build failed: " << int(status) << " " << (error ? error : "") << '\n'; + szs_device_scope_free(device); + return 4; + } + double const build_seconds = + std::chrono::duration(std::chrono::steady_clock::now() - build_start).count(); + std::cout << "k=" << unsigned(max_distance) << " build=" << build_seconds << "s\n"; + + std::uint8_t const first_bound = max_distance <= 2 ? max_distance : 3; + for (std::uint8_t bound = first_bound; bound <= max_distance; ++bound) { + sparse_matches_t matches; + sz_size_t required = 0; + sz_status_t sizing_status = index_find(index, device, &query_sequence, bound, matches, + &required, &error); + if (sizing_status != sz_success_k && sizing_status != sz_unexpected_dimensions_k) { + std::cerr << "output sizing failed: " << int(sizing_status) << " " << (error ? error : "") << '\n'; + index_free(index); + szs_device_scope_free(device); + return 5; + } + matches.resize(required); + + for (int repeat = 0; repeat != query_repeats; ++repeat) { + for (std::size_t offset = 0; offset < cache_evict_buffer.size(); offset += 64) { + ++cache_evict_buffer[offset]; + cache_evict_checksum += cache_evict_buffer[offset]; + } + auto const start = std::chrono::steady_clock::now(); + sz_size_t matches_count = 0; + for (std::size_t batch = 0; batch != batches_per_repeat; ++batch) { + sz_size_t batch_matches = 0; + if (sz_status_t status = index_find(index, device, &query_sequence, bound, matches, + &batch_matches, &error); + status != sz_success_k) { + std::cerr << "search failed: " << int(status) << " " << (error ? error : "") << '\n'; + index_free(index); + szs_device_scope_free(device); + return 6; + } + if (batch_matches != required) { + std::cerr << "search returned a different result count after output sizing\n"; + index_free(index); + szs_device_scope_free(device); + return 6; + } + matches_count += batch_matches; + } + double const elapsed = + std::chrono::duration(std::chrono::steady_clock::now() - start).count(); + std::cout << "k=" << unsigned(bound) << " query=" << elapsed / batches_per_repeat + << "s matches=" << matches_count / batches_per_repeat << " threads=" << query_threads + << " batches=" << batches_per_repeat << " cache_evict_bytes=" << cache_evict_bytes + << " output_element_bytes=" + << sizeof(sz_u64_t) + sizeof(sz_u32_t) + sizeof(sz_u8_t) << '\n'; + } + if (!dump_prefix.empty()) { + std::string const path = dump_prefix + ".k" + std::to_string(bound) + ".bin"; + if (!dump_matches(dictionary.size(), queries.size(), bound, matches, path)) { + std::cerr << "dump failed: " << path << '\n'; + index_free(index); + szs_device_scope_free(device); + return 7; + } + } + } + index_free(index); + } + szs_device_scope_free(device); + if (cache_evict_bytes) std::cerr << "cache_evict_checksum=" << cache_evict_checksum << '\n'; + return 0; +} + +int main(int argc, char **argv) { + if (argc < 3 || argc > 5) { + std::cerr << "usage: levenshtein_index DICTIONARY QUERIES [QUERY_LIMIT] [DUMP_PREFIX]\n"; + return 2; + } + std::size_t const query_limit = argc >= 4 ? std::stoull(argv[3]) : 0; + std::string const dump_prefix = argc == 5 ? argv[4] : ""; + auto const dictionary = load_lines(argv[1]); + auto const queries = load_lines(argv[2], query_limit); + bool const utf8 = std::getenv("SZ_LEVENSHTEIN_UTF8") != nullptr; + std::cout << "dictionary=" << dictionary.size() << " queries=" << queries.size() + << " semantics=" << (utf8 ? "utf8-codepoints" : "bytes") << '\n'; + + std::vector max_distances = {1, 2, 4}; + int const query_repeats = std::getenv("SZ_LEVENSHTEIN_REPEATS") + ? std::stoi(std::getenv("SZ_LEVENSHTEIN_REPEATS")) + : 3; + std::size_t const query_threads = std::getenv("SZ_LEVENSHTEIN_THREADS") + ? std::stoull(std::getenv("SZ_LEVENSHTEIN_THREADS")) + : 1; + std::size_t const batches_per_repeat = std::getenv("SZ_LEVENSHTEIN_BATCHES_PER_REPEAT") + ? std::stoull(std::getenv("SZ_LEVENSHTEIN_BATCHES_PER_REPEAT")) + : 1; + std::size_t const cache_evict_mb = std::getenv("SZ_LEVENSHTEIN_CACHE_EVICT_MB") + ? std::stoull(std::getenv("SZ_LEVENSHTEIN_CACHE_EVICT_MB")) + : 0; + if (query_repeats <= 0 || query_threads == 0 || batches_per_repeat == 0 || + cache_evict_mb > std::size_t(-1) / (1024 * 1024)) { + std::cerr << "repeat, thread, batch, or cache setting is invalid\n"; + return 2; + } + if (char const *requested_max = std::getenv("SZ_LEVENSHTEIN_MAX_DISTANCE")) { + int const parsed = std::stoi(requested_max); + if (parsed != 1 && parsed != 2 && parsed != 4) { + std::cerr << "SZ_LEVENSHTEIN_MAX_DISTANCE must be 1, 2, or 4\n"; + return 2; + } + max_distances = {static_cast(parsed)}; + } + std::size_t const cache_evict_bytes = cache_evict_mb * 1024 * 1024; + return utf8 ? run(dictionary, queries, max_distances, dump_prefix, query_repeats, query_threads, + batches_per_repeat, cache_evict_bytes) + : run(dictionary, queries, max_distances, dump_prefix, query_repeats, query_threads, + batches_per_repeat, cache_evict_bytes); +} From 295bb86111d4a616f73010060cb98511bcc4fb2e Mon Sep 17 00:00:00 2001 From: Guillaume de Rouville Date: Sat, 15 Aug 2026 23:39:16 -0700 Subject: [PATCH 03/17] Add: RapidFuzz result checker Use RapidFuzz as the reference for the byte benchmark. It receives the same dictionary, queries, and inclusive distance bounds as StringZilla. It scans the complete dictionary for every query, so it checks correctness rather than serving as the closest indexed performance comparison. Both runners write the same file format. Comparing those files checks every dictionary ID and exact distance for every query, not only the final match count. The complete results matched at k=1 and k=2 on the recorded English run. Signed-off-by: Guillaume de Rouville --- levenshtein/rapidfuzz.cpp | 107 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 levenshtein/rapidfuzz.cpp diff --git a/levenshtein/rapidfuzz.cpp b/levenshtein/rapidfuzz.cpp new file mode 100644 index 0000000..1ae369f --- /dev/null +++ b/levenshtein/rapidfuzz.cpp @@ -0,0 +1,107 @@ +/** + * @brief Native RapidFuzz baseline for repeated immutable-dictionary Levenshtein retrieval. + * + * This file is intentionally not part of the default build because it requires rapidfuzz-cpp. + * Pin the dependency revision and compile it explicitly, for example: + * + * g++ -std=c++20 -O3 -DNDEBUG -march=native -I rapidfuzz-cpp \ + * levenshtein/rapidfuzz.cpp -o rapidfuzz_levenshtein_index + * + * `RF_REPEATS=0` skips timing and only emits exact comparison artifacts when `DUMP_PREFIX` is present. + */ +#include + +#include +#include +#include +#include +#include +#include +#include + +struct match_t { + std::uint32_t id; + std::uint8_t distance; +}; + +static std::vector load_lines(char const *path, std::size_t limit = 0) { + std::ifstream input(path); + if (!input) throw std::runtime_error(std::string("failed to open ") + path); + std::vector lines; + std::string line; + while ((!limit || lines.size() != limit) && std::getline(input, line)) { + if (!line.empty() && line.back() == '\r') line.pop_back(); + lines.push_back(std::move(line)); + } + return lines; +} + +static bool dump_matches(std::vector const &dictionary, std::vector const &queries, + std::uint8_t bound, std::string const &path) { + std::ofstream output(path, std::ios::binary); + if (!output) return false; + char const magic[8] = {'S', 'Z', 'L', 'E', 'V', '0', '0', '1'}; + std::uint64_t const dictionary_size = dictionary.size(); + std::uint64_t const queries_size = queries.size(); + output.write(magic, sizeof(magic)); + output.write(reinterpret_cast(&dictionary_size), sizeof(dictionary_size)); + output.write(reinterpret_cast(&queries_size), sizeof(queries_size)); + output.write(reinterpret_cast(&bound), sizeof(bound)); + + std::vector matches; + for (auto const &query : queries) { + matches.clear(); + rapidfuzz::CachedLevenshtein scorer(query); + for (std::uint32_t id = 0; id != dictionary.size(); ++id) { + std::size_t const distance = scorer.distance(dictionary[id], bound); + if (distance <= bound) matches.push_back({id, static_cast(distance)}); + } + std::uint64_t const matches_size = matches.size(); + output.write(reinterpret_cast(&matches_size), sizeof(matches_size)); + for (auto const &match : matches) { + output.write(reinterpret_cast(&match.id), sizeof(match.id)); + output.write(reinterpret_cast(&match.distance), sizeof(match.distance)); + } + } + return output.good(); +} + +int main(int argc, char **argv) { + if (argc < 3 || argc > 5) { + std::cerr << "usage: rapidfuzz_levenshtein_index DICTIONARY QUERIES [QUERY_LIMIT] [DUMP_PREFIX]\n"; + return 2; + } + std::size_t const query_limit = argc >= 4 ? std::stoull(argv[3]) : 0; + std::string const dump_prefix = argc == 5 ? argv[4] : ""; + auto const dictionary = load_lines(argv[1]); + auto const queries = load_lines(argv[2], query_limit); + int const repeats = std::getenv("RF_REPEATS") ? std::stoi(std::getenv("RF_REPEATS")) : 3; + int const max_distance = std::getenv("RF_MAX_DISTANCE") ? std::stoi(std::getenv("RF_MAX_DISTANCE")) : 4; + if (max_distance < 1 || max_distance > 4) { + std::cerr << "RF_MAX_DISTANCE must be between 1 and 4\n"; + return 2; + } + std::cout << "dictionary=" << dictionary.size() << " queries=" << queries.size() << '\n'; + + for (std::uint8_t bound = 1; bound <= max_distance; ++bound) { + for (int repeat = 0; repeat != repeats; ++repeat) { + std::size_t matches_count = 0; + auto const start = std::chrono::steady_clock::now(); + for (auto const &query : queries) { + rapidfuzz::CachedLevenshtein scorer(query); + for (auto const &candidate : dictionary) + if (scorer.distance(candidate, bound) <= bound) ++matches_count; + } + double const elapsed = + std::chrono::duration(std::chrono::steady_clock::now() - start).count(); + std::cout << "k=" << unsigned(bound) << " query=" << elapsed << "s matches=" << matches_count << '\n'; + } + if (!dump_prefix.empty()) { + std::string const path = dump_prefix + ".k" + std::to_string(bound) + ".bin"; + if (!dump_matches(dictionary, queries, bound, path)) { + std::cerr << "dump failed: " << path << '\n'; + return 3; + } + } + } +} From 510a0fc974b9ccc551deb4d524da3654a441983c Mon Sep 17 00:00:00 2001 From: Guillaume de Rouville Date: Sat, 15 Aug 2026 23:41:47 -0700 Subject: [PATCH 04/17] Add: Exact SymSpell dictionary comparison Add the closest reusable index found for this workload, pinned to the official SymSpell-Rust repository. SymSpell normally lowercases words, counts an adjacent swap as one edit, and cannot preserve duplicate IDs. The runner therefore uses unique lowercase input and checks every suggestion again with plain Levenshtein distance. Sorting is only needed for the result file and stays outside the timer. On the recorded English run, StringZilla took 1.948 ms at k=1 and 41.555 ms at k=2. SymSpell took 20.931 and 455.857 ms. Both complete result files matched RapidFuzz. The comparison is part of the normal StringWars Cargo target and dependency lock instead of a separate Rust project. Signed-off-by: Guillaume de Rouville --- Cargo.lock | 25 +++++ Cargo.toml | 14 +++ levenshtein/bench.rs | 236 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 275 insertions(+) create mode 100644 levenshtein/bench.rs diff --git a/Cargo.lock b/Cargo.lock index f004abf..d95a389 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -27,6 +27,7 @@ dependencies = [ "const-random", "getrandom 0.3.4", "once_cell", + "serde", "version_check 0.9.5", "zerocopy", ] @@ -1691,6 +1692,15 @@ version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" +[[package]] +name = "gxhash" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3ce1bab7aa741d4e7042b2aae415b78741f267a98a7271ea226cd5ba6c43d7d" +dependencies = [ + "rustversion", +] + [[package]] name = "h2" version = "0.4.15" @@ -4784,6 +4794,7 @@ dependencies = [ "sodiumoxide", "stringtape", "stringzilla", + "symspell_rs", "twox-hash", "unicase", "unicode-linebreak", @@ -4868,6 +4879,20 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "symspell_rs" +version = "6.8.3" +source = "git+https://github.com/wolfgarbe/symspell_rs.git?rev=df6b21abbb3500b99e9e75857a616ff60f30f50f#df6b21abbb3500b99e9e75857a616ff60f30f50f" +dependencies = [ + "ahash", + "gxhash", + "itertools 0.14.0", + "regex", + "smallvec", + "strsim 0.11.1", + "unicode-normalization", +] + [[package]] name = "syn" version = "0.15.44" diff --git a/Cargo.toml b/Cargo.toml index e2c800b..94a4276 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -64,6 +64,9 @@ bench_hash = [ "ring", # SHA256 "cityhash", # CityHash (x86_64 only) ] +bench_levenshtein = [ + "symspell_rs", # SymSpell deletion index +] bench_memory = [ "rand", # Randomize Buffer "zeroize", # Obfuscate Buffer @@ -175,6 +178,11 @@ version = "0.2.11" optional = true version = "0.5.0" +[dependencies.symspell_rs] +git = "https://github.com/wolfgarbe/symspell_rs.git" +optional = true +rev = "df6b21abbb3500b99e9e75857a616ff60f30f50f" + [dependencies.forkunion] optional = true version = "3.0" @@ -316,6 +324,12 @@ name = "bench_hash" path = "hash/bench.rs" required-features = ["bench_hash"] +[[bench]] +harness = false +name = "bench_levenshtein" +path = "levenshtein/bench.rs" +required-features = ["bench_levenshtein"] + [[bench]] harness = false name = "bench_sequence" diff --git a/levenshtein/bench.rs b/levenshtein/bench.rs new file mode 100644 index 0000000..01fac61 --- /dev/null +++ b/levenshtein/bench.rs @@ -0,0 +1,236 @@ +//! Repeated bounded Levenshtein search over an immutable dictionary. +//! +//! The dictionary is built once and reused for every query. Use `STRINGWARS_FILTER=symspell` +//! to select this comparison. + +#[allow(dead_code)] +#[path = "../utils.rs"] +mod utils; + +use std::collections::HashMap; +use std::env; +use std::fs::{self, File}; +use std::io::{BufWriter, Write}; +use std::time::Instant; +use symspell_rs::{SymSpell, Verbosity}; +use utils::should_run; + +type AnyError = Box; + +#[derive(Clone, Copy)] +struct Match { + id: u32, + distance: u8, +} + +struct Settings { + repeats: usize, + min_distance: usize, + max_distance: usize, + dump_prefix: Option, + cache_evict_bytes: usize, +} + +fn env_usize(name: &str, default: usize) -> Result { + env::var(name).map_or(Ok(default), |value| Ok(value.parse()?)) +} + +fn load_lines(path: &str, limit: usize) -> Result, AnyError> { + let contents = fs::read_to_string(path)?; + Ok(contents + .lines() + .take(if limit == 0 { usize::MAX } else { limit }) + .map(|line| line.trim_end_matches('\r').to_owned()) + .collect()) +} + +fn evict_cache(buffer: &mut [u8], checksum: &mut u64) { + for value in buffer.iter_mut().step_by(64) { + *value = value.wrapping_add(1); + *checksum = checksum.wrapping_add(*value as u64); + } + std::hint::black_box(*checksum); +} + +fn levenshtein_within(first: &str, second: &str, bound: usize) -> Option { + let first: Vec = first.chars().collect(); + let second: Vec = second.chars().collect(); + if first.len().abs_diff(second.len()) > bound { + return None; + } + + let mut previous: Vec = (0..=first.len()).collect(); + let mut current = vec![0; first.len() + 1]; + for (row, &second_char) in second.iter().enumerate() { + current[0] = row + 1; + let mut row_min = current[0]; + for (column, &first_char) in first.iter().enumerate() { + current[column + 1] = (previous[column + 1] + 1) + .min(current[column] + 1) + .min(previous[column] + usize::from(first_char != second_char)); + row_min = row_min.min(current[column + 1]); + } + if row_min > bound { + return None; + } + std::mem::swap(&mut previous, &mut current); + } + (previous[first.len()] <= bound).then_some(previous[first.len()]) +} + +fn symspell_search( + symspell: &SymSpell, + ids: &HashMap, + query: &str, + bound: usize, +) -> Vec { + symspell + .lookup(query, Verbosity::All, bound, &None, None, false) + .into_iter() + .filter_map(|suggestion| { + levenshtein_within(query, &suggestion.term, bound).map(|distance| Match { + id: ids[&suggestion.term], + distance: distance as u8, + }) + }) + .collect() +} + +fn dump_symspell_matches( + symspell: &SymSpell, + ids: &HashMap, + dictionary_size: usize, + queries: &[String], + bound: usize, + path: &str, +) -> Result<(), AnyError> { + let mut output = BufWriter::new(File::create(path)?); + output.write_all(b"SZLEV001")?; + output.write_all(&(dictionary_size as u64).to_ne_bytes())?; + output.write_all(&(queries.len() as u64).to_ne_bytes())?; + output.write_all(&[bound as u8])?; + for query in queries { + let mut matches = symspell_search(symspell, ids, query, bound); + matches.sort_unstable_by_key(|found| (found.id, found.distance)); + output.write_all(&(matches.len() as u64).to_ne_bytes())?; + for found in matches { + output.write_all(&found.id.to_ne_bytes())?; + output.write_all(&[found.distance])?; + } + } + Ok(()) +} + +fn run_symspell( + dictionary: &[String], + queries: &[String], + settings: &Settings, +) -> Result<(), AnyError> { + if dictionary + .iter() + .chain(queries) + .any(|text| text.to_lowercase() != *text) + { + return Err("SymSpell requires lowercase input for case-sensitive comparison".into()); + } + + let mut ids = HashMap::with_capacity(dictionary.len()); + for (id, word) in dictionary.iter().enumerate() { + if ids.insert(word.clone(), id as u32).is_some() { + return Err("SymSpell cannot preserve duplicate dictionary entries".into()); + } + } + + println!("# levenshtein/symspell"); + println!( + "dictionary={} queries={} semantics=utf8-codepoints+exact-levenshtein-filter output=id+distance", + dictionary.len(), + queries.len() + ); + let mut cache = vec![0u8; settings.cache_evict_bytes]; + let mut cache_checksum = 0u64; + + for bound in settings.min_distance..=settings.max_distance { + let build_start = Instant::now(); + let mut symspell = SymSpell::new(bound, None, 7, 1); + for word in dictionary { + symspell.create_dictionary_entry(word, 1); + } + println!( + "k={bound} build={:.6}s indexed_words={}", + build_start.elapsed().as_secs_f64(), + symspell.get_dictionary_size() + ); + + for repeat in 0..settings.repeats { + evict_cache(&mut cache, &mut cache_checksum); + let start = Instant::now(); + let mut matches_count = 0usize; + let mut checksum = 0u64; + for query in queries { + for found in symspell_search(&symspell, &ids, query, bound) { + matches_count += 1; + checksum = checksum.wrapping_add( + ((found.id as u64) << 8 | found.distance as u64) + .wrapping_mul(0x9E37_79B1_85EB_CA87), + ); + } + } + println!( + "k={bound} repeat={repeat} query={:.6}s matches={matches_count} checksum={checksum:016x} cache_evict_bytes={}", + start.elapsed().as_secs_f64(), + cache.len() + ); + } + + if let Some(prefix) = &settings.dump_prefix { + dump_symspell_matches( + &symspell, + &ids, + dictionary.len(), + queries, + bound, + &format!("{prefix}.symspell.k{bound}.bin"), + )?; + } + } + Ok(()) +} + +fn main() -> Result<(), AnyError> { + let args: Vec = env::args() + .filter(|argument| argument != "--bench") + .collect(); + if args.len() < 3 || args.len() > 4 { + eprintln!("usage: bench_levenshtein DICTIONARY QUERIES [QUERY_LIMIT]"); + std::process::exit(2); + } + + let query_limit = args.get(3).map_or(Ok(0), |value| value.parse::())?; + let dictionary = load_lines(&args[1], 0)?; + let queries = load_lines(&args[2], query_limit)?; + let cache_evict_mb = env_usize("STRINGWARS_CACHE_EVICT_MB", 0)?; + let settings = Settings { + repeats: env_usize("STRINGWARS_REPEATS", 3)?, + min_distance: env_usize("STRINGWARS_MIN_DISTANCE", 1)?, + max_distance: env_usize("STRINGWARS_MAX_DISTANCE", 2)?, + dump_prefix: env::var("STRINGWARS_DUMP_PREFIX").ok(), + cache_evict_bytes: cache_evict_mb + .checked_mul(1024 * 1024) + .ok_or("STRINGWARS_CACHE_EVICT_MB is too large")?, + }; + if settings.repeats == 0 + || settings.min_distance == 0 + || settings.min_distance > settings.max_distance + || settings.max_distance > 4 + { + return Err( + "expected positive repeats and 1 <= minimum distance <= maximum distance <= 4".into(), + ); + } + + if should_run("levenshtein/symspell") { + run_symspell(&dictionary, &queries, &settings)?; + } + Ok(()) +} From 75c4d3f35dbe8c767b2a48616e79cbc16a1c2b75 Mon Sep 17 00:00:00 2001 From: Guillaume de Rouville Date: Sat, 15 Aug 2026 23:43:43 -0700 Subject: [PATCH 05/17] Add: Rust fst Levenshtein comparison Add Rust fst 0.4.7 to the shared dictionary-search benchmark. fst returns dictionary IDs without exact distances and cannot keep duplicate keys. The comparable run therefore uses unique ASCII input, where its character behavior and StringZilla's byte behavior agree. On the English run, fst took 0.866, 5.469, 62.284, and 226.019 seconds at bounds one through four. StringZilla took 1.948 ms, 41.555 ms, 5.461 seconds, and 16.223 seconds. Match counts agreed, but fst does less output work, so that difference stays visible. The optional Unicode path first checks three one-character examples. The pinned fst version misses valid substitutions there, so the runner stops instead of timing answers that do not match. Signed-off-by: Guillaume de Rouville --- Cargo.lock | 16 ++++++ Cargo.toml | 6 +++ levenshtein/bench.rs | 119 ++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 139 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d95a389..f5e3a8e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1510,6 +1510,15 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "fst" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ab85b9b05e3978cc9a9cf8fea7f01b494e1a09ed3037e16ba39edc7a29eb61a" +dependencies = [ + "utf8-ranges", +] + [[package]] name = "futures" version = "0.3.33" @@ -4772,6 +4781,7 @@ dependencies = [ "fastbloom", "foldhash 0.2.0", "forkunion", + "fst", "getrandom 0.3.4", "icu", "libc", @@ -5340,6 +5350,12 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" +[[package]] +name = "utf8-ranges" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcfc827f90e53a02eaef5e535ee14266c1d569214c6aa70133a624d8a3164ba" + [[package]] name = "utf8_iter" version = "1.0.4" diff --git a/Cargo.toml b/Cargo.toml index 94a4276..a30ad8a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -65,6 +65,7 @@ bench_hash = [ "cityhash", # CityHash (x86_64 only) ] bench_levenshtein = [ + "fst", # Levenshtein automaton over a finite-state dictionary "symspell_rs", # SymSpell deletion index ] bench_memory = [ @@ -178,6 +179,11 @@ version = "0.2.11" optional = true version = "0.5.0" +[dependencies.fst] +features = ["levenshtein"] +optional = true +version = "0.4.7" + [dependencies.symspell_rs] git = "https://github.com/wolfgarbe/symspell_rs.git" optional = true diff --git a/levenshtein/bench.rs b/levenshtein/bench.rs index 01fac61..d6b7551 100644 --- a/levenshtein/bench.rs +++ b/levenshtein/bench.rs @@ -1,12 +1,14 @@ //! Repeated bounded Levenshtein search over an immutable dictionary. //! -//! The dictionary is built once and reused for every query. Use `STRINGWARS_FILTER=symspell` -//! to select this comparison. +//! The dictionary is built once and reused for every query. Use `STRINGWARS_FILTER` to select +//! `levenshtein/fst` or `levenshtein/symspell`. #[allow(dead_code)] #[path = "../utils.rs"] mod utils; +use fst::automaton::Levenshtein; +use fst::{IntoStreamer, Map, Streamer}; use std::collections::HashMap; use std::env; use std::fs::{self, File}; @@ -44,6 +46,12 @@ fn load_lines(path: &str, limit: usize) -> Result, AnyError> { .collect()) } +fn checksum_id(checksum: u64, id: u64) -> u64 { + checksum + .wrapping_mul(0x9E37_79B1_85EB_CA87) + .wrapping_add(id) +} + fn evict_cache(buffer: &mut [u8], checksum: &mut u64) { for value in buffer.iter_mut().step_by(64) { *value = value.wrapping_add(1); @@ -197,6 +205,110 @@ fn run_symspell( Ok(()) } +fn verify_fst_unicode_contract() -> Result<(), AnyError> { + let mut keys = vec!["é", "Ѐ", "А"]; + keys.sort_unstable(); + let map = Map::from_iter(keys.iter().enumerate().map(|(id, key)| (*key, id as u64)))?; + let mut matches = 0usize; + for query in &keys { + let automaton = Levenshtein::new(query, 1)?; + let mut stream = map.search(&automaton).into_stream(); + while stream.next().is_some() { + matches += 1; + } + } + if matches != 9 { + return Err(format!( + "fst Unicode smoke test failed: expected 9 one-character matches, observed {matches}" + ) + .into()); + } + Ok(()) +} + +fn run_fst(dictionary: &[String], queries: &[String], settings: &Settings) -> Result<(), AnyError> { + let allow_unicode = env::var_os("STRINGWARS_ALLOW_FST_UNICODE").is_some(); + if allow_unicode { + verify_fst_unicode_contract()?; + } else if dictionary + .iter() + .chain(queries) + .any(|text| !text.is_ascii()) + { + return Err("fst requires ASCII input for byte-for-byte semantic parity".into()); + } + + let mut keyed_words: Vec<(&str, u64)> = dictionary + .iter() + .enumerate() + .map(|(id, word)| (word.as_str(), id as u64)) + .collect(); + keyed_words.sort_unstable_by_key(|&(word, _)| word); + if keyed_words.windows(2).any(|pair| pair[0].0 == pair[1].0) { + return Err("fst cannot preserve duplicate dictionary entries".into()); + } + + let build_start = Instant::now(); + let map = Map::from_iter(keyed_words)?; + println!("# levenshtein/fst"); + println!( + "dictionary={} queries={} semantics={} build={:.6}s fst_bytes={} output=id-only", + dictionary.len(), + queries.len(), + if allow_unicode { + "unicode-codepoints" + } else { + "ascii-byte-parity" + }, + build_start.elapsed().as_secs_f64(), + map.as_fst().as_bytes().len() + ); + + let state_limit = env::var("STRINGWARS_FST_STATE_LIMIT") + .ok() + .map(|value| value.parse::()) + .transpose()?; + let mut cache = vec![0u8; settings.cache_evict_bytes]; + let mut cache_checksum = 0u64; + for bound in settings.min_distance..=settings.max_distance { + for repeat in 0..settings.repeats { + evict_cache(&mut cache, &mut cache_checksum); + let start = Instant::now(); + let mut matches_count = 0usize; + let mut checksum = 0u64; + let mut failed = None; + for query in queries { + let automaton = match state_limit { + Some(limit) => Levenshtein::new_with_limit(query, bound as u32, limit), + None => Levenshtein::new(query, bound as u32), + }; + let automaton = match automaton { + Ok(automaton) => automaton, + Err(error) => { + failed = Some(error); + break; + } + }; + let mut stream = map.search(&automaton).into_stream(); + while let Some((_, id)) = stream.next() { + matches_count += 1; + checksum = checksum_id(checksum, id); + } + } + if let Some(error) = failed { + println!("k={bound} skipped={error}"); + break; + } + println!( + "k={bound} repeat={repeat} query={:.6}s matches={matches_count} checksum={checksum:016x} cache_evict_bytes={}", + start.elapsed().as_secs_f64(), + cache.len() + ); + } + } + Ok(()) +} + fn main() -> Result<(), AnyError> { let args: Vec = env::args() .filter(|argument| argument != "--bench") @@ -229,6 +341,9 @@ fn main() -> Result<(), AnyError> { ); } + if should_run("levenshtein/fst") { + run_fst(&dictionary, &queries, &settings)?; + } if should_run("levenshtein/symspell") { run_symspell(&dictionary, &queries, &settings)?; } From c5f23fec996a782f67d1675b4a31a33be7cd5ac4 Mon Sep 17 00:00:00 2001 From: Guillaume de Rouville Date: Sat, 15 Aug 2026 23:45:32 -0700 Subject: [PATCH 06/17] Add: Tantivy fuzzy search comparison Add Tantivy 0.26.1 through its public fuzzy term query, with adjacent swaps disabled. The runner builds one in-memory index, waits for its indexing work to finish, then collects every matching document address. Build and query time are reported separately. Tantivy returns IDs without exact distances and supports bounds one and two, so the output states those differences. On the English run, Tantivy took 485.1 ms at k=1 and 4.183 seconds at k=2. StringZilla took 1.948 and 41.555 ms. Most of this commit is the generated Cargo lock update for Tantivy's search and indexing dependencies. Signed-off-by: Guillaume de Rouville --- Cargo.lock | 426 ++++++++++++++++++++++++++++++++++++++++++- Cargo.toml | 5 + levenshtein/bench.rs | 77 +++++++- 3 files changed, 505 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f5e3a8e..6d49c32 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -104,6 +104,15 @@ dependencies = [ "object", ] +[[package]] +name = "arc-swap" +version = "1.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b" +dependencies = [ + "rustversion", +] + [[package]] name = "argminmax" version = "0.6.3" @@ -610,6 +619,15 @@ dependencies = [ "serde_core", ] +[[package]] +name = "bitpacking" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96a7139abd3d9cebf8cd6f920a389cf3dc9576172e32f4563f188cae3c3eb019" +dependencies = [ + "crunchy", +] + [[package]] name = "blake3" version = "1.8.5" @@ -656,6 +674,31 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "bon" +version = "3.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a602c73c7b0148ec6d12af6fd5cc7a46e2eacc8878271a999abac56eed12f561" +dependencies = [ + "bon-macros", + "rustversion", +] + +[[package]] +name = "bon-macros" +version = "3.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dee98b0db6a962de883bf5d20362dee4d7ca0d12fe39a7c6c73c844e1cd7c1f" +dependencies = [ + "darling", + "ident_case", + "prettyplease", + "proc-macro2 1.0.107", + "quote 1.0.47", + "rustversion", + "syn 2.0.119", +] + [[package]] name = "borsh" version = "1.8.0" @@ -837,13 +880,19 @@ dependencies = [ "shlex 2.0.1", ] +[[package]] +name = "census" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f4c707c6a209cbe82d10abd08e1ea8995e9ea937d2550646e02798948992be0" + [[package]] name = "cexpr" version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fce5b5fb86b0c57c20c834c1b412fd09c77c8a59b9473f86272709e78874cd1d" dependencies = [ - "nom", + "nom 4.2.3", ] [[package]] @@ -1181,12 +1230,27 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "datasketches" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c286de4e81ea2590afc24d754e0f83810c566f50a1388fa75ebd57928c0d9745" + [[package]] name = "debug_unsafe" version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7eed2c4702fa172d1ce21078faa7c5203e69f5394d48cc436d25928394a867a2" +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +dependencies = [ + "serde_core", +] + [[package]] name = "derefable" version = "0.1.0" @@ -1247,6 +1311,12 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "downcast-rs" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "117240f60069e65410b3ae1bb213295bd828f707b5bec6596a1afc8793ce0cbc" + [[package]] name = "dyn-clone" version = "1.0.20" @@ -1313,6 +1383,17 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +[[package]] +name = "erased-serde" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2add8a07dd6a8d93ff627029c51de145e12686fbc36ecb298ac22e74cf02dec" +dependencies = [ + "serde", + "serde_core", + "typeid", +] + [[package]] name = "errno" version = "0.3.14" @@ -1384,6 +1465,12 @@ dependencies = [ "siphasher 1.0.3", ] +[[package]] +name = "fastdivide" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afc2bd4d5a73106dd53d10d73d3401c2f32730ba2c0b93ddb888a8983680471" + [[package]] name = "fastrand" version = "2.5.0" @@ -1814,6 +1901,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "htmlescape" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163" + [[package]] name = "http" version = "1.4.2" @@ -2395,6 +2488,15 @@ dependencies = [ "serde_core", ] +[[package]] +name = "inventory" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4f0c30c76f2f4ccee3fe55a2435f691ca00c0e4bd87abe4f4a851b1d4dac39b" +dependencies = [ + "rustversion", +] + [[package]] name = "ipnet" version = "2.12.0" @@ -2461,6 +2563,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "levenshtein_automata" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c2cdeb66e45e9f36bfad5bbdb4d2384e70936afbee843c6f6543f0c551ebb25" + [[package]] name = "lexical-core" version = "1.0.6" @@ -2579,6 +2687,15 @@ version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +[[package]] +name = "lru" +version = "0.16.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f66e8d5d03f609abc3a39e6f08e4164ebf1447a732906d39eb9b99b7919ef39" +dependencies = [ + "hashbrown 0.16.1", +] + [[package]] name = "lru-slab" version = "0.1.2" @@ -2604,6 +2721,12 @@ dependencies = [ "libc", ] +[[package]] +name = "lz4_flex" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e" + [[package]] name = "matrixmultiply" version = "0.3.11" @@ -2614,6 +2737,15 @@ dependencies = [ "rawpointer", ] +[[package]] +name = "measure_time" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51c55d61e72fc3ab704396c5fa16f4c184db37978ae4e94ca8959693a235fc0e" +dependencies = [ + "log", +] + [[package]] name = "memchr" version = "2.8.3" @@ -2629,6 +2761,12 @@ dependencies = [ "libc", ] +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.8.9" @@ -2659,6 +2797,12 @@ dependencies = [ "serde", ] +[[package]] +name = "murmurhash32" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2195bf6aa996a481483b29d62a7663eed3fe39600c460e323f8ff41e90bdd89b" + [[package]] name = "murmurhash32" version = "0.4.0" @@ -2716,6 +2860,16 @@ dependencies = [ "version_check 0.1.5", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "now" version = "0.1.3" @@ -2753,6 +2907,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + [[package]] name = "num-derive" version = "0.4.2" @@ -2865,6 +3025,12 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "oneshot" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "269bca4c2591a28585d6bf10d9ed0332b7d76900a1b02bec41bdc3a2cdcda107" + [[package]] name = "openssl" version = "0.10.81" @@ -2917,6 +3083,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ownedbytes" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fbd56f7631767e61784dc43f8580f403f4475bd4aaa4da003e6295e1bab4a7e" +dependencies = [ + "stable_deref_trait", +] + [[package]] name = "parking" version = "2.2.1" @@ -3720,6 +3895,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -3729,6 +3910,16 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2 1.0.107", + "syn 2.0.119", +] + [[package]] name = "probabilistic-collections" version = "0.7.0" @@ -4616,6 +4807,15 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" +[[package]] +name = "sketches-ddsketch" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "513c3f5f732bfd6fbb187619c2dfe9d2f25f1a2976f01d575f0fd329d565df56" +dependencies = [ + "serde", +] + [[package]] name = "slab" version = "0.4.12" @@ -4786,7 +4986,7 @@ dependencies = [ "icu", "libc", "memchr", - "murmurhash32", + "murmurhash32 0.4.0", "openssl", "pcre2", "perf-event", @@ -4805,6 +5005,7 @@ dependencies = [ "stringtape", "stringzilla", "symspell_rs", + "tantivy", "twox-hash", "unicase", "unicode-linebreak", @@ -4970,6 +5171,167 @@ dependencies = [ "windows", ] +[[package]] +name = "tantivy" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edde6a10743fff00a4e1a8c9ef020bf5f3cbad301b7d2d39f2b07f123c4eac07" +dependencies = [ + "aho-corasick", + "arc-swap", + "base64", + "bitpacking", + "bon", + "byteorder", + "census", + "crc32fast", + "crossbeam-channel", + "datasketches", + "downcast-rs", + "fastdivide", + "fnv", + "fs4", + "htmlescape", + "itertools 0.14.0", + "levenshtein_automata", + "log", + "lru", + "lz4_flex", + "measure_time", + "memmap2", + "once_cell", + "oneshot", + "rayon", + "regex", + "rust-stemmers", + "rustc-hash", + "serde", + "serde_json", + "sketches-ddsketch", + "smallvec", + "tantivy-bitpacker", + "tantivy-columnar", + "tantivy-common", + "tantivy-fst", + "tantivy-query-grammar", + "tantivy-stacker", + "tantivy-tokenizer-api", + "tempfile", + "thiserror 2.0.19", + "time", + "typetag", + "uuid", + "winapi", +] + +[[package]] +name = "tantivy-bitpacker" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fed3d674429bcd2de5d0a6d1aa5495fed8afd9c5ecce993019caf7615f53fa4" +dependencies = [ + "bitpacking", +] + +[[package]] +name = "tantivy-columnar" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c57166f5bcfd478f370ab8445afb4678dce44801fa5ce5c451aaf8595583c5dc" +dependencies = [ + "downcast-rs", + "fastdivide", + "itertools 0.14.0", + "serde", + "tantivy-bitpacker", + "tantivy-common", + "tantivy-sstable", + "tantivy-stacker", +] + +[[package]] +name = "tantivy-common" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbf10915aa75da3c3b0d58b58853d2e889efbaf32d4982a4c3715dde6bba23e5" +dependencies = [ + "async-trait", + "byteorder", + "ownedbytes", + "serde", + "time", +] + +[[package]] +name = "tantivy-fst" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d60769b80ad7953d8a7b2c70cdfe722bbcdcac6bccc8ac934c40c034d866fc18" +dependencies = [ + "byteorder", + "regex-syntax", + "utf8-ranges", +] + +[[package]] +name = "tantivy-query-grammar" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfadb8526b6da90704feb293b0701a6aae62ea14983143344be2dc5ce30f1d82" +dependencies = [ + "fnv", + "nom 7.1.3", + "ordered-float", + "serde", + "serde_json", +] + +[[package]] +name = "tantivy-sstable" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a2cfc3ac5164cbadc28965ffb145a8f47582a60ae5897859ad8d4316596c606" +dependencies = [ + "futures-util", + "itertools 0.14.0", + "tantivy-bitpacker", + "tantivy-common", + "tantivy-fst", + "zstd", +] + +[[package]] +name = "tantivy-stacker" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cbb051742da9d53ca9e8fff43a9b10e319338b24e2c0e15d0372df19ffeb951" +dependencies = [ + "murmurhash32 0.3.1", + "tantivy-common", +] + +[[package]] +name = "tantivy-tokenizer-api" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eac258c2c6390673f2685813afeeafcb8c4e0ee7de8dd3fc46838dcc37263f98" +dependencies = [ + "serde", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + [[package]] name = "term_size" version = "0.3.2" @@ -5048,6 +5410,36 @@ dependencies = [ "cfg-if 1.0.4", ] +[[package]] +name = "time" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "tiny-keccak" version = "2.0.2" @@ -5260,12 +5652,42 @@ dependencies = [ "rand 0.10.2", ] +[[package]] +name = "typeid" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" + [[package]] name = "typenum" version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" +[[package]] +name = "typetag" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c90e86058a30d42a1a928dfb4b49bb33c98c3a2b4909492e6b0881cd94798ec2" +dependencies = [ + "erased-serde", + "inventory", + "once_cell", + "serde", + "typetag-impl", +] + +[[package]] +name = "typetag-impl" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f153acc4e99a5f2a5aefa09fb078be54e26271b2813f6041200b224c098d8328" +dependencies = [ + "proc-macro2 1.0.107", + "quote 1.0.47", + "syn 3.0.2", +] + [[package]] name = "unicase" version = "2.9.0" diff --git a/Cargo.toml b/Cargo.toml index a30ad8a..dad34b6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -67,6 +67,7 @@ bench_hash = [ bench_levenshtein = [ "fst", # Levenshtein automaton over a finite-state dictionary "symspell_rs", # SymSpell deletion index + "tantivy", # Search-engine fuzzy term lookup ] bench_memory = [ "rand", # Randomize Buffer @@ -189,6 +190,10 @@ git = "https://github.com/wolfgarbe/symspell_rs.git" optional = true rev = "df6b21abbb3500b99e9e75857a616ff60f30f50f" +[dependencies.tantivy] +optional = true +version = "0.26.1" + [dependencies.forkunion] optional = true version = "3.0" diff --git a/levenshtein/bench.rs b/levenshtein/bench.rs index d6b7551..d9b7a82 100644 --- a/levenshtein/bench.rs +++ b/levenshtein/bench.rs @@ -1,7 +1,7 @@ //! Repeated bounded Levenshtein search over an immutable dictionary. //! //! The dictionary is built once and reused for every query. Use `STRINGWARS_FILTER` to select -//! `levenshtein/fst` or `levenshtein/symspell`. +//! `levenshtein/fst`, `levenshtein/symspell`, or `levenshtein/tantivy`. #[allow(dead_code)] #[path = "../utils.rs"] @@ -15,6 +15,10 @@ use std::fs::{self, File}; use std::io::{BufWriter, Write}; use std::time::Instant; use symspell_rs::{SymSpell, Verbosity}; +use tantivy::collector::DocSetCollector; +use tantivy::query::FuzzyTermQuery; +use tantivy::schema::{Schema, STRING}; +use tantivy::{doc, Index, Term}; use utils::should_run; type AnyError = Box; @@ -309,6 +313,74 @@ fn run_fst(dictionary: &[String], queries: &[String], settings: &Settings) -> Re Ok(()) } +fn run_tantivy( + dictionary: &[String], + queries: &[String], + settings: &Settings, +) -> Result<(), AnyError> { + let mut schema_builder = Schema::builder(); + let word_field = schema_builder.add_text_field("word", STRING); + let index = Index::create_in_ram(schema_builder.build()); + let build_start = Instant::now(); + let mut writer = index.writer(50_000_000)?; + for word in dictionary { + writer.add_document(doc!(word_field => word.as_str()))?; + } + writer.commit()?; + writer.wait_merging_threads()?; + let reader = index.reader()?; + let searcher = reader.searcher(); + println!("# levenshtein/tantivy"); + println!( + "dictionary={} queries={} build={:.6}s segments={} output=id-only", + dictionary.len(), + queries.len(), + build_start.elapsed().as_secs_f64(), + searcher.segment_readers().len() + ); + + let mut cache = vec![0u8; settings.cache_evict_bytes]; + let mut cache_checksum = 0u64; + for bound in settings.min_distance..=settings.max_distance { + if bound > u8::MAX as usize { + break; + } + for repeat in 0..settings.repeats { + evict_cache(&mut cache, &mut cache_checksum); + let start = Instant::now(); + let mut matches_count = 0usize; + let mut checksum = 0u64; + let mut failed = None; + for query_text in queries { + let term = Term::from_field_text(word_field, query_text); + let query = FuzzyTermQuery::new(term, bound as u8, false); + let matches = match searcher.search(&query, &DocSetCollector) { + Ok(matches) => matches, + Err(error) => { + failed = Some(error); + break; + } + }; + matches_count += matches.len(); + for address in matches { + let id = (u64::from(address.segment_ord) << 32) | u64::from(address.doc_id); + checksum = checksum_id(checksum, id); + } + } + if let Some(error) = failed { + println!("k={bound} skipped={error}"); + break; + } + println!( + "k={bound} repeat={repeat} query={:.6}s matches={matches_count} checksum={checksum:016x} cache_evict_bytes={}", + start.elapsed().as_secs_f64(), + cache.len() + ); + } + } + Ok(()) +} + fn main() -> Result<(), AnyError> { let args: Vec = env::args() .filter(|argument| argument != "--bench") @@ -347,5 +419,8 @@ fn main() -> Result<(), AnyError> { if should_run("levenshtein/symspell") { run_symspell(&dictionary, &queries, &settings)?; } + if should_run("levenshtein/tantivy") { + run_tantivy(&dictionary, &queries, &settings)?; + } Ok(()) } From 8b4ff4746964abbcd7e71422876a9b9151fd4626 Mon Sep 17 00:00:00 2001 From: Guillaume de Rouville Date: Sat, 15 Aug 2026 23:45:50 -0700 Subject: [PATCH 07/17] Add: Lucene Levenshtein automaton comparison Add Lucene 10.3.1 on OpenJDK 21 through its public fuzzy query and Levenshtein automaton. The normal fuzzy query applies a shorter-term rule that plain Levenshtein search does not have. Its different match count is reported but not used for a speed ratio. The automaton mode disables adjacent swaps and returns the expected totals, so that is the comparable mode. Lucene returns hit counts without dictionary IDs or exact distances. On the English run, its exact automaton took 1.676 seconds at k=1 and 14.965 seconds at k=2. StringZilla took 1.948 and 41.555 ms. Signed-off-by: Guillaume de Rouville --- levenshtein/lucene/pom.xml | 34 ++++++++ .../LevenshteinIndexBenchmark.java | 87 +++++++++++++++++++ 2 files changed, 121 insertions(+) create mode 100644 levenshtein/lucene/pom.xml create mode 100644 levenshtein/lucene/src/main/java/com/stringzilla/LevenshteinIndexBenchmark.java diff --git a/levenshtein/lucene/pom.xml b/levenshtein/lucene/pom.xml new file mode 100644 index 0000000..dc39762 --- /dev/null +++ b/levenshtein/lucene/pom.xml @@ -0,0 +1,34 @@ + + + 4.0.0 + com.stringzilla + levenshtein-index-lucene + 1.0-SNAPSHOT + + 21 + UTF-8 + + + + org.apache.lucene + lucene-core + 10.3.1 + + + org.apache.lucene + lucene-analysis-common + 10.3.1 + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + + + diff --git a/levenshtein/lucene/src/main/java/com/stringzilla/LevenshteinIndexBenchmark.java b/levenshtein/lucene/src/main/java/com/stringzilla/LevenshteinIndexBenchmark.java new file mode 100644 index 0000000..9f0c627 --- /dev/null +++ b/levenshtein/lucene/src/main/java/com/stringzilla/LevenshteinIndexBenchmark.java @@ -0,0 +1,87 @@ +package com.stringzilla; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import org.apache.lucene.analysis.core.KeywordAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.FuzzyQuery; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TotalHitCountCollector; +import org.apache.lucene.store.ByteBuffersDirectory; +import org.apache.lucene.search.AutomatonQuery; +import org.apache.lucene.util.automaton.LevenshteinAutomata; + +public final class LevenshteinIndexBenchmark { + private static List loadAscii(Path path) throws Exception { + List lines = Files.readAllLines(path, StandardCharsets.UTF_8); + for (String line : lines) + for (int index = 0; index != line.length(); ++index) + if (line.charAt(index) > 0x7f) + throw new IllegalArgumentException("non-ASCII input would compare different semantics: " + path); + return lines; + } + + public static void main(String[] args) throws Exception { + if (args.length != 4) { + System.err.println("usage: LevenshteinIndexBenchmark DICTIONARY QUERIES MAX_DISTANCE fuzzy|automaton"); + System.exit(2); + } + List dictionary = loadAscii(Path.of(args[0])); + List queries = loadAscii(Path.of(args[1])); + int maxDistance = Integer.parseInt(args[2]); + boolean exactAutomaton = args[3].equals("automaton"); + if (!exactAutomaton && !args[3].equals("fuzzy")) { + System.err.println("mode must be fuzzy or automaton"); + System.exit(2); + } + if (maxDistance < 1 || maxDistance > FuzzyQuery.defaultMaxEdits) { + System.err.printf("MAX_DISTANCE must be between 1 and %d%n", FuzzyQuery.defaultMaxEdits); + System.exit(2); + } + + long buildStart = System.nanoTime(); + ByteBuffersDirectory directory = new ByteBuffersDirectory(); + IndexWriterConfig config = new IndexWriterConfig(new KeywordAnalyzer()); + try (IndexWriter writer = new IndexWriter(directory, config)) { + for (String word : dictionary) { + Document document = new Document(); + document.add(new StringField("term", word, Field.Store.NO)); + writer.addDocument(document); + } + writer.forceMerge(1); + } + DirectoryReader reader = DirectoryReader.open(directory); + IndexSearcher searcher = new IndexSearcher(reader); + double buildSeconds = (System.nanoTime() - buildStart) * 1e-9; + System.out.printf("dictionary=%d queries=%d build=%.6fs%n", dictionary.size(), queries.size(), buildSeconds); + + for (int bound = 1; bound <= maxDistance; ++bound) { + for (int repeat = 0; repeat != 3; ++repeat) { + long matches = 0; + long start = System.nanoTime(); + for (String query : queries) { + Term term = new Term("term", query); + Query fuzzy = exactAutomaton + ? new AutomatonQuery(term, new LevenshteinAutomata(query, false).toAutomaton(bound)) + : new FuzzyQuery(term, bound, 0, Integer.MAX_VALUE, false); + TotalHitCountCollector collector = new TotalHitCountCollector(); + searcher.search(fuzzy, collector); + matches += collector.getTotalHits(); + } + double seconds = (System.nanoTime() - start) * 1e-9; + System.out.printf("k=%d query=%.6fs matches=%d%n", bound, seconds, matches); + } + } + reader.close(); + directory.close(); + } +} From f2bf0d38f3be5ee355d55d7584871b1722d4b76d Mon Sep 17 00:00:00 2001 From: Guillaume de Rouville Date: Sat, 15 Aug 2026 23:45:56 -0700 Subject: [PATCH 08/17] Add: UTF-8 Levenshtein result checker Add the RapidFuzz reference for Unicode search. It validates and decodes UTF-8 before timing, then measures distance between Unicode characters. The helper extracts 348,980 unique lowercase terms from the Simplified Chinese dictionary without changing their text. StringZilla and RapidFuzz produced identical complete result files on both the small non-ASCII test and the natural Chinese data. The Chinese run returned 2,219,220 matches at k=1 and 343,237,926 at k=2. StringZilla took 16.017 ms and 5.149 seconds. The very large k=2 output is an important limit on broader speed claims. Signed-off-by: Guillaume de Rouville --- levenshtein/dictionary_terms.py | 27 +++++++ levenshtein/rapidfuzz_utf8.cpp | 130 ++++++++++++++++++++++++++++++++ 2 files changed, 157 insertions(+) create mode 100644 levenshtein/dictionary_terms.py create mode 100644 levenshtein/rapidfuzz_utf8.cpp diff --git a/levenshtein/dictionary_terms.py b/levenshtein/dictionary_terms.py new file mode 100644 index 0000000..1490737 --- /dev/null +++ b/levenshtein/dictionary_terms.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +"""Extract unique already-lowercase terms from a whitespace-delimited frequency dictionary.""" + +import argparse + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("input") + parser.add_argument("output") + args = parser.parse_args() + + seen: set[str] = set() + with open(args.input, encoding="utf-8") as source, open(args.output, "w", encoding="utf-8") as target: + for line in source: + fields = line.split() + if not fields: + continue + term = fields[0] + if term.lower() != term or term in seen: + continue + seen.add(term) + target.write(term + "\n") + + +if __name__ == "__main__": + main() diff --git a/levenshtein/rapidfuzz_utf8.cpp b/levenshtein/rapidfuzz_utf8.cpp new file mode 100644 index 0000000..e05af1b --- /dev/null +++ b/levenshtein/rapidfuzz_utf8.cpp @@ -0,0 +1,130 @@ +/** + * @brief Native RapidFuzz baseline for validated UTF-8/codepoint immutable-dictionary retrieval. + * + * This is intentionally outside the default build because it requires rapidfuzz-cpp. UTF-8 decoding happens while + * loading the corpus, outside the timed query loop, favoring RapidFuzz relative to StringZilla's per-query facade. + */ +#include + +#include +#include +#include +#include +#include +#include +#include + +struct match_t { + std::uint32_t id; + std::uint8_t distance; +}; + +static bool decode_utf8(std::string const &source, std::u32string &destination) { + destination.clear(); + for (std::size_t i = 0; i != source.size();) { + std::uint8_t const lead = static_cast(source[i]); + char32_t rune = 0; + std::size_t length = 0; + if (lead < 0x80) rune = lead, length = 1; + else if (lead >= 0xC2 && lead <= 0xDF && i + 1 < source.size()) + rune = (char32_t(lead & 0x1F) << 6) | (static_cast(source[i + 1]) & 0x3F), length = 2; + else if (lead >= 0xE0 && lead <= 0xEF && i + 2 < source.size()) + rune = (char32_t(lead & 0x0F) << 12) | ((static_cast(source[i + 1]) & 0x3F) << 6) | + (static_cast(source[i + 2]) & 0x3F), + length = 3; + else if (lead >= 0xF0 && lead <= 0xF4 && i + 3 < source.size()) + rune = (char32_t(lead & 0x07) << 18) | ((static_cast(source[i + 1]) & 0x3F) << 12) | + ((static_cast(source[i + 2]) & 0x3F) << 6) | + (static_cast(source[i + 3]) & 0x3F), + length = 4; + else + return false; + for (std::size_t continuation = 1; continuation != length; ++continuation) + if ((static_cast(source[i + continuation]) & 0xC0) != 0x80) return false; + if ((length == 3 && ((lead == 0xE0 && static_cast(source[i + 1]) < 0xA0) || + (lead == 0xED && static_cast(source[i + 1]) >= 0xA0))) || + (length == 4 && ((lead == 0xF0 && static_cast(source[i + 1]) < 0x90) || + (lead == 0xF4 && static_cast(source[i + 1]) >= 0x90)))) + return false; + destination.push_back(rune); + i += length; + } + return true; +} + +static std::vector load_lines(char const *path, std::size_t limit = 0) { + std::ifstream input(path); + if (!input) throw std::runtime_error(std::string("failed to open ") + path); + std::vector lines; + std::string encoded; + while ((!limit || lines.size() != limit) && std::getline(input, encoded)) { + if (!encoded.empty() && encoded.back() == '\r') encoded.pop_back(); + std::u32string decoded; + if (!decode_utf8(encoded, decoded)) throw std::runtime_error(std::string("invalid UTF-8 in ") + path); + lines.push_back(std::move(decoded)); + } + return lines; +} + +static bool dump_matches(std::vector const &dictionary, + std::vector const &queries, std::uint8_t bound, std::string const &path) { + std::ofstream output(path, std::ios::binary); + if (!output) return false; + char const magic[8] = {'S', 'Z', 'L', 'E', 'V', '0', '0', '1'}; + std::uint64_t const dictionary_size = dictionary.size(), queries_size = queries.size(); + output.write(magic, sizeof(magic)); + output.write(reinterpret_cast(&dictionary_size), sizeof(dictionary_size)); + output.write(reinterpret_cast(&queries_size), sizeof(queries_size)); + output.write(reinterpret_cast(&bound), sizeof(bound)); + + std::vector matches; + for (auto const &query : queries) { + matches.clear(); + rapidfuzz::CachedLevenshtein scorer(query); + for (std::uint32_t id = 0; id != dictionary.size(); ++id) { + std::size_t const distance = scorer.distance(dictionary[id], bound); + if (distance <= bound) matches.push_back({id, static_cast(distance)}); + } + std::uint64_t const matches_size = matches.size(); + output.write(reinterpret_cast(&matches_size), sizeof(matches_size)); + for (auto const &match : matches) { + output.write(reinterpret_cast(&match.id), sizeof(match.id)); + output.write(reinterpret_cast(&match.distance), sizeof(match.distance)); + } + } + return output.good(); +} + +int main(int argc, char **argv) { + if (argc < 3 || argc > 5) { + std::cerr << "usage: rapidfuzz_levenshtein_index_utf8 DICTIONARY QUERIES [QUERY_LIMIT] [DUMP_PREFIX]\n"; + return 2; + } + std::size_t const query_limit = argc >= 4 ? std::stoull(argv[3]) : 0; + std::string const dump_prefix = argc == 5 ? argv[4] : ""; + auto const dictionary = load_lines(argv[1]); + auto const queries = load_lines(argv[2], query_limit); + int const repeats = std::getenv("RF_REPEATS") ? std::stoi(std::getenv("RF_REPEATS")) : 3; + int const max_distance = std::getenv("RF_MAX_DISTANCE") ? std::stoi(std::getenv("RF_MAX_DISTANCE")) : 4; + if (max_distance < 1 || max_distance > 4) return 2; + std::cout << "dictionary=" << dictionary.size() << " queries=" << queries.size() + << " semantics=utf8-codepoints\n"; + + for (std::uint8_t bound = 1; bound <= max_distance; ++bound) { + for (int repeat = 0; repeat != repeats; ++repeat) { + std::size_t matches_count = 0; + auto const start = std::chrono::steady_clock::now(); + for (auto const &query : queries) { + rapidfuzz::CachedLevenshtein scorer(query); + for (auto const &candidate : dictionary) + if (scorer.distance(candidate, bound) <= bound) ++matches_count; + } + double const elapsed = + std::chrono::duration(std::chrono::steady_clock::now() - start).count(); + std::cout << "k=" << unsigned(bound) << " query=" << elapsed << "s matches=" << matches_count << '\n'; + } + if (!dump_prefix.empty() && !dump_matches(dictionary, queries, bound, + dump_prefix + ".k" + std::to_string(bound) + ".bin")) + return 3; + } +} From 6032a152d994a646b4bfd36c2e2ea961f28fd3bc Mon Sep 17 00:00:00 2001 From: Guillaume de Rouville Date: Sun, 16 Aug 2026 17:22:49 -0700 Subject: [PATCH 09/17] Improve: Cover each Levenshtein edit shape The first mixed workload changed sampled words mostly through substitutions. That was deterministic, but it did not exercise the insertion and deletion behavior that a Levenshtein index is designed to handle. Generate exact queries, each one-edit operation, several two-edit combinations, adjacent swaps, and a five-symbol extension in equal proportions. A swap still costs two plain Levenshtein edits. The mode name describes only how the query was made from its sampled source. It makes no claim about distance from other dictionary entries. The complete-result oracle remains responsible for the answer. Use the same selection and mutation rules for bytes and decoded UTF-8 codepoints. Skip source words that cannot support a requested operation instead of silently turning it into a different edit. Signed-off-by: Guillaume de Rouville --- levenshtein/queries.cpp | 177 +++++++++++++++++++++++++++------------- 1 file changed, 121 insertions(+), 56 deletions(-) diff --git a/levenshtein/queries.cpp b/levenshtein/queries.cpp index ea19fed..e6163af 100644 --- a/levenshtein/queries.cpp +++ b/levenshtein/queries.cpp @@ -1,13 +1,14 @@ /** * @brief Deterministic query generator for immutable-dictionary Levenshtein benchmarks. * - * Generates exact, one-edit, two-edit, five-symbol length-extended, or mixed queries by sampling lines from an - * existing dictionary. The extension is guaranteed beyond four edits from its sampled source, not necessarily from - * every other dictionary entry. Mutations use bytes by default or validated Unicode codepoints when - * `SZ_LEVENSHTEIN_UTF8` is set. + * Samples dictionary entries and applies a named edit operation. The mixed workload gives equal weight to exact + * queries, substitutions, insertions, deletions, adjacent swaps, and several two-edit combinations. Labels describe + * how a query was made from its source word; the result oracle still decides which dictionary entries match. + * Mutations use bytes by default or validated Unicode codepoints when `SZ_LEVENSHTEIN_UTF8` is set. */ #include +#include #include #include #include @@ -35,16 +36,6 @@ static std::uint64_t splitmix64(std::uint64_t &state) { return value ^ (value >> 31); } -static std::size_t substitute(std::string &query, std::uint64_t random) { - if (query.empty()) { - query.push_back('~'); - return 0; - } - std::size_t const position = random % query.size(); - query[position] = query[position] == '~' ? '^' : '~'; - return position; -} - static bool decode_utf8(std::string const &encoded, std::vector &decoded) { decoded.clear(); char const *position = encoded.data(), *end = position + encoded.size(); @@ -70,19 +61,92 @@ static std::string encode_utf8(std::vector const &decoded) { return encoded; } -static std::size_t substitute(std::vector &query, std::uint64_t random) { - if (query.empty()) { - query.push_back('~'); - return 0; - } +template +static std::size_t substitute(sequence_type_ &query, std::uint64_t random) { std::size_t const position = random % query.size(); query[position] = query[position] == '~' ? '^' : '~'; return position; } +static bool known_mode(std::string_view mode) { + return mode == "exact" || mode == "substitute1" || mode == "insert1" || mode == "delete1" || + mode == "substitute2" || mode == "insert_delete" || mode == "insert2" || mode == "delete2" || + mode == "transpose" || mode == "source_plus_5" || mode == "mixed"; +} + +template +static bool supports_mode(sequence_type_ const &query, std::string_view mode) { + if ((mode == "substitute1" || mode == "delete1" || mode == "insert_delete") && query.empty()) return false; + if ((mode == "substitute2" || mode == "delete2") && query.size() < 2) return false; + if (mode == "transpose") { + for (std::size_t index = 1; index != query.size(); ++index) + if (query[index - 1] != query[index]) return true; + return false; + } + return true; +} + +template +static void mutate(sequence_type_ &query, std::string_view mode, std::uint64_t &random_state) { + using symbol_t = typename sequence_type_::value_type; + auto const marker = [](symbol_t symbol) { return symbol == symbol_t('~') ? symbol_t('^') : symbol_t('~'); }; + auto const insert_one = [&] { + std::size_t const position = splitmix64(random_state) % (query.size() + 1); + symbol_t const inserted = position < query.size() ? marker(query[position]) : symbol_t('~'); + query.insert(query.begin() + position, inserted); + }; + auto const delete_one = [&] { + std::size_t const position = splitmix64(random_state) % query.size(); + query.erase(query.begin() + position); + }; + + if (mode == "exact") return; + if (mode == "substitute1") { + substitute(query, splitmix64(random_state)); + return; + } + if (mode == "insert1") { + insert_one(); + return; + } + if (mode == "delete1") { + delete_one(); + return; + } + if (mode == "substitute2") { + std::size_t const first = substitute(query, splitmix64(random_state)); + std::size_t second = splitmix64(random_state) % (query.size() - 1); + if (second >= first) ++second; + query[second] = marker(query[second]); + return; + } + if (mode == "insert_delete") { + delete_one(); + insert_one(); + return; + } + if (mode == "insert2") { + insert_one(); + insert_one(); + return; + } + if (mode == "delete2") { + delete_one(); + delete_one(); + return; + } + if (mode == "transpose") { + std::size_t position = splitmix64(random_state) % (query.size() - 1); + while (query[position] == query[position + 1]) position = (position + 1) % (query.size() - 1); + std::swap(query[position], query[position + 1]); + return; + } + query.insert(query.begin(), 5, symbol_t('~')); +} + int main(int argc, char **argv) { if (argc != 6) { - std::cerr << "usage: levenshtein_index_queries DICTIONARY OUTPUT COUNT exact|edit1|edit2|reject|mixed SEED\n"; + std::cerr << "usage: levenshtein_index_queries DICTIONARY OUTPUT COUNT MODE SEED\n"; return 2; } auto const dictionary = load_lines(argv[1]); @@ -97,57 +161,58 @@ int main(int argc, char **argv) { } std::size_t const count = std::stoull(argv[3]); std::string_view const requested_mode = argv[4]; + if (!known_mode(requested_mode)) { + std::cerr << "unknown mode: " << requested_mode << '\n'; + return 5; + } std::uint64_t random_state = std::stoull(argv[5]); bool const utf8 = std::getenv("SZ_LEVENSHTEIN_UTF8") != nullptr; for (std::size_t query_index = 0; query_index != count; ++query_index) { - std::uint64_t const sample_random = splitmix64(random_state); - std::string query = dictionary[sample_random % dictionary.size()]; std::string_view mode = requested_mode; if (mode == "mixed") { - static constexpr std::string_view modes[] = {"exact", "edit1", "edit2", "reject"}; - mode = modes[query_index % 4]; + static constexpr std::string_view modes[] = {"exact", "substitute1", "insert1", "delete1", + "substitute2", "insert_delete", "insert2", "delete2", + "transpose", "source_plus_5"}; + mode = modes[query_index % (sizeof(modes) / sizeof(modes[0]))]; } if (utf8) { std::vector decoded; - if (!decode_utf8(query, decoded)) { - std::cerr << "invalid UTF-8 dictionary entry\n"; - return 6; - } - if (mode == "edit1" || mode == "edit2") { - std::size_t const first_position = substitute(decoded, splitmix64(random_state)); - if (mode == "edit2") { - if (decoded.size() > 1) { - std::size_t second_position = splitmix64(random_state) % (decoded.size() - 1); - if (second_position >= first_position) ++second_position; - decoded[second_position] = decoded[second_position] == '~' ? '^' : '~'; - } - else - decoded.push_back('~'); + bool selected = false; + for (std::size_t attempt = 0; attempt != dictionary.size(); ++attempt) { + std::string const &source = dictionary[splitmix64(random_state) % dictionary.size()]; + if (!decode_utf8(source, decoded)) { + std::cerr << "invalid UTF-8 dictionary entry\n"; + return 6; } + if (supports_mode(decoded, mode)) { + selected = true; + break; + } + } + if (!selected) { + std::cerr << "dictionary has no entry suitable for mode " << mode << '\n'; + return 6; } - if (mode == "reject") decoded.insert(decoded.begin(), 5, '~'); - query = encode_utf8(decoded); + mutate(decoded, mode, random_state); + output << encode_utf8(decoded) << '\n'; } else { - if (mode == "edit1" || mode == "edit2") { - std::size_t const first_position = substitute(query, splitmix64(random_state)); - if (mode == "edit2") { - if (query.size() > 1) { - std::size_t second_position = splitmix64(random_state) % (query.size() - 1); - if (second_position >= first_position) ++second_position; - query[second_position] = query[second_position] == '~' ? '^' : '~'; - } - else - query.push_back('~'); + std::string query; + bool selected = false; + for (std::size_t attempt = 0; attempt != dictionary.size(); ++attempt) { + query = dictionary[splitmix64(random_state) % dictionary.size()]; + if (supports_mode(query, mode)) { + selected = true; + break; } } - if (mode == "reject") query.insert(0, "~~~~~"); // Beyond four edits from the sampled source word. - } - if (mode != "exact" && mode != "edit1" && mode != "edit2" && mode != "reject") { - std::cerr << "unknown mode: " << mode << '\n'; - return 5; + if (!selected) { + std::cerr << "dictionary has no entry suitable for mode " << mode << '\n'; + return 6; + } + mutate(query, mode, random_state); + output << query << '\n'; } - output << query << '\n'; } return output.good() ? 0 : 6; } From 6c4594dc5a25807d16704fac09685f726048cda9 Mon Sep 17 00:00:00 2001 From: Guillaume de Rouville Date: Sun, 16 Aug 2026 17:23:07 -0700 Subject: [PATCH 10/17] Improve: Separate Levenshtein timing modes The StringZilla runner used one untimed search to discover the exact result capacity, then timed the same batch with warm reader memory and perfect output sizing. That is a useful steady-state measurement, but it should not stand in for every query cost. Report four named modes instead: a fresh reader with sizing and retry, a warm pre-sized batch, a reusable growable service buffer, and single-query p50, p95, and p99 latency. Keep dictionary construction separate and print the work included by each line. A threshold-specialized run still builds the low-bound indexes independently. A shared run builds one index at the requested maximum and queries every smaller bound, which matches a deployed service that accepts different thresholds. Allow the exact StringZilla and RapidFuzz track to sweep through distance 254. The ecosystem adapters keep their smaller limits, so large-bound crossover results remain a separate complete-output comparison rather than being mixed into the low-bound table. Signed-off-by: Guillaume de Rouville --- levenshtein/stringzilla.cpp | 186 ++++++++++++++++++++++++++++++++++-- 1 file changed, 176 insertions(+), 10 deletions(-) diff --git a/levenshtein/stringzilla.cpp b/levenshtein/stringzilla.cpp index d11693d..36ae004 100644 --- a/levenshtein/stringzilla.cpp +++ b/levenshtein/stringzilla.cpp @@ -6,7 +6,9 @@ #include #include #include +#include #include +#include #include #include @@ -41,6 +43,24 @@ struct sparse_matches_t { } }; +static bool mode_enabled(std::string_view modes, std::string_view requested) { + while (!modes.empty()) { + std::size_t const separator = modes.find(','); + std::string_view const mode = modes.substr(0, separator); + if (mode == requested) return true; + if (separator == std::string_view::npos) break; + modes.remove_prefix(separator + 1); + } + return false; +} + +static double percentile(std::vector values, double fraction) { + if (values.empty()) return 0; + std::sort(values.begin(), values.end()); + std::size_t const rank = static_cast(fraction * (values.size() - 1)); + return values[rank]; +} + static bool dump_matches(std::size_t dictionary_size, std::size_t queries_size, std::uint8_t bound, sparse_matches_t const &matches, std::string const &path) { std::vector>> grouped(queries_size); @@ -106,7 +126,7 @@ template static int run(std::vector const &dictionary, std::vector const &queries, std::vector const &max_distances, std::string const &dump_prefix, int query_repeats, std::size_t query_threads, std::size_t batches_per_repeat, - std::size_t cache_evict_bytes) { + std::size_t cache_evict_bytes, std::string_view modes, bool shared_index) { std::vector dictionary_views = make_views(dictionary); std::vector query_views = make_views(queries); sz_sequence_t dictionary_sequence, query_sequence; @@ -135,8 +155,56 @@ static int run(std::vector const &dictionary, std::vector(std::chrono::steady_clock::now() - build_start).count(); std::cout << "k=" << unsigned(max_distance) << " build=" << build_seconds << "s\n"; - std::uint8_t const first_bound = max_distance <= 2 ? max_distance : 3; + std::uint8_t const first_bound = shared_index ? 1 : max_distance <= 2 ? max_distance : 3; for (std::uint8_t bound = first_bound; bound <= max_distance; ++bound) { + if (mode_enabled(modes, "cold")) { + for (int repeat = 0; repeat != query_repeats; ++repeat) { + void *cold_index = NULL; + auto const cold_build_start = std::chrono::steady_clock::now(); + if (sz_status_t status = index_init(&dictionary_sequence, max_distance, &cold_index, &error); + status != sz_success_k) { + std::cerr << "cold build failed: " << int(status) << " " << (error ? error : "") << '\n'; + index_free(index); + szs_device_scope_free(device); + return 5; + } + double const cold_build_seconds = + std::chrono::duration(std::chrono::steady_clock::now() - cold_build_start).count(); + for (std::size_t offset = 0; offset < cache_evict_buffer.size(); offset += 64) { + ++cache_evict_buffer[offset]; + cache_evict_checksum += cache_evict_buffer[offset]; + } + sparse_matches_t cold_matches; + auto const start = std::chrono::steady_clock::now(); + sz_size_t cold_required = 0; + sz_status_t status = index_find(cold_index, device, &query_sequence, bound, cold_matches, + &cold_required, &error); + if (status != sz_success_k && status != sz_unexpected_dimensions_k) { + std::cerr << "cold sizing failed: " << int(status) << " " << (error ? error : "") << '\n'; + index_free(cold_index); + index_free(index); + szs_device_scope_free(device); + return 5; + } + cold_matches.resize(cold_required); + sz_size_t cold_count = 0; + status = index_find(cold_index, device, &query_sequence, bound, cold_matches, &cold_count, + &error); + double const elapsed = + std::chrono::duration(std::chrono::steady_clock::now() - start).count(); + index_free(cold_index); + if (status != sz_success_k || cold_count != cold_required) { + std::cerr << "cold search failed: " << int(status) << " " << (error ? error : "") << '\n'; + index_free(index); + szs_device_scope_free(device); + return 5; + } + std::cout << "k=" << unsigned(bound) << " mode=cold_end_to_end repeat=" << repeat + << " query=" << elapsed << "s matches=" << cold_count + << " prep_build=" << cold_build_seconds << "s threads=" << query_threads << '\n'; + } + } + sparse_matches_t matches; sz_size_t required = 0; sz_status_t sizing_status = index_find(index, device, &query_sequence, bound, matches, @@ -148,8 +216,18 @@ static int run(std::vector const &dictionary, std::vector(index, device, &query_sequence, bound, matches, &materialized, + &error); + status != sz_success_k || materialized != required) { + std::cerr << "correctness materialization failed: " << int(status) << " " << (error ? error : "") + << '\n'; + index_free(index); + szs_device_scope_free(device); + return 6; + } - for (int repeat = 0; repeat != query_repeats; ++repeat) { + for (int repeat = 0; mode_enabled(modes, "warm") && repeat != query_repeats; ++repeat) { for (std::size_t offset = 0; offset < cache_evict_buffer.size(); offset += 64) { ++cache_evict_buffer[offset]; cache_evict_checksum += cache_evict_buffer[offset]; @@ -176,12 +254,90 @@ static int run(std::vector const &dictionary, std::vector(std::chrono::steady_clock::now() - start).count(); - std::cout << "k=" << unsigned(bound) << " query=" << elapsed / batches_per_repeat - << "s matches=" << matches_count / batches_per_repeat << " threads=" << query_threads - << " batches=" << batches_per_repeat << " cache_evict_bytes=" << cache_evict_bytes + std::cout << "k=" << unsigned(bound) << " mode=warm_presized repeat=" << repeat + << " query=" << elapsed / batches_per_repeat << "s matches=" + << matches_count / batches_per_repeat << " threads=" << query_threads << " batches=" + << batches_per_repeat << " cache_evict_bytes=" << cache_evict_bytes << " output_element_bytes=" << sizeof(sz_u64_t) + sizeof(sz_u32_t) + sizeof(sz_u8_t) << '\n'; } + + for (int repeat = 0; mode_enabled(modes, "steady") && repeat != query_repeats; ++repeat) { + for (std::size_t offset = 0; offset < cache_evict_buffer.size(); offset += 64) { + ++cache_evict_buffer[offset]; + cache_evict_checksum += cache_evict_buffer[offset]; + } + sparse_matches_t growable; + if (queries.size() > std::numeric_limits::max() / 8) { + std::cerr << "query count is too large for the starting output estimate\n"; + index_free(index); + szs_device_scope_free(device); + return 6; + } + auto const start = std::chrono::steady_clock::now(); + growable.resize(queries.size() * 8); + sz_size_t found = 0; + sz_status_t status = index_find(index, device, &query_sequence, bound, growable, &found, &error); + bool const retried = status == sz_unexpected_dimensions_k && found > growable.query_ids.size(); + if (retried) { + growable.resize(found); + status = index_find(index, device, &query_sequence, bound, growable, &found, &error); + } + double const elapsed = + std::chrono::duration(std::chrono::steady_clock::now() - start).count(); + if (status != sz_success_k || found != required) { + std::cerr << "steady search failed: " << int(status) << " " << (error ? error : "") << '\n'; + index_free(index); + szs_device_scope_free(device); + return 6; + } + std::cout << "k=" << unsigned(bound) << " mode=steady_growable repeat=" << repeat + << " query=" << elapsed << "s matches=" << found << " retried=" << retried + << " threads=" << query_threads << '\n'; + } + + if (mode_enabled(modes, "latency")) { + std::vector latencies; + latencies.reserve(queries.size()); + sparse_matches_t service_matches; + service_matches.resize(8); + std::size_t service_capacity = 8; + sz_size_t service_matches_count = 0; + for (std::size_t query_index = 0; query_index != queries.size(); ++query_index) { + sz_string_view_t one_view = query_views[query_index]; + sz_sequence_t one_query; + sz_sequence_from_string_views(&one_view, 1, &one_query); + auto const start = std::chrono::steady_clock::now(); + sz_size_t found = 0; + sz_status_t status = index_find(index, device, &one_query, bound, service_matches, &found, + &error); + if (status == sz_unexpected_dimensions_k && found > service_capacity) { + service_capacity = found; + service_matches.resize(service_capacity); + status = index_find(index, device, &one_query, bound, service_matches, &found, &error); + } + double const elapsed = + std::chrono::duration(std::chrono::steady_clock::now() - start).count(); + if (status != sz_success_k) { + std::cerr << "latency search failed: " << int(status) << " " << (error ? error : "") << '\n'; + index_free(index); + szs_device_scope_free(device); + return 6; + } + latencies.push_back(elapsed); + service_matches_count += found; + } + if (service_matches_count != required) { + std::cerr << "latency search returned a different result count\n"; + index_free(index); + szs_device_scope_free(device); + return 6; + } + std::cout << "k=" << unsigned(bound) << " mode=single_query_latency samples=" << latencies.size() + << " p50=" << percentile(latencies, 0.50) << "s p95=" << percentile(latencies, 0.95) + << "s p99=" << percentile(latencies, 0.99) << "s matches=" << service_matches_count + << " final_capacity=" << service_capacity << '\n'; + } if (!dump_prefix.empty()) { std::string const path = dump_prefix + ".k" + std::to_string(bound) + ".bin"; if (!dump_matches(dictionary.size(), queries.size(), bound, matches, path)) { @@ -209,6 +365,10 @@ int main(int argc, char **argv) { auto const dictionary = load_lines(argv[1]); auto const queries = load_lines(argv[2], query_limit); bool const utf8 = std::getenv("SZ_LEVENSHTEIN_UTF8") != nullptr; + std::string const modes = std::getenv("SZ_LEVENSHTEIN_MODES") ? std::getenv("SZ_LEVENSHTEIN_MODES") + : "warm,steady,latency"; + bool const shared_index = std::getenv("SZ_LEVENSHTEIN_INDEX_PLAN") && + std::string_view(std::getenv("SZ_LEVENSHTEIN_INDEX_PLAN")) == "shared"; std::cout << "dictionary=" << dictionary.size() << " queries=" << queries.size() << " semantics=" << (utf8 ? "utf8-codepoints" : "bytes") << '\n'; @@ -232,15 +392,21 @@ int main(int argc, char **argv) { } if (char const *requested_max = std::getenv("SZ_LEVENSHTEIN_MAX_DISTANCE")) { int const parsed = std::stoi(requested_max); - if (parsed != 1 && parsed != 2 && parsed != 4) { - std::cerr << "SZ_LEVENSHTEIN_MAX_DISTANCE must be 1, 2, or 4\n"; + if (parsed < 1 || parsed >= std::numeric_limits::max()) { + std::cerr << "SZ_LEVENSHTEIN_MAX_DISTANCE must be between 1 and 254\n"; return 2; } max_distances = {static_cast(parsed)}; } + if (shared_index && !std::getenv("SZ_LEVENSHTEIN_MAX_DISTANCE")) max_distances = {4}; + if (!mode_enabled(modes, "cold") && !mode_enabled(modes, "warm") && !mode_enabled(modes, "steady") && + !mode_enabled(modes, "latency")) { + std::cerr << "SZ_LEVENSHTEIN_MODES must include cold, warm, steady, or latency\n"; + return 2; + } std::size_t const cache_evict_bytes = cache_evict_mb * 1024 * 1024; return utf8 ? run(dictionary, queries, max_distances, dump_prefix, query_repeats, query_threads, - batches_per_repeat, cache_evict_bytes) + batches_per_repeat, cache_evict_bytes, modes, shared_index) : run(dictionary, queries, max_distances, dump_prefix, query_repeats, query_threads, - batches_per_repeat, cache_evict_bytes); + batches_per_repeat, cache_evict_bytes, modes, shared_index); } From 240be3f61f8a183bbcd4b22121d1b1752b504d45 Mon Sep 17 00:00:00 2001 From: Guillaume de Rouville Date: Sun, 16 Aug 2026 17:23:22 -0700 Subject: [PATCH 11/17] Improve: Separate RapidFuzz output work Counting matches and returning every query ID, dictionary ID, and distance are different amounts of work. Report them as separate RapidFuzz modes so a count-only result is never compared directly with StringZilla full output. The materialized mode keeps one result vector across repeats, computes the same checksum fields as the StringZilla runner, and continues to write the common binary artifact used for exact result comparison. Allow the byte and UTF-8 correctness runners through distance 254. This supports the separate large-bound crossover sweep without changing the smaller limits of tools whose own APIs stop earlier. Signed-off-by: Guillaume de Rouville --- levenshtein/rapidfuzz.cpp | 59 ++++++++++++++++++++++++++-------- levenshtein/rapidfuzz_utf8.cpp | 3 +- 2 files changed, 48 insertions(+), 14 deletions(-) diff --git a/levenshtein/rapidfuzz.cpp b/levenshtein/rapidfuzz.cpp index 1ae369f..a6bd8ed 100644 --- a/levenshtein/rapidfuzz.cpp +++ b/levenshtein/rapidfuzz.cpp @@ -16,10 +16,12 @@ #include #include #include +#include #include #include struct match_t { + std::uint64_t query_id; std::uint32_t id; std::uint8_t distance; }; @@ -54,7 +56,7 @@ static bool dump_matches(std::vector const &dictionary, std::vector rapidfuzz::CachedLevenshtein scorer(query); for (std::uint32_t id = 0; id != dictionary.size(); ++id) { std::size_t const distance = scorer.distance(dictionary[id], bound); - if (distance <= bound) matches.push_back({id, static_cast(distance)}); + if (distance <= bound) matches.push_back({0, id, static_cast(distance)}); } std::uint64_t const matches_size = matches.size(); output.write(reinterpret_cast(&matches_size), sizeof(matches_size)); @@ -77,24 +79,55 @@ int main(int argc, char **argv) { auto const queries = load_lines(argv[2], query_limit); int const repeats = std::getenv("RF_REPEATS") ? std::stoi(std::getenv("RF_REPEATS")) : 3; int const max_distance = std::getenv("RF_MAX_DISTANCE") ? std::stoi(std::getenv("RF_MAX_DISTANCE")) : 4; - if (max_distance < 1 || max_distance > 4) { - std::cerr << "RF_MAX_DISTANCE must be between 1 and 4\n"; + std::string const requested_mode = std::getenv("RF_MODE") ? std::getenv("RF_MODE") : "both"; + if (max_distance < 1 || max_distance >= std::numeric_limits::max()) { + std::cerr << "RF_MAX_DISTANCE must be between 1 and 254\n"; + return 2; + } + if (requested_mode != "count" && requested_mode != "materialized" && requested_mode != "both") { + std::cerr << "RF_MODE must be count, materialized, or both\n"; return 2; } std::cout << "dictionary=" << dictionary.size() << " queries=" << queries.size() << '\n'; for (std::uint8_t bound = 1; bound <= max_distance; ++bound) { - for (int repeat = 0; repeat != repeats; ++repeat) { - std::size_t matches_count = 0; - auto const start = std::chrono::steady_clock::now(); - for (auto const &query : queries) { - rapidfuzz::CachedLevenshtein scorer(query); - for (auto const &candidate : dictionary) - if (scorer.distance(candidate, bound) <= bound) ++matches_count; + if (requested_mode == "count" || requested_mode == "both") { + for (int repeat = 0; repeat != repeats; ++repeat) { + std::size_t matches_count = 0; + auto const start = std::chrono::steady_clock::now(); + for (auto const &query : queries) { + rapidfuzz::CachedLevenshtein scorer(query); + for (auto const &candidate : dictionary) + if (scorer.distance(candidate, bound) <= bound) ++matches_count; + } + double const elapsed = + std::chrono::duration(std::chrono::steady_clock::now() - start).count(); + std::cout << "k=" << unsigned(bound) << " mode=count repeat=" << repeat << " query=" << elapsed + << "s matches=" << matches_count << " output=count-only\n"; + } + } + if (requested_mode == "materialized" || requested_mode == "both") { + std::vector matches; + for (int repeat = 0; repeat != repeats; ++repeat) { + matches.clear(); + auto const start = std::chrono::steady_clock::now(); + for (std::uint64_t query_id = 0; query_id != queries.size(); ++query_id) { + rapidfuzz::CachedLevenshtein scorer(queries[query_id]); + for (std::uint32_t id = 0; id != dictionary.size(); ++id) { + std::size_t const distance = scorer.distance(dictionary[id], bound); + if (distance <= bound) + matches.push_back({query_id, id, static_cast(distance)}); + } + } + double const elapsed = + std::chrono::duration(std::chrono::steady_clock::now() - start).count(); + std::uint64_t checksum = 0; + for (match_t const &match : matches) + checksum += ((match.query_id * dictionary.size() + match.id) << 8) | match.distance; + std::cout << "k=" << unsigned(bound) << " mode=materialized repeat=" << repeat << " query=" + << elapsed << "s matches=" << matches.size() << " checksum=" << std::hex << checksum + << std::dec << " output=query-id+dictionary-id+distance\n"; } - double const elapsed = - std::chrono::duration(std::chrono::steady_clock::now() - start).count(); - std::cout << "k=" << unsigned(bound) << " query=" << elapsed << "s matches=" << matches_count << '\n'; } if (!dump_prefix.empty()) { std::string const path = dump_prefix + ".k" + std::to_string(bound) + ".bin"; diff --git a/levenshtein/rapidfuzz_utf8.cpp b/levenshtein/rapidfuzz_utf8.cpp index e05af1b..6e103d9 100644 --- a/levenshtein/rapidfuzz_utf8.cpp +++ b/levenshtein/rapidfuzz_utf8.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -106,7 +107,7 @@ int main(int argc, char **argv) { auto const queries = load_lines(argv[2], query_limit); int const repeats = std::getenv("RF_REPEATS") ? std::stoi(std::getenv("RF_REPEATS")) : 3; int const max_distance = std::getenv("RF_MAX_DISTANCE") ? std::stoi(std::getenv("RF_MAX_DISTANCE")) : 4; - if (max_distance < 1 || max_distance > 4) return 2; + if (max_distance < 1 || max_distance >= std::numeric_limits::max()) return 2; std::cout << "dictionary=" << dictionary.size() << " queries=" << queries.size() << " semantics=utf8-codepoints\n"; From 43cef57d7240f5f5c06c836ad640613f5b7d9846 Mon Sep 17 00:00:00 2001 From: Guillaume de Rouville Date: Sun, 16 Aug 2026 17:24:20 -0700 Subject: [PATCH 12/17] Fix: Reuse SymSpell verification memory SymSpell does not return the same contract as StringZilla. Its native suggestions may use different distance behavior, and the adapter still needs to recover every exact plain-Levenshtein ID before a direct timing comparison is valid. The first compatibility adapter decoded strings and allocated dynamic-programming rows while processing every suggestion. That measured avoidable harness work rather than the cost of the lookup. Decode the dictionary and queries once, then reuse the verifier rows and match buffer. Report native SymSpell output and exact compatibility output as separate modes. The exact mode remains restricted to unique lowercase dictionaries because those are real limits of this adapter. The benchmark target compiles with the pinned SymSpell revision, and its tiny complete-output artifact matches the StringZilla runner. Signed-off-by: Guillaume de Rouville --- levenshtein/bench.rs | 200 ++++++++++++++++++++++++++++--------------- 1 file changed, 129 insertions(+), 71 deletions(-) diff --git a/levenshtein/bench.rs b/levenshtein/bench.rs index d9b7a82..ff6b9b7 100644 --- a/levenshtein/bench.rs +++ b/levenshtein/bench.rs @@ -51,9 +51,7 @@ fn load_lines(path: &str, limit: usize) -> Result, AnyError> { } fn checksum_id(checksum: u64, id: u64) -> u64 { - checksum - .wrapping_mul(0x9E37_79B1_85EB_CA87) - .wrapping_add(id) + checksum.wrapping_mul(0x9E37_79B1_85EB_CA87).wrapping_add(id) } fn evict_cache(buffer: &mut [u8], checksum: &mut u64) { @@ -64,68 +62,96 @@ fn evict_cache(buffer: &mut [u8], checksum: &mut u64) { std::hint::black_box(*checksum); } -fn levenshtein_within(first: &str, second: &str, bound: usize) -> Option { - let first: Vec = first.chars().collect(); - let second: Vec = second.chars().collect(); +struct DistanceScratch { + previous: Vec, + current: Vec, +} + +fn levenshtein_within(first: &[char], second: &[char], bound: usize, scratch: &mut DistanceScratch) -> Option { if first.len().abs_diff(second.len()) > bound { return None; } - let mut previous: Vec = (0..=first.len()).collect(); - let mut current = vec![0; first.len() + 1]; + scratch.previous.resize(first.len() + 1, 0); + scratch.current.resize(first.len() + 1, 0); + for (column, value) in scratch.previous.iter_mut().enumerate() { + *value = column; + } for (row, &second_char) in second.iter().enumerate() { - current[0] = row + 1; - let mut row_min = current[0]; + scratch.current[0] = row + 1; + let mut row_min = scratch.current[0]; for (column, &first_char) in first.iter().enumerate() { - current[column + 1] = (previous[column + 1] + 1) - .min(current[column] + 1) - .min(previous[column] + usize::from(first_char != second_char)); - row_min = row_min.min(current[column + 1]); + scratch.current[column + 1] = (scratch.previous[column + 1] + 1) + .min(scratch.current[column] + 1) + .min(scratch.previous[column] + usize::from(first_char != second_char)); + row_min = row_min.min(scratch.current[column + 1]); } if row_min > bound { return None; } - std::mem::swap(&mut previous, &mut current); + std::mem::swap(&mut scratch.previous, &mut scratch.current); } - (previous[first.len()] <= bound).then_some(previous[first.len()]) + (scratch.previous[first.len()] <= bound).then_some(scratch.previous[first.len()]) } fn symspell_search( symspell: &SymSpell, ids: &HashMap, + decoded_dictionary: &[Vec], query: &str, + decoded_query: &[char], bound: usize, -) -> Vec { - symspell - .lookup(query, Verbosity::All, bound, &None, None, false) - .into_iter() - .filter_map(|suggestion| { - levenshtein_within(query, &suggestion.term, bound).map(|distance| Match { - id: ids[&suggestion.term], + distance_scratch: &mut DistanceScratch, + matches: &mut Vec, +) { + matches.clear(); + for suggestion in symspell.lookup(query, Verbosity::All, bound, &None, None, false) { + let id = ids[&suggestion.term]; + if let Some(distance) = + levenshtein_within(decoded_query, &decoded_dictionary[id as usize], bound, distance_scratch) + { + matches.push(Match { + id, distance: distance as u8, - }) - }) - .collect() + }); + } + } } fn dump_symspell_matches( symspell: &SymSpell, ids: &HashMap, + decoded_dictionary: &[Vec], + decoded_queries: &[Vec], dictionary_size: usize, queries: &[String], bound: usize, path: &str, ) -> Result<(), AnyError> { let mut output = BufWriter::new(File::create(path)?); + let mut distance_scratch = DistanceScratch { + previous: Vec::new(), + current: Vec::new(), + }; + let mut matches = Vec::new(); output.write_all(b"SZLEV001")?; output.write_all(&(dictionary_size as u64).to_ne_bytes())?; output.write_all(&(queries.len() as u64).to_ne_bytes())?; output.write_all(&[bound as u8])?; - for query in queries { - let mut matches = symspell_search(symspell, ids, query, bound); + for (query, decoded_query) in queries.iter().zip(decoded_queries) { + symspell_search( + symspell, + ids, + decoded_dictionary, + query, + decoded_query, + bound, + &mut distance_scratch, + &mut matches, + ); matches.sort_unstable_by_key(|found| (found.id, found.distance)); output.write_all(&(matches.len() as u64).to_ne_bytes())?; - for found in matches { + for found in &matches { output.write_all(&found.id.to_ne_bytes())?; output.write_all(&[found.distance])?; } @@ -133,11 +159,7 @@ fn dump_symspell_matches( Ok(()) } -fn run_symspell( - dictionary: &[String], - queries: &[String], - settings: &Settings, -) -> Result<(), AnyError> { +fn run_symspell(dictionary: &[String], queries: &[String], settings: &Settings) -> Result<(), AnyError> { if dictionary .iter() .chain(queries) @@ -146,16 +168,24 @@ fn run_symspell( return Err("SymSpell requires lowercase input for case-sensitive comparison".into()); } + let setup_start = Instant::now(); let mut ids = HashMap::with_capacity(dictionary.len()); for (id, word) in dictionary.iter().enumerate() { if ids.insert(word.clone(), id as u32).is_some() { return Err("SymSpell cannot preserve duplicate dictionary entries".into()); } } + let decoded_dictionary: Vec> = dictionary.iter().map(|word| word.chars().collect()).collect(); + let decoded_queries: Vec> = queries.iter().map(|query| query.chars().collect()).collect(); + let setup_seconds = setup_start.elapsed().as_secs_f64(); + let requested_mode = env::var("STRINGWARS_SYMSPELL_MODE").unwrap_or_else(|_| "both".to_owned()); + if requested_mode != "raw" && requested_mode != "exact" && requested_mode != "both" { + return Err("STRINGWARS_SYMSPELL_MODE must be raw, exact, or both".into()); + } println!("# levenshtein/symspell"); println!( - "dictionary={} queries={} semantics=utf8-codepoints+exact-levenshtein-filter output=id+distance", + "dictionary={} queries={} adapter_setup={setup_seconds:.6}s", dictionary.len(), queries.len() ); @@ -174,31 +204,72 @@ fn run_symspell( symspell.get_dictionary_size() ); - for repeat in 0..settings.repeats { - evict_cache(&mut cache, &mut cache_checksum); - let start = Instant::now(); - let mut matches_count = 0usize; - let mut checksum = 0u64; - for query in queries { - for found in symspell_search(&symspell, &ids, query, bound) { - matches_count += 1; - checksum = checksum.wrapping_add( - ((found.id as u64) << 8 | found.distance as u64) - .wrapping_mul(0x9E37_79B1_85EB_CA87), + if requested_mode == "raw" || requested_mode == "both" { + for repeat in 0..settings.repeats { + evict_cache(&mut cache, &mut cache_checksum); + let start = Instant::now(); + let mut matches_count = 0usize; + let mut checksum = 0u64; + for query in queries { + for suggestion in symspell.lookup(query, Verbosity::All, bound, &None, None, false) { + matches_count += 1; + checksum = checksum.wrapping_add( + ((ids[&suggestion.term] as u64) << 8 | suggestion.distance as u64) + .wrapping_mul(0x9E37_79B1_85EB_CA87), + ); + } + } + println!( + "k={bound} mode=native repeat={repeat} query={:.6}s matches={matches_count} checksum={checksum:016x} cache_evict_bytes={} semantics=symspell output=id+native-distance", + start.elapsed().as_secs_f64(), + cache.len() + ); + } + } + + if requested_mode == "exact" || requested_mode == "both" { + for repeat in 0..settings.repeats { + evict_cache(&mut cache, &mut cache_checksum); + let start = Instant::now(); + let mut matches_count = 0usize; + let mut checksum = 0u64; + let mut distance_scratch = DistanceScratch { + previous: Vec::new(), + current: Vec::new(), + }; + let mut matches = Vec::new(); + for (query, decoded_query) in queries.iter().zip(&decoded_queries) { + symspell_search( + &symspell, + &ids, + &decoded_dictionary, + query, + decoded_query, + bound, + &mut distance_scratch, + &mut matches, ); + for found in &matches { + matches_count += 1; + checksum = checksum.wrapping_add( + ((found.id as u64) << 8 | found.distance as u64).wrapping_mul(0x9E37_79B1_85EB_CA87), + ); + } } + println!( + "k={bound} mode=exact_compatibility repeat={repeat} query={:.6}s matches={matches_count} checksum={checksum:016x} cache_evict_bytes={} semantics=unicode-codepoints output=id+exact-distance", + start.elapsed().as_secs_f64(), + cache.len() + ); } - println!( - "k={bound} repeat={repeat} query={:.6}s matches={matches_count} checksum={checksum:016x} cache_evict_bytes={}", - start.elapsed().as_secs_f64(), - cache.len() - ); } if let Some(prefix) = &settings.dump_prefix { dump_symspell_matches( &symspell, &ids, + &decoded_dictionary, + &decoded_queries, dictionary.len(), queries, bound, @@ -222,10 +293,9 @@ fn verify_fst_unicode_contract() -> Result<(), AnyError> { } } if matches != 9 { - return Err(format!( - "fst Unicode smoke test failed: expected 9 one-character matches, observed {matches}" - ) - .into()); + return Err( + format!("fst Unicode smoke test failed: expected 9 one-character matches, observed {matches}").into(), + ); } Ok(()) } @@ -234,11 +304,7 @@ fn run_fst(dictionary: &[String], queries: &[String], settings: &Settings) -> Re let allow_unicode = env::var_os("STRINGWARS_ALLOW_FST_UNICODE").is_some(); if allow_unicode { verify_fst_unicode_contract()?; - } else if dictionary - .iter() - .chain(queries) - .any(|text| !text.is_ascii()) - { + } else if dictionary.iter().chain(queries).any(|text| !text.is_ascii()) { return Err("fst requires ASCII input for byte-for-byte semantic parity".into()); } @@ -313,11 +379,7 @@ fn run_fst(dictionary: &[String], queries: &[String], settings: &Settings) -> Re Ok(()) } -fn run_tantivy( - dictionary: &[String], - queries: &[String], - settings: &Settings, -) -> Result<(), AnyError> { +fn run_tantivy(dictionary: &[String], queries: &[String], settings: &Settings) -> Result<(), AnyError> { let mut schema_builder = Schema::builder(); let word_field = schema_builder.add_text_field("word", STRING); let index = Index::create_in_ram(schema_builder.build()); @@ -382,9 +444,7 @@ fn run_tantivy( } fn main() -> Result<(), AnyError> { - let args: Vec = env::args() - .filter(|argument| argument != "--bench") - .collect(); + let args: Vec = env::args().filter(|argument| argument != "--bench").collect(); if args.len() < 3 || args.len() > 4 { eprintln!("usage: bench_levenshtein DICTIONARY QUERIES [QUERY_LIMIT]"); std::process::exit(2); @@ -408,9 +468,7 @@ fn main() -> Result<(), AnyError> { || settings.min_distance > settings.max_distance || settings.max_distance > 4 { - return Err( - "expected positive repeats and 1 <= minimum distance <= maximum distance <= 4".into(), - ); + return Err("expected positive repeats and 1 <= minimum distance <= maximum distance <= 4".into()); } if should_run("levenshtein/fst") { From 56e54891bcc7d376469953c7bd6f2c5cccdf20e1 Mon Sep 17 00:00:00 2001 From: Guillaume de Rouville Date: Sun, 16 Aug 2026 17:24:35 -0700 Subject: [PATCH 13/17] Docs: Explain repeated Levenshtein benchmarks Repeated dictionary retrieval is not the same workload as the existing dense distance matrices, so give it one focused page and link that page from the root benchmark catalogue. Document the exact output contract before listing tools. StringZilla, RapidFuzz, and the exact SymSpell mode can be compared only when they return the same original IDs and distances. Native suggestions, unique terms, and counts remain useful context but do not support a direct speedup claim. Describe the generated edit shapes, cold and warm timing modes, threshold-specialized and shared indexes, required machine metadata, and the common binary result check. Keep the larger-bound crossover separate because several ecosystem tools stop at small distances. Results and raw machine logs are intentionally absent. They belong in the draft PR and archived run artifacts after the protocol passes, not as permanent numbers in the benchmark source tree. Signed-off-by: Guillaume de Rouville --- README.md | 6 +++ levenshtein/README.md | 108 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 114 insertions(+) create mode 100644 levenshtein/README.md diff --git a/README.md b/README.md index 8c35408..00fe26d 100644 --- a/README.md +++ b/README.md @@ -255,6 +255,12 @@ stringzilla ██████ See [similarities/README.md](similarities/README.md) for details +### Repeated Levenshtein Dictionary Search + +Repeated fuzzy search has a different shape from a distance matrix: one dictionary is built once, then searched many times. The benchmark separates full-output comparisons from tools that return only unique terms, IDs, counts, or native suggestion distances. + +See [levenshtein/README.md](levenshtein/README.md) for the workloads, timing modes, correctness checks, and reproduction commands. + ### Fingerprinting Converting variable-length strings into fixed-length sketches (like Min-Hashing) enables fast approximate matching in large-scale retrieval. diff --git a/levenshtein/README.md b/levenshtein/README.md new file mode 100644 index 0000000..ddb2e72 --- /dev/null +++ b/levenshtein/README.md @@ -0,0 +1,108 @@ +# Repeated Levenshtein Dictionary Search + +This benchmark covers the case where a dictionary is built once and searched many times. It is separate from the dense distance matrices in [`similarities/`](../similarities/). + +Each exact result contains the query ID, the original dictionary ID, and the plain Levenshtein distance. Duplicate dictionary entries keep separate IDs. An adjacent swap counts as two edits. + +## Comparisons + +Direct timing comparisons require the same complete output: + +* StringZilla returns every original ID and exact distance. +* RapidFuzz scans the dictionary and materializes the same result. It is also the correctness oracle. +* SymSpell has an exact compatibility mode that checks its suggestions again with allocation-free plain Levenshtein distance. It is comparable only on unique lowercase dictionaries. + +The native SymSpell, Rust `fst`, Tantivy, and Lucene modes return different information. Their results provide useful ecosystem context, but they are not used for direct speedup claims. + +Every comparable runner writes the same binary result format. The files must match before timings are reported. + +## Queries + +`queries.cpp` creates deterministic queries from an existing dictionary. The mixed workload gives equal weight to: + +* exact queries; +* one substitution, insertion, or deletion; +* two substitutions, insertions, or deletions; +* one insertion plus one deletion; +* one adjacent swap; +* a five-symbol extension of the sampled source word. + +These labels describe how the query was created from one source word. They do not assume that the query has no other dictionary matches. The result oracle decides the complete answer. + +Final runs should include short English words, longer URLs, DNA strings, valid non-ASCII text, and a duplicate-heavy synthetic dictionary. Each run records the dictionary and query hashes. + +## Timing + +StringZilla reports four separate measurements: + +* `cold_end_to_end` starts with a fresh index reader and includes output sizing, allocation, retry, and materialization. Dictionary construction is reported separately. +* `warm_presized` measures repeated batches after scratch and exact output capacity are available. +* `steady_growable` starts from eight output slots per query and includes a resize and retry when needed. +* `single_query_latency` reports p50, p95, and p99 for one-query calls with a reusable grow-only output buffer. + +Threshold-specialized runs build separate indexes for `k=1`, `k=2`, and `k=4`. Shared-index runs build once for `k=4` and query the same index at every bound from one through four. + +The same complete-output comparison can sweep larger bounds by setting both runners to the same maximum. This is kept separate from the low-bound ecosystem table because several indexed tools only support small edit distances. It checks that StringZilla changes its internal search path without changing results or developing a performance cliff. + +Published runs use at least 20 measured repetitions, keep raw output, randomize runner order, pin CPU and memory placement, and record compiler versions, dependency revisions, CPU frequency settings, result counts, output bytes, build time, retained index size, peak build memory, and reader scratch. Warm and cold results are never combined into one number. + +## Reproducing the correctness check + +Build StringZilla first, then compile the query generator and the two complete-output runners from the StringWars root. Pin the RapidFuzz revision used by the final run. + +```bash +cmake -S ../StringZilla -B ../StringZilla/build -DCMAKE_BUILD_TYPE=Release +cmake --build ../StringZilla/build -j --target stringzillas_cpus_static + +g++ -std=c++20 -O3 -DNDEBUG -I ../StringZilla/include \ + levenshtein/queries.cpp -o levenshtein_queries + +g++ -std=c++20 -O3 -DNDEBUG -march=native -DSZ_DYNAMIC_DISPATCH=1 \ + -I ../StringZilla/include -I ../StringZilla/forkunion/include \ + levenshtein/stringzilla.cpp ../StringZilla/build/libstringzillas_cpus_static.a \ + ../StringZilla/build/forkunion/libforkunion_static.a -pthread -o stringzilla_levenshtein + +g++ -std=c++20 -O3 -DNDEBUG -march=native -I ../rapidfuzz-cpp \ + levenshtein/rapidfuzz.cpp -o rapidfuzz_levenshtein + +./levenshtein_queries words_alpha.txt queries.txt 10000 mixed 243 + +SZ_LEVENSHTEIN_MAX_DISTANCE=1 SZ_LEVENSHTEIN_REPEATS=20 \ + SZ_LEVENSHTEIN_MODES=warm,steady,latency \ + ./stringzilla_levenshtein words_alpha.txt queries.txt 10000 stringzilla-results + +SZ_LEVENSHTEIN_MAX_DISTANCE=2 SZ_LEVENSHTEIN_REPEATS=20 \ + SZ_LEVENSHTEIN_MODES=warm,steady,latency \ + ./stringzilla_levenshtein words_alpha.txt queries.txt 10000 stringzilla-results + +RF_MAX_DISTANCE=2 RF_REPEATS=20 RF_MODE=materialized \ + ./rapidfuzz_levenshtein words_alpha.txt queries.txt 10000 rapidfuzz-results + +cmp stringzilla-results.k1.bin rapidfuzz-results.k1.bin +cmp stringzilla-results.k2.bin rapidfuzz-results.k2.bin +``` + +To check the larger-bound crossover with one index, run: + +```bash +SZ_LEVENSHTEIN_INDEX_PLAN=shared SZ_LEVENSHTEIN_MAX_DISTANCE=10 \ + SZ_LEVENSHTEIN_REPEATS=20 SZ_LEVENSHTEIN_MODES=warm,steady \ + ./stringzilla_levenshtein words_alpha.txt queries.txt 10000 stringzilla-wide + +RF_MAX_DISTANCE=10 RF_REPEATS=20 RF_MODE=materialized \ + ./rapidfuzz_levenshtein words_alpha.txt queries.txt 10000 rapidfuzz-wide + +for k in 1 2 3 4 5 6 7 8 9 10; do + cmp "stringzilla-wide.k${k}.bin" "rapidfuzz-wide.k${k}.bin" +done +``` + +Run the Rust adapters through the normal benchmark target: + +```bash +RUSTFLAGS="-C target-cpu=native" STRINGWARS_REPEATS=20 \ + cargo bench --features bench_levenshtein --bench bench_levenshtein -- \ + words_alpha.txt queries.txt 10000 +``` + +The final result tables and raw run artifacts are added only after this protocol passes on every reported machine. From 7f4261b61a507f2d85d5e569f16da9f12bbb06c6 Mon Sep 17 00:00:00 2001 From: Guillaume de Rouville Date: Mon, 17 Aug 2026 14:56:03 -0700 Subject: [PATCH 14/17] Fix: Stabilize Tantivy result checksums Tantivy returns the same document set on every repeat, but DocSetCollector does not promise one iteration order. Feeding those addresses into an order-sensitive checksum made identical results print different checksums and weakened the benchmark evidence. Combine document addresses with a commutative checksum instead. Match counts and timed work are unchanged, while five repeated runs now report one checksum at both k=1 and k=2. Signed-off-by: Guillaume de Rouville --- levenshtein/bench.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/levenshtein/bench.rs b/levenshtein/bench.rs index ff6b9b7..0b3db40 100644 --- a/levenshtein/bench.rs +++ b/levenshtein/bench.rs @@ -426,7 +426,9 @@ fn run_tantivy(dictionary: &[String], queries: &[String], settings: &Settings) - matches_count += matches.len(); for address in matches { let id = (u64::from(address.segment_ord) << 32) | u64::from(address.doc_id); - checksum = checksum_id(checksum, id); + // `DocSetCollector` does not promise iteration order. Hash the set commutatively so + // identical matches produce one reproducible checksum across repeats. + checksum = checksum.wrapping_add(id.wrapping_mul(0x9E37_79B1_85EB_CA87)); } } if let Some(error) = failed { From 6144596c3fe90aca84df58e2f237e72d67e95231 Mon Sep 17 00:00:00 2001 From: Guillaume de Rouville Date: Mon, 17 Aug 2026 15:05:13 -0700 Subject: [PATCH 15/17] Improve: Configure Lucene benchmark repetitions The Lucene runner always measured three repetitions while the benchmark protocol requires at least twenty for final evidence. That made it too easy to publish a Lucene number collected under a different method from the other runners. Read the shared STRINGWARS_REPEATS setting, keep three as the convenient local default, and reject invalid values. This also let the current comparison use five repetitions consistently across every available adapter. Signed-off-by: Guillaume de Rouville --- .../java/com/stringzilla/LevenshteinIndexBenchmark.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/levenshtein/lucene/src/main/java/com/stringzilla/LevenshteinIndexBenchmark.java b/levenshtein/lucene/src/main/java/com/stringzilla/LevenshteinIndexBenchmark.java index 9f0c627..0b6eff2 100644 --- a/levenshtein/lucene/src/main/java/com/stringzilla/LevenshteinIndexBenchmark.java +++ b/levenshtein/lucene/src/main/java/com/stringzilla/LevenshteinIndexBenchmark.java @@ -47,6 +47,12 @@ public static void main(String[] args) throws Exception { System.err.printf("MAX_DISTANCE must be between 1 and %d%n", FuzzyQuery.defaultMaxEdits); System.exit(2); } + String repeatsVariable = System.getenv("STRINGWARS_REPEATS"); + int repeats = repeatsVariable == null ? 3 : Integer.parseInt(repeatsVariable); + if (repeats < 1) { + System.err.println("STRINGWARS_REPEATS must be positive"); + System.exit(2); + } long buildStart = System.nanoTime(); ByteBuffersDirectory directory = new ByteBuffersDirectory(); @@ -65,7 +71,7 @@ public static void main(String[] args) throws Exception { System.out.printf("dictionary=%d queries=%d build=%.6fs%n", dictionary.size(), queries.size(), buildSeconds); for (int bound = 1; bound <= maxDistance; ++bound) { - for (int repeat = 0; repeat != 3; ++repeat) { + for (int repeat = 0; repeat != repeats; ++repeat) { long matches = 0; long start = System.nanoTime(); for (String query : queries) { From 847035fc66045e82e5b6add16a0e883d6723ec23 Mon Sep 17 00:00:00 2001 From: Guillaume de Rouville Date: Mon, 17 Aug 2026 17:03:11 -0700 Subject: [PATCH 16/17] Add: Reproducible Levenshtein comparison runs The individual adapters already print the work they perform, but reproducing the full comparison still required a long sequence of manual build, query generation, validation, and timing commands. That made it easy to omit one adapter, reuse a favorable order, or publish timings before checking the complete answers. Add one focused driver for the shared k=1 and k=2 track. It builds the existing C++, Rust, and Lucene runners, generates the mixed workload, and refuses to time invalid results. StringZilla, RapidFuzz, and exact SymSpell must produce byte-identical artifacts. FST, Tantivy, and Lucene must return the same match totals. Each measured repetition starts fresh processes in a seeded shuffled order. The driver preserves every runner log and writes a small manifest with input hashes, repository and dependency revisions, commands, environment settings, wall time, and peak process memory. Host-specific controls and published machine logs remain outside the source tree. The six-adapter validation passed on the included small dictionary, and a fresh source build reached the same checks without prepared binaries. Signed-off-by: Guillaume de Rouville --- levenshtein/README.md | 62 ++---- levenshtein/run.py | 444 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 455 insertions(+), 51 deletions(-) create mode 100644 levenshtein/run.py diff --git a/levenshtein/README.md b/levenshtein/README.md index ddb2e72..402c5dc 100644 --- a/levenshtein/README.md +++ b/levenshtein/README.md @@ -46,63 +46,23 @@ The same complete-output comparison can sweep larger bounds by setting both runn Published runs use at least 20 measured repetitions, keep raw output, randomize runner order, pin CPU and memory placement, and record compiler versions, dependency revisions, CPU frequency settings, result counts, output bytes, build time, retained index size, peak build memory, and reader scratch. Warm and cold results are never combined into one number. -## Reproducing the correctness check +## Running the comparison -Build StringZilla first, then compile the query generator and the two complete-output runners from the StringWars root. Pin the RapidFuzz revision used by the final run. +The shared low-bound track expects a unique lowercase ASCII dictionary because that is the common contract supported by all six adapters. From the StringWars root: ```bash -cmake -S ../StringZilla -B ../StringZilla/build -DCMAKE_BUILD_TYPE=Release -cmake --build ../StringZilla/build -j --target stringzillas_cpus_static - -g++ -std=c++20 -O3 -DNDEBUG -I ../StringZilla/include \ - levenshtein/queries.cpp -o levenshtein_queries - -g++ -std=c++20 -O3 -DNDEBUG -march=native -DSZ_DYNAMIC_DISPATCH=1 \ - -I ../StringZilla/include -I ../StringZilla/forkunion/include \ - levenshtein/stringzilla.cpp ../StringZilla/build/libstringzillas_cpus_static.a \ - ../StringZilla/build/forkunion/libforkunion_static.a -pthread -o stringzilla_levenshtein - -g++ -std=c++20 -O3 -DNDEBUG -march=native -I ../rapidfuzz-cpp \ - levenshtein/rapidfuzz.cpp -o rapidfuzz_levenshtein - -./levenshtein_queries words_alpha.txt queries.txt 10000 mixed 243 - -SZ_LEVENSHTEIN_MAX_DISTANCE=1 SZ_LEVENSHTEIN_REPEATS=20 \ - SZ_LEVENSHTEIN_MODES=warm,steady,latency \ - ./stringzilla_levenshtein words_alpha.txt queries.txt 10000 stringzilla-results - -SZ_LEVENSHTEIN_MAX_DISTANCE=2 SZ_LEVENSHTEIN_REPEATS=20 \ - SZ_LEVENSHTEIN_MODES=warm,steady,latency \ - ./stringzilla_levenshtein words_alpha.txt queries.txt 10000 stringzilla-results - -RF_MAX_DISTANCE=2 RF_REPEATS=20 RF_MODE=materialized \ - ./rapidfuzz_levenshtein words_alpha.txt queries.txt 10000 rapidfuzz-results - -cmp stringzilla-results.k1.bin rapidfuzz-results.k1.bin -cmp stringzilla-results.k2.bin rapidfuzz-results.k2.bin +python3 levenshtein/run.py words_alpha.txt \ + --stringzilla-root ../StringZilla \ + --rapidfuzz-root ../rapidfuzz-cpp \ + --cpu 2 ``` -To check the larger-bound crossover with one index, run: - -```bash -SZ_LEVENSHTEIN_INDEX_PLAN=shared SZ_LEVENSHTEIN_MAX_DISTANCE=10 \ - SZ_LEVENSHTEIN_REPEATS=20 SZ_LEVENSHTEIN_MODES=warm,steady \ - ./stringzilla_levenshtein words_alpha.txt queries.txt 10000 stringzilla-wide +The command builds every adapter, generates 10,000 mixed queries with seed 243, and validates results before recording timings. StringZilla, RapidFuzz, and exact SymSpell must write identical result files at `k=1` and `k=2`. FST, Tantivy, and Lucene must return the same totals. Missing output, zero work, or any mismatch stops the run. -RF_MAX_DISTANCE=10 RF_REPEATS=20 RF_MODE=materialized \ - ./rapidfuzz_levenshtein words_alpha.txt queries.txt 10000 rapidfuzz-wide +The default 20 repetitions run each adapter in a deterministic shuffled order. Every repetition starts a new process. StringZilla uses separate threshold-specific indexes for `k=1` and `k=2` and keeps cold, warm, growable, and single-query latency measurements separate. -for k in 1 2 3 4 5 6 7 8 9 10; do - cmp "stringzilla-wide.k${k}.bin" "rapidfuzz-wide.k${k}.bin" -done -``` +`target/levenshtein/manifest.json` records input hashes, repository revisions, tool versions, execution order, commands, relevant environment settings, wall time, and peak process memory. The neighboring `logs/` directory keeps the runners' original key-value output. These generated files are archived with a published run instead of committed to the repository. -Run the Rust adapters through the normal benchmark target: - -```bash -RUSTFLAGS="-C target-cpu=native" STRINGWARS_REPEATS=20 \ - cargo bench --features bench_levenshtein --bench bench_levenshtein -- \ - words_alpha.txt queries.txt 10000 -``` +CI uses the same command with the small dictionary under `levenshtein/data/`, one repetition, and `--validation-only`. `--skip-build` is available when the expected products already exist under the selected build directory. -The final result tables and raw run artifacts are added only after this protocol passes on every reported machine. +The larger-bound crossover remains a separate StringZilla and RapidFuzz run. It is not mixed into this command because SymSpell, Tantivy, and Lucene stop at small bounds, and because high-result sweeps answer a different performance question. diff --git a/levenshtein/run.py b/levenshtein/run.py new file mode 100644 index 0000000..7dabb61 --- /dev/null +++ b/levenshtein/run.py @@ -0,0 +1,444 @@ +#!/usr/bin/env python3 +"""Build, validate, and run the repeated Levenshtein benchmark.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import platform +import random +import re +import shutil +import struct +import subprocess +import time +from datetime import UTC, datetime +from pathlib import Path + +ADAPTERS = ("stringzilla", "rapidfuzz", "symspell", "fst", "tantivy", "lucene") + + +def arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("dictionary", type=Path) + parser.add_argument("--stringzilla-root", type=Path, required=True) + parser.add_argument("--rapidfuzz-root", type=Path, required=True) + parser.add_argument("--build-dir", type=Path, default=Path("target/levenshtein")) + parser.add_argument("--query-count", type=int, default=10_000) + parser.add_argument("--repeats", type=int, default=20) + parser.add_argument("--seed", type=int, default=243) + parser.add_argument("--cpu", type=int) + parser.add_argument("--validation-only", action="store_true") + parser.add_argument("--skip-build", action="store_true") + parser.add_argument("--stringzilla-modes", default="cold,warm,steady,latency") + args = parser.parse_args() + if args.query_count < 1 or args.repeats < 1: + parser.error("query count and repetitions must be positive") + for name in ("dictionary", "stringzilla_root", "rapidfuzz_root"): + setattr(args, name, getattr(args, name).resolve()) + args.build_dir = args.build_dir.resolve() + return args + + +def run(command: list[str], *, cwd: Path, env: dict[str, str] | None = None) -> subprocess.CompletedProcess[str]: + completed = subprocess.run( + command, + cwd=cwd, + env={**os.environ, **(env or {})}, + text=True, + capture_output=True, + check=False, + ) + if completed.returncode: + raise RuntimeError( + f"command failed ({completed.returncode}): {' '.join(command)}\n{completed.stdout}\n{completed.stderr}" + ) + return completed + + +def compile_products(args: argparse.Namespace, root: Path) -> dict[str, Path | str]: + build = args.build_dir + binary = build / "bin" + binary.mkdir(parents=True, exist_ok=True) + products: dict[str, Path | str] = { + "queries": binary / "queries", + "stringzilla": binary / "stringzilla", + "rapidfuzz": binary / "rapidfuzz", + "rust": binary / "bench_levenshtein", + "lucene_classpath": build / "lucene-classpath.txt", + } + if args.skip_build: + missing = [str(path) for path in products.values() if isinstance(path, Path) and not path.exists()] + if missing: + raise RuntimeError("missing build products: " + ", ".join(missing)) + products["lucene_classpath"] = Path(products["lucene_classpath"]).read_text().strip() + return products + + stringzilla_build = build / "stringzilla" + run( + ["cmake", "-S", str(args.stringzilla_root), "-B", str(stringzilla_build), "-DCMAKE_BUILD_TYPE=Release"], + cwd=root, + ) + run(["cmake", "--build", str(stringzilla_build), "-j", "--target", "stringzillas_cpus_static"], cwd=root) + stringzilla_library = next(stringzilla_build.rglob("libstringzillas_cpus_static.a"), None) + forkunion_library = next(stringzilla_build.rglob("libforkunion_static.a"), None) + if not stringzilla_library or not forkunion_library: + raise RuntimeError("StringZilla build did not produce the expected static libraries") + + cxx_flags = ["g++", "-std=c++20", "-O3", "-DNDEBUG", "-march=native"] + include = args.stringzilla_root / "include" + run( + [*cxx_flags, "-I", str(include), str(root / "levenshtein/queries.cpp"), "-o", str(products["queries"])], + cwd=root, + ) + run( + [ + *cxx_flags, + "-DSZ_DYNAMIC_DISPATCH=1", + "-I", + str(include), + "-I", + str(args.stringzilla_root / "forkunion/include"), + str(root / "levenshtein/stringzilla.cpp"), + str(stringzilla_library), + str(forkunion_library), + "-pthread", + "-o", + str(products["stringzilla"]), + ], + cwd=root, + ) + run( + [ + *cxx_flags, + "-I", + str(args.rapidfuzz_root), + str(root / "levenshtein/rapidfuzz.cpp"), + "-o", + str(products["rapidfuzz"]), + ], + cwd=root, + ) + + cargo_target = build / "cargo" + cargo = run( + [ + "cargo", + "build", + "--release", + "--locked", + "--features", + "bench_levenshtein", + "--bench", + "bench_levenshtein", + "--message-format=json", + ], + cwd=root, + env={"CARGO_TARGET_DIR": str(cargo_target), "RUSTFLAGS": "-C target-cpu=native"}, + ) + executable = None + for line in cargo.stdout.splitlines(): + try: + message = json.loads(line) + except json.JSONDecodeError: + continue + if message.get("target", {}).get("name") == "bench_levenshtein" and message.get("executable"): + executable = Path(message["executable"]) + if not executable: + raise RuntimeError("Cargo did not report the Levenshtein benchmark executable") + shutil.copy2(executable, products["rust"]) + + lucene = build / "lucene" + shutil.copytree(root / "levenshtein/lucene", lucene, dirs_exist_ok=True) + dependencies = build / "lucene-dependencies" + run( + [ + "mvn", + "-q", + "-f", + str(lucene / "pom.xml"), + "package", + "dependency:copy-dependencies", + f"-DoutputDirectory={dependencies}", + ], + cwd=root, + ) + jars = sorted(dependencies.glob("*.jar")) + classes = lucene / "target/classes" + if not jars or not classes.is_dir(): + raise RuntimeError("Maven did not produce Lucene classes and dependencies") + classpath = os.pathsep.join([str(classes), *(str(jar) for jar in jars)]) + Path(products["lucene_classpath"]).write_text(classpath) + products["lucene_classpath"] = classpath + return products + + +def hash_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + while block := source.read(1024 * 1024): + digest.update(block) + return digest.hexdigest() + + +def git_revision(path: Path) -> str | None: + completed = run(["git", "-C", str(path), "rev-parse", "HEAD"], cwd=path) + return completed.stdout.strip() or None + + +def version(command: list[str]) -> str | None: + try: + completed = subprocess.run(command, text=True, capture_output=True, check=False) + except OSError: + return None + output = completed.stdout.strip() or completed.stderr.strip() + return output.splitlines()[0] if output else None + + +def machine() -> dict[str, object]: + model = None + cpuinfo = Path("/proc/cpuinfo") + if cpuinfo.exists(): + match = re.search(r"^model name\s*:\s*(.+)$", cpuinfo.read_text(), re.MULTILINE) + model = match.group(1) if match else None + governor = Path("/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor") + return { + "platform": platform.platform(), + "cpu_count": os.cpu_count(), + "cpu_model": model, + "frequency_governor": governor.read_text().strip() if governor.exists() else None, + "tools": { + "g++": version(["g++", "--version"]), + "rustc": version(["rustc", "--version"]), + "java": version(["java", "-version"]), + "cmake": version(["cmake", "--version"]), + }, + } + + +def artifact_matches(path: Path) -> int: + data = path.read_bytes() + if len(data) < 25 or data[:8] != b"SZLEV001": + raise RuntimeError(f"invalid result artifact: {path}") + query_count = struct.unpack_from("=Q", data, 16)[0] + cursor, matches = 25, 0 + for _ in range(query_count): + count = struct.unpack_from("=Q", data, cursor)[0] + cursor += 8 + count * 5 + matches += count + if cursor != len(data): + raise RuntimeError(f"malformed result artifact: {path}") + return matches + + +def command_for( + adapter: str, + args: argparse.Namespace, + products: dict[str, Path | str], + queries: Path, + *, + validation: bool, + stringzilla_bound: int | None = None, +) -> tuple[list[str], dict[str, str]]: + dictionary, limit = str(args.dictionary), str(args.query_count) + artifacts = args.build_dir / "artifacts" + if adapter == "stringzilla": + command = [str(products[adapter]), dictionary, str(queries), limit] + if validation: + command.append(str(artifacts / adapter)) + return command, { + "SZ_LEVENSHTEIN_MAX_DISTANCE": str(stringzilla_bound or 2), + "SZ_LEVENSHTEIN_REPEATS": "1", + "SZ_LEVENSHTEIN_MODES": "warm" if validation else args.stringzilla_modes, + } + if adapter == "rapidfuzz": + command = [str(products[adapter]), dictionary, str(queries), limit] + if validation: + command.append(str(artifacts / adapter)) + return command, {"RF_MAX_DISTANCE": "2", "RF_REPEATS": "0" if validation else "1", "RF_MODE": "materialized"} + if adapter in {"symspell", "fst", "tantivy"}: + environment = { + "STRINGWARS_FILTER": f"levenshtein/{adapter}", + "STRINGWARS_REPEATS": "1", + "STRINGWARS_MIN_DISTANCE": "1", + "STRINGWARS_MAX_DISTANCE": "2", + } + if adapter == "symspell": + environment["STRINGWARS_SYMSPELL_MODE"] = "exact" if validation else "both" + if validation: + environment["STRINGWARS_DUMP_PREFIX"] = str(artifacts / adapter) + return [str(products["rust"]), dictionary, str(queries), limit], environment + return [ + "java", + "-cp", + str(products["lucene_classpath"]), + "com.stringzilla.LevenshteinIndexBenchmark", + dictionary, + str(queries), + "2", + "automaton", + ], {"STRINGWARS_REPEATS": "1"} + + +def measured_run( + adapter: str, + command: list[str], + environment: dict[str, str], + args: argparse.Namespace, + root: Path, + log: Path, +) -> dict[str, object]: + if args.cpu is not None: + command = ["taskset", "-c", str(args.cpu), *command] + timing = log.with_suffix(".rss") + measured = ["/usr/bin/time", "-f", "%M", "-o", str(timing), *command] + started = time.perf_counter() + completed = run(measured, cwd=root, env=environment) + wall = time.perf_counter() - started + log.write_text(completed.stdout) + if completed.stderr: + log.with_suffix(".stderr").write_text(completed.stderr) + return { + "adapter": adapter, + "command": command, + "environment": environment, + "log": str(log.relative_to(args.build_dir)), + "wall_seconds": wall, + "peak_rss_bytes": int(timing.read_text().strip()) * 1024, + } + + +def result_counts(text: str) -> dict[int, set[int]]: + counts: dict[int, set[int]] = {} + for line in text.splitlines(): + bound = re.search(r"(?:^| )k=(\d+)", line) + matches = re.search(r"(?:^| )matches=(\d+)", line) + if bound and matches: + counts.setdefault(int(bound.group(1)), set()).add(int(matches.group(1))) + return counts + + +def validate( + args: argparse.Namespace, products: dict[str, Path | str], queries: Path, root: Path, manifest: dict[str, object] +) -> None: + artifacts = args.build_dir / "artifacts" + logs = args.build_dir / "logs" + artifacts.mkdir(exist_ok=True) + logs.mkdir(exist_ok=True) + validation: dict[str, dict[str, object]] = {} + for adapter in ADAPTERS: + if adapter == "stringzilla": + for bound in (1, 2): + command, environment = command_for( + adapter, args, products, queries, validation=True, stringzilla_bound=bound + ) + key = f"{adapter}-k{bound}" + validation[key] = measured_run(adapter, command, environment, args, root, logs / f"validate-{key}.log") + continue + command, environment = command_for(adapter, args, products, queries, validation=True) + entry = measured_run(adapter, command, environment, args, root, logs / f"validate-{adapter}.log") + validation[adapter] = entry + + expected: dict[int, int] = {} + for bound in (1, 2): + files = { + "stringzilla": artifacts / f"stringzilla.k{bound}.bin", + "rapidfuzz": artifacts / f"rapidfuzz.k{bound}.bin", + "symspell": artifacts / f"symspell.symspell.k{bound}.bin", + } + reference = files["stringzilla"].read_bytes() + for adapter, path in files.items(): + if path.read_bytes() != reference: + raise RuntimeError(f"exact results differ at k={bound}: stringzilla != {adapter}") + expected[bound] = artifact_matches(files["stringzilla"]) + if expected[bound] == 0: + raise RuntimeError(f"validation produced no matches at k={bound}") + + for adapter in ("fst", "tantivy", "lucene"): + observed = result_counts((args.build_dir / str(validation[adapter]["log"])).read_text()) + for bound, count in expected.items(): + if observed.get(bound) != {count}: + raise RuntimeError(f"{adapter} returned {observed.get(bound)} at k={bound}; expected {count}") + manifest["expected_matches"] = expected + manifest["validation"] = validation + + +def main() -> int: + args = arguments() + root = Path(__file__).resolve().parent.parent + args.build_dir.mkdir(parents=True, exist_ok=True) + words = args.dictionary.read_text().splitlines() + if not words or len(words) != len(set(words)) or any(not word.isascii() or word.lower() != word for word in words): + raise RuntimeError("the shared adapter track requires a non-empty, unique, lowercase ASCII dictionary") + + products = compile_products(args, root) + queries = args.build_dir / "queries.txt" + run( + [str(products["queries"]), str(args.dictionary), str(queries), str(args.query_count), "mixed", str(args.seed)], + cwd=root, + ) + manifest: dict[str, object] = { + "schema": 1, + "started_at": datetime.now(UTC).isoformat(), + "machine": {**machine(), "pinned_cpu": args.cpu}, + "inputs": { + "dictionary": str(args.dictionary), + "dictionary_sha256": hash_file(args.dictionary), + "queries": str(queries), + "queries_sha256": hash_file(queries), + "query_count": args.query_count, + "seed": args.seed, + }, + "revisions": { + "stringwars": git_revision(root), + "stringzilla": git_revision(args.stringzilla_root), + "rapidfuzz": git_revision(args.rapidfuzz_root), + "cargo_lock_sha256": hash_file(root / "Cargo.lock"), + "lucene_pom_sha256": hash_file(root / "levenshtein/lucene/pom.xml"), + }, + "repeats": args.repeats, + "runs": [], + } + output = args.build_dir / "manifest.json" + output.write_text(json.dumps(manifest, indent=2) + "\n") + try: + validate(args, products, queries, root, manifest) + + if not args.validation_only: + randomizer = random.Random(args.seed) + logs = args.build_dir / "logs" + for repetition in range(args.repeats): + order = list(ADAPTERS) + randomizer.shuffle(order) + for position, adapter in enumerate(order): + bounds = (1, 2) if adapter == "stringzilla" else (None,) + for bound in bounds: + command, environment = command_for( + adapter, args, products, queries, validation=False, stringzilla_bound=bound + ) + suffix = f"-k{bound}" if bound else "" + entry = measured_run( + adapter, + command, + environment, + args, + root, + logs / f"run-{repetition:02d}-{position}-{adapter}{suffix}.log", + ) + entry.update({"repetition": repetition, "position": position, "bound": bound}) + manifest["runs"].append(entry) + output.write_text(json.dumps(manifest, indent=2) + "\n") + manifest["completed_at"] = datetime.now(UTC).isoformat() + except Exception as error: + manifest["error"] = str(error) + output.write_text(json.dumps(manifest, indent=2) + "\n") + raise + output.write_text(json.dumps(manifest, indent=2) + "\n") + print(output) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From dde76292117f3e9c35394e864ad8986f12217719 Mon Sep 17 00:00:00 2001 From: Guillaume de Rouville Date: Mon, 17 Aug 2026 17:03:26 -0700 Subject: [PATCH 17/17] Test: Validate Levenshtein adapters in CI The benchmark branch had no automated check, so a runner could stop compiling or drift to a different result contract without making the PR fail. Add one small unique lowercase dictionary and run the same public driver in validation-only mode. CI builds all six adapters against the reviewed StringZilla and RapidFuzz revisions, compares every exact artifact at k=1 and k=2, checks the ID-only and count-only totals, and uploads the manifest and raw logs. This is deliberately a correctness job, not a hosted-runner performance claim. It uses 100 deterministic queries and one repetition, while controlled benchmark machines keep the separate 20-run protocol. Signed-off-by: Guillaume de Rouville --- .github/workflows/levenshtein.yml | 67 ++++++++++++++++++++++++++++++ levenshtein/data/ci-dictionary.txt | 24 +++++++++++ 2 files changed, 91 insertions(+) create mode 100644 .github/workflows/levenshtein.yml create mode 100644 levenshtein/data/ci-dictionary.txt diff --git a/.github/workflows/levenshtein.yml b/.github/workflows/levenshtein.yml new file mode 100644 index 0000000..ec5fc5e --- /dev/null +++ b/.github/workflows/levenshtein.yml @@ -0,0 +1,67 @@ +name: Levenshtein dictionary search + +on: + pull_request: + paths: + - ".github/workflows/levenshtein.yml" + - "Cargo.lock" + - "Cargo.toml" + - "levenshtein/**" + push: + branches: [main] + paths: + - ".github/workflows/levenshtein.yml" + - "Cargo.lock" + - "Cargo.toml" + - "levenshtein/**" + +permissions: + contents: read + +concurrency: + group: levenshtein-${{ github.ref }} + cancel-in-progress: true + +jobs: + exact-results: + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + + - uses: actions/checkout@v4 + with: + repository: grouville/StringZilla + ref: 19b7faed60051867cb3bf8b1367d3da84bc1b209 + path: dependencies/StringZilla + submodules: recursive + + - uses: actions/checkout@v4 + with: + repository: rapidfuzz/rapidfuzz-cpp + ref: b5830af53bd1b3c7460a8de1e9f7095df99b3470 + path: dependencies/rapidfuzz-cpp + + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: "21" + + - name: Validate every adapter + run: | + python3 levenshtein/run.py levenshtein/data/ci-dictionary.txt \ + --stringzilla-root dependencies/StringZilla \ + --rapidfuzz-root dependencies/rapidfuzz-cpp \ + --build-dir target/levenshtein-ci \ + --query-count 100 \ + --repeats 1 \ + --validation-only + + - name: Keep validation evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: levenshtein-validation + path: | + target/levenshtein-ci/manifest.json + target/levenshtein-ci/logs/ diff --git a/levenshtein/data/ci-dictionary.txt b/levenshtein/data/ci-dictionary.txt new file mode 100644 index 0000000..90453b4 --- /dev/null +++ b/levenshtein/data/ci-dictionary.txt @@ -0,0 +1,24 @@ +back +book +booked +booking +books +boon +brook +cook +cookie +cookies +crook +distance +hook +instance +kitchen +kitten +look +shook +sitting +spook +strange +string +strong +took