diff --git a/.github/workflows/levenshtein.yml b/.github/workflows/levenshtein.yml new file mode 100644 index 0000000..ec5fc5e --- /dev/null +++ b/.github/workflows/levenshtein.yml @@ -0,0 +1,67 @@ +name: Levenshtein dictionary search + +on: + pull_request: + paths: + - ".github/workflows/levenshtein.yml" + - "Cargo.lock" + - "Cargo.toml" + - "levenshtein/**" + push: + branches: [main] + paths: + - ".github/workflows/levenshtein.yml" + - "Cargo.lock" + - "Cargo.toml" + - "levenshtein/**" + +permissions: + contents: read + +concurrency: + group: levenshtein-${{ github.ref }} + cancel-in-progress: true + +jobs: + exact-results: + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + + - uses: actions/checkout@v4 + with: + repository: grouville/StringZilla + ref: 19b7faed60051867cb3bf8b1367d3da84bc1b209 + path: dependencies/StringZilla + submodules: recursive + + - uses: actions/checkout@v4 + with: + repository: rapidfuzz/rapidfuzz-cpp + ref: b5830af53bd1b3c7460a8de1e9f7095df99b3470 + path: dependencies/rapidfuzz-cpp + + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: "21" + + - name: Validate every adapter + run: | + python3 levenshtein/run.py levenshtein/data/ci-dictionary.txt \ + --stringzilla-root dependencies/StringZilla \ + --rapidfuzz-root dependencies/rapidfuzz-cpp \ + --build-dir target/levenshtein-ci \ + --query-count 100 \ + --repeats 1 \ + --validation-only + + - name: Keep validation evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: levenshtein-validation + path: | + target/levenshtein-ci/manifest.json + target/levenshtein-ci/logs/ diff --git a/Cargo.lock b/Cargo.lock index f004abf..6d49c32 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -27,6 +27,7 @@ dependencies = [ "const-random", "getrandom 0.3.4", "once_cell", + "serde", "version_check 0.9.5", "zerocopy", ] @@ -103,6 +104,15 @@ dependencies = [ "object", ] +[[package]] +name = "arc-swap" +version = "1.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b" +dependencies = [ + "rustversion", +] + [[package]] name = "argminmax" version = "0.6.3" @@ -609,6 +619,15 @@ dependencies = [ "serde_core", ] +[[package]] +name = "bitpacking" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96a7139abd3d9cebf8cd6f920a389cf3dc9576172e32f4563f188cae3c3eb019" +dependencies = [ + "crunchy", +] + [[package]] name = "blake3" version = "1.8.5" @@ -655,6 +674,31 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "bon" +version = "3.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a602c73c7b0148ec6d12af6fd5cc7a46e2eacc8878271a999abac56eed12f561" +dependencies = [ + "bon-macros", + "rustversion", +] + +[[package]] +name = "bon-macros" +version = "3.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dee98b0db6a962de883bf5d20362dee4d7ca0d12fe39a7c6c73c844e1cd7c1f" +dependencies = [ + "darling", + "ident_case", + "prettyplease", + "proc-macro2 1.0.107", + "quote 1.0.47", + "rustversion", + "syn 2.0.119", +] + [[package]] name = "borsh" version = "1.8.0" @@ -836,13 +880,19 @@ dependencies = [ "shlex 2.0.1", ] +[[package]] +name = "census" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f4c707c6a209cbe82d10abd08e1ea8995e9ea937d2550646e02798948992be0" + [[package]] name = "cexpr" version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fce5b5fb86b0c57c20c834c1b412fd09c77c8a59b9473f86272709e78874cd1d" dependencies = [ - "nom", + "nom 4.2.3", ] [[package]] @@ -1180,12 +1230,27 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "datasketches" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c286de4e81ea2590afc24d754e0f83810c566f50a1388fa75ebd57928c0d9745" + [[package]] name = "debug_unsafe" version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7eed2c4702fa172d1ce21078faa7c5203e69f5394d48cc436d25928394a867a2" +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +dependencies = [ + "serde_core", +] + [[package]] name = "derefable" version = "0.1.0" @@ -1246,6 +1311,12 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "downcast-rs" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "117240f60069e65410b3ae1bb213295bd828f707b5bec6596a1afc8793ce0cbc" + [[package]] name = "dyn-clone" version = "1.0.20" @@ -1312,6 +1383,17 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +[[package]] +name = "erased-serde" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2add8a07dd6a8d93ff627029c51de145e12686fbc36ecb298ac22e74cf02dec" +dependencies = [ + "serde", + "serde_core", + "typeid", +] + [[package]] name = "errno" version = "0.3.14" @@ -1383,6 +1465,12 @@ dependencies = [ "siphasher 1.0.3", ] +[[package]] +name = "fastdivide" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afc2bd4d5a73106dd53d10d73d3401c2f32730ba2c0b93ddb888a8983680471" + [[package]] name = "fastrand" version = "2.5.0" @@ -1509,6 +1597,15 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "fst" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ab85b9b05e3978cc9a9cf8fea7f01b494e1a09ed3037e16ba39edc7a29eb61a" +dependencies = [ + "utf8-ranges", +] + [[package]] name = "futures" version = "0.3.33" @@ -1691,6 +1788,15 @@ version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" +[[package]] +name = "gxhash" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3ce1bab7aa741d4e7042b2aae415b78741f267a98a7271ea226cd5ba6c43d7d" +dependencies = [ + "rustversion", +] + [[package]] name = "h2" version = "0.4.15" @@ -1795,6 +1901,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "htmlescape" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163" + [[package]] name = "http" version = "1.4.2" @@ -2376,6 +2488,15 @@ dependencies = [ "serde_core", ] +[[package]] +name = "inventory" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4f0c30c76f2f4ccee3fe55a2435f691ca00c0e4bd87abe4f4a851b1d4dac39b" +dependencies = [ + "rustversion", +] + [[package]] name = "ipnet" version = "2.12.0" @@ -2442,6 +2563,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "levenshtein_automata" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c2cdeb66e45e9f36bfad5bbdb4d2384e70936afbee843c6f6543f0c551ebb25" + [[package]] name = "lexical-core" version = "1.0.6" @@ -2560,6 +2687,15 @@ version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +[[package]] +name = "lru" +version = "0.16.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f66e8d5d03f609abc3a39e6f08e4164ebf1447a732906d39eb9b99b7919ef39" +dependencies = [ + "hashbrown 0.16.1", +] + [[package]] name = "lru-slab" version = "0.1.2" @@ -2585,6 +2721,12 @@ dependencies = [ "libc", ] +[[package]] +name = "lz4_flex" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e" + [[package]] name = "matrixmultiply" version = "0.3.11" @@ -2595,6 +2737,15 @@ dependencies = [ "rawpointer", ] +[[package]] +name = "measure_time" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51c55d61e72fc3ab704396c5fa16f4c184db37978ae4e94ca8959693a235fc0e" +dependencies = [ + "log", +] + [[package]] name = "memchr" version = "2.8.3" @@ -2610,6 +2761,12 @@ dependencies = [ "libc", ] +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.8.9" @@ -2640,6 +2797,12 @@ dependencies = [ "serde", ] +[[package]] +name = "murmurhash32" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2195bf6aa996a481483b29d62a7663eed3fe39600c460e323f8ff41e90bdd89b" + [[package]] name = "murmurhash32" version = "0.4.0" @@ -2697,6 +2860,16 @@ dependencies = [ "version_check 0.1.5", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "now" version = "0.1.3" @@ -2734,6 +2907,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + [[package]] name = "num-derive" version = "0.4.2" @@ -2846,6 +3025,12 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "oneshot" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "269bca4c2591a28585d6bf10d9ed0332b7d76900a1b02bec41bdc3a2cdcda107" + [[package]] name = "openssl" version = "0.10.81" @@ -2898,6 +3083,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ownedbytes" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fbd56f7631767e61784dc43f8580f403f4475bd4aaa4da003e6295e1bab4a7e" +dependencies = [ + "stable_deref_trait", +] + [[package]] name = "parking" version = "2.2.1" @@ -3701,6 +3895,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -3710,6 +3910,16 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2 1.0.107", + "syn 2.0.119", +] + [[package]] name = "probabilistic-collections" version = "0.7.0" @@ -4597,6 +4807,15 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" +[[package]] +name = "sketches-ddsketch" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "513c3f5f732bfd6fbb187619c2dfe9d2f25f1a2976f01d575f0fd329d565df56" +dependencies = [ + "serde", +] + [[package]] name = "slab" version = "0.4.12" @@ -4762,11 +4981,12 @@ dependencies = [ "fastbloom", "foldhash 0.2.0", "forkunion", + "fst", "getrandom 0.3.4", "icu", "libc", "memchr", - "murmurhash32", + "murmurhash32 0.4.0", "openssl", "pcre2", "perf-event", @@ -4784,6 +5004,8 @@ dependencies = [ "sodiumoxide", "stringtape", "stringzilla", + "symspell_rs", + "tantivy", "twox-hash", "unicase", "unicode-linebreak", @@ -4868,6 +5090,20 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "symspell_rs" +version = "6.8.3" +source = "git+https://github.com/wolfgarbe/symspell_rs.git?rev=df6b21abbb3500b99e9e75857a616ff60f30f50f#df6b21abbb3500b99e9e75857a616ff60f30f50f" +dependencies = [ + "ahash", + "gxhash", + "itertools 0.14.0", + "regex", + "smallvec", + "strsim 0.11.1", + "unicode-normalization", +] + [[package]] name = "syn" version = "0.15.44" @@ -4935,6 +5171,167 @@ dependencies = [ "windows", ] +[[package]] +name = "tantivy" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edde6a10743fff00a4e1a8c9ef020bf5f3cbad301b7d2d39f2b07f123c4eac07" +dependencies = [ + "aho-corasick", + "arc-swap", + "base64", + "bitpacking", + "bon", + "byteorder", + "census", + "crc32fast", + "crossbeam-channel", + "datasketches", + "downcast-rs", + "fastdivide", + "fnv", + "fs4", + "htmlescape", + "itertools 0.14.0", + "levenshtein_automata", + "log", + "lru", + "lz4_flex", + "measure_time", + "memmap2", + "once_cell", + "oneshot", + "rayon", + "regex", + "rust-stemmers", + "rustc-hash", + "serde", + "serde_json", + "sketches-ddsketch", + "smallvec", + "tantivy-bitpacker", + "tantivy-columnar", + "tantivy-common", + "tantivy-fst", + "tantivy-query-grammar", + "tantivy-stacker", + "tantivy-tokenizer-api", + "tempfile", + "thiserror 2.0.19", + "time", + "typetag", + "uuid", + "winapi", +] + +[[package]] +name = "tantivy-bitpacker" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fed3d674429bcd2de5d0a6d1aa5495fed8afd9c5ecce993019caf7615f53fa4" +dependencies = [ + "bitpacking", +] + +[[package]] +name = "tantivy-columnar" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c57166f5bcfd478f370ab8445afb4678dce44801fa5ce5c451aaf8595583c5dc" +dependencies = [ + "downcast-rs", + "fastdivide", + "itertools 0.14.0", + "serde", + "tantivy-bitpacker", + "tantivy-common", + "tantivy-sstable", + "tantivy-stacker", +] + +[[package]] +name = "tantivy-common" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbf10915aa75da3c3b0d58b58853d2e889efbaf32d4982a4c3715dde6bba23e5" +dependencies = [ + "async-trait", + "byteorder", + "ownedbytes", + "serde", + "time", +] + +[[package]] +name = "tantivy-fst" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d60769b80ad7953d8a7b2c70cdfe722bbcdcac6bccc8ac934c40c034d866fc18" +dependencies = [ + "byteorder", + "regex-syntax", + "utf8-ranges", +] + +[[package]] +name = "tantivy-query-grammar" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfadb8526b6da90704feb293b0701a6aae62ea14983143344be2dc5ce30f1d82" +dependencies = [ + "fnv", + "nom 7.1.3", + "ordered-float", + "serde", + "serde_json", +] + +[[package]] +name = "tantivy-sstable" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a2cfc3ac5164cbadc28965ffb145a8f47582a60ae5897859ad8d4316596c606" +dependencies = [ + "futures-util", + "itertools 0.14.0", + "tantivy-bitpacker", + "tantivy-common", + "tantivy-fst", + "zstd", +] + +[[package]] +name = "tantivy-stacker" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cbb051742da9d53ca9e8fff43a9b10e319338b24e2c0e15d0372df19ffeb951" +dependencies = [ + "murmurhash32 0.3.1", + "tantivy-common", +] + +[[package]] +name = "tantivy-tokenizer-api" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eac258c2c6390673f2685813afeeafcb8c4e0ee7de8dd3fc46838dcc37263f98" +dependencies = [ + "serde", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + [[package]] name = "term_size" version = "0.3.2" @@ -5013,6 +5410,36 @@ dependencies = [ "cfg-if 1.0.4", ] +[[package]] +name = "time" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "tiny-keccak" version = "2.0.2" @@ -5225,12 +5652,42 @@ dependencies = [ "rand 0.10.2", ] +[[package]] +name = "typeid" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" + [[package]] name = "typenum" version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" +[[package]] +name = "typetag" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c90e86058a30d42a1a928dfb4b49bb33c98c3a2b4909492e6b0881cd94798ec2" +dependencies = [ + "erased-serde", + "inventory", + "once_cell", + "serde", + "typetag-impl", +] + +[[package]] +name = "typetag-impl" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f153acc4e99a5f2a5aefa09fb078be54e26271b2813f6041200b224c098d8328" +dependencies = [ + "proc-macro2 1.0.107", + "quote 1.0.47", + "syn 3.0.2", +] + [[package]] name = "unicase" version = "2.9.0" @@ -5315,6 +5772,12 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" +[[package]] +name = "utf8-ranges" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcfc827f90e53a02eaef5e535ee14266c1d569214c6aa70133a624d8a3164ba" + [[package]] name = "utf8_iter" version = "1.0.4" diff --git a/Cargo.toml b/Cargo.toml index e2c800b..dad34b6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -64,6 +64,11 @@ bench_hash = [ "ring", # SHA256 "cityhash", # CityHash (x86_64 only) ] +bench_levenshtein = [ + "fst", # Levenshtein automaton over a finite-state dictionary + "symspell_rs", # SymSpell deletion index + "tantivy", # Search-engine fuzzy term lookup +] bench_memory = [ "rand", # Randomize Buffer "zeroize", # Obfuscate Buffer @@ -175,6 +180,20 @@ version = "0.2.11" optional = true version = "0.5.0" +[dependencies.fst] +features = ["levenshtein"] +optional = true +version = "0.4.7" + +[dependencies.symspell_rs] +git = "https://github.com/wolfgarbe/symspell_rs.git" +optional = true +rev = "df6b21abbb3500b99e9e75857a616ff60f30f50f" + +[dependencies.tantivy] +optional = true +version = "0.26.1" + [dependencies.forkunion] optional = true version = "3.0" @@ -316,6 +335,12 @@ name = "bench_hash" path = "hash/bench.rs" required-features = ["bench_hash"] +[[bench]] +harness = false +name = "bench_levenshtein" +path = "levenshtein/bench.rs" +required-features = ["bench_levenshtein"] + [[bench]] harness = false name = "bench_sequence" diff --git a/README.md b/README.md index 8c35408..00fe26d 100644 --- a/README.md +++ b/README.md @@ -255,6 +255,12 @@ stringzilla ██████ See [similarities/README.md](similarities/README.md) for details +### Repeated Levenshtein Dictionary Search + +Repeated fuzzy search has a different shape from a distance matrix: one dictionary is built once, then searched many times. The benchmark separates full-output comparisons from tools that return only unique terms, IDs, counts, or native suggestion distances. + +See [levenshtein/README.md](levenshtein/README.md) for the workloads, timing modes, correctness checks, and reproduction commands. + ### Fingerprinting Converting variable-length strings into fixed-length sketches (like Min-Hashing) enables fast approximate matching in large-scale retrieval. diff --git a/levenshtein/README.md b/levenshtein/README.md new file mode 100644 index 0000000..402c5dc --- /dev/null +++ b/levenshtein/README.md @@ -0,0 +1,68 @@ +# Repeated Levenshtein Dictionary Search + +This benchmark covers the case where a dictionary is built once and searched many times. It is separate from the dense distance matrices in [`similarities/`](../similarities/). + +Each exact result contains the query ID, the original dictionary ID, and the plain Levenshtein distance. Duplicate dictionary entries keep separate IDs. An adjacent swap counts as two edits. + +## Comparisons + +Direct timing comparisons require the same complete output: + +* StringZilla returns every original ID and exact distance. +* RapidFuzz scans the dictionary and materializes the same result. It is also the correctness oracle. +* SymSpell has an exact compatibility mode that checks its suggestions again with allocation-free plain Levenshtein distance. It is comparable only on unique lowercase dictionaries. + +The native SymSpell, Rust `fst`, Tantivy, and Lucene modes return different information. Their results provide useful ecosystem context, but they are not used for direct speedup claims. + +Every comparable runner writes the same binary result format. The files must match before timings are reported. + +## Queries + +`queries.cpp` creates deterministic queries from an existing dictionary. The mixed workload gives equal weight to: + +* exact queries; +* one substitution, insertion, or deletion; +* two substitutions, insertions, or deletions; +* one insertion plus one deletion; +* one adjacent swap; +* a five-symbol extension of the sampled source word. + +These labels describe how the query was created from one source word. They do not assume that the query has no other dictionary matches. The result oracle decides the complete answer. + +Final runs should include short English words, longer URLs, DNA strings, valid non-ASCII text, and a duplicate-heavy synthetic dictionary. Each run records the dictionary and query hashes. + +## Timing + +StringZilla reports four separate measurements: + +* `cold_end_to_end` starts with a fresh index reader and includes output sizing, allocation, retry, and materialization. Dictionary construction is reported separately. +* `warm_presized` measures repeated batches after scratch and exact output capacity are available. +* `steady_growable` starts from eight output slots per query and includes a resize and retry when needed. +* `single_query_latency` reports p50, p95, and p99 for one-query calls with a reusable grow-only output buffer. + +Threshold-specialized runs build separate indexes for `k=1`, `k=2`, and `k=4`. Shared-index runs build once for `k=4` and query the same index at every bound from one through four. + +The same complete-output comparison can sweep larger bounds by setting both runners to the same maximum. This is kept separate from the low-bound ecosystem table because several indexed tools only support small edit distances. It checks that StringZilla changes its internal search path without changing results or developing a performance cliff. + +Published runs use at least 20 measured repetitions, keep raw output, randomize runner order, pin CPU and memory placement, and record compiler versions, dependency revisions, CPU frequency settings, result counts, output bytes, build time, retained index size, peak build memory, and reader scratch. Warm and cold results are never combined into one number. + +## Running the comparison + +The shared low-bound track expects a unique lowercase ASCII dictionary because that is the common contract supported by all six adapters. From the StringWars root: + +```bash +python3 levenshtein/run.py words_alpha.txt \ + --stringzilla-root ../StringZilla \ + --rapidfuzz-root ../rapidfuzz-cpp \ + --cpu 2 +``` + +The command builds every adapter, generates 10,000 mixed queries with seed 243, and validates results before recording timings. StringZilla, RapidFuzz, and exact SymSpell must write identical result files at `k=1` and `k=2`. FST, Tantivy, and Lucene must return the same totals. Missing output, zero work, or any mismatch stops the run. + +The default 20 repetitions run each adapter in a deterministic shuffled order. Every repetition starts a new process. StringZilla uses separate threshold-specific indexes for `k=1` and `k=2` and keeps cold, warm, growable, and single-query latency measurements separate. + +`target/levenshtein/manifest.json` records input hashes, repository revisions, tool versions, execution order, commands, relevant environment settings, wall time, and peak process memory. The neighboring `logs/` directory keeps the runners' original key-value output. These generated files are archived with a published run instead of committed to the repository. + +CI uses the same command with the small dictionary under `levenshtein/data/`, one repetition, and `--validation-only`. `--skip-build` is available when the expected products already exist under the selected build directory. + +The larger-bound crossover remains a separate StringZilla and RapidFuzz run. It is not mixed into this command because SymSpell, Tantivy, and Lucene stop at small bounds, and because high-result sweeps answer a different performance question. diff --git a/levenshtein/bench.rs b/levenshtein/bench.rs new file mode 100644 index 0000000..0b3db40 --- /dev/null +++ b/levenshtein/bench.rs @@ -0,0 +1,486 @@ +//! Repeated bounded Levenshtein search over an immutable dictionary. +//! +//! The dictionary is built once and reused for every query. Use `STRINGWARS_FILTER` to select +//! `levenshtein/fst`, `levenshtein/symspell`, or `levenshtein/tantivy`. + +#[allow(dead_code)] +#[path = "../utils.rs"] +mod utils; + +use fst::automaton::Levenshtein; +use fst::{IntoStreamer, Map, Streamer}; +use std::collections::HashMap; +use std::env; +use std::fs::{self, File}; +use std::io::{BufWriter, Write}; +use std::time::Instant; +use symspell_rs::{SymSpell, Verbosity}; +use tantivy::collector::DocSetCollector; +use tantivy::query::FuzzyTermQuery; +use tantivy::schema::{Schema, STRING}; +use tantivy::{doc, Index, Term}; +use utils::should_run; + +type AnyError = Box; + +#[derive(Clone, Copy)] +struct Match { + id: u32, + distance: u8, +} + +struct Settings { + repeats: usize, + min_distance: usize, + max_distance: usize, + dump_prefix: Option, + cache_evict_bytes: usize, +} + +fn env_usize(name: &str, default: usize) -> Result { + env::var(name).map_or(Ok(default), |value| Ok(value.parse()?)) +} + +fn load_lines(path: &str, limit: usize) -> Result, AnyError> { + let contents = fs::read_to_string(path)?; + Ok(contents + .lines() + .take(if limit == 0 { usize::MAX } else { limit }) + .map(|line| line.trim_end_matches('\r').to_owned()) + .collect()) +} + +fn checksum_id(checksum: u64, id: u64) -> u64 { + checksum.wrapping_mul(0x9E37_79B1_85EB_CA87).wrapping_add(id) +} + +fn evict_cache(buffer: &mut [u8], checksum: &mut u64) { + for value in buffer.iter_mut().step_by(64) { + *value = value.wrapping_add(1); + *checksum = checksum.wrapping_add(*value as u64); + } + std::hint::black_box(*checksum); +} + +struct DistanceScratch { + previous: Vec, + current: Vec, +} + +fn levenshtein_within(first: &[char], second: &[char], bound: usize, scratch: &mut DistanceScratch) -> Option { + if first.len().abs_diff(second.len()) > bound { + return None; + } + + scratch.previous.resize(first.len() + 1, 0); + scratch.current.resize(first.len() + 1, 0); + for (column, value) in scratch.previous.iter_mut().enumerate() { + *value = column; + } + for (row, &second_char) in second.iter().enumerate() { + scratch.current[0] = row + 1; + let mut row_min = scratch.current[0]; + for (column, &first_char) in first.iter().enumerate() { + scratch.current[column + 1] = (scratch.previous[column + 1] + 1) + .min(scratch.current[column] + 1) + .min(scratch.previous[column] + usize::from(first_char != second_char)); + row_min = row_min.min(scratch.current[column + 1]); + } + if row_min > bound { + return None; + } + std::mem::swap(&mut scratch.previous, &mut scratch.current); + } + (scratch.previous[first.len()] <= bound).then_some(scratch.previous[first.len()]) +} + +fn symspell_search( + symspell: &SymSpell, + ids: &HashMap, + decoded_dictionary: &[Vec], + query: &str, + decoded_query: &[char], + bound: usize, + distance_scratch: &mut DistanceScratch, + matches: &mut Vec, +) { + matches.clear(); + for suggestion in symspell.lookup(query, Verbosity::All, bound, &None, None, false) { + let id = ids[&suggestion.term]; + if let Some(distance) = + levenshtein_within(decoded_query, &decoded_dictionary[id as usize], bound, distance_scratch) + { + matches.push(Match { + id, + distance: distance as u8, + }); + } + } +} + +fn dump_symspell_matches( + symspell: &SymSpell, + ids: &HashMap, + decoded_dictionary: &[Vec], + decoded_queries: &[Vec], + dictionary_size: usize, + queries: &[String], + bound: usize, + path: &str, +) -> Result<(), AnyError> { + let mut output = BufWriter::new(File::create(path)?); + let mut distance_scratch = DistanceScratch { + previous: Vec::new(), + current: Vec::new(), + }; + let mut matches = Vec::new(); + output.write_all(b"SZLEV001")?; + output.write_all(&(dictionary_size as u64).to_ne_bytes())?; + output.write_all(&(queries.len() as u64).to_ne_bytes())?; + output.write_all(&[bound as u8])?; + for (query, decoded_query) in queries.iter().zip(decoded_queries) { + symspell_search( + symspell, + ids, + decoded_dictionary, + query, + decoded_query, + bound, + &mut distance_scratch, + &mut matches, + ); + matches.sort_unstable_by_key(|found| (found.id, found.distance)); + output.write_all(&(matches.len() as u64).to_ne_bytes())?; + for found in &matches { + output.write_all(&found.id.to_ne_bytes())?; + output.write_all(&[found.distance])?; + } + } + Ok(()) +} + +fn run_symspell(dictionary: &[String], queries: &[String], settings: &Settings) -> Result<(), AnyError> { + if dictionary + .iter() + .chain(queries) + .any(|text| text.to_lowercase() != *text) + { + return Err("SymSpell requires lowercase input for case-sensitive comparison".into()); + } + + let setup_start = Instant::now(); + let mut ids = HashMap::with_capacity(dictionary.len()); + for (id, word) in dictionary.iter().enumerate() { + if ids.insert(word.clone(), id as u32).is_some() { + return Err("SymSpell cannot preserve duplicate dictionary entries".into()); + } + } + let decoded_dictionary: Vec> = dictionary.iter().map(|word| word.chars().collect()).collect(); + let decoded_queries: Vec> = queries.iter().map(|query| query.chars().collect()).collect(); + let setup_seconds = setup_start.elapsed().as_secs_f64(); + let requested_mode = env::var("STRINGWARS_SYMSPELL_MODE").unwrap_or_else(|_| "both".to_owned()); + if requested_mode != "raw" && requested_mode != "exact" && requested_mode != "both" { + return Err("STRINGWARS_SYMSPELL_MODE must be raw, exact, or both".into()); + } + + println!("# levenshtein/symspell"); + println!( + "dictionary={} queries={} adapter_setup={setup_seconds:.6}s", + dictionary.len(), + queries.len() + ); + let mut cache = vec![0u8; settings.cache_evict_bytes]; + let mut cache_checksum = 0u64; + + for bound in settings.min_distance..=settings.max_distance { + let build_start = Instant::now(); + let mut symspell = SymSpell::new(bound, None, 7, 1); + for word in dictionary { + symspell.create_dictionary_entry(word, 1); + } + println!( + "k={bound} build={:.6}s indexed_words={}", + build_start.elapsed().as_secs_f64(), + symspell.get_dictionary_size() + ); + + if requested_mode == "raw" || requested_mode == "both" { + for repeat in 0..settings.repeats { + evict_cache(&mut cache, &mut cache_checksum); + let start = Instant::now(); + let mut matches_count = 0usize; + let mut checksum = 0u64; + for query in queries { + for suggestion in symspell.lookup(query, Verbosity::All, bound, &None, None, false) { + matches_count += 1; + checksum = checksum.wrapping_add( + ((ids[&suggestion.term] as u64) << 8 | suggestion.distance as u64) + .wrapping_mul(0x9E37_79B1_85EB_CA87), + ); + } + } + println!( + "k={bound} mode=native repeat={repeat} query={:.6}s matches={matches_count} checksum={checksum:016x} cache_evict_bytes={} semantics=symspell output=id+native-distance", + start.elapsed().as_secs_f64(), + cache.len() + ); + } + } + + if requested_mode == "exact" || requested_mode == "both" { + for repeat in 0..settings.repeats { + evict_cache(&mut cache, &mut cache_checksum); + let start = Instant::now(); + let mut matches_count = 0usize; + let mut checksum = 0u64; + let mut distance_scratch = DistanceScratch { + previous: Vec::new(), + current: Vec::new(), + }; + let mut matches = Vec::new(); + for (query, decoded_query) in queries.iter().zip(&decoded_queries) { + symspell_search( + &symspell, + &ids, + &decoded_dictionary, + query, + decoded_query, + bound, + &mut distance_scratch, + &mut matches, + ); + for found in &matches { + matches_count += 1; + checksum = checksum.wrapping_add( + ((found.id as u64) << 8 | found.distance as u64).wrapping_mul(0x9E37_79B1_85EB_CA87), + ); + } + } + println!( + "k={bound} mode=exact_compatibility repeat={repeat} query={:.6}s matches={matches_count} checksum={checksum:016x} cache_evict_bytes={} semantics=unicode-codepoints output=id+exact-distance", + start.elapsed().as_secs_f64(), + cache.len() + ); + } + } + + if let Some(prefix) = &settings.dump_prefix { + dump_symspell_matches( + &symspell, + &ids, + &decoded_dictionary, + &decoded_queries, + dictionary.len(), + queries, + bound, + &format!("{prefix}.symspell.k{bound}.bin"), + )?; + } + } + Ok(()) +} + +fn verify_fst_unicode_contract() -> Result<(), AnyError> { + let mut keys = vec!["é", "Ѐ", "А"]; + keys.sort_unstable(); + let map = Map::from_iter(keys.iter().enumerate().map(|(id, key)| (*key, id as u64)))?; + let mut matches = 0usize; + for query in &keys { + let automaton = Levenshtein::new(query, 1)?; + let mut stream = map.search(&automaton).into_stream(); + while stream.next().is_some() { + matches += 1; + } + } + if matches != 9 { + return Err( + format!("fst Unicode smoke test failed: expected 9 one-character matches, observed {matches}").into(), + ); + } + Ok(()) +} + +fn run_fst(dictionary: &[String], queries: &[String], settings: &Settings) -> Result<(), AnyError> { + let allow_unicode = env::var_os("STRINGWARS_ALLOW_FST_UNICODE").is_some(); + if allow_unicode { + verify_fst_unicode_contract()?; + } else if dictionary.iter().chain(queries).any(|text| !text.is_ascii()) { + return Err("fst requires ASCII input for byte-for-byte semantic parity".into()); + } + + let mut keyed_words: Vec<(&str, u64)> = dictionary + .iter() + .enumerate() + .map(|(id, word)| (word.as_str(), id as u64)) + .collect(); + keyed_words.sort_unstable_by_key(|&(word, _)| word); + if keyed_words.windows(2).any(|pair| pair[0].0 == pair[1].0) { + return Err("fst cannot preserve duplicate dictionary entries".into()); + } + + let build_start = Instant::now(); + let map = Map::from_iter(keyed_words)?; + println!("# levenshtein/fst"); + println!( + "dictionary={} queries={} semantics={} build={:.6}s fst_bytes={} output=id-only", + dictionary.len(), + queries.len(), + if allow_unicode { + "unicode-codepoints" + } else { + "ascii-byte-parity" + }, + build_start.elapsed().as_secs_f64(), + map.as_fst().as_bytes().len() + ); + + let state_limit = env::var("STRINGWARS_FST_STATE_LIMIT") + .ok() + .map(|value| value.parse::()) + .transpose()?; + let mut cache = vec![0u8; settings.cache_evict_bytes]; + let mut cache_checksum = 0u64; + for bound in settings.min_distance..=settings.max_distance { + for repeat in 0..settings.repeats { + evict_cache(&mut cache, &mut cache_checksum); + let start = Instant::now(); + let mut matches_count = 0usize; + let mut checksum = 0u64; + let mut failed = None; + for query in queries { + let automaton = match state_limit { + Some(limit) => Levenshtein::new_with_limit(query, bound as u32, limit), + None => Levenshtein::new(query, bound as u32), + }; + let automaton = match automaton { + Ok(automaton) => automaton, + Err(error) => { + failed = Some(error); + break; + } + }; + let mut stream = map.search(&automaton).into_stream(); + while let Some((_, id)) = stream.next() { + matches_count += 1; + checksum = checksum_id(checksum, id); + } + } + if let Some(error) = failed { + println!("k={bound} skipped={error}"); + break; + } + println!( + "k={bound} repeat={repeat} query={:.6}s matches={matches_count} checksum={checksum:016x} cache_evict_bytes={}", + start.elapsed().as_secs_f64(), + cache.len() + ); + } + } + Ok(()) +} + +fn run_tantivy(dictionary: &[String], queries: &[String], settings: &Settings) -> Result<(), AnyError> { + let mut schema_builder = Schema::builder(); + let word_field = schema_builder.add_text_field("word", STRING); + let index = Index::create_in_ram(schema_builder.build()); + let build_start = Instant::now(); + let mut writer = index.writer(50_000_000)?; + for word in dictionary { + writer.add_document(doc!(word_field => word.as_str()))?; + } + writer.commit()?; + writer.wait_merging_threads()?; + let reader = index.reader()?; + let searcher = reader.searcher(); + println!("# levenshtein/tantivy"); + println!( + "dictionary={} queries={} build={:.6}s segments={} output=id-only", + dictionary.len(), + queries.len(), + build_start.elapsed().as_secs_f64(), + searcher.segment_readers().len() + ); + + let mut cache = vec![0u8; settings.cache_evict_bytes]; + let mut cache_checksum = 0u64; + for bound in settings.min_distance..=settings.max_distance { + if bound > u8::MAX as usize { + break; + } + for repeat in 0..settings.repeats { + evict_cache(&mut cache, &mut cache_checksum); + let start = Instant::now(); + let mut matches_count = 0usize; + let mut checksum = 0u64; + let mut failed = None; + for query_text in queries { + let term = Term::from_field_text(word_field, query_text); + let query = FuzzyTermQuery::new(term, bound as u8, false); + let matches = match searcher.search(&query, &DocSetCollector) { + Ok(matches) => matches, + Err(error) => { + failed = Some(error); + break; + } + }; + matches_count += matches.len(); + for address in matches { + let id = (u64::from(address.segment_ord) << 32) | u64::from(address.doc_id); + // `DocSetCollector` does not promise iteration order. Hash the set commutatively so + // identical matches produce one reproducible checksum across repeats. + checksum = checksum.wrapping_add(id.wrapping_mul(0x9E37_79B1_85EB_CA87)); + } + } + if let Some(error) = failed { + println!("k={bound} skipped={error}"); + break; + } + println!( + "k={bound} repeat={repeat} query={:.6}s matches={matches_count} checksum={checksum:016x} cache_evict_bytes={}", + start.elapsed().as_secs_f64(), + cache.len() + ); + } + } + Ok(()) +} + +fn main() -> Result<(), AnyError> { + let args: Vec = env::args().filter(|argument| argument != "--bench").collect(); + if args.len() < 3 || args.len() > 4 { + eprintln!("usage: bench_levenshtein DICTIONARY QUERIES [QUERY_LIMIT]"); + std::process::exit(2); + } + + let query_limit = args.get(3).map_or(Ok(0), |value| value.parse::())?; + let dictionary = load_lines(&args[1], 0)?; + let queries = load_lines(&args[2], query_limit)?; + let cache_evict_mb = env_usize("STRINGWARS_CACHE_EVICT_MB", 0)?; + let settings = Settings { + repeats: env_usize("STRINGWARS_REPEATS", 3)?, + min_distance: env_usize("STRINGWARS_MIN_DISTANCE", 1)?, + max_distance: env_usize("STRINGWARS_MAX_DISTANCE", 2)?, + dump_prefix: env::var("STRINGWARS_DUMP_PREFIX").ok(), + cache_evict_bytes: cache_evict_mb + .checked_mul(1024 * 1024) + .ok_or("STRINGWARS_CACHE_EVICT_MB is too large")?, + }; + if settings.repeats == 0 + || settings.min_distance == 0 + || settings.min_distance > settings.max_distance + || settings.max_distance > 4 + { + return Err("expected positive repeats and 1 <= minimum distance <= maximum distance <= 4".into()); + } + + if should_run("levenshtein/fst") { + run_fst(&dictionary, &queries, &settings)?; + } + if should_run("levenshtein/symspell") { + run_symspell(&dictionary, &queries, &settings)?; + } + if should_run("levenshtein/tantivy") { + run_tantivy(&dictionary, &queries, &settings)?; + } + Ok(()) +} diff --git a/levenshtein/data/ci-dictionary.txt b/levenshtein/data/ci-dictionary.txt new file mode 100644 index 0000000..90453b4 --- /dev/null +++ b/levenshtein/data/ci-dictionary.txt @@ -0,0 +1,24 @@ +back +book +booked +booking +books +boon +brook +cook +cookie +cookies +crook +distance +hook +instance +kitchen +kitten +look +shook +sitting +spook +strange +string +strong +took diff --git a/levenshtein/dictionary_terms.py b/levenshtein/dictionary_terms.py new file mode 100644 index 0000000..1490737 --- /dev/null +++ b/levenshtein/dictionary_terms.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +"""Extract unique already-lowercase terms from a whitespace-delimited frequency dictionary.""" + +import argparse + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("input") + parser.add_argument("output") + args = parser.parse_args() + + seen: set[str] = set() + with open(args.input, encoding="utf-8") as source, open(args.output, "w", encoding="utf-8") as target: + for line in source: + fields = line.split() + if not fields: + continue + term = fields[0] + if term.lower() != term or term in seen: + continue + seen.add(term) + target.write(term + "\n") + + +if __name__ == "__main__": + main() diff --git a/levenshtein/lucene/pom.xml b/levenshtein/lucene/pom.xml new file mode 100644 index 0000000..dc39762 --- /dev/null +++ b/levenshtein/lucene/pom.xml @@ -0,0 +1,34 @@ + + + 4.0.0 + com.stringzilla + levenshtein-index-lucene + 1.0-SNAPSHOT + + 21 + UTF-8 + + + + org.apache.lucene + lucene-core + 10.3.1 + + + org.apache.lucene + lucene-analysis-common + 10.3.1 + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + + + diff --git a/levenshtein/lucene/src/main/java/com/stringzilla/LevenshteinIndexBenchmark.java b/levenshtein/lucene/src/main/java/com/stringzilla/LevenshteinIndexBenchmark.java new file mode 100644 index 0000000..0b6eff2 --- /dev/null +++ b/levenshtein/lucene/src/main/java/com/stringzilla/LevenshteinIndexBenchmark.java @@ -0,0 +1,93 @@ +package com.stringzilla; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import org.apache.lucene.analysis.core.KeywordAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.FuzzyQuery; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TotalHitCountCollector; +import org.apache.lucene.store.ByteBuffersDirectory; +import org.apache.lucene.search.AutomatonQuery; +import org.apache.lucene.util.automaton.LevenshteinAutomata; + +public final class LevenshteinIndexBenchmark { + private static List loadAscii(Path path) throws Exception { + List lines = Files.readAllLines(path, StandardCharsets.UTF_8); + for (String line : lines) + for (int index = 0; index != line.length(); ++index) + if (line.charAt(index) > 0x7f) + throw new IllegalArgumentException("non-ASCII input would compare different semantics: " + path); + return lines; + } + + public static void main(String[] args) throws Exception { + if (args.length != 4) { + System.err.println("usage: LevenshteinIndexBenchmark DICTIONARY QUERIES MAX_DISTANCE fuzzy|automaton"); + System.exit(2); + } + List dictionary = loadAscii(Path.of(args[0])); + List queries = loadAscii(Path.of(args[1])); + int maxDistance = Integer.parseInt(args[2]); + boolean exactAutomaton = args[3].equals("automaton"); + if (!exactAutomaton && !args[3].equals("fuzzy")) { + System.err.println("mode must be fuzzy or automaton"); + System.exit(2); + } + if (maxDistance < 1 || maxDistance > FuzzyQuery.defaultMaxEdits) { + System.err.printf("MAX_DISTANCE must be between 1 and %d%n", FuzzyQuery.defaultMaxEdits); + System.exit(2); + } + String repeatsVariable = System.getenv("STRINGWARS_REPEATS"); + int repeats = repeatsVariable == null ? 3 : Integer.parseInt(repeatsVariable); + if (repeats < 1) { + System.err.println("STRINGWARS_REPEATS must be positive"); + System.exit(2); + } + + long buildStart = System.nanoTime(); + ByteBuffersDirectory directory = new ByteBuffersDirectory(); + IndexWriterConfig config = new IndexWriterConfig(new KeywordAnalyzer()); + try (IndexWriter writer = new IndexWriter(directory, config)) { + for (String word : dictionary) { + Document document = new Document(); + document.add(new StringField("term", word, Field.Store.NO)); + writer.addDocument(document); + } + writer.forceMerge(1); + } + DirectoryReader reader = DirectoryReader.open(directory); + IndexSearcher searcher = new IndexSearcher(reader); + double buildSeconds = (System.nanoTime() - buildStart) * 1e-9; + System.out.printf("dictionary=%d queries=%d build=%.6fs%n", dictionary.size(), queries.size(), buildSeconds); + + for (int bound = 1; bound <= maxDistance; ++bound) { + for (int repeat = 0; repeat != repeats; ++repeat) { + long matches = 0; + long start = System.nanoTime(); + for (String query : queries) { + Term term = new Term("term", query); + Query fuzzy = exactAutomaton + ? new AutomatonQuery(term, new LevenshteinAutomata(query, false).toAutomaton(bound)) + : new FuzzyQuery(term, bound, 0, Integer.MAX_VALUE, false); + TotalHitCountCollector collector = new TotalHitCountCollector(); + searcher.search(fuzzy, collector); + matches += collector.getTotalHits(); + } + double seconds = (System.nanoTime() - start) * 1e-9; + System.out.printf("k=%d query=%.6fs matches=%d%n", bound, seconds, matches); + } + } + reader.close(); + directory.close(); + } +} diff --git a/levenshtein/queries.cpp b/levenshtein/queries.cpp new file mode 100644 index 0000000..e6163af --- /dev/null +++ b/levenshtein/queries.cpp @@ -0,0 +1,218 @@ +/** + * @brief Deterministic query generator for immutable-dictionary Levenshtein benchmarks. + * + * Samples dictionary entries and applies a named edit operation. The mixed workload gives equal weight to exact + * queries, substitutions, insertions, deletions, adjacent swaps, and several two-edit combinations. Labels describe + * how a query was made from its source word; the result oracle still decides which dictionary entries match. + * Mutations use bytes by default or validated Unicode codepoints when `SZ_LEVENSHTEIN_UTF8` is set. + */ +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +static std::vector load_lines(char const *path) { + std::ifstream input(path); + if (!input) throw std::runtime_error(std::string("failed to open ") + path); + std::vector lines; + std::string line; + while (std::getline(input, line)) { + if (!line.empty() && line.back() == '\r') line.pop_back(); + lines.push_back(std::move(line)); + } + return lines; +} + +static std::uint64_t splitmix64(std::uint64_t &state) { + std::uint64_t value = (state += 0x9E3779B97F4A7C15ull); + value = (value ^ (value >> 30)) * 0xBF58476D1CE4E5B9ull; + value = (value ^ (value >> 27)) * 0x94D049BB133111EBull; + return value ^ (value >> 31); +} + +static bool decode_utf8(std::string const &encoded, std::vector &decoded) { + decoded.clear(); + char const *position = encoded.data(), *end = position + encoded.size(); + while (position != end) { + sz_rune_t rune = 0; + sz_rune_length_t const length = sz_rune_decode(position, end, &rune); + if (length == sz_rune_invalid_k) return false; + decoded.push_back(rune); + position += length; + } + return true; +} + +static std::string encode_utf8(std::vector const &decoded) { + std::string encoded; + encoded.reserve(decoded.size() * 3); + for (sz_rune_t rune : decoded) { + sz_u8_t bytes[4]; + sz_rune_length_t const length = sz_rune_encode(rune, bytes); + if (length == sz_rune_invalid_k) throw std::runtime_error("invalid Unicode codepoint"); + encoded.append(reinterpret_cast(bytes), length); + } + return encoded; +} + +template +static std::size_t substitute(sequence_type_ &query, std::uint64_t random) { + std::size_t const position = random % query.size(); + query[position] = query[position] == '~' ? '^' : '~'; + return position; +} + +static bool known_mode(std::string_view mode) { + return mode == "exact" || mode == "substitute1" || mode == "insert1" || mode == "delete1" || + mode == "substitute2" || mode == "insert_delete" || mode == "insert2" || mode == "delete2" || + mode == "transpose" || mode == "source_plus_5" || mode == "mixed"; +} + +template +static bool supports_mode(sequence_type_ const &query, std::string_view mode) { + if ((mode == "substitute1" || mode == "delete1" || mode == "insert_delete") && query.empty()) return false; + if ((mode == "substitute2" || mode == "delete2") && query.size() < 2) return false; + if (mode == "transpose") { + for (std::size_t index = 1; index != query.size(); ++index) + if (query[index - 1] != query[index]) return true; + return false; + } + return true; +} + +template +static void mutate(sequence_type_ &query, std::string_view mode, std::uint64_t &random_state) { + using symbol_t = typename sequence_type_::value_type; + auto const marker = [](symbol_t symbol) { return symbol == symbol_t('~') ? symbol_t('^') : symbol_t('~'); }; + auto const insert_one = [&] { + std::size_t const position = splitmix64(random_state) % (query.size() + 1); + symbol_t const inserted = position < query.size() ? marker(query[position]) : symbol_t('~'); + query.insert(query.begin() + position, inserted); + }; + auto const delete_one = [&] { + std::size_t const position = splitmix64(random_state) % query.size(); + query.erase(query.begin() + position); + }; + + if (mode == "exact") return; + if (mode == "substitute1") { + substitute(query, splitmix64(random_state)); + return; + } + if (mode == "insert1") { + insert_one(); + return; + } + if (mode == "delete1") { + delete_one(); + return; + } + if (mode == "substitute2") { + std::size_t const first = substitute(query, splitmix64(random_state)); + std::size_t second = splitmix64(random_state) % (query.size() - 1); + if (second >= first) ++second; + query[second] = marker(query[second]); + return; + } + if (mode == "insert_delete") { + delete_one(); + insert_one(); + return; + } + if (mode == "insert2") { + insert_one(); + insert_one(); + return; + } + if (mode == "delete2") { + delete_one(); + delete_one(); + return; + } + if (mode == "transpose") { + std::size_t position = splitmix64(random_state) % (query.size() - 1); + while (query[position] == query[position + 1]) position = (position + 1) % (query.size() - 1); + std::swap(query[position], query[position + 1]); + return; + } + query.insert(query.begin(), 5, symbol_t('~')); +} + +int main(int argc, char **argv) { + if (argc != 6) { + std::cerr << "usage: levenshtein_index_queries DICTIONARY OUTPUT COUNT MODE SEED\n"; + return 2; + } + auto const dictionary = load_lines(argv[1]); + if (dictionary.empty()) { + std::cerr << "dictionary is empty\n"; + return 3; + } + std::ofstream output(argv[2]); + if (!output) { + std::cerr << "failed to create " << argv[2] << '\n'; + return 4; + } + std::size_t const count = std::stoull(argv[3]); + std::string_view const requested_mode = argv[4]; + if (!known_mode(requested_mode)) { + std::cerr << "unknown mode: " << requested_mode << '\n'; + return 5; + } + std::uint64_t random_state = std::stoull(argv[5]); + bool const utf8 = std::getenv("SZ_LEVENSHTEIN_UTF8") != nullptr; + for (std::size_t query_index = 0; query_index != count; ++query_index) { + std::string_view mode = requested_mode; + if (mode == "mixed") { + static constexpr std::string_view modes[] = {"exact", "substitute1", "insert1", "delete1", + "substitute2", "insert_delete", "insert2", "delete2", + "transpose", "source_plus_5"}; + mode = modes[query_index % (sizeof(modes) / sizeof(modes[0]))]; + } + if (utf8) { + std::vector decoded; + bool selected = false; + for (std::size_t attempt = 0; attempt != dictionary.size(); ++attempt) { + std::string const &source = dictionary[splitmix64(random_state) % dictionary.size()]; + if (!decode_utf8(source, decoded)) { + std::cerr << "invalid UTF-8 dictionary entry\n"; + return 6; + } + if (supports_mode(decoded, mode)) { + selected = true; + break; + } + } + if (!selected) { + std::cerr << "dictionary has no entry suitable for mode " << mode << '\n'; + return 6; + } + mutate(decoded, mode, random_state); + output << encode_utf8(decoded) << '\n'; + } + else { + std::string query; + bool selected = false; + for (std::size_t attempt = 0; attempt != dictionary.size(); ++attempt) { + query = dictionary[splitmix64(random_state) % dictionary.size()]; + if (supports_mode(query, mode)) { + selected = true; + break; + } + } + if (!selected) { + std::cerr << "dictionary has no entry suitable for mode " << mode << '\n'; + return 6; + } + mutate(query, mode, random_state); + output << query << '\n'; + } + } + return output.good() ? 0 : 6; +} diff --git a/levenshtein/rapidfuzz.cpp b/levenshtein/rapidfuzz.cpp new file mode 100644 index 0000000..a6bd8ed --- /dev/null +++ b/levenshtein/rapidfuzz.cpp @@ -0,0 +1,140 @@ +/** + * @brief Native RapidFuzz baseline for repeated immutable-dictionary Levenshtein retrieval. + * + * This file is intentionally not part of the default build because it requires rapidfuzz-cpp. + * Pin the dependency revision and compile it explicitly, for example: + * + * g++ -std=c++20 -O3 -DNDEBUG -march=native -I rapidfuzz-cpp \ + * levenshtein/rapidfuzz.cpp -o rapidfuzz_levenshtein_index + * + * `RF_REPEATS=0` skips timing and only emits exact comparison artifacts when `DUMP_PREFIX` is present. + */ +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +struct match_t { + std::uint64_t query_id; + std::uint32_t id; + std::uint8_t distance; +}; + +static std::vector load_lines(char const *path, std::size_t limit = 0) { + std::ifstream input(path); + if (!input) throw std::runtime_error(std::string("failed to open ") + path); + std::vector lines; + std::string line; + while ((!limit || lines.size() != limit) && std::getline(input, line)) { + if (!line.empty() && line.back() == '\r') line.pop_back(); + lines.push_back(std::move(line)); + } + return lines; +} + +static bool dump_matches(std::vector const &dictionary, std::vector const &queries, + std::uint8_t bound, std::string const &path) { + std::ofstream output(path, std::ios::binary); + if (!output) return false; + char const magic[8] = {'S', 'Z', 'L', 'E', 'V', '0', '0', '1'}; + std::uint64_t const dictionary_size = dictionary.size(); + std::uint64_t const queries_size = queries.size(); + output.write(magic, sizeof(magic)); + output.write(reinterpret_cast(&dictionary_size), sizeof(dictionary_size)); + output.write(reinterpret_cast(&queries_size), sizeof(queries_size)); + output.write(reinterpret_cast(&bound), sizeof(bound)); + + std::vector matches; + for (auto const &query : queries) { + matches.clear(); + rapidfuzz::CachedLevenshtein scorer(query); + for (std::uint32_t id = 0; id != dictionary.size(); ++id) { + std::size_t const distance = scorer.distance(dictionary[id], bound); + if (distance <= bound) matches.push_back({0, id, static_cast(distance)}); + } + std::uint64_t const matches_size = matches.size(); + output.write(reinterpret_cast(&matches_size), sizeof(matches_size)); + for (auto const &match : matches) { + output.write(reinterpret_cast(&match.id), sizeof(match.id)); + output.write(reinterpret_cast(&match.distance), sizeof(match.distance)); + } + } + return output.good(); +} + +int main(int argc, char **argv) { + if (argc < 3 || argc > 5) { + std::cerr << "usage: rapidfuzz_levenshtein_index DICTIONARY QUERIES [QUERY_LIMIT] [DUMP_PREFIX]\n"; + return 2; + } + std::size_t const query_limit = argc >= 4 ? std::stoull(argv[3]) : 0; + std::string const dump_prefix = argc == 5 ? argv[4] : ""; + auto const dictionary = load_lines(argv[1]); + auto const queries = load_lines(argv[2], query_limit); + int const repeats = std::getenv("RF_REPEATS") ? std::stoi(std::getenv("RF_REPEATS")) : 3; + int const max_distance = std::getenv("RF_MAX_DISTANCE") ? std::stoi(std::getenv("RF_MAX_DISTANCE")) : 4; + std::string const requested_mode = std::getenv("RF_MODE") ? std::getenv("RF_MODE") : "both"; + if (max_distance < 1 || max_distance >= std::numeric_limits::max()) { + std::cerr << "RF_MAX_DISTANCE must be between 1 and 254\n"; + return 2; + } + if (requested_mode != "count" && requested_mode != "materialized" && requested_mode != "both") { + std::cerr << "RF_MODE must be count, materialized, or both\n"; + return 2; + } + std::cout << "dictionary=" << dictionary.size() << " queries=" << queries.size() << '\n'; + + for (std::uint8_t bound = 1; bound <= max_distance; ++bound) { + if (requested_mode == "count" || requested_mode == "both") { + for (int repeat = 0; repeat != repeats; ++repeat) { + std::size_t matches_count = 0; + auto const start = std::chrono::steady_clock::now(); + for (auto const &query : queries) { + rapidfuzz::CachedLevenshtein scorer(query); + for (auto const &candidate : dictionary) + if (scorer.distance(candidate, bound) <= bound) ++matches_count; + } + double const elapsed = + std::chrono::duration(std::chrono::steady_clock::now() - start).count(); + std::cout << "k=" << unsigned(bound) << " mode=count repeat=" << repeat << " query=" << elapsed + << "s matches=" << matches_count << " output=count-only\n"; + } + } + if (requested_mode == "materialized" || requested_mode == "both") { + std::vector matches; + for (int repeat = 0; repeat != repeats; ++repeat) { + matches.clear(); + auto const start = std::chrono::steady_clock::now(); + for (std::uint64_t query_id = 0; query_id != queries.size(); ++query_id) { + rapidfuzz::CachedLevenshtein scorer(queries[query_id]); + for (std::uint32_t id = 0; id != dictionary.size(); ++id) { + std::size_t const distance = scorer.distance(dictionary[id], bound); + if (distance <= bound) + matches.push_back({query_id, id, static_cast(distance)}); + } + } + double const elapsed = + std::chrono::duration(std::chrono::steady_clock::now() - start).count(); + std::uint64_t checksum = 0; + for (match_t const &match : matches) + checksum += ((match.query_id * dictionary.size() + match.id) << 8) | match.distance; + std::cout << "k=" << unsigned(bound) << " mode=materialized repeat=" << repeat << " query=" + << elapsed << "s matches=" << matches.size() << " checksum=" << std::hex << checksum + << std::dec << " output=query-id+dictionary-id+distance\n"; + } + } + if (!dump_prefix.empty()) { + std::string const path = dump_prefix + ".k" + std::to_string(bound) + ".bin"; + if (!dump_matches(dictionary, queries, bound, path)) { + std::cerr << "dump failed: " << path << '\n'; + return 3; + } + } + } +} diff --git a/levenshtein/rapidfuzz_utf8.cpp b/levenshtein/rapidfuzz_utf8.cpp new file mode 100644 index 0000000..6e103d9 --- /dev/null +++ b/levenshtein/rapidfuzz_utf8.cpp @@ -0,0 +1,131 @@ +/** + * @brief Native RapidFuzz baseline for validated UTF-8/codepoint immutable-dictionary retrieval. + * + * This is intentionally outside the default build because it requires rapidfuzz-cpp. UTF-8 decoding happens while + * loading the corpus, outside the timed query loop, favoring RapidFuzz relative to StringZilla's per-query facade. + */ +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +struct match_t { + std::uint32_t id; + std::uint8_t distance; +}; + +static bool decode_utf8(std::string const &source, std::u32string &destination) { + destination.clear(); + for (std::size_t i = 0; i != source.size();) { + std::uint8_t const lead = static_cast(source[i]); + char32_t rune = 0; + std::size_t length = 0; + if (lead < 0x80) rune = lead, length = 1; + else if (lead >= 0xC2 && lead <= 0xDF && i + 1 < source.size()) + rune = (char32_t(lead & 0x1F) << 6) | (static_cast(source[i + 1]) & 0x3F), length = 2; + else if (lead >= 0xE0 && lead <= 0xEF && i + 2 < source.size()) + rune = (char32_t(lead & 0x0F) << 12) | ((static_cast(source[i + 1]) & 0x3F) << 6) | + (static_cast(source[i + 2]) & 0x3F), + length = 3; + else if (lead >= 0xF0 && lead <= 0xF4 && i + 3 < source.size()) + rune = (char32_t(lead & 0x07) << 18) | ((static_cast(source[i + 1]) & 0x3F) << 12) | + ((static_cast(source[i + 2]) & 0x3F) << 6) | + (static_cast(source[i + 3]) & 0x3F), + length = 4; + else + return false; + for (std::size_t continuation = 1; continuation != length; ++continuation) + if ((static_cast(source[i + continuation]) & 0xC0) != 0x80) return false; + if ((length == 3 && ((lead == 0xE0 && static_cast(source[i + 1]) < 0xA0) || + (lead == 0xED && static_cast(source[i + 1]) >= 0xA0))) || + (length == 4 && ((lead == 0xF0 && static_cast(source[i + 1]) < 0x90) || + (lead == 0xF4 && static_cast(source[i + 1]) >= 0x90)))) + return false; + destination.push_back(rune); + i += length; + } + return true; +} + +static std::vector load_lines(char const *path, std::size_t limit = 0) { + std::ifstream input(path); + if (!input) throw std::runtime_error(std::string("failed to open ") + path); + std::vector lines; + std::string encoded; + while ((!limit || lines.size() != limit) && std::getline(input, encoded)) { + if (!encoded.empty() && encoded.back() == '\r') encoded.pop_back(); + std::u32string decoded; + if (!decode_utf8(encoded, decoded)) throw std::runtime_error(std::string("invalid UTF-8 in ") + path); + lines.push_back(std::move(decoded)); + } + return lines; +} + +static bool dump_matches(std::vector const &dictionary, + std::vector const &queries, std::uint8_t bound, std::string const &path) { + std::ofstream output(path, std::ios::binary); + if (!output) return false; + char const magic[8] = {'S', 'Z', 'L', 'E', 'V', '0', '0', '1'}; + std::uint64_t const dictionary_size = dictionary.size(), queries_size = queries.size(); + output.write(magic, sizeof(magic)); + output.write(reinterpret_cast(&dictionary_size), sizeof(dictionary_size)); + output.write(reinterpret_cast(&queries_size), sizeof(queries_size)); + output.write(reinterpret_cast(&bound), sizeof(bound)); + + std::vector matches; + for (auto const &query : queries) { + matches.clear(); + rapidfuzz::CachedLevenshtein scorer(query); + for (std::uint32_t id = 0; id != dictionary.size(); ++id) { + std::size_t const distance = scorer.distance(dictionary[id], bound); + if (distance <= bound) matches.push_back({id, static_cast(distance)}); + } + std::uint64_t const matches_size = matches.size(); + output.write(reinterpret_cast(&matches_size), sizeof(matches_size)); + for (auto const &match : matches) { + output.write(reinterpret_cast(&match.id), sizeof(match.id)); + output.write(reinterpret_cast(&match.distance), sizeof(match.distance)); + } + } + return output.good(); +} + +int main(int argc, char **argv) { + if (argc < 3 || argc > 5) { + std::cerr << "usage: rapidfuzz_levenshtein_index_utf8 DICTIONARY QUERIES [QUERY_LIMIT] [DUMP_PREFIX]\n"; + return 2; + } + std::size_t const query_limit = argc >= 4 ? std::stoull(argv[3]) : 0; + std::string const dump_prefix = argc == 5 ? argv[4] : ""; + auto const dictionary = load_lines(argv[1]); + auto const queries = load_lines(argv[2], query_limit); + int const repeats = std::getenv("RF_REPEATS") ? std::stoi(std::getenv("RF_REPEATS")) : 3; + int const max_distance = std::getenv("RF_MAX_DISTANCE") ? std::stoi(std::getenv("RF_MAX_DISTANCE")) : 4; + if (max_distance < 1 || max_distance >= std::numeric_limits::max()) return 2; + std::cout << "dictionary=" << dictionary.size() << " queries=" << queries.size() + << " semantics=utf8-codepoints\n"; + + for (std::uint8_t bound = 1; bound <= max_distance; ++bound) { + for (int repeat = 0; repeat != repeats; ++repeat) { + std::size_t matches_count = 0; + auto const start = std::chrono::steady_clock::now(); + for (auto const &query : queries) { + rapidfuzz::CachedLevenshtein scorer(query); + for (auto const &candidate : dictionary) + if (scorer.distance(candidate, bound) <= bound) ++matches_count; + } + double const elapsed = + std::chrono::duration(std::chrono::steady_clock::now() - start).count(); + std::cout << "k=" << unsigned(bound) << " query=" << elapsed << "s matches=" << matches_count << '\n'; + } + if (!dump_prefix.empty() && !dump_matches(dictionary, queries, bound, + dump_prefix + ".k" + std::to_string(bound) + ".bin")) + return 3; + } +} diff --git a/levenshtein/run.py b/levenshtein/run.py new file mode 100644 index 0000000..7dabb61 --- /dev/null +++ b/levenshtein/run.py @@ -0,0 +1,444 @@ +#!/usr/bin/env python3 +"""Build, validate, and run the repeated Levenshtein benchmark.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import platform +import random +import re +import shutil +import struct +import subprocess +import time +from datetime import UTC, datetime +from pathlib import Path + +ADAPTERS = ("stringzilla", "rapidfuzz", "symspell", "fst", "tantivy", "lucene") + + +def arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("dictionary", type=Path) + parser.add_argument("--stringzilla-root", type=Path, required=True) + parser.add_argument("--rapidfuzz-root", type=Path, required=True) + parser.add_argument("--build-dir", type=Path, default=Path("target/levenshtein")) + parser.add_argument("--query-count", type=int, default=10_000) + parser.add_argument("--repeats", type=int, default=20) + parser.add_argument("--seed", type=int, default=243) + parser.add_argument("--cpu", type=int) + parser.add_argument("--validation-only", action="store_true") + parser.add_argument("--skip-build", action="store_true") + parser.add_argument("--stringzilla-modes", default="cold,warm,steady,latency") + args = parser.parse_args() + if args.query_count < 1 or args.repeats < 1: + parser.error("query count and repetitions must be positive") + for name in ("dictionary", "stringzilla_root", "rapidfuzz_root"): + setattr(args, name, getattr(args, name).resolve()) + args.build_dir = args.build_dir.resolve() + return args + + +def run(command: list[str], *, cwd: Path, env: dict[str, str] | None = None) -> subprocess.CompletedProcess[str]: + completed = subprocess.run( + command, + cwd=cwd, + env={**os.environ, **(env or {})}, + text=True, + capture_output=True, + check=False, + ) + if completed.returncode: + raise RuntimeError( + f"command failed ({completed.returncode}): {' '.join(command)}\n{completed.stdout}\n{completed.stderr}" + ) + return completed + + +def compile_products(args: argparse.Namespace, root: Path) -> dict[str, Path | str]: + build = args.build_dir + binary = build / "bin" + binary.mkdir(parents=True, exist_ok=True) + products: dict[str, Path | str] = { + "queries": binary / "queries", + "stringzilla": binary / "stringzilla", + "rapidfuzz": binary / "rapidfuzz", + "rust": binary / "bench_levenshtein", + "lucene_classpath": build / "lucene-classpath.txt", + } + if args.skip_build: + missing = [str(path) for path in products.values() if isinstance(path, Path) and not path.exists()] + if missing: + raise RuntimeError("missing build products: " + ", ".join(missing)) + products["lucene_classpath"] = Path(products["lucene_classpath"]).read_text().strip() + return products + + stringzilla_build = build / "stringzilla" + run( + ["cmake", "-S", str(args.stringzilla_root), "-B", str(stringzilla_build), "-DCMAKE_BUILD_TYPE=Release"], + cwd=root, + ) + run(["cmake", "--build", str(stringzilla_build), "-j", "--target", "stringzillas_cpus_static"], cwd=root) + stringzilla_library = next(stringzilla_build.rglob("libstringzillas_cpus_static.a"), None) + forkunion_library = next(stringzilla_build.rglob("libforkunion_static.a"), None) + if not stringzilla_library or not forkunion_library: + raise RuntimeError("StringZilla build did not produce the expected static libraries") + + cxx_flags = ["g++", "-std=c++20", "-O3", "-DNDEBUG", "-march=native"] + include = args.stringzilla_root / "include" + run( + [*cxx_flags, "-I", str(include), str(root / "levenshtein/queries.cpp"), "-o", str(products["queries"])], + cwd=root, + ) + run( + [ + *cxx_flags, + "-DSZ_DYNAMIC_DISPATCH=1", + "-I", + str(include), + "-I", + str(args.stringzilla_root / "forkunion/include"), + str(root / "levenshtein/stringzilla.cpp"), + str(stringzilla_library), + str(forkunion_library), + "-pthread", + "-o", + str(products["stringzilla"]), + ], + cwd=root, + ) + run( + [ + *cxx_flags, + "-I", + str(args.rapidfuzz_root), + str(root / "levenshtein/rapidfuzz.cpp"), + "-o", + str(products["rapidfuzz"]), + ], + cwd=root, + ) + + cargo_target = build / "cargo" + cargo = run( + [ + "cargo", + "build", + "--release", + "--locked", + "--features", + "bench_levenshtein", + "--bench", + "bench_levenshtein", + "--message-format=json", + ], + cwd=root, + env={"CARGO_TARGET_DIR": str(cargo_target), "RUSTFLAGS": "-C target-cpu=native"}, + ) + executable = None + for line in cargo.stdout.splitlines(): + try: + message = json.loads(line) + except json.JSONDecodeError: + continue + if message.get("target", {}).get("name") == "bench_levenshtein" and message.get("executable"): + executable = Path(message["executable"]) + if not executable: + raise RuntimeError("Cargo did not report the Levenshtein benchmark executable") + shutil.copy2(executable, products["rust"]) + + lucene = build / "lucene" + shutil.copytree(root / "levenshtein/lucene", lucene, dirs_exist_ok=True) + dependencies = build / "lucene-dependencies" + run( + [ + "mvn", + "-q", + "-f", + str(lucene / "pom.xml"), + "package", + "dependency:copy-dependencies", + f"-DoutputDirectory={dependencies}", + ], + cwd=root, + ) + jars = sorted(dependencies.glob("*.jar")) + classes = lucene / "target/classes" + if not jars or not classes.is_dir(): + raise RuntimeError("Maven did not produce Lucene classes and dependencies") + classpath = os.pathsep.join([str(classes), *(str(jar) for jar in jars)]) + Path(products["lucene_classpath"]).write_text(classpath) + products["lucene_classpath"] = classpath + return products + + +def hash_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + while block := source.read(1024 * 1024): + digest.update(block) + return digest.hexdigest() + + +def git_revision(path: Path) -> str | None: + completed = run(["git", "-C", str(path), "rev-parse", "HEAD"], cwd=path) + return completed.stdout.strip() or None + + +def version(command: list[str]) -> str | None: + try: + completed = subprocess.run(command, text=True, capture_output=True, check=False) + except OSError: + return None + output = completed.stdout.strip() or completed.stderr.strip() + return output.splitlines()[0] if output else None + + +def machine() -> dict[str, object]: + model = None + cpuinfo = Path("/proc/cpuinfo") + if cpuinfo.exists(): + match = re.search(r"^model name\s*:\s*(.+)$", cpuinfo.read_text(), re.MULTILINE) + model = match.group(1) if match else None + governor = Path("/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor") + return { + "platform": platform.platform(), + "cpu_count": os.cpu_count(), + "cpu_model": model, + "frequency_governor": governor.read_text().strip() if governor.exists() else None, + "tools": { + "g++": version(["g++", "--version"]), + "rustc": version(["rustc", "--version"]), + "java": version(["java", "-version"]), + "cmake": version(["cmake", "--version"]), + }, + } + + +def artifact_matches(path: Path) -> int: + data = path.read_bytes() + if len(data) < 25 or data[:8] != b"SZLEV001": + raise RuntimeError(f"invalid result artifact: {path}") + query_count = struct.unpack_from("=Q", data, 16)[0] + cursor, matches = 25, 0 + for _ in range(query_count): + count = struct.unpack_from("=Q", data, cursor)[0] + cursor += 8 + count * 5 + matches += count + if cursor != len(data): + raise RuntimeError(f"malformed result artifact: {path}") + return matches + + +def command_for( + adapter: str, + args: argparse.Namespace, + products: dict[str, Path | str], + queries: Path, + *, + validation: bool, + stringzilla_bound: int | None = None, +) -> tuple[list[str], dict[str, str]]: + dictionary, limit = str(args.dictionary), str(args.query_count) + artifacts = args.build_dir / "artifacts" + if adapter == "stringzilla": + command = [str(products[adapter]), dictionary, str(queries), limit] + if validation: + command.append(str(artifacts / adapter)) + return command, { + "SZ_LEVENSHTEIN_MAX_DISTANCE": str(stringzilla_bound or 2), + "SZ_LEVENSHTEIN_REPEATS": "1", + "SZ_LEVENSHTEIN_MODES": "warm" if validation else args.stringzilla_modes, + } + if adapter == "rapidfuzz": + command = [str(products[adapter]), dictionary, str(queries), limit] + if validation: + command.append(str(artifacts / adapter)) + return command, {"RF_MAX_DISTANCE": "2", "RF_REPEATS": "0" if validation else "1", "RF_MODE": "materialized"} + if adapter in {"symspell", "fst", "tantivy"}: + environment = { + "STRINGWARS_FILTER": f"levenshtein/{adapter}", + "STRINGWARS_REPEATS": "1", + "STRINGWARS_MIN_DISTANCE": "1", + "STRINGWARS_MAX_DISTANCE": "2", + } + if adapter == "symspell": + environment["STRINGWARS_SYMSPELL_MODE"] = "exact" if validation else "both" + if validation: + environment["STRINGWARS_DUMP_PREFIX"] = str(artifacts / adapter) + return [str(products["rust"]), dictionary, str(queries), limit], environment + return [ + "java", + "-cp", + str(products["lucene_classpath"]), + "com.stringzilla.LevenshteinIndexBenchmark", + dictionary, + str(queries), + "2", + "automaton", + ], {"STRINGWARS_REPEATS": "1"} + + +def measured_run( + adapter: str, + command: list[str], + environment: dict[str, str], + args: argparse.Namespace, + root: Path, + log: Path, +) -> dict[str, object]: + if args.cpu is not None: + command = ["taskset", "-c", str(args.cpu), *command] + timing = log.with_suffix(".rss") + measured = ["/usr/bin/time", "-f", "%M", "-o", str(timing), *command] + started = time.perf_counter() + completed = run(measured, cwd=root, env=environment) + wall = time.perf_counter() - started + log.write_text(completed.stdout) + if completed.stderr: + log.with_suffix(".stderr").write_text(completed.stderr) + return { + "adapter": adapter, + "command": command, + "environment": environment, + "log": str(log.relative_to(args.build_dir)), + "wall_seconds": wall, + "peak_rss_bytes": int(timing.read_text().strip()) * 1024, + } + + +def result_counts(text: str) -> dict[int, set[int]]: + counts: dict[int, set[int]] = {} + for line in text.splitlines(): + bound = re.search(r"(?:^| )k=(\d+)", line) + matches = re.search(r"(?:^| )matches=(\d+)", line) + if bound and matches: + counts.setdefault(int(bound.group(1)), set()).add(int(matches.group(1))) + return counts + + +def validate( + args: argparse.Namespace, products: dict[str, Path | str], queries: Path, root: Path, manifest: dict[str, object] +) -> None: + artifacts = args.build_dir / "artifacts" + logs = args.build_dir / "logs" + artifacts.mkdir(exist_ok=True) + logs.mkdir(exist_ok=True) + validation: dict[str, dict[str, object]] = {} + for adapter in ADAPTERS: + if adapter == "stringzilla": + for bound in (1, 2): + command, environment = command_for( + adapter, args, products, queries, validation=True, stringzilla_bound=bound + ) + key = f"{adapter}-k{bound}" + validation[key] = measured_run(adapter, command, environment, args, root, logs / f"validate-{key}.log") + continue + command, environment = command_for(adapter, args, products, queries, validation=True) + entry = measured_run(adapter, command, environment, args, root, logs / f"validate-{adapter}.log") + validation[adapter] = entry + + expected: dict[int, int] = {} + for bound in (1, 2): + files = { + "stringzilla": artifacts / f"stringzilla.k{bound}.bin", + "rapidfuzz": artifacts / f"rapidfuzz.k{bound}.bin", + "symspell": artifacts / f"symspell.symspell.k{bound}.bin", + } + reference = files["stringzilla"].read_bytes() + for adapter, path in files.items(): + if path.read_bytes() != reference: + raise RuntimeError(f"exact results differ at k={bound}: stringzilla != {adapter}") + expected[bound] = artifact_matches(files["stringzilla"]) + if expected[bound] == 0: + raise RuntimeError(f"validation produced no matches at k={bound}") + + for adapter in ("fst", "tantivy", "lucene"): + observed = result_counts((args.build_dir / str(validation[adapter]["log"])).read_text()) + for bound, count in expected.items(): + if observed.get(bound) != {count}: + raise RuntimeError(f"{adapter} returned {observed.get(bound)} at k={bound}; expected {count}") + manifest["expected_matches"] = expected + manifest["validation"] = validation + + +def main() -> int: + args = arguments() + root = Path(__file__).resolve().parent.parent + args.build_dir.mkdir(parents=True, exist_ok=True) + words = args.dictionary.read_text().splitlines() + if not words or len(words) != len(set(words)) or any(not word.isascii() or word.lower() != word for word in words): + raise RuntimeError("the shared adapter track requires a non-empty, unique, lowercase ASCII dictionary") + + products = compile_products(args, root) + queries = args.build_dir / "queries.txt" + run( + [str(products["queries"]), str(args.dictionary), str(queries), str(args.query_count), "mixed", str(args.seed)], + cwd=root, + ) + manifest: dict[str, object] = { + "schema": 1, + "started_at": datetime.now(UTC).isoformat(), + "machine": {**machine(), "pinned_cpu": args.cpu}, + "inputs": { + "dictionary": str(args.dictionary), + "dictionary_sha256": hash_file(args.dictionary), + "queries": str(queries), + "queries_sha256": hash_file(queries), + "query_count": args.query_count, + "seed": args.seed, + }, + "revisions": { + "stringwars": git_revision(root), + "stringzilla": git_revision(args.stringzilla_root), + "rapidfuzz": git_revision(args.rapidfuzz_root), + "cargo_lock_sha256": hash_file(root / "Cargo.lock"), + "lucene_pom_sha256": hash_file(root / "levenshtein/lucene/pom.xml"), + }, + "repeats": args.repeats, + "runs": [], + } + output = args.build_dir / "manifest.json" + output.write_text(json.dumps(manifest, indent=2) + "\n") + try: + validate(args, products, queries, root, manifest) + + if not args.validation_only: + randomizer = random.Random(args.seed) + logs = args.build_dir / "logs" + for repetition in range(args.repeats): + order = list(ADAPTERS) + randomizer.shuffle(order) + for position, adapter in enumerate(order): + bounds = (1, 2) if adapter == "stringzilla" else (None,) + for bound in bounds: + command, environment = command_for( + adapter, args, products, queries, validation=False, stringzilla_bound=bound + ) + suffix = f"-k{bound}" if bound else "" + entry = measured_run( + adapter, + command, + environment, + args, + root, + logs / f"run-{repetition:02d}-{position}-{adapter}{suffix}.log", + ) + entry.update({"repetition": repetition, "position": position, "bound": bound}) + manifest["runs"].append(entry) + output.write_text(json.dumps(manifest, indent=2) + "\n") + manifest["completed_at"] = datetime.now(UTC).isoformat() + except Exception as error: + manifest["error"] = str(error) + output.write_text(json.dumps(manifest, indent=2) + "\n") + raise + output.write_text(json.dumps(manifest, indent=2) + "\n") + print(output) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/levenshtein/stringzilla.cpp b/levenshtein/stringzilla.cpp new file mode 100644 index 0000000..36ae004 --- /dev/null +++ b/levenshtein/stringzilla.cpp @@ -0,0 +1,412 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static std::vector load_lines(char const *path, std::size_t limit = 0) { + std::ifstream input(path); + if (!input) throw std::runtime_error(std::string("failed to open ") + path); + std::vector lines; + std::string line; + while ((!limit || lines.size() != limit) && std::getline(input, line)) { + if (!line.empty() && line.back() == '\r') line.pop_back(); + lines.push_back(std::move(line)); + } + return lines; +} + +static std::vector make_views(std::vector const &strings) { + std::vector views; + views.reserve(strings.size()); + for (auto const &string : strings) views.push_back({string.data(), string.size()}); + return views; +} + +struct sparse_matches_t { + std::vector query_ids; + std::vector dictionary_ids; + std::vector distances; + + void resize(std::size_t count) { + query_ids.resize(count); + dictionary_ids.resize(count); + distances.resize(count); + } +}; + +static bool mode_enabled(std::string_view modes, std::string_view requested) { + while (!modes.empty()) { + std::size_t const separator = modes.find(','); + std::string_view const mode = modes.substr(0, separator); + if (mode == requested) return true; + if (separator == std::string_view::npos) break; + modes.remove_prefix(separator + 1); + } + return false; +} + +static double percentile(std::vector values, double fraction) { + if (values.empty()) return 0; + std::sort(values.begin(), values.end()); + std::size_t const rank = static_cast(fraction * (values.size() - 1)); + return values[rank]; +} + +static bool dump_matches(std::size_t dictionary_size, std::size_t queries_size, std::uint8_t bound, + sparse_matches_t const &matches, std::string const &path) { + std::vector>> grouped(queries_size); + for (std::size_t match = 0; match != matches.query_ids.size(); ++match) { + if (matches.query_ids[match] >= queries_size || matches.dictionary_ids[match] >= dictionary_size || + matches.distances[match] > bound) + return false; + grouped[matches.query_ids[match]].push_back({matches.dictionary_ids[match], matches.distances[match]}); + } + for (auto &query_matches : grouped) std::sort(query_matches.begin(), query_matches.end()); + + std::ofstream output(path, std::ios::binary); + if (!output) return false; + char const magic[8] = {'S', 'Z', 'L', 'E', 'V', '0', '0', '1'}; + std::uint64_t const dictionary_size_u64 = dictionary_size; + std::uint64_t const queries_size_u64 = queries_size; + output.write(magic, sizeof(magic)); + output.write(reinterpret_cast(&dictionary_size_u64), sizeof(dictionary_size_u64)); + output.write(reinterpret_cast(&queries_size_u64), sizeof(queries_size_u64)); + output.write(reinterpret_cast(&bound), sizeof(bound)); + for (auto const &query_matches : grouped) { + std::uint64_t const matches_size = query_matches.size(); + output.write(reinterpret_cast(&matches_size), sizeof(matches_size)); + for (auto const &match : query_matches) { + output.write(reinterpret_cast(&match.first), sizeof(match.first)); + output.write(reinterpret_cast(&match.second), sizeof(match.second)); + } + } + return output.good(); +} + +template +static sz_status_t index_init(sz_sequence_t const *dictionary, sz_size_t max_distance, void **index, + char const **error) { + if constexpr (utf8_) + return szs_levenshtein_index_utf8_init(dictionary, max_distance, NULL, sz_caps_sp_k, index, error); + else return szs_levenshtein_index_init(dictionary, max_distance, NULL, sz_caps_sp_k, index, error); +} + +template +static sz_status_t index_find(void *index, szs_device_scope_t device, sz_sequence_t const *queries, + sz_size_t bound, sparse_matches_t &matches, sz_size_t *matches_found, + char const **error) { + sz_size_t const capacity = matches.query_ids.size(); + sz_u64_t *query_ids = capacity ? matches.query_ids.data() : NULL; + sz_u32_t *dictionary_ids = capacity ? matches.dictionary_ids.data() : NULL; + sz_u8_t *distances = capacity ? matches.distances.data() : NULL; + if constexpr (utf8_) + return szs_levenshtein_index_utf8_find(index, device, queries, bound, query_ids, dictionary_ids, distances, + capacity, matches_found, error); + else + return szs_levenshtein_index_find(index, device, queries, bound, query_ids, dictionary_ids, distances, + capacity, matches_found, error); +} + +template +static void index_free(void *index) { + if constexpr (utf8_) szs_levenshtein_index_utf8_free(index); + else szs_levenshtein_index_free(index); +} + +template +static int run(std::vector const &dictionary, std::vector const &queries, + std::vector const &max_distances, std::string const &dump_prefix, + int query_repeats, std::size_t query_threads, std::size_t batches_per_repeat, + std::size_t cache_evict_bytes, std::string_view modes, bool shared_index) { + std::vector dictionary_views = make_views(dictionary); + std::vector query_views = make_views(queries); + sz_sequence_t dictionary_sequence, query_sequence; + sz_sequence_from_string_views(dictionary_views.data(), dictionary_views.size(), &dictionary_sequence); + sz_sequence_from_string_views(query_views.data(), query_views.size(), &query_sequence); + + char const *error = NULL; + szs_device_scope_t device = NULL; + if (szs_device_scope_init_cpu_cores(query_threads, &device, &error) != sz_success_k) { + std::cerr << "device initialization failed: " << (error ? error : "unknown error") << '\n'; + return 3; + } + + std::vector cache_evict_buffer(cache_evict_bytes); + std::uint64_t cache_evict_checksum = 0; + for (std::uint8_t max_distance : max_distances) { + void *index = NULL; + auto const build_start = std::chrono::steady_clock::now(); + if (sz_status_t status = index_init(&dictionary_sequence, max_distance, &index, &error); + status != sz_success_k) { + std::cerr << "build failed: " << int(status) << " " << (error ? error : "") << '\n'; + szs_device_scope_free(device); + return 4; + } + double const build_seconds = + std::chrono::duration(std::chrono::steady_clock::now() - build_start).count(); + std::cout << "k=" << unsigned(max_distance) << " build=" << build_seconds << "s\n"; + + std::uint8_t const first_bound = shared_index ? 1 : max_distance <= 2 ? max_distance : 3; + for (std::uint8_t bound = first_bound; bound <= max_distance; ++bound) { + if (mode_enabled(modes, "cold")) { + for (int repeat = 0; repeat != query_repeats; ++repeat) { + void *cold_index = NULL; + auto const cold_build_start = std::chrono::steady_clock::now(); + if (sz_status_t status = index_init(&dictionary_sequence, max_distance, &cold_index, &error); + status != sz_success_k) { + std::cerr << "cold build failed: " << int(status) << " " << (error ? error : "") << '\n'; + index_free(index); + szs_device_scope_free(device); + return 5; + } + double const cold_build_seconds = + std::chrono::duration(std::chrono::steady_clock::now() - cold_build_start).count(); + for (std::size_t offset = 0; offset < cache_evict_buffer.size(); offset += 64) { + ++cache_evict_buffer[offset]; + cache_evict_checksum += cache_evict_buffer[offset]; + } + sparse_matches_t cold_matches; + auto const start = std::chrono::steady_clock::now(); + sz_size_t cold_required = 0; + sz_status_t status = index_find(cold_index, device, &query_sequence, bound, cold_matches, + &cold_required, &error); + if (status != sz_success_k && status != sz_unexpected_dimensions_k) { + std::cerr << "cold sizing failed: " << int(status) << " " << (error ? error : "") << '\n'; + index_free(cold_index); + index_free(index); + szs_device_scope_free(device); + return 5; + } + cold_matches.resize(cold_required); + sz_size_t cold_count = 0; + status = index_find(cold_index, device, &query_sequence, bound, cold_matches, &cold_count, + &error); + double const elapsed = + std::chrono::duration(std::chrono::steady_clock::now() - start).count(); + index_free(cold_index); + if (status != sz_success_k || cold_count != cold_required) { + std::cerr << "cold search failed: " << int(status) << " " << (error ? error : "") << '\n'; + index_free(index); + szs_device_scope_free(device); + return 5; + } + std::cout << "k=" << unsigned(bound) << " mode=cold_end_to_end repeat=" << repeat + << " query=" << elapsed << "s matches=" << cold_count + << " prep_build=" << cold_build_seconds << "s threads=" << query_threads << '\n'; + } + } + + sparse_matches_t matches; + sz_size_t required = 0; + sz_status_t sizing_status = index_find(index, device, &query_sequence, bound, matches, + &required, &error); + if (sizing_status != sz_success_k && sizing_status != sz_unexpected_dimensions_k) { + std::cerr << "output sizing failed: " << int(sizing_status) << " " << (error ? error : "") << '\n'; + index_free(index); + szs_device_scope_free(device); + return 5; + } + matches.resize(required); + sz_size_t materialized = 0; + if (sz_status_t status = index_find(index, device, &query_sequence, bound, matches, &materialized, + &error); + status != sz_success_k || materialized != required) { + std::cerr << "correctness materialization failed: " << int(status) << " " << (error ? error : "") + << '\n'; + index_free(index); + szs_device_scope_free(device); + return 6; + } + + for (int repeat = 0; mode_enabled(modes, "warm") && repeat != query_repeats; ++repeat) { + for (std::size_t offset = 0; offset < cache_evict_buffer.size(); offset += 64) { + ++cache_evict_buffer[offset]; + cache_evict_checksum += cache_evict_buffer[offset]; + } + auto const start = std::chrono::steady_clock::now(); + sz_size_t matches_count = 0; + for (std::size_t batch = 0; batch != batches_per_repeat; ++batch) { + sz_size_t batch_matches = 0; + if (sz_status_t status = index_find(index, device, &query_sequence, bound, matches, + &batch_matches, &error); + status != sz_success_k) { + std::cerr << "search failed: " << int(status) << " " << (error ? error : "") << '\n'; + index_free(index); + szs_device_scope_free(device); + return 6; + } + if (batch_matches != required) { + std::cerr << "search returned a different result count after output sizing\n"; + index_free(index); + szs_device_scope_free(device); + return 6; + } + matches_count += batch_matches; + } + double const elapsed = + std::chrono::duration(std::chrono::steady_clock::now() - start).count(); + std::cout << "k=" << unsigned(bound) << " mode=warm_presized repeat=" << repeat + << " query=" << elapsed / batches_per_repeat << "s matches=" + << matches_count / batches_per_repeat << " threads=" << query_threads << " batches=" + << batches_per_repeat << " cache_evict_bytes=" << cache_evict_bytes + << " output_element_bytes=" + << sizeof(sz_u64_t) + sizeof(sz_u32_t) + sizeof(sz_u8_t) << '\n'; + } + + for (int repeat = 0; mode_enabled(modes, "steady") && repeat != query_repeats; ++repeat) { + for (std::size_t offset = 0; offset < cache_evict_buffer.size(); offset += 64) { + ++cache_evict_buffer[offset]; + cache_evict_checksum += cache_evict_buffer[offset]; + } + sparse_matches_t growable; + if (queries.size() > std::numeric_limits::max() / 8) { + std::cerr << "query count is too large for the starting output estimate\n"; + index_free(index); + szs_device_scope_free(device); + return 6; + } + auto const start = std::chrono::steady_clock::now(); + growable.resize(queries.size() * 8); + sz_size_t found = 0; + sz_status_t status = index_find(index, device, &query_sequence, bound, growable, &found, &error); + bool const retried = status == sz_unexpected_dimensions_k && found > growable.query_ids.size(); + if (retried) { + growable.resize(found); + status = index_find(index, device, &query_sequence, bound, growable, &found, &error); + } + double const elapsed = + std::chrono::duration(std::chrono::steady_clock::now() - start).count(); + if (status != sz_success_k || found != required) { + std::cerr << "steady search failed: " << int(status) << " " << (error ? error : "") << '\n'; + index_free(index); + szs_device_scope_free(device); + return 6; + } + std::cout << "k=" << unsigned(bound) << " mode=steady_growable repeat=" << repeat + << " query=" << elapsed << "s matches=" << found << " retried=" << retried + << " threads=" << query_threads << '\n'; + } + + if (mode_enabled(modes, "latency")) { + std::vector latencies; + latencies.reserve(queries.size()); + sparse_matches_t service_matches; + service_matches.resize(8); + std::size_t service_capacity = 8; + sz_size_t service_matches_count = 0; + for (std::size_t query_index = 0; query_index != queries.size(); ++query_index) { + sz_string_view_t one_view = query_views[query_index]; + sz_sequence_t one_query; + sz_sequence_from_string_views(&one_view, 1, &one_query); + auto const start = std::chrono::steady_clock::now(); + sz_size_t found = 0; + sz_status_t status = index_find(index, device, &one_query, bound, service_matches, &found, + &error); + if (status == sz_unexpected_dimensions_k && found > service_capacity) { + service_capacity = found; + service_matches.resize(service_capacity); + status = index_find(index, device, &one_query, bound, service_matches, &found, &error); + } + double const elapsed = + std::chrono::duration(std::chrono::steady_clock::now() - start).count(); + if (status != sz_success_k) { + std::cerr << "latency search failed: " << int(status) << " " << (error ? error : "") << '\n'; + index_free(index); + szs_device_scope_free(device); + return 6; + } + latencies.push_back(elapsed); + service_matches_count += found; + } + if (service_matches_count != required) { + std::cerr << "latency search returned a different result count\n"; + index_free(index); + szs_device_scope_free(device); + return 6; + } + std::cout << "k=" << unsigned(bound) << " mode=single_query_latency samples=" << latencies.size() + << " p50=" << percentile(latencies, 0.50) << "s p95=" << percentile(latencies, 0.95) + << "s p99=" << percentile(latencies, 0.99) << "s matches=" << service_matches_count + << " final_capacity=" << service_capacity << '\n'; + } + if (!dump_prefix.empty()) { + std::string const path = dump_prefix + ".k" + std::to_string(bound) + ".bin"; + if (!dump_matches(dictionary.size(), queries.size(), bound, matches, path)) { + std::cerr << "dump failed: " << path << '\n'; + index_free(index); + szs_device_scope_free(device); + return 7; + } + } + } + index_free(index); + } + szs_device_scope_free(device); + if (cache_evict_bytes) std::cerr << "cache_evict_checksum=" << cache_evict_checksum << '\n'; + return 0; +} + +int main(int argc, char **argv) { + if (argc < 3 || argc > 5) { + std::cerr << "usage: levenshtein_index DICTIONARY QUERIES [QUERY_LIMIT] [DUMP_PREFIX]\n"; + return 2; + } + std::size_t const query_limit = argc >= 4 ? std::stoull(argv[3]) : 0; + std::string const dump_prefix = argc == 5 ? argv[4] : ""; + auto const dictionary = load_lines(argv[1]); + auto const queries = load_lines(argv[2], query_limit); + bool const utf8 = std::getenv("SZ_LEVENSHTEIN_UTF8") != nullptr; + std::string const modes = std::getenv("SZ_LEVENSHTEIN_MODES") ? std::getenv("SZ_LEVENSHTEIN_MODES") + : "warm,steady,latency"; + bool const shared_index = std::getenv("SZ_LEVENSHTEIN_INDEX_PLAN") && + std::string_view(std::getenv("SZ_LEVENSHTEIN_INDEX_PLAN")) == "shared"; + std::cout << "dictionary=" << dictionary.size() << " queries=" << queries.size() + << " semantics=" << (utf8 ? "utf8-codepoints" : "bytes") << '\n'; + + std::vector max_distances = {1, 2, 4}; + int const query_repeats = std::getenv("SZ_LEVENSHTEIN_REPEATS") + ? std::stoi(std::getenv("SZ_LEVENSHTEIN_REPEATS")) + : 3; + std::size_t const query_threads = std::getenv("SZ_LEVENSHTEIN_THREADS") + ? std::stoull(std::getenv("SZ_LEVENSHTEIN_THREADS")) + : 1; + std::size_t const batches_per_repeat = std::getenv("SZ_LEVENSHTEIN_BATCHES_PER_REPEAT") + ? std::stoull(std::getenv("SZ_LEVENSHTEIN_BATCHES_PER_REPEAT")) + : 1; + std::size_t const cache_evict_mb = std::getenv("SZ_LEVENSHTEIN_CACHE_EVICT_MB") + ? std::stoull(std::getenv("SZ_LEVENSHTEIN_CACHE_EVICT_MB")) + : 0; + if (query_repeats <= 0 || query_threads == 0 || batches_per_repeat == 0 || + cache_evict_mb > std::size_t(-1) / (1024 * 1024)) { + std::cerr << "repeat, thread, batch, or cache setting is invalid\n"; + return 2; + } + if (char const *requested_max = std::getenv("SZ_LEVENSHTEIN_MAX_DISTANCE")) { + int const parsed = std::stoi(requested_max); + if (parsed < 1 || parsed >= std::numeric_limits::max()) { + std::cerr << "SZ_LEVENSHTEIN_MAX_DISTANCE must be between 1 and 254\n"; + return 2; + } + max_distances = {static_cast(parsed)}; + } + if (shared_index && !std::getenv("SZ_LEVENSHTEIN_MAX_DISTANCE")) max_distances = {4}; + if (!mode_enabled(modes, "cold") && !mode_enabled(modes, "warm") && !mode_enabled(modes, "steady") && + !mode_enabled(modes, "latency")) { + std::cerr << "SZ_LEVENSHTEIN_MODES must include cold, warm, steady, or latency\n"; + return 2; + } + std::size_t const cache_evict_bytes = cache_evict_mb * 1024 * 1024; + return utf8 ? run(dictionary, queries, max_distances, dump_prefix, query_repeats, query_threads, + batches_per_repeat, cache_evict_bytes, modes, shared_index) + : run(dictionary, queries, max_distances, dump_prefix, query_repeats, query_threads, + batches_per_repeat, cache_evict_bytes, modes, shared_index); +}