diff --git a/CHANGELOG.md b/CHANGELOG.md index c3a3368..0211cc6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- **Opt-in `vecq` vector store for Brain Mode.** Set `brain.vector_store: vecq` in `.cora.yaml` to replace the usearch HNSW index with vecq 4-bit quantized scan (pure Rust, deterministic, ~6x smaller). Opt-in only; keyed persistence awaits vecq#32, so indexes rebuild fresh on load for now (#542). +- **Opt-in `vecq` vector store for Brain Mode.** Set `brain.vector_store: vecq` in `.cora.yaml` to replace the usearch HNSW index with a vecq quantized scan (pure Rust, deterministic, ~5x smaller). Keyed persistence included: symbol ids survive reload, so a fresh process serves the index as-is and `cora index` no longer re-embeds unchanged projects (#542, #547). +- **`brain.vector_bits` quantization-width knob.** `residual` (default) | `4` | `5` | `6` — 4-bit base codes with second-pass residual rescoring, or plain Lloyd-Max widths. The default is residual: best recall@10 at 4-bit scan speed in a recall study on cora's own embeddings, ahead of plain 5-bit at 1k/5k/13k-symbol scales. Changing the width rebuilds the index once on the next `cora index` instead of silently serving the old width; unknown values fall back to `residual`. + +### Fixed + +- **Vector signal never fired in a fresh process.** `cora brain` and MCP `brain_search` only saw the vector index if the same process had run the embed — otherwise results silently degraded to FTS-only. The search path now lazy-loads the on-disk index once per process, with a dimension guard against backend switches (#545). +- **Stale embed fingerprints after a global vector-index rebuild.** The vector index is a single file shared by all projects; rebuilding it (width/dims change, legacy file, corruption) wiped every project's vectors while their fingerprints still said "embedded" — the incremental path would skip those symbols forever. A rebuild now clears fingerprints for all projects, and the usearch dims-mismatch path (which deleted the index without clearing) joins the same heal. + +### Changed + +- **vecq-core dependency 0.2.0 → 0.3.0.** Picks up the 4-bit+residual mode, plain 5/6-bit widths, runtime-detected AVX2 scoring, and file formats v1.3–v1.5 with the keyed-slot table. Pre-0.3.0 `.vecq` files carry no key table and rebuild once with a warning, then upgrade to the keyed format. ## [0.14.0] - 2026-08-28 diff --git a/Cargo.lock b/Cargo.lock index 645686a..e9ee6a2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2399,9 +2399,9 @@ checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" [[package]] name = "vecq-core" -version = "0.2.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5f5e3c486177edd6c21108dfe621a7e9c85730c0dbb0ce76673e6682a68e90b" +checksum = "543ddd2e748c26c39bd8ca5ab9c43eb01ba0ca13750a1428b08e8c7dc0d9def7" [[package]] name = "version_check" diff --git a/Cargo.toml b/Cargo.toml index f1084c9..c77e731 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -72,7 +72,7 @@ rusqlite = { version = "0.31", features = ["bundled"] } # Vector search (Phase 3 — Brain Mode) usearch = "2" -vecq-core = "0.2.0" +vecq-core = "0.3.0" fs2 = "0.4" # Self-update (upgrade command) diff --git a/src/config/schema.rs b/src/config/schema.rs index 9d60c79..4a6e633 100644 --- a/src/config/schema.rs +++ b/src/config/schema.rs @@ -451,15 +451,29 @@ fn default_vector_store() -> String { "usearch".to_string() } -#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq)] +fn default_vector_bits() -> String { + "residual".to_string() +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct BrainConfig { /// Vector store backend for Brain Mode search. /// /// - `"usearch"` (default) — HNSW graph, f32 - /// - `"vecq"` — 4-bit quantized brute-force scan (pure Rust, deterministic, - /// ~6x smaller index; recall trade absorbed by RRF fusion) + /// - `"vecq"` — quantized brute-force scan (pure Rust, deterministic, + /// ~5x smaller index; recall trade absorbed by RRF fusion) #[serde(default = "default_vector_store")] pub vector_store: String, + /// Quantization width for the `vecq` store (ignored by `usearch`). + /// + /// - `"residual"` (default) — 4-bit base + second-pass residual codes: + /// best recall at 4-bit scan speed on cora's default embeddings + /// - `"4"` / `"5"` / `"6"` — plain Lloyd-Max width + /// + /// Invalid values fall back to `"residual"`. Changing the width rebuilds + /// the index once on the next `cora index`. + #[serde(default = "default_vector_bits")] + pub vector_bits: String, /// Embedding backend selection. /// /// - `"auto"` (default) — best available: pretrained → hashing @@ -471,6 +485,19 @@ pub struct BrainConfig { pub embedding: BrainEmbeddingMode, } +// Manual Default so `Config::default()` carries the documented values +// ("usearch"/"residual") — the derived one would leave empty strings, and +// the store/width parsers only meet those values leniently. +impl Default for BrainConfig { + fn default() -> Self { + Self { + vector_store: default_vector_store(), + vector_bits: default_vector_bits(), + embedding: BrainEmbeddingMode::default(), + } + } +} + /// `.cora.yaml` `brain:` section — mirrors the subset of [`BrainConfig`] /// that is meaningful in a config file. #[derive(Debug, Clone, Serialize, Deserialize, Default)] @@ -478,6 +505,8 @@ pub struct BrainSection { #[serde(skip_serializing_if = "Option::is_none")] pub vector_store: Option, #[serde(skip_serializing_if = "Option::is_none")] + pub vector_bits: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub embedding: Option, } @@ -795,6 +824,9 @@ impl CoraFile { if let Some(v) = &brain.vector_store { config.brain.vector_store.clone_from(v); } + if let Some(v) = &brain.vector_bits { + config.brain.vector_bits.clone_from(v); + } if let Some(v) = &brain.embedding { config.brain.embedding = v.clone(); } @@ -870,6 +902,48 @@ mod tests { assert_eq!(cfg.output.format, "pretty"); } + #[test] + fn brain_vector_bits_defaults_to_residual() { + let cfg = Config::default(); + assert_eq!(cfg.brain.vector_bits, "residual"); + assert_eq!(cfg.brain.vector_store, "usearch"); + } + + #[test] + fn merge_brain_vector_store_and_bits() { + let mut cfg = Config::default(); + let cora = CoraFile::from_str( + r" +brain: + vector_store: vecq + vector_bits: '6' + embedding: hashing +", + ) + .unwrap(); + + cora.merge_into(&mut cfg).unwrap(); + + assert_eq!(cfg.brain.vector_store, "vecq"); + assert_eq!(cfg.brain.vector_bits, "6"); + } + + #[test] + fn merge_brain_vector_bits_absent_keeps_default() { + let mut cfg = Config::default(); + let cora = CoraFile::from_str( + r" +brain: + vector_store: vecq +", + ) + .unwrap(); + + cora.merge_into(&mut cfg).unwrap(); + + assert_eq!(cfg.brain.vector_bits, "residual"); + } + #[test] fn merge_provider_overrides() { let mut cfg = Config::default(); diff --git a/src/index/brain.rs b/src/index/brain.rs index b9746f8..de88f0f 100644 --- a/src/index/brain.rs +++ b/src/index/brain.rs @@ -62,17 +62,20 @@ fn vector_index_path() -> std::path::PathBuf { /// The index file contains usearch binary data — we can't easily read the /// dimensionality without loading it. Instead, we check the `embedding_dims` /// column in the projects table, which is written during embedding. -fn check_dimension_compat(vi_path: &std::path::Path, expected_dims: usize) { +fn check_dimension_compat(vi_path: &std::path::Path, expected_dims: usize) -> bool { // Attempt to load the index just to check its dimensions. // If it fails (empty, corrupt, etc.), we'll create fresh — no warning needed. + // Returns true when a stale index was removed: the vector index is a + // single file shared by every project, so its fingerprints are now all + // stale and callers must clear them. let Ok(file) = std::fs::File::open(vi_path) else { - return; + return false; }; let Ok(metadata) = file.metadata() else { - return; + return false; }; if metadata.len() == 0 { - return; // Empty file — will create fresh + return false; // Empty file — will create fresh } // Try to load and check dims @@ -102,18 +105,22 @@ fn check_dimension_compat(vi_path: &std::path::Path, expected_dims: usize) { // Delete the stale index so embed_project creates a fresh one if let Err(e) = std::fs::remove_file(vi_path) { tracing::warn!(" Failed to remove stale index: {e}"); + false } else { let keys_path = vi_path.with_extension("keys"); let _ = std::fs::remove_file(&keys_path); tracing::info!(" Removed stale vector index — will create fresh on next index"); + true } } Ok(Some(_)) => { // Dimensions match — all good + false } _ => { // Couldn't read dimensions (corrupt, unsupported, etc.) // embed_project will handle creation + false } } } @@ -140,9 +147,11 @@ pub fn embed_project(conn: &Connection, project_id: i64) -> Result { // Check for existing index with mismatched dimensions (e.g. user rebuilt // with/without pretrained-embed feature). Warn but continue — the index // will be corrupted for searches until re-indexed. - if vi_path.exists() { - check_dimension_compat(&vi_path, active); - } + let dims_stale = if vi_path.exists() { + check_dimension_compat(&vi_path, active) + } else { + false + }; // Acquire write lock — block searches while embedding. // This is fine because embedding only happens during `cora index`, @@ -158,19 +167,21 @@ pub fn embed_project(conn: &Connection, project_id: i64) -> Result { cache.as_mut().unwrap() }; - // ── Self-heal after vecq reload-discard (#542, vecq#32) ────────── - // A vecq index reloaded from disk is rebuilt empty (upstream lacks key - // serialization) and arrives dirty. Stored fingerprints would skip all - // unchanged symbols, permanently excluding them from vector search. - // Clear fingerprints so this run re-embeds every symbol in the project. - if vi.is_dirty() && vi.is_empty() { - let cleared = conn.execute( - "UPDATE symbols SET embed_fingerprint = NULL WHERE project_id = ?1", - rusqlite::params![project_id], - )?; + // ── Self-heal after a global index rebuild (#542, vecq#32) ─────── + // The vector index is one shared file across projects. A vecq reload + // that had to rebuild it (legacy pre-0.3.0 file, width change, dim + // change, corruption) or a stale usearch index removed for a dims + // mismatch leaves every project's vectors gone while their stored + // fingerprints still say "embedded" — the incremental path would skip + // those symbols forever. Clear fingerprints for ALL projects: the + // current project re-embeds in this run; every other project re-embeds + // on its next `cora index` run that reaches embed_project + // (cross-project scheduling: #545). + if dims_stale || (vi.is_dirty() && vi.is_empty()) { + let cleared = conn.execute("UPDATE symbols SET embed_fingerprint = NULL", [])?; tracing::info!( cleared, - "vector index rebuilt from empty (vecq reload) — re-embedding all symbols" + "vector index rebuilt from empty — all projects re-embed on their next index run" ); } @@ -372,17 +383,17 @@ fn fts5_search(conn: &Connection, project_id: i64, query: &str, limit: usize) -> } /// True when the on-disk vector index exists but will be discarded on reload — -/// i.e. vecq backend with a non-empty `.vecq` file that cannot restore its -/// key map (vecq#32). `cora index` uses this to force a re-embed even when -/// no files changed, so the vector signal heals instead of staying empty. +/// i.e. vecq backend whose `.vecq` file predates keyed persistence (vecq#32), +/// is unreadable, or was built with different dims. `cora index` uses this to +/// force a re-embed even when no files changed, so the vector signal heals +/// instead of staying empty. Healthy keyed files (vecq-core 0.3.0+) reload +/// as-is and return false. pub fn vector_index_needs_rebuild() -> bool { if crate::index::vector::current_vector_store() != crate::index::vector::VectorStoreKind::Vecq { return false; } let path = vector_index_path().with_extension("vecq"); - std::fs::metadata(&path) - .map(|m| m.len() > 24) - .unwrap_or(false) + path.exists() && crate::index::vector::vecq_file_needs_rebuild(&path, active_dims()) } /// Return a read guard over the vector index cache, loading the on-disk diff --git a/src/index/mod.rs b/src/index/mod.rs index 1ef8e99..32b520e 100644 --- a/src/index/mod.rs +++ b/src/index/mod.rs @@ -532,8 +532,8 @@ fn index_project_with_id( ); // Embed symbols into vector index for brain search — when files changed, - // or when the stored index was discarded on reload (vecq #542: a fresh - // process rebuilds the vecq index empty; if all files are unchanged this + // or when the stored index cannot serve keyed search after reload + // (legacy pre-0.3.0 vecq file, vecq #542: if all files are unchanged this // would be the only chance to re-embed — without it the vector signal // stays dead until a file actually changes). if stats.files_indexed > 0 || brain::vector_index_needs_rebuild() { diff --git a/src/index/vector.rs b/src/index/vector.rs index 51ff239..4d6e3df 100644 --- a/src/index/vector.rs +++ b/src/index/vector.rs @@ -25,7 +25,7 @@ pub enum VectorStoreKind { /// HNSW graph over f32 vectors (the historical default). #[default] Usearch, - /// vecq 4-bit quantized scan — pure Rust, deterministic, ~6x smaller. + /// vecq quantized scan — pure Rust, deterministic, ~5x smaller. Vecq, } @@ -38,9 +38,70 @@ impl VectorStoreKind { } } +/// vecq quantization width (`brain.vector_bits` in `.cora.yaml`). +/// +/// Default is 4-bit + residual rescore: on cora's default hashing-trick +/// embeddings it measured the best recall@10 at 4-bit scan speed across +/// 1k/5k/13k-symbol indexes (recall study, 2026-08) — plain 5-bit, the +/// vecq-core default, trailed by ~4-7 points at every scale. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum VectorBits { + /// 4-bit base codes + second-pass residual rescoring (format v1.4). + #[default] + Residual, + /// Plain 4-bit Lloyd-Max (max compression, lowest recall). + B4, + /// Plain 5-bit Lloyd-Max (vecq-core's out-of-the-box default). + B5, + /// Plain 6-bit Lloyd-Max (highest plain recall, slowest scan). + B6, +} + +impl VectorBits { + /// Lenient like [`VectorStoreKind::parse`]: empty/unknown → Residual. + pub fn parse(s: &str) -> Self { + match s.trim().to_lowercase().as_str() { + "4" => Self::B4, + "5" => Self::B5, + "6" => Self::B6, + _ => Self::Residual, // "residual", "" and anything unknown + } + } + + fn create(&self, dims: usize) -> vecq_core::VecqIndex { + match self { + Self::Residual => vecq_core::VecqIndex::with_residual(dims, VECQ_SEED), + Self::B4 => Self::create_plain(dims, 4), + Self::B5 => Self::create_plain(dims, 5), + Self::B6 => Self::create_plain(dims, 6), + } + } + + fn create_plain(dims: usize, bits: u8) -> vecq_core::VecqIndex { + let mut index = vecq_core::VecqIndex::new(dims, VECQ_SEED); + index.set_bits(bits); + index + } + + /// Whether an on-disk index was built at this width. A mismatch triggers + /// one rebuild so a config change actually takes effect instead of + /// silently serving the old width forever. + fn matches(&self, index: &vecq_core::VecqIndex) -> bool { + match self { + Self::Residual => index.is_residual(), + Self::B4 => !index.is_residual() && index.bits() == 4, + Self::B5 => !index.is_residual() && index.bits() == 5, + Self::B6 => !index.is_residual() && index.bits() == 6, + } + } +} + static VECTOR_STORE: LazyLock> = LazyLock::new(|| std::sync::RwLock::new(VectorStoreKind::Usearch)); +static VECTOR_BITS: LazyLock> = + LazyLock::new(|| std::sync::RwLock::new(VectorBits::default())); + /// Select the physical vector store used by NEW indexes (process-wide). /// Existing on-disk indexes keep their own format via file extension. pub fn set_vector_store(kind: VectorStoreKind) { @@ -51,14 +112,30 @@ pub fn current_vector_store() -> VectorStoreKind { *VECTOR_STORE.read().unwrap() } -/// Apply `brain.vector_store` from a loaded config (any process entry that -/// touches Brain search/embedding must call this before opening the index, -/// or `.vecq`/`.usearch` files silently diverge between runs). +/// Select the vecq quantization width used by NEW indexes (process-wide). +pub fn set_vector_bits(bits: VectorBits) { + *VECTOR_BITS.write().unwrap() = bits; +} + +pub fn current_vector_bits() -> VectorBits { + *VECTOR_BITS.read().unwrap() +} + +/// Apply `brain.vector_store` + `brain.vector_bits` from a loaded config (any +/// process entry that touches Brain search/embedding must call this before +/// opening the index, or `.vecq`/`.usearch` files silently diverge between +/// runs). pub fn apply_config_store(config: Option<&crate::config::schema::Config>) { - let kind = config - .map(|c| VectorStoreKind::parse(&c.brain.vector_store)) + let (kind, bits) = config + .map(|c| { + ( + VectorStoreKind::parse(&c.brain.vector_store), + VectorBits::parse(&c.brain.vector_bits), + ) + }) .unwrap_or_default(); set_vector_store(kind); + set_vector_bits(bits); } /// Hashing-trick embedding dimensions (zero-dependency fallback). @@ -113,9 +190,7 @@ impl CodeVectorIndex { symbol_to_key: HashMap::new(), next_key: 0, }, - VectorStoreKind::Vecq => { - Inner::Vecq(Box::new(vecq_core::VecqIndex::new(dims, VECQ_SEED))) - } + VectorStoreKind::Vecq => Inner::Vecq(Box::new(current_vector_bits().create(dims))), }, path: None, dirty: false, @@ -161,7 +236,6 @@ impl CodeVectorIndex { let mut lock_file = acquire_file_lock(path)?; let mut buffer = Vec::new(); - let mut dirty = false; if lock_file.metadata().context("read vecq metadata")?.len() > 0 { use std::io::{Read, Seek, SeekFrom}; lock_file @@ -172,21 +246,57 @@ impl CodeVectorIndex { .context("read vecq file")?; } - // KNOWN LIMITATION (#542): vecq-core 0.2.0 `from_bytes` restores codes - // and scales but NOT the keyed map (see codecoradev/vecq#32), so a - // reloaded index would silently search empty. Until key serialization - // ships upstream, reload always rebuilds fresh. To keep the next - // `cora index` honest, mark the index dirty here — `embed_project` - // will then clear embed fingerprints and re-embed everything instead - // of skipping unchanged symbols against an empty index. - if buffer.len() >= 24 { - tracing::warn!( - "vecq persistence lacks key serialization upstream (vecq#32) — \ - recreating index; `cora index` will re-embed all symbols" - ); - dirty = true; + let bits = current_vector_bits(); + let mut index = bits.create(dims); + let mut dirty = false; + if !buffer.is_empty() { + match vecq_core::VecqIndex::from_bytes(&buffer) { + Ok(loaded) + if loaded.dim() == dims && bits.matches(&loaded) && vecq_has_keys(&loaded) => + { + index = loaded; + } + // Dim mismatch (e.g. rebuilt with/without pretrained-embed): + // keeping the loaded dims would panic on the next insert. + Ok(loaded) if loaded.dim() != dims => { + tracing::warn!( + "vecq index dims {} != current {dims} — recreating; \ + `cora index` will re-embed all symbols", + loaded.dim() + ); + dirty = true; + } + Ok(loaded) if !bits.matches(&loaded) => { + tracing::warn!( + "vecq index width ({}) != configured brain.vector_bits ({bits:?}) — \ + recreating; `cora index` will re-embed all symbols", + if loaded.is_residual() { + "residual".to_string() + } else { + format!("{}-bit", loaded.bits()) + } + ); + dirty = true; + } + // Legacy vecq-core 0.2.x file (vecq#32): parses fine but + // carries no keyed-slot table, and cora's symbol ids live in + // the key map — searching it would silently return nothing. + Ok(_) => { + tracing::warn!( + "vecq file predates keyed persistence (vecq#32) — \ + recreating index; `cora index` will re-embed all symbols" + ); + dirty = true; + } + Err(e) => { + tracing::warn!( + "vecq index unreadable ({e}) — recreating; \ + `cora index` will re-embed all symbols" + ); + dirty = true; + } + } } - let index = vecq_core::VecqIndex::new(dims, VECQ_SEED); Ok(Self { inner: Inner::Vecq(Box::new(index)), @@ -446,6 +556,37 @@ fn create_usearch_index(dims: usize) -> Result { Index::new(&options).context("create usearch index") } +/// cora only inserts via `add_keyed`, so every live slot of a healthy index +/// carries a key. Files written by vecq-core 0.2.x (before format v1.3) +/// load with vectors but no key table — searching them would silently +/// return nothing (vecq#32). +fn vecq_has_keys(index: &vecq_core::VecqIndex) -> bool { + index.is_empty() || (0..index.slots()).any(|s| index.key_of(s).is_some()) +} + +/// True when the `.vecq` file exists but cannot serve keyed search after +/// reload — legacy vecq#32 file, unreadable, built with different dims, or +/// at a different width than the configured `brain.vector_bits`. Mirrors +/// exactly what [`CodeVectorIndex::load_or_create`] will decide, so +/// `cora index` can force a re-embed when the index is about to be rebuilt. +pub fn vecq_file_needs_rebuild(path: &Path, dims: usize) -> bool { + let bytes = match std::fs::read(path) { + Ok(b) => b, + Err(_) => return false, // no file / unreadable — nothing to rebuild + }; + if bytes.is_empty() { + return false; + } + match vecq_core::VecqIndex::from_bytes(&bytes) { + Ok(loaded) => { + loaded.dim() != dims + || !current_vector_bits().matches(&loaded) + || !vecq_has_keys(&loaded) + } + Err(_) => true, + } +} + /// Convert cosine distance (0..2) to cosine similarity (0..1). pub fn cosine_distance_to_similarity(distance: f32) -> f32 { (1.0 - distance).clamp(0.0, 1.0) @@ -591,52 +732,151 @@ mod tests { } #[test] - fn test_vecq_backend_roundtrip() { - let _g = with_store_lock(); - set_vector_store(VectorStoreKind::Vecq); - let mut idx = CodeVectorIndex::new(64).unwrap(); + fn test_vecq_backend_roundtrip_all_widths() { + for bits in [ + VectorBits::Residual, + VectorBits::B4, + VectorBits::B5, + VectorBits::B6, + ] { + let _g = with_store_lock(); + set_vector_store(VectorStoreKind::Vecq); + set_vector_bits(bits); + let mut idx = CodeVectorIndex::new(64).unwrap(); - idx.insert(1, &make_unit_vec(64, 0)).unwrap(); - idx.insert(2, &make_unit_vec(64, 1)).unwrap(); - assert_eq!(idx.len(), 2); + idx.insert(1, &make_unit_vec(64, 0)).unwrap(); + idx.insert(2, &make_unit_vec(64, 1)).unwrap(); + assert_eq!(idx.len(), 2); - // Replace semantics - idx.insert(1, &make_unit_vec(64, 2)).unwrap(); - assert_eq!(idx.len(), 2); + // Replace semantics + idx.insert(1, &make_unit_vec(64, 2)).unwrap(); + assert_eq!(idx.len(), 2); - // Removal - assert!(idx.remove(2)); - assert!(!idx.remove(2)); - assert_eq!(idx.len(), 1); + // Removal + assert!(idx.remove(2)); + assert!(!idx.remove(2)); + assert_eq!(idx.len(), 1); - let results = idx.search(&make_unit_vec(64, 2), 5); - assert_eq!(results.len(), 1); - assert_eq!(results[0].0, 1); - // distance conversion: similarity ~1.0 -> distance ~0.0 - assert!(results[0].1 < 0.05, "dist={:?}", results[0].1); + let results = idx.search(&make_unit_vec(64, 2), 5); + assert_eq!(results.len(), 1); + assert_eq!(results[0].0, 1); + // distance conversion: similarity ~1.0 -> distance ~0.0 + assert!(results[0].1 < 0.05, "dist={:?}", results[0].1); - set_vector_store(VectorStoreKind::Usearch); + set_vector_bits(VectorBits::default()); + set_vector_store(VectorStoreKind::Usearch); + } } #[test] - fn test_vecq_save_and_load_uses_own_extension() { + fn test_vecq_save_and_load_restores_keys() { let _g = with_store_lock(); set_vector_store(VectorStoreKind::Vecq); + set_vector_bits(VectorBits::default()); // residual let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("test.usearch"); // caller names it; backend picks .vecq { let mut idx = CodeVectorIndex::new(64).unwrap(); idx.insert(10, &make_unit_vec(64, 0)).unwrap(); + idx.insert(20, &make_unit_vec(64, 1)).unwrap(); idx.path = Some(path.clone()); idx.save().unwrap(); } assert!(dir.path().join("test.vecq").exists()); - // Upstream gap (vecq#32): keys are not serialized, so reload rebuilds - // fresh instead of serving a silently-empty index. + // vecq-core 0.3.0 keyed persistence (vecq#32 closed): symbol-id keys + // survive reload, so a fresh process serves the index as-is. let idx2 = CodeVectorIndex::load_or_create(&path, 64).unwrap(); - assert!(idx2.is_empty(), "reload must not serve a keyless index"); + assert_eq!(idx2.len(), 2, "keyed map must survive reload"); + assert!(!idx2.is_dirty(), "healthy keyed file loads clean"); + + let results = idx2.search(&make_unit_vec(64, 0), 1); + assert_eq!(results[0].0, 10); + set_vector_store(VectorStoreKind::Usearch); + } + + #[test] + fn test_vecq_legacy_keyless_file_rebuilds() { + let _g = with_store_lock(); + set_vector_store(VectorStoreKind::Vecq); + // The simulated legacy file is plain 5-bit; match that width so the + // load hits the keyless arm rather than the width-mismatch arm. + set_vector_bits(VectorBits::B5); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("test.usearch"); + let vecq_path = dir.path().join("test.vecq"); + + // Simulate a vecq-core 0.2.x file (vecq#32): vectors serialized + // without a keyed-slot table, which is what unkeyed `add()` produces. + let mut legacy = vecq_core::VecqIndex::new(64, VECQ_SEED); + legacy.add(&make_unit_vec(64, 0)); + std::fs::write(&vecq_path, legacy.to_bytes()).unwrap(); + + let idx = CodeVectorIndex::load_or_create(&path, 64).unwrap(); + assert!(idx.is_empty(), "keyless file must not be served"); + assert!(idx.is_dirty(), "keyless file must trigger a rebuild"); + set_vector_bits(VectorBits::default()); + set_vector_store(VectorStoreKind::Usearch); + } + + #[test] + fn test_vecq_width_switch_rebuilds() { + let _g = with_store_lock(); + set_vector_store(VectorStoreKind::Vecq); + set_vector_bits(VectorBits::default()); // residual + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("test.usearch"); + + { + let mut idx = CodeVectorIndex::new(64).unwrap(); + idx.insert(10, &make_unit_vec(64, 0)).unwrap(); + idx.path = Some(path.clone()); + idx.save().unwrap(); + } + + // A healthy keyed index must NOT be kept when the configured width + // changed — the rebuild makes brain.vector_bits actually take effect. + set_vector_bits(VectorBits::B5); + let idx = CodeVectorIndex::load_or_create(&path, 64).unwrap(); + assert!(idx.is_empty(), "width-mismatched file must not be served"); + assert!(idx.is_dirty(), "width change must trigger a rebuild"); + set_vector_bits(VectorBits::default()); + set_vector_store(VectorStoreKind::Usearch); + } + + #[test] + fn test_vecq_corrupt_file_rebuilds() { + let _g = with_store_lock(); + set_vector_store(VectorStoreKind::Vecq); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("test.usearch"); + std::fs::write(dir.path().join("test.vecq"), vec![0u8; 64]).unwrap(); + + let idx = CodeVectorIndex::load_or_create(&path, 64).unwrap(); + assert!(idx.is_empty()); + assert!(idx.is_dirty()); + set_vector_store(VectorStoreKind::Usearch); + } + + #[test] + fn test_vecq_dim_mismatch_rebuilds() { + let _g = with_store_lock(); + set_vector_store(VectorStoreKind::Vecq); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("test.usearch"); + + { + let mut idx = CodeVectorIndex::new(64).unwrap(); + idx.insert(10, &make_unit_vec(64, 0)).unwrap(); + idx.path = Some(path.clone()); + idx.save().unwrap(); + } + + let idx = CodeVectorIndex::load_or_create(&path, 128).unwrap(); + assert!(idx.is_empty()); + assert!(idx.is_dirty()); + assert_eq!(idx.dims(), 128); set_vector_store(VectorStoreKind::Usearch); } @@ -649,6 +889,17 @@ mod tests { assert_eq!(VectorStoreKind::parse("bogus"), VectorStoreKind::Usearch); } + #[test] + fn vector_bits_parse_is_lenient() { + assert_eq!(VectorBits::parse("residual"), VectorBits::Residual); + assert_eq!(VectorBits::parse(" RESIDUAL "), VectorBits::Residual); + assert_eq!(VectorBits::parse("4"), VectorBits::B4); + assert_eq!(VectorBits::parse("5"), VectorBits::B5); + assert_eq!(VectorBits::parse("6"), VectorBits::B6); + assert_eq!(VectorBits::parse(""), VectorBits::Residual); + assert_eq!(VectorBits::parse("bogus"), VectorBits::Residual); + } + #[test] fn test_cosine_distance_to_similarity() { assert!((cosine_distance_to_similarity(0.0) - 1.0).abs() < f32::EPSILON);