From 0bfd003c1abb37bb0420a0cf7fb9ec33d4f63c04 Mon Sep 17 00:00:00 2001 From: ajianaz Date: Fri, 28 Aug 2026 14:14:30 +0700 Subject: [PATCH 1/2] feat(brain): opt-in vecq vector store backend MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add `brain.vector_store: usearch|vecq` (default usearch). vecq-core 0.2.0 replaces the HNSW store with a 4-bit quantized brute-force scan: pure Rust (no C++ FFI), bit-identical deterministic results, ~6x smaller index, 14x faster builds — query latency stays sub-ms at cora's typical index sizes and the recall trade is absorbed by Brain's RRF fusion (#542). - CodeVectorIndex gains an enum backend (Usearch | Vecq); every backend owns its file extension (.usearch / .vecq) so switching stores never misreads the other's format - config applied at every Brain entry point (index, watch, brain cmd, MCP brain_search) so CLI and MCP cannot silently diverge - vecq insert/remove map symbol ids to native u64 keys (tombstones); search converts similarity to the cosine-distance contract Known limitation (documented + tested): vecq-core 0.2.0 from_bytes does not serialize the keyed map (vecq#32), so reload rebuilds fresh instead of serving a silently-empty index; `cora index` re-embeds. Persistence lands when vecq ships key serialization. Regression tests: backend roundtrip (insert/replace/remove/search + distance conversion), per-backend persistence extensions, lenient store-kind parsing. Signed-off-by: ajianaz --- CHANGELOG.md | 4 + Cargo.lock | 7 + Cargo.toml | 1 + src/commands/watch.rs | 1 + src/config/schema.rs | 11 ++ src/index/vector.rs | 355 ++++++++++++++++++++++++++++++++++++------ src/main.rs | 11 ++ src/mcp/tools.rs | 1 + 8 files changed, 345 insertions(+), 46 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4679547..c3a3368 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- **Opt-in `vecq` vector store for Brain Mode.** Set `brain.vector_store: vecq` in `.cora.yaml` to replace the usearch HNSW index with vecq 4-bit quantized scan (pure Rust, deterministic, ~6x smaller). Opt-in only; keyed persistence awaits vecq#32, so indexes rebuild fresh on load for now (#542). + ## [0.14.0] - 2026-08-28 ### Fixed diff --git a/Cargo.lock b/Cargo.lock index cf0a2d3..645686a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -350,6 +350,7 @@ dependencies = [ "tree-sitter-scala", "tree-sitter-typescript", "usearch", + "vecq-core", "which", ] @@ -2396,6 +2397,12 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +[[package]] +name = "vecq-core" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5f5e3c486177edd6c21108dfe621a7e9c85730c0dbb0ce76673e6682a68e90b" + [[package]] name = "version_check" version = "0.9.5" diff --git a/Cargo.toml b/Cargo.toml index 4623ae6..f1084c9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -72,6 +72,7 @@ rusqlite = { version = "0.31", features = ["bundled"] } # Vector search (Phase 3 — Brain Mode) usearch = "2" +vecq-core = "0.2.0" fs2 = "0.4" # Self-update (upgrade command) diff --git a/src/commands/watch.rs b/src/commands/watch.rs index 0d4c0da..c18920e 100644 --- a/src/commands/watch.rs +++ b/src/commands/watch.rs @@ -51,6 +51,7 @@ pub fn run_watch( .map(|c| c.brain.embedding.to_string()) .unwrap_or_else(|| "auto".to_string()); crate::embed::resolve_backend(&brain_mode); + crate::index::vector::apply_config_store(config.as_ref()); let skip_ref: Option<&[String]> = skip_patterns.as_deref(); diff --git a/src/config/schema.rs b/src/config/schema.rs index 56bdd67..bbd481b 100644 --- a/src/config/schema.rs +++ b/src/config/schema.rs @@ -445,8 +445,19 @@ pub struct AnalysisConfig { /// By default (`auto`), cora selects the best available backend at runtime: /// pretrained 768d (if compiled with `pretrained-embed` feature) → hashing 256d fallback. /// Users can force a specific backend via `.cora.yaml`. +fn default_vector_store() -> String { + "usearch".to_string() +} + #[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq)] pub struct BrainConfig { + /// Vector store backend for Brain Mode search. + /// + /// - `"usearch"` (default) — HNSW graph, f32 + /// - `"vecq"` — 4-bit quantized brute-force scan (pure Rust, deterministic, + /// ~6x smaller index; recall trade absorbed by RRF fusion) + #[serde(default = "default_vector_store")] + pub vector_store: String, /// Embedding backend selection. /// /// - `"auto"` (default) — best available: pretrained → hashing diff --git a/src/index/vector.rs b/src/index/vector.rs index 24448cb..401a99c 100644 --- a/src/index/vector.rs +++ b/src/index/vector.rs @@ -12,9 +12,54 @@ use fs2::FileExt; use std::collections::HashMap; use std::fs::File; use std::path::{Path, PathBuf}; +use std::sync::LazyLock; use usearch::{Index, IndexOptions, MetricKind, ScalarKind}; const USEARCH_EXT: &str = "usearch"; +const VECQ_EXT: &str = "vecq"; +const VECQ_SEED: u64 = 42; + +/// Which physical vector store backs `CodeVectorIndex` (#542). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum VectorStoreKind { + /// HNSW graph over f32 vectors (the historical default). + #[default] + Usearch, + /// vecq 4-bit quantized scan — pure Rust, deterministic, ~6x smaller. + Vecq, +} + +impl VectorStoreKind { + pub fn parse(s: &str) -> Self { + match s.trim().to_lowercase().as_str() { + "vecq" => Self::Vecq, + _ => Self::Usearch, // "usearch", "" and anything unknown + } + } +} + +static VECTOR_STORE: LazyLock> = + LazyLock::new(|| std::sync::RwLock::new(VectorStoreKind::Usearch)); + +/// Select the physical vector store used by NEW indexes (process-wide). +/// Existing on-disk indexes keep their own format via file extension. +pub fn set_vector_store(kind: VectorStoreKind) { + *VECTOR_STORE.write().unwrap() = kind; +} + +pub fn current_vector_store() -> VectorStoreKind { + *VECTOR_STORE.read().unwrap() +} + +/// Apply `brain.vector_store` from a loaded config (any process entry that +/// touches Brain search/embedding must call this before opening the index, +/// or `.vecq`/`.usearch` files silently diverge between runs). +pub fn apply_config_store(config: Option<&crate::config::schema::Config>) { + let kind = config + .map(|c| VectorStoreKind::parse(&c.brain.vector_store)) + .unwrap_or_default(); + set_vector_store(kind); +} /// Hashing-trick embedding dimensions (zero-dependency fallback). pub const FALLBACK_DIMS: usize = 256; @@ -36,26 +81,42 @@ pub const DEFAULT_DIMS: usize = FALLBACK_DIMS; /// - Disk persistence via buffer-based serialization (same as uteke) /// - Cross-process safety via exclusive file lock pub struct CodeVectorIndex { - index: Index, - /// Maps usearch integer key → symbol database ID. - key_to_symbol: HashMap, - /// Maps symbol database ID → usearch integer key. - symbol_to_key: HashMap, - next_key: u64, + inner: Inner, path: Option, dirty: bool, _lock_file: Option, } +enum Inner { + Usearch { + index: Index, + /// Maps usearch integer key → symbol database ID. + key_to_symbol: HashMap, + /// Maps symbol database ID → usearch integer key. + symbol_to_key: HashMap, + next_key: u64, + }, + /// vecq keys are u64 natively (tombstone removal + compact), so symbol + /// IDs map directly — no sidecar needed. + Vecq(Box), +} + impl CodeVectorIndex { - /// Create a new empty in-memory index. + /// Create a new empty in-memory index (backend selected by + /// [`current_vector_store`]). pub fn new(dims: usize) -> Result { - let index = create_usearch_index(dims)?; Ok(Self { - index, - key_to_symbol: HashMap::new(), - symbol_to_key: HashMap::new(), - next_key: 0, + inner: match current_vector_store() { + VectorStoreKind::Usearch => Inner::Usearch { + index: create_usearch_index(dims)?, + key_to_symbol: HashMap::new(), + symbol_to_key: HashMap::new(), + next_key: 0, + }, + VectorStoreKind::Vecq => { + Inner::Vecq(Box::new(vecq_core::VecqIndex::new(dims, VECQ_SEED))) + } + }, path: None, dirty: false, _lock_file: None, @@ -64,7 +125,19 @@ impl CodeVectorIndex { /// Load from disk, or create empty if not exists. /// Acquires exclusive file lock for cross-process safety. + /// + /// Each backend owns its own file extension (`.usearch` vs `.vecq`), so + /// switching `brain.vector_store` never misreads the other's format. pub fn load_or_create(path: &Path, dims: usize) -> Result { + match current_vector_store() { + VectorStoreKind::Vecq => { + Self::load_or_create_vecq(&path.with_extension(VECQ_EXT), dims) + } + VectorStoreKind::Usearch => Self::load_or_create_usearch(path, dims), + } + } + + fn load_or_create_usearch(path: &Path, dims: usize) -> Result { if !path.exists() { std::fs::write(path, []).context("create usearch file")?; } @@ -81,6 +154,44 @@ impl CodeVectorIndex { Ok(idx) } + fn load_or_create_vecq(path: &Path, dims: usize) -> Result { + if !path.exists() { + std::fs::write(path, []).context("create vecq file")?; + } + let mut lock_file = acquire_file_lock(path)?; + + let mut buffer = Vec::new(); + if lock_file.metadata().context("read vecq metadata")?.len() > 0 { + use std::io::{Read, Seek, SeekFrom}; + lock_file + .seek(SeekFrom::Start(0)) + .context("seek vecq file")?; + lock_file + .read_to_end(&mut buffer) + .context("read vecq file")?; + } + + // KNOWN LIMITATION (#542): vecq-core 0.2.0 `from_bytes` restores codes + // and scales but NOT the keyed map (see codecoradev/vecq#32), so a + // reloaded index would silently search empty. Until key serialization + // ships upstream, reload always rebuilds fresh — safe (never + // mis-searches); `cora index` re-embeds on the next run. + if buffer.len() >= 24 { + tracing::warn!( + "vecq persistence lacks key serialization upstream (vecq#32) — recreating index; run `cora index` to re-embed symbols" + ); + } + let index = vecq_core::VecqIndex::new(dims, VECQ_SEED); + + Ok(Self { + inner: Inner::Vecq(Box::new(index)), + path: Some(path.to_path_buf()), + dirty: false, + _lock_file: Some(lock_file), + }) + } + + #[allow(clippy::too_many_lines)] fn load_from_file(file: &mut File, path: &Path) -> Result { use std::io::{Read, Seek, SeekFrom}; @@ -115,10 +226,12 @@ impl CodeVectorIndex { } Ok(Self { - index, - key_to_symbol, - symbol_to_key, - next_key, + inner: Inner::Usearch { + index, + key_to_symbol, + symbol_to_key, + next_key, + }, path: None, dirty: false, _lock_file: None, @@ -127,10 +240,31 @@ impl CodeVectorIndex { /// Save index and key mappings to disk. pub fn save(&mut self) -> Result<()> { + if let Inner::Vecq(index) = &mut self.inner { + let path = self + .path + .as_ref() + .context("vecq index has no path")? + .with_extension(VECQ_EXT); + let buffer = index.to_bytes(); + let tmp_path = path.with_extension(format!("{VECQ_EXT}.tmp")); + std::fs::write(&tmp_path, &buffer).context("write temp vecq index")?; + std::fs::rename(&tmp_path, path).context("rename temp vecq")?; + self.dirty = false; + return Ok(()); + } if let Some(ref path) = self.path { - let buf_len = self.index.serialized_length(); + let Inner::Usearch { + index, + key_to_symbol, + .. + } = &self.inner + else { + unreachable!("vecq path returned earlier"); + }; + let buf_len = index.serialized_length(); let mut buffer = vec![0u8; buf_len]; - self.index + index .save_to_buffer(&mut buffer) .context("save usearch to buffer")?; @@ -141,7 +275,7 @@ impl CodeVectorIndex { // Save key mapping sidecar let mapping_path = path.with_extension("keys"); let mut lines = Vec::new(); - for (&key, &sym_id) in &self.key_to_symbol { + for (&key, &sym_id) in key_to_symbol { lines.push(format!("{key}\t{sym_id}")); } atomic_write(&mapping_path, lines.join("\n").as_bytes())?; @@ -153,30 +287,43 @@ impl CodeVectorIndex { /// Insert a symbol embedding. If symbol ID exists, replaces it. pub fn insert(&mut self, symbol_id: i64, embedding: &[f32]) -> Result<()> { + if let Inner::Vecq(index) = &mut self.inner { + let key = symbol_id as u64; + if index.contains_key(key) { + index.remove_keyed(key); + } + index.add_keyed(key, embedding); + self.dirty = true; + return Ok(()); + } + let Inner::Usearch { + index, + key_to_symbol, + symbol_to_key, + next_key, + } = &mut self.inner + else { + unreachable!("usearch insert path guarded above"); + }; + // Remove old entry if exists - if let Some(&old_key) = self.symbol_to_key.get(&symbol_id) { - self.key_to_symbol.remove(&old_key); - self.index - .remove(old_key) - .context("remove old usearch entry")?; + if let Some(&old_key) = symbol_to_key.get(&symbol_id) { + key_to_symbol.remove(&old_key); + index.remove(old_key).context("remove old usearch entry")?; } - let key = self.next_key; - self.next_key += 1; - self.key_to_symbol.insert(key, symbol_id); - self.symbol_to_key.insert(symbol_id, key); + let key = *next_key; + *next_key += 1; + key_to_symbol.insert(key, symbol_id); + symbol_to_key.insert(symbol_id, key); // Auto-reserve if at capacity - if self.index.size() >= self.index.capacity() { - let new_cap = (self.index.capacity() + 1024).max(1024); - self.index - .reserve(new_cap) - .context("reserve usearch capacity")?; + if index.size() >= index.capacity() { + let new_cap = (index.capacity() + 1024).max(1024); + index.reserve(new_cap).context("reserve usearch capacity")?; } - self.index - .add(key, embedding) - .context("insert into usearch")?; + index.add(key, embedding).context("insert into usearch")?; self.dirty = true; Ok(()) @@ -184,10 +331,24 @@ impl CodeVectorIndex { /// Remove a symbol by database ID. Incremental, no rebuild. pub fn remove(&mut self, symbol_id: i64) -> bool { - if let Some(&key) = self.symbol_to_key.get(&symbol_id) { - self.key_to_symbol.remove(&key); - self.symbol_to_key.remove(&symbol_id); - if let Err(e) = self.index.remove(key) { + if let Inner::Vecq(index) = &mut self.inner { + let removed = index.remove_keyed(symbol_id as u64); + self.dirty |= removed; + return removed; + } + let Inner::Usearch { + index, + key_to_symbol, + symbol_to_key, + next_key: _, + } = &mut self.inner + else { + return false; + }; + if let Some(&key) = symbol_to_key.get(&symbol_id) { + key_to_symbol.remove(&key); + symbol_to_key.remove(&symbol_id); + if let Err(e) = index.remove(key) { tracing::error!("Failed to remove from usearch: {e}"); } self.dirty = true; @@ -199,11 +360,30 @@ impl CodeVectorIndex { /// Search for k nearest neighbors. Returns (symbol_id, cosine_distance) pairs. pub fn search(&self, query: &[f32], k: usize) -> Vec<(i64, f32)> { - if self.index.size() == 0 { + if let Inner::Vecq(index) = &self.inner { + if index.is_empty() { + return Vec::new(); + } + let count = k.max(1); + return index + .search_keyed(query, count) + .into_iter() + .map(|(key, sim)| (key as i64, 1.0 - sim)) + .collect(); + } + if self.index_size() == 0 { return Vec::new(); } let count = k.max(1); - let results = match self.index.search(query, count) { + let Inner::Usearch { + index, + key_to_symbol, + .. + } = &self.inner + else { + unreachable!("vecq path returned earlier"); + }; + let results = match index.search(query, count) { Ok(r) => r, Err(e) => { tracing::error!("usearch search failed: {e}"); @@ -215,18 +395,24 @@ impl CodeVectorIndex { .keys .iter() .zip(results.distances.iter()) - .filter_map(|(key, dist)| self.key_to_symbol.get(key).map(|&id| (id, *dist))) + .filter_map(|(key, dist)| key_to_symbol.get(key).map(|&id| (id, *dist))) .collect() } /// Number of vectors in the index. pub fn len(&self) -> usize { - self.index.size() + match &self.inner { + Inner::Usearch { index, .. } => index.size(), + Inner::Vecq(index) => index.len(), + } } /// Embedding dimensionality. pub fn dims(&self) -> usize { - self.index.dimensions() + match &self.inner { + Inner::Usearch { index, .. } => index.dimensions(), + Inner::Vecq(index) => index.dim(), + } } /// Whether the index has unsaved changes. @@ -236,7 +422,12 @@ impl CodeVectorIndex { /// Whether the index is empty. pub fn is_empty(&self) -> bool { - self.index.size() == 0 + self.len() == 0 + } + + #[allow(dead_code)] + fn index_size(&self) -> usize { + self.len() } } @@ -292,8 +483,17 @@ mod tests { v } + /// The global store selector is process-wide — serialize every test that + /// constructs an index so backend flips never race parallel tests. + static STORE_LOCK: LazyLock> = LazyLock::new(|| std::sync::Mutex::new(())); + + fn with_store_lock() -> std::sync::MutexGuard<'static, ()> { + STORE_LOCK.lock().unwrap_or_else(|e| e.into_inner()) + } + #[test] fn test_empty_search() { + let _g = with_store_lock(); let idx = CodeVectorIndex::new(768).unwrap(); assert!(idx.is_empty()); let results = idx.search(&[0.0f32; 768], 5); @@ -302,6 +502,7 @@ mod tests { #[test] fn test_insert_and_search() { + let _g = with_store_lock(); let mut idx = CodeVectorIndex::new(768).unwrap(); let v1 = make_unit_vec(768, 0); // unit vector along dim 0 @@ -330,6 +531,7 @@ mod tests { #[test] fn test_replace_on_duplicate_insert() { + let _g = with_store_lock(); let mut idx = CodeVectorIndex::new(64).unwrap(); let v1 = make_unit_vec(64, 0); @@ -348,6 +550,7 @@ mod tests { #[test] fn test_remove() { + let _g = with_store_lock(); let mut idx = CodeVectorIndex::new(64).unwrap(); idx.insert(1, &make_unit_vec(64, 0)).unwrap(); @@ -363,6 +566,7 @@ mod tests { #[test] fn test_save_and_load() { + let _g = with_store_lock(); let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("test.usearch"); @@ -381,6 +585,65 @@ mod tests { assert_eq!(results[0].0, 10); } + #[test] + fn test_vecq_backend_roundtrip() { + let _g = with_store_lock(); + set_vector_store(VectorStoreKind::Vecq); + let mut idx = CodeVectorIndex::new(64).unwrap(); + + idx.insert(1, &make_unit_vec(64, 0)).unwrap(); + idx.insert(2, &make_unit_vec(64, 1)).unwrap(); + assert_eq!(idx.len(), 2); + + // Replace semantics + idx.insert(1, &make_unit_vec(64, 2)).unwrap(); + assert_eq!(idx.len(), 2); + + // Removal + assert!(idx.remove(2)); + assert!(!idx.remove(2)); + assert_eq!(idx.len(), 1); + + let results = idx.search(&make_unit_vec(64, 2), 5); + assert_eq!(results.len(), 1); + assert_eq!(results[0].0, 1); + // distance conversion: similarity ~1.0 -> distance ~0.0 + assert!(results[0].1 < 0.05, "dist={:?}", results[0].1); + + set_vector_store(VectorStoreKind::Usearch); + } + + #[test] + fn test_vecq_save_and_load_uses_own_extension() { + let _g = with_store_lock(); + set_vector_store(VectorStoreKind::Vecq); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("test.usearch"); // caller names it; backend picks .vecq + + { + let mut idx = CodeVectorIndex::new(64).unwrap(); + idx.insert(10, &make_unit_vec(64, 0)).unwrap(); + idx.path = Some(path.clone()); + idx.save().unwrap(); + } + + assert!(dir.path().join("test.vecq").exists()); + // Upstream gap (vecq#32): keys are not serialized, so reload rebuilds + // fresh instead of serving a silently-empty index. + let idx2 = CodeVectorIndex::load_or_create(&path, 64).unwrap(); + assert!(idx2.is_empty(), "reload must not serve a keyless index"); + set_vector_store(VectorStoreKind::Usearch); + } + + #[test] + fn vector_store_parse_is_lenient() { + assert_eq!(VectorStoreKind::parse("vecq"), VectorStoreKind::Vecq); + assert_eq!(VectorStoreKind::parse("VECQ "), VectorStoreKind::Vecq); + assert_eq!(VectorStoreKind::parse("usearch"), VectorStoreKind::Usearch); + assert_eq!(VectorStoreKind::parse(""), VectorStoreKind::Usearch); + assert_eq!(VectorStoreKind::parse("bogus"), VectorStoreKind::Usearch); + } + #[test] fn test_cosine_distance_to_similarity() { assert!((cosine_distance_to_similarity(0.0) - 1.0).abs() < f32::EPSILON); diff --git a/src/main.rs b/src/main.rs index daf9329..61c1394 100644 --- a/src/main.rs +++ b/src/main.rs @@ -789,6 +789,7 @@ async fn main() -> Result<()> { .map(|c| c.brain.embedding.to_string()) .unwrap_or_else(|| "auto".to_string()); crate::embed::resolve_backend(&brain_mode); + index::vector::apply_config_store(config.as_ref()); eprintln!("{}", "🔍 Indexing project...".cyan()); let stats = index::index_project_with_skip( @@ -1197,6 +1198,16 @@ async fn main() -> Result<()> { .unwrap_or_else(|| "auto".to_string()); crate::embed::resolve_backend(&brain_mode); + let brain_cfg = crate::config::loader::load_config( + cli.global.config.as_deref(), + None, + None, + None, + None, + false, + ) + .ok(); + index::vector::apply_config_store(brain_cfg.as_ref()); let results = index::brain::brain_search(&conn, project_id, &query_str, limit)?; if json { diff --git a/src/mcp/tools.rs b/src/mcp/tools.rs index 6b27765..6a84c43 100644 --- a/src/mcp/tools.rs +++ b/src/mcp/tools.rs @@ -1001,6 +1001,7 @@ fn handle_brain_search(params: &serde_json::Value) -> ToolResult { Err(e) => return ToolResult::error(e.to_string()), }; + crate::index::vector::apply_config_store(load_project_config().ok().as_ref()); match crate::index::brain::brain_search(&conn, project_id, query, limit) { Ok(results) => { if results.is_empty() { From 514ef43267f92c74b103c5287e53cf291c20b3c2 Mon Sep 17 00:00:00 2001 From: ajianaz Date: Sat, 29 Aug 2026 19:03:52 +0700 Subject: [PATCH 2/2] fix(brain): wire brain section config, self-heal vecq reload, dedupe config load MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three fixes from PR #543 review: 1. Config wiring: .cora.yaml 'brain:' section was never parsed — CoraFile had no brain field, so vector_store: vecq silently fell back to usearch and embedding mode was ignored. Add BrainSection + field-wise merge_into (matches other sections). 2. Vecq reload self-heal: a reloaded vecq index is rebuilt empty (upstream vecq#32 lacks key serialization) and arrived with dirty=false, so a no-change 'cora index' run saved an empty index over the stored one — permanently losing vector signal. Now reload marks the index dirty, embed_project clears stale embed fingerprints when it sees dirty+empty, and cora index forces re-embed when the on-disk vecq file needs a rebuild. 3. Brain command: load_config was called twice; fold into one load shared by resolve_backend and apply_config_store. Verified: 937/937 tests pass; e2e with CODECORA_HOME sandbox — vector_store: vecq now creates cora_index.vecq config now works, and a no-change re-run logs 'cleared=2, re-embedding all symbols' instead of silently persisting an empty index. --- src/config/schema.rs | 22 ++++++++++++++++++++++ src/index/brain.rs | 30 ++++++++++++++++++++++++++++++ src/index/mod.rs | 8 ++++++-- src/index/vector.rs | 13 +++++++++---- src/main.rs | 18 +++++------------- 5 files changed, 72 insertions(+), 19 deletions(-) diff --git a/src/config/schema.rs b/src/config/schema.rs index bbd481b..9d60c79 100644 --- a/src/config/schema.rs +++ b/src/config/schema.rs @@ -328,6 +328,8 @@ pub struct CoraFile { pub profile: Option, #[serde(skip_serializing_if = "Option::is_none")] pub analysis: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub brain: Option, } #[derive(Debug, Clone, Serialize, Deserialize, Default)] @@ -469,6 +471,16 @@ pub struct BrainConfig { pub embedding: BrainEmbeddingMode, } +/// `.cora.yaml` `brain:` section — mirrors the subset of [`BrainConfig`] +/// that is meaningful in a config file. +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct BrainSection { + #[serde(skip_serializing_if = "Option::is_none")] + pub vector_store: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub embedding: Option, +} + /// Embedding backend mode for Brain Mode. #[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, Eq)] #[serde(rename_all = "lowercase")] @@ -777,6 +789,16 @@ impl CoraFile { .clone_from(&analysis.entry_point_patterns); } } + // Merge brain config (Brain Mode vector store / embedding backend). + // Field-wise like the other sections: unset fields keep Config defaults. + if let Some(brain) = &self.brain { + if let Some(v) = &brain.vector_store { + config.brain.vector_store.clone_from(v); + } + if let Some(v) = &brain.embedding { + config.brain.embedding = v.clone(); + } + } Ok(()) } } diff --git a/src/index/brain.rs b/src/index/brain.rs index 49b892e..16b8c06 100644 --- a/src/index/brain.rs +++ b/src/index/brain.rs @@ -158,6 +158,22 @@ pub fn embed_project(conn: &Connection, project_id: i64) -> Result { cache.as_mut().unwrap() }; + // ── Self-heal after vecq reload-discard (#542, vecq#32) ────────── + // A vecq index reloaded from disk is rebuilt empty (upstream lacks key + // serialization) and arrives dirty. Stored fingerprints would skip all + // unchanged symbols, permanently excluding them from vector search. + // Clear fingerprints so this run re-embeds every symbol in the project. + if vi.is_dirty() && vi.is_empty() { + let cleared = conn.execute( + "UPDATE symbols SET embed_fingerprint = NULL WHERE project_id = ?1", + rusqlite::params![project_id], + )?; + tracing::info!( + cleared, + "vector index rebuilt from empty (vecq reload) — re-embedding all symbols" + ); + } + // ── Incremental: fetch stored fingerprints ────────────────────── // Only re-embed symbols whose name+signature has changed. let mut stmt = conn.prepare( @@ -355,6 +371,20 @@ fn fts5_search(conn: &Connection, project_id: i64, query: &str, limit: usize) -> } } +/// True when the on-disk vector index exists but will be discarded on reload — +/// i.e. vecq backend with a non-empty `.vecq` file that cannot restore its +/// key map (vecq#32). `cora index` uses this to force a re-embed even when +/// no files changed, so the vector signal heals instead of staying empty. +pub fn vector_index_needs_rebuild() -> bool { + if crate::index::vector::current_vector_store() != crate::index::vector::VectorStoreKind::Vecq { + return false; + } + let path = vector_index_path().with_extension("vecq"); + std::fs::metadata(&path) + .map(|m| m.len() > 24) + .unwrap_or(false) +} + /// usearch vector search → (symbol_id, cosine_similarity) pairs, filtered to project. /// Uses cached vector index and cached project ID set — no disk I/O per query. fn vector_search(conn: &Connection, project_id: i64, query: &str, limit: usize) -> Vec<(i64, f32)> { diff --git a/src/index/mod.rs b/src/index/mod.rs index 5ed9ae9..1ef8e99 100644 --- a/src/index/mod.rs +++ b/src/index/mod.rs @@ -531,8 +531,12 @@ fn index_project_with_id( stats.files_scanned, stats.files_indexed, stats.symbols_indexed, stats.errors ); - // Embed symbols into vector index for brain search — only when files changed - if stats.files_indexed > 0 { + // Embed symbols into vector index for brain search — when files changed, + // or when the stored index was discarded on reload (vecq #542: a fresh + // process rebuilds the vecq index empty; if all files are unchanged this + // would be the only chance to re-embed — without it the vector signal + // stays dead until a file actually changes). + if stats.files_indexed > 0 || brain::vector_index_needs_rebuild() { match brain::embed_project(conn, project_id) { Ok(n) => { stats.embedded_symbols = Some(n); diff --git a/src/index/vector.rs b/src/index/vector.rs index 401a99c..51ff239 100644 --- a/src/index/vector.rs +++ b/src/index/vector.rs @@ -161,6 +161,7 @@ impl CodeVectorIndex { let mut lock_file = acquire_file_lock(path)?; let mut buffer = Vec::new(); + let mut dirty = false; if lock_file.metadata().context("read vecq metadata")?.len() > 0 { use std::io::{Read, Seek, SeekFrom}; lock_file @@ -174,19 +175,23 @@ impl CodeVectorIndex { // KNOWN LIMITATION (#542): vecq-core 0.2.0 `from_bytes` restores codes // and scales but NOT the keyed map (see codecoradev/vecq#32), so a // reloaded index would silently search empty. Until key serialization - // ships upstream, reload always rebuilds fresh — safe (never - // mis-searches); `cora index` re-embeds on the next run. + // ships upstream, reload always rebuilds fresh. To keep the next + // `cora index` honest, mark the index dirty here — `embed_project` + // will then clear embed fingerprints and re-embed everything instead + // of skipping unchanged symbols against an empty index. if buffer.len() >= 24 { tracing::warn!( - "vecq persistence lacks key serialization upstream (vecq#32) — recreating index; run `cora index` to re-embed symbols" + "vecq persistence lacks key serialization upstream (vecq#32) — \ + recreating index; `cora index` will re-embed all symbols" ); + dirty = true; } let index = vecq_core::VecqIndex::new(dims, VECQ_SEED); Ok(Self { inner: Inner::Vecq(Box::new(index)), path: Some(path.to_path_buf()), - dirty: false, + dirty, _lock_file: Some(lock_file), }) } diff --git a/src/main.rs b/src/main.rs index 61c1394..1558085 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1185,19 +1185,6 @@ async fn main() -> Result<()> { let project_id = index::ensure_project(&conn, &project_root)?; // Resolve embedding backend from config for query embedding - let brain_mode = crate::config::loader::load_config( - cli.global.config.as_deref(), - None, - None, - None, - None, - false, - ) - .ok() - .map(|c| c.brain.embedding.to_string()) - .unwrap_or_else(|| "auto".to_string()); - crate::embed::resolve_backend(&brain_mode); - let brain_cfg = crate::config::loader::load_config( cli.global.config.as_deref(), None, @@ -1207,6 +1194,11 @@ async fn main() -> Result<()> { false, ) .ok(); + let brain_mode = brain_cfg + .as_ref() + .map(|c| c.brain.embedding.to_string()) + .unwrap_or_else(|| "auto".to_string()); + crate::embed::resolve_backend(&brain_mode); index::vector::apply_config_store(brain_cfg.as_ref()); let results = index::brain::brain_search(&conn, project_id, &query_str, limit)?;