feat(brain): vecq-core 0.3.0 — keyed persistence, residual default, brain.vector_bits knob

CHANGELOG.md

@@ -9,7 +9,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
-- **Opt-in `vecq` vector store for Brain Mode.** Set `brain.vector_store: vecq` in `.cora.yaml` to replace the usearch HNSW index with vecq 4-bit quantized scan (pure Rust, deterministic, ~6x smaller). Opt-in only; keyed persistence awaits vecq#32, so indexes rebuild fresh on load for now (#542).
+- **Opt-in `vecq` vector store for Brain Mode.** Set `brain.vector_store: vecq` in `.cora.yaml` to replace the usearch HNSW index with a vecq quantized scan (pure Rust, deterministic, ~5x smaller). Keyed persistence included: symbol ids survive reload, so a fresh process serves the index as-is and `cora index` no longer re-embeds unchanged projects (#542, #547).
+- **`brain.vector_bits` quantization-width knob.** `residual` (default) | `4` | `5` | `6` — 4-bit base codes with second-pass residual rescoring, or plain Lloyd-Max widths. The default is residual: best recall@10 at 4-bit scan speed in a recall study on cora's own embeddings, ahead of plain 5-bit at 1k/5k/13k-symbol scales. Changing the width rebuilds the index once on the next `cora index` instead of silently serving the old width; unknown values fall back to `residual`.
+
+### Fixed
+
+- **Vector signal never fired in a fresh process.** `cora brain` and MCP `brain_search` only saw the vector index if the same process had run the embed — otherwise results silently degraded to FTS-only. The search path now lazy-loads the on-disk index once per process, with a dimension guard against backend switches (#545).
+- **Stale embed fingerprints after a global vector-index rebuild.** The vector index is a single file shared by all projects; rebuilding it (width/dims change, legacy file, corruption) wiped every project's vectors while their fingerprints still said "embedded" — the incremental path would skip those symbols forever. A rebuild now clears fingerprints for all projects, and the usearch dims-mismatch path (which deleted the index without clearing) joins the same heal.
+
+### Changed
+
+- **vecq-core dependency 0.2.0 → 0.3.0.** Picks up the 4-bit+residual mode, plain 5/6-bit widths, runtime-detected AVX2 scoring, and file formats v1.3–v1.5 with the keyed-slot table. Pre-0.3.0 `.vecq` files carry no key table and rebuild once with a warning, then upgrade to the keyed format.
 
 ## [0.14.0] - 2026-08-28
 

Cargo.toml

@@ -72,7 +72,7 @@ rusqlite = { version = "0.31", features = ["bundled"] }
 
 # Vector search (Phase 3 — Brain Mode)
 usearch = "2"
-vecq-core = "0.2.0"
+vecq-core = "0.3.0"
 fs2 = "0.4"
 
 # Self-update (upgrade command)

src/config/schema.rs

@@ -451,15 +451,29 @@ fn default_vector_store() -> String {
     "usearch".to_string()
 }
 
-#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq)]
+fn default_vector_bits() -> String {
+    "residual".to_string()
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
 pub struct BrainConfig {
     /// Vector store backend for Brain Mode search.
     ///
     /// - `"usearch"` (default) — HNSW graph, f32
-    /// - `"vecq"` — 4-bit quantized brute-force scan (pure Rust, deterministic,
-    ///   ~6x smaller index; recall trade absorbed by RRF fusion)
+    /// - `"vecq"` — quantized brute-force scan (pure Rust, deterministic,
+    ///   ~5x smaller index; recall trade absorbed by RRF fusion)
     #[serde(default = "default_vector_store")]
     pub vector_store: String,
+    /// Quantization width for the `vecq` store (ignored by `usearch`).
+    ///
+    /// - `"residual"` (default) — 4-bit base + second-pass residual codes:
+    ///   best recall at 4-bit scan speed on cora's default embeddings
+    /// - `"4"` / `"5"` / `"6"` — plain Lloyd-Max width
+    ///
+    /// Invalid values fall back to `"residual"`. Changing the width rebuilds
+    /// the index once on the next `cora index`.
+    #[serde(default = "default_vector_bits")]
+    pub vector_bits: String,
     /// Embedding backend selection.
     ///
     /// - `"auto"` (default) — best available: pretrained → hashing
@@ -471,13 +485,28 @@ pub struct BrainConfig {
     pub embedding: BrainEmbeddingMode,
 }
 
+// Manual Default so `Config::default()` carries the documented values
+// ("usearch"/"residual") — the derived one would leave empty strings, and
+// the store/width parsers only meet those values leniently.
+impl Default for BrainConfig {
+    fn default() -> Self {
+        Self {
+            vector_store: default_vector_store(),
+            vector_bits: default_vector_bits(),
+            embedding: BrainEmbeddingMode::default(),
+        }
+    }
+}
+
 /// `.cora.yaml` `brain:` section — mirrors the subset of [`BrainConfig`]
 /// that is meaningful in a config file.
 #[derive(Debug, Clone, Serialize, Deserialize, Default)]
 pub struct BrainSection {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub vector_store: Option<String>,
     #[serde(skip_serializing_if = "Option::is_none")]
+    pub vector_bits: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub embedding: Option<BrainEmbeddingMode>,
 }
 
@@ -795,6 +824,9 @@ impl CoraFile {
             if let Some(v) = &brain.vector_store {
                 config.brain.vector_store.clone_from(v);
             }
+            if let Some(v) = &brain.vector_bits {
+                config.brain.vector_bits.clone_from(v);
+            }
             if let Some(v) = &brain.embedding {
                 config.brain.embedding = v.clone();
             }
@@ -870,6 +902,48 @@ mod tests {
         assert_eq!(cfg.output.format, "pretty");
     }
 
+    #[test]
+    fn brain_vector_bits_defaults_to_residual() {
+        let cfg = Config::default();
+        assert_eq!(cfg.brain.vector_bits, "residual");
+        assert_eq!(cfg.brain.vector_store, "usearch");
+    }
+
+    #[test]
+    fn merge_brain_vector_store_and_bits() {
+        let mut cfg = Config::default();
+        let cora = CoraFile::from_str(
+            r"
+brain:
+  vector_store: vecq
+  vector_bits: '6'
+  embedding: hashing
+",
+        )
+        .unwrap();
+
+        cora.merge_into(&mut cfg).unwrap();
+
+        assert_eq!(cfg.brain.vector_store, "vecq");
+        assert_eq!(cfg.brain.vector_bits, "6");
+    }
+
+    #[test]
+    fn merge_brain_vector_bits_absent_keeps_default() {
+        let mut cfg = Config::default();
+        let cora = CoraFile::from_str(
+            r"
+brain:
+  vector_store: vecq
+",
+        )
+        .unwrap();
+
+        cora.merge_into(&mut cfg).unwrap();
+
+        assert_eq!(cfg.brain.vector_bits, "residual");
+    }
+
     #[test]
     fn merge_provider_overrides() {
         let mut cfg = Config::default();

src/index/brain.rs

@@ -62,17 +62,20 @@ fn vector_index_path() -> std::path::PathBuf {
 /// The index file contains usearch binary data — we can't easily read the
 /// dimensionality without loading it. Instead, we check the `embedding_dims`
 /// column in the projects table, which is written during embedding.
-fn check_dimension_compat(vi_path: &std::path::Path, expected_dims: usize) {
+fn check_dimension_compat(vi_path: &std::path::Path, expected_dims: usize) -> bool {
     // Attempt to load the index just to check its dimensions.
     // If it fails (empty, corrupt, etc.), we'll create fresh — no warning needed.
+    // Returns true when a stale index was removed: the vector index is a
+    // single file shared by every project, so its fingerprints are now all
+    // stale and callers must clear them.
     let Ok(file) = std::fs::File::open(vi_path) else {
-        return;
+        return false;
     };
     let Ok(metadata) = file.metadata() else {
-        return;
+        return false;
     };
     if metadata.len() == 0 {
-        return; // Empty file — will create fresh
+        return false; // Empty file — will create fresh
     }
 
     // Try to load and check dims
@@ -102,18 +105,22 @@ fn check_dimension_compat(vi_path: &std::path::Path, expected_dims: usize) {
             // Delete the stale index so embed_project creates a fresh one
             if let Err(e) = std::fs::remove_file(vi_path) {
                 tracing::warn!("  Failed to remove stale index: {e}");
+                false
             } else {
                 let keys_path = vi_path.with_extension("keys");
                 let _ = std::fs::remove_file(&keys_path);
                 tracing::info!("  Removed stale vector index — will create fresh on next index");
+                true
             }
         }
         Ok(Some(_)) => {
             // Dimensions match — all good
+            false
         }
         _ => {
             // Couldn't read dimensions (corrupt, unsupported, etc.)
             // embed_project will handle creation
+            false
         }
     }
 }
@@ -140,9 +147,11 @@ pub fn embed_project(conn: &Connection, project_id: i64) -> Result<usize> {
     // Check for existing index with mismatched dimensions (e.g. user rebuilt
     // with/without pretrained-embed feature). Warn but continue — the index
     // will be corrupted for searches until re-indexed.
-    if vi_path.exists() {
-        check_dimension_compat(&vi_path, active);
-    }
+    let dims_stale = if vi_path.exists() {
+        check_dimension_compat(&vi_path, active)
+    } else {
+        false
+    };
 
     // Acquire write lock — block searches while embedding.
     // This is fine because embedding only happens during `cora index`,
@@ -158,19 +167,21 @@ pub fn embed_project(conn: &Connection, project_id: i64) -> Result<usize> {
         cache.as_mut().unwrap()
     };
 
-    // ── Self-heal after vecq reload-discard (#542, vecq#32) ──────────
-    // A vecq index reloaded from disk is rebuilt empty (upstream lacks key
-    // serialization) and arrives dirty. Stored fingerprints would skip all
-    // unchanged symbols, permanently excluding them from vector search.
-    // Clear fingerprints so this run re-embeds every symbol in the project.
-    if vi.is_dirty() && vi.is_empty() {
-        let cleared = conn.execute(
-            "UPDATE symbols SET embed_fingerprint = NULL WHERE project_id = ?1",
-            rusqlite::params![project_id],
-        )?;
+    // ── Self-heal after a global index rebuild (#542, vecq#32) ───────
+    // The vector index is one shared file across projects. A vecq reload
+    // that had to rebuild it (legacy pre-0.3.0 file, width change, dim
+    // change, corruption) or a stale usearch index removed for a dims
+    // mismatch leaves every project's vectors gone while their stored
+    // fingerprints still say "embedded" — the incremental path would skip
+    // those symbols forever. Clear fingerprints for ALL projects: the
+    // current project re-embeds in this run; every other project re-embeds
+    // on its next `cora index` run that reaches embed_project
+    // (cross-project scheduling: #545).
+    if dims_stale || (vi.is_dirty() && vi.is_empty()) {
+        let cleared = conn.execute("UPDATE symbols SET embed_fingerprint = NULL", [])?;
         tracing::info!(
             cleared,
-            "vector index rebuilt from empty (vecq reload) — re-embedding all symbols"
+            "vector index rebuilt from empty — all projects re-embed on their next index run"
         );
     }
 
@@ -372,17 +383,17 @@ fn fts5_search(conn: &Connection, project_id: i64, query: &str, limit: usize) ->
 }
 
 /// True when the on-disk vector index exists but will be discarded on reload —
-/// i.e. vecq backend with a non-empty `.vecq` file that cannot restore its
-/// key map (vecq#32). `cora index` uses this to force a re-embed even when
-/// no files changed, so the vector signal heals instead of staying empty.
+/// i.e. vecq backend whose `.vecq` file predates keyed persistence (vecq#32),
+/// is unreadable, or was built with different dims. `cora index` uses this to
+/// force a re-embed even when no files changed, so the vector signal heals
+/// instead of staying empty. Healthy keyed files (vecq-core 0.3.0+) reload
+/// as-is and return false.
 pub fn vector_index_needs_rebuild() -> bool {
     if crate::index::vector::current_vector_store() != crate::index::vector::VectorStoreKind::Vecq {
         return false;
     }
     let path = vector_index_path().with_extension("vecq");
-    std::fs::metadata(&path)
-        .map(|m| m.len() > 24)
-        .unwrap_or(false)
+    path.exists() && crate::index::vector::vecq_file_needs_rebuild(&path, active_dims())
 }
 
 /// Return a read guard over the vector index cache, loading the on-disk

src/index/mod.rs

@@ -532,8 +532,8 @@ fn index_project_with_id(
     );
 
     // Embed symbols into vector index for brain search — when files changed,
-    // or when the stored index was discarded on reload (vecq #542: a fresh
-    // process rebuilds the vecq index empty; if all files are unchanged this
+    // or when the stored index cannot serve keyed search after reload
+    // (legacy pre-0.3.0 vecq file, vecq #542: if all files are unchanged this
     // would be the only chance to re-embed — without it the vector signal
     // stays dead until a file actually changes).
     if stats.files_indexed > 0 || brain::vector_index_needs_rebuild() {