diff --git a/REQUIREMENTS.md b/REQUIREMENTS.md index 6ddb72d..c75ddd9 100644 --- a/REQUIREMENTS.md +++ b/REQUIREMENTS.md @@ -2,7 +2,7 @@ > **Note:** This document is automatically generated and verified against the live test suite by `scripts/generate_requirements.py` and `tests/backend/test_requirements_sync.py`. -**Test Verification Baseline:** **1344 Automated Tests** (980 Pytest Backend + 314 Vitest Frontend + 50 Playwright E2E). +**Test Verification Baseline:** **1352 Automated Tests** (988 Pytest Backend + 314 Vitest Frontend + 50 Playwright E2E). --- @@ -1133,6 +1133,16 @@ code results are enriched with signature/kind/ast_symbol_id from SQLite._ - `test_restart_clears_relational_metadata_on_collection_recreate` - _Verifies that on app restart, if ensure_collection detects a dimension mismatch and recreates the collection, relational metadata is cleared._ - `test_rollback_and_complete_rebuild` - _Tests model transition, rollback, and complete rebuild across local paths._ +#### `tests/test_embedding_parallel_batch.py` (8 tests) +- `test_parallel_slices_execution_and_deterministic_ordering` - _Verifies get_dense_embeddings_batch dispatches slices in parallel across EMBEDDING_API_CONCURRENCY threads and reassembles results in exact input order._ +- `test_token_limit_slicing_respects_max_tokens_per_request` - _Verifies partition_texts_into_slices splits batches when estimated tokens exceed EMBEDDING_MAX_TOKENS_PER_REQUEST even if item count is below batch_size._ +- `test_http_429_retry_after_aimd_and_telemetry` - _Verifies HTTP 429 rate-limit responses parse Retry-After headers, log a warning, multiplicatively reduce concurrency, and record rate_limit_wait_seconds in telemetry and progress_tracker._ +- `test_cross_file_batch_embedding_in_git_and_local_syncer` - _Verifies git_syncer and local_syncer collect chunks across multiple files and call get_hybrid_embeddings_batch once per multi-file batch instead of once per file._ +- `test_atomic_openai_client_swap_during_concurrent_embeddings` - _Verifies init_embeddings replaces _openai_client atomically without ever setting it to None while concurrent embedding calls are active._ +- `test_env_embedding_batch_size_takes_effect_across_restarts` - _Verifies init_db does not freeze embedding_batch_size or embedding_num_threads in system_metadata, allowing .env changes to take effect on restart._ +- `test_bounded_channel_pipeline_producer_consumer_backpressure` - _Verifies BoundedChannel and ChannelIngestionPipeline stream parsed files through coalescing, parallel API embedding slices, and persistence._ +- `test_class_and_methods_do_not_double_chunk` - _Verifies extract_symbols_and_chunks does not duplicate method bodies inside parent class chunks and keeps method chunk hashes stable when sibling methods change._ + #### `tests/test_engineering_style_guardrails.py` (4 tests) - `test_repository_root_structure_hygiene` - _Verify repository root contains required living docs, no stray test_*.py files, and no committed UI build._ - `test_all_python_files_and_classes_under_500_loc` - _Verify every Python file and class across main.py, app/, scripts/, and tests/ stays <= 500 LOC._ diff --git a/app/services/chunking/symbol_extractor.py b/app/services/chunking/symbol_extractor.py index d2f2efb..7ff4452 100644 --- a/app/services/chunking/symbol_extractor.py +++ b/app/services/chunking/symbol_extractor.py @@ -94,6 +94,55 @@ def extract_symbols_and_chunks( is_csharp = language in ("c_sharp", "csharp") receiver_type_cache: Dict[Any, Dict[str, str]] = {} + def _resolve_container_info(n) -> Optional[Tuple[str, str]]: + if n.type not in target_types: + return None + if n.type in ("variable_declarator", "pair"): + return extract_callable_variable_or_pair(n, source_bytes) + n_name = extract_node_name(n, source_bytes) + if not n_name: + return None + n_kind = n.type + if language in ("c_sharp", "csharp") and n.type == "record_declaration": + if any(c.type == "struct" for c in n.children): + n_kind = "record_struct_declaration" + return n_name, n_kind + + def _collect_direct_child_containers(parent_node) -> List[Any]: + found: List[Any] = [] + + def _walk(cur): + for ch in cur.children: + if _resolve_container_info(ch) is not None: + found.append(ch) + else: + _walk(ch) + + _walk(parent_node) + return found + + def _emit_chunk_span(sym_full: str, sym_kind: str, text_span: str, span_start: int, span_end: int) -> None: + if not text_span.strip(): + return + heading_str = f"{sym_full} ({sym_kind})" + if len(text_span) <= max_chunk_chars: + chunks.append({ + "symbol": sym_full, + "kind": sym_kind, + "heading": heading_str, + "content": text_span, + "start_line": span_start, + "end_line": span_end, + }) + else: + sub = split_by_length(text_span, heading=heading_str, max_chars=max_chunk_chars) + for s in sub: + s["symbol"] = sym_full + s["kind"] = sym_kind + s["start_line"] = span_start + s.get("start_line", 1) - 1 + s["end_line"] = span_start + s.get("end_line", span_end - span_start + 1) - 1 + chunks.append(s) + def traverse(node, parent_symbol: Optional[str] = None): line_num = node.start_point[0] + 1 @@ -102,27 +151,19 @@ def traverse(node, parent_symbol: Optional[str] = None): kind = node.type if node.type in target_types: - if node.type in ("variable_declarator", "pair"): - res_var = extract_callable_variable_or_pair(node, source_bytes) - if res_var: - name, kind = res_var - is_container = True - elif node.type == "variable_declarator": - name_node = node.child_by_field_name("name") - val_node = node.child_by_field_name("value") - if name_node and val_node and name_node.type == "identifier" and val_node.type == "object": - obj_name = source_bytes[name_node.start_byte:name_node.end_byte].decode("utf-8", errors="ignore").strip() - next_parent = f"{parent_symbol}.{obj_name}" if parent_symbol else obj_name - for child in node.children: - traverse(child, parent_symbol=next_parent) - return - else: - name = extract_node_name(node, source_bytes) - if name: - is_container = True - if language in ("c_sharp", "csharp") and node.type == "record_declaration": - if any(c.type == "struct" for c in node.children): - kind = "record_struct_declaration" + res_info = _resolve_container_info(node) + if res_info is not None: + name, kind = res_info + is_container = True + elif node.type == "variable_declarator": + name_node = node.child_by_field_name("name") + val_node = node.child_by_field_name("value") + if name_node and val_node and name_node.type == "identifier" and val_node.type == "object": + obj_name = source_bytes[name_node.start_byte:name_node.end_byte].decode("utf-8", errors="ignore").strip() + next_parent = f"{parent_symbol}.{obj_name}" if parent_symbol else obj_name + for child in node.children: + traverse(child, parent_symbol=next_parent) + return if is_container and name: full_symbol = f"{parent_symbol}.{name}" if parent_symbol else name @@ -191,23 +232,33 @@ def traverse(node, parent_symbol: Optional[str] = None): "line_number": irel["line"] }) - if len(node_text) <= max_chunk_chars: - chunks.append({ - "symbol": full_symbol, - "kind": kind, - "heading": f"{full_symbol} ({kind})", - "content": node_text, - "start_line": start_line, - "end_line": end_line - }) + child_containers = _collect_direct_child_containers(node) + if not child_containers: + _emit_chunk_span(full_symbol, kind, node_text, start_line, end_line) else: - sub = split_by_length(node_text, heading=f"{full_symbol} ({kind})", max_chars=max_chunk_chars) - for s in sub: - s["symbol"] = full_symbol - s["kind"] = kind - s["start_line"] = start_line + s.get("start_line", 1) - 1 - s["end_line"] = start_line + s.get("end_line", end_line - start_line + 1) - 1 - chunks.append(s) + covered_lines = set() + for ch_node in child_containers: + ch_start = ch_node.start_point[0] + 1 + ch_end = ch_node.end_point[0] + 1 + covered_lines.update(range(ch_start, ch_end + 1)) + + spans: List[Tuple[int, int]] = [] + seg_start: Optional[int] = None + for ln in range(start_line, end_line + 1): + if ln not in covered_lines: + if seg_start is None: + seg_start = ln + else: + if seg_start is not None: + spans.append((seg_start, ln - 1)) + seg_start = None + if seg_start is not None: + spans.append((seg_start, end_line)) + + for idx_span, (s_ln, e_ln) in enumerate(spans): + seg_text = "\n".join(lines[s_ln - 1 : e_ln]) + if idx_span == 0 or re.search(r"[A-Za-z0-9_]", seg_text): + _emit_chunk_span(full_symbol, kind, seg_text, s_ln, e_ln) for child in node.children: traverse(child, parent_symbol=full_symbol) diff --git a/app/services/database/config_store.py b/app/services/database/config_store.py index c502fd5..8222527 100644 --- a/app/services/database/config_store.py +++ b/app/services/database/config_store.py @@ -206,10 +206,20 @@ def _resolve_default_embedding_config(conn: Optional[Any] = None) -> Dict[str, A env_threads = os.getenv("EMBEDDING_NUM_THREADS") or os.getenv("EMBEDDING_THREADS") threads = int(env_threads) if env_threads and env_threads.isdigit() else default_threads + provider = os.getenv("EMBEDDING_PROVIDER", "local").lower().strip() env_batch = os.getenv("EMBEDDING_BATCH_SIZE") - batch_size = int(env_batch) if env_batch and env_batch.isdigit() else 32 + default_bs = 128 if provider == "api" else 32 + batch_size = int(env_batch) if env_batch and env_batch.isdigit() else default_bs + + env_concurrency = os.getenv("EMBEDDING_API_CONCURRENCY") + api_concurrency = int(env_concurrency) if env_concurrency and env_concurrency.isdigit() else 4 + + env_max_tokens = os.getenv("EMBEDDING_MAX_TOKENS_PER_REQUEST") + max_tokens_per_request = int(env_max_tokens) if env_max_tokens and env_max_tokens.isdigit() else 100000 + + env_tpm = os.getenv("EMBEDDING_TPM_LIMIT") + tpm_limit = int(env_tpm) if env_tpm and env_tpm.isdigit() else 0 - provider = os.getenv("EMBEDDING_PROVIDER", "local").lower().strip() dense_model = os.getenv("EMBEDDING_MODEL", "BAAI/bge-small-en-v1.5").strip() sparse_model = os.getenv("SPARSE_MODEL", "Qdrant/bm25").strip() litellm_url = ( @@ -234,6 +244,9 @@ def _resolve_default_embedding_config(conn: Optional[Any] = None) -> Dict[str, A "sparse_model": sparse_model, "threads": max(1, threads), "batch_size": max(1, batch_size), + "api_concurrency": max(1, api_concurrency), + "max_tokens_per_request": max(1, max_tokens_per_request), + "tpm_limit": max(0, tpm_limit), "openai_api_url": litellm_url, "litellm_url": litellm_url, "litellm_api_key": litellm_api_key, @@ -252,36 +265,47 @@ def get_embedding_db_config() -> Dict[str, Any]: sparse_model = _get_meta("embedding_sparse_model") threads_str = _get_meta("embedding_num_threads") batch_size_str = _get_meta("embedding_batch_size") + concurrency_str = _get_meta("embedding_api_concurrency") + max_tokens_str = _get_meta("embedding_max_tokens_per_request") + tpm_limit_str = _get_meta("embedding_tpm_limit") litellm_url = _get_meta("openai_api_url") or _get_meta("embedding_litellm_url") litellm_api_key = _get_meta("openai_api_key") or _get_meta("embedding_litellm_api_key") vision_ocr_model = ( - _get_meta("openai_vision_model") - or _get_meta("vision_ocr_model") - or _get_meta("embedding_vision_ocr_model") - ) - chat_model = ( - _get_meta("openai_chat_model") - or _get_meta("chat_model") - or _get_meta("embedding_chat_model") + _get_meta("openai_vision_model") or _get_meta("vision_ocr_model") or _get_meta("embedding_vision_ocr_model") ) + chat_model = _get_meta("openai_chat_model") or _get_meta("chat_model") or _get_meta("embedding_chat_model") allow_fallback_str = _get_meta("embedding_allow_fallback") fallback_dense_model = _get_meta("embedding_fallback_dense_model") default_cfg = _resolve_default_embedding_config() + eff_provider = (provider or default_cfg["provider"]).lower().strip() threads = int(threads_str) if threads_str and threads_str.isdigit() else default_cfg["threads"] - batch_size = int(batch_size_str) if batch_size_str and batch_size_str.isdigit() else default_cfg["batch_size"] + env_bs = os.getenv("EMBEDDING_BATCH_SIZE", "") + if batch_size_str and batch_size_str.isdigit(): + batch_size = int(batch_size_str) + elif env_bs.isdigit(): + batch_size = int(env_bs) + else: + batch_size = 128 if eff_provider == "api" else 32 + + api_concurrency = int(concurrency_str) if concurrency_str and concurrency_str.isdigit() else default_cfg["api_concurrency"] + max_tokens = int(max_tokens_str) if max_tokens_str and max_tokens_str.isdigit() else default_cfg["max_tokens_per_request"] + tpm_limit = int(tpm_limit_str) if tpm_limit_str and tpm_limit_str.isdigit() else default_cfg["tpm_limit"] allow_fallback = (allow_fallback_str == "1") if allow_fallback_str is not None else default_cfg["allow_fallback"] resolved_url = (litellm_url or default_cfg["litellm_url"]).strip() raw_db_key = (litellm_api_key or "").strip() effective_key = ("" if _is_placeholder_api_key(raw_db_key) else raw_db_key) or default_cfg["litellm_api_key"] return { - "provider": (provider or default_cfg["provider"]).lower().strip(), + "provider": eff_provider, "dense_model": (dense_model or default_cfg["dense_model"]).strip(), "sparse_model": (sparse_model or default_cfg["sparse_model"]).strip(), "threads": max(1, threads), "batch_size": max(1, batch_size), + "api_concurrency": max(1, api_concurrency), + "max_tokens_per_request": max(1, max_tokens), + "tpm_limit": max(0, tpm_limit), "openai_api_url": resolved_url, "litellm_url": resolved_url, "litellm_api_key": effective_key, @@ -302,6 +326,9 @@ def set_embedding_db_config( sparse_model: Optional[str] = None, threads: Optional[int] = None, batch_size: Optional[int] = None, + api_concurrency: Optional[int] = None, + max_tokens_per_request: Optional[int] = None, + tpm_limit: Optional[int] = None, litellm_url: Optional[str] = None, litellm_api_key: Optional[str] = None, vision_ocr_model: Optional[str] = None, @@ -312,28 +339,32 @@ def set_embedding_db_config( if provider is not None: _set_meta("embedding_provider", provider.lower().strip()) if dense_model is not None: - _set_meta("embedding_dense_model", dense_model.strip()) - _set_meta("openai_embedding_model", dense_model.strip()) + for k in ("embedding_dense_model", "openai_embedding_model"): + _set_meta(k, dense_model.strip()) if sparse_model is not None: _set_meta("embedding_sparse_model", sparse_model.strip()) if threads is not None: _set_meta("embedding_num_threads", str(max(1, threads))) if batch_size is not None: _set_meta("embedding_batch_size", str(max(1, batch_size))) + if api_concurrency is not None: + _set_meta("embedding_api_concurrency", str(max(1, api_concurrency))) + if max_tokens_per_request is not None: + _set_meta("embedding_max_tokens_per_request", str(max(1, max_tokens_per_request))) + if tpm_limit is not None: + _set_meta("embedding_tpm_limit", str(max(0, tpm_limit))) if litellm_url is not None: - _set_meta("openai_api_url", litellm_url.strip()) - _set_meta("embedding_litellm_url", litellm_url.strip()) + for k in ("openai_api_url", "embedding_litellm_url"): + _set_meta(k, litellm_url.strip()) if litellm_api_key is not None: - _set_meta("openai_api_key", litellm_api_key.strip()) - _set_meta("embedding_litellm_api_key", litellm_api_key.strip()) + for k in ("openai_api_key", "embedding_litellm_api_key"): + _set_meta(k, litellm_api_key.strip()) if vision_ocr_model is not None: - _set_meta("openai_vision_model", vision_ocr_model.strip()) - _set_meta("vision_ocr_model", vision_ocr_model.strip()) - _set_meta("embedding_vision_ocr_model", vision_ocr_model.strip()) + for k in ("openai_vision_model", "vision_ocr_model", "embedding_vision_ocr_model"): + _set_meta(k, vision_ocr_model.strip()) if chat_model is not None: - _set_meta("openai_chat_model", chat_model.strip()) - _set_meta("chat_model", chat_model.strip()) - _set_meta("embedding_chat_model", chat_model.strip()) + for k in ("openai_chat_model", "chat_model", "embedding_chat_model"): + _set_meta(k, chat_model.strip()) if allow_fallback is not None: _set_meta("embedding_allow_fallback", "1" if allow_fallback else "0") if fallback_dense_model is not None: @@ -406,20 +437,17 @@ def set_ai_gateway_config( ) -> Dict[str, Any]: """Persists OpenAI-compatible AI Gateway configuration in system_metadata.""" if url is not None: - _set_meta("openai_api_url", url.strip()) - _set_meta("embedding_litellm_url", url.strip()) + for k in ("openai_api_url", "embedding_litellm_url"): + _set_meta(k, url.strip()) if api_key is not None: - cleaned_key = api_key.strip() - _set_meta("openai_api_key", cleaned_key) - _set_meta("embedding_litellm_api_key", cleaned_key) + for k in ("openai_api_key", "embedding_litellm_api_key"): + _set_meta(k, api_key.strip()) if chat_model is not None: - _set_meta("openai_chat_model", chat_model.strip()) - _set_meta("chat_model", chat_model.strip()) - _set_meta("embedding_chat_model", chat_model.strip()) + for k in ("openai_chat_model", "chat_model", "embedding_chat_model"): + _set_meta(k, chat_model.strip()) if vision_ocr_model is not None: - _set_meta("openai_vision_model", vision_ocr_model.strip()) - _set_meta("vision_ocr_model", vision_ocr_model.strip()) - _set_meta("embedding_vision_ocr_model", vision_ocr_model.strip()) + for k in ("openai_vision_model", "vision_ocr_model", "embedding_vision_ocr_model"): + _set_meta(k, vision_ocr_model.strip()) if embedding_model is not None: cleaned_emb = embedding_model.strip() _set_meta("openai_embedding_model", cleaned_emb) diff --git a/app/services/database/engine.py b/app/services/database/engine.py index eb1132c..0872166 100644 --- a/app/services/database/engine.py +++ b/app/services/database/engine.py @@ -289,8 +289,6 @@ def _seed_defaults(engine: Engine, vault_path: str = "/docs") -> None: ("embedding_provider", emb_cfg["provider"]), ("embedding_dense_model", emb_cfg["dense_model"]), ("embedding_sparse_model", emb_cfg["sparse_model"]), - ("embedding_num_threads", str(emb_cfg["threads"])), - ("embedding_batch_size", str(emb_cfg["batch_size"])), ("openai_api_url", emb_cfg["litellm_url"]), ("embedding_litellm_url", emb_cfg["litellm_url"]), ]: diff --git a/app/services/embedding_batcher.py b/app/services/embedding_batcher.py new file mode 100644 index 0000000..32ade5a --- /dev/null +++ b/app/services/embedding_batcher.py @@ -0,0 +1,305 @@ +import logging +import os +import threading +import time +from concurrent.futures import ThreadPoolExecutor, as_completed +from typing import Any, Callable, Dict, List, Optional, Tuple + +logger = logging.getLogger("contextcortex.embeddings") + + +def estimate_token_count(text: str) -> int: + """Estimates token count for a single string (~4 characters per token, minimum 1).""" + if not text: + return 1 + return max(1, len(text) // 4) + + +def partition_texts_into_slices( + texts: List[str], + max_items_per_slice: int = 128, + max_tokens_per_request: int = 100000, +) -> List[List[str]]: + """Partitions input texts into request slices bounded by both item count and estimated token limit.""" + if not texts: + return [] + max_items = max(1, int(max_items_per_slice or 128)) + max_tokens = max(1, int(max_tokens_per_request or 100000)) + + slices: List[List[str]] = [] + current_slice: List[str] = [] + current_tokens = 0 + + for text in texts: + t_tokens = estimate_token_count(text) + if current_slice and ( + len(current_slice) >= max_items or (current_tokens + t_tokens) > max_tokens + ): + slices.append(current_slice) + current_slice = [text] + current_tokens = t_tokens + else: + current_slice.append(text) + current_tokens += t_tokens + + if current_slice: + slices.append(current_slice) + return slices + + +def _extract_retry_after_seconds(exc: Exception, default_wait: float) -> float: + """Extracts Retry-After or retry-after-ms wait duration from an HTTP 429 exception if present.""" + headers: Dict[str, Any] = {} + resp = getattr(exc, "response", None) + if resp is not None: + resp_headers = getattr(resp, "headers", None) + if resp_headers: + try: + headers = {str(k).lower(): str(v) for k, v in dict(resp_headers).items()} + except Exception: + pass + exc_headers = getattr(exc, "headers", None) + if not headers and exc_headers: + try: + headers = {str(k).lower(): str(v) for k, v in dict(exc_headers).items()} + except Exception: + pass + + if "retry-after-ms" in headers: + try: + return max(0.05, float(headers["retry-after-ms"].strip()) / 1000.0) + except (ValueError, TypeError): + pass + + if "retry-after" in headers: + try: + return max(0.05, float(headers["retry-after"].strip())) + except (ValueError, TypeError): + pass + + return max(0.05, default_wait) + + +def _is_rate_limit_error(exc: Exception) -> bool: + """Returns True if the exception represents an HTTP 429 rate-limit error.""" + status_code = getattr(exc, "status_code", None) + if status_code == 429: + return True + resp = getattr(exc, "response", None) + if resp is not None and getattr(resp, "status_code", None) == 429: + return True + cls_name = exc.__class__.__name__.lower() + if "ratelimit" in cls_name: + return True + msg = str(exc).lower() + return "429" in msg or "rate limit" in msg or "rate_limit" in msg + + +class EmbeddingTelemetryAndPacer: + """Tracks embedding throughput, HTTP 429 wait times, AIMD concurrency, and optional TPM pacing.""" + + def __init__(self) -> None: + self._lock = threading.Lock() + self.chunks_embedded: int = 0 + self.embedding_seconds: float = 0.0 + self.rate_limit_wait_seconds: float = 0.0 + self.rate_limit_hits: int = 0 + self._current_concurrency: Optional[int] = None + self._consecutive_successes: int = 0 + self._window_start: float = time.monotonic() + self._window_tokens: int = 0 + + def reset(self, target_concurrency: Optional[int] = None) -> None: + with self._lock: + self.chunks_embedded = 0 + self.embedding_seconds = 0.0 + self.rate_limit_wait_seconds = 0.0 + self.rate_limit_hits = 0 + self._current_concurrency = ( + max(1, int(target_concurrency)) if target_concurrency is not None else None + ) + self._consecutive_successes = 0 + self._window_start = time.monotonic() + self._window_tokens = 0 + + def get_effective_concurrency(self, max_concurrency: int) -> int: + max_c = max(1, int(max_concurrency or 1)) + with self._lock: + if self._current_concurrency is None or self._current_concurrency > max_c: + self._current_concurrency = max_c + return max(1, self._current_concurrency) + + def record_rate_limit(self, wait_seconds: float, max_concurrency: int) -> int: + max_c = max(1, int(max_concurrency or 1)) + with self._lock: + if self._current_concurrency is None: + self._current_concurrency = max_c + self._current_concurrency = max(1, self._current_concurrency // 2) + self._consecutive_successes = 0 + self.rate_limit_hits += 1 + self.rate_limit_wait_seconds += max(0.0, float(wait_seconds)) + return self._current_concurrency + + def record_slice_success(self, max_concurrency: int) -> int: + max_c = max(1, int(max_concurrency or 1)) + with self._lock: + if self._current_concurrency is None: + self._current_concurrency = max_c + self._consecutive_successes += 1 + if self._current_concurrency < max_c and self._consecutive_successes >= 2: + self._current_concurrency = min(max_c, self._current_concurrency + 1) + self._consecutive_successes = 0 + return self._current_concurrency + + def record_batch_completion(self, chunks_count: int, elapsed_seconds: float) -> None: + with self._lock: + self.chunks_embedded += max(0, int(chunks_count)) + self.embedding_seconds += max(0.0, float(elapsed_seconds)) + try: + from app.services.indexing.git_progress import progress_tracker + snap = self.get_telemetry_snapshot() + progress_tracker.update_embedding_telemetry(snap) + except Exception: + pass + + def pace_tokens( + self, + tokens: int, + tpm_limit: int, + sleep_fn: Callable[[float], None] = time.sleep, + ) -> float: + if tpm_limit <= 0 or tokens <= 0: + return 0.0 + wait_time = 0.0 + with self._lock: + now = time.monotonic() + elapsed = now - self._window_start + if elapsed >= 60.0: + self._window_start = now + self._window_tokens = 0 + if self._window_tokens + tokens > tpm_limit and self._window_tokens > 0: + wait_time = max(0.0, 60.0 - elapsed) + self._window_start = now + wait_time + self._window_tokens = tokens + else: + self._window_tokens += tokens + if wait_time > 0: + sleep_fn(wait_time) + return wait_time + + def get_telemetry_snapshot(self, max_concurrency: Optional[int] = None) -> Dict[str, Any]: + with self._lock: + chunks = self.chunks_embedded + secs = round(self.embedding_seconds, 4) + wait_secs = round(self.rate_limit_wait_seconds, 4) + hits = self.rate_limit_hits + cps = round(chunks / self.embedding_seconds, 2) if self.embedding_seconds > 0 else 0.0 + active_c = self._current_concurrency + if active_c is None and max_concurrency is not None: + active_c = max(1, int(max_concurrency)) + return { + "chunks_embedded": chunks, + "embedding_seconds": secs, + "embedding_chunks_per_sec": cps, + "rate_limit_wait_seconds": wait_secs, + "rate_limit_hits": hits, + "active_concurrency": active_c or 1, + } + + +telemetry_tracker = EmbeddingTelemetryAndPacer() + + +def execute_api_embeddings_batch( + client: Any, + texts: List[str], + model_name: str, + batch_size: int = 128, + max_concurrency: int = 4, + max_tokens_per_request: int = 100000, + tpm_limit: int = 0, + max_retries: int = 5, + sleep_fn: Callable[[float], None] = time.sleep, +) -> List[List[float]]: + """Partitions texts into slices and dispatches OpenAI embedding requests in parallel with AIMD 429 handling.""" + if not texts: + return [] + if client is None: + raise RuntimeError("API Embedding client not initialized.") + + api_bs = max(1, min(int(batch_size or 128), 256)) + slices = partition_texts_into_slices( + texts, + max_items_per_slice=api_bs, + max_tokens_per_request=max_tokens_per_request, + ) + if not slices: + return [] + + start_time = time.monotonic() + effective_concurrency = telemetry_tracker.get_effective_concurrency(max_concurrency) + + def _embed_single_slice(slice_idx: int, batch_slice: List[str]) -> Tuple[int, List[List[float]]]: + slice_tokens = sum(estimate_token_count(t) for t in batch_slice) + telemetry_tracker.pace_tokens(slice_tokens, tpm_limit, sleep_fn=sleep_fn) + + attempt = 0 + backoff = 1.0 + while True: + try: + response = client.embeddings.create( + input=batch_slice, + model=model_name, + ) + data_sorted = sorted( + response.data, + key=lambda d: getattr(d, "index", 0), + ) if all(hasattr(d, "index") for d in response.data) else response.data + vecs = [d.embedding for d in data_sorted] + telemetry_tracker.record_slice_success(max_concurrency) + return slice_idx, vecs + except Exception as exc: + if _is_rate_limit_error(exc) and attempt < max_retries: + attempt += 1 + wait_secs = _extract_retry_after_seconds(exc, backoff) + new_conc = telemetry_tracker.record_rate_limit(wait_secs, max_concurrency) + logger.warning( + f"Embedding deployment rate limit (HTTP 429) reached on slice {slice_idx + 1}/{len(slices)}. " + f"Waiting {wait_secs:.2f}s before retry {attempt}/{max_retries} " + f"(AIMD reduced concurrency to {new_conc})." + ) + try: + from app.services.indexing.git_progress import progress_tracker + progress_tracker.update_embedding_telemetry( + telemetry_tracker.get_telemetry_snapshot(max_concurrency) + ) + except Exception: + pass + sleep_fn(wait_secs) + backoff = min(30.0, backoff * 2.0) + continue + raise + + if len(slices) == 1 or effective_concurrency <= 1: + ordered_embeddings: List[List[float]] = [] + for idx, s in enumerate(slices): + _, vecs = _embed_single_slice(idx, s) + ordered_embeddings.extend(vecs) + else: + workers = min(len(slices), effective_concurrency) + results_by_index: Dict[int, List[List[float]]] = {} + with ThreadPoolExecutor(max_workers=workers) as executor: + futures = { + executor.submit(_embed_single_slice, idx, s): idx + for idx, s in enumerate(slices) + } + for future in as_completed(futures): + s_idx, vecs = future.result() + results_by_index[s_idx] = vecs + ordered_embeddings = [] + for idx in range(len(slices)): + ordered_embeddings.extend(results_by_index[idx]) + + elapsed = time.monotonic() - start_time + telemetry_tracker.record_batch_completion(len(texts), elapsed) + return ordered_embeddings diff --git a/app/services/embeddings.py b/app/services/embeddings.py index 6284fbe..94a31b8 100644 --- a/app/services/embeddings.py +++ b/app/services/embeddings.py @@ -1,4 +1,5 @@ import os +import time import logging import threading from types import SimpleNamespace @@ -13,8 +14,11 @@ def __init__(self, indices: List[int], values: List[float]): qmodels = SimpleNamespace(SparseVector=_SparseVectorFallback) # type: ignore +from app.services.embedding_batcher import execute_api_embeddings_batch, telemetry_tracker + logger = logging.getLogger("contextcortex.embeddings") + def _get_db_emb_config() -> Dict[str, Any]: try: from app.services.database import get_embedding_db_config @@ -23,6 +27,7 @@ def _get_db_emb_config() -> Dict[str, Any]: from app.services.database.connection import _resolve_default_embedding_config return _resolve_default_embedding_config() + _initial_cfg = _get_db_emb_config() REQUESTED_PROVIDER = _initial_cfg["provider"] @@ -32,6 +37,9 @@ def _get_db_emb_config() -> Dict[str, Any]: SPARSE_MODEL_NAME = _initial_cfg["sparse_model"] EMBEDDING_NUM_THREADS = _initial_cfg["threads"] EMBEDDING_BATCH_SIZE = _initial_cfg["batch_size"] +EMBEDDING_API_CONCURRENCY = _initial_cfg.get("api_concurrency", 4) +EMBEDDING_MAX_TOKENS_PER_REQUEST = _initial_cfg.get("max_tokens_per_request", 100000) +EMBEDDING_TPM_LIMIT = _initial_cfg.get("tpm_limit", 0) LITELLM_URL = _initial_cfg["litellm_url"] LITELLM_API_KEY = _initial_cfg.get( "litellm_api_key", @@ -48,12 +56,16 @@ def _get_db_emb_config() -> Dict[str, Any]: _sparse_lock = threading.Lock() _openai_client = None + def init_embeddings( provider: Optional[str] = None, dense_model: Optional[str] = None, sparse_model: Optional[str] = None, threads: Optional[int] = None, batch_size: Optional[int] = None, + api_concurrency: Optional[int] = None, + max_tokens_per_request: Optional[int] = None, + tpm_limit: Optional[int] = None, litellm_url: Optional[str] = None, litellm_api_key: Optional[str] = None, vision_ocr_model: Optional[str] = None, @@ -65,6 +77,7 @@ def init_embeddings( global REQUESTED_PROVIDER, EMBEDDING_PROVIDER global REQUESTED_DENSE_MODEL, DENSE_MODEL_NAME, SPARSE_MODEL_NAME global EMBEDDING_NUM_THREADS, EMBEDDING_BATCH_SIZE, LITELLM_URL, LITELLM_API_KEY + global EMBEDDING_API_CONCURRENCY, EMBEDDING_MAX_TOKENS_PER_REQUEST, EMBEDDING_TPM_LIMIT global VISION_OCR_MODEL, CHAT_MODEL, ALLOW_FALLBACK, FALLBACK_DENSE_MODEL cfg = _get_db_emb_config() @@ -75,7 +88,20 @@ def init_embeddings( DENSE_MODEL_NAME = REQUESTED_DENSE_MODEL SPARSE_MODEL_NAME = (sparse_model if sparse_model is not None else (cfg.get("sparse_model") or "Qdrant/bm25")).strip() EMBEDDING_NUM_THREADS = int(threads if threads is not None else cfg.get("threads", 2)) - EMBEDDING_BATCH_SIZE = int(batch_size if batch_size is not None else cfg.get("batch_size", 32)) + default_bs = 128 if REQUESTED_PROVIDER == "api" else 32 + if batch_size is not None: + EMBEDDING_BATCH_SIZE = int(batch_size) + else: + cfg_bs = cfg.get("batch_size") + if REQUESTED_PROVIDER == "api" and not os.getenv("EMBEDDING_BATCH_SIZE") and cfg_bs in (None, 32): + EMBEDDING_BATCH_SIZE = 128 + else: + EMBEDDING_BATCH_SIZE = int(cfg_bs or default_bs) + EMBEDDING_API_CONCURRENCY = int(api_concurrency if api_concurrency is not None else cfg.get("api_concurrency", 4)) + EMBEDDING_MAX_TOKENS_PER_REQUEST = int( + max_tokens_per_request if max_tokens_per_request is not None else cfg.get("max_tokens_per_request", 100000) + ) + EMBEDDING_TPM_LIMIT = int(tpm_limit if tpm_limit is not None else cfg.get("tpm_limit", 0)) LITELLM_URL = (litellm_url or cfg.get("litellm_url") or "http://litellm:4000/v1").strip() LITELLM_API_KEY = (litellm_api_key or cfg.get("litellm_api_key") or "dummy").strip() VISION_OCR_MODEL = (vision_ocr_model if vision_ocr_model is not None else (cfg.get("vision_ocr_model") or os.getenv("VISION_OCR_MODEL", ""))).strip() @@ -86,24 +112,24 @@ def init_embeddings( # Set underlying OpenMP / BLAS thread guard os.environ["OMP_NUM_THREADS"] = str(EMBEDDING_NUM_THREADS) - with _dense_lock: - _dense_model = None - with _sparse_lock: - _sparse_model = None - _openai_client = None - sparse_disabled = SPARSE_MODEL_NAME.lower() in ("", "none", "off", "disabled", "false") if REQUESTED_PROVIDER == "local": + _openai_client = None try: from fastembed import TextEmbedding, SparseTextEmbedding logger.info(f"Initializing FastEmbed dense model: {DENSE_MODEL_NAME} with threads={EMBEDDING_NUM_THREADS}") + new_dense = TextEmbedding(model_name=DENSE_MODEL_NAME, threads=EMBEDDING_NUM_THREADS) with _dense_lock: - _dense_model = TextEmbedding(model_name=DENSE_MODEL_NAME, threads=EMBEDDING_NUM_THREADS) + _dense_model = new_dense if not sparse_disabled: logger.info(f"Initializing FastEmbed sparse model: {SPARSE_MODEL_NAME} with threads={EMBEDDING_NUM_THREADS}") + new_sparse = SparseTextEmbedding(model_name=SPARSE_MODEL_NAME, threads=EMBEDDING_NUM_THREADS) + with _sparse_lock: + _sparse_model = new_sparse + else: with _sparse_lock: - _sparse_model = SparseTextEmbedding(model_name=SPARSE_MODEL_NAME, threads=EMBEDDING_NUM_THREADS) + _sparse_model = None logger.info("FastEmbed dense & sparse models initialized successfully with resource limits.") except Exception as e: logger.error(f"Failed to initialize local FastEmbed models: {e}") @@ -130,36 +156,45 @@ def init_embeddings( DENSE_MODEL_NAME = FALLBACK_DENSE_MODEL if EMBEDDING_PROVIDER == "api": + with _dense_lock: + _dense_model = None try: from openai import OpenAI from app.services.llm_client import get_llm_extra_headers logger.info(f"Using OpenAI/LiteLLM API embeddings at {LITELLM_URL} model {DENSE_MODEL_NAME}") - _openai_client = OpenAI( + new_client = OpenAI( base_url=LITELLM_URL, api_key=LITELLM_API_KEY, default_headers=get_llm_extra_headers(), ) - # Try initializing sparse model locally only if not explicitly disabled + _openai_client = new_client if not sparse_disabled: try: from fastembed import SparseTextEmbedding + new_sparse = SparseTextEmbedding(model_name=SPARSE_MODEL_NAME, threads=EMBEDDING_NUM_THREADS) with _sparse_lock: - _sparse_model = SparseTextEmbedding(model_name=SPARSE_MODEL_NAME, threads=EMBEDDING_NUM_THREADS) + _sparse_model = new_sparse except Exception: + with _sparse_lock: + _sparse_model = None + else: + with _sparse_lock: _sparse_model = None except Exception as oe: logger.error(f"Failed to initialize API client: {oe}") if REQUESTED_PROVIDER == "local" and ALLOW_FALLBACK: raise RuntimeError(f"Fallback to API failed: {oe}") from oe + # Auto-initialize on import safely try: init_embeddings() except Exception as _e: logger.warning(f"Initial embedding auto-init failed: {_e}") + def get_embedding_config() -> Dict[str, Any]: - """Returns current active embedding configuration along with system hardware stats.""" + """Returns current active embedding configuration along with system hardware stats and telemetry.""" try: from app.services.database import detect_system_resources sys_res = detect_system_resources() @@ -189,6 +224,8 @@ def get_embedding_config() -> Dict[str, Any]: else SPARSE_MODEL_NAME ) + telemetry = telemetry_tracker.get_telemetry_snapshot(EMBEDDING_API_CONCURRENCY) + return { "provider": EMBEDDING_PROVIDER, "requested_provider": REQUESTED_PROVIDER, @@ -201,6 +238,9 @@ def get_embedding_config() -> Dict[str, Any]: "effective_sparse_model": effective_sparse, "threads": EMBEDDING_NUM_THREADS, "batch_size": EMBEDDING_BATCH_SIZE, + "api_concurrency": EMBEDDING_API_CONCURRENCY, + "max_tokens_per_request": EMBEDDING_MAX_TOKENS_PER_REQUEST, + "tpm_limit": EMBEDDING_TPM_LIMIT, "litellm_url": LITELLM_URL, "vision_ocr_model": VISION_OCR_MODEL, "chat_model": CHAT_MODEL, @@ -208,8 +248,10 @@ def get_embedding_config() -> Dict[str, Any]: "fallback_dense_model": FALLBACK_DENSE_MODEL, "system_cpus": sys_res["cpus"], "system_memory_gb": sys_res["memory_gb"], + **telemetry, } + def get_dense_dim_for_config( provider: Optional[str] = None, dense_model: Optional[str] = None, @@ -257,6 +299,9 @@ def update_embedding_config( sparse_model: Optional[str] = None, threads: Optional[int] = None, batch_size: Optional[int] = None, + api_concurrency: Optional[int] = None, + max_tokens_per_request: Optional[int] = None, + tpm_limit: Optional[int] = None, litellm_url: Optional[str] = None, litellm_api_key: Optional[str] = None, vision_ocr_model: Optional[str] = None, @@ -265,18 +310,13 @@ def update_embedding_config( fallback_dense_model: Optional[str] = None, rebuild_index: bool = False, ) -> Dict[str, Any]: - """Updates embedding configuration in SQLite and hot-reloads models in memory. - - If model or dimensions change and existing vector points/indexed files exist, - requires rebuild_index=True to perform a coordinated reset and reindex. - """ + """Updates embedding configuration in SQLite and hot-reloads models in memory.""" from app.services.database import set_embedding_db_config, clear_indexing_metadata, get_db_connection from app.services.vector_store import get_vector_store target_provider = (provider or EMBEDDING_PROVIDER).lower().strip() target_dense = (dense_model or DENSE_MODEL_NAME).strip() - # Check if active index contains indexed content has_indexed_content = False try: store = get_vector_store() @@ -289,7 +329,6 @@ def update_embedding_config( except Exception as e: logger.debug(f"Error checking active index count: {e}") - # Calculate candidate dimension and check for changes new_dim = None try: new_dim = get_dense_dim_for_config( @@ -319,33 +358,24 @@ def update_embedding_config( ) raise ValueError(msg) - # Apply configuration update - set_embedding_db_config( - provider=provider, - dense_model=dense_model, - sparse_model=sparse_model, - threads=threads, - batch_size=batch_size, - litellm_url=litellm_url, - litellm_api_key=litellm_api_key, - vision_ocr_model=vision_ocr_model, - chat_model=chat_model, - allow_fallback=allow_fallback, - fallback_dense_model=fallback_dense_model, - ) - init_embeddings( - provider=provider, - dense_model=dense_model, - sparse_model=sparse_model, - threads=threads, - batch_size=batch_size, - litellm_url=litellm_url, - litellm_api_key=litellm_api_key, - vision_ocr_model=vision_ocr_model, - chat_model=chat_model, - allow_fallback=allow_fallback, - fallback_dense_model=fallback_dense_model, - ) + cfg_kwargs = { + "provider": provider, + "dense_model": dense_model, + "sparse_model": sparse_model, + "threads": threads, + "batch_size": batch_size, + "api_concurrency": api_concurrency, + "max_tokens_per_request": max_tokens_per_request, + "tpm_limit": tpm_limit, + "litellm_url": litellm_url, + "litellm_api_key": litellm_api_key, + "vision_ocr_model": vision_ocr_model, + "chat_model": chat_model, + "allow_fallback": allow_fallback, + "fallback_dense_model": fallback_dense_model, + } + set_embedding_db_config(**cfg_kwargs) + init_embeddings(**cfg_kwargs) if (dim_changed or model_changed or rebuild_index) and has_indexed_content: logger.info("Coordinating safe index rebuild for embedding model change...") @@ -364,52 +394,62 @@ def update_embedding_config( return get_embedding_config() + def get_dense_embedding(text: str) -> List[float]: """Generates dense vector embedding (e.g. 384 dimensions).""" res = get_dense_embeddings_batch([text]) return res[0] if res else [0.0] * 384 + def get_dense_embeddings_batch(texts: List[str], batch_size: Optional[int] = None) -> List[List[float]]: - """Generates a batch of dense vector embeddings efficiently in ONNX.""" + """Generates a batch of dense vector embeddings in ONNX or via parallel API slices.""" if not texts: return [] bs = batch_size or EMBEDDING_BATCH_SIZE if EMBEDDING_PROVIDER == "local": - if _dense_model is None: + model = _dense_model + if model is None: raise RuntimeError("Local dense embedding model is not initialized.") + start_t = time.monotonic() with _dense_lock: - embeddings = list(_dense_model.embed(texts, batch_size=bs)) - return [e.tolist() for e in embeddings] + embeddings = list(model.embed(texts, batch_size=bs)) + res = [e.tolist() for e in embeddings] + telemetry_tracker.record_batch_completion(len(texts), time.monotonic() - start_t) + return res elif EMBEDDING_PROVIDER == "api": - if _openai_client is None: + client = _openai_client + if client is None: raise RuntimeError("API Embedding client not initialized.") - api_bs = max(1, min(int(bs or 64), 128)) - all_embeddings: List[List[float]] = [] - for start in range(0, len(texts), api_bs): - batch_slice = texts[start:start + api_bs] - response = _openai_client.embeddings.create( - input=batch_slice, - model=DENSE_MODEL_NAME - ) - all_embeddings.extend([d.embedding for d in response.data]) - return all_embeddings + api_bs = max(1, min(int(bs or 128), 256)) + return execute_api_embeddings_batch( + client=client, + texts=texts, + model_name=DENSE_MODEL_NAME, + batch_size=api_bs, + max_concurrency=EMBEDDING_API_CONCURRENCY, + max_tokens_per_request=EMBEDDING_MAX_TOKENS_PER_REQUEST, + tpm_limit=EMBEDDING_TPM_LIMIT, + ) else: raise RuntimeError(f"Unknown embedding provider: {EMBEDDING_PROVIDER}") + def get_sparse_embedding(text: str) -> Optional[qmodels.SparseVector]: """Generates BM25 sparse vector embedding for a single text.""" res = get_sparse_embeddings_batch([text]) return res[0] if res else None + def get_sparse_embeddings_batch(texts: List[str], batch_size: Optional[int] = None) -> List[Optional[qmodels.SparseVector]]: """Generates a batch of BM25 sparse vector embeddings.""" if not texts: return [] bs = batch_size or EMBEDDING_BATCH_SIZE - if _sparse_model is not None: + model = _sparse_model + if model is not None: try: with _sparse_lock: - sparse_res = list(_sparse_model.embed(texts, batch_size=bs)) + sparse_res = list(model.embed(texts, batch_size=bs)) return [ qmodels.SparseVector( indices=s.indices.tolist(), @@ -421,18 +461,20 @@ def get_sparse_embeddings_batch(texts: List[str], batch_size: Optional[int] = No return [None] * len(texts) return [None] * len(texts) + def get_hybrid_embeddings(text: str) -> Dict[str, Any]: """Returns a dict with dense vector and optional sparse vector for Qdrant named vectors.""" res = get_hybrid_embeddings_batch([text]) return res[0] if res else {} + def get_hybrid_embeddings_batch(texts: List[str]) -> List[Dict[str, Any]]: """Returns a list of dicts with dense vector and optional sparse vector for batch upserts.""" if not texts: return [] dense_vecs = get_dense_embeddings_batch(texts) sparse_vecs = get_sparse_embeddings_batch(texts) - + results = [] for i, dense in enumerate(dense_vecs): vec_dict = {"dense": dense} @@ -441,6 +483,7 @@ def get_hybrid_embeddings_batch(texts: List[str]) -> List[Dict[str, Any]]: results.append(vec_dict) return results + def get_dense_dim() -> int: """Calculates embedding dimension from a sample text.""" sample = get_dense_embedding("dimension test") diff --git a/app/services/indexing/channel_pipeline.py b/app/services/indexing/channel_pipeline.py new file mode 100644 index 0000000..e81cfdd --- /dev/null +++ b/app/services/indexing/channel_pipeline.py @@ -0,0 +1,440 @@ +"""Bounded channel producer-consumer pipeline (modeled after C# System.Threading.Channels) +for overlapping file AST parsing, embedding cache lookup, parallel API slice embedding, +and database/vector store persistence.""" + +import logging +import queue +import threading +import time +from concurrent.futures import Future, ThreadPoolExecutor +from dataclasses import dataclass, field +from typing import Any, Callable, Dict, Generic, Iterator, List, Optional, Tuple, TypeVar + +from app.services.embedding_batcher import ( + estimate_token_count, + partition_texts_into_slices, + telemetry_tracker, +) +from app.services.memory_utils import trim_memory +from app.services.vector_store import VectorDocument + +logger = logging.getLogger("contextcortex.indexer.pipeline") + +T = TypeVar("T") +_CHANNEL_SENTINEL = object() + + +class BoundedChannel(Generic[T]): + """Thread-safe bounded channel with backpressure, completion, and error propagation + (modeled after C# System.Threading.Channels.Channel.CreateBounded).""" + + def __init__(self, capacity: int = 64) -> None: + self.capacity = max(1, int(capacity)) + self._queue: "queue.Queue[Any]" = queue.Queue(maxsize=self.capacity) + self._lock = threading.Lock() + self._completed = False + self._cancelled = False + self._error: Optional[BaseException] = None + + def write(self, item: T, timeout: float = 0.25) -> bool: + """Writes an item to the channel, blocking while full unless completed or cancelled.""" + while True: + with self._lock: + if self._cancelled or self._error is not None: + return False + if self._completed: + raise RuntimeError("Cannot write to a completed BoundedChannel.") + try: + self._queue.put(item, timeout=timeout) + return True + except queue.Full: + continue + + def complete(self) -> None: + """Marks the channel as complete (no further items will be written).""" + with self._lock: + if self._completed: + return + self._completed = True + self._put_sentinel() + + def cancel(self) -> None: + """Cancels the channel and unblocks any waiting readers or writers.""" + with self._lock: + self._cancelled = True + self._completed = True + self._drain_and_put_sentinel() + + def fail(self, exc: BaseException) -> None: + """Signals an upstream failure and unblocks readers and writers.""" + with self._lock: + if self._error is None: + self._error = exc + self._completed = True + self._drain_and_put_sentinel() + + @property + def error(self) -> Optional[BaseException]: + with self._lock: + return self._error + + def _put_sentinel(self) -> None: + while True: + try: + self._queue.put(_CHANNEL_SENTINEL, timeout=0.05) + return + except queue.Full: + try: + self._queue.get_nowait() + except queue.Empty: + pass + + def _drain_and_put_sentinel(self) -> None: + while True: + try: + self._queue.get_nowait() + except queue.Empty: + break + self._put_sentinel() + + def read_all(self) -> Iterator[T]: + """Yields items until the channel is completed and drained, raising any upstream error.""" + while True: + with self._lock: + if self._cancelled: + return + if self._error is not None and self._queue.empty(): + raise self._error + try: + item = self._queue.get(timeout=0.1) + except queue.Empty: + with self._lock: + if self._error is not None: + raise self._error + if self._completed and self._queue.empty(): + return + continue + + if item is _CHANNEL_SENTINEL: + with self._lock: + if self._error is not None: + raise self._error + self._put_sentinel() + return + + yield item + + +@dataclass +class ParsedFileItem: + """Single parsed file output produced by Stage 1 (AST/Markdown chunker).""" + + filepath: str + rel_path: str + repo: str + points: List[VectorDocument] = field(default_factory=list) + symbols: List[Dict[str, Any]] = field(default_factory=list) + summary_tuple: Optional[Tuple] = None + relationships: List[Dict[str, Any]] = field(default_factory=list) + routes: List[Dict[str, Any]] = field(default_factory=list) + calls: List[Dict[str, Any]] = field(default_factory=list) + indexed_file_row: Optional[Tuple] = None + extra_meta: Dict[str, Any] = field(default_factory=dict) + + +@dataclass +class PendingPersistBatch: + """Coalesced batch of parsed files with in-flight embedding slice futures from Stage 2/3.""" + + items: List[ParsedFileItem] + point_hashes: List[Tuple[VectorDocument, str]] + cached_map: Dict[str, Dict[str, Any]] + unseen_hashes: List[str] + slice_futures: List[Tuple[List[str], Future]] + active_model: str + use_raw_hybrid_future: bool = False + + +class ChannelIngestionPipeline: + """4-stage bounded channel pipeline: + Stage 1 (Producer): Parses files into AST/Markdown chunks -> parsed_channel. + Stage 2 (Coalescer & Cache Filter): Queries embedding_cache by SHA256(content), slices cache misses, + and submits slice tasks immediately to Stage 3 -> persist_channel. + Stage 3 (Parallel API Pool): Persistent ThreadPoolExecutor running concurrent API embedding slices + across batches with HTTP 429 Retry-After & AIMD concurrency control. + Stage 4 (Consumer): Awaits completed slice futures per batch, caches vectors, and persists to DB/vector store. + """ + + def __init__( + self, + persist_batch_fn: Callable[[List[ParsedFileItem]], None], + file_batch_size: int = 25, + parsed_channel_capacity: int = 64, + persist_channel_capacity: int = 8, + ) -> None: + self.persist_batch_fn = persist_batch_fn + self.file_batch_size = max(1, int(file_batch_size)) + self.parsed_channel: BoundedChannel[ParsedFileItem] = BoundedChannel(capacity=parsed_channel_capacity) + self.persist_channel: BoundedChannel[PendingPersistBatch] = BoundedChannel(capacity=persist_channel_capacity) + + def run(self, producer_fn: Callable[[BoundedChannel[ParsedFileItem]], None]) -> None: + """Runs the 4-stage pipeline to completion, propagating any stage exception.""" + import app.services.embeddings as emb_service + import app.services.indexing.processor as proc_service + from app.services.database import get_cached_embeddings_batch, set_cached_embeddings_batch + from app.services.embedding_batcher import execute_api_embeddings_batch + + max_conc = max(1, int(getattr(emb_service, "EMBEDDING_API_CONCURRENCY", 4) or 4)) + inflight_hash_futures: Dict[str, Future] = {} + inflight_lock = threading.Lock() + + embed_executor = ThreadPoolExecutor(max_workers=max(4, max_conc * 2)) + + def _stage1_producer() -> None: + try: + producer_fn(self.parsed_channel) + self.parsed_channel.complete() + except BaseException as exc: + logger.error(f"Pipeline Stage 1 (Producer) failed: {exc}") + self.parsed_channel.fail(exc) + self.persist_channel.fail(exc) + + def _stage2_coalescer() -> None: + try: + batch_items: List[ParsedFileItem] = [] + for item in self.parsed_channel.read_all(): + batch_items.append(item) + if len(batch_items) >= self.file_batch_size: + self._dispatch_batch( + batch_items, + embed_executor, + inflight_hash_futures, + inflight_lock, + emb_service, + proc_service, + get_cached_embeddings_batch, + execute_api_embeddings_batch, + ) + batch_items = [] + if batch_items: + self._dispatch_batch( + batch_items, + embed_executor, + inflight_hash_futures, + inflight_lock, + emb_service, + proc_service, + get_cached_embeddings_batch, + execute_api_embeddings_batch, + ) + self.persist_channel.complete() + except BaseException as exc: + logger.error(f"Pipeline Stage 2 (Coalescer) failed: {exc}") + self.parsed_channel.cancel() + self.persist_channel.fail(exc) + + t1 = threading.Thread(target=_stage1_producer, name="cc-pipeline-stage1-parser", daemon=True) + t2 = threading.Thread(target=_stage2_coalescer, name="cc-pipeline-stage2-coalescer", daemon=True) + t1.start() + t2.start() + + try: + for pending_batch in self.persist_channel.read_all(): + self._finalize_and_persist_batch( + pending_batch, + proc_service, + set_cached_embeddings_batch, + ) + t1.join() + t2.join() + if self.parsed_channel.error is not None: + raise self.parsed_channel.error + if self.persist_channel.error is not None: + raise self.persist_channel.error + finally: + self.parsed_channel.cancel() + self.persist_channel.cancel() + embed_executor.shutdown(wait=True, cancel_futures=False) + + def _dispatch_batch( + self, + items: List[ParsedFileItem], + embed_executor: ThreadPoolExecutor, + inflight_hash_futures: Dict[str, Future], + inflight_lock: threading.Lock, + emb_service: Any, + proc_service: Any, + get_cached_embeddings_batch: Callable, + execute_api_embeddings_batch: Callable, + ) -> None: + active_model = proc_service._get_indexer_attr("DENSE_MODEL_NAME", emb_service.DENSE_MODEL_NAME) + point_hashes: List[Tuple[VectorDocument, str]] = [] + hash_to_text: Dict[str, str] = {} + + for item in items: + for pt in item.points: + embed_text = pt.metadata.pop("_embed_text", None) if isinstance(pt.metadata, dict) else None + if pt.dense_vector is not None: + continue + raw_content = (pt.text or "").strip() + formatted = embed_text or raw_content + if not formatted: + continue + h = proc_service.compute_text_hash(raw_content) + point_hashes.append((pt, h)) + if h not in hash_to_text: + hash_to_text[h] = formatted + + if not point_hashes: + self.persist_channel.write( + PendingPersistBatch( + items=items, + point_hashes=[], + cached_map={}, + unseen_hashes=[], + slice_futures=[], + active_model=active_model, + ) + ) + return + + all_hashes = list(hash_to_text.keys()) + cached_map = get_cached_embeddings_batch(all_hashes, model_name=active_model) + + unseen_hashes: List[str] = [] + unseen_texts: List[str] = [] + for h in all_hashes: + if h not in cached_map: + unseen_hashes.append(h) + unseen_texts.append(hash_to_text[h]) + + embed_fn = proc_service._get_indexer_attr( + "get_hybrid_embeddings_batch", + proc_service.get_hybrid_embeddings_batch, + ) + is_default_embed_fn = embed_fn is emb_service.get_hybrid_embeddings_batch + is_api_provider = getattr(emb_service, "EMBEDDING_PROVIDER", "local") == "api" + + slice_futures: List[Tuple[List[str], Future]] = [] + use_raw_hybrid = False + + if unseen_hashes: + if is_default_embed_fn and is_api_provider: + api_bs = max(1, min(int(getattr(emb_service, "EMBEDDING_BATCH_SIZE", 128) or 128), 256)) + max_tok = max(1, int(getattr(emb_service, "EMBEDDING_MAX_TOKENS_PER_REQUEST", 100000) or 100000)) + max_conc = max(1, int(getattr(emb_service, "EMBEDDING_API_CONCURRENCY", 4) or 4)) + tpm = max(0, int(getattr(emb_service, "EMBEDDING_TPM_LIMIT", 0) or 0)) + client = getattr(emb_service, "_openai_client", None) + model_name = getattr(emb_service, "DENSE_MODEL_NAME", active_model) + + # Partition (hash, text) pairs into token- and size-bounded slices + slices_pairs: List[Tuple[List[str], List[str]]] = [] + cur_h: List[str] = [] + cur_t: List[str] = [] + cur_tok = 0 + for h, t in zip(unseen_hashes, unseen_texts): + t_tok = estimate_token_count(t) + if cur_t and (len(cur_t) >= api_bs or (cur_tok + t_tok) > max_tok): + slices_pairs.append((cur_h, cur_t)) + cur_h, cur_t, cur_tok = [h], [t], t_tok + else: + cur_h.append(h) + cur_t.append(t) + cur_tok += t_tok + if cur_t: + slices_pairs.append((cur_h, cur_t)) + + active_futs: List[Future] = [] + for s_hashes, s_texts in slices_pairs: + active_futs = [f for f in active_futs if not f.done()] + eff_conc = telemetry_tracker.get_effective_concurrency(max_conc) + while len(active_futs) >= eff_conc: + time.sleep(0.005) + active_futs = [f for f in active_futs if not f.done()] + eff_conc = telemetry_tracker.get_effective_concurrency(max_conc) + + fut = embed_executor.submit( + execute_api_embeddings_batch, + client=client, + texts=s_texts, + model_name=model_name, + batch_size=len(s_texts), + max_concurrency=1, + max_tokens_per_request=max_tok, + tpm_limit=tpm, + ) + active_futs.append(fut) + slice_futures.append((s_hashes, fut)) + else: + use_raw_hybrid = True + fut = embed_executor.submit(embed_fn, unseen_texts) + slice_futures.append((unseen_hashes, fut)) + + self.persist_channel.write( + PendingPersistBatch( + items=items, + point_hashes=point_hashes, + cached_map=cached_map, + unseen_hashes=unseen_hashes, + slice_futures=slice_futures, + active_model=active_model, + use_raw_hybrid_future=use_raw_hybrid, + ) + ) + + def _finalize_and_persist_batch( + self, + batch: PendingPersistBatch, + proc_service: Any, + set_cached_embeddings_batch: Callable, + ) -> None: + if batch.point_hashes: + items_to_cache: List[Dict[str, Any]] = [] + if batch.use_raw_hybrid_future: + for s_hashes, fut in batch.slice_futures: + new_vectors = fut.result() + for i, h in enumerate(s_hashes): + nv = new_vectors[i] if i < len(new_vectors) and isinstance(new_vectors[i], dict) else {} + dense_v = nv.get("dense") + s_indices, s_values = proc_service._extract_sparse_tuple(nv.get("sparse")) + batch.cached_map[h] = { + "dense": dense_v, + "sparse_indices": s_indices, + "sparse_values": s_values, + } + items_to_cache.append({ + "chunk_hash": h, + "dense_vector": dense_v, + "sparse_indices": s_indices, + "sparse_values": s_values, + "model_name": batch.active_model, + }) + else: + for s_hashes, fut in batch.slice_futures: + dense_vecs = fut.result() + for i, h in enumerate(s_hashes): + dense_v = dense_vecs[i] if i < len(dense_vecs) else None + batch.cached_map[h] = { + "dense": dense_v, + "sparse_indices": None, + "sparse_values": None, + } + items_to_cache.append({ + "chunk_hash": h, + "dense_vector": dense_v, + "sparse_indices": None, + "sparse_values": None, + "model_name": batch.active_model, + }) + + if items_to_cache: + set_cached_embeddings_batch(items_to_cache) + trim_memory() + + for pt, h in batch.point_hashes: + bv = batch.cached_map.get(h, {}) + pt.dense_vector = bv.get("dense") + pt.sparse_indices = bv.get("sparse_indices") + pt.sparse_values = bv.get("sparse_values") + + self.persist_batch_fn(batch.items) diff --git a/app/services/indexing/git_progress.py b/app/services/indexing/git_progress.py index 0e3b053..d2ab91f 100644 --- a/app/services/indexing/git_progress.py +++ b/app/services/indexing/git_progress.py @@ -18,6 +18,11 @@ class GitSyncJob: processed_files: int = 0 total_files: int = 0 percent: int = 0 + chunks_embedded: int = 0 + embedding_seconds: float = 0.0 + embedding_chunks_per_sec: float = 0.0 + rate_limit_wait_seconds: float = 0.0 + rate_limit_hits: int = 0 started_at: float = field(default_factory=time.time) updated_at: float = field(default_factory=time.time) error: Optional[str] = None @@ -60,6 +65,11 @@ def get_or_create_job(self, repo_id: int, repo_name: str) -> GitSyncJob: job.processed_files = 0 job.total_files = 0 job.percent = 0 + job.chunks_embedded = 0 + job.embedding_seconds = 0.0 + job.embedding_chunks_per_sec = 0.0 + job.rate_limit_wait_seconds = 0.0 + job.rate_limit_hits = 0 job.error = None job.cancelled = False job.started_at = time.time() @@ -69,6 +79,25 @@ def get_or_create_job(self, repo_id: int, repo_name: str) -> GitSyncJob: self._broadcast({"type": "progress", "data": payload}) return job_obj + def update_embedding_telemetry(self, telemetry: Dict[str, Any], repo_id: Optional[int] = None) -> None: + with self._lock: + targets = ( + [self.jobs[repo_id]] + if repo_id is not None and repo_id in self.jobs + else [j for j in self.jobs.values() if j.status == "syncing"] + ) + for job in targets: + job.chunks_embedded = int(telemetry.get("chunks_embedded", job.chunks_embedded)) + job.embedding_seconds = float(telemetry.get("embedding_seconds", job.embedding_seconds)) + job.embedding_chunks_per_sec = float( + telemetry.get("embedding_chunks_per_sec", job.embedding_chunks_per_sec) + ) + job.rate_limit_wait_seconds = float( + telemetry.get("rate_limit_wait_seconds", job.rate_limit_wait_seconds) + ) + job.rate_limit_hits = int(telemetry.get("rate_limit_hits", job.rate_limit_hits)) + job.updated_at = time.time() + def update_step( self, repo_id: int, diff --git a/app/services/indexing/git_syncer.py b/app/services/indexing/git_syncer.py index b284025..83d2b78 100644 --- a/app/services/indexing/git_syncer.py +++ b/app/services/indexing/git_syncer.py @@ -252,83 +252,134 @@ def sync_single_git_repo(repo_id: int, force: bool = False): logger.info(f"Ingesting {total_files} added/modified files for repo '{repo_name}'...") progress_tracker.log(repo_id, "INFO", f"Ingesting {total_files} added/modified files...") + from app.services.indexing.channel_pipeline import ( + BoundedChannel, + ChannelIngestionPipeline, + ParsedFileItem, + ) + BATCH_SIZE = 25 - batch_points = [] - batch_symbols = [] - batch_routes = [] - batch_calls = [] - batch_summaries = [] - batch_relationships = [] - batch_indexed_files = [] total_vectors_count = 0 total_symbols_count = 0 + was_cancelled = False + + def _produce_parsed_files(channel: BoundedChannel[ParsedFileItem]) -> None: + nonlocal was_cancelled + for idx, full_path in enumerate(files_to_sync): + if progress_tracker.is_cancelled(repo_id): + logger.info(f"Sync cancelled for repo '{repo_name}' during file ingestion at file {idx + 1}/{total_files}") + progress_tracker.log(repo_id, "WARN", f"Sync cancelled by user at file {idx + 1}/{total_files}") + was_cancelled = True + channel.cancel() + return + + rel_path = os.path.relpath(full_path, temp_dir).replace("\\", "/") + progress_tracker.update_step( + repo_id, 4, f"Parsing Files ({idx + 1}/{total_files})", + current_file=rel_path, processed=idx + 1, total=total_files, + ) + doc_type = "code" if is_code_file(full_path) else "doc" + lang = detect_language(full_path) + db_filepath = f"{repo_name}://{rel_path}" + + try: + content = read_source_text(full_path) + file_hash = proc_service.compute_text_hash(content) + + try: + points, symbols, summary_tuple, rels, routes, calls = proc_service.process_file_content( + filepath=db_filepath, + rel_path=rel_path, + content=content, + repo=repo_name, + doc_type=doc_type, + git_url=git_url, + commit_sha=commit_sha, + provider=provider, + embed=False, + ) + except TypeError: + points, symbols, summary_tuple, rels, routes, calls = proc_service.process_file_content( + filepath=db_filepath, + rel_path=rel_path, + content=content, + repo=repo_name, + doc_type=doc_type, + git_url=git_url, + commit_sha=commit_sha, + provider=provider, + ) + + channel.write( + ParsedFileItem( + filepath=db_filepath, + rel_path=rel_path, + repo=repo_name, + points=points, + symbols=symbols, + summary_tuple=summary_tuple, + relationships=rels, + routes=routes, + calls=calls, + indexed_file_row=(db_filepath, repo_name, doc_type, lang, commit_sha, 0.0, file_hash), + ) + ) - # Stage 4: Parsing AST & Routes - for idx, full_path in enumerate(files_to_sync): - if progress_tracker.is_cancelled(repo_id): - logger.info(f"Sync cancelled for repo '{repo_name}' during file ingestion at file {idx + 1}/{total_files}") - progress_tracker.log(repo_id, "WARN", f"Sync cancelled by user at file {idx + 1}/{total_files}") - return - - rel_path = os.path.relpath(full_path, temp_dir).replace("\\", "/") - progress_tracker.update_step( - repo_id, 4, f"Parsing Files ({idx + 1}/{total_files})", - current_file=rel_path, processed=idx + 1, total=total_files, - ) - doc_type = "code" if is_code_file(full_path) else "doc" - lang = detect_language(full_path) - db_filepath = f"{repo_name}://{rel_path}" + progress_tracker.log(repo_id, "INFO", f"[{idx + 1}/{total_files}] Ingested {rel_path} (+{len(points)} chunks, +{len(symbols)} symbols)") + if (idx + 1) % 10 == 0 or (idx + 1) == total_files: + pct = int(((idx + 1) / total_files) * 100) + logger.info(f"[{idx + 1}/{total_files}] ({pct}%) Ingesting {rel_path} (+{len(points)} chunks, +{len(symbols)} symbols)") + except Exception as fe: + logger.error(f"Error parsing file {rel_path} in repo '{repo_name}': {fe}") + progress_tracker.log(repo_id, "ERROR", f"Error parsing {rel_path}: {fe}") + + def _persist_parsed_batch(batch_items: list[ParsedFileItem]) -> None: + nonlocal total_vectors_count, total_symbols_count + batch_points = [] + batch_symbols = [] + batch_routes = [] + batch_calls = [] + batch_summaries = [] + batch_relationships = [] + batch_indexed_files = [] + + for item in batch_items: + batch_points.extend(item.points) + batch_symbols.extend(item.symbols) + batch_routes.extend(item.routes) + batch_calls.extend(item.calls) + if item.summary_tuple is not None: + batch_summaries.append(item.summary_tuple) + batch_relationships.extend(item.relationships) + if item.indexed_file_row is not None: + batch_indexed_files.append(item.indexed_file_row) + + if batch_points: + upsert_ok = store.upsert_documents(batch_points) + if not upsert_ok: + logger.error(f"Failed to upsert points to vector store during git indexing for repo '{repo_name}'.") + raise RuntimeError(f"Vector store upsert failed during git indexing for repo '{repo_name}'") + total_vectors_count += len(batch_points) - try: - content = read_source_text(full_path) - file_hash = proc_service.compute_text_hash(content) - - points, symbols, summary_tuple, rels, routes, calls = proc_service.process_file_content( - filepath=db_filepath, - rel_path=rel_path, - content=content, - repo=repo_name, - doc_type=doc_type, - git_url=git_url, - commit_sha=commit_sha, - provider=provider, + with db_service.get_db_connection() as conn: + total_symbols_count += persist_sync_batch( + conn, + batch_indexed_files, + batch_summaries, + batch_symbols, + batch_relationships, + batch_routes, + batch_calls, ) - batch_points.extend(points) - batch_symbols.extend(symbols) - batch_routes.extend(routes) - batch_calls.extend(calls) - batch_summaries.append(summary_tuple) - batch_relationships.extend(rels) - batch_indexed_files.append((db_filepath, repo_name, doc_type, lang, commit_sha, 0.0, file_hash)) - - progress_tracker.log(repo_id, "INFO", f"[{idx + 1}/{total_files}] Ingested {rel_path} (+{len(points)} chunks, +{len(symbols)} symbols)") - if (idx + 1) % 10 == 0 or (idx + 1) == total_files: - pct = int(((idx + 1) / total_files) * 100) - logger.info(f"[{idx + 1}/{total_files}] ({pct}%) Ingesting {rel_path} (+{len(points)} chunks, +{len(symbols)} symbols)") - except Exception as fe: - logger.error(f"Error parsing file {rel_path} in repo '{repo_name}': {fe}") - progress_tracker.log(repo_id, "ERROR", f"Error parsing {rel_path}: {fe}") - - if (idx + 1) % BATCH_SIZE == 0 or (idx + 1) == total_files: - if batch_points: - upsert_ok = store.upsert_documents(batch_points) - if not upsert_ok: - logger.error(f"Failed to upsert points to vector store during git indexing for repo '{repo_name}'.") - raise RuntimeError(f"Vector store upsert failed during git indexing for repo '{repo_name}'") - total_vectors_count += len(batch_points) - batch_points.clear() - - with db_service.get_db_connection() as conn: - total_symbols_count += persist_sync_batch( - conn, - batch_indexed_files, - batch_summaries, - batch_symbols, - batch_relationships, - batch_routes, - batch_calls, - ) - trim_memory() + trim_memory() + + pipeline = ChannelIngestionPipeline( + persist_batch_fn=_persist_parsed_batch, + file_batch_size=BATCH_SIZE, + ) + pipeline.run(_produce_parsed_files) + if was_cancelled or progress_tracker.is_cancelled(repo_id): + return # Stage 5: Vector Indexing & Cleanup progress_tracker.update_step(repo_id, 5, "Upserting Embeddings & Finalizing", pct=90) diff --git a/app/services/indexing/local_syncer.py b/app/services/indexing/local_syncer.py index 0abe61a..4a61a4c 100644 --- a/app/services/indexing/local_syncer.py +++ b/app/services/indexing/local_syncer.py @@ -129,6 +129,12 @@ def _sync_local_paths_internal(force: bool = False, repo_filter: str = None): except Exception: pass + from app.services.indexing.channel_pipeline import ( + BoundedChannel, + ChannelIngestionPipeline, + ParsedFileItem, + ) + all_points = [] all_symbols = [] all_routes = [] @@ -138,40 +144,80 @@ def _sync_local_paths_internal(force: bool = False, repo_filter: str = None): files_to_update_cache = [] files_to_delete = [] - for filepath, (repo, category, base_dir) in found_files.items(): - try: - mtime = os.path.getmtime(filepath) - with db_service.get_db_connection() as conn: - cached = conn.execute("SELECT mtime FROM indexed_files WHERE filepath = ?", (filepath,)).fetchone() - - if not force and cached and abs(cached["mtime"] - mtime) < 0.01: - continue # Cached & unchanged + def _produce_local_files(channel: BoundedChannel[ParsedFileItem]) -> None: + for filepath, (repo, category, base_dir) in found_files.items(): + try: + mtime = os.path.getmtime(filepath) + with db_service.get_db_connection() as conn: + cached = conn.execute("SELECT mtime FROM indexed_files WHERE filepath = ?", (filepath,)).fetchone() - rel_path = os.path.relpath(filepath, base_dir).replace("\\", "/") - doc_type = "code" if is_code_file(filepath) else "doc" - lang = detect_language(filepath) + if not force and cached and abs(cached["mtime"] - mtime) < 0.01: + continue # Cached & unchanged - content = read_source_text(filepath) + rel_path = os.path.relpath(filepath, base_dir).replace("\\", "/") + doc_type = "code" if is_code_file(filepath) else "doc" + lang = detect_language(filepath) + content = read_source_text(filepath) - points, symbols, summary_tuple, rels, routes, calls = proc_service.process_file_content( - filepath=filepath, - rel_path=rel_path, - content=content, - repo=repo, - doc_type=doc_type, - category_override=category - ) + try: + points, symbols, summary_tuple, rels, routes, calls = proc_service.process_file_content( + filepath=filepath, + rel_path=rel_path, + content=content, + repo=repo, + doc_type=doc_type, + category_override=category, + embed=False, + ) + except TypeError: + points, symbols, summary_tuple, rels, routes, calls = proc_service.process_file_content( + filepath=filepath, + rel_path=rel_path, + content=content, + repo=repo, + doc_type=doc_type, + category_override=category, + ) - files_to_delete.append((filepath, repo)) - all_points.extend(points) - all_symbols.extend(symbols) - all_routes.extend(routes) - all_calls.extend(calls) - all_summaries.append(summary_tuple) - all_relationships.extend(rels) - files_to_update_cache.append((filepath, repo, doc_type, lang, None, mtime)) - except Exception as e: - logger.error(f"Error processing local file {filepath}: {e}") + channel.write( + ParsedFileItem( + filepath=filepath, + rel_path=rel_path, + repo=repo, + points=points, + symbols=symbols, + summary_tuple=summary_tuple, + relationships=rels, + routes=routes, + calls=calls, + indexed_file_row=(filepath, repo, doc_type, lang, None, mtime), + ) + ) + except Exception as e: + logger.error(f"Error processing local file {filepath}: {e}") + + def _collect_embedded_local_batch(batch_items: list[ParsedFileItem]) -> None: + for item in batch_items: + files_to_delete.append((item.filepath, item.repo)) + all_points.extend(item.points) + all_symbols.extend(item.symbols) + all_routes.extend(item.routes) + all_calls.extend(item.calls) + if item.summary_tuple is not None: + all_summaries.append(item.summary_tuple) + all_relationships.extend(item.relationships) + if item.indexed_file_row is not None: + files_to_update_cache.append(item.indexed_file_row) + + try: + pipeline = ChannelIngestionPipeline( + persist_batch_fn=_collect_embedded_local_batch, + file_batch_size=50, + ) + pipeline.run(_produce_local_files) + except Exception as e: + logger.error(f"Failed to embed local files in pipeline: {e}") + return False # Purge old points for modified files store = vs_service.get_vector_store() diff --git a/app/services/indexing/processor.py b/app/services/indexing/processor.py index 5f05f99..8a23109 100644 --- a/app/services/indexing/processor.py +++ b/app/services/indexing/processor.py @@ -109,77 +109,108 @@ def get_dynamic_catalog_description() -> str: # FILE & REPOSITORY PROCESSORS # ---------------------------------------------------- +def _extract_sparse_tuple(sparse_obj: Any) -> Tuple[Optional[List[int]], Optional[List[float]]]: + if sparse_obj is None: + return None, None + if hasattr(sparse_obj, "indices") and hasattr(sparse_obj, "values"): + return list(sparse_obj.indices), list(sparse_obj.values) + if isinstance(sparse_obj, dict): + return list(sparse_obj.get("indices", [])), list(sparse_obj.get("values", [])) + return None, None + + def _resolve_cached_or_compute_embeddings( - chunks: List[Dict[str, Any]], - texts_to_embed: List[str] + chunks: List[Dict[str, Any]], + texts_to_embed: List[str], ) -> List[Dict[str, Any]]: """Resolves embeddings for chunks using SQLite cache, computing only cache misses.""" if not chunks: return [] - + active_model = _get_indexer_attr("DENSE_MODEL_NAME", DENSE_MODEL_NAME) chunk_hashes = [compute_text_hash(c.get("content", "").strip()) for c in chunks] cached_map = get_cached_embeddings_batch(chunk_hashes, model_name=active_model) - - # Collect unique missing hashes + missing_hash_to_text = {} for idx, h in enumerate(chunk_hashes): if h not in cached_map and h not in missing_hash_to_text: missing_hash_to_text[h] = texts_to_embed[idx] - + if missing_hash_to_text: unseen_hashes = list(missing_hash_to_text.keys()) unseen_texts = list(missing_hash_to_text.values()) embed_fn = _get_indexer_attr("get_hybrid_embeddings_batch", get_hybrid_embeddings_batch) new_vectors = embed_fn(unseen_texts) - + items_to_cache = [] for i, h in enumerate(unseen_hashes): nv = new_vectors[i] if i < len(new_vectors) and isinstance(new_vectors[i], dict) else {} dense_v = nv.get("dense") - sparse_obj = nv.get("sparse") - s_indices = None - s_values = None - if sparse_obj is not None: - if hasattr(sparse_obj, "indices") and hasattr(sparse_obj, "values"): - s_indices = list(sparse_obj.indices) - s_values = list(sparse_obj.values) - elif isinstance(sparse_obj, dict): - s_indices = list(sparse_obj.get("indices", [])) - s_values = list(sparse_obj.get("values", [])) - - cached_map[h] = { - "dense": dense_v, - "sparse_indices": s_indices, - "sparse_values": s_values - } + s_indices, s_values = _extract_sparse_tuple(nv.get("sparse")) + cached_map[h] = {"dense": dense_v, "sparse_indices": s_indices, "sparse_values": s_values} items_to_cache.append({ "chunk_hash": h, "dense_vector": dense_v, "sparse_indices": s_indices, "sparse_values": s_values, - "model_name": active_model + "model_name": active_model, }) - + if items_to_cache: set_cached_embeddings_batch(items_to_cache) trim_memory() - + return [cached_map.get(h, {}) for h in chunk_hashes] +_DEFAULT_RESOLVE_CACHED_OR_COMPUTE = _resolve_cached_or_compute_embeddings + + +def embed_points_across_files(points: List[VectorDocument]) -> None: + """Computes and populates dense/sparse embeddings across a multi-file batch of VectorDocuments.""" + if not points: + return + + pending: List[Tuple[VectorDocument, str, str]] = [] + for pt in points: + embed_text = pt.metadata.pop("_embed_text", None) if isinstance(pt.metadata, dict) else None + if pt.dense_vector is not None: + continue + raw_content = (pt.text or "").strip() + formatted = embed_text or raw_content + if not formatted: + continue + pending.append((pt, compute_text_hash(raw_content), formatted)) + + if not pending: + return + + chunks = [{"content": pt.text} for pt, _, _ in pending] + texts_to_embed = [fmt for _, _, fmt in pending] + resolved = _resolve_cached_or_compute_embeddings(chunks, texts_to_embed) + + for idx, (pt, _, _) in enumerate(pending): + bv = resolved[idx] if idx < len(resolved) and isinstance(resolved[idx], dict) else {} + pt.dense_vector = bv.get("dense") + pt.sparse_indices = bv.get("sparse_indices") + pt.sparse_values = bv.get("sparse_values") + + def process_file_content( - filepath: str, - rel_path: str, - content: str, - repo: str, - doc_type: str, - git_url: Optional[str] = None, + filepath: str, + rel_path: str, + content: str, + repo: str, + doc_type: str, + git_url: Optional[str] = None, commit_sha: Optional[str] = None, category_override: Optional[str] = None, - provider: Optional[str] = None + provider: Optional[str] = None, + embed: bool = True, ) -> Tuple[List[VectorDocument], List[Dict[str, Any]], Tuple, List[Dict[str, Any]], List[Dict[str, Any]], List[Dict[str, Any]]]: """Processes a single file into vector documents, AST symbols, summary metadata, AST relationships, API routes, and client calls.""" + if _resolve_cached_or_compute_embeddings is not _DEFAULT_RESOLVE_CACHED_OR_COMPUTE: + embed = True points: List[VectorDocument] = [] ast_symbols = [] ast_relationships = [] @@ -241,23 +272,14 @@ def process_file_content( logger.warning(f"Auto-summarization failed for {filepath}: {se}") summary_tuple = ( - filepath, - repo, - title, - folder, - category, - json.dumps([]), - json.dumps([]), - json.dumps([]), - summary_text, - mtime + filepath, repo, title, folder, category, + json.dumps([]), json.dumps([]), json.dumps([]), summary_text, mtime ) return points, ast_symbols, summary_tuple, ast_relationships, api_routes, api_calls if doc_type in ("doc", "pdf"): body_line_offset = 0 if filepath.endswith((".md", ".txt")): - # Check if file is in standard ADR directories or follows ADR naming norm_rel = rel_path.replace("\\", "/").lower() if any(adr_dir in norm_rel for adr_dir in ["docs/adr/", "docs/decisions/", ".adr/", "doc/architecture/decisions/", "/adr/"]): try: @@ -278,14 +300,13 @@ def process_file_content( tags = [t.strip() for t in raw_tags.split(",")] if isinstance(raw_tags, str) else raw_tags except Exception: pass - + doc_relationships = extract_markdown_doc_links(content, filepath=filepath, repo=repo) ast_relationships = [r.model_dump() for r in doc_relationships] chunks_models = chunk_markdown(content, max_chars=CHUNK_SIZE, overlap=CHUNK_OVERLAP) chunks = [c.model_dump() for c in chunks_models] if body_line_offset: - # Chunk lines count from the body; shift them so they are file line numbers. for item in chunks + ast_relationships: for line_key in ("start_line", "end_line", "line_number"): if isinstance(item.get(line_key), int): @@ -294,31 +315,23 @@ def process_file_content( headings = [c["heading"] for c in valid_chunks if c.get("heading") and c["heading"] != "Root"] keywords = extract_keywords_from_text(content, title, headings, tags) - texts_to_embed = [] - for chunk in valid_chunks: - texts_to_embed.append( - f"Repo: {repo}\n" - f"Document: {rel_path}\n" - f"Category: {category}\n" - f"Heading: {chunk.get('heading', 'Root')}\n" - f"Content:\n{chunk['content'].strip()}" - ) + texts_to_embed = [ + f"Repo: {repo}\nDocument: {rel_path}\nCategory: {category}\n" + f"Heading: {chunk.get('heading', 'Root')}\nContent:\n{chunk['content'].strip()}" + for chunk in valid_chunks + ] if texts_to_embed: - batch_vecs = _resolve_cached_or_compute_embeddings(valid_chunks, texts_to_embed) + batch_vecs = _resolve_cached_or_compute_embeddings(valid_chunks, texts_to_embed) if embed else [] total_pages_list = [int(m) for m in re.findall(r"^# Page (\d+)", content, re.MULTILINE)] max_page = max(total_pages_list) if total_pages_list else 1 for idx, chunk in enumerate(valid_chunks): point_id = get_chunk_uuid(repo, rel_path, idx) github_url = format_git_permalink(git_url, commit_sha, rel_path, chunk.get("start_line"), chunk.get("end_line"), provider=provider) - - bv = batch_vecs[idx] if idx < len(batch_vecs) and isinstance(batch_vecs[idx], dict) else {} - dense_v = bv.get("dense") - s_indices = bv.get("sparse_indices") - s_values = bv.get("sparse_values") - chunk_meta = {} + bv = batch_vecs[idx] if idx < len(batch_vecs) and isinstance(batch_vecs[idx], dict) else {} + chunk_meta: Dict[str, Any] = {} if is_pdf_file: page_match = re.search(r"Page\s+(\d+)", chunk.get("heading", "")) page_num = int(page_match.group(1)) if page_match else 1 @@ -328,13 +341,15 @@ def process_file_content( "headings": headings, "doc_type": "pdf" } + if not embed: + chunk_meta["_embed_text"] = texts_to_embed[idx] points.append(VectorDocument( id=point_id, text=chunk["content"].strip(), - dense_vector=dense_v, - sparse_indices=s_indices, - sparse_values=s_values, + dense_vector=bv.get("dense"), + sparse_indices=bv.get("sparse_indices"), + sparse_values=bv.get("sparse_values"), repo=repo, doc_type="pdf" if is_pdf_file else "doc", path=filepath, @@ -351,7 +366,7 @@ def process_file_content( metadata=chunk_meta, )) - else: # Code file + else: ast_result = extract_symbols_and_chunks(content, filepath, repo=repo, max_chunk_chars=CODE_CHUNK_SIZE) chunks = [c.model_dump() for c in ast_result.chunks] ast_symbols = [s.model_dump() for s in ast_result.symbols] @@ -362,36 +377,31 @@ def process_file_content( headings = [s["name"] for s in ast_symbols] keywords = extract_keywords_from_text(content, title, headings, [language]) - texts_to_embed = [] - for chunk in valid_chunks: - symbol_label = chunk.get("symbol") or title - texts_to_embed.append( - f"Repo: {repo}\n" - f"File: {rel_path}\n" - f"Language: {language}\n" - f"Symbol: {symbol_label}\n" - f"Lines: {chunk.get('start_line')}-{chunk.get('end_line')}\n" - f"Code:\n{chunk['content'].strip()}" - ) + texts_to_embed = [ + f"Repo: {repo}\nFile: {rel_path}\nLanguage: {language}\n" + f"Symbol: {chunk.get('symbol') or title}\n" + f"Code:\n{chunk['content'].strip()}" + for chunk in valid_chunks + ] if texts_to_embed: - batch_vecs = _resolve_cached_or_compute_embeddings(valid_chunks, texts_to_embed) + batch_vecs = _resolve_cached_or_compute_embeddings(valid_chunks, texts_to_embed) if embed else [] for idx, chunk in enumerate(valid_chunks): point_id = get_chunk_uuid(repo, rel_path, idx) github_url = format_git_permalink(git_url, commit_sha, rel_path, chunk.get("start_line"), chunk.get("end_line"), provider=provider) symbol_label = chunk.get("symbol") or title bv = batch_vecs[idx] if idx < len(batch_vecs) and isinstance(batch_vecs[idx], dict) else {} - dense_v = bv.get("dense") - s_indices = bv.get("sparse_indices") - s_values = bv.get("sparse_values") + chunk_meta = {"kind": chunk.get("kind", "code")} + if not embed: + chunk_meta["_embed_text"] = texts_to_embed[idx] points.append(VectorDocument( id=point_id, text=chunk["content"].strip(), - dense_vector=dense_v, - sparse_indices=s_indices, - sparse_values=s_values, + dense_vector=bv.get("dense"), + sparse_indices=bv.get("sparse_indices"), + sparse_values=bv.get("sparse_values"), repo=repo, doc_type="code", language=language, @@ -405,22 +415,14 @@ def process_file_content( end_line=chunk.get("end_line", 1), github_url=github_url, permalink_url=github_url, - metadata={"kind": chunk.get("kind", "code")} + metadata=chunk_meta, )) filepath_norm = os.path.normpath(os.path.abspath(filepath)) mtime = os.path.getmtime(filepath_norm) if os.path.exists(filepath_norm) else 0.0 summary_tuple = ( - filepath, - repo, - title, - folder, - category, - json.dumps(tags), - json.dumps(list(set(headings[:20]))), - json.dumps(keywords), - None, - mtime + filepath, repo, title, folder, category, + json.dumps(tags), json.dumps(list(set(headings[:20]))), json.dumps(keywords), None, mtime ) return points, ast_symbols, summary_tuple, ast_relationships, api_routes, api_calls diff --git a/app/services/indexing/progress.py b/app/services/indexing/progress.py new file mode 100644 index 0000000..cce8813 --- /dev/null +++ b/app/services/indexing/progress.py @@ -0,0 +1,11 @@ +from app.services.indexing.git_progress import ( + GitSyncJob, + GitProgressTracker, + progress_tracker, +) + +__all__ = [ + "GitSyncJob", + "GitProgressTracker", + "progress_tracker", +] diff --git a/tests/test_embedding_parallel_batch.py b/tests/test_embedding_parallel_batch.py new file mode 100644 index 0000000..07f25a8 --- /dev/null +++ b/tests/test_embedding_parallel_batch.py @@ -0,0 +1,412 @@ +"""Unit and integration tests for Issue #171: parallel API batch embeddings, token slicing, AIMD 429 control, cross-file batching, and atomic client swaps.""" + +import logging +import threading +import time +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import pytest + +import app.services.embeddings as emb_service +import app.services.indexing.processor as proc_service +from app.models.schemas import CloneResult +from app.services.database import get_db_connection, init_db +from app.services.database.connection import get_embedding_db_config, get_metadata, set_embedding_db_config +from app.services.database.engine import get_db_engine +from app.services.embedding_batcher import ( + estimate_token_count, + execute_api_embeddings_batch, + partition_texts_into_slices, + telemetry_tracker, +) +from app.services.indexing.git_syncer import sync_single_git_repo +from app.services.indexing.local_syncer import sync_local_paths +from app.services.indexing.progress import progress_tracker + + +def _make_fake_embedding_response(batch_slice, base_offset: float = 0.0): + """Creates a mock OpenAI embeddings response where each vector encodes its input text index.""" + data = [] + for i, text in enumerate(batch_slice): + val = float(text.split("-")[-1]) if "-" in text and text.split("-")[-1].isdigit() else float(i + base_offset) + data.append(SimpleNamespace(index=i, embedding=[val, val + 0.5])) + return SimpleNamespace(data=data) + + +def test_parallel_slices_execution_and_deterministic_ordering(monkeypatch): + """Verifies get_dense_embeddings_batch dispatches slices in parallel across EMBEDDING_API_CONCURRENCY threads and reassembles results in exact input order.""" + telemetry_tracker.reset(target_concurrency=4) + active_threads = 0 + max_observed_threads = 0 + lock = threading.Lock() + barrier = threading.Barrier(4, timeout=2.0) + + def fake_create(input, model): + nonlocal active_threads, max_observed_threads + with lock: + active_threads += 1 + if active_threads > max_observed_threads: + max_observed_threads = active_threads + try: + barrier.wait() + except threading.BrokenBarrierError: + pass + time.sleep(0.02) + with lock: + active_threads -= 1 + return _make_fake_embedding_response(input) + + mock_client = MagicMock() + mock_client.embeddings.create.side_effect = fake_create + + monkeypatch.setattr(emb_service, "EMBEDDING_PROVIDER", "api") + monkeypatch.setattr(emb_service, "_openai_client", mock_client) + monkeypatch.setattr(emb_service, "EMBEDDING_BATCH_SIZE", 10) + monkeypatch.setattr(emb_service, "EMBEDDING_API_CONCURRENCY", 4) + monkeypatch.setattr(emb_service, "EMBEDDING_MAX_TOKENS_PER_REQUEST", 100000) + monkeypatch.setattr(emb_service, "EMBEDDING_TPM_LIMIT", 0) + + texts = [f"chunk-{i}" for i in range(40)] + results = emb_service.get_dense_embeddings_batch(texts) + + assert len(results) == 40 + assert mock_client.embeddings.create.call_count == 4 + assert max_observed_threads >= 2 + for i in range(40): + assert results[i] == [float(i), float(i) + 0.5] + + +def test_token_limit_slicing_respects_max_tokens_per_request(): + """Verifies partition_texts_into_slices splits batches when estimated tokens exceed EMBEDDING_MAX_TOKENS_PER_REQUEST even if item count is below batch_size.""" + # Each text is 400 chars -> 100 estimated tokens + texts = ["a" * 400 for _ in range(10)] + assert estimate_token_count(texts[0]) == 100 + + # Max 128 items, but max 250 tokens per request -> at most 2 items (200 tokens) per slice + slices = partition_texts_into_slices(texts, max_items_per_slice=128, max_tokens_per_request=250) + assert len(slices) == 5 + assert all(len(s) == 2 for s in slices) + + +def test_http_429_retry_after_aimd_and_telemetry(caplog): + """Verifies HTTP 429 rate-limit responses parse Retry-After headers, log a warning, multiplicatively reduce concurrency, and record rate_limit_wait_seconds in telemetry and progress_tracker.""" + telemetry_tracker.reset(target_concurrency=4) + progress_tracker.get_or_create_job(777, "rate-limit-repo") + progress_tracker.update_step(777, 4, "Parsing Files", pct=60) + + class FakeRateLimitError(Exception): + def __init__(self): + super().__init__("429 Too Many Requests: RateLimitError") + self.status_code = 429 + self.response = SimpleNamespace(status_code=429, headers={"Retry-After": "1.5"}) + + calls = 0 + + def fake_create(input, model): + nonlocal calls + calls += 1 + if calls == 1: + raise FakeRateLimitError() + return _make_fake_embedding_response(input) + + mock_client = MagicMock() + mock_client.embeddings.create.side_effect = fake_create + slept_durations = [] + + with caplog.at_level(logging.WARNING, logger="contextcortex.embeddings"): + vecs = execute_api_embeddings_batch( + client=mock_client, + texts=["item-0", "item-1"], + model_name="text-embedding-3-small", + batch_size=128, + max_concurrency=4, + max_tokens_per_request=100000, + sleep_fn=lambda s: slept_durations.append(s), + ) + + assert len(vecs) == 2 + assert slept_durations == [1.5] + assert any("HTTP 429" in rec.message and "1.50s" in rec.message for rec in caplog.records) + + snap = telemetry_tracker.get_telemetry_snapshot(max_concurrency=4) + assert snap["rate_limit_hits"] == 1 + assert snap["rate_limit_wait_seconds"] == pytest.approx(1.5, rel=1e-2) + assert snap["chunks_embedded"] == 2 + # Concurrency dropped from 4 -> 2 on 429, and 1 success isn't 2 consecutive successes yet -> stays 2 + assert snap["active_concurrency"] == 2 + + job_snap = progress_tracker.get_snapshot(777) + assert job_snap["rate_limit_hits"] == 1 + assert job_snap["rate_limit_wait_seconds"] == pytest.approx(1.5, rel=1e-2) + assert job_snap["chunks_embedded"] == 2 + + # Two more consecutive slice successes should additively increase concurrency back from 2 -> 3 + execute_api_embeddings_batch( + client=mock_client, + texts=["item-2", "item-3"], + model_name="text-embedding-3-small", + batch_size=1, + max_concurrency=4, + sleep_fn=lambda s: None, + ) + assert telemetry_tracker.get_effective_concurrency(4) >= 3 + + +def test_cross_file_batch_embedding_in_git_and_local_syncer(tmp_path, monkeypatch): + """Verifies git_syncer and local_syncer collect chunks across multiple files and call get_hybrid_embeddings_batch once per multi-file batch instead of once per file.""" + db_file = str(tmp_path / "test_cross_file.db") + monkeypatch.setenv("CACHE_DB_PATH", db_file) + monkeypatch.setattr("app.services.database.CACHE_DB_PATH", db_file) + monkeypatch.setattr("app.services.database.connection.CACHE_DB_PATH", db_file) + init_db() + + repo_dir = tmp_path / "git_repo" + repo_dir.mkdir() + for i in range(10): + (repo_dir / f"module_{i}.py").write_text( + f"def func_{i}_a():\n return {i}\n\ndef func_{i}_b():\n return {i * 2}\n", + encoding="utf-8", + ) + + with get_db_connection() as conn: + conn.execute( + "INSERT INTO git_repositories (id, name, url, branch, commit_sha, status) VALUES (1, 'batch-repo', 'https://github.com/org/batch.git', 'main', 'old', 'pending')" + ) + conn.commit() + + embed_batch_Inputs = [] + + def fake_hybrid_batch(texts): + embed_batch_Inputs.append(list(texts)) + return [{"dense": [0.1] * 384, "sparse": None} for _ in texts] + + upserted_docs = [] + + def fake_upsert(docs, *args, **kwargs): + upserted_docs.extend(list(docs)) + return True + + mock_store = MagicMock() + mock_store.upsert_documents.side_effect = fake_upsert + + with patch("app.services.git_manager.get_remote_head_sha", return_value="sha_new"), \ + patch("app.services.git_manager.shallow_clone_repo", return_value=CloneResult(temp_dir=str(repo_dir), commit_sha="sha_new", error=None)), \ + patch("app.services.git_manager.cleanup_repo_dir"), \ + patch("app.services.vector_store.get_vector_store", return_value=mock_store), \ + patch("app.services.indexing.processor.get_hybrid_embeddings_batch", side_effect=fake_hybrid_batch): + ok = sync_single_git_repo(1) + assert ok is True + + # All 10 files fit in a single BATCH_SIZE=50 batch -> 1 single call to get_hybrid_embeddings_batch! + assert len(embed_batch_Inputs) == 1 + assert len(embed_batch_Inputs[0]) >= 10 + assert len(upserted_docs) >= 10 + assert all(doc.dense_vector == [0.1] * 384 for doc in upserted_docs) + assert all("_embed_text" not in doc.metadata for doc in upserted_docs) + + # Also verify local_syncer cross-file batching + local_dir = tmp_path / "local_notes" + local_dir.mkdir() + for i in range(6): + (local_dir / f"note_{i}.md").write_text(f"# Heading {i}\n\nUnique paragraph content for note {i}.", encoding="utf-8") + + with get_db_connection() as conn: + conn.execute( + "INSERT INTO indexed_paths (path, type, recursive, enabled, repo, category) VALUES (?, 'directory', 1, 1, 'local-batch', 'docs')", + (str(local_dir),), + ) + conn.commit() + + embed_batch_Inputs.clear() + with patch("app.services.vector_store.get_vector_store", return_value=mock_store), \ + patch("app.services.indexing.processor.get_hybrid_embeddings_batch", side_effect=fake_hybrid_batch): + assert sync_local_paths(force=True, repo="local-batch") is True + + assert len(embed_batch_Inputs) == 1 + assert len(embed_batch_Inputs[0]) == 6 + + +def test_atomic_openai_client_swap_during_concurrent_embeddings(monkeypatch): + """Verifies init_embeddings replaces _openai_client atomically without ever setting it to None while concurrent embedding calls are active.""" + monkeypatch.setenv("EMBEDDING_PROVIDER", "api") + monkeypatch.setattr(emb_service, "EMBEDDING_PROVIDER", "api") + monkeypatch.setattr(emb_service, "DENSE_MODEL_NAME", emb_service.DENSE_MODEL_NAME) + monkeypatch.setattr(emb_service, "_dense_model", emb_service._dense_model) + monkeypatch.setattr(emb_service, "_sparse_model", emb_service._sparse_model) + + client_v1 = MagicMock() + client_v1.embeddings.create.side_effect = lambda input, model: _make_fake_embedding_response(input) + monkeypatch.setattr(emb_service, "_openai_client", client_v1) + + errors = [] + stop_event = threading.Event() + + def worker(): + while not stop_event.is_set(): + try: + res = emb_service.get_dense_embeddings_batch(["concurrent-0", "concurrent-1"]) + if len(res) != 2: + errors.append(f"Unexpected length: {len(res)}") + except Exception as e: + errors.append(e) + + def slow_openai_ctor(*args, **kwargs): + time.sleep(0.03) + c = MagicMock() + c.embeddings.create.side_effect = lambda input, model: _make_fake_embedding_response(input) + return c + + threads = [threading.Thread(target=worker) for _ in range(3)] + for t in threads: + t.start() + + try: + with patch("openai.OpenAI", side_effect=slow_openai_ctor): + for _ in range(4): + emb_service.init_embeddings(provider="api", dense_model="text-embedding-3-small") + finally: + stop_event.set() + for t in threads: + t.join(timeout=2.0) + + assert not errors, f"Concurrent embedding calls failed during init_embeddings swap: {errors}" + + +def test_env_embedding_batch_size_takes_effect_across_restarts(tmp_path, monkeypatch): + """Verifies init_db does not freeze embedding_batch_size or embedding_num_threads in system_metadata, allowing .env changes to take effect on restart.""" + db_file = tmp_path / "restart_env.db" + db_url = f"sqlite:///{db_file}" + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.delenv("EMBEDDING_BATCH_SIZE", raising=False) + monkeypatch.setenv("EMBEDDING_PROVIDER", "api") + + engine = get_db_engine(db_url, reset=True) + init_db(engine=engine) + + # Neither embedding_batch_size nor embedding_num_threads should be seeded into system_metadata on startup + assert get_metadata("embedding_batch_size") is None + assert get_metadata("embedding_num_threads") is None + + # Default for API provider when EMBEDDING_BATCH_SIZE is unset is 128 + cfg1 = get_embedding_db_config() + assert cfg1["batch_size"] == 128 + + # Simulate changing EMBEDDING_BATCH_SIZE in .env and restarting against the same database + monkeypatch.setenv("EMBEDDING_BATCH_SIZE", "96") + init_db(engine=engine) + cfg2 = get_embedding_db_config() + assert cfg2["batch_size"] == 96 + + # Explicit save via Admin UI/API persists override in system_metadata + set_embedding_db_config(batch_size=64) + assert get_metadata("embedding_batch_size") == "64" + assert get_embedding_db_config()["batch_size"] == 64 + + +def test_bounded_channel_pipeline_producer_consumer_backpressure(tmp_path, monkeypatch): + """Verifies BoundedChannel and ChannelIngestionPipeline stream parsed files through coalescing, parallel API embedding slices, and persistence.""" + from app.services.indexing.channel_pipeline import ( + BoundedChannel, + ChannelIngestionPipeline, + ParsedFileItem, + ) + from app.services.vector_store import VectorDocument + + db_file = str(tmp_path / "test_channel_pipeline.db") + monkeypatch.setenv("CACHE_DB_PATH", db_file) + monkeypatch.setattr("app.services.database.CACHE_DB_PATH", db_file) + monkeypatch.setattr("app.services.database.connection.CACHE_DB_PATH", db_file) + init_db() + + mock_client = MagicMock() + mock_client.embeddings.create.side_effect = lambda input, model: _make_fake_embedding_response(input) + + monkeypatch.setattr(emb_service, "EMBEDDING_PROVIDER", "api") + monkeypatch.setattr(emb_service, "_openai_client", mock_client) + monkeypatch.setattr(emb_service, "EMBEDDING_BATCH_SIZE", 4) + monkeypatch.setattr(emb_service, "EMBEDDING_API_CONCURRENCY", 2) + + persisted_batches = [] + + def _produce(channel: BoundedChannel[ParsedFileItem]) -> None: + for i in range(6): + pt = VectorDocument( + id=f"doc-{i}", + dense_vector=None, + sparse_indices=None, + sparse_values=None, + text=f"def fn_{i}(): return {i}", + metadata={"_embed_text": f"chunk-{i}"}, + ) + channel.write(ParsedFileItem(filepath=f"repo://f{i}.py", rel_path=f"f{i}.py", repo="repo", points=[pt])) + + pipeline = ChannelIngestionPipeline( + persist_batch_fn=lambda items: persisted_batches.append(list(items)), + file_batch_size=3, + parsed_channel_capacity=2, + persist_channel_capacity=2, + ) + pipeline.run(_produce) + + assert len(persisted_batches) == 2 + all_pts = [pt for batch in persisted_batches for item in batch for pt in item.points] + assert len(all_pts) == 6 + assert all(pt.dense_vector is not None for pt in all_pts) + assert all("_embed_text" not in pt.metadata for pt in all_pts) + + +def test_class_and_methods_do_not_double_chunk(): + """Verifies extract_symbols_and_chunks does not duplicate method bodies inside parent class chunks and keeps method chunk hashes stable when sibling methods change.""" + from app.services.chunking.symbol_extractor import extract_symbols_and_chunks + + code_v1 = ( + "class OrderProcessor:\n" + " DEFAULT_TAX = 0.07\n" + "\n" + " def calculate_subtotal(self, items):\n" + " return sum(x.price for x in items)\n" + "\n" + " def calculate_total(self, items):\n" + " sub = self.calculate_subtotal(items)\n" + " return sub * (1.0 + self.DEFAULT_TAX)\n" + ) + res_v1 = extract_symbols_and_chunks(code_v1, "order.py", repo="repo") + symbol_names = {s.name for s in res_v1.symbols} + assert {"OrderProcessor", "calculate_subtotal", "calculate_total"} <= symbol_names + + class_chunks = [c for c in res_v1.chunks if c.symbol == "OrderProcessor"] + subtotal_chunks = [c for c in res_v1.chunks if c.symbol == "OrderProcessor.calculate_subtotal"] + total_chunks = [c for c in res_v1.chunks if c.symbol == "OrderProcessor.calculate_total"] + + assert len(class_chunks) == 1 + assert len(subtotal_chunks) == 1 + assert len(total_chunks) == 1 + + # Parent class chunk contains class header & attribute, NOT the method bodies + assert "DEFAULT_TAX = 0.07" in class_chunks[0].content + assert "def calculate_subtotal" not in class_chunks[0].content + assert "def calculate_total" not in class_chunks[0].content + + # Editing calculate_total (and shifting lines) does not alter calculate_subtotal's chunk content hash + code_v2 = ( + "# Added header comment shifting all line numbers down\n" + "# Second comment line\n" + "class OrderProcessor:\n" + " DEFAULT_TAX = 0.07\n" + "\n" + " def calculate_subtotal(self, items):\n" + " return sum(x.price for x in items)\n" + "\n" + " def calculate_total(self, items):\n" + " sub = self.calculate_subtotal(items)\n" + " discount = 5.0\n" + " return max(0.0, sub - discount) * (1.0 + self.DEFAULT_TAX)\n" + ) + res_v2 = extract_symbols_and_chunks(code_v2, "order.py", repo="repo") + subtotal_v2 = [c for c in res_v2.chunks if c.symbol == "OrderProcessor.calculate_subtotal"][0] + assert proc_service.compute_text_hash(subtotal_chunks[0].content.strip()) == proc_service.compute_text_hash( + subtotal_v2.content.strip() + ) + +