diff --git a/.env.example b/.env.example index 31f7b2a..49af1e3 100644 --- a/.env.example +++ b/.env.example @@ -99,6 +99,8 @@ EMBEDDING_BATCH_SIZE=32 # EMBEDDING_API_CONCURRENCY=4 # Estimated token cap per API batch request (default: 100000). # EMBEDDING_MAX_TOKENS_PER_REQUEST=100000 +# Estimated token cap for a single input chunk before truncation (default: 8000). +# EMBEDDING_MAX_TOKENS_PER_INPUT=8000 # Optional tokens-per-minute pacing limit for strict quotas (default: 0 / unlimited). # EMBEDDING_TPM_LIMIT=0 diff --git a/REQUIREMENTS.md b/REQUIREMENTS.md index 9e6d8ea..62af691 100644 --- a/REQUIREMENTS.md +++ b/REQUIREMENTS.md @@ -2,7 +2,7 @@ > **Note:** This document is automatically generated and verified against the live test suite by `scripts/generate_requirements.py` and `tests/backend/test_requirements_sync.py`. -**Test Verification Baseline:** **1357 Automated Tests** (993 Pytest Backend + 314 Vitest Frontend + 50 Playwright E2E). +**Test Verification Baseline:** **1361 Automated Tests** (997 Pytest Backend + 314 Vitest Frontend + 50 Playwright E2E). --- @@ -593,13 +593,17 @@ classDiagram - `test_sse_stream_live_events_and_unsubscription` - `test_sse_stream_keep_alive_ping` -#### `tests/backend/test_git_syncer_progress.py` (6 tests) +#### `tests/backend/test_git_syncer_progress.py` (7 tests) - `test_git_syncer_reports_stages_up_to_date` - `test_git_syncer_clone_failure` - `test_git_syncer_full_5_stage_sync` - `test_git_syncer_cancellation_during_stage_4` - `test_git_syncer_nonexistent_repo` - `test_git_progress_tracker_pending_cancellation` +- `test_git_syncer_failure_stops_producer_keeps_error_status_and_only_counts_persisted_files` - _Issues #179 & #180: Verifies that when batch 2 fails in the persist stage: +1. The producer stops before reading all remaining files and logs no 'No such file or directory' after temp_dir cleanup. +2. Only batch 1 files (actually saved to DB) are counted in processed_files and logged as 'Ingested'. +3. sync-status remains 'error' matching the repository table record._ #### `tests/backend/test_health_check_status.py` (12 tests) - `test_api_stats_reports_healthy_when_store_is_healthy` @@ -1132,6 +1136,11 @@ code results are enriched with signature/kind/ast_symbol_id from SQLite._ - `test_embedding_cache_invalidation` - `test_embedding_cache_empty_inputs` +#### `tests/test_embedding_input_limits_and_pipeline_recovery.py` (3 tests) +- `test_per_input_token_limit_truncates_oversized_chunk_before_batching` - _Issue #178: Verifies oversized inputs exceeding max_tokens_per_input are truncated before batching and configurable via EMBEDDING_MAX_TOKENS_PER_INPUT._ +- `test_http_400_oversized_input_splits_batch_skips_bad_chunk_and_continues_sync` - _Issue #178: Verifies that on an HTTP 400 error for a batch, the batcher splits and retries halves, skips only the bad input, logs its file path, and completes the repo sync._ +- `test_pipeline_producer_stops_when_persist_stage_raises` - _Issue #179: Verifies that when the persist stage raises, the producer stops before reading all remaining files and threads are joined before executor shutdown._ + #### `tests/test_embedding_model_transition.py` (6 tests) - `test_update_embedding_config_rejection_when_index_has_content` - _Verifies that changing dense model/dimension without rebuild_index=True raises ValueError if index has content._ - `test_api_save_embedding_settings_rejection_returns_400` - _Verifies POST /admin/api/settings/embedding returns HTTP 400 when change is rejected._ diff --git a/app/api/routers/ai_settings.py b/app/api/routers/ai_settings.py index 3dff591..4e9b12d 100644 --- a/app/api/routers/ai_settings.py +++ b/app/api/routers/ai_settings.py @@ -111,6 +111,10 @@ async def api_save_embedding_settings(payload: EmbeddingSettingsRequest): sparse_model=payload.sparse_model, threads=payload.threads, batch_size=payload.batch_size, + api_concurrency=payload.api_concurrency, + max_tokens_per_request=payload.max_tokens_per_request, + max_tokens_per_input=payload.max_tokens_per_input, + tpm_limit=payload.tpm_limit, litellm_url=payload.litellm_url, litellm_api_key=payload.litellm_api_key, vision_ocr_model=payload.vision_ocr_model, diff --git a/app/models/schemas.py b/app/models/schemas.py index 9bb53ff..5274d31 100644 --- a/app/models/schemas.py +++ b/app/models/schemas.py @@ -186,6 +186,10 @@ class EmbeddingSettingsRequest(BaseModel): sparse_model: Optional[str] = None threads: Optional[int] = Field(default=None, ge=1, le=128) batch_size: Optional[int] = Field(default=None, ge=1, le=1024) + api_concurrency: Optional[int] = Field(default=None, ge=1, le=128) + max_tokens_per_request: Optional[int] = Field(default=None, ge=1) + max_tokens_per_input: Optional[int] = Field(default=None, ge=1) + tpm_limit: Optional[int] = Field(default=None, ge=0) litellm_url: Optional[str] = None litellm_api_key: Optional[str] = None vision_ocr_model: Optional[str] = None diff --git a/app/services/database/config_store.py b/app/services/database/config_store.py index 8222527..be7aac1 100644 --- a/app/services/database/config_store.py +++ b/app/services/database/config_store.py @@ -173,14 +173,10 @@ def detect_system_resources() -> Dict[str, Any]: class _MEMORYSTATUSEX(ctypes.Structure): _fields_ = [ - ("dwLength", ctypes.c_ulong), - ("dwMemoryLoad", ctypes.c_ulong), - ("ullTotalPhys", ctypes.c_ulonglong), - ("ullAvailPhys", ctypes.c_ulonglong), - ("ullTotalPageFile", ctypes.c_ulonglong), - ("ullAvailPageFile", ctypes.c_ulonglong), - ("ullTotalVirtual", ctypes.c_ulonglong), - ("ullAvailVirtual", ctypes.c_ulonglong), + ("dwLength", ctypes.c_ulong), ("dwMemoryLoad", ctypes.c_ulong), + ("ullTotalPhys", ctypes.c_ulonglong), ("ullAvailPhys", ctypes.c_ulonglong), + ("ullTotalPageFile", ctypes.c_ulonglong), ("ullAvailPageFile", ctypes.c_ulonglong), + ("ullTotalVirtual", ctypes.c_ulonglong), ("ullAvailVirtual", ctypes.c_ulonglong), ("sullAvailExtendedVirtual", ctypes.c_ulonglong), ] @@ -217,45 +213,31 @@ def _resolve_default_embedding_config(conn: Optional[Any] = None) -> Dict[str, A env_max_tokens = os.getenv("EMBEDDING_MAX_TOKENS_PER_REQUEST") max_tokens_per_request = int(env_max_tokens) if env_max_tokens and env_max_tokens.isdigit() else 100000 + env_max_input_tokens = os.getenv("EMBEDDING_MAX_TOKENS_PER_INPUT") + max_tokens_per_input = int(env_max_input_tokens) if env_max_input_tokens and env_max_input_tokens.isdigit() else 8000 + env_tpm = os.getenv("EMBEDDING_TPM_LIMIT") tpm_limit = int(env_tpm) if env_tpm and env_tpm.isdigit() else 0 dense_model = os.getenv("EMBEDDING_MODEL", "BAAI/bge-small-en-v1.5").strip() sparse_model = os.getenv("SPARSE_MODEL", "Qdrant/bm25").strip() - litellm_url = ( - os.getenv("OPENAI_API_URL") - or os.getenv("OPENAI_BASE_URL") - or os.getenv("LITELLM_URL") - or "http://litellm:4000/v1" - ).strip() + litellm_url = (os.getenv("OPENAI_API_URL") or os.getenv("OPENAI_BASE_URL") or os.getenv("LITELLM_URL") or "http://litellm:4000/v1").strip() raw_api_key = (os.getenv("OPENAI_API_KEY") or os.getenv("LITELLM_API_KEY") or "").strip() litellm_api_key = "" if _is_placeholder_api_key(raw_api_key) else raw_api_key - vision_ocr_model = os.getenv("VISION_OCR_MODEL", "").strip() chat_model = os.getenv("CHAT_MODEL", "").strip() - - allow_fallback_env = os.getenv("EMBEDDING_ALLOW_FALLBACK", "false").lower().strip() - allow_fallback = allow_fallback_env in ("1", "true", "yes") + allow_fallback = os.getenv("EMBEDDING_ALLOW_FALLBACK", "false").lower().strip() in ("1", "true", "yes") fallback_dense_model = os.getenv("EMBEDDING_FALLBACK_DENSE_MODEL", "").strip() return { - "provider": provider, - "dense_model": dense_model, - "sparse_model": sparse_model, - "threads": max(1, threads), - "batch_size": max(1, batch_size), - "api_concurrency": max(1, api_concurrency), - "max_tokens_per_request": max(1, max_tokens_per_request), - "tpm_limit": max(0, tpm_limit), - "openai_api_url": litellm_url, - "litellm_url": litellm_url, - "litellm_api_key": litellm_api_key, - "vision_ocr_model": vision_ocr_model, - "chat_model": chat_model, - "allow_fallback": allow_fallback, - "fallback_dense_model": fallback_dense_model, - "system_cpus": sys_res["cpus"], - "system_memory_gb": sys_res["memory_gb"], + "provider": provider, "dense_model": dense_model, "sparse_model": sparse_model, + "threads": max(1, threads), "batch_size": max(1, batch_size), + "api_concurrency": max(1, api_concurrency), "max_tokens_per_request": max(1, max_tokens_per_request), + "max_tokens_per_input": max(1, max_tokens_per_input), "tpm_limit": max(0, tpm_limit), + "openai_api_url": litellm_url, "litellm_url": litellm_url, "litellm_api_key": litellm_api_key, + "vision_ocr_model": vision_ocr_model, "chat_model": chat_model, + "allow_fallback": allow_fallback, "fallback_dense_model": fallback_dense_model, + "system_cpus": sys_res["cpus"], "system_memory_gb": sys_res["memory_gb"], } @@ -267,6 +249,7 @@ def get_embedding_db_config() -> Dict[str, Any]: batch_size_str = _get_meta("embedding_batch_size") concurrency_str = _get_meta("embedding_api_concurrency") max_tokens_str = _get_meta("embedding_max_tokens_per_request") + max_input_tokens_str = _get_meta("embedding_max_tokens_per_input") tpm_limit_str = _get_meta("embedding_tpm_limit") litellm_url = _get_meta("openai_api_url") or _get_meta("embedding_litellm_url") litellm_api_key = _get_meta("openai_api_key") or _get_meta("embedding_litellm_api_key") @@ -291,6 +274,11 @@ def get_embedding_db_config() -> Dict[str, Any]: api_concurrency = int(concurrency_str) if concurrency_str and concurrency_str.isdigit() else default_cfg["api_concurrency"] max_tokens = int(max_tokens_str) if max_tokens_str and max_tokens_str.isdigit() else default_cfg["max_tokens_per_request"] + max_input_tokens = ( + int(max_input_tokens_str) + if max_input_tokens_str and max_input_tokens_str.isdigit() + else default_cfg["max_tokens_per_input"] + ) tpm_limit = int(tpm_limit_str) if tpm_limit_str and tpm_limit_str.isdigit() else default_cfg["tpm_limit"] allow_fallback = (allow_fallback_str == "1") if allow_fallback_str is not None else default_cfg["allow_fallback"] @@ -305,6 +293,7 @@ def get_embedding_db_config() -> Dict[str, Any]: "batch_size": max(1, batch_size), "api_concurrency": max(1, api_concurrency), "max_tokens_per_request": max(1, max_tokens), + "max_tokens_per_input": max(1, max_input_tokens), "tpm_limit": max(0, tpm_limit), "openai_api_url": resolved_url, "litellm_url": resolved_url, @@ -328,6 +317,7 @@ def set_embedding_db_config( batch_size: Optional[int] = None, api_concurrency: Optional[int] = None, max_tokens_per_request: Optional[int] = None, + max_tokens_per_input: Optional[int] = None, tpm_limit: Optional[int] = None, litellm_url: Optional[str] = None, litellm_api_key: Optional[str] = None, @@ -351,6 +341,8 @@ def set_embedding_db_config( _set_meta("embedding_api_concurrency", str(max(1, api_concurrency))) if max_tokens_per_request is not None: _set_meta("embedding_max_tokens_per_request", str(max(1, max_tokens_per_request))) + if max_tokens_per_input is not None: + _set_meta("embedding_max_tokens_per_input", str(max(1, max_tokens_per_input))) if tpm_limit is not None: _set_meta("embedding_tpm_limit", str(max(0, tpm_limit))) if litellm_url is not None: diff --git a/app/services/embedding_batcher.py b/app/services/embedding_batcher.py index 32ade5a..21d4e71 100644 --- a/app/services/embedding_batcher.py +++ b/app/services/embedding_batcher.py @@ -15,10 +15,43 @@ def estimate_token_count(text: str) -> int: return max(1, len(text) // 4) +def _extract_filepath_from_embed_text(text: str) -> Optional[str]: + """Extracts the File: or Document: path header from a formatted embedding string if present.""" + if not text: + return None + for line in text.splitlines()[:5]: + if line.startswith("File: "): + return line[len("File: "):].strip() + if line.startswith("Document: "): + return line[len("Document: "):].strip() + return None + + +def truncate_text_to_token_limit( + text: str, + max_tokens_per_input: int = 8000, + filepath: Optional[str] = None, +) -> str: + """Truncates a single input text so its estimated token count does not exceed max_tokens_per_input.""" + if not text or not max_tokens_per_input or max_tokens_per_input <= 0: + return text + max_chars = int(max_tokens_per_input) * 4 + if len(text) <= max_chars: + return text + est_tokens = estimate_token_count(text) + label = filepath or _extract_filepath_from_embed_text(text) or "" + logger.warning( + f"Truncating oversized embedding input from '{label}' " + f"(~{est_tokens} estimated tokens, {len(text)} chars) to max_tokens_per_input={max_tokens_per_input} ({max_chars} chars)." + ) + return text[:max_chars] + + def partition_texts_into_slices( texts: List[str], max_items_per_slice: int = 128, max_tokens_per_request: int = 100000, + max_tokens_per_input: Optional[int] = None, ) -> List[List[str]]: """Partitions input texts into request slices bounded by both item count and estimated token limit.""" if not texts: @@ -30,7 +63,12 @@ def partition_texts_into_slices( current_slice: List[str] = [] current_tokens = 0 - for text in texts: + for raw_text in texts: + text = ( + truncate_text_to_token_limit(raw_text, max_tokens_per_input=max_tokens_per_input) + if max_tokens_per_input and max_tokens_per_input > 0 + else raw_text + ) t_tokens = estimate_token_count(text) if current_slice and ( len(current_slice) >= max_items or (current_tokens + t_tokens) > max_tokens @@ -95,6 +133,29 @@ def _is_rate_limit_error(exc: Exception) -> bool: return "429" in msg or "rate limit" in msg or "rate_limit" in msg +def _is_bad_request_input_error(exc: Exception) -> bool: + """Returns True if the exception represents an HTTP 400 invalid input or token limit error.""" + status_code = getattr(exc, "status_code", None) + if status_code == 400: + return True + resp = getattr(exc, "response", None) + if resp is not None and getattr(resp, "status_code", None) == 400: + return True + cls_name = exc.__class__.__name__.lower() + if "badrequest" in cls_name or "invalidrequest" in cls_name: + return True + msg = str(exc).lower() + return ( + "error code: 400" in msg + or "status code 400" in msg + or "http 400" in msg + or "maximum input length" in msg + or "maximum context length" in msg + or "context_length_exceeded" in msg + or ("invalid 'input" in msg and "tokens" in msg) + ) + + class EmbeddingTelemetryAndPacer: """Tracks embedding throughput, HTTP 429 wait times, AIMD concurrency, and optional TPM pacing.""" @@ -217,29 +278,59 @@ def execute_api_embeddings_batch( batch_size: int = 128, max_concurrency: int = 4, max_tokens_per_request: int = 100000, + max_tokens_per_input: int = 8000, tpm_limit: int = 0, max_retries: int = 5, sleep_fn: Callable[[float], None] = time.sleep, -) -> List[List[float]]: - """Partitions texts into slices and dispatches OpenAI embedding requests in parallel with AIMD 429 handling.""" + filepaths: Optional[List[Optional[str]]] = None, +) -> List[Optional[List[float]]]: + """Partitions texts into slices and dispatches OpenAI embedding requests in parallel with AIMD 429 handling and HTTP 400 batch bisection.""" if not texts: return [] if client is None: raise RuntimeError("API Embedding client not initialized.") api_bs = max(1, min(int(batch_size or 128), 256)) - slices = partition_texts_into_slices( - texts, - max_items_per_slice=api_bs, - max_tokens_per_request=max_tokens_per_request, - ) + max_req_tokens = max(1, int(max_tokens_per_request or 100000)) + max_in_tokens = int(max_tokens_per_input) if max_tokens_per_input is not None else 8000 + + # Pre-truncate any oversized inputs and pair with optional filepaths before slicing + slices: List[Tuple[List[str], List[Optional[str]]]] = [] + cur_texts: List[str] = [] + cur_paths: List[Optional[str]] = [] + cur_tokens = 0 + + for idx, raw_t in enumerate(texts): + fp = filepaths[idx] if filepaths and idx < len(filepaths) else None + t = ( + truncate_text_to_token_limit(raw_t, max_tokens_per_input=max_in_tokens, filepath=fp) + if max_in_tokens > 0 + else raw_t + ) + t_tok = estimate_token_count(t) + if cur_texts and (len(cur_texts) >= api_bs or (cur_tokens + t_tok) > max_req_tokens): + slices.append((cur_texts, cur_paths)) + cur_texts = [t] + cur_paths = [fp] + cur_tokens = t_tok + else: + cur_texts.append(t) + cur_paths.append(fp) + cur_tokens += t_tok + + if cur_texts: + slices.append((cur_texts, cur_paths)) if not slices: return [] start_time = time.monotonic() effective_concurrency = telemetry_tracker.get_effective_concurrency(max_concurrency) - def _embed_single_slice(slice_idx: int, batch_slice: List[str]) -> Tuple[int, List[List[float]]]: + def _embed_items_with_bisect( + slice_idx: int, + batch_slice: List[str], + slice_paths: List[Optional[str]], + ) -> List[Optional[List[float]]]: slice_tokens = sum(estimate_token_count(t) for t in batch_slice) telemetry_tracker.pace_tokens(slice_tokens, tpm_limit, sleep_fn=sleep_fn) @@ -255,9 +346,9 @@ def _embed_single_slice(slice_idx: int, batch_slice: List[str]) -> Tuple[int, Li response.data, key=lambda d: getattr(d, "index", 0), ) if all(hasattr(d, "index") for d in response.data) else response.data - vecs = [d.embedding for d in data_sorted] + vecs: List[Optional[List[float]]] = [d.embedding for d in data_sorted] telemetry_tracker.record_slice_success(max_concurrency) - return slice_idx, vecs + return vecs except Exception as exc: if _is_rate_limit_error(exc) and attempt < max_retries: attempt += 1 @@ -278,20 +369,47 @@ def _embed_single_slice(slice_idx: int, batch_slice: List[str]) -> Tuple[int, Li sleep_fn(wait_secs) backoff = min(30.0, backoff * 2.0) continue + if _is_bad_request_input_error(exc): + if len(batch_slice) > 1: + mid = len(batch_slice) // 2 + logger.warning( + f"Embedding slice {slice_idx + 1}/{len(slices)} ({len(batch_slice)} inputs) failed with HTTP 400 ({exc}). " + f"Splitting batch into halves ({mid} and {len(batch_slice) - mid}) to isolate bad input." + ) + left_vecs = _embed_items_with_bisect(slice_idx, batch_slice[:mid], slice_paths[:mid]) + right_vecs = _embed_items_with_bisect(slice_idx, batch_slice[mid:], slice_paths[mid:]) + return left_vecs + right_vecs + bad_path = ( + (slice_paths[0] if slice_paths and slice_paths[0] else None) + or _extract_filepath_from_embed_text(batch_slice[0]) + or "" + ) + logger.error( + f"Skipping bad embedding input from '{bad_path}' (~{estimate_token_count(batch_slice[0])} estimated tokens) " + f"after HTTP 400 error: {exc}" + ) + return [None] raise + def _embed_single_slice( + slice_idx: int, + batch_slice: List[str], + slice_paths: List[Optional[str]], + ) -> Tuple[int, List[Optional[List[float]]]]: + return slice_idx, _embed_items_with_bisect(slice_idx, batch_slice, slice_paths) + if len(slices) == 1 or effective_concurrency <= 1: - ordered_embeddings: List[List[float]] = [] - for idx, s in enumerate(slices): - _, vecs = _embed_single_slice(idx, s) + ordered_embeddings: List[Optional[List[float]]] = [] + for idx, (s_texts, s_paths) in enumerate(slices): + _, vecs = _embed_single_slice(idx, s_texts, s_paths) ordered_embeddings.extend(vecs) else: workers = min(len(slices), effective_concurrency) - results_by_index: Dict[int, List[List[float]]] = {} + results_by_index: Dict[int, List[Optional[List[float]]]] = {} with ThreadPoolExecutor(max_workers=workers) as executor: futures = { - executor.submit(_embed_single_slice, idx, s): idx - for idx, s in enumerate(slices) + executor.submit(_embed_single_slice, idx, s_texts, s_paths): idx + for idx, (s_texts, s_paths) in enumerate(slices) } for future in as_completed(futures): s_idx, vecs = future.result() @@ -301,5 +419,6 @@ def _embed_single_slice(slice_idx: int, batch_slice: List[str]) -> Tuple[int, Li ordered_embeddings.extend(results_by_index[idx]) elapsed = time.monotonic() - start_time - telemetry_tracker.record_batch_completion(len(texts), elapsed) + embedded_count = sum(1 for v in ordered_embeddings if v is not None) + telemetry_tracker.record_batch_completion(embedded_count, elapsed) return ordered_embeddings diff --git a/app/services/embeddings.py b/app/services/embeddings.py index 94a31b8..32e5e89 100644 --- a/app/services/embeddings.py +++ b/app/services/embeddings.py @@ -39,6 +39,7 @@ def _get_db_emb_config() -> Dict[str, Any]: EMBEDDING_BATCH_SIZE = _initial_cfg["batch_size"] EMBEDDING_API_CONCURRENCY = _initial_cfg.get("api_concurrency", 4) EMBEDDING_MAX_TOKENS_PER_REQUEST = _initial_cfg.get("max_tokens_per_request", 100000) +EMBEDDING_MAX_TOKENS_PER_INPUT = _initial_cfg.get("max_tokens_per_input", 8000) EMBEDDING_TPM_LIMIT = _initial_cfg.get("tpm_limit", 0) LITELLM_URL = _initial_cfg["litellm_url"] LITELLM_API_KEY = _initial_cfg.get( @@ -65,6 +66,7 @@ def init_embeddings( batch_size: Optional[int] = None, api_concurrency: Optional[int] = None, max_tokens_per_request: Optional[int] = None, + max_tokens_per_input: Optional[int] = None, tpm_limit: Optional[int] = None, litellm_url: Optional[str] = None, litellm_api_key: Optional[str] = None, @@ -77,7 +79,7 @@ def init_embeddings( global REQUESTED_PROVIDER, EMBEDDING_PROVIDER global REQUESTED_DENSE_MODEL, DENSE_MODEL_NAME, SPARSE_MODEL_NAME global EMBEDDING_NUM_THREADS, EMBEDDING_BATCH_SIZE, LITELLM_URL, LITELLM_API_KEY - global EMBEDDING_API_CONCURRENCY, EMBEDDING_MAX_TOKENS_PER_REQUEST, EMBEDDING_TPM_LIMIT + global EMBEDDING_API_CONCURRENCY, EMBEDDING_MAX_TOKENS_PER_REQUEST, EMBEDDING_MAX_TOKENS_PER_INPUT, EMBEDDING_TPM_LIMIT global VISION_OCR_MODEL, CHAT_MODEL, ALLOW_FALLBACK, FALLBACK_DENSE_MODEL cfg = _get_db_emb_config() @@ -101,6 +103,9 @@ def init_embeddings( EMBEDDING_MAX_TOKENS_PER_REQUEST = int( max_tokens_per_request if max_tokens_per_request is not None else cfg.get("max_tokens_per_request", 100000) ) + EMBEDDING_MAX_TOKENS_PER_INPUT = int( + max_tokens_per_input if max_tokens_per_input is not None else cfg.get("max_tokens_per_input", 8000) + ) EMBEDDING_TPM_LIMIT = int(tpm_limit if tpm_limit is not None else cfg.get("tpm_limit", 0)) LITELLM_URL = (litellm_url or cfg.get("litellm_url") or "http://litellm:4000/v1").strip() LITELLM_API_KEY = (litellm_api_key or cfg.get("litellm_api_key") or "dummy").strip() @@ -240,6 +245,7 @@ def get_embedding_config() -> Dict[str, Any]: "batch_size": EMBEDDING_BATCH_SIZE, "api_concurrency": EMBEDDING_API_CONCURRENCY, "max_tokens_per_request": EMBEDDING_MAX_TOKENS_PER_REQUEST, + "max_tokens_per_input": EMBEDDING_MAX_TOKENS_PER_INPUT, "tpm_limit": EMBEDDING_TPM_LIMIT, "litellm_url": LITELLM_URL, "vision_ocr_model": VISION_OCR_MODEL, @@ -301,6 +307,7 @@ def update_embedding_config( batch_size: Optional[int] = None, api_concurrency: Optional[int] = None, max_tokens_per_request: Optional[int] = None, + max_tokens_per_input: Optional[int] = None, tpm_limit: Optional[int] = None, litellm_url: Optional[str] = None, litellm_api_key: Optional[str] = None, @@ -366,6 +373,7 @@ def update_embedding_config( "batch_size": batch_size, "api_concurrency": api_concurrency, "max_tokens_per_request": max_tokens_per_request, + "max_tokens_per_input": max_tokens_per_input, "tpm_limit": tpm_limit, "litellm_url": litellm_url, "litellm_api_key": litellm_api_key, @@ -428,6 +436,7 @@ def get_dense_embeddings_batch(texts: List[str], batch_size: Optional[int] = Non batch_size=api_bs, max_concurrency=EMBEDDING_API_CONCURRENCY, max_tokens_per_request=EMBEDDING_MAX_TOKENS_PER_REQUEST, + max_tokens_per_input=EMBEDDING_MAX_TOKENS_PER_INPUT, tpm_limit=EMBEDDING_TPM_LIMIT, ) else: diff --git a/app/services/indexing/channel_pipeline.py b/app/services/indexing/channel_pipeline.py index 5cf75f5..fdfa575 100644 --- a/app/services/indexing/channel_pipeline.py +++ b/app/services/indexing/channel_pipeline.py @@ -14,6 +14,7 @@ estimate_token_count, partition_texts_into_slices, telemetry_tracker, + truncate_text_to_token_limit, ) from app.services.memory_utils import trim_memory from app.services.vector_store import VectorDocument @@ -46,10 +47,19 @@ def write(self, item: T, timeout: float = 0.25) -> bool: raise RuntimeError("Cannot write to a completed BoundedChannel.") try: self._queue.put(item, timeout=timeout) + with self._lock: + if self._cancelled or self._error is not None: + return False return True except queue.Full: continue + @property + def is_closed(self) -> bool: + """Returns True if the channel has been cancelled or failed.""" + with self._lock: + return self._cancelled or self._error is not None + def complete(self) -> None: """Marks the channel as complete (no further items will be written).""" with self._lock: @@ -203,9 +213,11 @@ def _stage2_coalescer() -> None: try: batch_items: List[ParsedFileItem] = [] for item in self.parsed_channel.read_all(): + if self.persist_channel.is_closed: + return batch_items.append(item) if len(batch_items) >= self.file_batch_size: - self._dispatch_batch( + if not self._dispatch_batch( batch_items, embed_executor, inflight_hash_futures, @@ -214,10 +226,11 @@ def _stage2_coalescer() -> None: proc_service, get_cached_embeddings_batch, execute_api_embeddings_batch, - ) + ): + return batch_items = [] - if batch_items: - self._dispatch_batch( + if batch_items and not self.persist_channel.is_closed: + if not self._dispatch_batch( batch_items, embed_executor, inflight_hash_futures, @@ -226,7 +239,8 @@ def _stage2_coalescer() -> None: proc_service, get_cached_embeddings_batch, execute_api_embeddings_batch, - ) + ): + return self.persist_channel.complete() except BaseException as exc: logger.error(f"Pipeline Stage 2 (Coalescer) failed: {exc}") @@ -245,8 +259,8 @@ def _stage2_coalescer() -> None: proc_service, set_cached_embeddings_batch, ) - t1.join() - t2.join() + t1.join(timeout=10.0) + t2.join(timeout=10.0) if self.parsed_channel.error is not None: raise self.parsed_channel.error if self.persist_channel.error is not None: @@ -254,7 +268,9 @@ def _stage2_coalescer() -> None: finally: self.parsed_channel.cancel() self.persist_channel.cancel() - embed_executor.shutdown(wait=True, cancel_futures=False) + t1.join(timeout=10.0) + t2.join(timeout=10.0) + embed_executor.shutdown(wait=True, cancel_futures=True) def _dispatch_batch( self, @@ -266,10 +282,13 @@ def _dispatch_batch( proc_service: Any, get_cached_embeddings_batch: Callable, execute_api_embeddings_batch: Callable, - ) -> None: + ) -> bool: + if self.persist_channel.is_closed: + return False active_model = proc_service._get_indexer_attr("DENSE_MODEL_NAME", emb_service.DENSE_MODEL_NAME) point_hashes: List[Tuple[VectorDocument, str]] = [] hash_to_text: Dict[str, str] = {} + hash_to_filepath: Dict[str, str] = {} for item in items: for pt in item.points: @@ -284,9 +303,10 @@ def _dispatch_batch( point_hashes.append((pt, h)) if h not in hash_to_text: hash_to_text[h] = formatted + hash_to_filepath[h] = item.rel_path or item.filepath if not point_hashes: - self.persist_channel.write( + return self.persist_channel.write( PendingPersistBatch( items=items, point_hashes=[], @@ -296,17 +316,18 @@ def _dispatch_batch( active_model=active_model, ) ) - return all_hashes = list(hash_to_text.keys()) cached_map = get_cached_embeddings_batch(all_hashes, model_name=active_model) unseen_hashes: List[str] = [] unseen_texts: List[str] = [] + unseen_filepaths: List[str] = [] for h in all_hashes: if h not in cached_map: unseen_hashes.append(h) unseen_texts.append(hash_to_text[h]) + unseen_filepaths.append(hash_to_filepath.get(h, "")) embed_fn = proc_service._get_indexer_attr( "get_hybrid_embeddings_batch", @@ -322,33 +343,41 @@ def _dispatch_batch( if is_default_embed_fn and is_api_provider: api_bs = max(1, min(int(getattr(emb_service, "EMBEDDING_BATCH_SIZE", 128) or 128), 256)) max_tok = max(1, int(getattr(emb_service, "EMBEDDING_MAX_TOKENS_PER_REQUEST", 100000) or 100000)) + max_tok_input = max(1, int(getattr(emb_service, "EMBEDDING_MAX_TOKENS_PER_INPUT", 8000) or 8000)) max_conc = max(1, int(getattr(emb_service, "EMBEDDING_API_CONCURRENCY", 4) or 4)) tpm = max(0, int(getattr(emb_service, "EMBEDDING_TPM_LIMIT", 0) or 0)) client = getattr(emb_service, "_openai_client", None) model_name = getattr(emb_service, "DENSE_MODEL_NAME", active_model) - # Partition (hash, text) pairs into token- and size-bounded slices - slices_pairs: List[Tuple[List[str], List[str]]] = [] + # Truncate oversized inputs to max_tokens_per_input and partition into token- and size-bounded slices + slices_triples: List[Tuple[List[str], List[str], List[str]]] = [] cur_h: List[str] = [] cur_t: List[str] = [] + cur_fp: List[str] = [] cur_tok = 0 - for h, t in zip(unseen_hashes, unseen_texts): + for h, raw_t, fp in zip(unseen_hashes, unseen_texts, unseen_filepaths): + t = truncate_text_to_token_limit(raw_t, max_tokens_per_input=max_tok_input, filepath=fp) t_tok = estimate_token_count(t) if cur_t and (len(cur_t) >= api_bs or (cur_tok + t_tok) > max_tok): - slices_pairs.append((cur_h, cur_t)) - cur_h, cur_t, cur_tok = [h], [t], t_tok + slices_triples.append((cur_h, cur_t, cur_fp)) + cur_h, cur_t, cur_fp, cur_tok = [h], [t], [fp], t_tok else: cur_h.append(h) cur_t.append(t) + cur_fp.append(fp) cur_tok += t_tok if cur_t: - slices_pairs.append((cur_h, cur_t)) + slices_triples.append((cur_h, cur_t, cur_fp)) active_futs: List[Future] = [] - for s_hashes, s_texts in slices_pairs: + for s_hashes, s_texts, s_fps in slices_triples: + if self.persist_channel.is_closed or self.parsed_channel.is_closed: + return False active_futs = [f for f in active_futs if not f.done()] eff_conc = telemetry_tracker.get_effective_concurrency(max_conc) while len(active_futs) >= eff_conc: + if self.persist_channel.is_closed or self.parsed_channel.is_closed: + return False time.sleep(0.005) active_futs = [f for f in active_futs if not f.done()] eff_conc = telemetry_tracker.get_effective_concurrency(max_conc) @@ -361,16 +390,20 @@ def _dispatch_batch( batch_size=len(s_texts), max_concurrency=1, max_tokens_per_request=max_tok, + max_tokens_per_input=max_tok_input, tpm_limit=tpm, + filepaths=s_fps, ) active_futs.append(fut) slice_futures.append((s_hashes, fut)) else: + if self.persist_channel.is_closed or self.parsed_channel.is_closed: + return False use_raw_hybrid = True fut = embed_executor.submit(embed_fn, unseen_texts) slice_futures.append((unseen_hashes, fut)) - self.persist_channel.write( + return self.persist_channel.write( PendingPersistBatch( items=items, point_hashes=point_hashes, @@ -396,6 +429,8 @@ def _finalize_and_persist_batch( for i, h in enumerate(s_hashes): nv = new_vectors[i] if i < len(new_vectors) and isinstance(new_vectors[i], dict) else {} dense_v = nv.get("dense") + if dense_v is None: + continue s_indices, s_values = proc_service._extract_sparse_tuple(nv.get("sparse")) batch.cached_map[h] = { "dense": dense_v, @@ -414,6 +449,8 @@ def _finalize_and_persist_batch( dense_vecs = fut.result() for i, h in enumerate(s_hashes): dense_v = dense_vecs[i] if i < len(dense_vecs) else None + if dense_v is None: + continue batch.cached_map[h] = { "dense": dense_v, "sparse_indices": None, @@ -439,4 +476,12 @@ def _finalize_and_persist_batch( pt.sparse_indices = bv.get("sparse_indices") pt.sparse_values = bv.get("sparse_values") + for item in batch.items: + if any(pt.dense_vector is None for pt in item.points): + skipped = sum(1 for pt in item.points if pt.dense_vector is None) + logger.warning( + f"Skipping {skipped} unembedded chunk(s) in file '{item.rel_path or item.filepath}'." + ) + item.points = [pt for pt in item.points if pt.dense_vector is not None] + self.persist_batch_fn(batch.items) diff --git a/app/services/indexing/git_progress.py b/app/services/indexing/git_progress.py index d2ab91f..0dcf2e9 100644 --- a/app/services/indexing/git_progress.py +++ b/app/services/indexing/git_progress.py @@ -15,6 +15,7 @@ class GitSyncJob: total_steps: int = 5 step_name: str = "Connecting & Remote Check" current_file: Optional[str] = None + parsed_files: int = 0 processed_files: int = 0 total_files: int = 0 percent: int = 0 @@ -62,6 +63,7 @@ def get_or_create_job(self, repo_id: int, repo_name: str) -> GitSyncJob: job.step = 1 job.step_name = "Connecting & Remote Check" job.current_file = None + job.parsed_files = 0 job.processed_files = 0 job.total_files = 0 job.percent = 0 @@ -106,12 +108,13 @@ def update_step( current_file: Optional[str] = None, processed: Optional[int] = None, total: Optional[int] = None, - pct: Optional[int] = None + pct: Optional[int] = None, + parsed: Optional[int] = None, ): payload = None with self._lock: job = self.jobs.get(repo_id) - if not job: + if not job or job.cancelled or job.status == "error": return job.status = "syncing" job.step = step @@ -120,6 +123,8 @@ def update_step( job.current_file = current_file if processed is not None: job.processed_files = processed + if parsed is not None: + job.parsed_files = parsed if total is not None: job.total_files = total if pct is not None: @@ -164,6 +169,7 @@ def finish_job(self, repo_id: int, status: str = "synced", error: Optional[str] job.percent = 100 job.step_name = "Sync Complete" if job.total_files > 0: + job.parsed_files = job.total_files job.processed_files = job.total_files job.updated_at = time.time() payload = job.to_dict() diff --git a/app/services/indexing/git_syncer.py b/app/services/indexing/git_syncer.py index 3c605cf..a5dd172 100644 --- a/app/services/indexing/git_syncer.py +++ b/app/services/indexing/git_syncer.py @@ -34,30 +34,18 @@ def _get_repo_lock(repo_id: int) -> threading.Lock: from app.services.embeddings import * from app.services.git_manager import * from app.services.indexing.git_delta_scanner import ( - DEFAULT_SUPPORTED_EXTENSIONS, - _BUNDLED_REL_PREFIXES, - _IGNORED_DIR_NAMES, - _should_skip_indexed_file, - compute_git_repo_delta, - persist_recent_commits, - persist_sync_batch, + DEFAULT_SUPPORTED_EXTENSIONS, _BUNDLED_REL_PREFIXES, _IGNORED_DIR_NAMES, + _should_skip_indexed_file, compute_git_repo_delta, persist_recent_commits, persist_sync_batch, ) from app.services.indexing.git_progress import progress_tracker from app.services.indexing.local_syncer import sync_local_paths from app.services.indexing.processor import ( - compute_text_hash, - extract_keywords_from_text, - get_chunk_uuid, - get_dynamic_catalog_description, - process_file_content, + compute_text_hash, extract_keywords_from_text, get_chunk_uuid, + get_dynamic_catalog_description, process_file_content, ) from app.services.indexing.state import ( - CHUNK_OVERLAP, - CHUNK_SIZE, - VAULT_PATH, - ensure_collection, - indexing_lock, - trigger_list_changed_notification, + CHUNK_OVERLAP, CHUNK_SIZE, VAULT_PATH, + ensure_collection, indexing_lock, trigger_list_changed_notification, ) from app.services.memory_utils import trim_memory from app.services.vector_store import VectorStoreManager @@ -272,11 +260,14 @@ def _clean_stale_paths() -> None: BATCH_SIZE = 25 total_vectors_count = 0 total_symbols_count = 0 + persisted_files_count = 0 was_cancelled = False def _produce_parsed_files(channel: BoundedChannel[ParsedFileItem]) -> None: nonlocal was_cancelled for idx, full_path in enumerate(files_to_sync): + if channel.is_closed: + return if progress_tracker.is_cancelled(repo_id): logger.info(f"Sync cancelled for repo '{repo_name}' during file ingestion at file {idx + 1}/{total_files}") progress_tracker.log(repo_id, "WARN", f"Sync cancelled by user at file {idx + 1}/{total_files}") @@ -286,8 +277,8 @@ def _produce_parsed_files(channel: BoundedChannel[ParsedFileItem]) -> None: rel_path = os.path.relpath(full_path, temp_dir).replace("\\", "/") progress_tracker.update_step( - repo_id, 4, f"Parsing Files ({idx + 1}/{total_files})", - current_file=rel_path, processed=idx + 1, total=total_files, + repo_id, 4, f"Parsed {idx + 1}/{total_files}", + current_file=rel_path, parsed=idx + 1, total=total_files, ) doc_type = "code" if is_code_file(full_path) else "doc" lang = detect_language(full_path) @@ -299,53 +290,39 @@ def _produce_parsed_files(channel: BoundedChannel[ParsedFileItem]) -> None: try: points, symbols, summary_tuple, rels, routes, calls = proc_service.process_file_content( - filepath=db_filepath, - rel_path=rel_path, - content=content, - repo=repo_name, - doc_type=doc_type, - git_url=git_url, - commit_sha=commit_sha, - provider=provider, - embed=False, + filepath=db_filepath, rel_path=rel_path, content=content, repo=repo_name, + doc_type=doc_type, git_url=git_url, commit_sha=commit_sha, provider=provider, embed=False, ) except TypeError: points, symbols, summary_tuple, rels, routes, calls = proc_service.process_file_content( - filepath=db_filepath, - rel_path=rel_path, - content=content, - repo=repo_name, - doc_type=doc_type, - git_url=git_url, - commit_sha=commit_sha, - provider=provider, + filepath=db_filepath, rel_path=rel_path, content=content, repo=repo_name, + doc_type=doc_type, git_url=git_url, commit_sha=commit_sha, provider=provider, ) - channel.write( + if progress_tracker.is_cancelled(repo_id): + logger.info(f"Sync cancelled for repo '{repo_name}' during file ingestion at file {idx + 1}/{total_files}") + progress_tracker.log(repo_id, "WARN", f"Sync cancelled by user at file {idx + 1}/{total_files}") + was_cancelled = True + channel.cancel() + return + + if not channel.write( ParsedFileItem( - filepath=db_filepath, - rel_path=rel_path, - repo=repo_name, - points=points, - symbols=symbols, - summary_tuple=summary_tuple, - relationships=rels, - routes=routes, - calls=calls, + filepath=db_filepath, rel_path=rel_path, repo=repo_name, points=points, + symbols=symbols, summary_tuple=summary_tuple, relationships=rels, + routes=routes, calls=calls, indexed_file_row=(db_filepath, repo_name, doc_type, lang, commit_sha, 0.0, file_hash), ) - ) - - progress_tracker.log(repo_id, "INFO", f"[{idx + 1}/{total_files}] Ingested {rel_path} (+{len(points)} chunks, +{len(symbols)} symbols)") - if (idx + 1) % 10 == 0 or (idx + 1) == total_files: - pct = int(((idx + 1) / total_files) * 100) - logger.info(f"[{idx + 1}/{total_files}] ({pct}%) Ingesting {rel_path} (+{len(points)} chunks, +{len(symbols)} symbols)") + ): + return except Exception as fe: + if channel.is_closed or progress_tracker.is_cancelled(repo_id): + return logger.error(f"Error parsing file {rel_path} in repo '{repo_name}': {fe}") progress_tracker.log(repo_id, "ERROR", f"Error parsing {rel_path}: {fe}") def _persist_parsed_batch(batch_items: list[ParsedFileItem]) -> None: - nonlocal total_vectors_count, total_symbols_count + nonlocal total_vectors_count, total_symbols_count, persisted_files_count batch_points = [] batch_symbols = [] batch_routes = [] @@ -385,6 +362,30 @@ def _write_relational_batch() -> int: ) total_symbols_count += submit_db_write(_write_relational_batch) + + for item in batch_items: + persisted_files_count += 1 + progress_tracker.log( + repo_id, + "INFO", + f"[{persisted_files_count}/{total_files}] Ingested {item.rel_path} (+{len(item.points)} chunks, +{len(item.symbols)} symbols)", + ) + if persisted_files_count % 10 == 0 or persisted_files_count == total_files: + pct = int((persisted_files_count / total_files) * 100) if total_files > 0 else 100 + logger.info( + f"[{persisted_files_count}/{total_files}] ({pct}%) Ingesting {item.rel_path} (+{len(item.points)} chunks, +{len(item.symbols)} symbols)" + ) + + if batch_items: + progress_tracker.update_step( + repo_id, + 4, + f"Ingested {persisted_files_count}/{total_files}", + current_file=batch_items[-1].rel_path, + processed=persisted_files_count, + total=total_files, + ) + trim_memory() pipeline = ChannelIngestionPipeline( diff --git a/app/services/indexing/local_syncer.py b/app/services/indexing/local_syncer.py index 48978d3..d269b37 100644 --- a/app/services/indexing/local_syncer.py +++ b/app/services/indexing/local_syncer.py @@ -155,6 +155,8 @@ def _sync_local_paths_internal(force: bool = False, repo_filter: str = None): def _produce_local_files(channel: BoundedChannel[ParsedFileItem]) -> None: for filepath, (repo, category, base_dir) in found_files.items(): + if channel.is_closed: + return try: mtime = os.path.getmtime(filepath) with db_service.get_db_connection() as conn: @@ -188,7 +190,7 @@ def _produce_local_files(channel: BoundedChannel[ParsedFileItem]) -> None: category_override=category, ) - channel.write( + if not channel.write( ParsedFileItem( filepath=filepath, rel_path=rel_path, @@ -201,8 +203,11 @@ def _produce_local_files(channel: BoundedChannel[ParsedFileItem]) -> None: calls=calls, indexed_file_row=(filepath, repo, doc_type, lang, None, mtime), ) - ) + ): + return except Exception as e: + if channel.is_closed: + return logger.error(f"Error processing local file {filepath}: {e}") def _collect_embedded_local_batch(batch_items: list[ParsedFileItem]) -> None: diff --git a/app/services/indexing/processor.py b/app/services/indexing/processor.py index 8a23109..3cd0e11 100644 --- a/app/services/indexing/processor.py +++ b/app/services/indexing/processor.py @@ -146,6 +146,8 @@ def _resolve_cached_or_compute_embeddings( for i, h in enumerate(unseen_hashes): nv = new_vectors[i] if i < len(new_vectors) and isinstance(new_vectors[i], dict) else {} dense_v = nv.get("dense") + if dense_v is None: + continue s_indices, s_values = _extract_sparse_tuple(nv.get("sparse")) cached_map[h] = {"dense": dense_v, "sparse_indices": s_indices, "sparse_values": s_values} items_to_cache.append({ diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index fd03d2d..17c39ab 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -37,6 +37,7 @@ Configure the application by setting these environment variables in the system e | `EMBEDDING_BATCH_SIZE` | Number of text chunks sent per embedding call across files (clamped to `256` per API request). | `32` (`local`) / `128` (`api`) | `128` | | `EMBEDDING_API_CONCURRENCY` | Maximum concurrent in-flight HTTP `/embeddings` requests during indexing when `EMBEDDING_PROVIDER=api`. | `4` | `8` | | `EMBEDDING_MAX_TOKENS_PER_REQUEST` | Estimated token budget cap per API embedding request batch before splitting. | `100000` | `100000` | +| `EMBEDDING_MAX_TOKENS_PER_INPUT` | Estimated per-input token cap before truncating oversized chunks prior to batching. | `8000` | `8000` | | `EMBEDDING_TPM_LIMIT` | Optional client-side tokens-per-minute pacing limit (`0` disables proactive pacing and relies on automatic HTTP `429` backoff). | `0` | `1000000` | | `VISION_OCR_MODEL` | Multimodal vision model used for scanned PDF and diagram OCR fallback. | `gpt-4o-mini` | `gpt-4o-mini` | | `CHAT_MODEL` | Default chat completion model used for large-file executive summarization. | `gpt-4o-mini` | `gpt-4o-mini` | diff --git a/tests/backend/test_git_progress.py b/tests/backend/test_git_progress.py index 3e36516..65d9e3b 100644 --- a/tests/backend/test_git_progress.py +++ b/tests/backend/test_git_progress.py @@ -85,6 +85,12 @@ def test_tracker_finish_job_with_error(): assert job.error == "Authentication failed" assert job.percent == 20 + # Issue #180: Late update_step calls from background threads must not overwrite error status back to syncing + tracker.update_step(20, 4, "Parsed 100/100", processed=100, total=100) + assert job.status == "error" + assert job.error == "Authentication failed" + assert job.step == 2 + def test_tracker_log_ring_buffer_limit(): tracker = GitProgressTracker() diff --git a/tests/backend/test_git_syncer_progress.py b/tests/backend/test_git_syncer_progress.py index d7f07f4..b884047 100644 --- a/tests/backend/test_git_syncer_progress.py +++ b/tests/backend/test_git_syncer_progress.py @@ -151,3 +151,67 @@ def test_git_progress_tracker_pending_cancellation(): snap = tracker.get_snapshot(107) assert snap["status"] == "error" assert snap["cancelled"] is True + + +@patch("app.services.git_manager.get_remote_head_sha", return_value="new333") +@patch("app.services.git_manager.shallow_clone_repo") +@patch("app.services.vector_store.get_vector_store") +def test_git_syncer_failure_stops_producer_keeps_error_status_and_only_counts_persisted_files( + mock_store_getter, mock_clone, mock_sha, tmp_path, caplog +): + """Issues #179 & #180: Verifies that when batch 2 fails in the persist stage: + 1. The producer stops before reading all remaining files and logs no 'No such file or directory' after temp_dir cleanup. + 2. Only batch 1 files (actually saved to DB) are counted in processed_files and logged as 'Ingested'. + 3. sync-status remains 'error' matching the repository table record. + """ + import logging + import shutil + + _insert_repo(180, "partial-fail-repo", commit_sha="old111") + + repo_dir = tmp_path / "repo_180" + repo_dir.mkdir() + for i in range(60): + (repo_dir / f"mod_{i:03d}.py").write_text(f"def fn_{i}():\n return {i}\n", encoding="utf-8") + + mock_clone.return_value = CloneResult(temp_dir=str(repo_dir), commit_sha="new333", error=None) + + upsert_calls = 0 + + def upsert_side_effect(docs): + nonlocal upsert_calls + upsert_calls += 1 + if upsert_calls == 1: + return True + raise RuntimeError("Batch 2 persist failure") + + mock_store = MagicMock() + mock_store.upsert_documents.side_effect = upsert_side_effect + mock_store_getter.return_value = mock_store + + with caplog.at_level(logging.ERROR), \ + patch("app.services.git_manager.cleanup_repo_dir", side_effect=lambda d: shutil.rmtree(d, ignore_errors=True)): + res = sync_single_git_repo(180) + + assert not res + # No 'No such file or directory' errors logged after temp_dir cleanup + assert not any("No such file or directory" in rec.message for rec in caplog.records) + + job = progress_tracker.get_snapshot(180) + assert job is not None + # Status must remain 'error' (not overwritten back to 'syncing' by producer) + assert job["status"] == "error" + assert "Batch 2 persist failure" in (job["error"] or "") + + # Only batch 1 (25 files) was persisted to the database + with get_db_connection() as conn: + repo_row = conn.execute("SELECT status, last_error FROM git_repositories WHERE id = 180").fetchone() + saved_files_count = conn.execute("SELECT COUNT(*) FROM indexed_files WHERE repo = 'partial-fail-repo'").fetchone()[0] + + assert repo_row["status"] == "error" + assert saved_files_count == 25 + assert job["processed_files"] == 25 + + ingested_logs = [l["message"] for l in job["logs"] if "Ingested mod_" in l["message"]] + assert len(ingested_logs) == 25 + diff --git a/tests/test_embedding_input_limits_and_pipeline_recovery.py b/tests/test_embedding_input_limits_and_pipeline_recovery.py new file mode 100644 index 0000000..9cecda7 --- /dev/null +++ b/tests/test_embedding_input_limits_and_pipeline_recovery.py @@ -0,0 +1,186 @@ +"""Unit and integration tests for Issues #178 and #179: +- Per-input token limit (max_tokens_per_input) truncation before batching +- HTTP 400 batch bisection, bad-chunk skipping with filepath logging, and uninterrupted repo sync +- Immediate pipeline producer/coalescer shutdown when the persist stage fails +""" + +import logging +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import pytest + +import app.services.embeddings as emb_service +from app.models.schemas import CloneResult +from app.services.database import get_db_connection, init_db +from app.services.embedding_batcher import ( + estimate_token_count, + execute_api_embeddings_batch, + truncate_text_to_token_limit, +) +from app.services.indexing.channel_pipeline import ( + BoundedChannel, + ChannelIngestionPipeline, + ParsedFileItem, +) +from app.services.indexing.git_syncer import sync_single_git_repo +from app.services.vector_store import VectorDocument + + +def _make_fake_embedding_response(batch_slice, base_offset: float = 0.0): + data = [] + for i, text in enumerate(batch_slice): + val = float(text.split("-")[-1]) if "-" in text and text.split("-")[-1].isdigit() else float(i + base_offset) + data.append(SimpleNamespace(index=i, embedding=[val, val + 0.5])) + return SimpleNamespace(data=data) + + +def test_per_input_token_limit_truncates_oversized_chunk_before_batching(caplog): + """Issue #178: Verifies oversized inputs exceeding max_tokens_per_input are truncated before batching and configurable via EMBEDDING_MAX_TOKENS_PER_INPUT.""" + oversized_text = "Repo: demo\nFile: src/giant.css\nLanguage: css\nSymbol: giant.css\nCode:\n" + ("a" * 40000) + assert estimate_token_count(oversized_text) > 8000 + + with caplog.at_level(logging.WARNING, logger="contextcortex.embeddings"): + truncated = truncate_text_to_token_limit(oversized_text, max_tokens_per_input=8000, filepath="src/giant.css") + + assert len(truncated) == 8000 * 4 + assert estimate_token_count(truncated) == 8000 + assert any("src/giant.css" in rec.message for rec in caplog.records) + + received_inputs = [] + + def fake_create(input, model): + received_inputs.extend(input) + return _make_fake_embedding_response(input) + + mock_client = MagicMock() + mock_client.embeddings.create.side_effect = fake_create + + vecs = execute_api_embeddings_batch( + client=mock_client, + texts=[oversized_text, "short-text-1"], + model_name="text-embedding-3-small", + batch_size=128, + max_concurrency=2, + max_tokens_per_request=100000, + max_tokens_per_input=1000, + ) + assert len(vecs) == 2 + assert len(received_inputs) == 2 + assert len(received_inputs[0]) == 1000 * 4 + + +def test_http_400_oversized_input_splits_batch_skips_bad_chunk_and_continues_sync(tmp_path, monkeypatch, caplog): + """Issue #178: Verifies that on an HTTP 400 error for a batch, the batcher splits and retries halves, skips only the bad input, logs its file path, and completes the repo sync.""" + db_file = str(tmp_path / "test_400_bisect.db") + monkeypatch.setenv("CACHE_DB_PATH", db_file) + monkeypatch.setattr("app.services.database.CACHE_DB_PATH", db_file) + monkeypatch.setattr("app.services.database.connection.CACHE_DB_PATH", db_file) + init_db() + + repo_dir = tmp_path / "repo_400" + repo_dir.mkdir() + for i in range(6): + if i == 3: + (repo_dir / f"file_{i}.py").write_text( + "def bad_func():\n return 'POISON_CHUNK_OVER_8192_TOKENS'\n", + encoding="utf-8", + ) + else: + (repo_dir / f"file_{i}.py").write_text( + f"def good_func_{i}():\n return {i}\n", + encoding="utf-8", + ) + + with get_db_connection() as conn: + conn.execute( + "INSERT INTO git_repositories (id, name, url, branch, commit_sha, status) VALUES (178, 'bisect-repo', 'https://github.com/org/bisect.git', 'main', 'old', 'pending')" + ) + conn.commit() + + class FakeBadRequestError(Exception): + def __init__(self, idx: int): + super().__init__(f"Error code: 400 - Invalid 'input[{idx}]': maximum input length is 8192 tokens.") + self.status_code = 400 + self.response = SimpleNamespace(status_code=400, headers={}) + + def fake_create(input, model): + for idx, text in enumerate(input): + if "POISON_CHUNK_OVER_8192_TOKENS" in text: + raise FakeBadRequestError(idx) + return _make_fake_embedding_response(input) + + mock_client = MagicMock() + mock_client.embeddings.create.side_effect = fake_create + + monkeypatch.setattr(emb_service, "EMBEDDING_PROVIDER", "api") + monkeypatch.setattr(emb_service, "_openai_client", mock_client) + monkeypatch.setattr(emb_service, "EMBEDDING_BATCH_SIZE", 128) + monkeypatch.setattr(emb_service, "EMBEDDING_API_CONCURRENCY", 2) + + upserted_docs = [] + mock_store = MagicMock() + mock_store.upsert_documents.side_effect = lambda docs, *a, **kw: (upserted_docs.extend(docs) or True) + + with caplog.at_level(logging.WARNING), \ + patch("app.services.git_manager.get_remote_head_sha", return_value="sha_178"), \ + patch("app.services.git_manager.shallow_clone_repo", return_value=CloneResult(temp_dir=str(repo_dir), commit_sha="sha_178", error=None)), \ + patch("app.services.git_manager.cleanup_repo_dir"), \ + patch("app.services.vector_store.get_vector_store", return_value=mock_store): + ok = sync_single_git_repo(178) + + assert ok is True + assert len(upserted_docs) == 5 + assert all(doc.dense_vector is not None for doc in upserted_docs) + assert all(doc.rel_path != "file_3.py" for doc in upserted_docs) + assert any("file_3.py" in rec.message for rec in caplog.records) + + +def test_pipeline_producer_stops_when_persist_stage_raises(tmp_path, monkeypatch, caplog): + """Issue #179: Verifies that when the persist stage raises, the producer stops before reading all remaining files and threads are joined before executor shutdown.""" + db_file = str(tmp_path / "test_pipeline_stop.db") + monkeypatch.setenv("CACHE_DB_PATH", db_file) + monkeypatch.setattr("app.services.database.CACHE_DB_PATH", db_file) + monkeypatch.setattr("app.services.database.connection.CACHE_DB_PATH", db_file) + init_db() + + mock_client = MagicMock() + mock_client.embeddings.create.side_effect = lambda input, model: _make_fake_embedding_response(input) + monkeypatch.setattr(emb_service, "EMBEDDING_PROVIDER", "api") + monkeypatch.setattr(emb_service, "_openai_client", mock_client) + + produced_count = 0 + + def _produce(channel: BoundedChannel[ParsedFileItem]) -> None: + nonlocal produced_count + for i in range(100): + if channel.is_closed: + break + pt = VectorDocument( + id=f"doc-{i}", + dense_vector=None, + sparse_indices=None, + sparse_values=None, + text=f"def fn_{i}(): return {i}", + metadata={"_embed_text": f"chunk-{i}"}, + ) + item = ParsedFileItem(filepath=f"repo://f{i}.py", rel_path=f"f{i}.py", repo="repo", points=[pt]) + if not channel.write(item): + break + produced_count += 1 + + def _failing_persist(items: list[ParsedFileItem]) -> None: + raise RuntimeError("Simulated persist stage failure on batch 1") + + pipeline = ChannelIngestionPipeline( + persist_batch_fn=_failing_persist, + file_batch_size=2, + parsed_channel_capacity=2, + persist_channel_capacity=1, + ) + + with caplog.at_level(logging.ERROR), pytest.raises(RuntimeError, match="Simulated persist stage failure"): + pipeline.run(_produce) + + assert produced_count < 25 + assert not any("cannot schedule new futures after shutdown" in rec.message for rec in caplog.records) diff --git a/tests/test_embedding_parallel_batch.py b/tests/test_embedding_parallel_batch.py index 07f25a8..b9611f1 100644 --- a/tests/test_embedding_parallel_batch.py +++ b/tests/test_embedding_parallel_batch.py @@ -410,3 +410,4 @@ def test_class_and_methods_do_not_double_chunk(): ) +